diff --git a/AGENTS.md b/AGENTS.md index 518f5663598a31f13f8430653412bf755afb8a56..9499a8783bdd36839981cb19ae53e7c7c2857d50 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,5 +50,5 @@ When the user asks to start or work on a project: - Each project lives in its OWN HF Space - Use `python3` and the `huggingface_hub` library for all HF API calls - If python3 or huggingface_hub is not installed, install with pip -- Ask the user for the namespace/username to use for project Spaces +- Default namespace for all project Spaces is `ken-q` - One project at a time unless explicitly asked to multitask diff --git a/README.md b/README.md index ad1fd43ac5c978d2b350023982813ac99112d570..858a37968a91a04a7eb8aaf814ec232d98736c36 100644 --- a/README.md +++ b/README.md @@ -20,4 +20,4 @@ An AI coding agent running in your browser via Hugging Face Spaces, powered by N ## Usage -Visit `https://kenqtade-opencode-home.hf.space` and start coding. +Visit `https://ken-q-opencode-home.hf.space` and start coding. diff --git a/node_modules/.bin/hfjs b/node_modules/.bin/hfjs new file mode 120000 index 0000000000000000000000000000000000000000..71f70138023d9e8e63431adad6bfd1a83d10c016 --- /dev/null +++ b/node_modules/.bin/hfjs @@ -0,0 +1 @@ +../@huggingface/hub/dist/cli.js \ No newline at end of file diff --git a/node_modules/.package-lock.json b/node_modules/.package-lock.json new file mode 100644 index 0000000000000000000000000000000000000000..49ed0fdd9e729d36355f7f5664709bd4edfc26bb --- /dev/null +++ b/node_modules/.package-lock.json @@ -0,0 +1,123 @@ +{ + "name": "opencode-space", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "node_modules/@huggingface/blake3-jit": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/@huggingface/blake3-jit/-/blake3-jit-0.0.2.tgz", + "integrity": "sha512-Bq7B5qabyjrJfhBsl85Jd2QBtf+HzRD7h7A9GfN2lzrrsABhOa5evVPgzoCTxR7Ub0QFj7YDK1YkYRWBU25+2w==", + "license": "MIT" + }, + "node_modules/@huggingface/hub": { + "version": "2.13.2", + "resolved": "https://registry.npmjs.org/@huggingface/hub/-/hub-2.13.2.tgz", + "integrity": "sha512-I1Pbn9UcBHCt0M/SbZkKzYr3JVnbJp4YIFv74eVRH2Y22yT9Jpm6DtfM74tcjNLbC7SyFZ433znUw0szdFd4Fg==", + "license": "MIT", + "dependencies": { + "@huggingface/tasks": "^0.21.13", + "@huggingface/xetchunk-wasm": "^0.1.0" + }, + "bin": { + "hfjs": "dist/cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "cli-progress": "^3.12.0" + } + }, + "node_modules/@huggingface/tasks": { + "version": "0.21.20", + "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.21.20.tgz", + "integrity": "sha512-pGw7pQI8TYSbrOVfNRdrYeISDEYOvQzOCEBceYjEKOH2wOmooc6eGfbYRhAALDgNKZCSISPYYFszRxHta3FaNw==", + "license": "MIT" + }, + "node_modules/@huggingface/xetchunk-wasm": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@huggingface/xetchunk-wasm/-/xetchunk-wasm-0.1.0.tgz", + "integrity": "sha512-wWpp2qwPgf9kv1KLJjcDUk/OrpDOsFoQ3Qpz0U5LGn20csoymBf8eneOv6wm/GzPBzlWac1OYiR0aa1vT6aM2Q==", + "license": "MIT", + "dependencies": { + "@huggingface/blake3-jit": "0.0.2", + "gearhash-jit": "1.0.2" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-progress": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz", + "integrity": "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==", + "license": "MIT", + "optional": true, + "dependencies": { + "string-width": "^4.2.3" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT", + "optional": true + }, + "node_modules/gearhash-jit": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/gearhash-jit/-/gearhash-jit-1.0.2.tgz", + "integrity": "sha512-UhzJL4KXSdqAKepy/tZwmi2Rcy0YMmtiC4DQS4SURCuIWdh8ECZtnXK2ePRMLigfB61hRKdLK/Vgg2bSw73izQ==", + "license": "MIT" + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "optional": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "optional": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + } + } +} diff --git a/node_modules/@huggingface/blake3-jit/LICENSE b/node_modules/@huggingface/blake3-jit/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..665259fb9dd3e34154fc57eb7c070107330d822f --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Original work Copyright (c) LongYinan (https://github.com/Brooooooklyn/blake3-jit) +Modified work Copyright (c) Hugging Face + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/node_modules/@huggingface/blake3-jit/README.md b/node_modules/@huggingface/blake3-jit/README.md new file mode 100644 index 0000000000000000000000000000000000000000..17b4b7b4722a3d5e1e5acf28a123a5cd36d69b2d --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/README.md @@ -0,0 +1,56 @@ +# @huggingface/blake3-jit + +Temporary fork of [`blake3-jit`](https://github.com/Brooooooklyn/blake3-jit) by [@Brooooooklyn](https://github.com/Brooooooklyn) with performance enhancements for [Hugging Face Xet](https://huggingface.co/docs/hub/xet) content-defined chunking. + +This package will be deprecated once upstream `blake3-jit` exposes these changes. + +## Changes from upstream + +1. **`Hasher.reset()` method** — resets the hasher to process a new message with the same key/flags, reusing all internal buffers (zero allocations per hash). + +2. **Pre-allocated internal buffers** — `parentBlock`, `parentCv`, `chunkCv`, `outWords`, and `finalizeCv` are allocated once in the constructor and reused across `update`/`finalize` calls, significantly reducing GC pressure in hot loops. + +3. **`ChunkState.resetTo()` method** — allows reusing `ChunkState` instances instead of allocating new ones per chunk. + +4. **Removed `Uint32Array` view fast-path** in `ChunkState.update` — the byte-by-byte `readLittleEndianWordsFull` path was empirically faster and avoids `RangeError` on unaligned offsets. + +5. **Dual ESM/CJS output via `tshy`** — the upstream package is ESM-only; this fork uses [`tshy`](https://github.com/isaacs/tshy) to produce both ESM and CommonJS builds, required for compatibility with Node.js CJS consumers. + +These changes are also available as a patch file at [`packages/xetchunk-wasm/patches/blake3-jit.patch`](../xetchunk-wasm/patches/blake3-jit.patch) (applicable to the upstream dist bundle). + +## Installation + +```bash +npm install @huggingface/blake3-jit +``` + +## Usage + +```typescript +import { hash, Hasher } from "@huggingface/blake3-jit"; + +// One-shot hashing +const digest = hash(new Uint8Array([1, 2, 3])); + +// Incremental hashing with reset (zero-alloc reuse) +const hasher = new Hasher(); +hasher.update(chunk1); +const hash1 = hasher.finalize(); + +hasher.reset(); +hasher.update(chunk2); +const hash2 = hasher.finalize(); + +// Keyed hashing (MAC) +const mac = Hasher.newKeyed(key).update(message).finalize(); +``` + +## Upstream + +This is a fork of [blake3-jit](https://github.com/Brooooooklyn/blake3-jit) — a high-performance BLAKE3 implementation with runtime JIT WASM SIMD, created by [LongYinan (@Brooooooklyn)](https://github.com/Brooooooklyn). + +See the upstream repository for full documentation, benchmarks, and architecture details. + +## License + +MIT — see [LICENSE](./LICENSE) diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/compress.d.ts b/node_modules/@huggingface/blake3-jit/dist/commonjs/compress.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..dcb794007e895c0b6056f2a5f39f60995c151c01 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/compress.d.ts @@ -0,0 +1,37 @@ +/** + * BLAKE3 Compression Function - Highly Optimized + * + * Optimization techniques applied (from Fleek Network case study): + * 1. Use 16 SMI variables for state instead of TypedArray + * 2. Use 16 SMI variables for message words + * 3. Fully inlined G function (no function call overhead) + * 4. Use `| 0` for integer coercion (forces V8 to use 32-bit ALU) + * 5. Hardcoded permutation swaps using only 2 temporary variables + * 6. Offset-based access pattern (avoid creating new views) + * + * The compression function takes: + * - cv: 8-word chaining value + * - block: 16-word message block (64 bytes) + * - counter: 64-bit block counter + * - blockLen: number of input bytes in this block + * - flags: domain separation flags + * + * And outputs 8 or 16 words depending on whether this is a root node. + */ +/** + * Compress a single block. + * + * This is the hot path - every optimization matters here. + * + * @param cv - Chaining value array + * @param cvOff - Offset into cv + * @param block - Message block words + * @param blockOff - Offset into block + * @param out - Output array (8 or 16 words) + * @param outOff - Offset into out + * @param full - If true, output all 16 words (for XOF); if false, output 8 words + * @param counter - 64-bit block counter + * @param blockLen - Number of bytes in this block (0-64) + * @param flags - Domain separation flags + */ +export declare function compress(cv: Uint32Array, cvOff: number, block: Uint32Array, blockOff: number, out: Uint32Array, outOff: number, full: boolean, counter: number, blockLen: number, flags: number): void; diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/compress.js b/node_modules/@huggingface/blake3-jit/dist/commonjs/compress.js new file mode 100644 index 0000000000000000000000000000000000000000..c507cd5766fc7d25a4348a1203c9a495bdad57da --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/compress.js @@ -0,0 +1,919 @@ +"use strict"; +/** + * BLAKE3 Compression Function - Highly Optimized + * + * Optimization techniques applied (from Fleek Network case study): + * 1. Use 16 SMI variables for state instead of TypedArray + * 2. Use 16 SMI variables for message words + * 3. Fully inlined G function (no function call overhead) + * 4. Use `| 0` for integer coercion (forces V8 to use 32-bit ALU) + * 5. Hardcoded permutation swaps using only 2 temporary variables + * 6. Offset-based access pattern (avoid creating new views) + * + * The compression function takes: + * - cv: 8-word chaining value + * - block: 16-word message block (64 bytes) + * - counter: 64-bit block counter + * - blockLen: number of input bytes in this block + * - flags: domain separation flags + * + * And outputs 8 or 16 words depending on whether this is a root node. + */ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.compress = compress; +/** + * Compress a single block. + * + * This is the hot path - every optimization matters here. + * + * @param cv - Chaining value array + * @param cvOff - Offset into cv + * @param block - Message block words + * @param blockOff - Offset into block + * @param out - Output array (8 or 16 words) + * @param outOff - Offset into out + * @param full - If true, output all 16 words (for XOF); if false, output 8 words + * @param counter - 64-bit block counter + * @param blockLen - Number of bytes in this block (0-64) + * @param flags - Domain separation flags + */ +function compress(cv, cvOff, block, blockOff, out, outOff, full, counter, blockLen, flags) { + // Load message words into SMI variables for maximum performance + // V8 optimizes SMI arithmetic directly with the ALU + let m0 = block[blockOff] | 0; + let m1 = block[blockOff + 1] | 0; + let m2 = block[blockOff + 2] | 0; + let m3 = block[blockOff + 3] | 0; + let m4 = block[blockOff + 4] | 0; + let m5 = block[blockOff + 5] | 0; + let m6 = block[blockOff + 6] | 0; + let m7 = block[blockOff + 7] | 0; + let m8 = block[blockOff + 8] | 0; + let m9 = block[blockOff + 9] | 0; + let m10 = block[blockOff + 10] | 0; + let m11 = block[blockOff + 11] | 0; + let m12 = block[blockOff + 12] | 0; + let m13 = block[blockOff + 13] | 0; + let m14 = block[blockOff + 14] | 0; + let m15 = block[blockOff + 15] | 0; + // Initialize state: first 8 words from chaining value + let s0 = cv[cvOff] | 0; + let s1 = cv[cvOff + 1] | 0; + let s2 = cv[cvOff + 2] | 0; + let s3 = cv[cvOff + 3] | 0; + let s4 = cv[cvOff + 4] | 0; + let s5 = cv[cvOff + 5] | 0; + let s6 = cv[cvOff + 6] | 0; + let s7 = cv[cvOff + 7] | 0; + // Words 8-11: IV constants + let s8 = 0x6a09e667; + let s9 = 0xbb67ae85; + let s10 = 0x3c6ef372; + let s11 = 0xa54ff53a; + // Words 12-15: counter, blockLen, flags + // Note: counter is 64-bit, split into low and high 32-bit words + let s12 = counter | 0; + let s13 = (counter / 0x100000000) | 0; + let s14 = blockLen | 0; + let s15 = flags | 0; + // ============================================================ + // 7 rounds of mixing + // Each round consists of 4 column G functions and 4 diagonal G functions + // followed by a message word permutation (except for round 7) + // ============================================================ + // ROUND 1 (message schedule: 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15) + // Column G functions + // G(0, 4, 8, 12) with m0, m1 + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + // G(1, 5, 9, 13) with m2, m3 + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + // G(2, 6, 10, 14) with m4, m5 + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + // G(3, 7, 11, 15) with m6, m7 + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + // Diagonal G functions + // G(0, 5, 10, 15) with m8, m9 + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + // G(1, 6, 11, 12) with m10, m11 + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + // G(2, 7, 8, 13) with m12, m13 + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + // G(3, 4, 9, 14) with m14, m15 + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + // Permute message words for round 2 + // Permutation: [2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8] + // Using 2 temps for the two cycles in the permutation + { + const t0 = m0, t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + // ROUND 2 (message schedule: 2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + // Permute for round 3 + { + const t0 = m0, t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + // ROUND 3 (message schedule: 3,4,10,12,13,2,7,14,6,5,9,0,11,15,8,1) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + // Permute for round 4 + { + const t0 = m0, t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + // ROUND 4 (message schedule: 10,7,12,9,14,3,13,15,4,0,11,2,5,8,1,6) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + // Permute for round 5 + { + const t0 = m0, t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + // ROUND 5 (message schedule: 12,13,9,11,15,10,14,8,7,2,5,3,0,1,6,4) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + // Permute for round 6 + { + const t0 = m0, t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + // ROUND 6 (message schedule: 9,14,11,5,8,12,15,1,13,3,0,10,2,6,4,7) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + // Permute for round 7 + { + const t0 = m0, t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + // ROUND 7 (message schedule: 11,15,5,0,1,9,8,6,14,10,2,12,3,4,7,13) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + // ============================================================ + // Final XOR and output + // ============================================================ + // If full output needed (XOF mode), write words 8-15 first + // (written first in case out === cv) + if (full) { + out[outOff + 8] = s8 ^ cv[cvOff]; + out[outOff + 9] = s9 ^ cv[cvOff + 1]; + out[outOff + 10] = s10 ^ cv[cvOff + 2]; + out[outOff + 11] = s11 ^ cv[cvOff + 3]; + out[outOff + 12] = s12 ^ cv[cvOff + 4]; + out[outOff + 13] = s13 ^ cv[cvOff + 5]; + out[outOff + 14] = s14 ^ cv[cvOff + 6]; + out[outOff + 15] = s15 ^ cv[cvOff + 7]; + } + // Standard output: XOR state[0..7] with state[8..15] + out[outOff] = s0 ^ s8; + out[outOff + 1] = s1 ^ s9; + out[outOff + 2] = s2 ^ s10; + out[outOff + 3] = s3 ^ s11; + out[outOff + 4] = s4 ^ s12; + out[outOff + 5] = s5 ^ s13; + out[outOff + 6] = s6 ^ s14; + out[outOff + 7] = s7 ^ s15; +} diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/constants.d.ts b/node_modules/@huggingface/blake3-jit/dist/commonjs/constants.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..61cf697fdc06eb91ee9fcaffd6ba7e187f74eae0 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/constants.d.ts @@ -0,0 +1,34 @@ +/** + * BLAKE3 Constants + * + * IV values are the same as SHA-256: first 32 bits of the fractional parts + * of the square roots of the first 8 primes (2..19) + */ +export declare const IV: Uint32Array; +export declare const CHUNK_START = 1; +export declare const CHUNK_END: number; +export declare const PARENT: number; +export declare const ROOT: number; +export declare const KEYED_HASH: number; +export declare const DERIVE_KEY_CONTEXT: number; +export declare const DERIVE_KEY_MATERIAL: number; +export declare const OUT_LEN = 32; +export declare const KEY_LEN = 32; +export declare const BLOCK_LEN = 64; +export declare const CHUNK_LEN = 1024; +export declare const MAX_DEPTH = 54; +/** + * Precomputed message word permutations for all 7 rounds. + * + * The base permutation is: [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] + * Each subsequent permutation is the previous one with this permutation applied. + * + * These are the indices into the message block for each round. + * By precomputing these, we avoid runtime permutation overhead. + */ +export declare const MSG_SCHEDULE: ReadonlyArray>; +/** + * Flattened permutation table for compress function optimization. + * This enables direct indexed access: PERMUTATIONS[round * 16 + index] + */ +export declare const PERMUTATIONS: Uint8Array; diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/constants.js b/node_modules/@huggingface/blake3-jit/dist/commonjs/constants.js new file mode 100644 index 0000000000000000000000000000000000000000..8961c527b0f0ee6b640972c32bd377f6c4f614dc --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/constants.js @@ -0,0 +1,56 @@ +"use strict"; +/** + * BLAKE3 Constants + * + * IV values are the same as SHA-256: first 32 bits of the fractional parts + * of the square roots of the first 8 primes (2..19) + */ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PERMUTATIONS = exports.MSG_SCHEDULE = exports.MAX_DEPTH = exports.CHUNK_LEN = exports.BLOCK_LEN = exports.KEY_LEN = exports.OUT_LEN = exports.DERIVE_KEY_MATERIAL = exports.DERIVE_KEY_CONTEXT = exports.KEYED_HASH = exports.ROOT = exports.PARENT = exports.CHUNK_END = exports.CHUNK_START = exports.IV = void 0; +// Initialization Vector (same as SHA-256) +exports.IV = new Uint32Array([ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, +]); +// Domain separation flags +exports.CHUNK_START = 1; +exports.CHUNK_END = 1 << 1; +exports.PARENT = 1 << 2; +exports.ROOT = 1 << 3; +exports.KEYED_HASH = 1 << 4; +exports.DERIVE_KEY_CONTEXT = 1 << 5; +exports.DERIVE_KEY_MATERIAL = 1 << 6; +// Size constants +exports.OUT_LEN = 32; +exports.KEY_LEN = 32; +exports.BLOCK_LEN = 64; +exports.CHUNK_LEN = 1024; +// Maximum depth of the CV stack (supports up to 2^54 bytes input) +exports.MAX_DEPTH = 54; +/** + * Precomputed message word permutations for all 7 rounds. + * + * The base permutation is: [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] + * Each subsequent permutation is the previous one with this permutation applied. + * + * These are the indices into the message block for each round. + * By precomputing these, we avoid runtime permutation overhead. + */ +exports.MSG_SCHEDULE = [ + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8], + [3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1], + [10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6], + [12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4], + [9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7], + [11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13], +]; +/** + * Flattened permutation table for compress function optimization. + * This enables direct indexed access: PERMUTATIONS[round * 16 + index] + */ +exports.PERMUTATIONS = new Uint8Array([ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, + 14, 15, 8, 3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1, 10, 7, 12, 9, 14, 3, 13, 15, 4, + 0, 11, 2, 5, 8, 1, 6, 12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4, 9, 14, 11, 5, 8, 12, + 15, 1, 13, 3, 0, 10, 2, 6, 4, 7, 11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13, +]); diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/hash.d.ts b/node_modules/@huggingface/blake3-jit/dist/commonjs/hash.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2e22b4908846fbd6b0ff65856a47aec9f6da2a73 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/hash.d.ts @@ -0,0 +1,28 @@ +/** + * BLAKE3 Hash Function - Simple one-shot API + * + * This provides a simple hash() function optimized for different input sizes. + * For small inputs, uses pure JS. For large inputs, uses WASM SIMD. + */ +/** + * Hash input data and return the result. + * Automatically uses WASM SIMD for large inputs when available. + * + * @param input - Data to hash + * @param outputLength - Number of bytes to output (default: 32) + * @returns The hash output + */ +export declare function hash(input: Uint8Array, outputLength?: number): Uint8Array; +/** + * Pre-warm SIMD initialization (call early to avoid latency later). + */ +export declare function warmupSimd(): boolean; +/** + * Hash input data directly into a caller-provided output buffer. + * Zero-allocation for the common 32-byte case - ideal for performance-critical code. + * + * @param input - Data to hash + * @param output - Pre-allocated output buffer (must be at least outputLength bytes) + * @param outputLength - Number of bytes to output (default: 32, max: output.length) + */ +export declare function hashInto(input: Uint8Array, output: Uint8Array, outputLength?: number): void; diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/hash.js b/node_modules/@huggingface/blake3-jit/dist/commonjs/hash.js new file mode 100644 index 0000000000000000000000000000000000000000..c0c322699cbb1c33fc1cc14ea4e2faaa753250f5 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/hash.js @@ -0,0 +1,1038 @@ +"use strict"; +/** + * BLAKE3 Hash Function - Simple one-shot API + * + * This provides a simple hash() function optimized for different input sizes. + * For small inputs, uses pure JS. For large inputs, uses WASM SIMD. + */ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.hash = hash; +exports.warmupSimd = warmupSimd; +exports.hashInto = hashInto; +const compress_js_1 = require("./compress.js"); +const constants_js_1 = require("./constants.js"); +const utils_js_1 = require("./utils.js"); +const wasm_simd_js_1 = require("./wasm-simd.js"); +// Pre-allocated buffers for reuse (single-threaded optimization) +let blockWords = null; +// ===== Contiguous Hyper CV Stack (Optimization #6) ===== +// Maximum tree depth for practical inputs (2^64 chunks = essentially unlimited) +// Fixed allocation at module load - no runtime allocation +const CV_STACK_DEPTH = 64; +const HYPER_CV_STACK = new Uint32Array(CV_STACK_DEPTH * 8); // 64 CVs × 8 words = 512 words +// Pre-computed offsets for the first few stack levels (hot path optimization) +// Note: These can be used for further optimization if needed +// const CV_STACK_OFF_0 = 0; +// const CV_STACK_OFF_1 = 8; +// const CV_STACK_OFF_2 = 16; +// const CV_STACK_OFF_3 = 24; +// ===== Pre-allocated CV Pool with Views (avoids subarray() in hot paths) ===== +const CV_POOL_SIZE = 64; +const CV_POOL = new Uint32Array(CV_POOL_SIZE * 8); // 64 CVs × 8 words = 512 words +const CV_VIEWS = []; +for (let i = 0; i < CV_POOL_SIZE; i++) { + CV_VIEWS.push(CV_POOL.subarray(i * 8, i * 8 + 8)); +} +// SIMD initialization state +let simdAvailable = false; +// Threshold for switching to SIMD (must be > 1 chunk to benefit from parallelism) +const SIMD_THRESHOLD = 4 * constants_js_1.CHUNK_LEN; // 4KB - need at least 4 chunks for SIMD benefit +/** + * Initialize SIMD synchronously (lazy). + */ +function ensureSimdSync() { + if (simdAvailable) + return true; + simdAvailable = (0, wasm_simd_js_1.initSimdSync)(); + return simdAvailable; +} +// Reusable buffer for SIMD chunk CVs (4 chunks × 8 words) +const simdChunkCvs = new Uint32Array(32); +// ===== Module-level reusable buffers (single-threaded safe) ===== +// These eliminate heap allocations in hot paths +// For hashChunkWithWords() and hashChunkRoot() +const reusableTempCv = new Uint32Array(8); +// For hashPureJS() +const reusableChunkCv = new Uint32Array(8); +const reusablePureParentBlock = new Uint32Array(16); +const reusablePureParentCv = new Uint32Array(8); +// For hashSimd() - use flat array for 4 chunk CVs (access via subarray) +const reusableSimdCvs = new Uint32Array(32); // 4 × 8 words flat +// For hashSimd() parent compression +const reusableSimdParentBlock = new Uint32Array(16); +const reusableSimdParentCv = new Uint32Array(8); +// For hashSimd() parameters - TypedArrays instead of JS arrays +const reusableOffsets = new Uint32Array(4); +const reusableCounters = new Uint32Array(4); +const reusableBlockLens = new Uint32Array(4); +const reusableFlags = new Uint32Array(4); +// Reusable output buffer for common 32-byte hash (eliminates allocations) +const reusableOut8 = new Uint32Array(8); // Standard 32-byte output +// Pre-created view to avoid allocation in hot path (Task 1 optimization) +const reusableOut8View = new Uint8Array(reusableOut8.buffer, 0, 32); +// ===== Unrolled CV Copy Helper (Task 7 optimization) ===== +// V8 will inline this - avoids loop overhead in hot paths +function copyCV8(src, srcOff, dst, dstOff) { + dst[dstOff] = src[srcOff]; + dst[dstOff + 1] = src[srcOff + 1]; + dst[dstOff + 2] = src[srcOff + 2]; + dst[dstOff + 3] = src[srcOff + 3]; + dst[dstOff + 4] = src[srcOff + 4]; + dst[dstOff + 5] = src[srcOff + 5]; + dst[dstOff + 6] = src[srcOff + 6]; + dst[dstOff + 7] = src[srcOff + 7]; +} +/** + * Transpose 4 blocks (64 bytes each) into SIMD memory layout. + * The SIMD compress4x expects: [m0_0,m0_1,m0_2,m0_3, m1_0,m1_1,m1_2,m1_3, ...] + * where m{i}_{j} is message word i from block j. + * + * OPTIMIZED: Processes all 4 blocks together for each word position, + * writing 4 consecutive u32s at once for better cache locality. + * + * @param inputWords - Pre-created Uint32Array view of input (null if unaligned/non-LE). + * Created once per hash call to avoid allocation in hot loop. + */ +function transposeBlocksToSimd(input, offsets, // Starting offsets for each of 4 blocks +blockLens, // Length of each block (0-64 bytes) +mem32, blockCount, // 1-4 blocks +inputWords) { + // Fast path: all blocks are full 64-byte blocks with aligned LE input + const allFull = blockCount === 4 && + blockLens[0] === 64 && + blockLens[1] === 64 && + blockLens[2] === 64 && + blockLens[3] === 64; + if (allFull && + inputWords && + offsets[0] % 4 === 0 && + offsets[1] % 4 === 0 && + offsets[2] % 4 === 0 && + offsets[3] % 4 === 0) { + // Ultra-fast path: process all 4 blocks together, write 4 consecutive u32s per word + const wordOff0 = offsets[0] >>> 2; + const wordOff1 = offsets[1] >>> 2; + const wordOff2 = offsets[2] >>> 2; + const wordOff3 = offsets[3] >>> 2; + for (let w = 0; w < 16; w++) { + const dstBase = w * 4; + mem32[dstBase] = inputWords[wordOff0 + w]; + mem32[dstBase + 1] = inputWords[wordOff1 + w]; + mem32[dstBase + 2] = inputWords[wordOff2 + w]; + mem32[dstBase + 3] = inputWords[wordOff3 + w]; + } + return; + } + // Standard path: process each block independently (handles partial blocks) + for (let b = 0; b < blockCount; b++) { + const len = blockLens[b]; + const off = offsets[b]; + if (len === 64) { + // Full block + if (inputWords && off % 4 === 0) { + // Direct Uint32Array access for aligned LE blocks + const wordOff = off >>> 2; + for (let w = 0; w < 16; w++) { + mem32[w * 4 + b] = inputWords[wordOff + w]; + } + } + else { + // Byte-by-byte reconstruction + for (let w = 0; w < 16; w++) { + const srcOff = off + w * 4; + mem32[w * 4 + b] = + input[srcOff] | + (input[srcOff + 1] << 8) | + (input[srcOff + 2] << 16) | + (input[srcOff + 3] << 24); + } + } + } + else if (len === 0) { + // Zero block + for (let w = 0; w < 16; w++) { + mem32[w * 4 + b] = 0; + } + } + else { + // Partial block - handle word by word + for (let w = 0; w < 16; w++) { + const wordOff = w * 4; + if (wordOff >= len) { + mem32[w * 4 + b] = 0; + } + else if (wordOff + 4 <= len) { + const srcOff = off + wordOff; + mem32[w * 4 + b] = + input[srcOff] | + (input[srcOff + 1] << 8) | + (input[srcOff + 2] << 16) | + (input[srcOff + 3] << 24); + } + else { + // Partial word at end of block + let word = 0; + for (let i = 0; i < len - wordOff; i++) { + word |= input[off + wordOff + i] << (i * 8); + } + mem32[w * 4 + b] = word; + } + } + } + } + // Zero unused block slots + for (let b = blockCount; b < 4; b++) { + for (let w = 0; w < 16; w++) { + mem32[w * 4 + b] = 0; + } + } +} +/** + * Transpose 4 full chunks (4 × 16 blocks = 64 blocks) into batch SIMD memory. + * This is used for the batched compressChunks4x function that processes + * all 16 blocks in a single WASM call. + * + * Memory layout: BATCH_BLOCK_WORDS has 16 positions, each with 16 v128 values. + * Position p, word w: mem32[(p * 64) + (w * 4) + lane] + * + * OPTIMIZED: Processes all 4 chunks together for each (pos, word) pair, + * writing 4 consecutive u32s at once for better cache locality. + * + * @param input - Input data (must have at least 4 full chunks = 4096 bytes) + * @param chunkOffsets - Starting offsets for each of 4 chunks + * @param mem32 - WASM memory view + * @param inputWords - Pre-created Uint32Array view (null if unaligned) + */ +function transposeBatchToSimd(input, chunkOffsets, mem32, inputWords) { + const BATCH_BASE = wasm_simd_js_1.SIMD_MEMORY.BATCH_BLOCK_WORDS / 4; + // Get base word offsets for each chunk (pre-computed for fast path) + const chunk0WordBase = chunkOffsets[0] >>> 2; + const chunk1WordBase = chunkOffsets[1] >>> 2; + const chunk2WordBase = chunkOffsets[2] >>> 2; + const chunk3WordBase = chunkOffsets[3] >>> 2; + // Fast path: all chunks aligned and LE - process 4 consecutive u32s at once + if (inputWords && chunkOffsets[0] % 4 === 0) { + for (let pos = 0; pos < 16; pos++) { + const posBase = BATCH_BASE + pos * 64; // 16 words × 4 lanes = 64 + const blockWordOff = pos * 16; // 16 words per block (64 bytes / 4) + // Process all 16 words, writing 4 chunks at a time (cache-friendly: 16 bytes per write group) + for (let w = 0; w < 16; w++) { + const dstBase = posBase + w * 4; + // Read word w from all 4 chunks at positions that become consecutive in output + mem32[dstBase] = inputWords[chunk0WordBase + blockWordOff + w]; + mem32[dstBase + 1] = inputWords[chunk1WordBase + blockWordOff + w]; + mem32[dstBase + 2] = inputWords[chunk2WordBase + blockWordOff + w]; + mem32[dstBase + 3] = inputWords[chunk3WordBase + blockWordOff + w]; + } + } + } + else { + // Slow path: byte-by-byte reconstruction, still cache-friendly write pattern + for (let pos = 0; pos < 16; pos++) { + const posBase = BATCH_BASE + pos * 64; + const blockByteOff = pos * 64; // 64 bytes per block + for (let w = 0; w < 16; w++) { + const dstBase = posBase + w * 4; + const wordByteOff = w * 4; + // Chunk 0 + const off0 = chunkOffsets[0] + blockByteOff + wordByteOff; + mem32[dstBase] = + input[off0] | (input[off0 + 1] << 8) | (input[off0 + 2] << 16) | (input[off0 + 3] << 24); + // Chunk 1 + const off1 = chunkOffsets[1] + blockByteOff + wordByteOff; + mem32[dstBase + 1] = + input[off1] | (input[off1 + 1] << 8) | (input[off1 + 2] << 16) | (input[off1 + 3] << 24); + // Chunk 2 + const off2 = chunkOffsets[2] + blockByteOff + wordByteOff; + mem32[dstBase + 2] = + input[off2] | (input[off2 + 1] << 8) | (input[off2 + 2] << 16) | (input[off2 + 3] << 24); + // Chunk 3 + const off3 = chunkOffsets[3] + blockByteOff + wordByteOff; + mem32[dstBase + 3] = + input[off3] | (input[off3 + 1] << 8) | (input[off3 + 2] << 16) | (input[off3 + 3] << 24); + } + } + } +} +// Pre-computed memory offsets for SIMD operations (single-block mode) +const SIMD_CV_BASE = wasm_simd_js_1.SIMD_MEMORY.CHAINING_VALUES / 4; +const SIMD_OUT_BASE = wasm_simd_js_1.SIMD_MEMORY.OUTPUT / 4; +const SIMD_COUNTER_LOW_BASE = wasm_simd_js_1.SIMD_MEMORY.COUNTER_LOW / 4; +const SIMD_COUNTER_HIGH_BASE = wasm_simd_js_1.SIMD_MEMORY.COUNTER_HIGH / 4; +const SIMD_BLOCK_LEN_BASE = wasm_simd_js_1.SIMD_MEMORY.BLOCK_LEN / 4; +// Pre-computed memory offsets for batch SIMD operations (16-block mode) +const BATCH_CV_BASE = wasm_simd_js_1.SIMD_MEMORY.BATCH_CV / 4; +const BATCH_COUNTER_LOW_BASE = wasm_simd_js_1.SIMD_MEMORY.BATCH_COUNTER_LOW / 4; +const BATCH_FLAGS_BASE_OFFSET = wasm_simd_js_1.SIMD_MEMORY.BATCH_FLAGS_BASE / 4; +const BATCH_OUTPUT_BASE = wasm_simd_js_1.SIMD_MEMORY.BATCH_OUTPUT / 4; +// Reusable arrays for batch processing +const batchChunkOffsets = new Uint32Array(4); +const SIMD_FLAGS_BASE = wasm_simd_js_1.SIMD_MEMORY.FLAGS / 4; +/** + * Set up chaining values in SIMD memory (transposed layout). + * Optimized: unrolled loops for common case of 4 chunks. + * cvs is flat: [cv0_word0..cv0_word7, cv1_word0..cv1_word7, ...] + */ +function setupSimdCvs(cvs, // Flat array: 4 × 8 words +mem32, count) { + // Unrolled for 4 chunks (common case) + if (count === 4) { + for (let w = 0; w < 8; w++) { + const base = SIMD_CV_BASE + w * 4; + mem32[base] = cvs[w]; // cv0[w] + mem32[base + 1] = cvs[8 + w]; // cv1[w] + mem32[base + 2] = cvs[16 + w]; // cv2[w] + mem32[base + 3] = cvs[24 + w]; // cv3[w] + } + } + else { + for (let w = 0; w < 8; w++) { + const base = SIMD_CV_BASE + w * 4; + for (let c = 0; c < count; c++) { + mem32[base + c] = cvs[c * 8 + w]; + } + for (let c = count; c < 4; c++) { + mem32[base + c] = 0; + } + } + } +} +/** + * Set up SIMD parameters (counters, flags, block lengths). + */ +function setupSimdParams(mem32, counters, blockLens, flagsArr, count) { + // Most chunk counters fit in 32 bits, so counter high is usually 0 + for (let i = 0; i < count; i++) { + mem32[SIMD_COUNTER_LOW_BASE + i] = counters[i]; + mem32[SIMD_COUNTER_HIGH_BASE + i] = 0; // Assume counters fit in 32 bits + mem32[SIMD_BLOCK_LEN_BASE + i] = blockLens[i]; + mem32[SIMD_FLAGS_BASE + i] = flagsArr[i]; + } + // Zero unused slots + for (let i = count; i < 4; i++) { + mem32[SIMD_COUNTER_LOW_BASE + i] = 0; + mem32[SIMD_COUNTER_HIGH_BASE + i] = 0; + mem32[SIMD_BLOCK_LEN_BASE + i] = 0; + mem32[SIMD_FLAGS_BASE + i] = 0; + } +} +/** + * Read output CVs from SIMD memory (untranspose). + */ +function readSimdOutputCvs(mem32, outputCvs, // Flat array: 4 × 8 words +count) { + // Unrolled for 4 chunks (common case) + if (count === 4) { + for (let w = 0; w < 8; w++) { + const base = SIMD_OUT_BASE + w * 4; + outputCvs[w] = mem32[base]; + outputCvs[8 + w] = mem32[base + 1]; + outputCvs[16 + w] = mem32[base + 2]; + outputCvs[24 + w] = mem32[base + 3]; + } + } + else { + for (let w = 0; w < 8; w++) { + const base = SIMD_OUT_BASE + w * 4; + for (let c = 0; c < count; c++) { + outputCvs[c * 8 + w] = mem32[base + c]; + } + } + } +} +function getBlockWords() { + if (!blockWords) { + blockWords = new Uint32Array(16); + } + return blockWords; +} +/** + * Hash a single chunk (up to 1024 bytes) with pre-created inputWords view. + * This is the optimized version that avoids creating Uint32Array views per chunk. + * (Fleek optimization Step 8) + */ +function hashChunkWithWords(input, inputWords, // Pre-created view of entire input +inputOffset, inputLen, chunkCounter, flags, cv, cvOffset) { + // Use reusable temporary CV for intermediate blocks (single-threaded safe) + reusableTempCv.set(constants_js_1.IV); + // Process full blocks + const fullBlocks = inputLen >>> 6; // inputLen / 64 + const remainder = inputLen & 63; // inputLen % 64 + // Calculate word offset for this chunk within the pre-created view + const chunkWordOffset = inputOffset >>> 2; + // Fast path for full chunks with aligned little-endian input + if (inputWords && remainder === 0 && inputLen === constants_js_1.CHUNK_LEN) { + // All 16 blocks are full, use fast path exclusively + let wordOff = chunkWordOffset; + // Block 0 (CHUNK_START) + (0, compress_js_1.compress)(reusableTempCv, 0, inputWords, wordOff, reusableTempCv, 0, false, chunkCounter, constants_js_1.BLOCK_LEN, flags | constants_js_1.CHUNK_START); + wordOff += 16; + // Blocks 1-14 (no special flags) + for (let i = 1; i < 15; i++) { + (0, compress_js_1.compress)(reusableTempCv, 0, inputWords, wordOff, reusableTempCv, 0, false, chunkCounter, constants_js_1.BLOCK_LEN, flags); + wordOff += 16; + } + // Block 15 (CHUNK_END) + (0, compress_js_1.compress)(reusableTempCv, 0, inputWords, wordOff, reusableTempCv, 0, false, chunkCounter, constants_js_1.BLOCK_LEN, flags | constants_js_1.CHUNK_END); + cv.set(reusableTempCv, cvOffset); + return; + } + // Slower path for partial chunks or non-aligned input + const totalBlocks = fullBlocks + (remainder > 0 ? 1 : 0); + const block = getBlockWords(); + for (let blockIdx = 0; blockIdx < totalBlocks; blockIdx++) { + const isFirst = blockIdx === 0; + const isLast = blockIdx === totalBlocks - 1; + const blockStart = blockIdx << 6; + const blockLen = isLast && remainder > 0 ? remainder : constants_js_1.BLOCK_LEN; + // Determine flags for this block + let blockFlags = flags; + if (isFirst) + blockFlags |= constants_js_1.CHUNK_START; + if (isLast) + blockFlags |= constants_js_1.CHUNK_END; + // Load block words + if (isLast && remainder > 0) { + // Partial final block - need zero padding + (0, utils_js_1.readLittleEndianWordsPartial)(input, inputOffset + blockStart, blockLen, block); + } + else if (inputWords && chunkWordOffset + (blockStart >>> 2) + 16 <= inputWords.length) { + // Fast path: use pre-created view directly + (0, compress_js_1.compress)(reusableTempCv, 0, inputWords, chunkWordOffset + (blockStart >>> 2), reusableTempCv, 0, false, chunkCounter, blockLen, blockFlags); + continue; + } + else { + (0, utils_js_1.readLittleEndianWordsFull)(input, inputOffset + blockStart, block); + } + (0, compress_js_1.compress)(reusableTempCv, 0, block, 0, reusableTempCv, 0, false, chunkCounter, blockLen, blockFlags); + } + // Copy result to output + cv.set(reusableTempCv, cvOffset); +} +/** + * Hash input using pure JavaScript. + * Handles the full Merkle tree construction. + */ +function hashPureJS(input, outputLen) { + const inputLen = input.length; + // Special case: empty input + if (inputLen === 0) { + const block = getBlockWords(); + block.fill(0); + // Use reusable output buffer for common 32-byte case + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + (0, compress_js_1.compress)(constants_js_1.IV, 0, block, 0, out, 0, outputLen > 32, 0, 0, constants_js_1.CHUNK_START | constants_js_1.CHUNK_END | constants_js_1.ROOT); + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && utils_js_1.IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (utils_js_1.IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } + else { + (0, utils_js_1.writeLittleEndianBytesPartial)(out, 0, result, 0, outputLen); + } + return result; + } + // Calculate number of chunks + const numChunks = Math.ceil(inputLen / constants_js_1.CHUNK_LEN); + // Single chunk optimization + if (numChunks === 1) { + // Use reusable output buffer for common 32-byte case + const cv = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + hashChunkRoot(input, 0, inputLen, 0, 0, cv, outputLen > 32); + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && utils_js_1.IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (utils_js_1.IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(cv.buffer, 0, outputLen)); + } + else { + (0, utils_js_1.writeLittleEndianBytesPartial)(cv, 0, result, 0, outputLen); + } + return result; + } + // Multiple chunks - need Merkle tree + // Use the global contiguous CV stack (no allocation) + const stack = HYPER_CV_STACK; + let stackLen = 0; + // Use reusable buffers (single-threaded safe) + const chunkCv = reusableChunkCv; + const parentBlock = reusablePureParentBlock; + const parentCv = reusablePureParentCv; + // Create Uint32Array view ONCE for entire input (Fleek optimization Step 8) + // This avoids creating views inside each chunk/block processing + let inputWords = null; + const canUseFastPath = utils_js_1.IS_LITTLE_ENDIAN && input.byteOffset % 4 === 0; + if (canUseFastPath) { + inputWords = new Uint32Array(input.buffer, input.byteOffset, inputLen >>> 2); + } + // Determine how many full chunks we have + const fullChunks = inputLen >>> 10; // inputLen / 1024 + const lastChunkLen = inputLen & 1023; // inputLen % 1024 + // Process all full chunks with fast path (inlined for performance) + if (canUseFastPath && inputWords) { + for (let chunkIdx = 0; chunkIdx < fullChunks; chunkIdx++) { + // Inline chunk processing for full chunks + chunkCv.set(constants_js_1.IV); + let wordOff = chunkIdx << 8; // chunkIdx * 256 (CHUNK_LEN/4) + // Block 0 (CHUNK_START) + (0, compress_js_1.compress)(chunkCv, 0, inputWords, wordOff, chunkCv, 0, false, chunkIdx, constants_js_1.BLOCK_LEN, constants_js_1.CHUNK_START); + wordOff += 16; + // Blocks 1-14 (no special flags) + for (let b = 1; b < 15; b++) { + (0, compress_js_1.compress)(chunkCv, 0, inputWords, wordOff, chunkCv, 0, false, chunkIdx, constants_js_1.BLOCK_LEN, 0); + wordOff += 16; + } + // Block 15 (CHUNK_END) + (0, compress_js_1.compress)(chunkCv, 0, inputWords, wordOff, chunkCv, 0, false, chunkIdx, constants_js_1.BLOCK_LEN, constants_js_1.CHUNK_END); + // Merge completed subtrees (avoid subarray by using index math) + let totalChunks = chunkIdx + 1; + let cvSrcOff = 0; + let cvSrc = chunkCv; + // Check if this is the last chunk overall + const isLastChunk = chunkIdx === fullChunks - 1 && lastChunkLen === 0; + while ((totalChunks & 1) === 0 && stackLen > 0) { + // Skip final merge if it would produce the root; let finalization handle it with ROOT flag + if (stackLen === 1 && isLastChunk) { + break; + } + stackLen--; + const stackOff = stackLen * 8; + // Copy left CV from stack to parentBlock[0..7] (unrolled) + copyCV8(stack, stackOff, parentBlock, 0); + // Copy current CV to parentBlock[8..15] (unrolled) + copyCV8(cvSrc, cvSrcOff, parentBlock, 8); + (0, compress_js_1.compress)(constants_js_1.IV, 0, parentBlock, 0, parentCv, 0, false, 0, constants_js_1.BLOCK_LEN, constants_js_1.PARENT); + cvSrc = parentCv; + cvSrcOff = 0; + totalChunks >>>= 1; + } + // Push CV to stack (unrolled) + const stackOff = stackLen * 8; + copyCV8(cvSrc, cvSrcOff, stack, stackOff); + stackLen++; + } + // Process last partial chunk if any + if (lastChunkLen > 0) { + hashChunkWithWords(input, inputWords, fullChunks * constants_js_1.CHUNK_LEN, lastChunkLen, fullChunks, 0, chunkCv, 0); + let totalChunks = fullChunks + 1; + let newCv = chunkCv; + let newCvOffset = 0; + while ((totalChunks & 1) === 0 && stackLen > 0) { + // Skip final merge; this IS the last chunk, let finalization handle ROOT flag + if (stackLen === 1) { + break; + } + stackLen--; + const stackOff = stackLen * 8; + // Copy from stack to parentBlock[0..7] (unrolled) + copyCV8(stack, stackOff, parentBlock, 0); + // Copy from newCv to parentBlock[8..15] (unrolled) + copyCV8(newCv, newCvOffset, parentBlock, 8); + (0, compress_js_1.compress)(constants_js_1.IV, 0, parentBlock, 0, parentCv, 0, false, 0, constants_js_1.BLOCK_LEN, constants_js_1.PARENT); + newCv = parentCv; + newCvOffset = 0; + totalChunks >>>= 1; + } + // Push CV to stack (unrolled) + const pushOff = stackLen * 8; + copyCV8(newCv, newCvOffset, stack, pushOff); + stackLen++; + } + } + else { + // Slow path for unaligned or big-endian + for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) { + const chunkStart = chunkIdx * constants_js_1.CHUNK_LEN; + const chunkLen = Math.min(constants_js_1.CHUNK_LEN, inputLen - chunkStart); + hashChunkWithWords(input, inputWords, chunkStart, chunkLen, chunkIdx, 0, chunkCv, 0); + // Merge completed subtrees + let totalChunks = chunkIdx + 1; + let newCv = chunkCv; + let newCvOffset = 0; + // Check if this is the last chunk + const isLastChunk = chunkIdx === numChunks - 1; + while ((totalChunks & 1) === 0 && stackLen > 0) { + // Skip final merge if it would produce the root; let finalization handle it with ROOT flag + if (stackLen === 1 && isLastChunk) { + break; + } + stackLen--; + const stackOff = stackLen * 8; + // Copy from stack to parentBlock[0..7] (unrolled) + copyCV8(stack, stackOff, parentBlock, 0); + // Copy from newCv to parentBlock[8..15] (unrolled) + copyCV8(newCv, newCvOffset, parentBlock, 8); + (0, compress_js_1.compress)(constants_js_1.IV, 0, parentBlock, 0, parentCv, 0, false, 0, constants_js_1.BLOCK_LEN, constants_js_1.PARENT); + newCv = parentCv; + newCvOffset = 0; + totalChunks >>>= 1; + } + // Push CV to stack (unrolled) + const pushOff = stackLen * 8; + copyCV8(newCv, newCvOffset, stack, pushOff); + stackLen++; + } + } + // Finalize: merge remaining stack entries + while (stackLen > 1) { + stackLen--; + const rightOff = stackLen * 8; + stackLen--; + const leftOff = stackLen * 8; + // Copy left CV to parentBlock[0..7] and right CV to parentBlock[8..15] (unrolled) + copyCV8(stack, leftOff, parentBlock, 0); + copyCV8(stack, rightOff, parentBlock, 8); + if (stackLen === 0) { + // This is the root - use reusable output buffer for common 32-byte case + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + (0, compress_js_1.compress)(constants_js_1.IV, 0, parentBlock, 0, out, 0, outputLen > 32, 0, constants_js_1.BLOCK_LEN, constants_js_1.PARENT | constants_js_1.ROOT); + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && utils_js_1.IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (utils_js_1.IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } + else { + (0, utils_js_1.writeLittleEndianBytesPartial)(out, 0, result, 0, outputLen); + } + return result; + } + (0, compress_js_1.compress)(constants_js_1.IV, 0, parentBlock, 0, parentCv, 0, false, 0, constants_js_1.BLOCK_LEN, constants_js_1.PARENT); + // Push to stack (unrolled) + copyCV8(parentCv, 0, stack, stackLen * 8); + stackLen++; + } + // Single entry in stack - this is the root + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + const lastBlock = getBlockWords(); + lastBlock.fill(0); + // Copy first 8 words from stack (unrolled) + copyCV8(stack, 0, lastBlock, 0); + (0, compress_js_1.compress)(constants_js_1.IV, 0, lastBlock, 0, out, 0, outputLen > 32, 0, constants_js_1.BLOCK_LEN, constants_js_1.ROOT); + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && utils_js_1.IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (utils_js_1.IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } + else { + (0, utils_js_1.writeLittleEndianBytesPartial)(out, 0, result, 0, outputLen); + } + return result; +} +/** + * Hash a single chunk that is also the root (single chunk input). + */ +function hashChunkRoot(input, inputOffset, inputLen, chunkCounter, flags, out, fullOutput) { + // Use reusable tempCv (single-threaded safe) + reusableTempCv.set(constants_js_1.IV); + const block = getBlockWords(); + // Process full blocks + const fullBlocks = inputLen >>> 6; + const remainder = inputLen & 63; + const totalBlocks = fullBlocks + (remainder > 0 ? 1 : 0) || 1; // At least 1 block + // Create a Uint32Array view if possible + let inputWords = null; + if (utils_js_1.IS_LITTLE_ENDIAN && (input.byteOffset + inputOffset) % 4 === 0 && inputLen >= 4) { + inputWords = new Uint32Array(input.buffer, input.byteOffset + inputOffset, inputLen >>> 2); + } + for (let blockIdx = 0; blockIdx < totalBlocks; blockIdx++) { + const isFirst = blockIdx === 0; + const isLast = blockIdx === totalBlocks - 1; + const blockStart = blockIdx << 6; + const blockLen = isLast ? remainder || (inputLen > 0 ? constants_js_1.BLOCK_LEN : 0) : constants_js_1.BLOCK_LEN; + // Determine flags + let blockFlags = flags; + if (isFirst) + blockFlags |= constants_js_1.CHUNK_START; + if (isLast) + blockFlags |= constants_js_1.CHUNK_END | constants_js_1.ROOT; + // Load block + if (isLast && remainder > 0) { + (0, utils_js_1.readLittleEndianWordsPartial)(input, inputOffset + blockStart, blockLen, block); + } + else if (inputLen === 0) { + block.fill(0); + } + else if (inputWords && (blockStart >>> 2) + 16 <= inputWords.length) { + // Fast path + (0, compress_js_1.compress)(reusableTempCv, 0, inputWords, blockStart >>> 2, isLast ? out : reusableTempCv, 0, isLast && fullOutput, chunkCounter, blockLen, blockFlags); + continue; + } + else { + (0, utils_js_1.readLittleEndianWordsFull)(input, inputOffset + blockStart, block); + } + (0, compress_js_1.compress)(reusableTempCv, 0, block, 0, isLast ? out : reusableTempCv, 0, isLast && fullOutput, chunkCounter, blockLen, blockFlags); + } +} +/** + * Hash using WASM SIMD - processes 4 chunks in parallel. + * Falls back to pure JS if SIMD fails. + */ +function hashSimd(input, outputLen) { + const mem = (0, wasm_simd_js_1.getSimdMemory)(); + if (!mem) { + return hashPureJS(input, outputLen); + } + const { view32 } = mem; + const inputLen = input.length; + const numChunks = Math.ceil(inputLen / constants_js_1.CHUNK_LEN); + // For small inputs, pure JS is faster (no transpose overhead) + if (numChunks < 4) { + return hashPureJS(input, outputLen); + } + // Try to use WASM arena buffers (zero JS heap allocation) + // Falls back to JS buffers if arena not available + const arena = (0, wasm_simd_js_1.getArenaBuffers)(); + const useWasmParent = arena !== null; // Use WASM parent compress when arena available + let stack; + let tempCvs; + let parentBlock; + let parentCv; + if (arena) { + // Use WASM-backed arena buffers + stack = arena.cvStack; + tempCvs = arena.tempCvs; + parentBlock = arena.parentBlock; + parentCv = arena.chunkCv; + } + else { + // Fallback to JS heap buffers - use global contiguous stack (no allocation) + stack = HYPER_CV_STACK; + tempCvs = reusableSimdCvs; + parentBlock = reusableSimdParentBlock; + parentCv = reusableSimdParentCv; + } + let stackLen = 0; + // Use TypedArrays instead of JS arrays for block parameters + const offsets = reusableOffsets; + const counters = reusableCounters; + const blockLens = reusableBlockLens; + const flagsArr = reusableFlags; + // Create Uint32Array view once for entire hash call (optimization: avoid allocation in hot loop) + const inputWords = utils_js_1.IS_LITTLE_ENDIAN && input.byteOffset % 4 === 0 + ? new Uint32Array(input.buffer, input.byteOffset, input.byteLength >>> 2) + : null; + // Calculate number of full chunks (1024 bytes each) + const numFullChunks = inputLen >>> 10; // inputLen / 1024 + // Process chunks in groups of 4 + let chunkIdx = 0; + while (chunkIdx < numChunks) { + const groupSize = Math.min(4, numChunks - chunkIdx); + // === BATCH FAST PATH: 4 full chunks === + // Use compressChunks4x for groups of exactly 4 full chunks + // This reduces 16 WASM calls to 1 per group + const canUseBatchPath = groupSize === 4 && chunkIdx + 4 <= numFullChunks; + if (canUseBatchPath) { + // Set up chunk offsets for batch transpose + batchChunkOffsets[0] = chunkIdx * constants_js_1.CHUNK_LEN; + batchChunkOffsets[1] = (chunkIdx + 1) * constants_js_1.CHUNK_LEN; + batchChunkOffsets[2] = (chunkIdx + 2) * constants_js_1.CHUNK_LEN; + batchChunkOffsets[3] = (chunkIdx + 3) * constants_js_1.CHUNK_LEN; + // Transpose all 64 blocks (4 chunks × 16 blocks) at once + transposeBatchToSimd(input, batchChunkOffsets, view32, inputWords); + // Set up initial CVs (IV) in batch memory - transposed layout + for (let w = 0; w < 8; w++) { + const ivWord = constants_js_1.IV[w]; + const base = BATCH_CV_BASE + w * 4; + view32[base] = ivWord; + view32[base + 1] = ivWord; + view32[base + 2] = ivWord; + view32[base + 3] = ivWord; + } + // Set up counters in batch memory + view32[BATCH_COUNTER_LOW_BASE] = chunkIdx; + view32[BATCH_COUNTER_LOW_BASE + 1] = chunkIdx + 1; + view32[BATCH_COUNTER_LOW_BASE + 2] = chunkIdx + 2; + view32[BATCH_COUNTER_LOW_BASE + 3] = chunkIdx + 3; + // Set up base flags (0 - no keyed hashing) + view32[BATCH_FLAGS_BASE_OFFSET] = 0; + view32[BATCH_FLAGS_BASE_OFFSET + 1] = 0; + view32[BATCH_FLAGS_BASE_OFFSET + 2] = 0; + view32[BATCH_FLAGS_BASE_OFFSET + 3] = 0; + // Run batched compress (16 blocks × 4 chunks in one call!) + (0, wasm_simd_js_1.runCompressChunks4x)(); + // Read output CVs from batch output - untranspose to tempCvs + for (let w = 0; w < 8; w++) { + const base = BATCH_OUTPUT_BASE + w * 4; + tempCvs[w] = view32[base]; // chunk 0 + tempCvs[8 + w] = view32[base + 1]; // chunk 1 + tempCvs[16 + w] = view32[base + 2]; // chunk 2 + tempCvs[24 + w] = view32[base + 3]; // chunk 3 + } + } + else { + // === STANDARD PATH: block-by-block processing === + // Used for partial chunks or groups < 4 + // Initialize CVs for this group to IV (flat array: 4 × 8 words) + for (let g = 0; g < groupSize; g++) { + const base = g * 8; + tempCvs[base] = constants_js_1.IV[0]; + tempCvs[base + 1] = constants_js_1.IV[1]; + tempCvs[base + 2] = constants_js_1.IV[2]; + tempCvs[base + 3] = constants_js_1.IV[3]; + tempCvs[base + 4] = constants_js_1.IV[4]; + tempCvs[base + 5] = constants_js_1.IV[5]; + tempCvs[base + 6] = constants_js_1.IV[6]; + tempCvs[base + 7] = constants_js_1.IV[7]; + } + // Process all 16 blocks of each chunk in this group + for (let blockIdx = 0; blockIdx < 16; blockIdx++) { + // Calculate block offsets and parameters (reuse arrays) + for (let g = 0; g < groupSize; g++) { + const thisChunkIdx = chunkIdx + g; + const chunkStart = thisChunkIdx * constants_js_1.CHUNK_LEN; + const chunkLen = Math.min(constants_js_1.CHUNK_LEN, inputLen - chunkStart); + const thisBlockStart = chunkStart + blockIdx * constants_js_1.BLOCK_LEN; + // Determine block length for this specific block + const blockStartInChunk = blockIdx * constants_js_1.BLOCK_LEN; + let thisBlockLen = constants_js_1.BLOCK_LEN; + if (blockStartInChunk >= chunkLen) { + thisBlockLen = 0; + } + else if (blockStartInChunk + constants_js_1.BLOCK_LEN > chunkLen) { + thisBlockLen = chunkLen - blockStartInChunk; + } + offsets[g] = thisBlockStart; + counters[g] = thisChunkIdx; + // Determine flags + let flags = 0; + if (blockIdx === 0) + flags |= constants_js_1.CHUNK_START; + const totalBlocksInChunk = Math.ceil(chunkLen / constants_js_1.BLOCK_LEN) || 1; + if (blockIdx === totalBlocksInChunk - 1) + flags |= constants_js_1.CHUNK_END; + blockLens[g] = thisBlockLen; + flagsArr[g] = flags; + } + // Check if any blocks need processing + if (blockLens[0] === 0 && blockLens[1] === 0 && blockLens[2] === 0 && blockLens[3] === 0) + continue; + // Transpose blocks into SIMD memory (pass pre-created view to avoid allocation) + transposeBlocksToSimd(input, offsets, blockLens, view32, groupSize, inputWords); + // Set up CVs in SIMD memory + setupSimdCvs(tempCvs, view32, groupSize); + // Set up parameters + setupSimdParams(view32, counters, blockLens, flagsArr, groupSize); + // Run SIMD compress + (0, wasm_simd_js_1.runCompress4x)(); + // Read output CVs back + readSimdOutputCvs(view32, simdChunkCvs, groupSize); + // Update tempCvs - copy from simdChunkCvs (both are flat 32-word arrays) + // simdChunkCvs layout matches tempCvs: [cv0_w0..cv0_w7, cv1_w0..cv1_w7, ...] + // IMPORTANT: Only update CVs for chunks that had data in this block! + // Skipping this check would corrupt CVs for partial chunks after their final block. + for (let g = 0; g < groupSize; g++) { + if (blockLens[g] === 0) + continue; // Don't update CV for chunks with no data in this block + const base = g * 8; + tempCvs[base] = simdChunkCvs[base]; + tempCvs[base + 1] = simdChunkCvs[base + 1]; + tempCvs[base + 2] = simdChunkCvs[base + 2]; + tempCvs[base + 3] = simdChunkCvs[base + 3]; + tempCvs[base + 4] = simdChunkCvs[base + 4]; + tempCvs[base + 5] = simdChunkCvs[base + 5]; + tempCvs[base + 6] = simdChunkCvs[base + 6]; + tempCvs[base + 7] = simdChunkCvs[base + 7]; + } + } + } + // Merge each chunk's CV into the Merkle tree + for (let g = 0; g < groupSize; g++) { + const thisChunkIdx = chunkIdx + g; + // Merge completed subtrees + let totalChunks = thisChunkIdx + 1; + // Track newCv source - either from tempCvs or parentCv + let newCvBase = g * 8; // Offset into tempCvs + let newCvSrc = tempCvs; + // Check if this is the last chunk + const isLastChunk = thisChunkIdx === numChunks - 1; + while ((totalChunks & 1) === 0 && stackLen > 0) { + // Skip final merge if it would produce the root; let finalization handle it with ROOT flag + if (stackLen === 1 && isLastChunk) { + break; + } + // Pop left child + stackLen--; + const stackOff = stackLen * 8; + // Copy from stack to parentBlock[0..7] (unrolled) + copyCV8(stack, stackOff, parentBlock, 0); + // Copy from newCv source to parentBlock[8..15] (unrolled) + copyCV8(newCvSrc, newCvBase, parentBlock, 8); + if (useWasmParent) { + // WASM parent compress - data already in arena buffers + (0, wasm_simd_js_1.runCompressParent)(); + } + else { + (0, compress_js_1.compress)(constants_js_1.IV, 0, parentBlock, 0, parentCv, 0, false, 0, constants_js_1.BLOCK_LEN, constants_js_1.PARENT); + } + newCvSrc = parentCv; + newCvBase = 0; + totalChunks >>>= 1; + } + // Push to stack (unrolled) + const pushOff = stackLen * 8; + copyCV8(newCvSrc, newCvBase, stack, pushOff); + stackLen++; + } + chunkIdx += groupSize; + } + // Finalize: merge remaining stack entries + while (stackLen > 1) { + stackLen--; + const rightOff = stackLen * 8; + stackLen--; + const leftOff = stackLen * 8; + // Copy left CV to parentBlock[0..7] and right CV to parentBlock[8..15] (unrolled) + copyCV8(stack, leftOff, parentBlock, 0); + copyCV8(stack, rightOff, parentBlock, 8); + if (stackLen === 0) { + // This is the root - use reusable output buffer + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + (0, compress_js_1.compress)(constants_js_1.IV, 0, parentBlock, 0, out, 0, outputLen > 32, 0, constants_js_1.BLOCK_LEN, constants_js_1.PARENT | constants_js_1.ROOT); + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && utils_js_1.IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (utils_js_1.IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } + else { + (0, utils_js_1.writeLittleEndianBytesPartial)(out, 0, result, 0, outputLen); + } + return result; + } + if (useWasmParent) { + // WASM parent compress - data already in arena buffers + (0, wasm_simd_js_1.runCompressParent)(); + } + else { + (0, compress_js_1.compress)(constants_js_1.IV, 0, parentBlock, 0, parentCv, 0, false, 0, constants_js_1.BLOCK_LEN, constants_js_1.PARENT); + } + // Push to stack (unrolled) + copyCV8(parentCv, 0, stack, stackLen * 8); + stackLen++; + } + // Single entry in stack - finalize as root + if (stackLen === 1) { + const block = getBlockWords(); + block.fill(0); + // Copy first 8 words from stack (unrolled) + copyCV8(stack, 0, block, 0); + // Use reusable output buffer + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + (0, compress_js_1.compress)(constants_js_1.IV, 0, block, 0, out, 0, outputLen > 32, 0, constants_js_1.BLOCK_LEN, constants_js_1.ROOT); + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && utils_js_1.IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (utils_js_1.IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } + else { + (0, utils_js_1.writeLittleEndianBytesPartial)(out, 0, result, 0, outputLen); + } + return result; + } + // Should not reach here + return hashPureJS(input, outputLen); +} +/** + * Hash input data and return the result. + * Automatically uses WASM SIMD for large inputs when available. + * + * @param input - Data to hash + * @param outputLength - Number of bytes to output (default: 32) + * @returns The hash output + */ +function hash(input, outputLength = constants_js_1.OUT_LEN) { + // For large inputs, use SIMD for ~1.5x performance improvement + if (input.length >= SIMD_THRESHOLD && ensureSimdSync()) { + return hashSimd(input, outputLength); + } + return hashPureJS(input, outputLength); +} +/** + * Pre-warm SIMD initialization (call early to avoid latency later). + */ +function warmupSimd() { + return ensureSimdSync(); +} +/** + * Hash input data directly into a caller-provided output buffer. + * Zero-allocation for the common 32-byte case - ideal for performance-critical code. + * + * @param input - Data to hash + * @param output - Pre-allocated output buffer (must be at least outputLength bytes) + * @param outputLength - Number of bytes to output (default: 32, max: output.length) + */ +function hashInto(input, output, outputLength = constants_js_1.OUT_LEN) { + // Validate output buffer + if (output.length < outputLength) { + throw new Error(`Output buffer too small: ${output.length} < ${outputLength}`); + } + // For large inputs, use SIMD for ~1.5x performance improvement + if (input.length >= SIMD_THRESHOLD && ensureSimdSync()) { + hashSimdInto(input, output, outputLength); + return; + } + hashPureJSInto(input, output, outputLength); +} +/** + * Internal: Hash using pure JS, writing directly to output buffer. + */ +function hashPureJSInto(input, output, outputLen) { + const inputLen = input.length; + // Special case: empty input + if (inputLen === 0) { + const block = getBlockWords(); + block.fill(0); + const out = outputLen <= 32 ? reusableOut8 : new Uint32Array(16); + (0, compress_js_1.compress)(constants_js_1.IV, 0, block, 0, out, 0, outputLen > 32, 0, 0, constants_js_1.CHUNK_START | constants_js_1.CHUNK_END | constants_js_1.ROOT); + // Copy result to output + if (utils_js_1.IS_LITTLE_ENDIAN) { + output.set(new Uint8Array(out.buffer, out.byteOffset, outputLen)); + } + else { + (0, utils_js_1.writeLittleEndianBytesPartial)(out, 0, output, 0, outputLen); + } + return; + } + // Calculate number of chunks + const numChunks = Math.ceil(inputLen / constants_js_1.CHUNK_LEN); + // Single chunk optimization + if (numChunks === 1) { + const cv = outputLen <= 32 ? reusableOut8 : new Uint32Array(16); + hashChunkRoot(input, 0, inputLen, 0, 0, cv, outputLen > 32); + // Copy result to output + if (utils_js_1.IS_LITTLE_ENDIAN) { + output.set(new Uint8Array(cv.buffer, cv.byteOffset, outputLen)); + } + else { + (0, utils_js_1.writeLittleEndianBytesPartial)(cv, 0, output, 0, outputLen); + } + return; + } + // Multiple chunks - delegate to hashPureJS and copy result + const result = hashPureJS(input, outputLen); + output.set(result); +} +/** + * Internal: Hash using SIMD, writing directly to output buffer. + */ +function hashSimdInto(input, output, outputLen) { + // Delegate to hashSimd and copy result (SIMD path already optimized) + const result = hashSimd(input, outputLen); + output.set(result); +} diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/hasher.d.ts b/node_modules/@huggingface/blake3-jit/dist/commonjs/hasher.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..37ecf0d590f939d37d0bd762df953c358548fcdc --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/hasher.d.ts @@ -0,0 +1,108 @@ +/** + * BLAKE3 Hasher - Incremental hashing with support for all modes + * + * Supports: + * - Regular hashing + * - Keyed hashing (MAC) + * - Key derivation (derive_key) + * - XOF (eXtendable Output Function) mode + */ +/** + * Output state for XOF (eXtendable Output Function) mode. + * Allows reading arbitrary amounts of output. + */ +export declare class XofReader { + private inputCv; + private blockWords; + private counter; + private blockLen; + private flags; + private outputBlock; + private outputBlockOffset; + constructor(inputCv: Uint32Array, blockWords: Uint32Array, counter: number, blockLen: number, flags: number); + /** + * Read the next `length` bytes of output. + */ + read(length: number): Uint8Array; +} +/** + * Main BLAKE3 Hasher class. + * + * Usage: + * const hasher = new Hasher(); + * hasher.update(data); + * const hash = hasher.finalize(); + * + * Or with chaining: + * const hash = new Hasher().update(data).finalize(); + */ +export declare class Hasher { + private chunkState; + private keyWords; + private cvStack; + private cvStackLen; + private flags; + private parentBlock; + private parentCv; + private chunkCv; + private outWords; + private finalizeCv; + /** + * Create a new Hasher. + * + * @param keyWords - Initial key words (IV for regular hashing) + * @param flags - Domain separation flags + */ + constructor(keyWords?: Uint32Array, flags?: number); + /** + * Reset the hasher to process a new message with the same key/flags. + * Reuses all internal buffers — zero allocations. + */ + reset(): this; + /** + * Create a new keyed hasher (MAC). + * + * @param key - 32-byte key + */ + static newKeyed(key: Uint8Array): Hasher; + /** + * Create a new key derivation hasher. + * + * @param context - Context string for domain separation + */ + static newDeriveKey(context: string): Hasher; + /** + * Push a chaining value onto the stack. + */ + private pushCv; + /** + * Pop a chaining value from the stack. + */ + private popCv; + /** + * Add a chunk's chaining value and merge completed subtrees. + */ + private addChunkCv; + /** + * Update the hasher with input data. + * + * @param input - Data to hash + * @returns this (for chaining) + */ + update(input: Uint8Array): this; + /** + * Get the output parameters (for XOF mode or finalization). + */ + private finalizeOutput; + /** + * Finalize the hash and return the result. + * + * @param outputLength - Number of bytes to output (default: 32) + * @returns The hash output + */ + finalize(outputLength?: number): Uint8Array; + /** + * Finalize and return an XOF reader for arbitrary-length output. + */ + finalizeXof(): XofReader; +} diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/hasher.js b/node_modules/@huggingface/blake3-jit/dist/commonjs/hasher.js new file mode 100644 index 0000000000000000000000000000000000000000..ec44ea437671c95fc03ee3ead6515295e0f1445a --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/hasher.js @@ -0,0 +1,396 @@ +"use strict"; +/** + * BLAKE3 Hasher - Incremental hashing with support for all modes + * + * Supports: + * - Regular hashing + * - Keyed hashing (MAC) + * - Key derivation (derive_key) + * - XOF (eXtendable Output Function) mode + */ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Hasher = exports.XofReader = void 0; +const compress_js_1 = require("./compress.js"); +const constants_js_1 = require("./constants.js"); +const utils_js_1 = require("./utils.js"); +/** + * Output state for XOF (eXtendable Output Function) mode. + * Allows reading arbitrary amounts of output. + */ +class XofReader { + inputCv; + blockWords; + counter; + blockLen; + flags; + outputBlock; + outputBlockOffset; + constructor(inputCv, blockWords, counter, blockLen, flags) { + this.inputCv = inputCv; + this.blockWords = blockWords; + this.counter = counter; + this.blockLen = blockLen; + this.flags = flags | constants_js_1.ROOT; + this.outputBlock = new Uint32Array(16); + this.outputBlockOffset = 64; // Forces generation on first read + } + /** + * Read the next `length` bytes of output. + */ + read(length) { + const output = new Uint8Array(length); + let outputOffset = 0; + while (outputOffset < length) { + // Generate new output block if needed + if (this.outputBlockOffset >= 64) { + (0, compress_js_1.compress)(this.inputCv, 0, this.blockWords, 0, this.outputBlock, 0, true, // full 64-byte output + this.counter++, this.blockLen, this.flags); + this.outputBlockOffset = 0; + } + // Copy bytes from output block + const available = 64 - this.outputBlockOffset; + const toCopy = Math.min(available, length - outputOffset); + // Optimized copy using writeLittleEndianBytesPartial + const wordOffset = this.outputBlockOffset >>> 2; + const byteWithinWord = this.outputBlockOffset & 3; + if (byteWithinWord === 0 && toCopy >= 4) { + // Aligned copy - can use word-at-a-time + const fullWords = toCopy >>> 2; + (0, utils_js_1.writeLittleEndianBytesPartial)(this.outputBlock, wordOffset, output, outputOffset, fullWords << 2); + const bytesCopied = fullWords << 2; + outputOffset += bytesCopied; + this.outputBlockOffset += bytesCopied; + } + else { + // Byte-by-byte for unaligned access + for (let i = 0; i < toCopy; i++) { + const wordIdx = (this.outputBlockOffset + i) >>> 2; + const byteIdx = (this.outputBlockOffset + i) & 3; + output[outputOffset + i] = (this.outputBlock[wordIdx] >>> (byteIdx << 3)) & 0xff; + } + outputOffset += toCopy; + this.outputBlockOffset += toCopy; + } + } + return output; + } +} +exports.XofReader = XofReader; +/** + * Chunk state for processing input data. + * Each chunk is 1024 bytes and produces an 8-word chaining value. + */ +class ChunkState { + chainingValue; + chunkCounter; + blockWords; + blockLen; + blocksCompressed; + flags; + constructor(keyWords, chunkCounter, flags) { + this.chainingValue = new Uint32Array(keyWords); + this.chunkCounter = chunkCounter; + this.blockWords = new Uint32Array(16); + this.blockLen = 0; + this.blocksCompressed = 0; + this.flags = flags; + } + resetTo(keyWords, chunkCounter, flags) { + this.chainingValue.set(keyWords); + this.chunkCounter = chunkCounter; + this.blockLen = 0; + this.blocksCompressed = 0; + this.flags = flags; + } + /** + * Get the flags for the current block. + */ + startFlag() { + return this.blocksCompressed === 0 ? constants_js_1.CHUNK_START : 0; + } + /** + * Update the chunk state with input data. + * Returns the number of bytes consumed. + */ + update(input, inputOffset, inputLen) { + let consumed = 0; + while (inputLen > 0) { + // If we have a full block, compress it + if (this.blockLen === constants_js_1.BLOCK_LEN) { + (0, compress_js_1.compress)(this.chainingValue, 0, this.blockWords, 0, this.chainingValue, 0, false, this.chunkCounter, constants_js_1.BLOCK_LEN, this.flags | this.startFlag()); + this.blocksCompressed++; + this.blockLen = 0; + } + // Fill the block buffer + const want = constants_js_1.BLOCK_LEN - this.blockLen; + const take = Math.min(want, inputLen); + if (this.blockLen === 0 && take === constants_js_1.BLOCK_LEN) { + (0, utils_js_1.readLittleEndianWordsFull)(input, inputOffset, this.blockWords); + } + else { + // Partial block - byte-by-byte into correct position + for (let i = 0; i < take; i++) { + const pos = this.blockLen + i; + const wordIdx = pos >>> 2; + const byteIdx = pos & 3; + if (byteIdx === 0) { + this.blockWords[wordIdx] = input[inputOffset + i]; + } + else { + this.blockWords[wordIdx] |= input[inputOffset + i] << (byteIdx << 3); + } + } + } + this.blockLen += take; + inputOffset += take; + inputLen -= take; + consumed += take; + } + return consumed; + } + /** + * Finalize this chunk and return its output. + * Returns 8 words (chaining value) or 16 words (if root). + */ + output() { + // Zero-pad unused words in blockWords to avoid stale data from previous blocks + // This is necessary when a partial block follows a full block within the same chunk + const usedWords = (this.blockLen + 3) >>> 2; // ceil(blockLen / 4) + for (let i = usedWords; i < 16; i++) { + this.blockWords[i] = 0; + } + return { + inputCv: this.chainingValue, + blockWords: this.blockWords, + blockLen: this.blockLen, + counter: this.chunkCounter, + flags: this.flags | this.startFlag() | constants_js_1.CHUNK_END, + }; + } + /** + * Get the number of bytes in this chunk. + */ + len() { + return this.blocksCompressed * constants_js_1.BLOCK_LEN + this.blockLen; + } +} +/** + * Main BLAKE3 Hasher class. + * + * Usage: + * const hasher = new Hasher(); + * hasher.update(data); + * const hash = hasher.finalize(); + * + * Or with chaining: + * const hash = new Hasher().update(data).finalize(); + */ +class Hasher { + chunkState; + keyWords; + cvStack; + cvStackLen; + flags; + parentBlock; + parentCv; + chunkCv; + outWords; + finalizeCv; + /** + * Create a new Hasher. + * + * @param keyWords - Initial key words (IV for regular hashing) + * @param flags - Domain separation flags + */ + constructor(keyWords, flags) { + this.keyWords = keyWords ? new Uint32Array(keyWords) : new Uint32Array(constants_js_1.IV); + this.flags = flags ?? 0; + this.chunkState = new ChunkState(this.keyWords, 0, this.flags); + this.cvStack = new Uint32Array(constants_js_1.MAX_DEPTH * 8); + this.cvStackLen = 0; + this.parentBlock = new Uint32Array(16); + this.parentCv = new Uint32Array(8); + this.chunkCv = new Uint32Array(8); + this.outWords = new Uint32Array(16); + this.finalizeCv = new Uint32Array(8); + } + /** + * Reset the hasher to process a new message with the same key/flags. + * Reuses all internal buffers — zero allocations. + */ + reset() { + this.chunkState.resetTo(this.keyWords, 0, this.flags); + this.cvStackLen = 0; + return this; + } + /** + * Create a new keyed hasher (MAC). + * + * @param key - 32-byte key + */ + static newKeyed(key) { + if (key.length !== constants_js_1.KEY_LEN) { + throw new Error(`Key must be ${constants_js_1.KEY_LEN} bytes, got ${key.length}`); + } + const keyWords = new Uint32Array(8); + if (utils_js_1.IS_LITTLE_ENDIAN) { + const view = new Uint32Array(key.buffer, key.byteOffset, 8); + keyWords.set(view); + } + else { + for (let i = 0; i < 8; i++) { + const off = i * 4; + keyWords[i] = key[off] | (key[off + 1] << 8) | (key[off + 2] << 16) | (key[off + 3] << 24); + } + } + return new Hasher(keyWords, constants_js_1.KEYED_HASH); + } + /** + * Create a new key derivation hasher. + * + * @param context - Context string for domain separation + */ + static newDeriveKey(context) { + // First, hash the context string with DERIVE_KEY_CONTEXT flag + const contextBytes = (0, utils_js_1.encodeUTF8)(context); + const contextHasher = new Hasher(new Uint32Array(constants_js_1.IV), constants_js_1.DERIVE_KEY_CONTEXT); + contextHasher.update(contextBytes); + // Get the context key + const contextKey = new Uint32Array(8); + const output = contextHasher.finalizeOutput(); + (0, compress_js_1.compress)(output.inputCv, 0, output.blockWords, 0, contextKey, 0, false, output.counter, output.blockLen, output.flags | constants_js_1.ROOT); + // Return a hasher initialized with the context key + return new Hasher(contextKey, constants_js_1.DERIVE_KEY_MATERIAL); + } + /** + * Push a chaining value onto the stack. + */ + pushCv(cv, cvOffset) { + this.cvStack.set(cv.subarray(cvOffset, cvOffset + 8), this.cvStackLen * 8); + this.cvStackLen++; + } + /** + * Pop a chaining value from the stack. + */ + popCv(out, outOffset) { + this.cvStackLen--; + out.set(this.cvStack.subarray(this.cvStackLen * 8, (this.cvStackLen + 1) * 8), outOffset); + } + /** + * Add a chunk's chaining value and merge completed subtrees. + */ + addChunkCv(newCv, newCvOffset, totalChunks) { + const parentBlock = this.parentBlock; + const parentCv = this.parentCv; + while ((totalChunks & 1) === 0) { + // Pop left child, new CV is right child + this.popCv(parentBlock, 0); + parentBlock.set(newCv.subarray(newCvOffset, newCvOffset + 8), 8); + (0, compress_js_1.compress)(this.keyWords, 0, parentBlock, 0, parentCv, 0, false, 0, constants_js_1.BLOCK_LEN, this.flags | constants_js_1.PARENT); + newCv = parentCv; + newCvOffset = 0; + totalChunks >>>= 1; + } + this.pushCv(newCv, newCvOffset); + } + /** + * Update the hasher with input data. + * + * @param input - Data to hash + * @returns this (for chaining) + */ + update(input) { + let inputOffset = 0; + let inputLen = input.length; + // Fill the current chunk + while (inputLen > 0) { + // If current chunk is full, finalize it and start a new one + if (this.chunkState.len() === constants_js_1.CHUNK_LEN) { + const output = this.chunkState.output(); + const chunkCv = this.chunkCv; + (0, compress_js_1.compress)(output.inputCv, 0, output.blockWords, 0, chunkCv, 0, false, output.counter, output.blockLen, output.flags); + const totalChunks = this.chunkState.chunkCounter + 1; + this.addChunkCv(chunkCv, 0, totalChunks); + this.chunkState.resetTo(this.keyWords, totalChunks, this.flags); + } + // Fill the current chunk + const want = constants_js_1.CHUNK_LEN - this.chunkState.len(); + const take = Math.min(want, inputLen); + this.chunkState.update(input, inputOffset, take); + inputOffset += take; + inputLen -= take; + } + return this; + } + /** + * Get the output parameters (for XOF mode or finalization). + */ + finalizeOutput() { + let output = this.chunkState.output(); + let parentBlock = this.parentBlock; + let cv = this.finalizeCv; + // If there are chunks on the stack, merge them + if (this.cvStackLen > 0) { + // First compress the current chunk + (0, compress_js_1.compress)(output.inputCv, 0, output.blockWords, 0, cv, 0, false, output.counter, output.blockLen, output.flags); + // Merge with parent nodes from stack + while (this.cvStackLen > 0) { + this.cvStackLen--; + parentBlock.set(this.cvStack.subarray(this.cvStackLen * 8, (this.cvStackLen + 1) * 8), 0); + parentBlock.set(cv, 8); + if (this.cvStackLen > 0) { + (0, compress_js_1.compress)(this.keyWords, 0, parentBlock, 0, cv, 0, false, 0, constants_js_1.BLOCK_LEN, this.flags | constants_js_1.PARENT); + } + else { + // This is the root - return output params + return { + inputCv: this.keyWords, + blockWords: parentBlock, + blockLen: constants_js_1.BLOCK_LEN, + counter: 0, + flags: this.flags | constants_js_1.PARENT, + }; + } + } + } + // Single chunk case + return output; + } + /** + * Finalize the hash and return the result. + * + * @param outputLength - Number of bytes to output (default: 32) + * @returns The hash output + */ + finalize(outputLength = constants_js_1.OUT_LEN) { + const output = this.finalizeOutput(); + const result = new Uint8Array(outputLength); + if (outputLength <= 64) { + const outWords = this.outWords; + (0, compress_js_1.compress)(output.inputCv, 0, output.blockWords, 0, outWords, 0, outputLength > 32, // full output if > 32 bytes + output.counter, output.blockLen, output.flags | constants_js_1.ROOT); + if (utils_js_1.IS_LITTLE_ENDIAN) { + const outBytes = new Uint8Array(outWords.buffer); + result.set(outBytes.subarray(0, outputLength)); + } + else { + (0, utils_js_1.writeLittleEndianBytesPartial)(outWords, 0, result, 0, outputLength); + } + } + else { + // Multiple blocks - use XOF + const xof = this.finalizeXof(); + const full = xof.read(outputLength); + result.set(full); + } + return result; + } + /** + * Finalize and return an XOF reader for arbitrary-length output. + */ + finalizeXof() { + const output = this.finalizeOutput(); + return new XofReader(new Uint32Array(output.inputCv), new Uint32Array(output.blockWords), output.counter, output.blockLen, output.flags); + } +} +exports.Hasher = Hasher; diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/index.d.ts b/node_modules/@huggingface/blake3-jit/dist/commonjs/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..412f7b619154f7868d110617bfc8863e99c46c2c --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/index.d.ts @@ -0,0 +1,82 @@ +/** + * BLAKE3 - The fastest pure JavaScript implementation + * + * Features: + * - All 3 modes: hash, keyed (MAC), derive_key + * - XOF (eXtendable Output Function) support + * - Automatic WASM SIMD acceleration for large inputs + * - Zero dependencies + * - Tree-shakeable exports + * + * @example + * ```typescript + * import { hash, createKeyed, createDeriveKey } from 'blake3-jit'; + * + * // Simple hashing + * const digest = hash(new Uint8Array([1, 2, 3])); + * + * // Keyed hashing (MAC) + * const mac = createKeyed(key).update(data).finalize(); + * + * // Key derivation + * const derived = createDeriveKey("my context").update(material).finalize(64); + * ``` + */ +export { Hasher, XofReader } from "./hasher.js"; +export { hash, hashInto, warmupSimd } from "./hash.js"; +import { Hasher } from "./hasher.js"; +/** + * Create a new keyed hasher (MAC). + * + * @param key - 32-byte key + * @returns A new Hasher configured for keyed hashing + * + * @example + * ```typescript + * const key = new Uint8Array(32); // Your 32-byte key + * crypto.getRandomValues(key); + * + * const mac = createKeyed(key) + * .update(message) + * .finalize(); + * ``` + */ +export declare function createKeyed(key: Uint8Array): Hasher; +/** + * Create a new key derivation hasher. + * + * @param context - Context string for domain separation + * @returns A new Hasher configured for key derivation + * + * @example + * ```typescript + * const derivedKey = createDeriveKey("my-app encryption key v1") + * .update(inputKeyMaterial) + * .finalize(32); + * ``` + */ +export declare function createDeriveKey(context: string): Hasher; +/** + * Create a new regular hasher for incremental hashing. + * + * @returns A new Hasher + * + * @example + * ```typescript + * const hasher = createHasher(); + * hasher.update(chunk1); + * hasher.update(chunk2); + * const digest = hasher.finalize(); + * ``` + */ +export declare function createHasher(): Hasher; +import { hash, hashInto } from "./hash.js"; +declare const _default: { + hash: typeof hash; + hashInto: typeof hashInto; + Hasher: typeof Hasher; + createHasher: typeof createHasher; + createKeyed: typeof createKeyed; + createDeriveKey: typeof createDeriveKey; +}; +export default _default; diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/index.js b/node_modules/@huggingface/blake3-jit/dist/commonjs/index.js new file mode 100644 index 0000000000000000000000000000000000000000..04e7c8f0e0d907c864305ce446c707f78613eb7b --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/index.js @@ -0,0 +1,109 @@ +"use strict"; +/** + * BLAKE3 - The fastest pure JavaScript implementation + * + * Features: + * - All 3 modes: hash, keyed (MAC), derive_key + * - XOF (eXtendable Output Function) support + * - Automatic WASM SIMD acceleration for large inputs + * - Zero dependencies + * - Tree-shakeable exports + * + * @example + * ```typescript + * import { hash, createKeyed, createDeriveKey } from 'blake3-jit'; + * + * // Simple hashing + * const digest = hash(new Uint8Array([1, 2, 3])); + * + * // Keyed hashing (MAC) + * const mac = createKeyed(key).update(data).finalize(); + * + * // Key derivation + * const derived = createDeriveKey("my context").update(material).finalize(64); + * ``` + */ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.warmupSimd = exports.hashInto = exports.hash = exports.XofReader = exports.Hasher = void 0; +exports.createKeyed = createKeyed; +exports.createDeriveKey = createDeriveKey; +exports.createHasher = createHasher; +// Core exports +var hasher_js_1 = require("./hasher.js"); +Object.defineProperty(exports, "Hasher", { enumerable: true, get: function () { return hasher_js_1.Hasher; } }); +Object.defineProperty(exports, "XofReader", { enumerable: true, get: function () { return hasher_js_1.XofReader; } }); +var hash_js_1 = require("./hash.js"); +Object.defineProperty(exports, "hash", { enumerable: true, get: function () { return hash_js_1.hash; } }); +Object.defineProperty(exports, "hashInto", { enumerable: true, get: function () { return hash_js_1.hashInto; } }); +Object.defineProperty(exports, "warmupSimd", { enumerable: true, get: function () { return hash_js_1.warmupSimd; } }); +// Convenience imports +const hasher_js_2 = require("./hasher.js"); +/** + * Create a new keyed hasher (MAC). + * + * @param key - 32-byte key + * @returns A new Hasher configured for keyed hashing + * + * @example + * ```typescript + * const key = new Uint8Array(32); // Your 32-byte key + * crypto.getRandomValues(key); + * + * const mac = createKeyed(key) + * .update(message) + * .finalize(); + * ``` + */ +function createKeyed(key) { + return hasher_js_2.Hasher.newKeyed(key); +} +/** + * Create a new key derivation hasher. + * + * @param context - Context string for domain separation + * @returns A new Hasher configured for key derivation + * + * @example + * ```typescript + * const derivedKey = createDeriveKey("my-app encryption key v1") + * .update(inputKeyMaterial) + * .finalize(32); + * ``` + */ +function createDeriveKey(context) { + return hasher_js_2.Hasher.newDeriveKey(context); +} +/** + * Create a new regular hasher for incremental hashing. + * + * @returns A new Hasher + * + * @example + * ```typescript + * const hasher = createHasher(); + * hasher.update(chunk1); + * hasher.update(chunk2); + * const digest = hasher.finalize(); + * ``` + */ +function createHasher() { + return new hasher_js_2.Hasher(); +} +// Import for default export +const hash_js_2 = require("./hash.js"); +// Pre-warm SIMD in browser environments (non-blocking) +// This avoids initialization latency on first large hash +if (typeof globalThis !== "undefined" && typeof globalThis.document !== "undefined") { + queueMicrotask(() => { + (0, hash_js_2.warmupSimd)(); + }); +} +// Default export for convenience +exports.default = { + hash: hash_js_2.hash, + hashInto: hash_js_2.hashInto, + Hasher: hasher_js_2.Hasher, + createHasher, + createKeyed, + createDeriveKey, +}; diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/package.json b/node_modules/@huggingface/blake3-jit/dist/commonjs/package.json new file mode 100644 index 0000000000000000000000000000000000000000..5bbefffbabee392d1855491b84dc0a716b6a3bf2 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/package.json @@ -0,0 +1,3 @@ +{ + "type": "commonjs" +} diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/utils.d.ts b/node_modules/@huggingface/blake3-jit/dist/commonjs/utils.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..07a9a42794db7fc4875250155fcb055c7bad954d --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/utils.d.ts @@ -0,0 +1,90 @@ +/** + * BLAKE3 Utility Functions + * + * Optimized for little-endian systems (most user-facing systems). + * BLAKE3 is little-endian friendly - on little-endian systems we can + * create Uint32Array views directly over input buffers. + */ +/** + * Detect system endianness at module load time. + * On little-endian systems, the byte 0x01 will be at index 0. + */ +export declare const IS_LITTLE_ENDIAN: boolean; +/** + * Read 16 little-endian 32-bit words from a byte array into a Uint32Array. + * This is only needed on big-endian systems. + * + * @param input - Source byte array + * @param offset - Starting byte offset in input + * @param words - Destination Uint32Array (must have at least 16 elements) + */ +export declare function readLittleEndianWordsFull(input: Uint8Array, offset: number, words: Uint32Array): void; +/** + * Read N little-endian 32-bit words from a byte array. + * Handles partial reads (for final blocks). + * + * @param input - Source byte array + * @param offset - Starting byte offset + * @param words - Destination Uint32Array + * @param wordCount - Number of words to read + */ +export declare function readLittleEndianWords(input: Uint8Array, offset: number, words: Uint32Array, wordCount: number): void; +/** + * Read a partial block with zero padding. + * Used for the final block when input length is not a multiple of 64. + * + * @param input - Source byte array + * @param offset - Starting byte offset + * @param length - Number of bytes to read (< 64) + * @param words - Destination Uint32Array (must have 16 elements) + */ +export declare function readLittleEndianWordsPartial(input: Uint8Array, offset: number, length: number, words: Uint32Array): void; +/** + * Write 8 little-endian 32-bit words to a byte array. + * + * @param words - Source Uint32Array + * @param wordOffset - Starting word offset in source + * @param output - Destination byte array + * @param byteOffset - Starting byte offset in destination + */ +export declare function writeLittleEndianWords(words: Uint32Array, wordOffset: number, output: Uint8Array, byteOffset: number): void; +/** + * Write N bytes from 32-bit words to output. + * Used for variable-length output (XOF mode). + * + * @param words - Source Uint32Array + * @param wordOffset - Starting word offset + * @param output - Destination byte array + * @param byteOffset - Starting byte offset in destination + * @param byteCount - Number of bytes to write + */ +export declare function writeLittleEndianBytesPartial(words: Uint32Array, wordOffset: number, output: Uint8Array, byteOffset: number, byteCount: number): void; +/** + * Encode a UTF-8 string to Uint8Array. + * Used for derive_key context strings. + */ +export declare function encodeUTF8(str: string): Uint8Array; +/** + * Count trailing zero bits in a 32-bit number using De Bruijn multiplication. + * This is O(1) and branchless for non-zero inputs. + * + * For Merkle tree merge: ctz32(chunkCounter) tells us how many merges to do. + */ +export declare function ctz32(n: number): number; +/** + * Count trailing zero bits in a 64-bit number. + * Used to determine how many parent nodes to compute after adding a chunk. + * + * Note: JavaScript bitwise ops work on 32-bit signed integers, + * so we need to handle 64-bit numbers carefully. + */ +export declare function countTrailingZeros(n: number): number; +/** + * Create a Uint32Array view of a Uint8Array. + * Only works correctly on little-endian systems when the offset is 4-byte aligned. + * + * @param arr - Source byte array + * @param byteOffset - Starting byte offset (must be 4-byte aligned) + * @param wordLength - Number of 32-bit words + */ +export declare function uint32View(arr: Uint8Array, byteOffset: number, wordLength: number): Uint32Array; diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/utils.js b/node_modules/@huggingface/blake3-jit/dist/commonjs/utils.js new file mode 100644 index 0000000000000000000000000000000000000000..0a7c50f2490b09859e3a340e1210b3d663188e98 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/utils.js @@ -0,0 +1,226 @@ +"use strict"; +/** + * BLAKE3 Utility Functions + * + * Optimized for little-endian systems (most user-facing systems). + * BLAKE3 is little-endian friendly - on little-endian systems we can + * create Uint32Array views directly over input buffers. + */ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.IS_LITTLE_ENDIAN = void 0; +exports.readLittleEndianWordsFull = readLittleEndianWordsFull; +exports.readLittleEndianWords = readLittleEndianWords; +exports.readLittleEndianWordsPartial = readLittleEndianWordsPartial; +exports.writeLittleEndianWords = writeLittleEndianWords; +exports.writeLittleEndianBytesPartial = writeLittleEndianBytesPartial; +exports.encodeUTF8 = encodeUTF8; +exports.ctz32 = ctz32; +exports.countTrailingZeros = countTrailingZeros; +exports.uint32View = uint32View; +/** + * Detect system endianness at module load time. + * On little-endian systems, the byte 0x01 will be at index 0. + */ +exports.IS_LITTLE_ENDIAN = new Uint8Array(new Uint32Array([0x01020304]).buffer)[0] === 0x04; +/** + * Read 16 little-endian 32-bit words from a byte array into a Uint32Array. + * This is only needed on big-endian systems. + * + * @param input - Source byte array + * @param offset - Starting byte offset in input + * @param words - Destination Uint32Array (must have at least 16 elements) + */ +function readLittleEndianWordsFull(input, offset, words) { + for (let i = 0; i < 16; ++i, offset += 4) { + words[i] = + input[offset] | + (input[offset + 1] << 8) | + (input[offset + 2] << 16) | + (input[offset + 3] << 24); + } +} +/** + * Read N little-endian 32-bit words from a byte array. + * Handles partial reads (for final blocks). + * + * @param input - Source byte array + * @param offset - Starting byte offset + * @param words - Destination Uint32Array + * @param wordCount - Number of words to read + */ +function readLittleEndianWords(input, offset, words, wordCount) { + for (let i = 0; i < wordCount; ++i, offset += 4) { + words[i] = + input[offset] | + (input[offset + 1] << 8) | + (input[offset + 2] << 16) | + (input[offset + 3] << 24); + } +} +/** + * Read a partial block with zero padding. + * Used for the final block when input length is not a multiple of 64. + * + * @param input - Source byte array + * @param offset - Starting byte offset + * @param length - Number of bytes to read (< 64) + * @param words - Destination Uint32Array (must have 16 elements) + */ +function readLittleEndianWordsPartial(input, offset, length, words) { + // Zero out all words first + words.fill(0); + // Read full words + const fullWords = length >>> 2; + let i = 0; + for (; i < fullWords; ++i, offset += 4) { + words[i] = + input[offset] | + (input[offset + 1] << 8) | + (input[offset + 2] << 16) | + (input[offset + 3] << 24); + } + // Handle remaining bytes (0-3) + const remaining = length & 3; + if (remaining > 0) { + let word = input[offset]; + if (remaining > 1) + word |= input[offset + 1] << 8; + if (remaining > 2) + word |= input[offset + 2] << 16; + words[i] = word; + } +} +/** + * Write 8 little-endian 32-bit words to a byte array. + * + * @param words - Source Uint32Array + * @param wordOffset - Starting word offset in source + * @param output - Destination byte array + * @param byteOffset - Starting byte offset in destination + */ +function writeLittleEndianWords(words, wordOffset, output, byteOffset) { + for (let i = 0; i < 8; ++i, byteOffset += 4) { + const w = words[wordOffset + i]; + output[byteOffset] = w & 0xff; + output[byteOffset + 1] = (w >>> 8) & 0xff; + output[byteOffset + 2] = (w >>> 16) & 0xff; + output[byteOffset + 3] = (w >>> 24) & 0xff; + } +} +/** + * Write N bytes from 32-bit words to output. + * Used for variable-length output (XOF mode). + * + * @param words - Source Uint32Array + * @param wordOffset - Starting word offset + * @param output - Destination byte array + * @param byteOffset - Starting byte offset in destination + * @param byteCount - Number of bytes to write + */ +function writeLittleEndianBytesPartial(words, wordOffset, output, byteOffset, byteCount) { + const fullWords = byteCount >>> 2; + let i = 0; + // Write full words + for (; i < fullWords; ++i, byteOffset += 4) { + const w = words[wordOffset + i]; + output[byteOffset] = w & 0xff; + output[byteOffset + 1] = (w >>> 8) & 0xff; + output[byteOffset + 2] = (w >>> 16) & 0xff; + output[byteOffset + 3] = (w >>> 24) & 0xff; + } + // Write remaining bytes + const remaining = byteCount & 3; + if (remaining > 0) { + const w = words[wordOffset + i]; + output[byteOffset] = w & 0xff; + if (remaining > 1) + output[byteOffset + 1] = (w >>> 8) & 0xff; + if (remaining > 2) + output[byteOffset + 2] = (w >>> 16) & 0xff; + } +} +/** + * Encode a UTF-8 string to Uint8Array. + * Used for derive_key context strings. + */ +function encodeUTF8(str) { + if (typeof TextEncoder !== "undefined") { + return new TextEncoder().encode(str); + } + // Fallback for older environments + const bytes = []; + for (let i = 0; i < str.length; i++) { + let c = str.charCodeAt(i); + if (c < 0x80) { + bytes.push(c); + } + else if (c < 0x800) { + bytes.push(0xc0 | (c >> 6), 0x80 | (c & 0x3f)); + } + else if (c < 0xd800 || c >= 0xe000) { + bytes.push(0xe0 | (c >> 12), 0x80 | ((c >> 6) & 0x3f), 0x80 | (c & 0x3f)); + } + else { + // Surrogate pair + i++; + c = 0x10000 + (((c & 0x3ff) << 10) | (str.charCodeAt(i) & 0x3ff)); + bytes.push(0xf0 | (c >> 18), 0x80 | ((c >> 12) & 0x3f), 0x80 | ((c >> 6) & 0x3f), 0x80 | (c & 0x3f)); + } + } + return new Uint8Array(bytes); +} +/** + * De Bruijn lookup table for O(1) trailing zero count. + * The expression (n & -n) isolates the lowest set bit. + * Multiplying by the De Bruijn constant maps each power of 2 to a unique 5-bit index. + */ +const CTZ32_TABLE = new Uint8Array([ + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, + 18, 6, 11, 5, 10, 9, +]); +/** + * Count trailing zero bits in a 32-bit number using De Bruijn multiplication. + * This is O(1) and branchless for non-zero inputs. + * + * For Merkle tree merge: ctz32(chunkCounter) tells us how many merges to do. + */ +function ctz32(n) { + if (n === 0) + return 32; + // Use unsigned right shift to handle negative numbers correctly + return CTZ32_TABLE[(((n & -n) * 0x077cb531) >>> 27) & 31]; +} +/** + * Count trailing zero bits in a 64-bit number. + * Used to determine how many parent nodes to compute after adding a chunk. + * + * Note: JavaScript bitwise ops work on 32-bit signed integers, + * so we need to handle 64-bit numbers carefully. + */ +function countTrailingZeros(n) { + if (n === 0) + return 64; + // For numbers that fit in 32 bits + const low = n | 0; + if (low !== 0) { + // Use Math.clz32 trick: ctz(x) = 31 - clz32(x & -x) for non-zero x + return 31 - Math.clz32(low & -low); + } + // High 32 bits + const high = (n / 0x100000000) | 0; + if (high !== 0) { + return 32 + (31 - Math.clz32(high & -high)); + } + return 64; +} +/** + * Create a Uint32Array view of a Uint8Array. + * Only works correctly on little-endian systems when the offset is 4-byte aligned. + * + * @param arr - Source byte array + * @param byteOffset - Starting byte offset (must be 4-byte aligned) + * @param wordLength - Number of 32-bit words + */ +function uint32View(arr, byteOffset, wordLength) { + return new Uint32Array(arr.buffer, arr.byteOffset + byteOffset, wordLength); +} diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/wasm-simd.d.ts b/node_modules/@huggingface/blake3-jit/dist/commonjs/wasm-simd.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..5cfc598b3c44d4a92b2cf7812bfcae4f151a1de9 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/wasm-simd.d.ts @@ -0,0 +1,97 @@ +/** + * BLAKE3 WASM SIMD - Runtime bytecode generation + * + * Generates WebAssembly SIMD bytecode at runtime to process 4 compress + * operations in parallel using 128-bit SIMD vectors (i32x4). + * + * Key insight: One i32x4.add instruction performs 4 parallel additions, + * giving us 4x throughput for the same number of instructions. + * + * Memory layout (all values are transposed for SIMD access): + * 0-511: 4 x 16 message words (m0_0,m0_1,m0_2,m0_3, m1_0,m1_1,m1_2,m1_3, ...) + * 512-639: 4 x 8 chaining values + * 640-767: 4 x 8 output values + * 768-783: 4 x counter low + * 784-799: 4 x counter high + * 800-815: 4 x block length + * 816-831: 4 x flags + */ +/** + * Check if WASM SIMD is supported. + */ +export declare function isSimdSupported(): boolean; +export declare function initSimdSync(): boolean; +/** + * Memory offsets for SIMD data layout + * + * WASM Arena Pattern: All working buffers live in WASM memory (64KB page) + * This eliminates JS heap allocations during hashing operations. + */ +export declare const SIMD_MEMORY: { + readonly BLOCK_WORDS: 0; + readonly CHAINING_VALUES: 512; + readonly OUTPUT: 640; + readonly COUNTER_LOW: 768; + readonly COUNTER_HIGH: 784; + readonly BLOCK_LEN: 800; + readonly FLAGS: 816; + readonly BATCH_BLOCK_WORDS: 832; + readonly BATCH_CV: 4928; + readonly BATCH_COUNTER_LOW: 5056; + readonly BATCH_FLAGS_BASE: 5072; + readonly BATCH_OUTPUT: 5088; + readonly CV_STACK: 5216; + readonly PARENT_BLOCK: 7264; + readonly CHUNK_CV: 7328; + readonly TEMP_CVS: 7360; +}; +/** + * Get the WASM memory views for writing input data. + */ +export declare function getSimdMemory(): { + view: Uint8Array; + view32: Uint32Array; +} | null; +/** + * Get the arena buffers for Merkle tree operations. + * These TypedArray views are backed by WASM memory - zero JS heap allocation. + */ +export declare function getArenaBuffers(): { + cvStack: Uint32Array; + parentBlock: Uint32Array; + chunkCv: Uint32Array; + tempCvs: Uint32Array; +} | null; +/** + * Get the batch arena buffers for chunk-level batched operations. + * These TypedArray views are backed by WASM memory - zero JS heap allocation. + */ +export declare function getBatchArenaBuffers(): { + blockWords: Uint32Array; + cv: Uint32Array; + counterLow: Uint32Array; + flagsBase: Uint32Array; + output: Uint32Array; +} | null; +/** + * Run the compress4x function. + * Data must already be set up in WASM memory. + */ +export declare function runCompress4x(): void; +/** + * Run the compressChunks4x function. + * Processes 4 full chunks (16 blocks each) in a single WASM call. + * Data must already be set up in batch arena buffers. + */ +export declare function runCompressChunks4x(): void; +/** + * Run the compressParent function. + * Compresses a parent node: reads 16 words from PARENT_BLOCK, writes 8 words to CHUNK_CV. + * Data must already be set up in arena buffers (PARENT_BLOCK at offset 7264). + * Output is written to CHUNK_CV at offset 7328. + */ +export declare function runCompressParent(): void; +/** + * Check if SIMD is initialized and ready. + */ +export declare function isSimdReady(): boolean; diff --git a/node_modules/@huggingface/blake3-jit/dist/commonjs/wasm-simd.js b/node_modules/@huggingface/blake3-jit/dist/commonjs/wasm-simd.js new file mode 100644 index 0000000000000000000000000000000000000000..7adf50c1c036eda7b63e0c0fd941e0a5ca099a94 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/commonjs/wasm-simd.js @@ -0,0 +1,948 @@ +"use strict"; +/** + * BLAKE3 WASM SIMD - Runtime bytecode generation + * + * Generates WebAssembly SIMD bytecode at runtime to process 4 compress + * operations in parallel using 128-bit SIMD vectors (i32x4). + * + * Key insight: One i32x4.add instruction performs 4 parallel additions, + * giving us 4x throughput for the same number of instructions. + * + * Memory layout (all values are transposed for SIMD access): + * 0-511: 4 x 16 message words (m0_0,m0_1,m0_2,m0_3, m1_0,m1_1,m1_2,m1_3, ...) + * 512-639: 4 x 8 chaining values + * 640-767: 4 x 8 output values + * 768-783: 4 x counter low + * 784-799: 4 x counter high + * 800-815: 4 x block length + * 816-831: 4 x flags + */ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.SIMD_MEMORY = void 0; +exports.isSimdSupported = isSimdSupported; +exports.initSimdSync = initSimdSync; +exports.getSimdMemory = getSimdMemory; +exports.getArenaBuffers = getArenaBuffers; +exports.getBatchArenaBuffers = getBatchArenaBuffers; +exports.runCompress4x = runCompress4x; +exports.runCompressChunks4x = runCompressChunks4x; +exports.runCompressParent = runCompressParent; +exports.isSimdReady = isSimdReady; +// LEB128 encoding with minimum 2 bytes +// This fixes a V8 quirk where single-byte values 64-127 cause issues +// when followed by certain SIMD instructions +function toLebU32Min2(n) { + // Always use at least 2 bytes + return [(n & 0x7f) | 0x80, (n >>> 7) & 0x7f]; +} +// LEB128 encoding padded to exactly 5 bytes (for backpatching) +// Uses continuation bits for all but the last byte +function toLebU32Padded5(n) { + return [ + (n & 0x7f) | 0x80, + ((n >>> 7) & 0x7f) | 0x80, + ((n >>> 14) & 0x7f) | 0x80, + ((n >>> 21) & 0x7f) | 0x80, + (n >>> 28) & 0x0f, // Last byte has no continuation bit + ]; +} +// Signed LEB128 encoding for i32 constants (handles full 32-bit range) +// WASM i32.const uses signed LEB128 immediate +function toSignedLeb128_i32(n) { + const bytes = []; + // Treat as signed 32-bit integer + let value = n | 0; + let more = true; + while (more) { + let byte = value & 0x7f; + // Arithmetic right shift preserves sign + value >>= 7; + // Check if we're done: + // - If value is 0 and sign bit of byte is clear, we're done + // - If value is -1 and sign bit of byte is set, we're done + if ((value === 0 && (byte & 0x40) === 0) || (value === -1 && (byte & 0x40) !== 0)) { + more = false; + } + else { + byte |= 0x80; + } + bytes.push(byte); + } + return bytes; +} +// Precomputed message access order for all 7 rounds +const MSG_ACCESS_ORDER = [ + // Round 1: 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + // Round 2: 2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8 + 2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8, + // Round 3: 3,4,10,12,13,2,7,14,6,5,9,0,11,15,8,1 + 3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1, + // Round 4: 10,7,12,9,14,3,13,15,4,0,11,2,5,8,1,6 + 10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6, + // Round 5: 12,13,9,11,15,10,14,8,7,2,5,3,0,1,6,4 + 12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4, + // Round 6: 9,14,11,5,8,12,15,1,13,3,0,10,2,6,4,7 + 9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7, + // Round 7: 11,15,5,0,1,9,8,6,14,10,2,12,3,4,7,13 + 11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13, +]; +// BLAKE3 Constants (used in generated WASM code) +// CHUNK_START = 1, CHUNK_END = 2 are embedded directly in WASM bytecode +/** + * Generate the WASM module bytecode with compress4x, compressChunks4x, and compressParent functions. + */ +function generateWasmBytes() { + const code = []; + // Helper to append bytes + function put(bytes) { + code.push(...bytes); + } + // WASM module header + put([0x00, 0x61, 0x73, 0x6d]); // Magic + put([0x01, 0x00, 0x00, 0x00]); // Version + // Section 1: Types + put([0x01]); // Section ID + put([0x04]); // Section size + put([0x01]); // 1 type + put([0x60, 0x00, 0x00]); // func () -> () + // Section 2: Imports (memory from JS) + put([0x02]); // Section ID + put([0x0b]); // Section size + put([0x01]); // 1 import + put([0x02, 0x6a, 0x73]); // "js" + put([0x03, 0x6d, 0x65, 0x6d]); // "mem" + put([0x02, 0x00, 0x01]); // memory min=1, no max + // Section 3: Functions + put([0x03]); // Section ID + put([0x04]); // Section size (3 functions = 4 bytes) + put([0x03]); // 3 functions + put([0x00]); // Function 0: type index 0 + put([0x00]); // Function 1: type index 0 + put([0x00]); // Function 2: type index 0 + // Section 7: Exports + // Size calculation: 1 (count) + (1+10+1+1) + (1+16+1+1) + (1+14+1+1) = 1 + 13 + 19 + 17 = 50 bytes + put([0x07]); // Section ID + put([0x32]); // Section size (50 bytes) + put([0x03]); // 3 exports + // "compress4x" -> func 0 + put([0x0a]); // name length + put([0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x34, 0x78]); // "compress4x" + put([0x00, 0x00]); // func index 0 + // "compressChunks4x" -> func 1 + put([0x10]); // name length (16) + put([ + 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x34, 0x78, + ]); // "compressChunks4x" + put([0x00, 0x01]); // func index 1 + // "compressParent" -> func 2 + put([0x0e]); // name length (14) + put([0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74]); // "compressParent" + put([0x00, 0x02]); // func index 2 + // Section 10: Code + put([0x0a]); // Section ID + // Reserve 5 bytes for section size (LEB128 u32) + const sectionSizeOffset = code.length; + put([0x00, 0x00, 0x00, 0x00, 0x00]); + put([0x03]); // 3 functions + // === Function 0: compress4x === + // Reserve 5 bytes for function size + const funcSizeOffset = code.length; + put([0x00, 0x00, 0x00, 0x00, 0x00]); + const funcBodyStart = code.length; + // Local declarations: 32 v128 locals + // Variables $0-$15: message words (m0-m15) + // Variables $16-$31: state words (s0-s15) + put([0x01]); // 1 local declaration + put([0x20, 0x7b]); // 32 x v128 + // ===== Function body ===== + // Load message words from memory (offset 0-255) + // Each v128 is 16 bytes, so m[i] is at offset i*16 + // Note: we use toLebU32Min2 to avoid V8 quirk with single-byte values 64-127 + for (let i = 0; i < 16; i++) { + put([0x41, ...toLebU32Min2(i * 16)]); // i32.const offset (2+ byte LEB128) + put([0xfd, 0x00, 0x02, 0x00]); // v128.load align=4 offset=0 + put([0x21, i]); // local.set $i + } + // Load chaining values (offset 512-639) + // cv[i] at offset 512 + i*16 + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(512 + i * 16)]); // i32.const offset + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 16 + i]); // local.set $(16+i) + } + // Initialize state[8-15] from IV and parameters + // s8-s11 = IV[0-3] + const IV = [0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a]; + for (let i = 0; i < 4; i++) { + // Create v128 constant with all lanes set to IV[i] + const ivBytes = []; + for (let j = 0; j < 4; j++) { + ivBytes.push(IV[i] & 0xff); + ivBytes.push((IV[i] >>> 8) & 0xff); + ivBytes.push((IV[i] >>> 16) & 0xff); + ivBytes.push((IV[i] >>> 24) & 0xff); + } + put([0xfd, 0x0c, ...ivBytes]); // v128.const + put([0x21, 24 + i]); // local.set $(24+i) -> s8-s11 + } + // s12 = counter_low (offset 768) + put([0x41, ...toLebU32Min2(768)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 28]); // local.set $28 -> s12 + // s13 = counter_high (offset 784) + put([0x41, ...toLebU32Min2(784)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 29]); // local.set $29 -> s13 + // s14 = block_len (offset 800) + put([0x41, ...toLebU32Min2(800)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 30]); // local.set $30 -> s14 + // s15 = flags (offset 816) + put([0x41, ...toLebU32Min2(816)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 31]); // local.set $31 -> s15 + // ===== 7 rounds of mixing ===== + let msgIdx = 0; // Index into MSG_ACCESS_ORDER + // Helper to generate G function (inlined) + // G(a, b, c, d) with two message words + function g(a, b, c, d) { + const mx = MSG_ACCESS_ORDER[msgIdx++]; + const my = MSG_ACCESS_ORDER[msgIdx++]; + // Variables: a,b,c,d are state indices (16-31), mx,my are message indices (0-15) + // First half of G + // s[a] = s[a] + s[b] + m[mx] + put([0x20, 16 + a]); // local.get s[a] + put([0x20, 16 + b]); // local.get s[b] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x20, mx]); // local.get m[mx] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + a]); // local.set s[a] + // s[d] = rotr(s[d] ^ s[a], 16) - using i8x16.shuffle (single instruction vs shift+or) + // ROTR16 pattern: [2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13] + put([0x20, 16 + d]); // local.get s[d] + put([0x20, 16 + a]); // local.get s[a] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + d]); // local.tee s[d] + put([0x20, 16 + d]); // local.get s[d] (second operand for shuffle) + put([0xfd, 0x0d, 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13]); // i8x16.shuffle ROTR16 + put([0x21, 16 + d]); // local.set s[d] + // s[c] = s[c] + s[d] + put([0x20, 16 + c]); // local.get s[c] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + c]); // local.set s[c] + // s[b] = (s[b] ^ s[c]) >>> 12 + put([0x20, 16 + b]); // local.get s[b] + put([0x20, 16 + c]); // local.get s[c] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + b]); // local.tee s[b] + put([0x41, 0x0c]); // i32.const 12 + put([0xfd, 0xad, 0x01]); // i32x4.shr_u (opcode 173 = 0xAD) + put([0x20, 16 + b]); // local.get s[b] + put([0x41, 0x14]); // i32.const 20 + put([0xfd, 0xab, 0x01]); // i32x4.shl + put([0xfd, 0x50]); // v128.or + put([0x21, 16 + b]); // local.set s[b] + // Second half of G + // s[a] = s[a] + s[b] + m[my] + put([0x20, 16 + a]); // local.get s[a] + put([0x20, 16 + b]); // local.get s[b] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x20, my]); // local.get m[my] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + a]); // local.set s[a] + // s[d] = rotr(s[d] ^ s[a], 8) - using i8x16.shuffle (single instruction vs shift+or) + // ROTR8 pattern: [1,2,3,0, 5,6,7,4, 9,10,11,8, 13,14,15,12] + put([0x20, 16 + d]); // local.get s[d] + put([0x20, 16 + a]); // local.get s[a] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + d]); // local.tee s[d] + put([0x20, 16 + d]); // local.get s[d] (second operand for shuffle) + put([0xfd, 0x0d, 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12]); // i8x16.shuffle ROTR8 + put([0x21, 16 + d]); // local.set s[d] + // s[c] = s[c] + s[d] + put([0x20, 16 + c]); // local.get s[c] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + c]); // local.set s[c] + // s[b] = (s[b] ^ s[c]) >>> 7 + put([0x20, 16 + b]); // local.get s[b] + put([0x20, 16 + c]); // local.get s[c] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + b]); // local.tee s[b] + put([0x41, 0x07]); // i32.const 7 + put([0xfd, 0xad, 0x01]); // i32x4.shr_u (opcode 173 = 0xAD) + put([0x20, 16 + b]); // local.get s[b] + put([0x41, 0x19]); // i32.const 25 + put([0xfd, 0xab, 0x01]); // i32x4.shl + put([0xfd, 0x50]); // v128.or + put([0x21, 16 + b]); // local.set s[b] + } + // Generate all 7 rounds + for (let round = 0; round < 7; round++) { + // Column mixing + g(0, 4, 8, 12); + g(1, 5, 9, 13); + g(2, 6, 10, 14); + g(3, 7, 11, 15); + // Diagonal mixing + g(0, 5, 10, 15); + g(1, 6, 11, 12); + g(2, 7, 8, 13); + g(3, 4, 9, 14); + } + // ===== Final XOR and store output ===== + // out[i] = s[i] ^ s[i+8] for i in 0..7 + // Store at offset 640-767 + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(640 + i * 16)]); // i32.const offset + put([0x20, 16 + i]); // local.get s[i] + put([0x20, 24 + i]); // local.get s[i+8] + put([0xfd, 0x51]); // v128.xor + put([0xfd, 0x0b, 0x02, 0x00]); // v128.store align=4 + } + // End of function + put([0x0b]); // end + // Fill in function 0 size using padded LEB128 + const funcBodySize = code.length - funcBodyStart; + const funcSizeBytes = toLebU32Padded5(funcBodySize); + for (let i = 0; i < 5; i++) { + code[funcSizeOffset + i] = funcSizeBytes[i]; + } + // === Function 1: compressChunks4x === + // Reserve 5 bytes for function size + const func1SizeOffset = code.length; + put([0x00, 0x00, 0x00, 0x00, 0x00]); + const func1BodyStart = code.length; + // Generate the compressChunks4x function body + const compressChunksBody = generateCompressChunks4xBody(); + put(compressChunksBody); + // Fill in function 1 size using padded LEB128 + const func1BodySize = code.length - func1BodyStart; + const func1SizeBytes = toLebU32Padded5(func1BodySize); + for (let i = 0; i < 5; i++) { + code[func1SizeOffset + i] = func1SizeBytes[i]; + } + // === Function 2: compressParent === + // Reserve 5 bytes for function size + const func2SizeOffset = code.length; + put([0x00, 0x00, 0x00, 0x00, 0x00]); + const func2BodyStart = code.length; + // Generate the compressParent function body + const compressParentBody = generateCompressParentBody(); + put(compressParentBody); + // Fill in function 2 size using padded LEB128 + const func2BodySize = code.length - func2BodyStart; + const func2SizeBytes = toLebU32Padded5(func2BodySize); + for (let i = 0; i < 5; i++) { + code[func2SizeOffset + i] = func2SizeBytes[i]; + } + // Fill in section size using padded LEB128 + const sectionSize = code.length - sectionSizeOffset - 5; + const sectionSizeBytes = toLebU32Padded5(sectionSize); + for (let i = 0; i < 5; i++) { + code[sectionSizeOffset + i] = sectionSizeBytes[i]; + } + return new Uint8Array(code); +} +/** + * Generate compressChunks4x WASM function body. + * Processes all 16 blocks of 4 chunks in a single call. + */ +function generateCompressChunks4xBody() { + const code = []; + function put(bytes) { + code.push(...bytes); + } + // Local declarations: 32 v128 locals + 1 i32 for position + // Locals $0-$15: message words (reloaded each iteration) + // Locals $16-$31: state words (s0-s15) + // Local $32: position counter (i32) + put([0x02]); // 2 local declarations + put([0x20, 0x7b]); // 32 x v128 + put([0x01, 0x7f]); // 1 x i32 + const BATCH_BLOCK_WORDS = exports.SIMD_MEMORY.BATCH_BLOCK_WORDS; + const BATCH_CV = exports.SIMD_MEMORY.BATCH_CV; + const BATCH_COUNTER_LOW = exports.SIMD_MEMORY.BATCH_COUNTER_LOW; + const BATCH_FLAGS_BASE = exports.SIMD_MEMORY.BATCH_FLAGS_BASE; + const BATCH_OUTPUT = exports.SIMD_MEMORY.BATCH_OUTPUT; + // IV constants (same as compress4x) + const IV = [0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a]; + // Load initial CVs from BATCH_CV into locals $16-$23 + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(BATCH_CV + i * 16)]); // i32.const offset + put([0xfd, 0x00, 0x02, 0x00]); // v128.load align=4 offset=0 + put([0x21, 16 + i]); // local.set $(16+i) -> s0-s7 + } + // Initialize $32 (pos) = 0 + put([0x41, 0x00]); // i32.const 0 + put([0x21, 0x20]); // local.set $32 + // block $done + put([0x02, 0x40]); // block void + // loop $continue + put([0x03, 0x40]); // loop void + // === Load message words for position $pos === + // offset = BATCH_BLOCK_WORDS + pos * 256 + word * 16 + for (let w = 0; w < 16; w++) { + put([0x20, 0x20]); // local.get $32 (pos) + put([0x41, ...toLebU32Min2(256)]); // i32.const 256 + put([0x6c]); // i32.mul + put([0x41, ...toLebU32Min2(BATCH_BLOCK_WORDS + w * 16)]); // i32.const base + word*16 + put([0x6a]); // i32.add + put([0xfd, 0x00, 0x02, 0x00]); // v128.load align=4 offset=0 + put([0x21, w]); // local.set $w + } + // === Initialize state[8-15] === + // s8-s11 = IV[0-3] + for (let i = 0; i < 4; i++) { + const ivBytes = []; + for (let j = 0; j < 4; j++) { + ivBytes.push(IV[i] & 0xff); + ivBytes.push((IV[i] >>> 8) & 0xff); + ivBytes.push((IV[i] >>> 16) & 0xff); + ivBytes.push((IV[i] >>> 24) & 0xff); + } + put([0xfd, 0x0c, ...ivBytes]); // v128.const + put([0x21, 24 + i]); // local.set $(24+i) -> s8-s11 + } + // s12 = counter_low (from BATCH_COUNTER_LOW) + put([0x41, ...toLebU32Min2(BATCH_COUNTER_LOW)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 28]); // local.set $28 -> s12 + // s13 = 0 (counter high - assume fits in 32 bits) + put([0xfd, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); // v128.const 0 + put([0x21, 29]); // local.set $29 -> s13 + // s14 = 64 (block_len = 64 for full blocks) + const blockLen64 = []; + for (let j = 0; j < 4; j++) { + blockLen64.push(64, 0, 0, 0); // 64 in little-endian + } + put([0xfd, 0x0c, ...blockLen64]); // v128.const [64,64,64,64] + put([0x21, 30]); // local.set $30 -> s14 + // s15 = flags = base_flags | (pos == 0 ? 1 : 0) | (pos == 15 ? 2 : 0) + // First load base flags + put([0x41, ...toLebU32Min2(BATCH_FLAGS_BASE)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load base flags + // Compute position-dependent bits + // CHUNK_START (1) if pos == 0 + put([0x20, 0x20]); // local.get $32 (pos) + put([0x45]); // i32.eqz -> 1 if pos==0, 0 otherwise + // CHUNK_END (2) if pos == 15 + put([0x20, 0x20]); // local.get $32 (pos) + put([0x41, 0x0f]); // i32.const 15 + put([0x46]); // i32.eq -> 1 if pos==15, 0 otherwise + put([0x41, 0x01]); // i32.const 1 (shift amount) + put([0x74]); // i32.shl -> 2 if pos==15, 0 otherwise + // OR the two bits together + put([0x72]); // i32.or -> combined position bits + // Splat to v128 and OR with base flags (stack: base_flags, bits) + put([0xfd, 0x11]); // i32x4.splat + put([0xfd, 0x50]); // v128.or + put([0x21, 31]); // local.set $31 -> s15 + // === 7 rounds of mixing === + let msgIdx = 0; + function g(a, b, c, d) { + const mx = MSG_ACCESS_ORDER[msgIdx++]; + const my = MSG_ACCESS_ORDER[msgIdx++]; + // First half of G: s[a] = s[a] + s[b] + m[mx] + put([0x20, 16 + a]); // local.get s[a] + put([0x20, 16 + b]); // local.get s[b] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x20, mx]); // local.get m[mx] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + a]); // local.set s[a] + // s[d] = rotr(s[d] ^ s[a], 16) - byte shuffle + put([0x20, 16 + d]); // local.get s[d] + put([0x20, 16 + a]); // local.get s[a] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + d]); // local.tee s[d] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0x0d, 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13]); // i8x16.shuffle ROTR16 + put([0x21, 16 + d]); // local.set s[d] + // s[c] = s[c] + s[d] + put([0x20, 16 + c]); // local.get s[c] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + c]); // local.set s[c] + // s[b] = rotr(s[b] ^ s[c], 12) + put([0x20, 16 + b]); // local.get s[b] + put([0x20, 16 + c]); // local.get s[c] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + b]); // local.tee s[b] + put([0x41, 0x0c]); // i32.const 12 + put([0xfd, 0xad, 0x01]); // i32x4.shr_u (opcode 173 = 0xAD) + put([0x20, 16 + b]); // local.get s[b] + put([0x41, 0x14]); // i32.const 20 + put([0xfd, 0xab, 0x01]); // i32x4.shl + put([0xfd, 0x50]); // v128.or + put([0x21, 16 + b]); // local.set s[b] + // Second half: s[a] = s[a] + s[b] + m[my] + put([0x20, 16 + a]); // local.get s[a] + put([0x20, 16 + b]); // local.get s[b] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x20, my]); // local.get m[my] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + a]); // local.set s[a] + // s[d] = rotr(s[d] ^ s[a], 8) - byte shuffle + put([0x20, 16 + d]); // local.get s[d] + put([0x20, 16 + a]); // local.get s[a] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + d]); // local.tee s[d] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0x0d, 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12]); // i8x16.shuffle ROTR8 + put([0x21, 16 + d]); // local.set s[d] + // s[c] = s[c] + s[d] + put([0x20, 16 + c]); // local.get s[c] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + c]); // local.set s[c] + // s[b] = rotr(s[b] ^ s[c], 7) + put([0x20, 16 + b]); // local.get s[b] + put([0x20, 16 + c]); // local.get s[c] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + b]); // local.tee s[b] + put([0x41, 0x07]); // i32.const 7 + put([0xfd, 0xad, 0x01]); // i32x4.shr_u (opcode 173 = 0xAD) + put([0x20, 16 + b]); // local.get s[b] + put([0x41, 0x19]); // i32.const 25 + put([0xfd, 0xab, 0x01]); // i32x4.shl + put([0xfd, 0x50]); // v128.or + put([0x21, 16 + b]); // local.set s[b] + } + // Generate all 7 rounds + for (let round = 0; round < 7; round++) { + // Column mixing + g(0, 4, 8, 12); + g(1, 5, 9, 13); + g(2, 6, 10, 14); + g(3, 7, 11, 15); + // Diagonal mixing + g(0, 5, 10, 15); + g(1, 6, 11, 12); + g(2, 7, 8, 13); + g(3, 4, 9, 14); + } + // === Update CVs: cv[i] = s[i] ^ s[i+8] === + // Store back to state locals $16-$23 (the CV positions) + for (let i = 0; i < 8; i++) { + put([0x20, 16 + i]); // local.get s[i] + put([0x20, 24 + i]); // local.get s[i+8] + put([0xfd, 0x51]); // v128.xor + put([0x21, 16 + i]); // local.set $(16+i) - update CV + } + // === Loop control: pos++, continue if pos < 16 === + put([0x20, 0x20]); // local.get $32 (pos) + put([0x41, 0x01]); // i32.const 1 + put([0x6a]); // i32.add + put([0x22, 0x20]); // local.tee $32 (pos) + put([0x41, 0x10]); // i32.const 16 + put([0x49]); // i32.lt_u + put([0x0d, 0x00]); // br_if 0 (continue loop) + // end loop + put([0x0b]); // end + // end block + put([0x0b]); // end + // === Store final CVs to BATCH_OUTPUT === + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(BATCH_OUTPUT + i * 16)]); // i32.const offset + put([0x20, 16 + i]); // local.get $(16+i) - CV word + put([0xfd, 0x0b, 0x02, 0x00]); // v128.store align=4 + } + // end function + put([0x0b]); // end + return code; +} +/** + * Generate compressParent WASM function body. + * Performs a single parent node compression using scalar i32 operations. + * Reads 16 words from PARENT_BLOCK, writes 8 words to CHUNK_CV. + * Uses IV, counter=0, blockLen=64, flags=PARENT(4). + */ +function generateCompressParentBody() { + const code = []; + function put(bytes) { + code.push(...bytes); + } + // Local declarations: 32 i32 locals for state (s0-s15) and message (m0-m15) + put([0x01]); // 1 local declaration + put([0x20, 0x7f]); // 32 x i32 + // Message word indices: 0-15, State indices: 16-31 + // Locals $0-$15: message words (m0-m15) + // Locals $16-$31: state words (s0-s15) + const PARENT_BLOCK_OFFSET = exports.SIMD_MEMORY.PARENT_BLOCK; + const CHUNK_CV_OFFSET = exports.SIMD_MEMORY.CHUNK_CV; + // BLAKE3 IV + const IV = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, + ]; + // Load message words from PARENT_BLOCK (16 words at offset 7264) + for (let i = 0; i < 16; i++) { + put([0x41, ...toLebU32Min2(PARENT_BLOCK_OFFSET + i * 4)]); // i32.const offset + put([0x28, 0x02, 0x00]); // i32.load align=4 offset=0 + put([0x21, i]); // local.set $i (m0-m15) + } + // Initialize state s0-s7 = IV[0-7] + for (let i = 0; i < 8; i++) { + put([0x41, ...toSignedLeb128_i32(IV[i])]); // i32.const IV[i] + put([0x21, 16 + i]); // local.set $(16+i) -> s0-s7 + } + // Initialize state s8-s11 = IV[0-3] + for (let i = 0; i < 4; i++) { + put([0x41, ...toSignedLeb128_i32(IV[i])]); // i32.const IV[i] + put([0x21, 24 + i]); // local.set $(24+i) -> s8-s11 + } + // s12 = counter_low = 0 + put([0x41, 0x00]); // i32.const 0 + put([0x21, 28]); // local.set $28 -> s12 + // s13 = counter_high = 0 + put([0x41, 0x00]); // i32.const 0 + put([0x21, 29]); // local.set $29 -> s13 + // s14 = block_len = 64 + // Note: 0x40 alone is -64 in signed LEB128 (bit 6 is sign bit) + // For 64, we need [0xC0, 0x00] to avoid sign extension + put([0x41, 0xc0, 0x00]); // i32.const 64 + put([0x21, 30]); // local.set $30 -> s14 + // s15 = flags = PARENT = 4 + put([0x41, 0x04]); // i32.const 4 + put([0x21, 31]); // local.set $31 -> s15 + // Helper to generate scalar G function (inlined) + // G(a, b, c, d, mx, my) where a,b,c,d are state indices 0-15, mx,my are message indices 0-15 + function g(a, b, c, d, mx, my) { + const sa = 16 + a, sb = 16 + b, sc = 16 + c, sd = 16 + d; + // s[a] = (s[a] + s[b] + m[mx]) >>> 0 + put([0x20, sa]); // local.get s[a] + put([0x20, sb]); // local.get s[b] + put([0x6a]); // i32.add + put([0x20, mx]); // local.get m[mx] + put([0x6a]); // i32.add + put([0x21, sa]); // local.set s[a] + // s[d] = rotr(s[d] ^ s[a], 16) + put([0x20, sd]); // local.get s[d] + put([0x20, sa]); // local.get s[a] + put([0x73]); // i32.xor + put([0x41, 0x10]); // i32.const 16 + put([0x78]); // i32.rotr + put([0x21, sd]); // local.set s[d] + // s[c] = (s[c] + s[d]) >>> 0 + put([0x20, sc]); // local.get s[c] + put([0x20, sd]); // local.get s[d] + put([0x6a]); // i32.add + put([0x21, sc]); // local.set s[c] + // s[b] = rotr(s[b] ^ s[c], 12) + put([0x20, sb]); // local.get s[b] + put([0x20, sc]); // local.get s[c] + put([0x73]); // i32.xor + put([0x41, 0x0c]); // i32.const 12 + put([0x78]); // i32.rotr + put([0x21, sb]); // local.set s[b] + // s[a] = (s[a] + s[b] + m[my]) >>> 0 + put([0x20, sa]); // local.get s[a] + put([0x20, sb]); // local.get s[b] + put([0x6a]); // i32.add + put([0x20, my]); // local.get m[my] + put([0x6a]); // i32.add + put([0x21, sa]); // local.set s[a] + // s[d] = rotr(s[d] ^ s[a], 8) + put([0x20, sd]); // local.get s[d] + put([0x20, sa]); // local.get s[a] + put([0x73]); // i32.xor + put([0x41, 0x08]); // i32.const 8 + put([0x78]); // i32.rotr + put([0x21, sd]); // local.set s[d] + // s[c] = (s[c] + s[d]) >>> 0 + put([0x20, sc]); // local.get s[c] + put([0x20, sd]); // local.get s[d] + put([0x6a]); // i32.add + put([0x21, sc]); // local.set s[c] + // s[b] = rotr(s[b] ^ s[c], 7) + put([0x20, sb]); // local.get s[b] + put([0x20, sc]); // local.get s[c] + put([0x73]); // i32.xor + put([0x41, 0x07]); // i32.const 7 + put([0x78]); // i32.rotr + put([0x21, sb]); // local.set s[b] + } + // 7 rounds of mixing with permuted message schedule + let msgIdx = 0; + for (let round = 0; round < 7; round++) { + // Column mixing + g(0, 4, 8, 12, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(1, 5, 9, 13, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(2, 6, 10, 14, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(3, 7, 11, 15, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + // Diagonal mixing + g(0, 5, 10, 15, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(1, 6, 11, 12, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(2, 7, 8, 13, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(3, 4, 9, 14, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + } + // Store output: out[i] = s[i] ^ s[i+8] for i in 0..7 + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(CHUNK_CV_OFFSET + i * 4)]); // i32.const offset + put([0x20, 16 + i]); // local.get s[i] + put([0x20, 24 + i]); // local.get s[i+8] + put([0x73]); // i32.xor + put([0x36, 0x02, 0x00]); // i32.store align=4 offset=0 + } + // end function + put([0x0b]); // end + return code; +} +// Cached WASM instance +let wasmInstance = null; +let wasmMemory = null; +let wasmCompress4x = null; +let wasmCompressChunks4x = null; +let wasmCompressParent = null; +let wasmMemoryView = null; +let wasmMemoryView32 = null; +/** + * Check if WASM SIMD is supported. + */ +function isSimdSupported() { + try { + // Minimal WASM module with v128.const instruction to test SIMD support + const simdTest = new Uint8Array([ + 0x00, + 0x61, + 0x73, + 0x6d, // magic: \0asm + 0x01, + 0x00, + 0x00, + 0x00, // version: 1 + // Type section (id=1): () -> v128 + 0x01, // section id = 1 (type) + 0x05, // section length = 5 + 0x01, // 1 type + 0x60, + 0x00, + 0x01, + 0x7b, // func () -> v128 + // Function section (id=3) + 0x03, // section id = 3 (function) + 0x02, // section length = 2 + 0x01, // 1 function + 0x00, // type index 0 + // Code section (id=10) with v128.const + 0x0a, // section id = 10 (code) + 0x16, // section length = 22 + 0x01, // 1 function body + 0x14, // body length = 20 + 0x00, // 0 locals + 0xfd, + 0x0c, // v128.const opcode + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x0b, // end + ]); + return WebAssembly.validate(simdTest); + } + catch { + return false; + } +} +/** + * Set up arena views over WASM memory. + * Called after WASM memory is allocated. + */ +function setupArenaViews() { + if (!wasmMemory) + return; + const buffer = wasmMemory.buffer; + // Create TypedArray views over WASM memory for arena buffers + // These views are backed by WASM memory, eliminating JS heap allocation + arenaCvStack = new Uint32Array(buffer, exports.SIMD_MEMORY.CV_STACK, 64 * 8); // 64 levels × 8 words + arenaParentBlock = new Uint32Array(buffer, exports.SIMD_MEMORY.PARENT_BLOCK, 16); // 16 words + arenaChunkCv = new Uint32Array(buffer, exports.SIMD_MEMORY.CHUNK_CV, 8); // 8 words + arenaTempCvs = new Uint32Array(buffer, exports.SIMD_MEMORY.TEMP_CVS, 32); // 4 × 8 words + // Batch mode views + // 16 positions × 16 v128 words = 16 × 64 u32 words = 1024 words per position? No... + // In u32 terms: 16 positions × 16 words × 4 lanes = 1024 u32 values total + arenaBatchBlockWords = new Uint32Array(buffer, exports.SIMD_MEMORY.BATCH_BLOCK_WORDS, 16 * 16 * 4); // 16 pos × 16 words × 4 lanes + arenaBatchCv = new Uint32Array(buffer, exports.SIMD_MEMORY.BATCH_CV, 32); // 4 × 8 words + arenaBatchCounterLow = new Uint32Array(buffer, exports.SIMD_MEMORY.BATCH_COUNTER_LOW, 4); // 4 words + arenaBatchFlagsBase = new Uint32Array(buffer, exports.SIMD_MEMORY.BATCH_FLAGS_BASE, 4); // 4 words + arenaBatchOutput = new Uint32Array(buffer, exports.SIMD_MEMORY.BATCH_OUTPUT, 32); // 4 × 8 words +} +/** + * Initialize the WASM SIMD module synchronously. + * Call this once before using compress4x. + */ +// Cache generated WASM bytes to avoid regenerating on each init +let cachedWasmBytes = null; +function initSimdSync() { + if (wasmInstance) + return true; + if (!isSimdSupported()) { + return false; + } + try { + const wasmBytes = cachedWasmBytes || generateWasmBytes(); + cachedWasmBytes = wasmBytes; + wasmMemory = new WebAssembly.Memory({ initial: 1 }); + const importObject = { + js: { mem: wasmMemory }, + }; + const module = new WebAssembly.Module(wasmBytes.buffer); + wasmInstance = new WebAssembly.Instance(module, importObject); + wasmCompress4x = wasmInstance.exports.compress4x; + wasmCompressChunks4x = wasmInstance.exports.compressChunks4x; + wasmCompressParent = wasmInstance.exports.compressParent; + wasmMemoryView = new Uint8Array(wasmMemory.buffer); + wasmMemoryView32 = new Uint32Array(wasmMemory.buffer); + // Set up arena views for Merkle tree operations + setupArenaViews(); + return true; + } + catch (e) { + console.warn("Failed to initialize WASM SIMD:", e); + return false; + } +} +/** + * Memory offsets for SIMD data layout + * + * WASM Arena Pattern: All working buffers live in WASM memory (64KB page) + * This eliminates JS heap allocations during hashing operations. + */ +exports.SIMD_MEMORY = { + // SIMD compress4x working area (used by WASM code) - single block + BLOCK_WORDS: 0, // 4 x 16 words = 512 bytes (transposed layout) + CHAINING_VALUES: 512, // 4 x 8 words = 128 bytes + OUTPUT: 640, // 4 x 8 words = 128 bytes + COUNTER_LOW: 768, // 4 words = 16 bytes + COUNTER_HIGH: 784, // 4 words = 16 bytes + BLOCK_LEN: 800, // 4 words = 16 bytes + FLAGS: 816, // 4 words = 16 bytes + // End of single-block SIMD working area: 832 bytes + // SIMD compressChunks4x working area - 16 blocks batched + // Each block position has 16 v128 values (one per message word) = 256 bytes + // 16 block positions = 16 × 256 = 4096 bytes + BATCH_BLOCK_WORDS: 832, // 16 positions × 256 bytes = 4096 bytes (transposed), ends at 4928 + BATCH_CV: 4928, // 4 × 8 words × 4 bytes = 128 bytes (working CVs), ends at 5056 + BATCH_COUNTER_LOW: 5056, // 4 words × 4 bytes = 16 bytes (per-chunk counters), ends at 5072 + BATCH_FLAGS_BASE: 5072, // 4 words × 4 bytes = 16 bytes (base flags, no START/END), ends at 5088 + BATCH_OUTPUT: 5088, // 4 × 8 words × 4 bytes = 128 bytes (final output), ends at 5216 + // End of batch working area: 5216 bytes + // WASM Arena: JS working buffers (accessed via TypedArray views) + CV_STACK: 5216, // 64 levels × 8 words × 4 bytes = 2048 bytes, ends at 7264 + PARENT_BLOCK: 7264, // 16 words × 4 bytes = 64 bytes, ends at 7328 + CHUNK_CV: 7328, // 8 words × 4 bytes = 32 bytes, ends at 7360 + TEMP_CVS: 7360, // 4 × 8 words × 4 bytes = 128 bytes, ends at 7488 + // Total arena usage: ~7488 bytes (fits comfortably in 64KB page) +}; +// Arena views - created once when SIMD initializes +let arenaCvStack = null; +let arenaParentBlock = null; +let arenaChunkCv = null; +let arenaTempCvs = null; +// Batch mode arena views +let arenaBatchBlockWords = null; +let arenaBatchCv = null; +let arenaBatchCounterLow = null; +let arenaBatchFlagsBase = null; +let arenaBatchOutput = null; +/** + * Get the WASM memory views for writing input data. + */ +function getSimdMemory() { + if (!wasmMemoryView || !wasmMemoryView32) + return null; + return { view: wasmMemoryView, view32: wasmMemoryView32 }; +} +/** + * Get the arena buffers for Merkle tree operations. + * These TypedArray views are backed by WASM memory - zero JS heap allocation. + */ +function getArenaBuffers() { + if (!arenaCvStack || !arenaParentBlock || !arenaChunkCv || !arenaTempCvs) + return null; + return { + cvStack: arenaCvStack, + parentBlock: arenaParentBlock, + chunkCv: arenaChunkCv, + tempCvs: arenaTempCvs, + }; +} +/** + * Get the batch arena buffers for chunk-level batched operations. + * These TypedArray views are backed by WASM memory - zero JS heap allocation. + */ +function getBatchArenaBuffers() { + if (!arenaBatchBlockWords || + !arenaBatchCv || + !arenaBatchCounterLow || + !arenaBatchFlagsBase || + !arenaBatchOutput) + return null; + return { + blockWords: arenaBatchBlockWords, + cv: arenaBatchCv, + counterLow: arenaBatchCounterLow, + flagsBase: arenaBatchFlagsBase, + output: arenaBatchOutput, + }; +} +/** + * Run the compress4x function. + * Data must already be set up in WASM memory. + */ +function runCompress4x() { + if (!wasmCompress4x) { + throw new Error("WASM SIMD not initialized. Call initSimdSync() first."); + } + wasmCompress4x(); +} +/** + * Run the compressChunks4x function. + * Processes 4 full chunks (16 blocks each) in a single WASM call. + * Data must already be set up in batch arena buffers. + */ +function runCompressChunks4x() { + if (!wasmCompressChunks4x) { + throw new Error("WASM SIMD not initialized. Call initSimdSync() first."); + } + wasmCompressChunks4x(); +} +/** + * Run the compressParent function. + * Compresses a parent node: reads 16 words from PARENT_BLOCK, writes 8 words to CHUNK_CV. + * Data must already be set up in arena buffers (PARENT_BLOCK at offset 7264). + * Output is written to CHUNK_CV at offset 7328. + */ +function runCompressParent() { + if (!wasmCompressParent) { + throw new Error("WASM SIMD not initialized. Call initSimdSync() first."); + } + wasmCompressParent(); +} +/** + * Check if SIMD is initialized and ready. + */ +function isSimdReady() { + return wasmCompress4x !== null; +} diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/compress.d.ts b/node_modules/@huggingface/blake3-jit/dist/esm/compress.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..dcb794007e895c0b6056f2a5f39f60995c151c01 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/compress.d.ts @@ -0,0 +1,37 @@ +/** + * BLAKE3 Compression Function - Highly Optimized + * + * Optimization techniques applied (from Fleek Network case study): + * 1. Use 16 SMI variables for state instead of TypedArray + * 2. Use 16 SMI variables for message words + * 3. Fully inlined G function (no function call overhead) + * 4. Use `| 0` for integer coercion (forces V8 to use 32-bit ALU) + * 5. Hardcoded permutation swaps using only 2 temporary variables + * 6. Offset-based access pattern (avoid creating new views) + * + * The compression function takes: + * - cv: 8-word chaining value + * - block: 16-word message block (64 bytes) + * - counter: 64-bit block counter + * - blockLen: number of input bytes in this block + * - flags: domain separation flags + * + * And outputs 8 or 16 words depending on whether this is a root node. + */ +/** + * Compress a single block. + * + * This is the hot path - every optimization matters here. + * + * @param cv - Chaining value array + * @param cvOff - Offset into cv + * @param block - Message block words + * @param blockOff - Offset into block + * @param out - Output array (8 or 16 words) + * @param outOff - Offset into out + * @param full - If true, output all 16 words (for XOF); if false, output 8 words + * @param counter - 64-bit block counter + * @param blockLen - Number of bytes in this block (0-64) + * @param flags - Domain separation flags + */ +export declare function compress(cv: Uint32Array, cvOff: number, block: Uint32Array, blockOff: number, out: Uint32Array, outOff: number, full: boolean, counter: number, blockLen: number, flags: number): void; diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/compress.js b/node_modules/@huggingface/blake3-jit/dist/esm/compress.js new file mode 100644 index 0000000000000000000000000000000000000000..cfebe77fa4b9b14b9d005862dd1435261630adab --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/compress.js @@ -0,0 +1,916 @@ +/** + * BLAKE3 Compression Function - Highly Optimized + * + * Optimization techniques applied (from Fleek Network case study): + * 1. Use 16 SMI variables for state instead of TypedArray + * 2. Use 16 SMI variables for message words + * 3. Fully inlined G function (no function call overhead) + * 4. Use `| 0` for integer coercion (forces V8 to use 32-bit ALU) + * 5. Hardcoded permutation swaps using only 2 temporary variables + * 6. Offset-based access pattern (avoid creating new views) + * + * The compression function takes: + * - cv: 8-word chaining value + * - block: 16-word message block (64 bytes) + * - counter: 64-bit block counter + * - blockLen: number of input bytes in this block + * - flags: domain separation flags + * + * And outputs 8 or 16 words depending on whether this is a root node. + */ +/** + * Compress a single block. + * + * This is the hot path - every optimization matters here. + * + * @param cv - Chaining value array + * @param cvOff - Offset into cv + * @param block - Message block words + * @param blockOff - Offset into block + * @param out - Output array (8 or 16 words) + * @param outOff - Offset into out + * @param full - If true, output all 16 words (for XOF); if false, output 8 words + * @param counter - 64-bit block counter + * @param blockLen - Number of bytes in this block (0-64) + * @param flags - Domain separation flags + */ +export function compress(cv, cvOff, block, blockOff, out, outOff, full, counter, blockLen, flags) { + // Load message words into SMI variables for maximum performance + // V8 optimizes SMI arithmetic directly with the ALU + let m0 = block[blockOff] | 0; + let m1 = block[blockOff + 1] | 0; + let m2 = block[blockOff + 2] | 0; + let m3 = block[blockOff + 3] | 0; + let m4 = block[blockOff + 4] | 0; + let m5 = block[blockOff + 5] | 0; + let m6 = block[blockOff + 6] | 0; + let m7 = block[blockOff + 7] | 0; + let m8 = block[blockOff + 8] | 0; + let m9 = block[blockOff + 9] | 0; + let m10 = block[blockOff + 10] | 0; + let m11 = block[blockOff + 11] | 0; + let m12 = block[blockOff + 12] | 0; + let m13 = block[blockOff + 13] | 0; + let m14 = block[blockOff + 14] | 0; + let m15 = block[blockOff + 15] | 0; + // Initialize state: first 8 words from chaining value + let s0 = cv[cvOff] | 0; + let s1 = cv[cvOff + 1] | 0; + let s2 = cv[cvOff + 2] | 0; + let s3 = cv[cvOff + 3] | 0; + let s4 = cv[cvOff + 4] | 0; + let s5 = cv[cvOff + 5] | 0; + let s6 = cv[cvOff + 6] | 0; + let s7 = cv[cvOff + 7] | 0; + // Words 8-11: IV constants + let s8 = 0x6a09e667; + let s9 = 0xbb67ae85; + let s10 = 0x3c6ef372; + let s11 = 0xa54ff53a; + // Words 12-15: counter, blockLen, flags + // Note: counter is 64-bit, split into low and high 32-bit words + let s12 = counter | 0; + let s13 = (counter / 0x100000000) | 0; + let s14 = blockLen | 0; + let s15 = flags | 0; + // ============================================================ + // 7 rounds of mixing + // Each round consists of 4 column G functions and 4 diagonal G functions + // followed by a message word permutation (except for round 7) + // ============================================================ + // ROUND 1 (message schedule: 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15) + // Column G functions + // G(0, 4, 8, 12) with m0, m1 + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + // G(1, 5, 9, 13) with m2, m3 + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + // G(2, 6, 10, 14) with m4, m5 + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + // G(3, 7, 11, 15) with m6, m7 + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + // Diagonal G functions + // G(0, 5, 10, 15) with m8, m9 + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + // G(1, 6, 11, 12) with m10, m11 + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + // G(2, 7, 8, 13) with m12, m13 + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + // G(3, 4, 9, 14) with m14, m15 + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + // Permute message words for round 2 + // Permutation: [2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8] + // Using 2 temps for the two cycles in the permutation + { + const t0 = m0, t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + // ROUND 2 (message schedule: 2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + // Permute for round 3 + { + const t0 = m0, t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + // ROUND 3 (message schedule: 3,4,10,12,13,2,7,14,6,5,9,0,11,15,8,1) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + // Permute for round 4 + { + const t0 = m0, t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + // ROUND 4 (message schedule: 10,7,12,9,14,3,13,15,4,0,11,2,5,8,1,6) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + // Permute for round 5 + { + const t0 = m0, t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + // ROUND 5 (message schedule: 12,13,9,11,15,10,14,8,7,2,5,3,0,1,6,4) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + // Permute for round 6 + { + const t0 = m0, t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + // ROUND 6 (message schedule: 9,14,11,5,8,12,15,1,13,3,0,10,2,6,4,7) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + // Permute for round 7 + { + const t0 = m0, t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + // ROUND 7 (message schedule: 11,15,5,0,1,9,8,6,14,10,2,12,3,4,7,13) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + // ============================================================ + // Final XOR and output + // ============================================================ + // If full output needed (XOF mode), write words 8-15 first + // (written first in case out === cv) + if (full) { + out[outOff + 8] = s8 ^ cv[cvOff]; + out[outOff + 9] = s9 ^ cv[cvOff + 1]; + out[outOff + 10] = s10 ^ cv[cvOff + 2]; + out[outOff + 11] = s11 ^ cv[cvOff + 3]; + out[outOff + 12] = s12 ^ cv[cvOff + 4]; + out[outOff + 13] = s13 ^ cv[cvOff + 5]; + out[outOff + 14] = s14 ^ cv[cvOff + 6]; + out[outOff + 15] = s15 ^ cv[cvOff + 7]; + } + // Standard output: XOR state[0..7] with state[8..15] + out[outOff] = s0 ^ s8; + out[outOff + 1] = s1 ^ s9; + out[outOff + 2] = s2 ^ s10; + out[outOff + 3] = s3 ^ s11; + out[outOff + 4] = s4 ^ s12; + out[outOff + 5] = s5 ^ s13; + out[outOff + 6] = s6 ^ s14; + out[outOff + 7] = s7 ^ s15; +} diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/constants.d.ts b/node_modules/@huggingface/blake3-jit/dist/esm/constants.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..61cf697fdc06eb91ee9fcaffd6ba7e187f74eae0 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/constants.d.ts @@ -0,0 +1,34 @@ +/** + * BLAKE3 Constants + * + * IV values are the same as SHA-256: first 32 bits of the fractional parts + * of the square roots of the first 8 primes (2..19) + */ +export declare const IV: Uint32Array; +export declare const CHUNK_START = 1; +export declare const CHUNK_END: number; +export declare const PARENT: number; +export declare const ROOT: number; +export declare const KEYED_HASH: number; +export declare const DERIVE_KEY_CONTEXT: number; +export declare const DERIVE_KEY_MATERIAL: number; +export declare const OUT_LEN = 32; +export declare const KEY_LEN = 32; +export declare const BLOCK_LEN = 64; +export declare const CHUNK_LEN = 1024; +export declare const MAX_DEPTH = 54; +/** + * Precomputed message word permutations for all 7 rounds. + * + * The base permutation is: [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] + * Each subsequent permutation is the previous one with this permutation applied. + * + * These are the indices into the message block for each round. + * By precomputing these, we avoid runtime permutation overhead. + */ +export declare const MSG_SCHEDULE: ReadonlyArray>; +/** + * Flattened permutation table for compress function optimization. + * This enables direct indexed access: PERMUTATIONS[round * 16 + index] + */ +export declare const PERMUTATIONS: Uint8Array; diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/constants.js b/node_modules/@huggingface/blake3-jit/dist/esm/constants.js new file mode 100644 index 0000000000000000000000000000000000000000..1c49c5c154164ca91a55ab954fe777a2baaa34ee --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/constants.js @@ -0,0 +1,53 @@ +/** + * BLAKE3 Constants + * + * IV values are the same as SHA-256: first 32 bits of the fractional parts + * of the square roots of the first 8 primes (2..19) + */ +// Initialization Vector (same as SHA-256) +export const IV = new Uint32Array([ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, +]); +// Domain separation flags +export const CHUNK_START = 1; +export const CHUNK_END = 1 << 1; +export const PARENT = 1 << 2; +export const ROOT = 1 << 3; +export const KEYED_HASH = 1 << 4; +export const DERIVE_KEY_CONTEXT = 1 << 5; +export const DERIVE_KEY_MATERIAL = 1 << 6; +// Size constants +export const OUT_LEN = 32; +export const KEY_LEN = 32; +export const BLOCK_LEN = 64; +export const CHUNK_LEN = 1024; +// Maximum depth of the CV stack (supports up to 2^54 bytes input) +export const MAX_DEPTH = 54; +/** + * Precomputed message word permutations for all 7 rounds. + * + * The base permutation is: [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] + * Each subsequent permutation is the previous one with this permutation applied. + * + * These are the indices into the message block for each round. + * By precomputing these, we avoid runtime permutation overhead. + */ +export const MSG_SCHEDULE = [ + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8], + [3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1], + [10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6], + [12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4], + [9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7], + [11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13], +]; +/** + * Flattened permutation table for compress function optimization. + * This enables direct indexed access: PERMUTATIONS[round * 16 + index] + */ +export const PERMUTATIONS = new Uint8Array([ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, + 14, 15, 8, 3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1, 10, 7, 12, 9, 14, 3, 13, 15, 4, + 0, 11, 2, 5, 8, 1, 6, 12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4, 9, 14, 11, 5, 8, 12, + 15, 1, 13, 3, 0, 10, 2, 6, 4, 7, 11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13, +]); diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/hash.d.ts b/node_modules/@huggingface/blake3-jit/dist/esm/hash.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2e22b4908846fbd6b0ff65856a47aec9f6da2a73 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/hash.d.ts @@ -0,0 +1,28 @@ +/** + * BLAKE3 Hash Function - Simple one-shot API + * + * This provides a simple hash() function optimized for different input sizes. + * For small inputs, uses pure JS. For large inputs, uses WASM SIMD. + */ +/** + * Hash input data and return the result. + * Automatically uses WASM SIMD for large inputs when available. + * + * @param input - Data to hash + * @param outputLength - Number of bytes to output (default: 32) + * @returns The hash output + */ +export declare function hash(input: Uint8Array, outputLength?: number): Uint8Array; +/** + * Pre-warm SIMD initialization (call early to avoid latency later). + */ +export declare function warmupSimd(): boolean; +/** + * Hash input data directly into a caller-provided output buffer. + * Zero-allocation for the common 32-byte case - ideal for performance-critical code. + * + * @param input - Data to hash + * @param output - Pre-allocated output buffer (must be at least outputLength bytes) + * @param outputLength - Number of bytes to output (default: 32, max: output.length) + */ +export declare function hashInto(input: Uint8Array, output: Uint8Array, outputLength?: number): void; diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/hash.js b/node_modules/@huggingface/blake3-jit/dist/esm/hash.js new file mode 100644 index 0000000000000000000000000000000000000000..846153bcdf9c445903ccec866d4f5928984fb888 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/hash.js @@ -0,0 +1,1033 @@ +/** + * BLAKE3 Hash Function - Simple one-shot API + * + * This provides a simple hash() function optimized for different input sizes. + * For small inputs, uses pure JS. For large inputs, uses WASM SIMD. + */ +import { compress } from "./compress.js"; +import { IV, CHUNK_START, CHUNK_END, PARENT, ROOT, BLOCK_LEN, CHUNK_LEN, OUT_LEN, } from "./constants.js"; +import { IS_LITTLE_ENDIAN, readLittleEndianWordsFull, readLittleEndianWordsPartial, writeLittleEndianBytesPartial, } from "./utils.js"; +import { initSimdSync, getSimdMemory, getArenaBuffers, runCompress4x, runCompressChunks4x, runCompressParent, SIMD_MEMORY, } from "./wasm-simd.js"; +// Pre-allocated buffers for reuse (single-threaded optimization) +let blockWords = null; +// ===== Contiguous Hyper CV Stack (Optimization #6) ===== +// Maximum tree depth for practical inputs (2^64 chunks = essentially unlimited) +// Fixed allocation at module load - no runtime allocation +const CV_STACK_DEPTH = 64; +const HYPER_CV_STACK = new Uint32Array(CV_STACK_DEPTH * 8); // 64 CVs × 8 words = 512 words +// Pre-computed offsets for the first few stack levels (hot path optimization) +// Note: These can be used for further optimization if needed +// const CV_STACK_OFF_0 = 0; +// const CV_STACK_OFF_1 = 8; +// const CV_STACK_OFF_2 = 16; +// const CV_STACK_OFF_3 = 24; +// ===== Pre-allocated CV Pool with Views (avoids subarray() in hot paths) ===== +const CV_POOL_SIZE = 64; +const CV_POOL = new Uint32Array(CV_POOL_SIZE * 8); // 64 CVs × 8 words = 512 words +const CV_VIEWS = []; +for (let i = 0; i < CV_POOL_SIZE; i++) { + CV_VIEWS.push(CV_POOL.subarray(i * 8, i * 8 + 8)); +} +// SIMD initialization state +let simdAvailable = false; +// Threshold for switching to SIMD (must be > 1 chunk to benefit from parallelism) +const SIMD_THRESHOLD = 4 * CHUNK_LEN; // 4KB - need at least 4 chunks for SIMD benefit +/** + * Initialize SIMD synchronously (lazy). + */ +function ensureSimdSync() { + if (simdAvailable) + return true; + simdAvailable = initSimdSync(); + return simdAvailable; +} +// Reusable buffer for SIMD chunk CVs (4 chunks × 8 words) +const simdChunkCvs = new Uint32Array(32); +// ===== Module-level reusable buffers (single-threaded safe) ===== +// These eliminate heap allocations in hot paths +// For hashChunkWithWords() and hashChunkRoot() +const reusableTempCv = new Uint32Array(8); +// For hashPureJS() +const reusableChunkCv = new Uint32Array(8); +const reusablePureParentBlock = new Uint32Array(16); +const reusablePureParentCv = new Uint32Array(8); +// For hashSimd() - use flat array for 4 chunk CVs (access via subarray) +const reusableSimdCvs = new Uint32Array(32); // 4 × 8 words flat +// For hashSimd() parent compression +const reusableSimdParentBlock = new Uint32Array(16); +const reusableSimdParentCv = new Uint32Array(8); +// For hashSimd() parameters - TypedArrays instead of JS arrays +const reusableOffsets = new Uint32Array(4); +const reusableCounters = new Uint32Array(4); +const reusableBlockLens = new Uint32Array(4); +const reusableFlags = new Uint32Array(4); +// Reusable output buffer for common 32-byte hash (eliminates allocations) +const reusableOut8 = new Uint32Array(8); // Standard 32-byte output +// Pre-created view to avoid allocation in hot path (Task 1 optimization) +const reusableOut8View = new Uint8Array(reusableOut8.buffer, 0, 32); +// ===== Unrolled CV Copy Helper (Task 7 optimization) ===== +// V8 will inline this - avoids loop overhead in hot paths +function copyCV8(src, srcOff, dst, dstOff) { + dst[dstOff] = src[srcOff]; + dst[dstOff + 1] = src[srcOff + 1]; + dst[dstOff + 2] = src[srcOff + 2]; + dst[dstOff + 3] = src[srcOff + 3]; + dst[dstOff + 4] = src[srcOff + 4]; + dst[dstOff + 5] = src[srcOff + 5]; + dst[dstOff + 6] = src[srcOff + 6]; + dst[dstOff + 7] = src[srcOff + 7]; +} +/** + * Transpose 4 blocks (64 bytes each) into SIMD memory layout. + * The SIMD compress4x expects: [m0_0,m0_1,m0_2,m0_3, m1_0,m1_1,m1_2,m1_3, ...] + * where m{i}_{j} is message word i from block j. + * + * OPTIMIZED: Processes all 4 blocks together for each word position, + * writing 4 consecutive u32s at once for better cache locality. + * + * @param inputWords - Pre-created Uint32Array view of input (null if unaligned/non-LE). + * Created once per hash call to avoid allocation in hot loop. + */ +function transposeBlocksToSimd(input, offsets, // Starting offsets for each of 4 blocks +blockLens, // Length of each block (0-64 bytes) +mem32, blockCount, // 1-4 blocks +inputWords) { + // Fast path: all blocks are full 64-byte blocks with aligned LE input + const allFull = blockCount === 4 && + blockLens[0] === 64 && + blockLens[1] === 64 && + blockLens[2] === 64 && + blockLens[3] === 64; + if (allFull && + inputWords && + offsets[0] % 4 === 0 && + offsets[1] % 4 === 0 && + offsets[2] % 4 === 0 && + offsets[3] % 4 === 0) { + // Ultra-fast path: process all 4 blocks together, write 4 consecutive u32s per word + const wordOff0 = offsets[0] >>> 2; + const wordOff1 = offsets[1] >>> 2; + const wordOff2 = offsets[2] >>> 2; + const wordOff3 = offsets[3] >>> 2; + for (let w = 0; w < 16; w++) { + const dstBase = w * 4; + mem32[dstBase] = inputWords[wordOff0 + w]; + mem32[dstBase + 1] = inputWords[wordOff1 + w]; + mem32[dstBase + 2] = inputWords[wordOff2 + w]; + mem32[dstBase + 3] = inputWords[wordOff3 + w]; + } + return; + } + // Standard path: process each block independently (handles partial blocks) + for (let b = 0; b < blockCount; b++) { + const len = blockLens[b]; + const off = offsets[b]; + if (len === 64) { + // Full block + if (inputWords && off % 4 === 0) { + // Direct Uint32Array access for aligned LE blocks + const wordOff = off >>> 2; + for (let w = 0; w < 16; w++) { + mem32[w * 4 + b] = inputWords[wordOff + w]; + } + } + else { + // Byte-by-byte reconstruction + for (let w = 0; w < 16; w++) { + const srcOff = off + w * 4; + mem32[w * 4 + b] = + input[srcOff] | + (input[srcOff + 1] << 8) | + (input[srcOff + 2] << 16) | + (input[srcOff + 3] << 24); + } + } + } + else if (len === 0) { + // Zero block + for (let w = 0; w < 16; w++) { + mem32[w * 4 + b] = 0; + } + } + else { + // Partial block - handle word by word + for (let w = 0; w < 16; w++) { + const wordOff = w * 4; + if (wordOff >= len) { + mem32[w * 4 + b] = 0; + } + else if (wordOff + 4 <= len) { + const srcOff = off + wordOff; + mem32[w * 4 + b] = + input[srcOff] | + (input[srcOff + 1] << 8) | + (input[srcOff + 2] << 16) | + (input[srcOff + 3] << 24); + } + else { + // Partial word at end of block + let word = 0; + for (let i = 0; i < len - wordOff; i++) { + word |= input[off + wordOff + i] << (i * 8); + } + mem32[w * 4 + b] = word; + } + } + } + } + // Zero unused block slots + for (let b = blockCount; b < 4; b++) { + for (let w = 0; w < 16; w++) { + mem32[w * 4 + b] = 0; + } + } +} +/** + * Transpose 4 full chunks (4 × 16 blocks = 64 blocks) into batch SIMD memory. + * This is used for the batched compressChunks4x function that processes + * all 16 blocks in a single WASM call. + * + * Memory layout: BATCH_BLOCK_WORDS has 16 positions, each with 16 v128 values. + * Position p, word w: mem32[(p * 64) + (w * 4) + lane] + * + * OPTIMIZED: Processes all 4 chunks together for each (pos, word) pair, + * writing 4 consecutive u32s at once for better cache locality. + * + * @param input - Input data (must have at least 4 full chunks = 4096 bytes) + * @param chunkOffsets - Starting offsets for each of 4 chunks + * @param mem32 - WASM memory view + * @param inputWords - Pre-created Uint32Array view (null if unaligned) + */ +function transposeBatchToSimd(input, chunkOffsets, mem32, inputWords) { + const BATCH_BASE = SIMD_MEMORY.BATCH_BLOCK_WORDS / 4; + // Get base word offsets for each chunk (pre-computed for fast path) + const chunk0WordBase = chunkOffsets[0] >>> 2; + const chunk1WordBase = chunkOffsets[1] >>> 2; + const chunk2WordBase = chunkOffsets[2] >>> 2; + const chunk3WordBase = chunkOffsets[3] >>> 2; + // Fast path: all chunks aligned and LE - process 4 consecutive u32s at once + if (inputWords && chunkOffsets[0] % 4 === 0) { + for (let pos = 0; pos < 16; pos++) { + const posBase = BATCH_BASE + pos * 64; // 16 words × 4 lanes = 64 + const blockWordOff = pos * 16; // 16 words per block (64 bytes / 4) + // Process all 16 words, writing 4 chunks at a time (cache-friendly: 16 bytes per write group) + for (let w = 0; w < 16; w++) { + const dstBase = posBase + w * 4; + // Read word w from all 4 chunks at positions that become consecutive in output + mem32[dstBase] = inputWords[chunk0WordBase + blockWordOff + w]; + mem32[dstBase + 1] = inputWords[chunk1WordBase + blockWordOff + w]; + mem32[dstBase + 2] = inputWords[chunk2WordBase + blockWordOff + w]; + mem32[dstBase + 3] = inputWords[chunk3WordBase + blockWordOff + w]; + } + } + } + else { + // Slow path: byte-by-byte reconstruction, still cache-friendly write pattern + for (let pos = 0; pos < 16; pos++) { + const posBase = BATCH_BASE + pos * 64; + const blockByteOff = pos * 64; // 64 bytes per block + for (let w = 0; w < 16; w++) { + const dstBase = posBase + w * 4; + const wordByteOff = w * 4; + // Chunk 0 + const off0 = chunkOffsets[0] + blockByteOff + wordByteOff; + mem32[dstBase] = + input[off0] | (input[off0 + 1] << 8) | (input[off0 + 2] << 16) | (input[off0 + 3] << 24); + // Chunk 1 + const off1 = chunkOffsets[1] + blockByteOff + wordByteOff; + mem32[dstBase + 1] = + input[off1] | (input[off1 + 1] << 8) | (input[off1 + 2] << 16) | (input[off1 + 3] << 24); + // Chunk 2 + const off2 = chunkOffsets[2] + blockByteOff + wordByteOff; + mem32[dstBase + 2] = + input[off2] | (input[off2 + 1] << 8) | (input[off2 + 2] << 16) | (input[off2 + 3] << 24); + // Chunk 3 + const off3 = chunkOffsets[3] + blockByteOff + wordByteOff; + mem32[dstBase + 3] = + input[off3] | (input[off3 + 1] << 8) | (input[off3 + 2] << 16) | (input[off3 + 3] << 24); + } + } + } +} +// Pre-computed memory offsets for SIMD operations (single-block mode) +const SIMD_CV_BASE = SIMD_MEMORY.CHAINING_VALUES / 4; +const SIMD_OUT_BASE = SIMD_MEMORY.OUTPUT / 4; +const SIMD_COUNTER_LOW_BASE = SIMD_MEMORY.COUNTER_LOW / 4; +const SIMD_COUNTER_HIGH_BASE = SIMD_MEMORY.COUNTER_HIGH / 4; +const SIMD_BLOCK_LEN_BASE = SIMD_MEMORY.BLOCK_LEN / 4; +// Pre-computed memory offsets for batch SIMD operations (16-block mode) +const BATCH_CV_BASE = SIMD_MEMORY.BATCH_CV / 4; +const BATCH_COUNTER_LOW_BASE = SIMD_MEMORY.BATCH_COUNTER_LOW / 4; +const BATCH_FLAGS_BASE_OFFSET = SIMD_MEMORY.BATCH_FLAGS_BASE / 4; +const BATCH_OUTPUT_BASE = SIMD_MEMORY.BATCH_OUTPUT / 4; +// Reusable arrays for batch processing +const batchChunkOffsets = new Uint32Array(4); +const SIMD_FLAGS_BASE = SIMD_MEMORY.FLAGS / 4; +/** + * Set up chaining values in SIMD memory (transposed layout). + * Optimized: unrolled loops for common case of 4 chunks. + * cvs is flat: [cv0_word0..cv0_word7, cv1_word0..cv1_word7, ...] + */ +function setupSimdCvs(cvs, // Flat array: 4 × 8 words +mem32, count) { + // Unrolled for 4 chunks (common case) + if (count === 4) { + for (let w = 0; w < 8; w++) { + const base = SIMD_CV_BASE + w * 4; + mem32[base] = cvs[w]; // cv0[w] + mem32[base + 1] = cvs[8 + w]; // cv1[w] + mem32[base + 2] = cvs[16 + w]; // cv2[w] + mem32[base + 3] = cvs[24 + w]; // cv3[w] + } + } + else { + for (let w = 0; w < 8; w++) { + const base = SIMD_CV_BASE + w * 4; + for (let c = 0; c < count; c++) { + mem32[base + c] = cvs[c * 8 + w]; + } + for (let c = count; c < 4; c++) { + mem32[base + c] = 0; + } + } + } +} +/** + * Set up SIMD parameters (counters, flags, block lengths). + */ +function setupSimdParams(mem32, counters, blockLens, flagsArr, count) { + // Most chunk counters fit in 32 bits, so counter high is usually 0 + for (let i = 0; i < count; i++) { + mem32[SIMD_COUNTER_LOW_BASE + i] = counters[i]; + mem32[SIMD_COUNTER_HIGH_BASE + i] = 0; // Assume counters fit in 32 bits + mem32[SIMD_BLOCK_LEN_BASE + i] = blockLens[i]; + mem32[SIMD_FLAGS_BASE + i] = flagsArr[i]; + } + // Zero unused slots + for (let i = count; i < 4; i++) { + mem32[SIMD_COUNTER_LOW_BASE + i] = 0; + mem32[SIMD_COUNTER_HIGH_BASE + i] = 0; + mem32[SIMD_BLOCK_LEN_BASE + i] = 0; + mem32[SIMD_FLAGS_BASE + i] = 0; + } +} +/** + * Read output CVs from SIMD memory (untranspose). + */ +function readSimdOutputCvs(mem32, outputCvs, // Flat array: 4 × 8 words +count) { + // Unrolled for 4 chunks (common case) + if (count === 4) { + for (let w = 0; w < 8; w++) { + const base = SIMD_OUT_BASE + w * 4; + outputCvs[w] = mem32[base]; + outputCvs[8 + w] = mem32[base + 1]; + outputCvs[16 + w] = mem32[base + 2]; + outputCvs[24 + w] = mem32[base + 3]; + } + } + else { + for (let w = 0; w < 8; w++) { + const base = SIMD_OUT_BASE + w * 4; + for (let c = 0; c < count; c++) { + outputCvs[c * 8 + w] = mem32[base + c]; + } + } + } +} +function getBlockWords() { + if (!blockWords) { + blockWords = new Uint32Array(16); + } + return blockWords; +} +/** + * Hash a single chunk (up to 1024 bytes) with pre-created inputWords view. + * This is the optimized version that avoids creating Uint32Array views per chunk. + * (Fleek optimization Step 8) + */ +function hashChunkWithWords(input, inputWords, // Pre-created view of entire input +inputOffset, inputLen, chunkCounter, flags, cv, cvOffset) { + // Use reusable temporary CV for intermediate blocks (single-threaded safe) + reusableTempCv.set(IV); + // Process full blocks + const fullBlocks = inputLen >>> 6; // inputLen / 64 + const remainder = inputLen & 63; // inputLen % 64 + // Calculate word offset for this chunk within the pre-created view + const chunkWordOffset = inputOffset >>> 2; + // Fast path for full chunks with aligned little-endian input + if (inputWords && remainder === 0 && inputLen === CHUNK_LEN) { + // All 16 blocks are full, use fast path exclusively + let wordOff = chunkWordOffset; + // Block 0 (CHUNK_START) + compress(reusableTempCv, 0, inputWords, wordOff, reusableTempCv, 0, false, chunkCounter, BLOCK_LEN, flags | CHUNK_START); + wordOff += 16; + // Blocks 1-14 (no special flags) + for (let i = 1; i < 15; i++) { + compress(reusableTempCv, 0, inputWords, wordOff, reusableTempCv, 0, false, chunkCounter, BLOCK_LEN, flags); + wordOff += 16; + } + // Block 15 (CHUNK_END) + compress(reusableTempCv, 0, inputWords, wordOff, reusableTempCv, 0, false, chunkCounter, BLOCK_LEN, flags | CHUNK_END); + cv.set(reusableTempCv, cvOffset); + return; + } + // Slower path for partial chunks or non-aligned input + const totalBlocks = fullBlocks + (remainder > 0 ? 1 : 0); + const block = getBlockWords(); + for (let blockIdx = 0; blockIdx < totalBlocks; blockIdx++) { + const isFirst = blockIdx === 0; + const isLast = blockIdx === totalBlocks - 1; + const blockStart = blockIdx << 6; + const blockLen = isLast && remainder > 0 ? remainder : BLOCK_LEN; + // Determine flags for this block + let blockFlags = flags; + if (isFirst) + blockFlags |= CHUNK_START; + if (isLast) + blockFlags |= CHUNK_END; + // Load block words + if (isLast && remainder > 0) { + // Partial final block - need zero padding + readLittleEndianWordsPartial(input, inputOffset + blockStart, blockLen, block); + } + else if (inputWords && chunkWordOffset + (blockStart >>> 2) + 16 <= inputWords.length) { + // Fast path: use pre-created view directly + compress(reusableTempCv, 0, inputWords, chunkWordOffset + (blockStart >>> 2), reusableTempCv, 0, false, chunkCounter, blockLen, blockFlags); + continue; + } + else { + readLittleEndianWordsFull(input, inputOffset + blockStart, block); + } + compress(reusableTempCv, 0, block, 0, reusableTempCv, 0, false, chunkCounter, blockLen, blockFlags); + } + // Copy result to output + cv.set(reusableTempCv, cvOffset); +} +/** + * Hash input using pure JavaScript. + * Handles the full Merkle tree construction. + */ +function hashPureJS(input, outputLen) { + const inputLen = input.length; + // Special case: empty input + if (inputLen === 0) { + const block = getBlockWords(); + block.fill(0); + // Use reusable output buffer for common 32-byte case + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + compress(IV, 0, block, 0, out, 0, outputLen > 32, 0, 0, CHUNK_START | CHUNK_END | ROOT); + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } + else { + writeLittleEndianBytesPartial(out, 0, result, 0, outputLen); + } + return result; + } + // Calculate number of chunks + const numChunks = Math.ceil(inputLen / CHUNK_LEN); + // Single chunk optimization + if (numChunks === 1) { + // Use reusable output buffer for common 32-byte case + const cv = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + hashChunkRoot(input, 0, inputLen, 0, 0, cv, outputLen > 32); + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(cv.buffer, 0, outputLen)); + } + else { + writeLittleEndianBytesPartial(cv, 0, result, 0, outputLen); + } + return result; + } + // Multiple chunks - need Merkle tree + // Use the global contiguous CV stack (no allocation) + const stack = HYPER_CV_STACK; + let stackLen = 0; + // Use reusable buffers (single-threaded safe) + const chunkCv = reusableChunkCv; + const parentBlock = reusablePureParentBlock; + const parentCv = reusablePureParentCv; + // Create Uint32Array view ONCE for entire input (Fleek optimization Step 8) + // This avoids creating views inside each chunk/block processing + let inputWords = null; + const canUseFastPath = IS_LITTLE_ENDIAN && input.byteOffset % 4 === 0; + if (canUseFastPath) { + inputWords = new Uint32Array(input.buffer, input.byteOffset, inputLen >>> 2); + } + // Determine how many full chunks we have + const fullChunks = inputLen >>> 10; // inputLen / 1024 + const lastChunkLen = inputLen & 1023; // inputLen % 1024 + // Process all full chunks with fast path (inlined for performance) + if (canUseFastPath && inputWords) { + for (let chunkIdx = 0; chunkIdx < fullChunks; chunkIdx++) { + // Inline chunk processing for full chunks + chunkCv.set(IV); + let wordOff = chunkIdx << 8; // chunkIdx * 256 (CHUNK_LEN/4) + // Block 0 (CHUNK_START) + compress(chunkCv, 0, inputWords, wordOff, chunkCv, 0, false, chunkIdx, BLOCK_LEN, CHUNK_START); + wordOff += 16; + // Blocks 1-14 (no special flags) + for (let b = 1; b < 15; b++) { + compress(chunkCv, 0, inputWords, wordOff, chunkCv, 0, false, chunkIdx, BLOCK_LEN, 0); + wordOff += 16; + } + // Block 15 (CHUNK_END) + compress(chunkCv, 0, inputWords, wordOff, chunkCv, 0, false, chunkIdx, BLOCK_LEN, CHUNK_END); + // Merge completed subtrees (avoid subarray by using index math) + let totalChunks = chunkIdx + 1; + let cvSrcOff = 0; + let cvSrc = chunkCv; + // Check if this is the last chunk overall + const isLastChunk = chunkIdx === fullChunks - 1 && lastChunkLen === 0; + while ((totalChunks & 1) === 0 && stackLen > 0) { + // Skip final merge if it would produce the root; let finalization handle it with ROOT flag + if (stackLen === 1 && isLastChunk) { + break; + } + stackLen--; + const stackOff = stackLen * 8; + // Copy left CV from stack to parentBlock[0..7] (unrolled) + copyCV8(stack, stackOff, parentBlock, 0); + // Copy current CV to parentBlock[8..15] (unrolled) + copyCV8(cvSrc, cvSrcOff, parentBlock, 8); + compress(IV, 0, parentBlock, 0, parentCv, 0, false, 0, BLOCK_LEN, PARENT); + cvSrc = parentCv; + cvSrcOff = 0; + totalChunks >>>= 1; + } + // Push CV to stack (unrolled) + const stackOff = stackLen * 8; + copyCV8(cvSrc, cvSrcOff, stack, stackOff); + stackLen++; + } + // Process last partial chunk if any + if (lastChunkLen > 0) { + hashChunkWithWords(input, inputWords, fullChunks * CHUNK_LEN, lastChunkLen, fullChunks, 0, chunkCv, 0); + let totalChunks = fullChunks + 1; + let newCv = chunkCv; + let newCvOffset = 0; + while ((totalChunks & 1) === 0 && stackLen > 0) { + // Skip final merge; this IS the last chunk, let finalization handle ROOT flag + if (stackLen === 1) { + break; + } + stackLen--; + const stackOff = stackLen * 8; + // Copy from stack to parentBlock[0..7] (unrolled) + copyCV8(stack, stackOff, parentBlock, 0); + // Copy from newCv to parentBlock[8..15] (unrolled) + copyCV8(newCv, newCvOffset, parentBlock, 8); + compress(IV, 0, parentBlock, 0, parentCv, 0, false, 0, BLOCK_LEN, PARENT); + newCv = parentCv; + newCvOffset = 0; + totalChunks >>>= 1; + } + // Push CV to stack (unrolled) + const pushOff = stackLen * 8; + copyCV8(newCv, newCvOffset, stack, pushOff); + stackLen++; + } + } + else { + // Slow path for unaligned or big-endian + for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) { + const chunkStart = chunkIdx * CHUNK_LEN; + const chunkLen = Math.min(CHUNK_LEN, inputLen - chunkStart); + hashChunkWithWords(input, inputWords, chunkStart, chunkLen, chunkIdx, 0, chunkCv, 0); + // Merge completed subtrees + let totalChunks = chunkIdx + 1; + let newCv = chunkCv; + let newCvOffset = 0; + // Check if this is the last chunk + const isLastChunk = chunkIdx === numChunks - 1; + while ((totalChunks & 1) === 0 && stackLen > 0) { + // Skip final merge if it would produce the root; let finalization handle it with ROOT flag + if (stackLen === 1 && isLastChunk) { + break; + } + stackLen--; + const stackOff = stackLen * 8; + // Copy from stack to parentBlock[0..7] (unrolled) + copyCV8(stack, stackOff, parentBlock, 0); + // Copy from newCv to parentBlock[8..15] (unrolled) + copyCV8(newCv, newCvOffset, parentBlock, 8); + compress(IV, 0, parentBlock, 0, parentCv, 0, false, 0, BLOCK_LEN, PARENT); + newCv = parentCv; + newCvOffset = 0; + totalChunks >>>= 1; + } + // Push CV to stack (unrolled) + const pushOff = stackLen * 8; + copyCV8(newCv, newCvOffset, stack, pushOff); + stackLen++; + } + } + // Finalize: merge remaining stack entries + while (stackLen > 1) { + stackLen--; + const rightOff = stackLen * 8; + stackLen--; + const leftOff = stackLen * 8; + // Copy left CV to parentBlock[0..7] and right CV to parentBlock[8..15] (unrolled) + copyCV8(stack, leftOff, parentBlock, 0); + copyCV8(stack, rightOff, parentBlock, 8); + if (stackLen === 0) { + // This is the root - use reusable output buffer for common 32-byte case + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + compress(IV, 0, parentBlock, 0, out, 0, outputLen > 32, 0, BLOCK_LEN, PARENT | ROOT); + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } + else { + writeLittleEndianBytesPartial(out, 0, result, 0, outputLen); + } + return result; + } + compress(IV, 0, parentBlock, 0, parentCv, 0, false, 0, BLOCK_LEN, PARENT); + // Push to stack (unrolled) + copyCV8(parentCv, 0, stack, stackLen * 8); + stackLen++; + } + // Single entry in stack - this is the root + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + const lastBlock = getBlockWords(); + lastBlock.fill(0); + // Copy first 8 words from stack (unrolled) + copyCV8(stack, 0, lastBlock, 0); + compress(IV, 0, lastBlock, 0, out, 0, outputLen > 32, 0, BLOCK_LEN, ROOT); + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } + else { + writeLittleEndianBytesPartial(out, 0, result, 0, outputLen); + } + return result; +} +/** + * Hash a single chunk that is also the root (single chunk input). + */ +function hashChunkRoot(input, inputOffset, inputLen, chunkCounter, flags, out, fullOutput) { + // Use reusable tempCv (single-threaded safe) + reusableTempCv.set(IV); + const block = getBlockWords(); + // Process full blocks + const fullBlocks = inputLen >>> 6; + const remainder = inputLen & 63; + const totalBlocks = fullBlocks + (remainder > 0 ? 1 : 0) || 1; // At least 1 block + // Create a Uint32Array view if possible + let inputWords = null; + if (IS_LITTLE_ENDIAN && (input.byteOffset + inputOffset) % 4 === 0 && inputLen >= 4) { + inputWords = new Uint32Array(input.buffer, input.byteOffset + inputOffset, inputLen >>> 2); + } + for (let blockIdx = 0; blockIdx < totalBlocks; blockIdx++) { + const isFirst = blockIdx === 0; + const isLast = blockIdx === totalBlocks - 1; + const blockStart = blockIdx << 6; + const blockLen = isLast ? remainder || (inputLen > 0 ? BLOCK_LEN : 0) : BLOCK_LEN; + // Determine flags + let blockFlags = flags; + if (isFirst) + blockFlags |= CHUNK_START; + if (isLast) + blockFlags |= CHUNK_END | ROOT; + // Load block + if (isLast && remainder > 0) { + readLittleEndianWordsPartial(input, inputOffset + blockStart, blockLen, block); + } + else if (inputLen === 0) { + block.fill(0); + } + else if (inputWords && (blockStart >>> 2) + 16 <= inputWords.length) { + // Fast path + compress(reusableTempCv, 0, inputWords, blockStart >>> 2, isLast ? out : reusableTempCv, 0, isLast && fullOutput, chunkCounter, blockLen, blockFlags); + continue; + } + else { + readLittleEndianWordsFull(input, inputOffset + blockStart, block); + } + compress(reusableTempCv, 0, block, 0, isLast ? out : reusableTempCv, 0, isLast && fullOutput, chunkCounter, blockLen, blockFlags); + } +} +/** + * Hash using WASM SIMD - processes 4 chunks in parallel. + * Falls back to pure JS if SIMD fails. + */ +function hashSimd(input, outputLen) { + const mem = getSimdMemory(); + if (!mem) { + return hashPureJS(input, outputLen); + } + const { view32 } = mem; + const inputLen = input.length; + const numChunks = Math.ceil(inputLen / CHUNK_LEN); + // For small inputs, pure JS is faster (no transpose overhead) + if (numChunks < 4) { + return hashPureJS(input, outputLen); + } + // Try to use WASM arena buffers (zero JS heap allocation) + // Falls back to JS buffers if arena not available + const arena = getArenaBuffers(); + const useWasmParent = arena !== null; // Use WASM parent compress when arena available + let stack; + let tempCvs; + let parentBlock; + let parentCv; + if (arena) { + // Use WASM-backed arena buffers + stack = arena.cvStack; + tempCvs = arena.tempCvs; + parentBlock = arena.parentBlock; + parentCv = arena.chunkCv; + } + else { + // Fallback to JS heap buffers - use global contiguous stack (no allocation) + stack = HYPER_CV_STACK; + tempCvs = reusableSimdCvs; + parentBlock = reusableSimdParentBlock; + parentCv = reusableSimdParentCv; + } + let stackLen = 0; + // Use TypedArrays instead of JS arrays for block parameters + const offsets = reusableOffsets; + const counters = reusableCounters; + const blockLens = reusableBlockLens; + const flagsArr = reusableFlags; + // Create Uint32Array view once for entire hash call (optimization: avoid allocation in hot loop) + const inputWords = IS_LITTLE_ENDIAN && input.byteOffset % 4 === 0 + ? new Uint32Array(input.buffer, input.byteOffset, input.byteLength >>> 2) + : null; + // Calculate number of full chunks (1024 bytes each) + const numFullChunks = inputLen >>> 10; // inputLen / 1024 + // Process chunks in groups of 4 + let chunkIdx = 0; + while (chunkIdx < numChunks) { + const groupSize = Math.min(4, numChunks - chunkIdx); + // === BATCH FAST PATH: 4 full chunks === + // Use compressChunks4x for groups of exactly 4 full chunks + // This reduces 16 WASM calls to 1 per group + const canUseBatchPath = groupSize === 4 && chunkIdx + 4 <= numFullChunks; + if (canUseBatchPath) { + // Set up chunk offsets for batch transpose + batchChunkOffsets[0] = chunkIdx * CHUNK_LEN; + batchChunkOffsets[1] = (chunkIdx + 1) * CHUNK_LEN; + batchChunkOffsets[2] = (chunkIdx + 2) * CHUNK_LEN; + batchChunkOffsets[3] = (chunkIdx + 3) * CHUNK_LEN; + // Transpose all 64 blocks (4 chunks × 16 blocks) at once + transposeBatchToSimd(input, batchChunkOffsets, view32, inputWords); + // Set up initial CVs (IV) in batch memory - transposed layout + for (let w = 0; w < 8; w++) { + const ivWord = IV[w]; + const base = BATCH_CV_BASE + w * 4; + view32[base] = ivWord; + view32[base + 1] = ivWord; + view32[base + 2] = ivWord; + view32[base + 3] = ivWord; + } + // Set up counters in batch memory + view32[BATCH_COUNTER_LOW_BASE] = chunkIdx; + view32[BATCH_COUNTER_LOW_BASE + 1] = chunkIdx + 1; + view32[BATCH_COUNTER_LOW_BASE + 2] = chunkIdx + 2; + view32[BATCH_COUNTER_LOW_BASE + 3] = chunkIdx + 3; + // Set up base flags (0 - no keyed hashing) + view32[BATCH_FLAGS_BASE_OFFSET] = 0; + view32[BATCH_FLAGS_BASE_OFFSET + 1] = 0; + view32[BATCH_FLAGS_BASE_OFFSET + 2] = 0; + view32[BATCH_FLAGS_BASE_OFFSET + 3] = 0; + // Run batched compress (16 blocks × 4 chunks in one call!) + runCompressChunks4x(); + // Read output CVs from batch output - untranspose to tempCvs + for (let w = 0; w < 8; w++) { + const base = BATCH_OUTPUT_BASE + w * 4; + tempCvs[w] = view32[base]; // chunk 0 + tempCvs[8 + w] = view32[base + 1]; // chunk 1 + tempCvs[16 + w] = view32[base + 2]; // chunk 2 + tempCvs[24 + w] = view32[base + 3]; // chunk 3 + } + } + else { + // === STANDARD PATH: block-by-block processing === + // Used for partial chunks or groups < 4 + // Initialize CVs for this group to IV (flat array: 4 × 8 words) + for (let g = 0; g < groupSize; g++) { + const base = g * 8; + tempCvs[base] = IV[0]; + tempCvs[base + 1] = IV[1]; + tempCvs[base + 2] = IV[2]; + tempCvs[base + 3] = IV[3]; + tempCvs[base + 4] = IV[4]; + tempCvs[base + 5] = IV[5]; + tempCvs[base + 6] = IV[6]; + tempCvs[base + 7] = IV[7]; + } + // Process all 16 blocks of each chunk in this group + for (let blockIdx = 0; blockIdx < 16; blockIdx++) { + // Calculate block offsets and parameters (reuse arrays) + for (let g = 0; g < groupSize; g++) { + const thisChunkIdx = chunkIdx + g; + const chunkStart = thisChunkIdx * CHUNK_LEN; + const chunkLen = Math.min(CHUNK_LEN, inputLen - chunkStart); + const thisBlockStart = chunkStart + blockIdx * BLOCK_LEN; + // Determine block length for this specific block + const blockStartInChunk = blockIdx * BLOCK_LEN; + let thisBlockLen = BLOCK_LEN; + if (blockStartInChunk >= chunkLen) { + thisBlockLen = 0; + } + else if (blockStartInChunk + BLOCK_LEN > chunkLen) { + thisBlockLen = chunkLen - blockStartInChunk; + } + offsets[g] = thisBlockStart; + counters[g] = thisChunkIdx; + // Determine flags + let flags = 0; + if (blockIdx === 0) + flags |= CHUNK_START; + const totalBlocksInChunk = Math.ceil(chunkLen / BLOCK_LEN) || 1; + if (blockIdx === totalBlocksInChunk - 1) + flags |= CHUNK_END; + blockLens[g] = thisBlockLen; + flagsArr[g] = flags; + } + // Check if any blocks need processing + if (blockLens[0] === 0 && blockLens[1] === 0 && blockLens[2] === 0 && blockLens[3] === 0) + continue; + // Transpose blocks into SIMD memory (pass pre-created view to avoid allocation) + transposeBlocksToSimd(input, offsets, blockLens, view32, groupSize, inputWords); + // Set up CVs in SIMD memory + setupSimdCvs(tempCvs, view32, groupSize); + // Set up parameters + setupSimdParams(view32, counters, blockLens, flagsArr, groupSize); + // Run SIMD compress + runCompress4x(); + // Read output CVs back + readSimdOutputCvs(view32, simdChunkCvs, groupSize); + // Update tempCvs - copy from simdChunkCvs (both are flat 32-word arrays) + // simdChunkCvs layout matches tempCvs: [cv0_w0..cv0_w7, cv1_w0..cv1_w7, ...] + // IMPORTANT: Only update CVs for chunks that had data in this block! + // Skipping this check would corrupt CVs for partial chunks after their final block. + for (let g = 0; g < groupSize; g++) { + if (blockLens[g] === 0) + continue; // Don't update CV for chunks with no data in this block + const base = g * 8; + tempCvs[base] = simdChunkCvs[base]; + tempCvs[base + 1] = simdChunkCvs[base + 1]; + tempCvs[base + 2] = simdChunkCvs[base + 2]; + tempCvs[base + 3] = simdChunkCvs[base + 3]; + tempCvs[base + 4] = simdChunkCvs[base + 4]; + tempCvs[base + 5] = simdChunkCvs[base + 5]; + tempCvs[base + 6] = simdChunkCvs[base + 6]; + tempCvs[base + 7] = simdChunkCvs[base + 7]; + } + } + } + // Merge each chunk's CV into the Merkle tree + for (let g = 0; g < groupSize; g++) { + const thisChunkIdx = chunkIdx + g; + // Merge completed subtrees + let totalChunks = thisChunkIdx + 1; + // Track newCv source - either from tempCvs or parentCv + let newCvBase = g * 8; // Offset into tempCvs + let newCvSrc = tempCvs; + // Check if this is the last chunk + const isLastChunk = thisChunkIdx === numChunks - 1; + while ((totalChunks & 1) === 0 && stackLen > 0) { + // Skip final merge if it would produce the root; let finalization handle it with ROOT flag + if (stackLen === 1 && isLastChunk) { + break; + } + // Pop left child + stackLen--; + const stackOff = stackLen * 8; + // Copy from stack to parentBlock[0..7] (unrolled) + copyCV8(stack, stackOff, parentBlock, 0); + // Copy from newCv source to parentBlock[8..15] (unrolled) + copyCV8(newCvSrc, newCvBase, parentBlock, 8); + if (useWasmParent) { + // WASM parent compress - data already in arena buffers + runCompressParent(); + } + else { + compress(IV, 0, parentBlock, 0, parentCv, 0, false, 0, BLOCK_LEN, PARENT); + } + newCvSrc = parentCv; + newCvBase = 0; + totalChunks >>>= 1; + } + // Push to stack (unrolled) + const pushOff = stackLen * 8; + copyCV8(newCvSrc, newCvBase, stack, pushOff); + stackLen++; + } + chunkIdx += groupSize; + } + // Finalize: merge remaining stack entries + while (stackLen > 1) { + stackLen--; + const rightOff = stackLen * 8; + stackLen--; + const leftOff = stackLen * 8; + // Copy left CV to parentBlock[0..7] and right CV to parentBlock[8..15] (unrolled) + copyCV8(stack, leftOff, parentBlock, 0); + copyCV8(stack, rightOff, parentBlock, 8); + if (stackLen === 0) { + // This is the root - use reusable output buffer + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + compress(IV, 0, parentBlock, 0, out, 0, outputLen > 32, 0, BLOCK_LEN, PARENT | ROOT); + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } + else { + writeLittleEndianBytesPartial(out, 0, result, 0, outputLen); + } + return result; + } + if (useWasmParent) { + // WASM parent compress - data already in arena buffers + runCompressParent(); + } + else { + compress(IV, 0, parentBlock, 0, parentCv, 0, false, 0, BLOCK_LEN, PARENT); + } + // Push to stack (unrolled) + copyCV8(parentCv, 0, stack, stackLen * 8); + stackLen++; + } + // Single entry in stack - finalize as root + if (stackLen === 1) { + const block = getBlockWords(); + block.fill(0); + // Copy first 8 words from stack (unrolled) + copyCV8(stack, 0, block, 0); + // Use reusable output buffer + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + compress(IV, 0, block, 0, out, 0, outputLen > 32, 0, BLOCK_LEN, ROOT); + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } + else { + writeLittleEndianBytesPartial(out, 0, result, 0, outputLen); + } + return result; + } + // Should not reach here + return hashPureJS(input, outputLen); +} +/** + * Hash input data and return the result. + * Automatically uses WASM SIMD for large inputs when available. + * + * @param input - Data to hash + * @param outputLength - Number of bytes to output (default: 32) + * @returns The hash output + */ +export function hash(input, outputLength = OUT_LEN) { + // For large inputs, use SIMD for ~1.5x performance improvement + if (input.length >= SIMD_THRESHOLD && ensureSimdSync()) { + return hashSimd(input, outputLength); + } + return hashPureJS(input, outputLength); +} +/** + * Pre-warm SIMD initialization (call early to avoid latency later). + */ +export function warmupSimd() { + return ensureSimdSync(); +} +/** + * Hash input data directly into a caller-provided output buffer. + * Zero-allocation for the common 32-byte case - ideal for performance-critical code. + * + * @param input - Data to hash + * @param output - Pre-allocated output buffer (must be at least outputLength bytes) + * @param outputLength - Number of bytes to output (default: 32, max: output.length) + */ +export function hashInto(input, output, outputLength = OUT_LEN) { + // Validate output buffer + if (output.length < outputLength) { + throw new Error(`Output buffer too small: ${output.length} < ${outputLength}`); + } + // For large inputs, use SIMD for ~1.5x performance improvement + if (input.length >= SIMD_THRESHOLD && ensureSimdSync()) { + hashSimdInto(input, output, outputLength); + return; + } + hashPureJSInto(input, output, outputLength); +} +/** + * Internal: Hash using pure JS, writing directly to output buffer. + */ +function hashPureJSInto(input, output, outputLen) { + const inputLen = input.length; + // Special case: empty input + if (inputLen === 0) { + const block = getBlockWords(); + block.fill(0); + const out = outputLen <= 32 ? reusableOut8 : new Uint32Array(16); + compress(IV, 0, block, 0, out, 0, outputLen > 32, 0, 0, CHUNK_START | CHUNK_END | ROOT); + // Copy result to output + if (IS_LITTLE_ENDIAN) { + output.set(new Uint8Array(out.buffer, out.byteOffset, outputLen)); + } + else { + writeLittleEndianBytesPartial(out, 0, output, 0, outputLen); + } + return; + } + // Calculate number of chunks + const numChunks = Math.ceil(inputLen / CHUNK_LEN); + // Single chunk optimization + if (numChunks === 1) { + const cv = outputLen <= 32 ? reusableOut8 : new Uint32Array(16); + hashChunkRoot(input, 0, inputLen, 0, 0, cv, outputLen > 32); + // Copy result to output + if (IS_LITTLE_ENDIAN) { + output.set(new Uint8Array(cv.buffer, cv.byteOffset, outputLen)); + } + else { + writeLittleEndianBytesPartial(cv, 0, output, 0, outputLen); + } + return; + } + // Multiple chunks - delegate to hashPureJS and copy result + const result = hashPureJS(input, outputLen); + output.set(result); +} +/** + * Internal: Hash using SIMD, writing directly to output buffer. + */ +function hashSimdInto(input, output, outputLen) { + // Delegate to hashSimd and copy result (SIMD path already optimized) + const result = hashSimd(input, outputLen); + output.set(result); +} diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/hasher.d.ts b/node_modules/@huggingface/blake3-jit/dist/esm/hasher.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..37ecf0d590f939d37d0bd762df953c358548fcdc --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/hasher.d.ts @@ -0,0 +1,108 @@ +/** + * BLAKE3 Hasher - Incremental hashing with support for all modes + * + * Supports: + * - Regular hashing + * - Keyed hashing (MAC) + * - Key derivation (derive_key) + * - XOF (eXtendable Output Function) mode + */ +/** + * Output state for XOF (eXtendable Output Function) mode. + * Allows reading arbitrary amounts of output. + */ +export declare class XofReader { + private inputCv; + private blockWords; + private counter; + private blockLen; + private flags; + private outputBlock; + private outputBlockOffset; + constructor(inputCv: Uint32Array, blockWords: Uint32Array, counter: number, blockLen: number, flags: number); + /** + * Read the next `length` bytes of output. + */ + read(length: number): Uint8Array; +} +/** + * Main BLAKE3 Hasher class. + * + * Usage: + * const hasher = new Hasher(); + * hasher.update(data); + * const hash = hasher.finalize(); + * + * Or with chaining: + * const hash = new Hasher().update(data).finalize(); + */ +export declare class Hasher { + private chunkState; + private keyWords; + private cvStack; + private cvStackLen; + private flags; + private parentBlock; + private parentCv; + private chunkCv; + private outWords; + private finalizeCv; + /** + * Create a new Hasher. + * + * @param keyWords - Initial key words (IV for regular hashing) + * @param flags - Domain separation flags + */ + constructor(keyWords?: Uint32Array, flags?: number); + /** + * Reset the hasher to process a new message with the same key/flags. + * Reuses all internal buffers — zero allocations. + */ + reset(): this; + /** + * Create a new keyed hasher (MAC). + * + * @param key - 32-byte key + */ + static newKeyed(key: Uint8Array): Hasher; + /** + * Create a new key derivation hasher. + * + * @param context - Context string for domain separation + */ + static newDeriveKey(context: string): Hasher; + /** + * Push a chaining value onto the stack. + */ + private pushCv; + /** + * Pop a chaining value from the stack. + */ + private popCv; + /** + * Add a chunk's chaining value and merge completed subtrees. + */ + private addChunkCv; + /** + * Update the hasher with input data. + * + * @param input - Data to hash + * @returns this (for chaining) + */ + update(input: Uint8Array): this; + /** + * Get the output parameters (for XOF mode or finalization). + */ + private finalizeOutput; + /** + * Finalize the hash and return the result. + * + * @param outputLength - Number of bytes to output (default: 32) + * @returns The hash output + */ + finalize(outputLength?: number): Uint8Array; + /** + * Finalize and return an XOF reader for arbitrary-length output. + */ + finalizeXof(): XofReader; +} diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/hasher.js b/node_modules/@huggingface/blake3-jit/dist/esm/hasher.js new file mode 100644 index 0000000000000000000000000000000000000000..ee3616e1d617dfabb30f8dd1f38f9d18800cdcf1 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/hasher.js @@ -0,0 +1,391 @@ +/** + * BLAKE3 Hasher - Incremental hashing with support for all modes + * + * Supports: + * - Regular hashing + * - Keyed hashing (MAC) + * - Key derivation (derive_key) + * - XOF (eXtendable Output Function) mode + */ +import { compress } from "./compress.js"; +import { IV, CHUNK_START, CHUNK_END, PARENT, ROOT, KEYED_HASH, DERIVE_KEY_CONTEXT, DERIVE_KEY_MATERIAL, BLOCK_LEN, CHUNK_LEN, OUT_LEN, KEY_LEN, MAX_DEPTH, } from "./constants.js"; +import { IS_LITTLE_ENDIAN, readLittleEndianWordsFull, writeLittleEndianBytesPartial, encodeUTF8, } from "./utils.js"; +/** + * Output state for XOF (eXtendable Output Function) mode. + * Allows reading arbitrary amounts of output. + */ +export class XofReader { + inputCv; + blockWords; + counter; + blockLen; + flags; + outputBlock; + outputBlockOffset; + constructor(inputCv, blockWords, counter, blockLen, flags) { + this.inputCv = inputCv; + this.blockWords = blockWords; + this.counter = counter; + this.blockLen = blockLen; + this.flags = flags | ROOT; + this.outputBlock = new Uint32Array(16); + this.outputBlockOffset = 64; // Forces generation on first read + } + /** + * Read the next `length` bytes of output. + */ + read(length) { + const output = new Uint8Array(length); + let outputOffset = 0; + while (outputOffset < length) { + // Generate new output block if needed + if (this.outputBlockOffset >= 64) { + compress(this.inputCv, 0, this.blockWords, 0, this.outputBlock, 0, true, // full 64-byte output + this.counter++, this.blockLen, this.flags); + this.outputBlockOffset = 0; + } + // Copy bytes from output block + const available = 64 - this.outputBlockOffset; + const toCopy = Math.min(available, length - outputOffset); + // Optimized copy using writeLittleEndianBytesPartial + const wordOffset = this.outputBlockOffset >>> 2; + const byteWithinWord = this.outputBlockOffset & 3; + if (byteWithinWord === 0 && toCopy >= 4) { + // Aligned copy - can use word-at-a-time + const fullWords = toCopy >>> 2; + writeLittleEndianBytesPartial(this.outputBlock, wordOffset, output, outputOffset, fullWords << 2); + const bytesCopied = fullWords << 2; + outputOffset += bytesCopied; + this.outputBlockOffset += bytesCopied; + } + else { + // Byte-by-byte for unaligned access + for (let i = 0; i < toCopy; i++) { + const wordIdx = (this.outputBlockOffset + i) >>> 2; + const byteIdx = (this.outputBlockOffset + i) & 3; + output[outputOffset + i] = (this.outputBlock[wordIdx] >>> (byteIdx << 3)) & 0xff; + } + outputOffset += toCopy; + this.outputBlockOffset += toCopy; + } + } + return output; + } +} +/** + * Chunk state for processing input data. + * Each chunk is 1024 bytes and produces an 8-word chaining value. + */ +class ChunkState { + chainingValue; + chunkCounter; + blockWords; + blockLen; + blocksCompressed; + flags; + constructor(keyWords, chunkCounter, flags) { + this.chainingValue = new Uint32Array(keyWords); + this.chunkCounter = chunkCounter; + this.blockWords = new Uint32Array(16); + this.blockLen = 0; + this.blocksCompressed = 0; + this.flags = flags; + } + resetTo(keyWords, chunkCounter, flags) { + this.chainingValue.set(keyWords); + this.chunkCounter = chunkCounter; + this.blockLen = 0; + this.blocksCompressed = 0; + this.flags = flags; + } + /** + * Get the flags for the current block. + */ + startFlag() { + return this.blocksCompressed === 0 ? CHUNK_START : 0; + } + /** + * Update the chunk state with input data. + * Returns the number of bytes consumed. + */ + update(input, inputOffset, inputLen) { + let consumed = 0; + while (inputLen > 0) { + // If we have a full block, compress it + if (this.blockLen === BLOCK_LEN) { + compress(this.chainingValue, 0, this.blockWords, 0, this.chainingValue, 0, false, this.chunkCounter, BLOCK_LEN, this.flags | this.startFlag()); + this.blocksCompressed++; + this.blockLen = 0; + } + // Fill the block buffer + const want = BLOCK_LEN - this.blockLen; + const take = Math.min(want, inputLen); + if (this.blockLen === 0 && take === BLOCK_LEN) { + readLittleEndianWordsFull(input, inputOffset, this.blockWords); + } + else { + // Partial block - byte-by-byte into correct position + for (let i = 0; i < take; i++) { + const pos = this.blockLen + i; + const wordIdx = pos >>> 2; + const byteIdx = pos & 3; + if (byteIdx === 0) { + this.blockWords[wordIdx] = input[inputOffset + i]; + } + else { + this.blockWords[wordIdx] |= input[inputOffset + i] << (byteIdx << 3); + } + } + } + this.blockLen += take; + inputOffset += take; + inputLen -= take; + consumed += take; + } + return consumed; + } + /** + * Finalize this chunk and return its output. + * Returns 8 words (chaining value) or 16 words (if root). + */ + output() { + // Zero-pad unused words in blockWords to avoid stale data from previous blocks + // This is necessary when a partial block follows a full block within the same chunk + const usedWords = (this.blockLen + 3) >>> 2; // ceil(blockLen / 4) + for (let i = usedWords; i < 16; i++) { + this.blockWords[i] = 0; + } + return { + inputCv: this.chainingValue, + blockWords: this.blockWords, + blockLen: this.blockLen, + counter: this.chunkCounter, + flags: this.flags | this.startFlag() | CHUNK_END, + }; + } + /** + * Get the number of bytes in this chunk. + */ + len() { + return this.blocksCompressed * BLOCK_LEN + this.blockLen; + } +} +/** + * Main BLAKE3 Hasher class. + * + * Usage: + * const hasher = new Hasher(); + * hasher.update(data); + * const hash = hasher.finalize(); + * + * Or with chaining: + * const hash = new Hasher().update(data).finalize(); + */ +export class Hasher { + chunkState; + keyWords; + cvStack; + cvStackLen; + flags; + parentBlock; + parentCv; + chunkCv; + outWords; + finalizeCv; + /** + * Create a new Hasher. + * + * @param keyWords - Initial key words (IV for regular hashing) + * @param flags - Domain separation flags + */ + constructor(keyWords, flags) { + this.keyWords = keyWords ? new Uint32Array(keyWords) : new Uint32Array(IV); + this.flags = flags ?? 0; + this.chunkState = new ChunkState(this.keyWords, 0, this.flags); + this.cvStack = new Uint32Array(MAX_DEPTH * 8); + this.cvStackLen = 0; + this.parentBlock = new Uint32Array(16); + this.parentCv = new Uint32Array(8); + this.chunkCv = new Uint32Array(8); + this.outWords = new Uint32Array(16); + this.finalizeCv = new Uint32Array(8); + } + /** + * Reset the hasher to process a new message with the same key/flags. + * Reuses all internal buffers — zero allocations. + */ + reset() { + this.chunkState.resetTo(this.keyWords, 0, this.flags); + this.cvStackLen = 0; + return this; + } + /** + * Create a new keyed hasher (MAC). + * + * @param key - 32-byte key + */ + static newKeyed(key) { + if (key.length !== KEY_LEN) { + throw new Error(`Key must be ${KEY_LEN} bytes, got ${key.length}`); + } + const keyWords = new Uint32Array(8); + if (IS_LITTLE_ENDIAN) { + const view = new Uint32Array(key.buffer, key.byteOffset, 8); + keyWords.set(view); + } + else { + for (let i = 0; i < 8; i++) { + const off = i * 4; + keyWords[i] = key[off] | (key[off + 1] << 8) | (key[off + 2] << 16) | (key[off + 3] << 24); + } + } + return new Hasher(keyWords, KEYED_HASH); + } + /** + * Create a new key derivation hasher. + * + * @param context - Context string for domain separation + */ + static newDeriveKey(context) { + // First, hash the context string with DERIVE_KEY_CONTEXT flag + const contextBytes = encodeUTF8(context); + const contextHasher = new Hasher(new Uint32Array(IV), DERIVE_KEY_CONTEXT); + contextHasher.update(contextBytes); + // Get the context key + const contextKey = new Uint32Array(8); + const output = contextHasher.finalizeOutput(); + compress(output.inputCv, 0, output.blockWords, 0, contextKey, 0, false, output.counter, output.blockLen, output.flags | ROOT); + // Return a hasher initialized with the context key + return new Hasher(contextKey, DERIVE_KEY_MATERIAL); + } + /** + * Push a chaining value onto the stack. + */ + pushCv(cv, cvOffset) { + this.cvStack.set(cv.subarray(cvOffset, cvOffset + 8), this.cvStackLen * 8); + this.cvStackLen++; + } + /** + * Pop a chaining value from the stack. + */ + popCv(out, outOffset) { + this.cvStackLen--; + out.set(this.cvStack.subarray(this.cvStackLen * 8, (this.cvStackLen + 1) * 8), outOffset); + } + /** + * Add a chunk's chaining value and merge completed subtrees. + */ + addChunkCv(newCv, newCvOffset, totalChunks) { + const parentBlock = this.parentBlock; + const parentCv = this.parentCv; + while ((totalChunks & 1) === 0) { + // Pop left child, new CV is right child + this.popCv(parentBlock, 0); + parentBlock.set(newCv.subarray(newCvOffset, newCvOffset + 8), 8); + compress(this.keyWords, 0, parentBlock, 0, parentCv, 0, false, 0, BLOCK_LEN, this.flags | PARENT); + newCv = parentCv; + newCvOffset = 0; + totalChunks >>>= 1; + } + this.pushCv(newCv, newCvOffset); + } + /** + * Update the hasher with input data. + * + * @param input - Data to hash + * @returns this (for chaining) + */ + update(input) { + let inputOffset = 0; + let inputLen = input.length; + // Fill the current chunk + while (inputLen > 0) { + // If current chunk is full, finalize it and start a new one + if (this.chunkState.len() === CHUNK_LEN) { + const output = this.chunkState.output(); + const chunkCv = this.chunkCv; + compress(output.inputCv, 0, output.blockWords, 0, chunkCv, 0, false, output.counter, output.blockLen, output.flags); + const totalChunks = this.chunkState.chunkCounter + 1; + this.addChunkCv(chunkCv, 0, totalChunks); + this.chunkState.resetTo(this.keyWords, totalChunks, this.flags); + } + // Fill the current chunk + const want = CHUNK_LEN - this.chunkState.len(); + const take = Math.min(want, inputLen); + this.chunkState.update(input, inputOffset, take); + inputOffset += take; + inputLen -= take; + } + return this; + } + /** + * Get the output parameters (for XOF mode or finalization). + */ + finalizeOutput() { + let output = this.chunkState.output(); + let parentBlock = this.parentBlock; + let cv = this.finalizeCv; + // If there are chunks on the stack, merge them + if (this.cvStackLen > 0) { + // First compress the current chunk + compress(output.inputCv, 0, output.blockWords, 0, cv, 0, false, output.counter, output.blockLen, output.flags); + // Merge with parent nodes from stack + while (this.cvStackLen > 0) { + this.cvStackLen--; + parentBlock.set(this.cvStack.subarray(this.cvStackLen * 8, (this.cvStackLen + 1) * 8), 0); + parentBlock.set(cv, 8); + if (this.cvStackLen > 0) { + compress(this.keyWords, 0, parentBlock, 0, cv, 0, false, 0, BLOCK_LEN, this.flags | PARENT); + } + else { + // This is the root - return output params + return { + inputCv: this.keyWords, + blockWords: parentBlock, + blockLen: BLOCK_LEN, + counter: 0, + flags: this.flags | PARENT, + }; + } + } + } + // Single chunk case + return output; + } + /** + * Finalize the hash and return the result. + * + * @param outputLength - Number of bytes to output (default: 32) + * @returns The hash output + */ + finalize(outputLength = OUT_LEN) { + const output = this.finalizeOutput(); + const result = new Uint8Array(outputLength); + if (outputLength <= 64) { + const outWords = this.outWords; + compress(output.inputCv, 0, output.blockWords, 0, outWords, 0, outputLength > 32, // full output if > 32 bytes + output.counter, output.blockLen, output.flags | ROOT); + if (IS_LITTLE_ENDIAN) { + const outBytes = new Uint8Array(outWords.buffer); + result.set(outBytes.subarray(0, outputLength)); + } + else { + writeLittleEndianBytesPartial(outWords, 0, result, 0, outputLength); + } + } + else { + // Multiple blocks - use XOF + const xof = this.finalizeXof(); + const full = xof.read(outputLength); + result.set(full); + } + return result; + } + /** + * Finalize and return an XOF reader for arbitrary-length output. + */ + finalizeXof() { + const output = this.finalizeOutput(); + return new XofReader(new Uint32Array(output.inputCv), new Uint32Array(output.blockWords), output.counter, output.blockLen, output.flags); + } +} diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/index.d.ts b/node_modules/@huggingface/blake3-jit/dist/esm/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..412f7b619154f7868d110617bfc8863e99c46c2c --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/index.d.ts @@ -0,0 +1,82 @@ +/** + * BLAKE3 - The fastest pure JavaScript implementation + * + * Features: + * - All 3 modes: hash, keyed (MAC), derive_key + * - XOF (eXtendable Output Function) support + * - Automatic WASM SIMD acceleration for large inputs + * - Zero dependencies + * - Tree-shakeable exports + * + * @example + * ```typescript + * import { hash, createKeyed, createDeriveKey } from 'blake3-jit'; + * + * // Simple hashing + * const digest = hash(new Uint8Array([1, 2, 3])); + * + * // Keyed hashing (MAC) + * const mac = createKeyed(key).update(data).finalize(); + * + * // Key derivation + * const derived = createDeriveKey("my context").update(material).finalize(64); + * ``` + */ +export { Hasher, XofReader } from "./hasher.js"; +export { hash, hashInto, warmupSimd } from "./hash.js"; +import { Hasher } from "./hasher.js"; +/** + * Create a new keyed hasher (MAC). + * + * @param key - 32-byte key + * @returns A new Hasher configured for keyed hashing + * + * @example + * ```typescript + * const key = new Uint8Array(32); // Your 32-byte key + * crypto.getRandomValues(key); + * + * const mac = createKeyed(key) + * .update(message) + * .finalize(); + * ``` + */ +export declare function createKeyed(key: Uint8Array): Hasher; +/** + * Create a new key derivation hasher. + * + * @param context - Context string for domain separation + * @returns A new Hasher configured for key derivation + * + * @example + * ```typescript + * const derivedKey = createDeriveKey("my-app encryption key v1") + * .update(inputKeyMaterial) + * .finalize(32); + * ``` + */ +export declare function createDeriveKey(context: string): Hasher; +/** + * Create a new regular hasher for incremental hashing. + * + * @returns A new Hasher + * + * @example + * ```typescript + * const hasher = createHasher(); + * hasher.update(chunk1); + * hasher.update(chunk2); + * const digest = hasher.finalize(); + * ``` + */ +export declare function createHasher(): Hasher; +import { hash, hashInto } from "./hash.js"; +declare const _default: { + hash: typeof hash; + hashInto: typeof hashInto; + Hasher: typeof Hasher; + createHasher: typeof createHasher; + createKeyed: typeof createKeyed; + createDeriveKey: typeof createDeriveKey; +}; +export default _default; diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/index.js b/node_modules/@huggingface/blake3-jit/dist/esm/index.js new file mode 100644 index 0000000000000000000000000000000000000000..da5da7d7f08f16c65bea9d5add4855804a120ebd --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/index.js @@ -0,0 +1,98 @@ +/** + * BLAKE3 - The fastest pure JavaScript implementation + * + * Features: + * - All 3 modes: hash, keyed (MAC), derive_key + * - XOF (eXtendable Output Function) support + * - Automatic WASM SIMD acceleration for large inputs + * - Zero dependencies + * - Tree-shakeable exports + * + * @example + * ```typescript + * import { hash, createKeyed, createDeriveKey } from 'blake3-jit'; + * + * // Simple hashing + * const digest = hash(new Uint8Array([1, 2, 3])); + * + * // Keyed hashing (MAC) + * const mac = createKeyed(key).update(data).finalize(); + * + * // Key derivation + * const derived = createDeriveKey("my context").update(material).finalize(64); + * ``` + */ +// Core exports +export { Hasher, XofReader } from "./hasher.js"; +export { hash, hashInto, warmupSimd } from "./hash.js"; +// Convenience imports +import { Hasher } from "./hasher.js"; +/** + * Create a new keyed hasher (MAC). + * + * @param key - 32-byte key + * @returns A new Hasher configured for keyed hashing + * + * @example + * ```typescript + * const key = new Uint8Array(32); // Your 32-byte key + * crypto.getRandomValues(key); + * + * const mac = createKeyed(key) + * .update(message) + * .finalize(); + * ``` + */ +export function createKeyed(key) { + return Hasher.newKeyed(key); +} +/** + * Create a new key derivation hasher. + * + * @param context - Context string for domain separation + * @returns A new Hasher configured for key derivation + * + * @example + * ```typescript + * const derivedKey = createDeriveKey("my-app encryption key v1") + * .update(inputKeyMaterial) + * .finalize(32); + * ``` + */ +export function createDeriveKey(context) { + return Hasher.newDeriveKey(context); +} +/** + * Create a new regular hasher for incremental hashing. + * + * @returns A new Hasher + * + * @example + * ```typescript + * const hasher = createHasher(); + * hasher.update(chunk1); + * hasher.update(chunk2); + * const digest = hasher.finalize(); + * ``` + */ +export function createHasher() { + return new Hasher(); +} +// Import for default export +import { hash, hashInto, warmupSimd } from "./hash.js"; +// Pre-warm SIMD in browser environments (non-blocking) +// This avoids initialization latency on first large hash +if (typeof globalThis !== "undefined" && typeof globalThis.document !== "undefined") { + queueMicrotask(() => { + warmupSimd(); + }); +} +// Default export for convenience +export default { + hash, + hashInto, + Hasher, + createHasher, + createKeyed, + createDeriveKey, +}; diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/package.json b/node_modules/@huggingface/blake3-jit/dist/esm/package.json new file mode 100644 index 0000000000000000000000000000000000000000..3dbc1ca591c0557e35b6004aeba250e6a70b56e3 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/utils.d.ts b/node_modules/@huggingface/blake3-jit/dist/esm/utils.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..07a9a42794db7fc4875250155fcb055c7bad954d --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/utils.d.ts @@ -0,0 +1,90 @@ +/** + * BLAKE3 Utility Functions + * + * Optimized for little-endian systems (most user-facing systems). + * BLAKE3 is little-endian friendly - on little-endian systems we can + * create Uint32Array views directly over input buffers. + */ +/** + * Detect system endianness at module load time. + * On little-endian systems, the byte 0x01 will be at index 0. + */ +export declare const IS_LITTLE_ENDIAN: boolean; +/** + * Read 16 little-endian 32-bit words from a byte array into a Uint32Array. + * This is only needed on big-endian systems. + * + * @param input - Source byte array + * @param offset - Starting byte offset in input + * @param words - Destination Uint32Array (must have at least 16 elements) + */ +export declare function readLittleEndianWordsFull(input: Uint8Array, offset: number, words: Uint32Array): void; +/** + * Read N little-endian 32-bit words from a byte array. + * Handles partial reads (for final blocks). + * + * @param input - Source byte array + * @param offset - Starting byte offset + * @param words - Destination Uint32Array + * @param wordCount - Number of words to read + */ +export declare function readLittleEndianWords(input: Uint8Array, offset: number, words: Uint32Array, wordCount: number): void; +/** + * Read a partial block with zero padding. + * Used for the final block when input length is not a multiple of 64. + * + * @param input - Source byte array + * @param offset - Starting byte offset + * @param length - Number of bytes to read (< 64) + * @param words - Destination Uint32Array (must have 16 elements) + */ +export declare function readLittleEndianWordsPartial(input: Uint8Array, offset: number, length: number, words: Uint32Array): void; +/** + * Write 8 little-endian 32-bit words to a byte array. + * + * @param words - Source Uint32Array + * @param wordOffset - Starting word offset in source + * @param output - Destination byte array + * @param byteOffset - Starting byte offset in destination + */ +export declare function writeLittleEndianWords(words: Uint32Array, wordOffset: number, output: Uint8Array, byteOffset: number): void; +/** + * Write N bytes from 32-bit words to output. + * Used for variable-length output (XOF mode). + * + * @param words - Source Uint32Array + * @param wordOffset - Starting word offset + * @param output - Destination byte array + * @param byteOffset - Starting byte offset in destination + * @param byteCount - Number of bytes to write + */ +export declare function writeLittleEndianBytesPartial(words: Uint32Array, wordOffset: number, output: Uint8Array, byteOffset: number, byteCount: number): void; +/** + * Encode a UTF-8 string to Uint8Array. + * Used for derive_key context strings. + */ +export declare function encodeUTF8(str: string): Uint8Array; +/** + * Count trailing zero bits in a 32-bit number using De Bruijn multiplication. + * This is O(1) and branchless for non-zero inputs. + * + * For Merkle tree merge: ctz32(chunkCounter) tells us how many merges to do. + */ +export declare function ctz32(n: number): number; +/** + * Count trailing zero bits in a 64-bit number. + * Used to determine how many parent nodes to compute after adding a chunk. + * + * Note: JavaScript bitwise ops work on 32-bit signed integers, + * so we need to handle 64-bit numbers carefully. + */ +export declare function countTrailingZeros(n: number): number; +/** + * Create a Uint32Array view of a Uint8Array. + * Only works correctly on little-endian systems when the offset is 4-byte aligned. + * + * @param arr - Source byte array + * @param byteOffset - Starting byte offset (must be 4-byte aligned) + * @param wordLength - Number of 32-bit words + */ +export declare function uint32View(arr: Uint8Array, byteOffset: number, wordLength: number): Uint32Array; diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/utils.js b/node_modules/@huggingface/blake3-jit/dist/esm/utils.js new file mode 100644 index 0000000000000000000000000000000000000000..aebb07e4b919e634b1dce5facf2f47785a27afed --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/utils.js @@ -0,0 +1,214 @@ +/** + * BLAKE3 Utility Functions + * + * Optimized for little-endian systems (most user-facing systems). + * BLAKE3 is little-endian friendly - on little-endian systems we can + * create Uint32Array views directly over input buffers. + */ +/** + * Detect system endianness at module load time. + * On little-endian systems, the byte 0x01 will be at index 0. + */ +export const IS_LITTLE_ENDIAN = new Uint8Array(new Uint32Array([0x01020304]).buffer)[0] === 0x04; +/** + * Read 16 little-endian 32-bit words from a byte array into a Uint32Array. + * This is only needed on big-endian systems. + * + * @param input - Source byte array + * @param offset - Starting byte offset in input + * @param words - Destination Uint32Array (must have at least 16 elements) + */ +export function readLittleEndianWordsFull(input, offset, words) { + for (let i = 0; i < 16; ++i, offset += 4) { + words[i] = + input[offset] | + (input[offset + 1] << 8) | + (input[offset + 2] << 16) | + (input[offset + 3] << 24); + } +} +/** + * Read N little-endian 32-bit words from a byte array. + * Handles partial reads (for final blocks). + * + * @param input - Source byte array + * @param offset - Starting byte offset + * @param words - Destination Uint32Array + * @param wordCount - Number of words to read + */ +export function readLittleEndianWords(input, offset, words, wordCount) { + for (let i = 0; i < wordCount; ++i, offset += 4) { + words[i] = + input[offset] | + (input[offset + 1] << 8) | + (input[offset + 2] << 16) | + (input[offset + 3] << 24); + } +} +/** + * Read a partial block with zero padding. + * Used for the final block when input length is not a multiple of 64. + * + * @param input - Source byte array + * @param offset - Starting byte offset + * @param length - Number of bytes to read (< 64) + * @param words - Destination Uint32Array (must have 16 elements) + */ +export function readLittleEndianWordsPartial(input, offset, length, words) { + // Zero out all words first + words.fill(0); + // Read full words + const fullWords = length >>> 2; + let i = 0; + for (; i < fullWords; ++i, offset += 4) { + words[i] = + input[offset] | + (input[offset + 1] << 8) | + (input[offset + 2] << 16) | + (input[offset + 3] << 24); + } + // Handle remaining bytes (0-3) + const remaining = length & 3; + if (remaining > 0) { + let word = input[offset]; + if (remaining > 1) + word |= input[offset + 1] << 8; + if (remaining > 2) + word |= input[offset + 2] << 16; + words[i] = word; + } +} +/** + * Write 8 little-endian 32-bit words to a byte array. + * + * @param words - Source Uint32Array + * @param wordOffset - Starting word offset in source + * @param output - Destination byte array + * @param byteOffset - Starting byte offset in destination + */ +export function writeLittleEndianWords(words, wordOffset, output, byteOffset) { + for (let i = 0; i < 8; ++i, byteOffset += 4) { + const w = words[wordOffset + i]; + output[byteOffset] = w & 0xff; + output[byteOffset + 1] = (w >>> 8) & 0xff; + output[byteOffset + 2] = (w >>> 16) & 0xff; + output[byteOffset + 3] = (w >>> 24) & 0xff; + } +} +/** + * Write N bytes from 32-bit words to output. + * Used for variable-length output (XOF mode). + * + * @param words - Source Uint32Array + * @param wordOffset - Starting word offset + * @param output - Destination byte array + * @param byteOffset - Starting byte offset in destination + * @param byteCount - Number of bytes to write + */ +export function writeLittleEndianBytesPartial(words, wordOffset, output, byteOffset, byteCount) { + const fullWords = byteCount >>> 2; + let i = 0; + // Write full words + for (; i < fullWords; ++i, byteOffset += 4) { + const w = words[wordOffset + i]; + output[byteOffset] = w & 0xff; + output[byteOffset + 1] = (w >>> 8) & 0xff; + output[byteOffset + 2] = (w >>> 16) & 0xff; + output[byteOffset + 3] = (w >>> 24) & 0xff; + } + // Write remaining bytes + const remaining = byteCount & 3; + if (remaining > 0) { + const w = words[wordOffset + i]; + output[byteOffset] = w & 0xff; + if (remaining > 1) + output[byteOffset + 1] = (w >>> 8) & 0xff; + if (remaining > 2) + output[byteOffset + 2] = (w >>> 16) & 0xff; + } +} +/** + * Encode a UTF-8 string to Uint8Array. + * Used for derive_key context strings. + */ +export function encodeUTF8(str) { + if (typeof TextEncoder !== "undefined") { + return new TextEncoder().encode(str); + } + // Fallback for older environments + const bytes = []; + for (let i = 0; i < str.length; i++) { + let c = str.charCodeAt(i); + if (c < 0x80) { + bytes.push(c); + } + else if (c < 0x800) { + bytes.push(0xc0 | (c >> 6), 0x80 | (c & 0x3f)); + } + else if (c < 0xd800 || c >= 0xe000) { + bytes.push(0xe0 | (c >> 12), 0x80 | ((c >> 6) & 0x3f), 0x80 | (c & 0x3f)); + } + else { + // Surrogate pair + i++; + c = 0x10000 + (((c & 0x3ff) << 10) | (str.charCodeAt(i) & 0x3ff)); + bytes.push(0xf0 | (c >> 18), 0x80 | ((c >> 12) & 0x3f), 0x80 | ((c >> 6) & 0x3f), 0x80 | (c & 0x3f)); + } + } + return new Uint8Array(bytes); +} +/** + * De Bruijn lookup table for O(1) trailing zero count. + * The expression (n & -n) isolates the lowest set bit. + * Multiplying by the De Bruijn constant maps each power of 2 to a unique 5-bit index. + */ +const CTZ32_TABLE = new Uint8Array([ + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, + 18, 6, 11, 5, 10, 9, +]); +/** + * Count trailing zero bits in a 32-bit number using De Bruijn multiplication. + * This is O(1) and branchless for non-zero inputs. + * + * For Merkle tree merge: ctz32(chunkCounter) tells us how many merges to do. + */ +export function ctz32(n) { + if (n === 0) + return 32; + // Use unsigned right shift to handle negative numbers correctly + return CTZ32_TABLE[(((n & -n) * 0x077cb531) >>> 27) & 31]; +} +/** + * Count trailing zero bits in a 64-bit number. + * Used to determine how many parent nodes to compute after adding a chunk. + * + * Note: JavaScript bitwise ops work on 32-bit signed integers, + * so we need to handle 64-bit numbers carefully. + */ +export function countTrailingZeros(n) { + if (n === 0) + return 64; + // For numbers that fit in 32 bits + const low = n | 0; + if (low !== 0) { + // Use Math.clz32 trick: ctz(x) = 31 - clz32(x & -x) for non-zero x + return 31 - Math.clz32(low & -low); + } + // High 32 bits + const high = (n / 0x100000000) | 0; + if (high !== 0) { + return 32 + (31 - Math.clz32(high & -high)); + } + return 64; +} +/** + * Create a Uint32Array view of a Uint8Array. + * Only works correctly on little-endian systems when the offset is 4-byte aligned. + * + * @param arr - Source byte array + * @param byteOffset - Starting byte offset (must be 4-byte aligned) + * @param wordLength - Number of 32-bit words + */ +export function uint32View(arr, byteOffset, wordLength) { + return new Uint32Array(arr.buffer, arr.byteOffset + byteOffset, wordLength); +} diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/wasm-simd.d.ts b/node_modules/@huggingface/blake3-jit/dist/esm/wasm-simd.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..5cfc598b3c44d4a92b2cf7812bfcae4f151a1de9 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/wasm-simd.d.ts @@ -0,0 +1,97 @@ +/** + * BLAKE3 WASM SIMD - Runtime bytecode generation + * + * Generates WebAssembly SIMD bytecode at runtime to process 4 compress + * operations in parallel using 128-bit SIMD vectors (i32x4). + * + * Key insight: One i32x4.add instruction performs 4 parallel additions, + * giving us 4x throughput for the same number of instructions. + * + * Memory layout (all values are transposed for SIMD access): + * 0-511: 4 x 16 message words (m0_0,m0_1,m0_2,m0_3, m1_0,m1_1,m1_2,m1_3, ...) + * 512-639: 4 x 8 chaining values + * 640-767: 4 x 8 output values + * 768-783: 4 x counter low + * 784-799: 4 x counter high + * 800-815: 4 x block length + * 816-831: 4 x flags + */ +/** + * Check if WASM SIMD is supported. + */ +export declare function isSimdSupported(): boolean; +export declare function initSimdSync(): boolean; +/** + * Memory offsets for SIMD data layout + * + * WASM Arena Pattern: All working buffers live in WASM memory (64KB page) + * This eliminates JS heap allocations during hashing operations. + */ +export declare const SIMD_MEMORY: { + readonly BLOCK_WORDS: 0; + readonly CHAINING_VALUES: 512; + readonly OUTPUT: 640; + readonly COUNTER_LOW: 768; + readonly COUNTER_HIGH: 784; + readonly BLOCK_LEN: 800; + readonly FLAGS: 816; + readonly BATCH_BLOCK_WORDS: 832; + readonly BATCH_CV: 4928; + readonly BATCH_COUNTER_LOW: 5056; + readonly BATCH_FLAGS_BASE: 5072; + readonly BATCH_OUTPUT: 5088; + readonly CV_STACK: 5216; + readonly PARENT_BLOCK: 7264; + readonly CHUNK_CV: 7328; + readonly TEMP_CVS: 7360; +}; +/** + * Get the WASM memory views for writing input data. + */ +export declare function getSimdMemory(): { + view: Uint8Array; + view32: Uint32Array; +} | null; +/** + * Get the arena buffers for Merkle tree operations. + * These TypedArray views are backed by WASM memory - zero JS heap allocation. + */ +export declare function getArenaBuffers(): { + cvStack: Uint32Array; + parentBlock: Uint32Array; + chunkCv: Uint32Array; + tempCvs: Uint32Array; +} | null; +/** + * Get the batch arena buffers for chunk-level batched operations. + * These TypedArray views are backed by WASM memory - zero JS heap allocation. + */ +export declare function getBatchArenaBuffers(): { + blockWords: Uint32Array; + cv: Uint32Array; + counterLow: Uint32Array; + flagsBase: Uint32Array; + output: Uint32Array; +} | null; +/** + * Run the compress4x function. + * Data must already be set up in WASM memory. + */ +export declare function runCompress4x(): void; +/** + * Run the compressChunks4x function. + * Processes 4 full chunks (16 blocks each) in a single WASM call. + * Data must already be set up in batch arena buffers. + */ +export declare function runCompressChunks4x(): void; +/** + * Run the compressParent function. + * Compresses a parent node: reads 16 words from PARENT_BLOCK, writes 8 words to CHUNK_CV. + * Data must already be set up in arena buffers (PARENT_BLOCK at offset 7264). + * Output is written to CHUNK_CV at offset 7328. + */ +export declare function runCompressParent(): void; +/** + * Check if SIMD is initialized and ready. + */ +export declare function isSimdReady(): boolean; diff --git a/node_modules/@huggingface/blake3-jit/dist/esm/wasm-simd.js b/node_modules/@huggingface/blake3-jit/dist/esm/wasm-simd.js new file mode 100644 index 0000000000000000000000000000000000000000..886fc14950ef82f20b081a17a0a379d3198a7ae4 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/dist/esm/wasm-simd.js @@ -0,0 +1,936 @@ +/** + * BLAKE3 WASM SIMD - Runtime bytecode generation + * + * Generates WebAssembly SIMD bytecode at runtime to process 4 compress + * operations in parallel using 128-bit SIMD vectors (i32x4). + * + * Key insight: One i32x4.add instruction performs 4 parallel additions, + * giving us 4x throughput for the same number of instructions. + * + * Memory layout (all values are transposed for SIMD access): + * 0-511: 4 x 16 message words (m0_0,m0_1,m0_2,m0_3, m1_0,m1_1,m1_2,m1_3, ...) + * 512-639: 4 x 8 chaining values + * 640-767: 4 x 8 output values + * 768-783: 4 x counter low + * 784-799: 4 x counter high + * 800-815: 4 x block length + * 816-831: 4 x flags + */ +// LEB128 encoding with minimum 2 bytes +// This fixes a V8 quirk where single-byte values 64-127 cause issues +// when followed by certain SIMD instructions +function toLebU32Min2(n) { + // Always use at least 2 bytes + return [(n & 0x7f) | 0x80, (n >>> 7) & 0x7f]; +} +// LEB128 encoding padded to exactly 5 bytes (for backpatching) +// Uses continuation bits for all but the last byte +function toLebU32Padded5(n) { + return [ + (n & 0x7f) | 0x80, + ((n >>> 7) & 0x7f) | 0x80, + ((n >>> 14) & 0x7f) | 0x80, + ((n >>> 21) & 0x7f) | 0x80, + (n >>> 28) & 0x0f, // Last byte has no continuation bit + ]; +} +// Signed LEB128 encoding for i32 constants (handles full 32-bit range) +// WASM i32.const uses signed LEB128 immediate +function toSignedLeb128_i32(n) { + const bytes = []; + // Treat as signed 32-bit integer + let value = n | 0; + let more = true; + while (more) { + let byte = value & 0x7f; + // Arithmetic right shift preserves sign + value >>= 7; + // Check if we're done: + // - If value is 0 and sign bit of byte is clear, we're done + // - If value is -1 and sign bit of byte is set, we're done + if ((value === 0 && (byte & 0x40) === 0) || (value === -1 && (byte & 0x40) !== 0)) { + more = false; + } + else { + byte |= 0x80; + } + bytes.push(byte); + } + return bytes; +} +// Precomputed message access order for all 7 rounds +const MSG_ACCESS_ORDER = [ + // Round 1: 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + // Round 2: 2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8 + 2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8, + // Round 3: 3,4,10,12,13,2,7,14,6,5,9,0,11,15,8,1 + 3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1, + // Round 4: 10,7,12,9,14,3,13,15,4,0,11,2,5,8,1,6 + 10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6, + // Round 5: 12,13,9,11,15,10,14,8,7,2,5,3,0,1,6,4 + 12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4, + // Round 6: 9,14,11,5,8,12,15,1,13,3,0,10,2,6,4,7 + 9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7, + // Round 7: 11,15,5,0,1,9,8,6,14,10,2,12,3,4,7,13 + 11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13, +]; +// BLAKE3 Constants (used in generated WASM code) +// CHUNK_START = 1, CHUNK_END = 2 are embedded directly in WASM bytecode +/** + * Generate the WASM module bytecode with compress4x, compressChunks4x, and compressParent functions. + */ +function generateWasmBytes() { + const code = []; + // Helper to append bytes + function put(bytes) { + code.push(...bytes); + } + // WASM module header + put([0x00, 0x61, 0x73, 0x6d]); // Magic + put([0x01, 0x00, 0x00, 0x00]); // Version + // Section 1: Types + put([0x01]); // Section ID + put([0x04]); // Section size + put([0x01]); // 1 type + put([0x60, 0x00, 0x00]); // func () -> () + // Section 2: Imports (memory from JS) + put([0x02]); // Section ID + put([0x0b]); // Section size + put([0x01]); // 1 import + put([0x02, 0x6a, 0x73]); // "js" + put([0x03, 0x6d, 0x65, 0x6d]); // "mem" + put([0x02, 0x00, 0x01]); // memory min=1, no max + // Section 3: Functions + put([0x03]); // Section ID + put([0x04]); // Section size (3 functions = 4 bytes) + put([0x03]); // 3 functions + put([0x00]); // Function 0: type index 0 + put([0x00]); // Function 1: type index 0 + put([0x00]); // Function 2: type index 0 + // Section 7: Exports + // Size calculation: 1 (count) + (1+10+1+1) + (1+16+1+1) + (1+14+1+1) = 1 + 13 + 19 + 17 = 50 bytes + put([0x07]); // Section ID + put([0x32]); // Section size (50 bytes) + put([0x03]); // 3 exports + // "compress4x" -> func 0 + put([0x0a]); // name length + put([0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x34, 0x78]); // "compress4x" + put([0x00, 0x00]); // func index 0 + // "compressChunks4x" -> func 1 + put([0x10]); // name length (16) + put([ + 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x34, 0x78, + ]); // "compressChunks4x" + put([0x00, 0x01]); // func index 1 + // "compressParent" -> func 2 + put([0x0e]); // name length (14) + put([0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74]); // "compressParent" + put([0x00, 0x02]); // func index 2 + // Section 10: Code + put([0x0a]); // Section ID + // Reserve 5 bytes for section size (LEB128 u32) + const sectionSizeOffset = code.length; + put([0x00, 0x00, 0x00, 0x00, 0x00]); + put([0x03]); // 3 functions + // === Function 0: compress4x === + // Reserve 5 bytes for function size + const funcSizeOffset = code.length; + put([0x00, 0x00, 0x00, 0x00, 0x00]); + const funcBodyStart = code.length; + // Local declarations: 32 v128 locals + // Variables $0-$15: message words (m0-m15) + // Variables $16-$31: state words (s0-s15) + put([0x01]); // 1 local declaration + put([0x20, 0x7b]); // 32 x v128 + // ===== Function body ===== + // Load message words from memory (offset 0-255) + // Each v128 is 16 bytes, so m[i] is at offset i*16 + // Note: we use toLebU32Min2 to avoid V8 quirk with single-byte values 64-127 + for (let i = 0; i < 16; i++) { + put([0x41, ...toLebU32Min2(i * 16)]); // i32.const offset (2+ byte LEB128) + put([0xfd, 0x00, 0x02, 0x00]); // v128.load align=4 offset=0 + put([0x21, i]); // local.set $i + } + // Load chaining values (offset 512-639) + // cv[i] at offset 512 + i*16 + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(512 + i * 16)]); // i32.const offset + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 16 + i]); // local.set $(16+i) + } + // Initialize state[8-15] from IV and parameters + // s8-s11 = IV[0-3] + const IV = [0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a]; + for (let i = 0; i < 4; i++) { + // Create v128 constant with all lanes set to IV[i] + const ivBytes = []; + for (let j = 0; j < 4; j++) { + ivBytes.push(IV[i] & 0xff); + ivBytes.push((IV[i] >>> 8) & 0xff); + ivBytes.push((IV[i] >>> 16) & 0xff); + ivBytes.push((IV[i] >>> 24) & 0xff); + } + put([0xfd, 0x0c, ...ivBytes]); // v128.const + put([0x21, 24 + i]); // local.set $(24+i) -> s8-s11 + } + // s12 = counter_low (offset 768) + put([0x41, ...toLebU32Min2(768)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 28]); // local.set $28 -> s12 + // s13 = counter_high (offset 784) + put([0x41, ...toLebU32Min2(784)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 29]); // local.set $29 -> s13 + // s14 = block_len (offset 800) + put([0x41, ...toLebU32Min2(800)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 30]); // local.set $30 -> s14 + // s15 = flags (offset 816) + put([0x41, ...toLebU32Min2(816)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 31]); // local.set $31 -> s15 + // ===== 7 rounds of mixing ===== + let msgIdx = 0; // Index into MSG_ACCESS_ORDER + // Helper to generate G function (inlined) + // G(a, b, c, d) with two message words + function g(a, b, c, d) { + const mx = MSG_ACCESS_ORDER[msgIdx++]; + const my = MSG_ACCESS_ORDER[msgIdx++]; + // Variables: a,b,c,d are state indices (16-31), mx,my are message indices (0-15) + // First half of G + // s[a] = s[a] + s[b] + m[mx] + put([0x20, 16 + a]); // local.get s[a] + put([0x20, 16 + b]); // local.get s[b] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x20, mx]); // local.get m[mx] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + a]); // local.set s[a] + // s[d] = rotr(s[d] ^ s[a], 16) - using i8x16.shuffle (single instruction vs shift+or) + // ROTR16 pattern: [2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13] + put([0x20, 16 + d]); // local.get s[d] + put([0x20, 16 + a]); // local.get s[a] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + d]); // local.tee s[d] + put([0x20, 16 + d]); // local.get s[d] (second operand for shuffle) + put([0xfd, 0x0d, 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13]); // i8x16.shuffle ROTR16 + put([0x21, 16 + d]); // local.set s[d] + // s[c] = s[c] + s[d] + put([0x20, 16 + c]); // local.get s[c] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + c]); // local.set s[c] + // s[b] = (s[b] ^ s[c]) >>> 12 + put([0x20, 16 + b]); // local.get s[b] + put([0x20, 16 + c]); // local.get s[c] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + b]); // local.tee s[b] + put([0x41, 0x0c]); // i32.const 12 + put([0xfd, 0xad, 0x01]); // i32x4.shr_u (opcode 173 = 0xAD) + put([0x20, 16 + b]); // local.get s[b] + put([0x41, 0x14]); // i32.const 20 + put([0xfd, 0xab, 0x01]); // i32x4.shl + put([0xfd, 0x50]); // v128.or + put([0x21, 16 + b]); // local.set s[b] + // Second half of G + // s[a] = s[a] + s[b] + m[my] + put([0x20, 16 + a]); // local.get s[a] + put([0x20, 16 + b]); // local.get s[b] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x20, my]); // local.get m[my] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + a]); // local.set s[a] + // s[d] = rotr(s[d] ^ s[a], 8) - using i8x16.shuffle (single instruction vs shift+or) + // ROTR8 pattern: [1,2,3,0, 5,6,7,4, 9,10,11,8, 13,14,15,12] + put([0x20, 16 + d]); // local.get s[d] + put([0x20, 16 + a]); // local.get s[a] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + d]); // local.tee s[d] + put([0x20, 16 + d]); // local.get s[d] (second operand for shuffle) + put([0xfd, 0x0d, 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12]); // i8x16.shuffle ROTR8 + put([0x21, 16 + d]); // local.set s[d] + // s[c] = s[c] + s[d] + put([0x20, 16 + c]); // local.get s[c] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + c]); // local.set s[c] + // s[b] = (s[b] ^ s[c]) >>> 7 + put([0x20, 16 + b]); // local.get s[b] + put([0x20, 16 + c]); // local.get s[c] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + b]); // local.tee s[b] + put([0x41, 0x07]); // i32.const 7 + put([0xfd, 0xad, 0x01]); // i32x4.shr_u (opcode 173 = 0xAD) + put([0x20, 16 + b]); // local.get s[b] + put([0x41, 0x19]); // i32.const 25 + put([0xfd, 0xab, 0x01]); // i32x4.shl + put([0xfd, 0x50]); // v128.or + put([0x21, 16 + b]); // local.set s[b] + } + // Generate all 7 rounds + for (let round = 0; round < 7; round++) { + // Column mixing + g(0, 4, 8, 12); + g(1, 5, 9, 13); + g(2, 6, 10, 14); + g(3, 7, 11, 15); + // Diagonal mixing + g(0, 5, 10, 15); + g(1, 6, 11, 12); + g(2, 7, 8, 13); + g(3, 4, 9, 14); + } + // ===== Final XOR and store output ===== + // out[i] = s[i] ^ s[i+8] for i in 0..7 + // Store at offset 640-767 + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(640 + i * 16)]); // i32.const offset + put([0x20, 16 + i]); // local.get s[i] + put([0x20, 24 + i]); // local.get s[i+8] + put([0xfd, 0x51]); // v128.xor + put([0xfd, 0x0b, 0x02, 0x00]); // v128.store align=4 + } + // End of function + put([0x0b]); // end + // Fill in function 0 size using padded LEB128 + const funcBodySize = code.length - funcBodyStart; + const funcSizeBytes = toLebU32Padded5(funcBodySize); + for (let i = 0; i < 5; i++) { + code[funcSizeOffset + i] = funcSizeBytes[i]; + } + // === Function 1: compressChunks4x === + // Reserve 5 bytes for function size + const func1SizeOffset = code.length; + put([0x00, 0x00, 0x00, 0x00, 0x00]); + const func1BodyStart = code.length; + // Generate the compressChunks4x function body + const compressChunksBody = generateCompressChunks4xBody(); + put(compressChunksBody); + // Fill in function 1 size using padded LEB128 + const func1BodySize = code.length - func1BodyStart; + const func1SizeBytes = toLebU32Padded5(func1BodySize); + for (let i = 0; i < 5; i++) { + code[func1SizeOffset + i] = func1SizeBytes[i]; + } + // === Function 2: compressParent === + // Reserve 5 bytes for function size + const func2SizeOffset = code.length; + put([0x00, 0x00, 0x00, 0x00, 0x00]); + const func2BodyStart = code.length; + // Generate the compressParent function body + const compressParentBody = generateCompressParentBody(); + put(compressParentBody); + // Fill in function 2 size using padded LEB128 + const func2BodySize = code.length - func2BodyStart; + const func2SizeBytes = toLebU32Padded5(func2BodySize); + for (let i = 0; i < 5; i++) { + code[func2SizeOffset + i] = func2SizeBytes[i]; + } + // Fill in section size using padded LEB128 + const sectionSize = code.length - sectionSizeOffset - 5; + const sectionSizeBytes = toLebU32Padded5(sectionSize); + for (let i = 0; i < 5; i++) { + code[sectionSizeOffset + i] = sectionSizeBytes[i]; + } + return new Uint8Array(code); +} +/** + * Generate compressChunks4x WASM function body. + * Processes all 16 blocks of 4 chunks in a single call. + */ +function generateCompressChunks4xBody() { + const code = []; + function put(bytes) { + code.push(...bytes); + } + // Local declarations: 32 v128 locals + 1 i32 for position + // Locals $0-$15: message words (reloaded each iteration) + // Locals $16-$31: state words (s0-s15) + // Local $32: position counter (i32) + put([0x02]); // 2 local declarations + put([0x20, 0x7b]); // 32 x v128 + put([0x01, 0x7f]); // 1 x i32 + const BATCH_BLOCK_WORDS = SIMD_MEMORY.BATCH_BLOCK_WORDS; + const BATCH_CV = SIMD_MEMORY.BATCH_CV; + const BATCH_COUNTER_LOW = SIMD_MEMORY.BATCH_COUNTER_LOW; + const BATCH_FLAGS_BASE = SIMD_MEMORY.BATCH_FLAGS_BASE; + const BATCH_OUTPUT = SIMD_MEMORY.BATCH_OUTPUT; + // IV constants (same as compress4x) + const IV = [0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a]; + // Load initial CVs from BATCH_CV into locals $16-$23 + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(BATCH_CV + i * 16)]); // i32.const offset + put([0xfd, 0x00, 0x02, 0x00]); // v128.load align=4 offset=0 + put([0x21, 16 + i]); // local.set $(16+i) -> s0-s7 + } + // Initialize $32 (pos) = 0 + put([0x41, 0x00]); // i32.const 0 + put([0x21, 0x20]); // local.set $32 + // block $done + put([0x02, 0x40]); // block void + // loop $continue + put([0x03, 0x40]); // loop void + // === Load message words for position $pos === + // offset = BATCH_BLOCK_WORDS + pos * 256 + word * 16 + for (let w = 0; w < 16; w++) { + put([0x20, 0x20]); // local.get $32 (pos) + put([0x41, ...toLebU32Min2(256)]); // i32.const 256 + put([0x6c]); // i32.mul + put([0x41, ...toLebU32Min2(BATCH_BLOCK_WORDS + w * 16)]); // i32.const base + word*16 + put([0x6a]); // i32.add + put([0xfd, 0x00, 0x02, 0x00]); // v128.load align=4 offset=0 + put([0x21, w]); // local.set $w + } + // === Initialize state[8-15] === + // s8-s11 = IV[0-3] + for (let i = 0; i < 4; i++) { + const ivBytes = []; + for (let j = 0; j < 4; j++) { + ivBytes.push(IV[i] & 0xff); + ivBytes.push((IV[i] >>> 8) & 0xff); + ivBytes.push((IV[i] >>> 16) & 0xff); + ivBytes.push((IV[i] >>> 24) & 0xff); + } + put([0xfd, 0x0c, ...ivBytes]); // v128.const + put([0x21, 24 + i]); // local.set $(24+i) -> s8-s11 + } + // s12 = counter_low (from BATCH_COUNTER_LOW) + put([0x41, ...toLebU32Min2(BATCH_COUNTER_LOW)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 28]); // local.set $28 -> s12 + // s13 = 0 (counter high - assume fits in 32 bits) + put([0xfd, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); // v128.const 0 + put([0x21, 29]); // local.set $29 -> s13 + // s14 = 64 (block_len = 64 for full blocks) + const blockLen64 = []; + for (let j = 0; j < 4; j++) { + blockLen64.push(64, 0, 0, 0); // 64 in little-endian + } + put([0xfd, 0x0c, ...blockLen64]); // v128.const [64,64,64,64] + put([0x21, 30]); // local.set $30 -> s14 + // s15 = flags = base_flags | (pos == 0 ? 1 : 0) | (pos == 15 ? 2 : 0) + // First load base flags + put([0x41, ...toLebU32Min2(BATCH_FLAGS_BASE)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load base flags + // Compute position-dependent bits + // CHUNK_START (1) if pos == 0 + put([0x20, 0x20]); // local.get $32 (pos) + put([0x45]); // i32.eqz -> 1 if pos==0, 0 otherwise + // CHUNK_END (2) if pos == 15 + put([0x20, 0x20]); // local.get $32 (pos) + put([0x41, 0x0f]); // i32.const 15 + put([0x46]); // i32.eq -> 1 if pos==15, 0 otherwise + put([0x41, 0x01]); // i32.const 1 (shift amount) + put([0x74]); // i32.shl -> 2 if pos==15, 0 otherwise + // OR the two bits together + put([0x72]); // i32.or -> combined position bits + // Splat to v128 and OR with base flags (stack: base_flags, bits) + put([0xfd, 0x11]); // i32x4.splat + put([0xfd, 0x50]); // v128.or + put([0x21, 31]); // local.set $31 -> s15 + // === 7 rounds of mixing === + let msgIdx = 0; + function g(a, b, c, d) { + const mx = MSG_ACCESS_ORDER[msgIdx++]; + const my = MSG_ACCESS_ORDER[msgIdx++]; + // First half of G: s[a] = s[a] + s[b] + m[mx] + put([0x20, 16 + a]); // local.get s[a] + put([0x20, 16 + b]); // local.get s[b] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x20, mx]); // local.get m[mx] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + a]); // local.set s[a] + // s[d] = rotr(s[d] ^ s[a], 16) - byte shuffle + put([0x20, 16 + d]); // local.get s[d] + put([0x20, 16 + a]); // local.get s[a] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + d]); // local.tee s[d] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0x0d, 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13]); // i8x16.shuffle ROTR16 + put([0x21, 16 + d]); // local.set s[d] + // s[c] = s[c] + s[d] + put([0x20, 16 + c]); // local.get s[c] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + c]); // local.set s[c] + // s[b] = rotr(s[b] ^ s[c], 12) + put([0x20, 16 + b]); // local.get s[b] + put([0x20, 16 + c]); // local.get s[c] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + b]); // local.tee s[b] + put([0x41, 0x0c]); // i32.const 12 + put([0xfd, 0xad, 0x01]); // i32x4.shr_u (opcode 173 = 0xAD) + put([0x20, 16 + b]); // local.get s[b] + put([0x41, 0x14]); // i32.const 20 + put([0xfd, 0xab, 0x01]); // i32x4.shl + put([0xfd, 0x50]); // v128.or + put([0x21, 16 + b]); // local.set s[b] + // Second half: s[a] = s[a] + s[b] + m[my] + put([0x20, 16 + a]); // local.get s[a] + put([0x20, 16 + b]); // local.get s[b] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x20, my]); // local.get m[my] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + a]); // local.set s[a] + // s[d] = rotr(s[d] ^ s[a], 8) - byte shuffle + put([0x20, 16 + d]); // local.get s[d] + put([0x20, 16 + a]); // local.get s[a] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + d]); // local.tee s[d] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0x0d, 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12]); // i8x16.shuffle ROTR8 + put([0x21, 16 + d]); // local.set s[d] + // s[c] = s[c] + s[d] + put([0x20, 16 + c]); // local.get s[c] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + c]); // local.set s[c] + // s[b] = rotr(s[b] ^ s[c], 7) + put([0x20, 16 + b]); // local.get s[b] + put([0x20, 16 + c]); // local.get s[c] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + b]); // local.tee s[b] + put([0x41, 0x07]); // i32.const 7 + put([0xfd, 0xad, 0x01]); // i32x4.shr_u (opcode 173 = 0xAD) + put([0x20, 16 + b]); // local.get s[b] + put([0x41, 0x19]); // i32.const 25 + put([0xfd, 0xab, 0x01]); // i32x4.shl + put([0xfd, 0x50]); // v128.or + put([0x21, 16 + b]); // local.set s[b] + } + // Generate all 7 rounds + for (let round = 0; round < 7; round++) { + // Column mixing + g(0, 4, 8, 12); + g(1, 5, 9, 13); + g(2, 6, 10, 14); + g(3, 7, 11, 15); + // Diagonal mixing + g(0, 5, 10, 15); + g(1, 6, 11, 12); + g(2, 7, 8, 13); + g(3, 4, 9, 14); + } + // === Update CVs: cv[i] = s[i] ^ s[i+8] === + // Store back to state locals $16-$23 (the CV positions) + for (let i = 0; i < 8; i++) { + put([0x20, 16 + i]); // local.get s[i] + put([0x20, 24 + i]); // local.get s[i+8] + put([0xfd, 0x51]); // v128.xor + put([0x21, 16 + i]); // local.set $(16+i) - update CV + } + // === Loop control: pos++, continue if pos < 16 === + put([0x20, 0x20]); // local.get $32 (pos) + put([0x41, 0x01]); // i32.const 1 + put([0x6a]); // i32.add + put([0x22, 0x20]); // local.tee $32 (pos) + put([0x41, 0x10]); // i32.const 16 + put([0x49]); // i32.lt_u + put([0x0d, 0x00]); // br_if 0 (continue loop) + // end loop + put([0x0b]); // end + // end block + put([0x0b]); // end + // === Store final CVs to BATCH_OUTPUT === + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(BATCH_OUTPUT + i * 16)]); // i32.const offset + put([0x20, 16 + i]); // local.get $(16+i) - CV word + put([0xfd, 0x0b, 0x02, 0x00]); // v128.store align=4 + } + // end function + put([0x0b]); // end + return code; +} +/** + * Generate compressParent WASM function body. + * Performs a single parent node compression using scalar i32 operations. + * Reads 16 words from PARENT_BLOCK, writes 8 words to CHUNK_CV. + * Uses IV, counter=0, blockLen=64, flags=PARENT(4). + */ +function generateCompressParentBody() { + const code = []; + function put(bytes) { + code.push(...bytes); + } + // Local declarations: 32 i32 locals for state (s0-s15) and message (m0-m15) + put([0x01]); // 1 local declaration + put([0x20, 0x7f]); // 32 x i32 + // Message word indices: 0-15, State indices: 16-31 + // Locals $0-$15: message words (m0-m15) + // Locals $16-$31: state words (s0-s15) + const PARENT_BLOCK_OFFSET = SIMD_MEMORY.PARENT_BLOCK; + const CHUNK_CV_OFFSET = SIMD_MEMORY.CHUNK_CV; + // BLAKE3 IV + const IV = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, + ]; + // Load message words from PARENT_BLOCK (16 words at offset 7264) + for (let i = 0; i < 16; i++) { + put([0x41, ...toLebU32Min2(PARENT_BLOCK_OFFSET + i * 4)]); // i32.const offset + put([0x28, 0x02, 0x00]); // i32.load align=4 offset=0 + put([0x21, i]); // local.set $i (m0-m15) + } + // Initialize state s0-s7 = IV[0-7] + for (let i = 0; i < 8; i++) { + put([0x41, ...toSignedLeb128_i32(IV[i])]); // i32.const IV[i] + put([0x21, 16 + i]); // local.set $(16+i) -> s0-s7 + } + // Initialize state s8-s11 = IV[0-3] + for (let i = 0; i < 4; i++) { + put([0x41, ...toSignedLeb128_i32(IV[i])]); // i32.const IV[i] + put([0x21, 24 + i]); // local.set $(24+i) -> s8-s11 + } + // s12 = counter_low = 0 + put([0x41, 0x00]); // i32.const 0 + put([0x21, 28]); // local.set $28 -> s12 + // s13 = counter_high = 0 + put([0x41, 0x00]); // i32.const 0 + put([0x21, 29]); // local.set $29 -> s13 + // s14 = block_len = 64 + // Note: 0x40 alone is -64 in signed LEB128 (bit 6 is sign bit) + // For 64, we need [0xC0, 0x00] to avoid sign extension + put([0x41, 0xc0, 0x00]); // i32.const 64 + put([0x21, 30]); // local.set $30 -> s14 + // s15 = flags = PARENT = 4 + put([0x41, 0x04]); // i32.const 4 + put([0x21, 31]); // local.set $31 -> s15 + // Helper to generate scalar G function (inlined) + // G(a, b, c, d, mx, my) where a,b,c,d are state indices 0-15, mx,my are message indices 0-15 + function g(a, b, c, d, mx, my) { + const sa = 16 + a, sb = 16 + b, sc = 16 + c, sd = 16 + d; + // s[a] = (s[a] + s[b] + m[mx]) >>> 0 + put([0x20, sa]); // local.get s[a] + put([0x20, sb]); // local.get s[b] + put([0x6a]); // i32.add + put([0x20, mx]); // local.get m[mx] + put([0x6a]); // i32.add + put([0x21, sa]); // local.set s[a] + // s[d] = rotr(s[d] ^ s[a], 16) + put([0x20, sd]); // local.get s[d] + put([0x20, sa]); // local.get s[a] + put([0x73]); // i32.xor + put([0x41, 0x10]); // i32.const 16 + put([0x78]); // i32.rotr + put([0x21, sd]); // local.set s[d] + // s[c] = (s[c] + s[d]) >>> 0 + put([0x20, sc]); // local.get s[c] + put([0x20, sd]); // local.get s[d] + put([0x6a]); // i32.add + put([0x21, sc]); // local.set s[c] + // s[b] = rotr(s[b] ^ s[c], 12) + put([0x20, sb]); // local.get s[b] + put([0x20, sc]); // local.get s[c] + put([0x73]); // i32.xor + put([0x41, 0x0c]); // i32.const 12 + put([0x78]); // i32.rotr + put([0x21, sb]); // local.set s[b] + // s[a] = (s[a] + s[b] + m[my]) >>> 0 + put([0x20, sa]); // local.get s[a] + put([0x20, sb]); // local.get s[b] + put([0x6a]); // i32.add + put([0x20, my]); // local.get m[my] + put([0x6a]); // i32.add + put([0x21, sa]); // local.set s[a] + // s[d] = rotr(s[d] ^ s[a], 8) + put([0x20, sd]); // local.get s[d] + put([0x20, sa]); // local.get s[a] + put([0x73]); // i32.xor + put([0x41, 0x08]); // i32.const 8 + put([0x78]); // i32.rotr + put([0x21, sd]); // local.set s[d] + // s[c] = (s[c] + s[d]) >>> 0 + put([0x20, sc]); // local.get s[c] + put([0x20, sd]); // local.get s[d] + put([0x6a]); // i32.add + put([0x21, sc]); // local.set s[c] + // s[b] = rotr(s[b] ^ s[c], 7) + put([0x20, sb]); // local.get s[b] + put([0x20, sc]); // local.get s[c] + put([0x73]); // i32.xor + put([0x41, 0x07]); // i32.const 7 + put([0x78]); // i32.rotr + put([0x21, sb]); // local.set s[b] + } + // 7 rounds of mixing with permuted message schedule + let msgIdx = 0; + for (let round = 0; round < 7; round++) { + // Column mixing + g(0, 4, 8, 12, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(1, 5, 9, 13, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(2, 6, 10, 14, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(3, 7, 11, 15, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + // Diagonal mixing + g(0, 5, 10, 15, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(1, 6, 11, 12, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(2, 7, 8, 13, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(3, 4, 9, 14, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + } + // Store output: out[i] = s[i] ^ s[i+8] for i in 0..7 + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(CHUNK_CV_OFFSET + i * 4)]); // i32.const offset + put([0x20, 16 + i]); // local.get s[i] + put([0x20, 24 + i]); // local.get s[i+8] + put([0x73]); // i32.xor + put([0x36, 0x02, 0x00]); // i32.store align=4 offset=0 + } + // end function + put([0x0b]); // end + return code; +} +// Cached WASM instance +let wasmInstance = null; +let wasmMemory = null; +let wasmCompress4x = null; +let wasmCompressChunks4x = null; +let wasmCompressParent = null; +let wasmMemoryView = null; +let wasmMemoryView32 = null; +/** + * Check if WASM SIMD is supported. + */ +export function isSimdSupported() { + try { + // Minimal WASM module with v128.const instruction to test SIMD support + const simdTest = new Uint8Array([ + 0x00, + 0x61, + 0x73, + 0x6d, // magic: \0asm + 0x01, + 0x00, + 0x00, + 0x00, // version: 1 + // Type section (id=1): () -> v128 + 0x01, // section id = 1 (type) + 0x05, // section length = 5 + 0x01, // 1 type + 0x60, + 0x00, + 0x01, + 0x7b, // func () -> v128 + // Function section (id=3) + 0x03, // section id = 3 (function) + 0x02, // section length = 2 + 0x01, // 1 function + 0x00, // type index 0 + // Code section (id=10) with v128.const + 0x0a, // section id = 10 (code) + 0x16, // section length = 22 + 0x01, // 1 function body + 0x14, // body length = 20 + 0x00, // 0 locals + 0xfd, + 0x0c, // v128.const opcode + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x0b, // end + ]); + return WebAssembly.validate(simdTest); + } + catch { + return false; + } +} +/** + * Set up arena views over WASM memory. + * Called after WASM memory is allocated. + */ +function setupArenaViews() { + if (!wasmMemory) + return; + const buffer = wasmMemory.buffer; + // Create TypedArray views over WASM memory for arena buffers + // These views are backed by WASM memory, eliminating JS heap allocation + arenaCvStack = new Uint32Array(buffer, SIMD_MEMORY.CV_STACK, 64 * 8); // 64 levels × 8 words + arenaParentBlock = new Uint32Array(buffer, SIMD_MEMORY.PARENT_BLOCK, 16); // 16 words + arenaChunkCv = new Uint32Array(buffer, SIMD_MEMORY.CHUNK_CV, 8); // 8 words + arenaTempCvs = new Uint32Array(buffer, SIMD_MEMORY.TEMP_CVS, 32); // 4 × 8 words + // Batch mode views + // 16 positions × 16 v128 words = 16 × 64 u32 words = 1024 words per position? No... + // In u32 terms: 16 positions × 16 words × 4 lanes = 1024 u32 values total + arenaBatchBlockWords = new Uint32Array(buffer, SIMD_MEMORY.BATCH_BLOCK_WORDS, 16 * 16 * 4); // 16 pos × 16 words × 4 lanes + arenaBatchCv = new Uint32Array(buffer, SIMD_MEMORY.BATCH_CV, 32); // 4 × 8 words + arenaBatchCounterLow = new Uint32Array(buffer, SIMD_MEMORY.BATCH_COUNTER_LOW, 4); // 4 words + arenaBatchFlagsBase = new Uint32Array(buffer, SIMD_MEMORY.BATCH_FLAGS_BASE, 4); // 4 words + arenaBatchOutput = new Uint32Array(buffer, SIMD_MEMORY.BATCH_OUTPUT, 32); // 4 × 8 words +} +/** + * Initialize the WASM SIMD module synchronously. + * Call this once before using compress4x. + */ +// Cache generated WASM bytes to avoid regenerating on each init +let cachedWasmBytes = null; +export function initSimdSync() { + if (wasmInstance) + return true; + if (!isSimdSupported()) { + return false; + } + try { + const wasmBytes = cachedWasmBytes || generateWasmBytes(); + cachedWasmBytes = wasmBytes; + wasmMemory = new WebAssembly.Memory({ initial: 1 }); + const importObject = { + js: { mem: wasmMemory }, + }; + const module = new WebAssembly.Module(wasmBytes.buffer); + wasmInstance = new WebAssembly.Instance(module, importObject); + wasmCompress4x = wasmInstance.exports.compress4x; + wasmCompressChunks4x = wasmInstance.exports.compressChunks4x; + wasmCompressParent = wasmInstance.exports.compressParent; + wasmMemoryView = new Uint8Array(wasmMemory.buffer); + wasmMemoryView32 = new Uint32Array(wasmMemory.buffer); + // Set up arena views for Merkle tree operations + setupArenaViews(); + return true; + } + catch (e) { + console.warn("Failed to initialize WASM SIMD:", e); + return false; + } +} +/** + * Memory offsets for SIMD data layout + * + * WASM Arena Pattern: All working buffers live in WASM memory (64KB page) + * This eliminates JS heap allocations during hashing operations. + */ +export const SIMD_MEMORY = { + // SIMD compress4x working area (used by WASM code) - single block + BLOCK_WORDS: 0, // 4 x 16 words = 512 bytes (transposed layout) + CHAINING_VALUES: 512, // 4 x 8 words = 128 bytes + OUTPUT: 640, // 4 x 8 words = 128 bytes + COUNTER_LOW: 768, // 4 words = 16 bytes + COUNTER_HIGH: 784, // 4 words = 16 bytes + BLOCK_LEN: 800, // 4 words = 16 bytes + FLAGS: 816, // 4 words = 16 bytes + // End of single-block SIMD working area: 832 bytes + // SIMD compressChunks4x working area - 16 blocks batched + // Each block position has 16 v128 values (one per message word) = 256 bytes + // 16 block positions = 16 × 256 = 4096 bytes + BATCH_BLOCK_WORDS: 832, // 16 positions × 256 bytes = 4096 bytes (transposed), ends at 4928 + BATCH_CV: 4928, // 4 × 8 words × 4 bytes = 128 bytes (working CVs), ends at 5056 + BATCH_COUNTER_LOW: 5056, // 4 words × 4 bytes = 16 bytes (per-chunk counters), ends at 5072 + BATCH_FLAGS_BASE: 5072, // 4 words × 4 bytes = 16 bytes (base flags, no START/END), ends at 5088 + BATCH_OUTPUT: 5088, // 4 × 8 words × 4 bytes = 128 bytes (final output), ends at 5216 + // End of batch working area: 5216 bytes + // WASM Arena: JS working buffers (accessed via TypedArray views) + CV_STACK: 5216, // 64 levels × 8 words × 4 bytes = 2048 bytes, ends at 7264 + PARENT_BLOCK: 7264, // 16 words × 4 bytes = 64 bytes, ends at 7328 + CHUNK_CV: 7328, // 8 words × 4 bytes = 32 bytes, ends at 7360 + TEMP_CVS: 7360, // 4 × 8 words × 4 bytes = 128 bytes, ends at 7488 + // Total arena usage: ~7488 bytes (fits comfortably in 64KB page) +}; +// Arena views - created once when SIMD initializes +let arenaCvStack = null; +let arenaParentBlock = null; +let arenaChunkCv = null; +let arenaTempCvs = null; +// Batch mode arena views +let arenaBatchBlockWords = null; +let arenaBatchCv = null; +let arenaBatchCounterLow = null; +let arenaBatchFlagsBase = null; +let arenaBatchOutput = null; +/** + * Get the WASM memory views for writing input data. + */ +export function getSimdMemory() { + if (!wasmMemoryView || !wasmMemoryView32) + return null; + return { view: wasmMemoryView, view32: wasmMemoryView32 }; +} +/** + * Get the arena buffers for Merkle tree operations. + * These TypedArray views are backed by WASM memory - zero JS heap allocation. + */ +export function getArenaBuffers() { + if (!arenaCvStack || !arenaParentBlock || !arenaChunkCv || !arenaTempCvs) + return null; + return { + cvStack: arenaCvStack, + parentBlock: arenaParentBlock, + chunkCv: arenaChunkCv, + tempCvs: arenaTempCvs, + }; +} +/** + * Get the batch arena buffers for chunk-level batched operations. + * These TypedArray views are backed by WASM memory - zero JS heap allocation. + */ +export function getBatchArenaBuffers() { + if (!arenaBatchBlockWords || + !arenaBatchCv || + !arenaBatchCounterLow || + !arenaBatchFlagsBase || + !arenaBatchOutput) + return null; + return { + blockWords: arenaBatchBlockWords, + cv: arenaBatchCv, + counterLow: arenaBatchCounterLow, + flagsBase: arenaBatchFlagsBase, + output: arenaBatchOutput, + }; +} +/** + * Run the compress4x function. + * Data must already be set up in WASM memory. + */ +export function runCompress4x() { + if (!wasmCompress4x) { + throw new Error("WASM SIMD not initialized. Call initSimdSync() first."); + } + wasmCompress4x(); +} +/** + * Run the compressChunks4x function. + * Processes 4 full chunks (16 blocks each) in a single WASM call. + * Data must already be set up in batch arena buffers. + */ +export function runCompressChunks4x() { + if (!wasmCompressChunks4x) { + throw new Error("WASM SIMD not initialized. Call initSimdSync() first."); + } + wasmCompressChunks4x(); +} +/** + * Run the compressParent function. + * Compresses a parent node: reads 16 words from PARENT_BLOCK, writes 8 words to CHUNK_CV. + * Data must already be set up in arena buffers (PARENT_BLOCK at offset 7264). + * Output is written to CHUNK_CV at offset 7328. + */ +export function runCompressParent() { + if (!wasmCompressParent) { + throw new Error("WASM SIMD not initialized. Call initSimdSync() first."); + } + wasmCompressParent(); +} +/** + * Check if SIMD is initialized and ready. + */ +export function isSimdReady() { + return wasmCompress4x !== null; +} diff --git a/node_modules/@huggingface/blake3-jit/package.json b/node_modules/@huggingface/blake3-jit/package.json new file mode 100644 index 0000000000000000000000000000000000000000..a5378dc2b47f2b16437c73456e507561c87e928a --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/package.json @@ -0,0 +1,49 @@ +{ + "name": "@huggingface/blake3-jit", + "version": "0.0.2", + "description": "Temporary fork of blake3-jit with Hasher.reset() and pre-allocated buffers. Will be deprecated once upstream blake3-jit exposes reset().", + "keywords": [ + "blake3", + "hash", + "fast", + "wasm" + ], + "license": "MIT", + "author": "Hugging Face", + "publishConfig": { + "access": "public" + }, + "files": [ + "dist", + "src", + "LICENSE", + "README.md" + ], + "type": "module", + "sideEffects": false, + "scripts": { + "prepare": "tshy" + }, + "tshy": { + "exports": { + ".": "./src/index.ts", + "./package.json": "./package.json" + } + }, + "exports": { + ".": { + "import": { + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/index.js" + }, + "require": { + "types": "./dist/commonjs/index.d.ts", + "default": "./dist/commonjs/index.js" + } + }, + "./package.json": "./package.json" + }, + "main": "./dist/commonjs/index.js", + "types": "./dist/commonjs/index.d.ts", + "module": "./dist/esm/index.js" +} diff --git a/node_modules/@huggingface/blake3-jit/src/compress.ts b/node_modules/@huggingface/blake3-jit/src/compress.ts new file mode 100644 index 0000000000000000000000000000000000000000..7c2cbc7c2113638c298f14370982834f7ae894dd --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/src/compress.ts @@ -0,0 +1,954 @@ +/** + * BLAKE3 Compression Function - Highly Optimized + * + * Optimization techniques applied (from Fleek Network case study): + * 1. Use 16 SMI variables for state instead of TypedArray + * 2. Use 16 SMI variables for message words + * 3. Fully inlined G function (no function call overhead) + * 4. Use `| 0` for integer coercion (forces V8 to use 32-bit ALU) + * 5. Hardcoded permutation swaps using only 2 temporary variables + * 6. Offset-based access pattern (avoid creating new views) + * + * The compression function takes: + * - cv: 8-word chaining value + * - block: 16-word message block (64 bytes) + * - counter: 64-bit block counter + * - blockLen: number of input bytes in this block + * - flags: domain separation flags + * + * And outputs 8 or 16 words depending on whether this is a root node. + */ + +/** + * Compress a single block. + * + * This is the hot path - every optimization matters here. + * + * @param cv - Chaining value array + * @param cvOff - Offset into cv + * @param block - Message block words + * @param blockOff - Offset into block + * @param out - Output array (8 or 16 words) + * @param outOff - Offset into out + * @param full - If true, output all 16 words (for XOF); if false, output 8 words + * @param counter - 64-bit block counter + * @param blockLen - Number of bytes in this block (0-64) + * @param flags - Domain separation flags + */ +export function compress( + cv: Uint32Array, + cvOff: number, + block: Uint32Array, + blockOff: number, + out: Uint32Array, + outOff: number, + full: boolean, + counter: number, + blockLen: number, + flags: number, +): void { + // Load message words into SMI variables for maximum performance + // V8 optimizes SMI arithmetic directly with the ALU + let m0 = block[blockOff] | 0; + let m1 = block[blockOff + 1] | 0; + let m2 = block[blockOff + 2] | 0; + let m3 = block[blockOff + 3] | 0; + let m4 = block[blockOff + 4] | 0; + let m5 = block[blockOff + 5] | 0; + let m6 = block[blockOff + 6] | 0; + let m7 = block[blockOff + 7] | 0; + let m8 = block[blockOff + 8] | 0; + let m9 = block[blockOff + 9] | 0; + let m10 = block[blockOff + 10] | 0; + let m11 = block[blockOff + 11] | 0; + let m12 = block[blockOff + 12] | 0; + let m13 = block[blockOff + 13] | 0; + let m14 = block[blockOff + 14] | 0; + let m15 = block[blockOff + 15] | 0; + + // Initialize state: first 8 words from chaining value + let s0 = cv[cvOff] | 0; + let s1 = cv[cvOff + 1] | 0; + let s2 = cv[cvOff + 2] | 0; + let s3 = cv[cvOff + 3] | 0; + let s4 = cv[cvOff + 4] | 0; + let s5 = cv[cvOff + 5] | 0; + let s6 = cv[cvOff + 6] | 0; + let s7 = cv[cvOff + 7] | 0; + + // Words 8-11: IV constants + let s8 = 0x6a09e667; + let s9 = 0xbb67ae85; + let s10 = 0x3c6ef372; + let s11 = 0xa54ff53a; + + // Words 12-15: counter, blockLen, flags + // Note: counter is 64-bit, split into low and high 32-bit words + let s12 = counter | 0; + let s13 = (counter / 0x100000000) | 0; + let s14 = blockLen | 0; + let s15 = flags | 0; + + // ============================================================ + // 7 rounds of mixing + // Each round consists of 4 column G functions and 4 diagonal G functions + // followed by a message word permutation (except for round 7) + // ============================================================ + + // ROUND 1 (message schedule: 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15) + // Column G functions + // G(0, 4, 8, 12) with m0, m1 + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + // G(1, 5, 9, 13) with m2, m3 + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + // G(2, 6, 10, 14) with m4, m5 + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + // G(3, 7, 11, 15) with m6, m7 + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + // Diagonal G functions + // G(0, 5, 10, 15) with m8, m9 + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + // G(1, 6, 11, 12) with m10, m11 + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + // G(2, 7, 8, 13) with m12, m13 + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + // G(3, 4, 9, 14) with m14, m15 + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + + // Permute message words for round 2 + // Permutation: [2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8] + // Using 2 temps for the two cycles in the permutation + { + const t0 = m0, + t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + + // ROUND 2 (message schedule: 2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + + // Permute for round 3 + { + const t0 = m0, + t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + + // ROUND 3 (message schedule: 3,4,10,12,13,2,7,14,6,5,9,0,11,15,8,1) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + + // Permute for round 4 + { + const t0 = m0, + t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + + // ROUND 4 (message schedule: 10,7,12,9,14,3,13,15,4,0,11,2,5,8,1,6) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + + // Permute for round 5 + { + const t0 = m0, + t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + + // ROUND 5 (message schedule: 12,13,9,11,15,10,14,8,7,2,5,3,0,1,6,4) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + + // Permute for round 6 + { + const t0 = m0, + t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + + // ROUND 6 (message schedule: 9,14,11,5,8,12,15,1,13,3,0,10,2,6,4,7) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + + // Permute for round 7 + { + const t0 = m0, + t1 = m1; + m0 = m2; + m2 = m3; + m3 = m10; + m10 = m12; + m12 = m9; + m9 = m11; + m11 = m5; + m5 = t0; + m1 = m6; + m6 = m4; + m4 = m7; + m7 = m13; + m13 = m14; + m14 = m15; + m15 = m8; + m8 = t1; + } + + // ROUND 7 (message schedule: 11,15,5,0,1,9,8,6,14,10,2,12,3,4,7,13) + s0 = (((s0 + s4) | 0) + m0) | 0; + s12 ^= s0; + s12 = (s12 >>> 16) | (s12 << 16); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 12) | (s4 << 20); + s0 = (((s0 + s4) | 0) + m1) | 0; + s12 ^= s0; + s12 = (s12 >>> 8) | (s12 << 24); + s8 = (s8 + s12) | 0; + s4 ^= s8; + s4 = (s4 >>> 7) | (s4 << 25); + s1 = (((s1 + s5) | 0) + m2) | 0; + s13 ^= s1; + s13 = (s13 >>> 16) | (s13 << 16); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 12) | (s5 << 20); + s1 = (((s1 + s5) | 0) + m3) | 0; + s13 ^= s1; + s13 = (s13 >>> 8) | (s13 << 24); + s9 = (s9 + s13) | 0; + s5 ^= s9; + s5 = (s5 >>> 7) | (s5 << 25); + s2 = (((s2 + s6) | 0) + m4) | 0; + s14 ^= s2; + s14 = (s14 >>> 16) | (s14 << 16); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 12) | (s6 << 20); + s2 = (((s2 + s6) | 0) + m5) | 0; + s14 ^= s2; + s14 = (s14 >>> 8) | (s14 << 24); + s10 = (s10 + s14) | 0; + s6 ^= s10; + s6 = (s6 >>> 7) | (s6 << 25); + s3 = (((s3 + s7) | 0) + m6) | 0; + s15 ^= s3; + s15 = (s15 >>> 16) | (s15 << 16); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 12) | (s7 << 20); + s3 = (((s3 + s7) | 0) + m7) | 0; + s15 ^= s3; + s15 = (s15 >>> 8) | (s15 << 24); + s11 = (s11 + s15) | 0; + s7 ^= s11; + s7 = (s7 >>> 7) | (s7 << 25); + s0 = (((s0 + s5) | 0) + m8) | 0; + s15 ^= s0; + s15 = (s15 >>> 16) | (s15 << 16); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 12) | (s5 << 20); + s0 = (((s0 + s5) | 0) + m9) | 0; + s15 ^= s0; + s15 = (s15 >>> 8) | (s15 << 24); + s10 = (s10 + s15) | 0; + s5 ^= s10; + s5 = (s5 >>> 7) | (s5 << 25); + s1 = (((s1 + s6) | 0) + m10) | 0; + s12 ^= s1; + s12 = (s12 >>> 16) | (s12 << 16); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 12) | (s6 << 20); + s1 = (((s1 + s6) | 0) + m11) | 0; + s12 ^= s1; + s12 = (s12 >>> 8) | (s12 << 24); + s11 = (s11 + s12) | 0; + s6 ^= s11; + s6 = (s6 >>> 7) | (s6 << 25); + s2 = (((s2 + s7) | 0) + m12) | 0; + s13 ^= s2; + s13 = (s13 >>> 16) | (s13 << 16); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 12) | (s7 << 20); + s2 = (((s2 + s7) | 0) + m13) | 0; + s13 ^= s2; + s13 = (s13 >>> 8) | (s13 << 24); + s8 = (s8 + s13) | 0; + s7 ^= s8; + s7 = (s7 >>> 7) | (s7 << 25); + s3 = (((s3 + s4) | 0) + m14) | 0; + s14 ^= s3; + s14 = (s14 >>> 16) | (s14 << 16); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 12) | (s4 << 20); + s3 = (((s3 + s4) | 0) + m15) | 0; + s14 ^= s3; + s14 = (s14 >>> 8) | (s14 << 24); + s9 = (s9 + s14) | 0; + s4 ^= s9; + s4 = (s4 >>> 7) | (s4 << 25); + + // ============================================================ + // Final XOR and output + // ============================================================ + + // If full output needed (XOF mode), write words 8-15 first + // (written first in case out === cv) + if (full) { + out[outOff + 8] = s8 ^ cv[cvOff]; + out[outOff + 9] = s9 ^ cv[cvOff + 1]; + out[outOff + 10] = s10 ^ cv[cvOff + 2]; + out[outOff + 11] = s11 ^ cv[cvOff + 3]; + out[outOff + 12] = s12 ^ cv[cvOff + 4]; + out[outOff + 13] = s13 ^ cv[cvOff + 5]; + out[outOff + 14] = s14 ^ cv[cvOff + 6]; + out[outOff + 15] = s15 ^ cv[cvOff + 7]; + } + + // Standard output: XOR state[0..7] with state[8..15] + out[outOff] = s0 ^ s8; + out[outOff + 1] = s1 ^ s9; + out[outOff + 2] = s2 ^ s10; + out[outOff + 3] = s3 ^ s11; + out[outOff + 4] = s4 ^ s12; + out[outOff + 5] = s5 ^ s13; + out[outOff + 6] = s6 ^ s14; + out[outOff + 7] = s7 ^ s15; +} diff --git a/node_modules/@huggingface/blake3-jit/src/constants.ts b/node_modules/@huggingface/blake3-jit/src/constants.ts new file mode 100644 index 0000000000000000000000000000000000000000..26a64604c6507c75fabd90437c3500efa20e6a45 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/src/constants.ts @@ -0,0 +1,59 @@ +/** + * BLAKE3 Constants + * + * IV values are the same as SHA-256: first 32 bits of the fractional parts + * of the square roots of the first 8 primes (2..19) + */ + +// Initialization Vector (same as SHA-256) +export const IV = new Uint32Array([ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, +]); + +// Domain separation flags +export const CHUNK_START = 1; +export const CHUNK_END = 1 << 1; +export const PARENT = 1 << 2; +export const ROOT = 1 << 3; +export const KEYED_HASH = 1 << 4; +export const DERIVE_KEY_CONTEXT = 1 << 5; +export const DERIVE_KEY_MATERIAL = 1 << 6; + +// Size constants +export const OUT_LEN = 32; +export const KEY_LEN = 32; +export const BLOCK_LEN = 64; +export const CHUNK_LEN = 1024; + +// Maximum depth of the CV stack (supports up to 2^54 bytes input) +export const MAX_DEPTH = 54; + +/** + * Precomputed message word permutations for all 7 rounds. + * + * The base permutation is: [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8] + * Each subsequent permutation is the previous one with this permutation applied. + * + * These are the indices into the message block for each round. + * By precomputing these, we avoid runtime permutation overhead. + */ +export const MSG_SCHEDULE: ReadonlyArray> = [ + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8], + [3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1], + [10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6], + [12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4], + [9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7], + [11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13], +]; + +/** + * Flattened permutation table for compress function optimization. + * This enables direct indexed access: PERMUTATIONS[round * 16 + index] + */ +export const PERMUTATIONS = new Uint8Array([ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, + 14, 15, 8, 3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1, 10, 7, 12, 9, 14, 3, 13, 15, 4, + 0, 11, 2, 5, 8, 1, 6, 12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4, 9, 14, 11, 5, 8, 12, + 15, 1, 13, 3, 0, 10, 2, 6, 4, 7, 11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13, +]); diff --git a/node_modules/@huggingface/blake3-jit/src/hash.ts b/node_modules/@huggingface/blake3-jit/src/hash.ts new file mode 100644 index 0000000000000000000000000000000000000000..a7a99e66a7e11adf0e72167b15b37d97a5f5599f --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/src/hash.ts @@ -0,0 +1,1318 @@ +/** + * BLAKE3 Hash Function - Simple one-shot API + * + * This provides a simple hash() function optimized for different input sizes. + * For small inputs, uses pure JS. For large inputs, uses WASM SIMD. + */ + +import { compress } from "./compress.js"; +import { + IV, + CHUNK_START, + CHUNK_END, + PARENT, + ROOT, + BLOCK_LEN, + CHUNK_LEN, + OUT_LEN, +} from "./constants.js"; +import { + IS_LITTLE_ENDIAN, + readLittleEndianWordsFull, + readLittleEndianWordsPartial, + writeLittleEndianBytesPartial, +} from "./utils.js"; +import { + initSimdSync, + getSimdMemory, + getArenaBuffers, + runCompress4x, + runCompressChunks4x, + runCompressParent, + SIMD_MEMORY, +} from "./wasm-simd.js"; + +// Pre-allocated buffers for reuse (single-threaded optimization) +let blockWords: Uint32Array | null = null; + +// ===== Contiguous Hyper CV Stack (Optimization #6) ===== +// Maximum tree depth for practical inputs (2^64 chunks = essentially unlimited) +// Fixed allocation at module load - no runtime allocation +const CV_STACK_DEPTH = 64; +const HYPER_CV_STACK = new Uint32Array(CV_STACK_DEPTH * 8); // 64 CVs × 8 words = 512 words + +// Pre-computed offsets for the first few stack levels (hot path optimization) +// Note: These can be used for further optimization if needed +// const CV_STACK_OFF_0 = 0; +// const CV_STACK_OFF_1 = 8; +// const CV_STACK_OFF_2 = 16; +// const CV_STACK_OFF_3 = 24; + +// ===== Pre-allocated CV Pool with Views (avoids subarray() in hot paths) ===== +const CV_POOL_SIZE = 64; +const CV_POOL = new Uint32Array(CV_POOL_SIZE * 8); // 64 CVs × 8 words = 512 words +const CV_VIEWS: Uint32Array[] = []; +for (let i = 0; i < CV_POOL_SIZE; i++) { + CV_VIEWS.push(CV_POOL.subarray(i * 8, i * 8 + 8)); +} + +// SIMD initialization state +let simdAvailable = false; + +// Threshold for switching to SIMD (must be > 1 chunk to benefit from parallelism) +const SIMD_THRESHOLD = 4 * CHUNK_LEN; // 4KB - need at least 4 chunks for SIMD benefit + +/** + * Initialize SIMD synchronously (lazy). + */ +function ensureSimdSync(): boolean { + if (simdAvailable) return true; + simdAvailable = initSimdSync(); + return simdAvailable; +} + +// Reusable buffer for SIMD chunk CVs (4 chunks × 8 words) +const simdChunkCvs = new Uint32Array(32); + +// ===== Module-level reusable buffers (single-threaded safe) ===== +// These eliminate heap allocations in hot paths + +// For hashChunkWithWords() and hashChunkRoot() +const reusableTempCv = new Uint32Array(8); + +// For hashPureJS() +const reusableChunkCv = new Uint32Array(8); +const reusablePureParentBlock = new Uint32Array(16); +const reusablePureParentCv = new Uint32Array(8); + +// For hashSimd() - use flat array for 4 chunk CVs (access via subarray) +const reusableSimdCvs = new Uint32Array(32); // 4 × 8 words flat + +// For hashSimd() parent compression +const reusableSimdParentBlock = new Uint32Array(16); +const reusableSimdParentCv = new Uint32Array(8); + +// For hashSimd() parameters - TypedArrays instead of JS arrays +const reusableOffsets = new Uint32Array(4); +const reusableCounters = new Uint32Array(4); +const reusableBlockLens = new Uint32Array(4); +const reusableFlags = new Uint32Array(4); + +// Reusable output buffer for common 32-byte hash (eliminates allocations) +const reusableOut8 = new Uint32Array(8); // Standard 32-byte output +// Pre-created view to avoid allocation in hot path (Task 1 optimization) +const reusableOut8View = new Uint8Array(reusableOut8.buffer, 0, 32); + +// ===== Unrolled CV Copy Helper (Task 7 optimization) ===== +// V8 will inline this - avoids loop overhead in hot paths +function copyCV8(src: Uint32Array, srcOff: number, dst: Uint32Array, dstOff: number): void { + dst[dstOff] = src[srcOff]; + dst[dstOff + 1] = src[srcOff + 1]; + dst[dstOff + 2] = src[srcOff + 2]; + dst[dstOff + 3] = src[srcOff + 3]; + dst[dstOff + 4] = src[srcOff + 4]; + dst[dstOff + 5] = src[srcOff + 5]; + dst[dstOff + 6] = src[srcOff + 6]; + dst[dstOff + 7] = src[srcOff + 7]; +} + +/** + * Transpose 4 blocks (64 bytes each) into SIMD memory layout. + * The SIMD compress4x expects: [m0_0,m0_1,m0_2,m0_3, m1_0,m1_1,m1_2,m1_3, ...] + * where m{i}_{j} is message word i from block j. + * + * OPTIMIZED: Processes all 4 blocks together for each word position, + * writing 4 consecutive u32s at once for better cache locality. + * + * @param inputWords - Pre-created Uint32Array view of input (null if unaligned/non-LE). + * Created once per hash call to avoid allocation in hot loop. + */ +function transposeBlocksToSimd( + input: Uint8Array, + offsets: Uint32Array, // Starting offsets for each of 4 blocks + blockLens: Uint32Array, // Length of each block (0-64 bytes) + mem32: Uint32Array, + blockCount: number, // 1-4 blocks + inputWords: Uint32Array | null, // Pre-created view passed from caller +): void { + // Fast path: all blocks are full 64-byte blocks with aligned LE input + const allFull = + blockCount === 4 && + blockLens[0] === 64 && + blockLens[1] === 64 && + blockLens[2] === 64 && + blockLens[3] === 64; + + if ( + allFull && + inputWords && + offsets[0] % 4 === 0 && + offsets[1] % 4 === 0 && + offsets[2] % 4 === 0 && + offsets[3] % 4 === 0 + ) { + // Ultra-fast path: process all 4 blocks together, write 4 consecutive u32s per word + const wordOff0 = offsets[0] >>> 2; + const wordOff1 = offsets[1] >>> 2; + const wordOff2 = offsets[2] >>> 2; + const wordOff3 = offsets[3] >>> 2; + + for (let w = 0; w < 16; w++) { + const dstBase = w * 4; + mem32[dstBase] = inputWords[wordOff0 + w]; + mem32[dstBase + 1] = inputWords[wordOff1 + w]; + mem32[dstBase + 2] = inputWords[wordOff2 + w]; + mem32[dstBase + 3] = inputWords[wordOff3 + w]; + } + return; + } + + // Standard path: process each block independently (handles partial blocks) + for (let b = 0; b < blockCount; b++) { + const len = blockLens[b]; + const off = offsets[b]; + + if (len === 64) { + // Full block + if (inputWords && off % 4 === 0) { + // Direct Uint32Array access for aligned LE blocks + const wordOff = off >>> 2; + for (let w = 0; w < 16; w++) { + mem32[w * 4 + b] = inputWords[wordOff + w]; + } + } else { + // Byte-by-byte reconstruction + for (let w = 0; w < 16; w++) { + const srcOff = off + w * 4; + mem32[w * 4 + b] = + input[srcOff] | + (input[srcOff + 1] << 8) | + (input[srcOff + 2] << 16) | + (input[srcOff + 3] << 24); + } + } + } else if (len === 0) { + // Zero block + for (let w = 0; w < 16; w++) { + mem32[w * 4 + b] = 0; + } + } else { + // Partial block - handle word by word + for (let w = 0; w < 16; w++) { + const wordOff = w * 4; + if (wordOff >= len) { + mem32[w * 4 + b] = 0; + } else if (wordOff + 4 <= len) { + const srcOff = off + wordOff; + mem32[w * 4 + b] = + input[srcOff] | + (input[srcOff + 1] << 8) | + (input[srcOff + 2] << 16) | + (input[srcOff + 3] << 24); + } else { + // Partial word at end of block + let word = 0; + for (let i = 0; i < len - wordOff; i++) { + word |= input[off + wordOff + i] << (i * 8); + } + mem32[w * 4 + b] = word; + } + } + } + } + + // Zero unused block slots + for (let b = blockCount; b < 4; b++) { + for (let w = 0; w < 16; w++) { + mem32[w * 4 + b] = 0; + } + } +} + +/** + * Transpose 4 full chunks (4 × 16 blocks = 64 blocks) into batch SIMD memory. + * This is used for the batched compressChunks4x function that processes + * all 16 blocks in a single WASM call. + * + * Memory layout: BATCH_BLOCK_WORDS has 16 positions, each with 16 v128 values. + * Position p, word w: mem32[(p * 64) + (w * 4) + lane] + * + * OPTIMIZED: Processes all 4 chunks together for each (pos, word) pair, + * writing 4 consecutive u32s at once for better cache locality. + * + * @param input - Input data (must have at least 4 full chunks = 4096 bytes) + * @param chunkOffsets - Starting offsets for each of 4 chunks + * @param mem32 - WASM memory view + * @param inputWords - Pre-created Uint32Array view (null if unaligned) + */ +function transposeBatchToSimd( + input: Uint8Array, + chunkOffsets: Uint32Array, + mem32: Uint32Array, + inputWords: Uint32Array | null, +): void { + const BATCH_BASE = SIMD_MEMORY.BATCH_BLOCK_WORDS / 4; + + // Get base word offsets for each chunk (pre-computed for fast path) + const chunk0WordBase = chunkOffsets[0] >>> 2; + const chunk1WordBase = chunkOffsets[1] >>> 2; + const chunk2WordBase = chunkOffsets[2] >>> 2; + const chunk3WordBase = chunkOffsets[3] >>> 2; + + // Fast path: all chunks aligned and LE - process 4 consecutive u32s at once + if (inputWords && chunkOffsets[0] % 4 === 0) { + for (let pos = 0; pos < 16; pos++) { + const posBase = BATCH_BASE + pos * 64; // 16 words × 4 lanes = 64 + const blockWordOff = pos * 16; // 16 words per block (64 bytes / 4) + + // Process all 16 words, writing 4 chunks at a time (cache-friendly: 16 bytes per write group) + for (let w = 0; w < 16; w++) { + const dstBase = posBase + w * 4; + // Read word w from all 4 chunks at positions that become consecutive in output + mem32[dstBase] = inputWords[chunk0WordBase + blockWordOff + w]; + mem32[dstBase + 1] = inputWords[chunk1WordBase + blockWordOff + w]; + mem32[dstBase + 2] = inputWords[chunk2WordBase + blockWordOff + w]; + mem32[dstBase + 3] = inputWords[chunk3WordBase + blockWordOff + w]; + } + } + } else { + // Slow path: byte-by-byte reconstruction, still cache-friendly write pattern + for (let pos = 0; pos < 16; pos++) { + const posBase = BATCH_BASE + pos * 64; + const blockByteOff = pos * 64; // 64 bytes per block + + for (let w = 0; w < 16; w++) { + const dstBase = posBase + w * 4; + const wordByteOff = w * 4; + + // Chunk 0 + const off0 = chunkOffsets[0] + blockByteOff + wordByteOff; + mem32[dstBase] = + input[off0] | (input[off0 + 1] << 8) | (input[off0 + 2] << 16) | (input[off0 + 3] << 24); + + // Chunk 1 + const off1 = chunkOffsets[1] + blockByteOff + wordByteOff; + mem32[dstBase + 1] = + input[off1] | (input[off1 + 1] << 8) | (input[off1 + 2] << 16) | (input[off1 + 3] << 24); + + // Chunk 2 + const off2 = chunkOffsets[2] + blockByteOff + wordByteOff; + mem32[dstBase + 2] = + input[off2] | (input[off2 + 1] << 8) | (input[off2 + 2] << 16) | (input[off2 + 3] << 24); + + // Chunk 3 + const off3 = chunkOffsets[3] + blockByteOff + wordByteOff; + mem32[dstBase + 3] = + input[off3] | (input[off3 + 1] << 8) | (input[off3 + 2] << 16) | (input[off3 + 3] << 24); + } + } + } +} + +// Pre-computed memory offsets for SIMD operations (single-block mode) +const SIMD_CV_BASE = SIMD_MEMORY.CHAINING_VALUES / 4; +const SIMD_OUT_BASE = SIMD_MEMORY.OUTPUT / 4; +const SIMD_COUNTER_LOW_BASE = SIMD_MEMORY.COUNTER_LOW / 4; +const SIMD_COUNTER_HIGH_BASE = SIMD_MEMORY.COUNTER_HIGH / 4; +const SIMD_BLOCK_LEN_BASE = SIMD_MEMORY.BLOCK_LEN / 4; + +// Pre-computed memory offsets for batch SIMD operations (16-block mode) +const BATCH_CV_BASE = SIMD_MEMORY.BATCH_CV / 4; +const BATCH_COUNTER_LOW_BASE = SIMD_MEMORY.BATCH_COUNTER_LOW / 4; +const BATCH_FLAGS_BASE_OFFSET = SIMD_MEMORY.BATCH_FLAGS_BASE / 4; +const BATCH_OUTPUT_BASE = SIMD_MEMORY.BATCH_OUTPUT / 4; + +// Reusable arrays for batch processing +const batchChunkOffsets = new Uint32Array(4); +const SIMD_FLAGS_BASE = SIMD_MEMORY.FLAGS / 4; + +/** + * Set up chaining values in SIMD memory (transposed layout). + * Optimized: unrolled loops for common case of 4 chunks. + * cvs is flat: [cv0_word0..cv0_word7, cv1_word0..cv1_word7, ...] + */ +function setupSimdCvs( + cvs: Uint32Array, // Flat array: 4 × 8 words + mem32: Uint32Array, + count: number, +): void { + // Unrolled for 4 chunks (common case) + if (count === 4) { + for (let w = 0; w < 8; w++) { + const base = SIMD_CV_BASE + w * 4; + mem32[base] = cvs[w]; // cv0[w] + mem32[base + 1] = cvs[8 + w]; // cv1[w] + mem32[base + 2] = cvs[16 + w]; // cv2[w] + mem32[base + 3] = cvs[24 + w]; // cv3[w] + } + } else { + for (let w = 0; w < 8; w++) { + const base = SIMD_CV_BASE + w * 4; + for (let c = 0; c < count; c++) { + mem32[base + c] = cvs[c * 8 + w]; + } + for (let c = count; c < 4; c++) { + mem32[base + c] = 0; + } + } + } +} + +/** + * Set up SIMD parameters (counters, flags, block lengths). + */ +function setupSimdParams( + mem32: Uint32Array, + counters: Uint32Array, + blockLens: Uint32Array, + flagsArr: Uint32Array, + count: number, +): void { + // Most chunk counters fit in 32 bits, so counter high is usually 0 + for (let i = 0; i < count; i++) { + mem32[SIMD_COUNTER_LOW_BASE + i] = counters[i]; + mem32[SIMD_COUNTER_HIGH_BASE + i] = 0; // Assume counters fit in 32 bits + mem32[SIMD_BLOCK_LEN_BASE + i] = blockLens[i]; + mem32[SIMD_FLAGS_BASE + i] = flagsArr[i]; + } + // Zero unused slots + for (let i = count; i < 4; i++) { + mem32[SIMD_COUNTER_LOW_BASE + i] = 0; + mem32[SIMD_COUNTER_HIGH_BASE + i] = 0; + mem32[SIMD_BLOCK_LEN_BASE + i] = 0; + mem32[SIMD_FLAGS_BASE + i] = 0; + } +} + +/** + * Read output CVs from SIMD memory (untranspose). + */ +function readSimdOutputCvs( + mem32: Uint32Array, + outputCvs: Uint32Array, // Flat array: 4 × 8 words + count: number, +): void { + // Unrolled for 4 chunks (common case) + if (count === 4) { + for (let w = 0; w < 8; w++) { + const base = SIMD_OUT_BASE + w * 4; + outputCvs[w] = mem32[base]; + outputCvs[8 + w] = mem32[base + 1]; + outputCvs[16 + w] = mem32[base + 2]; + outputCvs[24 + w] = mem32[base + 3]; + } + } else { + for (let w = 0; w < 8; w++) { + const base = SIMD_OUT_BASE + w * 4; + for (let c = 0; c < count; c++) { + outputCvs[c * 8 + w] = mem32[base + c]; + } + } + } +} + +function getBlockWords(): Uint32Array { + if (!blockWords) { + blockWords = new Uint32Array(16); + } + return blockWords; +} + +/** + * Hash a single chunk (up to 1024 bytes) with pre-created inputWords view. + * This is the optimized version that avoids creating Uint32Array views per chunk. + * (Fleek optimization Step 8) + */ +function hashChunkWithWords( + input: Uint8Array, + inputWords: Uint32Array | null, // Pre-created view of entire input + inputOffset: number, + inputLen: number, + chunkCounter: number, + flags: number, + cv: Uint32Array, + cvOffset: number, +): void { + // Use reusable temporary CV for intermediate blocks (single-threaded safe) + reusableTempCv.set(IV); + + // Process full blocks + const fullBlocks = inputLen >>> 6; // inputLen / 64 + const remainder = inputLen & 63; // inputLen % 64 + + // Calculate word offset for this chunk within the pre-created view + const chunkWordOffset = inputOffset >>> 2; + + // Fast path for full chunks with aligned little-endian input + if (inputWords && remainder === 0 && inputLen === CHUNK_LEN) { + // All 16 blocks are full, use fast path exclusively + let wordOff = chunkWordOffset; + // Block 0 (CHUNK_START) + compress( + reusableTempCv, + 0, + inputWords, + wordOff, + reusableTempCv, + 0, + false, + chunkCounter, + BLOCK_LEN, + flags | CHUNK_START, + ); + wordOff += 16; + // Blocks 1-14 (no special flags) + for (let i = 1; i < 15; i++) { + compress( + reusableTempCv, + 0, + inputWords, + wordOff, + reusableTempCv, + 0, + false, + chunkCounter, + BLOCK_LEN, + flags, + ); + wordOff += 16; + } + // Block 15 (CHUNK_END) + compress( + reusableTempCv, + 0, + inputWords, + wordOff, + reusableTempCv, + 0, + false, + chunkCounter, + BLOCK_LEN, + flags | CHUNK_END, + ); + + cv.set(reusableTempCv, cvOffset); + return; + } + + // Slower path for partial chunks or non-aligned input + const totalBlocks = fullBlocks + (remainder > 0 ? 1 : 0); + const block = getBlockWords(); + + for (let blockIdx = 0; blockIdx < totalBlocks; blockIdx++) { + const isFirst = blockIdx === 0; + const isLast = blockIdx === totalBlocks - 1; + const blockStart = blockIdx << 6; + const blockLen = isLast && remainder > 0 ? remainder : BLOCK_LEN; + + // Determine flags for this block + let blockFlags = flags; + if (isFirst) blockFlags |= CHUNK_START; + if (isLast) blockFlags |= CHUNK_END; + + // Load block words + if (isLast && remainder > 0) { + // Partial final block - need zero padding + readLittleEndianWordsPartial(input, inputOffset + blockStart, blockLen, block); + } else if (inputWords && chunkWordOffset + (blockStart >>> 2) + 16 <= inputWords.length) { + // Fast path: use pre-created view directly + compress( + reusableTempCv, + 0, + inputWords, + chunkWordOffset + (blockStart >>> 2), + reusableTempCv, + 0, + false, + chunkCounter, + blockLen, + blockFlags, + ); + continue; + } else { + readLittleEndianWordsFull(input, inputOffset + blockStart, block); + } + + compress( + reusableTempCv, + 0, + block, + 0, + reusableTempCv, + 0, + false, + chunkCounter, + blockLen, + blockFlags, + ); + } + + // Copy result to output + cv.set(reusableTempCv, cvOffset); +} + +/** + * Hash input using pure JavaScript. + * Handles the full Merkle tree construction. + */ +function hashPureJS(input: Uint8Array, outputLen: number): Uint8Array { + const inputLen = input.length; + + // Special case: empty input + if (inputLen === 0) { + const block = getBlockWords(); + block.fill(0); + // Use reusable output buffer for common 32-byte case + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + + compress(IV, 0, block, 0, out, 0, outputLen > 32, 0, 0, CHUNK_START | CHUNK_END | ROOT); + + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } else { + writeLittleEndianBytesPartial(out, 0, result, 0, outputLen); + } + return result; + } + + // Calculate number of chunks + const numChunks = Math.ceil(inputLen / CHUNK_LEN); + + // Single chunk optimization + if (numChunks === 1) { + // Use reusable output buffer for common 32-byte case + const cv = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + hashChunkRoot(input, 0, inputLen, 0, 0, cv, outputLen > 32); + + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(cv.buffer, 0, outputLen)); + } else { + writeLittleEndianBytesPartial(cv, 0, result, 0, outputLen); + } + return result; + } + + // Multiple chunks - need Merkle tree + // Use the global contiguous CV stack (no allocation) + const stack = HYPER_CV_STACK; + let stackLen = 0; + + // Use reusable buffers (single-threaded safe) + const chunkCv = reusableChunkCv; + const parentBlock = reusablePureParentBlock; + const parentCv = reusablePureParentCv; + + // Create Uint32Array view ONCE for entire input (Fleek optimization Step 8) + // This avoids creating views inside each chunk/block processing + let inputWords: Uint32Array | null = null; + const canUseFastPath = IS_LITTLE_ENDIAN && input.byteOffset % 4 === 0; + if (canUseFastPath) { + inputWords = new Uint32Array(input.buffer, input.byteOffset, inputLen >>> 2); + } + + // Determine how many full chunks we have + const fullChunks = inputLen >>> 10; // inputLen / 1024 + const lastChunkLen = inputLen & 1023; // inputLen % 1024 + + // Process all full chunks with fast path (inlined for performance) + if (canUseFastPath && inputWords) { + for (let chunkIdx = 0; chunkIdx < fullChunks; chunkIdx++) { + // Inline chunk processing for full chunks + chunkCv.set(IV); + let wordOff = chunkIdx << 8; // chunkIdx * 256 (CHUNK_LEN/4) + + // Block 0 (CHUNK_START) + compress( + chunkCv, + 0, + inputWords, + wordOff, + chunkCv, + 0, + false, + chunkIdx, + BLOCK_LEN, + CHUNK_START, + ); + wordOff += 16; + // Blocks 1-14 (no special flags) + for (let b = 1; b < 15; b++) { + compress(chunkCv, 0, inputWords, wordOff, chunkCv, 0, false, chunkIdx, BLOCK_LEN, 0); + wordOff += 16; + } + // Block 15 (CHUNK_END) + compress(chunkCv, 0, inputWords, wordOff, chunkCv, 0, false, chunkIdx, BLOCK_LEN, CHUNK_END); + + // Merge completed subtrees (avoid subarray by using index math) + let totalChunks = chunkIdx + 1; + let cvSrcOff = 0; + let cvSrc = chunkCv; + + // Check if this is the last chunk overall + const isLastChunk = chunkIdx === fullChunks - 1 && lastChunkLen === 0; + + while ((totalChunks & 1) === 0 && stackLen > 0) { + // Skip final merge if it would produce the root; let finalization handle it with ROOT flag + if (stackLen === 1 && isLastChunk) { + break; + } + stackLen--; + const stackOff = stackLen * 8; + // Copy left CV from stack to parentBlock[0..7] (unrolled) + copyCV8(stack, stackOff, parentBlock, 0); + // Copy current CV to parentBlock[8..15] (unrolled) + copyCV8(cvSrc, cvSrcOff, parentBlock, 8); + + compress(IV, 0, parentBlock, 0, parentCv, 0, false, 0, BLOCK_LEN, PARENT); + cvSrc = parentCv; + cvSrcOff = 0; + totalChunks >>>= 1; + } + + // Push CV to stack (unrolled) + const stackOff = stackLen * 8; + copyCV8(cvSrc, cvSrcOff, stack, stackOff); + stackLen++; + } + + // Process last partial chunk if any + if (lastChunkLen > 0) { + hashChunkWithWords( + input, + inputWords, + fullChunks * CHUNK_LEN, + lastChunkLen, + fullChunks, + 0, + chunkCv, + 0, + ); + + let totalChunks = fullChunks + 1; + let newCv = chunkCv; + let newCvOffset = 0; + + while ((totalChunks & 1) === 0 && stackLen > 0) { + // Skip final merge; this IS the last chunk, let finalization handle ROOT flag + if (stackLen === 1) { + break; + } + stackLen--; + const stackOff = stackLen * 8; + // Copy from stack to parentBlock[0..7] (unrolled) + copyCV8(stack, stackOff, parentBlock, 0); + // Copy from newCv to parentBlock[8..15] (unrolled) + copyCV8(newCv, newCvOffset, parentBlock, 8); + compress(IV, 0, parentBlock, 0, parentCv, 0, false, 0, BLOCK_LEN, PARENT); + newCv = parentCv; + newCvOffset = 0; + totalChunks >>>= 1; + } + + // Push CV to stack (unrolled) + const pushOff = stackLen * 8; + copyCV8(newCv, newCvOffset, stack, pushOff); + stackLen++; + } + } else { + // Slow path for unaligned or big-endian + for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) { + const chunkStart = chunkIdx * CHUNK_LEN; + const chunkLen = Math.min(CHUNK_LEN, inputLen - chunkStart); + + hashChunkWithWords(input, inputWords, chunkStart, chunkLen, chunkIdx, 0, chunkCv, 0); + + // Merge completed subtrees + let totalChunks = chunkIdx + 1; + let newCv = chunkCv; + let newCvOffset = 0; + + // Check if this is the last chunk + const isLastChunk = chunkIdx === numChunks - 1; + + while ((totalChunks & 1) === 0 && stackLen > 0) { + // Skip final merge if it would produce the root; let finalization handle it with ROOT flag + if (stackLen === 1 && isLastChunk) { + break; + } + stackLen--; + const stackOff = stackLen * 8; + // Copy from stack to parentBlock[0..7] (unrolled) + copyCV8(stack, stackOff, parentBlock, 0); + // Copy from newCv to parentBlock[8..15] (unrolled) + copyCV8(newCv, newCvOffset, parentBlock, 8); + compress(IV, 0, parentBlock, 0, parentCv, 0, false, 0, BLOCK_LEN, PARENT); + newCv = parentCv; + newCvOffset = 0; + totalChunks >>>= 1; + } + + // Push CV to stack (unrolled) + const pushOff = stackLen * 8; + copyCV8(newCv, newCvOffset, stack, pushOff); + stackLen++; + } + } + + // Finalize: merge remaining stack entries + while (stackLen > 1) { + stackLen--; + const rightOff = stackLen * 8; + stackLen--; + const leftOff = stackLen * 8; + // Copy left CV to parentBlock[0..7] and right CV to parentBlock[8..15] (unrolled) + copyCV8(stack, leftOff, parentBlock, 0); + copyCV8(stack, rightOff, parentBlock, 8); + + if (stackLen === 0) { + // This is the root - use reusable output buffer for common 32-byte case + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + compress(IV, 0, parentBlock, 0, out, 0, outputLen > 32, 0, BLOCK_LEN, PARENT | ROOT); + + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } else { + writeLittleEndianBytesPartial(out, 0, result, 0, outputLen); + } + return result; + } + + compress(IV, 0, parentBlock, 0, parentCv, 0, false, 0, BLOCK_LEN, PARENT); + + // Push to stack (unrolled) + copyCV8(parentCv, 0, stack, stackLen * 8); + stackLen++; + } + + // Single entry in stack - this is the root + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + const lastBlock = getBlockWords(); + lastBlock.fill(0); + // Copy first 8 words from stack (unrolled) + copyCV8(stack, 0, lastBlock, 0); + + compress(IV, 0, lastBlock, 0, out, 0, outputLen > 32, 0, BLOCK_LEN, ROOT); + + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } else { + writeLittleEndianBytesPartial(out, 0, result, 0, outputLen); + } + return result; +} + +/** + * Hash a single chunk that is also the root (single chunk input). + */ +function hashChunkRoot( + input: Uint8Array, + inputOffset: number, + inputLen: number, + chunkCounter: number, + flags: number, + out: Uint32Array, + fullOutput: boolean, +): void { + // Use reusable tempCv (single-threaded safe) + reusableTempCv.set(IV); + + const block = getBlockWords(); + + // Process full blocks + const fullBlocks = inputLen >>> 6; + const remainder = inputLen & 63; + const totalBlocks = fullBlocks + (remainder > 0 ? 1 : 0) || 1; // At least 1 block + + // Create a Uint32Array view if possible + let inputWords: Uint32Array | null = null; + if (IS_LITTLE_ENDIAN && (input.byteOffset + inputOffset) % 4 === 0 && inputLen >= 4) { + inputWords = new Uint32Array(input.buffer, input.byteOffset + inputOffset, inputLen >>> 2); + } + + for (let blockIdx = 0; blockIdx < totalBlocks; blockIdx++) { + const isFirst = blockIdx === 0; + const isLast = blockIdx === totalBlocks - 1; + const blockStart = blockIdx << 6; + const blockLen = isLast ? remainder || (inputLen > 0 ? BLOCK_LEN : 0) : BLOCK_LEN; + + // Determine flags + let blockFlags = flags; + if (isFirst) blockFlags |= CHUNK_START; + if (isLast) blockFlags |= CHUNK_END | ROOT; + + // Load block + if (isLast && remainder > 0) { + readLittleEndianWordsPartial(input, inputOffset + blockStart, blockLen, block); + } else if (inputLen === 0) { + block.fill(0); + } else if (inputWords && (blockStart >>> 2) + 16 <= inputWords.length) { + // Fast path + compress( + reusableTempCv, + 0, + inputWords, + blockStart >>> 2, + isLast ? out : reusableTempCv, + 0, + isLast && fullOutput, + chunkCounter, + blockLen, + blockFlags, + ); + continue; + } else { + readLittleEndianWordsFull(input, inputOffset + blockStart, block); + } + + compress( + reusableTempCv, + 0, + block, + 0, + isLast ? out : reusableTempCv, + 0, + isLast && fullOutput, + chunkCounter, + blockLen, + blockFlags, + ); + } +} + +/** + * Hash using WASM SIMD - processes 4 chunks in parallel. + * Falls back to pure JS if SIMD fails. + */ +function hashSimd(input: Uint8Array, outputLen: number): Uint8Array { + const mem = getSimdMemory(); + if (!mem) { + return hashPureJS(input, outputLen); + } + + const { view32 } = mem; + const inputLen = input.length; + const numChunks = Math.ceil(inputLen / CHUNK_LEN); + + // For small inputs, pure JS is faster (no transpose overhead) + if (numChunks < 4) { + return hashPureJS(input, outputLen); + } + + // Try to use WASM arena buffers (zero JS heap allocation) + // Falls back to JS buffers if arena not available + const arena = getArenaBuffers(); + const useWasmParent = arena !== null; // Use WASM parent compress when arena available + let stack: Uint32Array; + let tempCvs: Uint32Array; + let parentBlock: Uint32Array; + let parentCv: Uint32Array; + + if (arena) { + // Use WASM-backed arena buffers + stack = arena.cvStack; + tempCvs = arena.tempCvs; + parentBlock = arena.parentBlock; + parentCv = arena.chunkCv; + } else { + // Fallback to JS heap buffers - use global contiguous stack (no allocation) + stack = HYPER_CV_STACK; + tempCvs = reusableSimdCvs; + parentBlock = reusableSimdParentBlock; + parentCv = reusableSimdParentCv; + } + + let stackLen = 0; + + // Use TypedArrays instead of JS arrays for block parameters + const offsets = reusableOffsets; + const counters = reusableCounters; + const blockLens = reusableBlockLens; + const flagsArr = reusableFlags; + + // Create Uint32Array view once for entire hash call (optimization: avoid allocation in hot loop) + const inputWords = + IS_LITTLE_ENDIAN && input.byteOffset % 4 === 0 + ? new Uint32Array(input.buffer, input.byteOffset, input.byteLength >>> 2) + : null; + + // Calculate number of full chunks (1024 bytes each) + const numFullChunks = inputLen >>> 10; // inputLen / 1024 + + // Process chunks in groups of 4 + let chunkIdx = 0; + while (chunkIdx < numChunks) { + const groupSize = Math.min(4, numChunks - chunkIdx); + + // === BATCH FAST PATH: 4 full chunks === + // Use compressChunks4x for groups of exactly 4 full chunks + // This reduces 16 WASM calls to 1 per group + const canUseBatchPath = groupSize === 4 && chunkIdx + 4 <= numFullChunks; + + if (canUseBatchPath) { + // Set up chunk offsets for batch transpose + batchChunkOffsets[0] = chunkIdx * CHUNK_LEN; + batchChunkOffsets[1] = (chunkIdx + 1) * CHUNK_LEN; + batchChunkOffsets[2] = (chunkIdx + 2) * CHUNK_LEN; + batchChunkOffsets[3] = (chunkIdx + 3) * CHUNK_LEN; + + // Transpose all 64 blocks (4 chunks × 16 blocks) at once + transposeBatchToSimd(input, batchChunkOffsets, view32, inputWords); + + // Set up initial CVs (IV) in batch memory - transposed layout + for (let w = 0; w < 8; w++) { + const ivWord = IV[w]; + const base = BATCH_CV_BASE + w * 4; + view32[base] = ivWord; + view32[base + 1] = ivWord; + view32[base + 2] = ivWord; + view32[base + 3] = ivWord; + } + + // Set up counters in batch memory + view32[BATCH_COUNTER_LOW_BASE] = chunkIdx; + view32[BATCH_COUNTER_LOW_BASE + 1] = chunkIdx + 1; + view32[BATCH_COUNTER_LOW_BASE + 2] = chunkIdx + 2; + view32[BATCH_COUNTER_LOW_BASE + 3] = chunkIdx + 3; + + // Set up base flags (0 - no keyed hashing) + view32[BATCH_FLAGS_BASE_OFFSET] = 0; + view32[BATCH_FLAGS_BASE_OFFSET + 1] = 0; + view32[BATCH_FLAGS_BASE_OFFSET + 2] = 0; + view32[BATCH_FLAGS_BASE_OFFSET + 3] = 0; + + // Run batched compress (16 blocks × 4 chunks in one call!) + runCompressChunks4x(); + + // Read output CVs from batch output - untranspose to tempCvs + for (let w = 0; w < 8; w++) { + const base = BATCH_OUTPUT_BASE + w * 4; + tempCvs[w] = view32[base]; // chunk 0 + tempCvs[8 + w] = view32[base + 1]; // chunk 1 + tempCvs[16 + w] = view32[base + 2]; // chunk 2 + tempCvs[24 + w] = view32[base + 3]; // chunk 3 + } + } else { + // === STANDARD PATH: block-by-block processing === + // Used for partial chunks or groups < 4 + + // Initialize CVs for this group to IV (flat array: 4 × 8 words) + for (let g = 0; g < groupSize; g++) { + const base = g * 8; + tempCvs[base] = IV[0]; + tempCvs[base + 1] = IV[1]; + tempCvs[base + 2] = IV[2]; + tempCvs[base + 3] = IV[3]; + tempCvs[base + 4] = IV[4]; + tempCvs[base + 5] = IV[5]; + tempCvs[base + 6] = IV[6]; + tempCvs[base + 7] = IV[7]; + } + + // Process all 16 blocks of each chunk in this group + for (let blockIdx = 0; blockIdx < 16; blockIdx++) { + // Calculate block offsets and parameters (reuse arrays) + + for (let g = 0; g < groupSize; g++) { + const thisChunkIdx = chunkIdx + g; + const chunkStart = thisChunkIdx * CHUNK_LEN; + const chunkLen = Math.min(CHUNK_LEN, inputLen - chunkStart); + const thisBlockStart = chunkStart + blockIdx * BLOCK_LEN; + + // Determine block length for this specific block + const blockStartInChunk = blockIdx * BLOCK_LEN; + let thisBlockLen = BLOCK_LEN; + if (blockStartInChunk >= chunkLen) { + thisBlockLen = 0; + } else if (blockStartInChunk + BLOCK_LEN > chunkLen) { + thisBlockLen = chunkLen - blockStartInChunk; + } + + offsets[g] = thisBlockStart; + counters[g] = thisChunkIdx; + + // Determine flags + let flags = 0; + if (blockIdx === 0) flags |= CHUNK_START; + const totalBlocksInChunk = Math.ceil(chunkLen / BLOCK_LEN) || 1; + if (blockIdx === totalBlocksInChunk - 1) flags |= CHUNK_END; + + blockLens[g] = thisBlockLen; + flagsArr[g] = flags; + } + + // Check if any blocks need processing + if (blockLens[0] === 0 && blockLens[1] === 0 && blockLens[2] === 0 && blockLens[3] === 0) + continue; + + // Transpose blocks into SIMD memory (pass pre-created view to avoid allocation) + transposeBlocksToSimd(input, offsets, blockLens, view32, groupSize, inputWords); + + // Set up CVs in SIMD memory + setupSimdCvs(tempCvs, view32, groupSize); + + // Set up parameters + setupSimdParams(view32, counters, blockLens, flagsArr, groupSize); + + // Run SIMD compress + runCompress4x(); + + // Read output CVs back + readSimdOutputCvs(view32, simdChunkCvs, groupSize); + + // Update tempCvs - copy from simdChunkCvs (both are flat 32-word arrays) + // simdChunkCvs layout matches tempCvs: [cv0_w0..cv0_w7, cv1_w0..cv1_w7, ...] + // IMPORTANT: Only update CVs for chunks that had data in this block! + // Skipping this check would corrupt CVs for partial chunks after their final block. + for (let g = 0; g < groupSize; g++) { + if (blockLens[g] === 0) continue; // Don't update CV for chunks with no data in this block + const base = g * 8; + tempCvs[base] = simdChunkCvs[base]; + tempCvs[base + 1] = simdChunkCvs[base + 1]; + tempCvs[base + 2] = simdChunkCvs[base + 2]; + tempCvs[base + 3] = simdChunkCvs[base + 3]; + tempCvs[base + 4] = simdChunkCvs[base + 4]; + tempCvs[base + 5] = simdChunkCvs[base + 5]; + tempCvs[base + 6] = simdChunkCvs[base + 6]; + tempCvs[base + 7] = simdChunkCvs[base + 7]; + } + } + } + + // Merge each chunk's CV into the Merkle tree + for (let g = 0; g < groupSize; g++) { + const thisChunkIdx = chunkIdx + g; + + // Merge completed subtrees + let totalChunks = thisChunkIdx + 1; + // Track newCv source - either from tempCvs or parentCv + let newCvBase = g * 8; // Offset into tempCvs + let newCvSrc = tempCvs; + + // Check if this is the last chunk + const isLastChunk = thisChunkIdx === numChunks - 1; + + while ((totalChunks & 1) === 0 && stackLen > 0) { + // Skip final merge if it would produce the root; let finalization handle it with ROOT flag + if (stackLen === 1 && isLastChunk) { + break; + } + // Pop left child + stackLen--; + const stackOff = stackLen * 8; + // Copy from stack to parentBlock[0..7] (unrolled) + copyCV8(stack, stackOff, parentBlock, 0); + // Copy from newCv source to parentBlock[8..15] (unrolled) + copyCV8(newCvSrc, newCvBase, parentBlock, 8); + + if (useWasmParent) { + // WASM parent compress - data already in arena buffers + runCompressParent(); + } else { + compress(IV, 0, parentBlock, 0, parentCv, 0, false, 0, BLOCK_LEN, PARENT); + } + + newCvSrc = parentCv; + newCvBase = 0; + totalChunks >>>= 1; + } + + // Push to stack (unrolled) + const pushOff = stackLen * 8; + copyCV8(newCvSrc, newCvBase, stack, pushOff); + stackLen++; + } + + chunkIdx += groupSize; + } + + // Finalize: merge remaining stack entries + while (stackLen > 1) { + stackLen--; + const rightOff = stackLen * 8; + stackLen--; + const leftOff = stackLen * 8; + // Copy left CV to parentBlock[0..7] and right CV to parentBlock[8..15] (unrolled) + copyCV8(stack, leftOff, parentBlock, 0); + copyCV8(stack, rightOff, parentBlock, 8); + + if (stackLen === 0) { + // This is the root - use reusable output buffer + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + compress(IV, 0, parentBlock, 0, out, 0, outputLen > 32, 0, BLOCK_LEN, PARENT | ROOT); + + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } else { + writeLittleEndianBytesPartial(out, 0, result, 0, outputLen); + } + return result; + } + + if (useWasmParent) { + // WASM parent compress - data already in arena buffers + runCompressParent(); + } else { + compress(IV, 0, parentBlock, 0, parentCv, 0, false, 0, BLOCK_LEN, PARENT); + } + + // Push to stack (unrolled) + copyCV8(parentCv, 0, stack, stackLen * 8); + stackLen++; + } + + // Single entry in stack - finalize as root + if (stackLen === 1) { + const block = getBlockWords(); + block.fill(0); + // Copy first 8 words from stack (unrolled) + copyCV8(stack, 0, block, 0); + + // Use reusable output buffer + const out = outputLen === 32 ? reusableOut8 : new Uint32Array(outputLen > 32 ? 16 : 8); + compress(IV, 0, block, 0, out, 0, outputLen > 32, 0, BLOCK_LEN, ROOT); + + // Return result - use pre-created view for common 32-byte case + if (outputLen === 32 && IS_LITTLE_ENDIAN) { + return reusableOut8View.slice(); + } + const result = new Uint8Array(outputLen); + if (IS_LITTLE_ENDIAN) { + result.set(new Uint8Array(out.buffer, 0, outputLen)); + } else { + writeLittleEndianBytesPartial(out, 0, result, 0, outputLen); + } + return result; + } + + // Should not reach here + return hashPureJS(input, outputLen); +} + +/** + * Hash input data and return the result. + * Automatically uses WASM SIMD for large inputs when available. + * + * @param input - Data to hash + * @param outputLength - Number of bytes to output (default: 32) + * @returns The hash output + */ +export function hash(input: Uint8Array, outputLength: number = OUT_LEN): Uint8Array { + // For large inputs, use SIMD for ~1.5x performance improvement + if (input.length >= SIMD_THRESHOLD && ensureSimdSync()) { + return hashSimd(input, outputLength); + } + return hashPureJS(input, outputLength); +} + +/** + * Pre-warm SIMD initialization (call early to avoid latency later). + */ +export function warmupSimd(): boolean { + return ensureSimdSync(); +} + +/** + * Hash input data directly into a caller-provided output buffer. + * Zero-allocation for the common 32-byte case - ideal for performance-critical code. + * + * @param input - Data to hash + * @param output - Pre-allocated output buffer (must be at least outputLength bytes) + * @param outputLength - Number of bytes to output (default: 32, max: output.length) + */ +export function hashInto( + input: Uint8Array, + output: Uint8Array, + outputLength: number = OUT_LEN, +): void { + // Validate output buffer + if (output.length < outputLength) { + throw new Error(`Output buffer too small: ${output.length} < ${outputLength}`); + } + + // For large inputs, use SIMD for ~1.5x performance improvement + if (input.length >= SIMD_THRESHOLD && ensureSimdSync()) { + hashSimdInto(input, output, outputLength); + return; + } + + hashPureJSInto(input, output, outputLength); +} + +/** + * Internal: Hash using pure JS, writing directly to output buffer. + */ +function hashPureJSInto(input: Uint8Array, output: Uint8Array, outputLen: number): void { + const inputLen = input.length; + + // Special case: empty input + if (inputLen === 0) { + const block = getBlockWords(); + block.fill(0); + const out = outputLen <= 32 ? reusableOut8 : new Uint32Array(16); + + compress(IV, 0, block, 0, out, 0, outputLen > 32, 0, 0, CHUNK_START | CHUNK_END | ROOT); + + // Copy result to output + if (IS_LITTLE_ENDIAN) { + output.set(new Uint8Array(out.buffer, out.byteOffset, outputLen)); + } else { + writeLittleEndianBytesPartial(out, 0, output, 0, outputLen); + } + return; + } + + // Calculate number of chunks + const numChunks = Math.ceil(inputLen / CHUNK_LEN); + + // Single chunk optimization + if (numChunks === 1) { + const cv = outputLen <= 32 ? reusableOut8 : new Uint32Array(16); + hashChunkRoot(input, 0, inputLen, 0, 0, cv, outputLen > 32); + + // Copy result to output + if (IS_LITTLE_ENDIAN) { + output.set(new Uint8Array(cv.buffer, cv.byteOffset, outputLen)); + } else { + writeLittleEndianBytesPartial(cv, 0, output, 0, outputLen); + } + return; + } + + // Multiple chunks - delegate to hashPureJS and copy result + const result = hashPureJS(input, outputLen); + output.set(result); +} + +/** + * Internal: Hash using SIMD, writing directly to output buffer. + */ +function hashSimdInto(input: Uint8Array, output: Uint8Array, outputLen: number): void { + // Delegate to hashSimd and copy result (SIMD path already optimized) + const result = hashSimd(input, outputLen); + output.set(result); +} diff --git a/node_modules/@huggingface/blake3-jit/src/hasher.ts b/node_modules/@huggingface/blake3-jit/src/hasher.ts new file mode 100644 index 0000000000000000000000000000000000000000..e0865a5e154a619706a49ab4c5c3866ccd855b19 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/src/hasher.ts @@ -0,0 +1,576 @@ +/** + * BLAKE3 Hasher - Incremental hashing with support for all modes + * + * Supports: + * - Regular hashing + * - Keyed hashing (MAC) + * - Key derivation (derive_key) + * - XOF (eXtendable Output Function) mode + */ + +import { compress } from "./compress.js"; +import { + IV, + CHUNK_START, + CHUNK_END, + PARENT, + ROOT, + KEYED_HASH, + DERIVE_KEY_CONTEXT, + DERIVE_KEY_MATERIAL, + BLOCK_LEN, + CHUNK_LEN, + OUT_LEN, + KEY_LEN, + MAX_DEPTH, +} from "./constants.js"; +import { + IS_LITTLE_ENDIAN, + readLittleEndianWordsFull, + writeLittleEndianBytesPartial, + encodeUTF8, +} from "./utils.js"; + +/** + * Output state for XOF (eXtendable Output Function) mode. + * Allows reading arbitrary amounts of output. + */ +export class XofReader { + private inputCv: Uint32Array; + private blockWords: Uint32Array; + private counter: number; + private blockLen: number; + private flags: number; + private outputBlock: Uint32Array; + private outputBlockOffset: number; + + constructor( + inputCv: Uint32Array, + blockWords: Uint32Array, + counter: number, + blockLen: number, + flags: number, + ) { + this.inputCv = inputCv; + this.blockWords = blockWords; + this.counter = counter; + this.blockLen = blockLen; + this.flags = flags | ROOT; + this.outputBlock = new Uint32Array(16); + this.outputBlockOffset = 64; // Forces generation on first read + } + + /** + * Read the next `length` bytes of output. + */ + read(length: number): Uint8Array { + const output = new Uint8Array(length); + let outputOffset = 0; + + while (outputOffset < length) { + // Generate new output block if needed + if (this.outputBlockOffset >= 64) { + compress( + this.inputCv, + 0, + this.blockWords, + 0, + this.outputBlock, + 0, + true, // full 64-byte output + this.counter++, + this.blockLen, + this.flags, + ); + this.outputBlockOffset = 0; + } + + // Copy bytes from output block + const available = 64 - this.outputBlockOffset; + const toCopy = Math.min(available, length - outputOffset); + + // Optimized copy using writeLittleEndianBytesPartial + const wordOffset = this.outputBlockOffset >>> 2; + const byteWithinWord = this.outputBlockOffset & 3; + + if (byteWithinWord === 0 && toCopy >= 4) { + // Aligned copy - can use word-at-a-time + const fullWords = toCopy >>> 2; + writeLittleEndianBytesPartial( + this.outputBlock, + wordOffset, + output, + outputOffset, + fullWords << 2, + ); + const bytesCopied = fullWords << 2; + outputOffset += bytesCopied; + this.outputBlockOffset += bytesCopied; + } else { + // Byte-by-byte for unaligned access + for (let i = 0; i < toCopy; i++) { + const wordIdx = (this.outputBlockOffset + i) >>> 2; + const byteIdx = (this.outputBlockOffset + i) & 3; + output[outputOffset + i] = (this.outputBlock[wordIdx] >>> (byteIdx << 3)) & 0xff; + } + outputOffset += toCopy; + this.outputBlockOffset += toCopy; + } + } + + return output; + } +} + +/** + * Chunk state for processing input data. + * Each chunk is 1024 bytes and produces an 8-word chaining value. + */ +class ChunkState { + chainingValue: Uint32Array; + chunkCounter: number; + blockWords: Uint32Array; + blockLen: number; + blocksCompressed: number; + flags: number; + + constructor(keyWords: Uint32Array, chunkCounter: number, flags: number) { + this.chainingValue = new Uint32Array(keyWords); + this.chunkCounter = chunkCounter; + this.blockWords = new Uint32Array(16); + this.blockLen = 0; + this.blocksCompressed = 0; + this.flags = flags; + } + + resetTo(keyWords: Uint32Array, chunkCounter: number, flags: number): void { + this.chainingValue.set(keyWords); + this.chunkCounter = chunkCounter; + this.blockLen = 0; + this.blocksCompressed = 0; + this.flags = flags; + } + + /** + * Get the flags for the current block. + */ + private startFlag(): number { + return this.blocksCompressed === 0 ? CHUNK_START : 0; + } + + /** + * Update the chunk state with input data. + * Returns the number of bytes consumed. + */ + update(input: Uint8Array, inputOffset: number, inputLen: number): number { + let consumed = 0; + + while (inputLen > 0) { + // If we have a full block, compress it + if (this.blockLen === BLOCK_LEN) { + compress( + this.chainingValue, + 0, + this.blockWords, + 0, + this.chainingValue, + 0, + false, + this.chunkCounter, + BLOCK_LEN, + this.flags | this.startFlag(), + ); + this.blocksCompressed++; + this.blockLen = 0; + } + + // Fill the block buffer + const want = BLOCK_LEN - this.blockLen; + const take = Math.min(want, inputLen); + + if (this.blockLen === 0 && take === BLOCK_LEN) { + readLittleEndianWordsFull(input, inputOffset, this.blockWords); + } else { + // Partial block - byte-by-byte into correct position + for (let i = 0; i < take; i++) { + const pos = this.blockLen + i; + const wordIdx = pos >>> 2; + const byteIdx = pos & 3; + + if (byteIdx === 0) { + this.blockWords[wordIdx] = input[inputOffset + i]; + } else { + this.blockWords[wordIdx] |= input[inputOffset + i] << (byteIdx << 3); + } + } + } + + this.blockLen += take; + inputOffset += take; + inputLen -= take; + consumed += take; + } + + return consumed; + } + + /** + * Finalize this chunk and return its output. + * Returns 8 words (chaining value) or 16 words (if root). + */ + output(): { + inputCv: Uint32Array; + blockWords: Uint32Array; + blockLen: number; + counter: number; + flags: number; + } { + // Zero-pad unused words in blockWords to avoid stale data from previous blocks + // This is necessary when a partial block follows a full block within the same chunk + const usedWords = (this.blockLen + 3) >>> 2; // ceil(blockLen / 4) + for (let i = usedWords; i < 16; i++) { + this.blockWords[i] = 0; + } + + return { + inputCv: this.chainingValue, + blockWords: this.blockWords, + blockLen: this.blockLen, + counter: this.chunkCounter, + flags: this.flags | this.startFlag() | CHUNK_END, + }; + } + + /** + * Get the number of bytes in this chunk. + */ + len(): number { + return this.blocksCompressed * BLOCK_LEN + this.blockLen; + } +} + +/** + * Main BLAKE3 Hasher class. + * + * Usage: + * const hasher = new Hasher(); + * hasher.update(data); + * const hash = hasher.finalize(); + * + * Or with chaining: + * const hash = new Hasher().update(data).finalize(); + */ +export class Hasher { + private chunkState: ChunkState; + private keyWords: Uint32Array; + private cvStack: Uint32Array; + private cvStackLen: number; + private flags: number; + private parentBlock: Uint32Array; + private parentCv: Uint32Array; + private chunkCv: Uint32Array; + private outWords: Uint32Array; + private finalizeCv: Uint32Array; + + /** + * Create a new Hasher. + * + * @param keyWords - Initial key words (IV for regular hashing) + * @param flags - Domain separation flags + */ + constructor(keyWords?: Uint32Array, flags?: number) { + this.keyWords = keyWords ? new Uint32Array(keyWords) : new Uint32Array(IV); + this.flags = flags ?? 0; + this.chunkState = new ChunkState(this.keyWords, 0, this.flags); + this.cvStack = new Uint32Array(MAX_DEPTH * 8); + this.cvStackLen = 0; + this.parentBlock = new Uint32Array(16); + this.parentCv = new Uint32Array(8); + this.chunkCv = new Uint32Array(8); + this.outWords = new Uint32Array(16); + this.finalizeCv = new Uint32Array(8); + } + + /** + * Reset the hasher to process a new message with the same key/flags. + * Reuses all internal buffers — zero allocations. + */ + reset(): this { + this.chunkState.resetTo(this.keyWords, 0, this.flags); + this.cvStackLen = 0; + return this; + } + + /** + * Create a new keyed hasher (MAC). + * + * @param key - 32-byte key + */ + static newKeyed(key: Uint8Array): Hasher { + if (key.length !== KEY_LEN) { + throw new Error(`Key must be ${KEY_LEN} bytes, got ${key.length}`); + } + + const keyWords = new Uint32Array(8); + if (IS_LITTLE_ENDIAN) { + const view = new Uint32Array(key.buffer, key.byteOffset, 8); + keyWords.set(view); + } else { + for (let i = 0; i < 8; i++) { + const off = i * 4; + keyWords[i] = key[off] | (key[off + 1] << 8) | (key[off + 2] << 16) | (key[off + 3] << 24); + } + } + + return new Hasher(keyWords, KEYED_HASH); + } + + /** + * Create a new key derivation hasher. + * + * @param context - Context string for domain separation + */ + static newDeriveKey(context: string): Hasher { + // First, hash the context string with DERIVE_KEY_CONTEXT flag + const contextBytes = encodeUTF8(context); + const contextHasher = new Hasher(new Uint32Array(IV), DERIVE_KEY_CONTEXT); + contextHasher.update(contextBytes); + + // Get the context key + const contextKey = new Uint32Array(8); + const output = contextHasher.finalizeOutput(); + compress( + output.inputCv, + 0, + output.blockWords, + 0, + contextKey, + 0, + false, + output.counter, + output.blockLen, + output.flags | ROOT, + ); + + // Return a hasher initialized with the context key + return new Hasher(contextKey, DERIVE_KEY_MATERIAL); + } + + /** + * Push a chaining value onto the stack. + */ + private pushCv(cv: Uint32Array, cvOffset: number): void { + this.cvStack.set(cv.subarray(cvOffset, cvOffset + 8), this.cvStackLen * 8); + this.cvStackLen++; + } + + /** + * Pop a chaining value from the stack. + */ + private popCv(out: Uint32Array, outOffset: number): void { + this.cvStackLen--; + out.set(this.cvStack.subarray(this.cvStackLen * 8, (this.cvStackLen + 1) * 8), outOffset); + } + + /** + * Add a chunk's chaining value and merge completed subtrees. + */ + private addChunkCv(newCv: Uint32Array, newCvOffset: number, totalChunks: number): void { + const parentBlock = this.parentBlock; + const parentCv = this.parentCv; + + while ((totalChunks & 1) === 0) { + // Pop left child, new CV is right child + this.popCv(parentBlock, 0); + parentBlock.set(newCv.subarray(newCvOffset, newCvOffset + 8), 8); + + compress( + this.keyWords, + 0, + parentBlock, + 0, + parentCv, + 0, + false, + 0, + BLOCK_LEN, + this.flags | PARENT, + ); + + newCv = parentCv; + newCvOffset = 0; + totalChunks >>>= 1; + } + + this.pushCv(newCv, newCvOffset); + } + + /** + * Update the hasher with input data. + * + * @param input - Data to hash + * @returns this (for chaining) + */ + update(input: Uint8Array): this { + let inputOffset = 0; + let inputLen = input.length; + + // Fill the current chunk + while (inputLen > 0) { + // If current chunk is full, finalize it and start a new one + if (this.chunkState.len() === CHUNK_LEN) { + const output = this.chunkState.output(); + const chunkCv = this.chunkCv; + + compress( + output.inputCv, + 0, + output.blockWords, + 0, + chunkCv, + 0, + false, + output.counter, + output.blockLen, + output.flags, + ); + + const totalChunks = this.chunkState.chunkCounter + 1; + this.addChunkCv(chunkCv, 0, totalChunks); + + this.chunkState.resetTo(this.keyWords, totalChunks, this.flags); + } + + // Fill the current chunk + const want = CHUNK_LEN - this.chunkState.len(); + const take = Math.min(want, inputLen); + + this.chunkState.update(input, inputOffset, take); + inputOffset += take; + inputLen -= take; + } + + return this; + } + + /** + * Get the output parameters (for XOF mode or finalization). + */ + private finalizeOutput(): { + inputCv: Uint32Array; + blockWords: Uint32Array; + blockLen: number; + counter: number; + flags: number; + } { + let output = this.chunkState.output(); + let parentBlock = this.parentBlock; + let cv = this.finalizeCv; + + // If there are chunks on the stack, merge them + if (this.cvStackLen > 0) { + // First compress the current chunk + compress( + output.inputCv, + 0, + output.blockWords, + 0, + cv, + 0, + false, + output.counter, + output.blockLen, + output.flags, + ); + + // Merge with parent nodes from stack + while (this.cvStackLen > 0) { + this.cvStackLen--; + parentBlock.set(this.cvStack.subarray(this.cvStackLen * 8, (this.cvStackLen + 1) * 8), 0); + parentBlock.set(cv, 8); + + if (this.cvStackLen > 0) { + compress( + this.keyWords, + 0, + parentBlock, + 0, + cv, + 0, + false, + 0, + BLOCK_LEN, + this.flags | PARENT, + ); + } else { + // This is the root - return output params + return { + inputCv: this.keyWords, + blockWords: parentBlock, + blockLen: BLOCK_LEN, + counter: 0, + flags: this.flags | PARENT, + }; + } + } + } + + // Single chunk case + return output; + } + + /** + * Finalize the hash and return the result. + * + * @param outputLength - Number of bytes to output (default: 32) + * @returns The hash output + */ + finalize(outputLength: number = OUT_LEN): Uint8Array { + const output = this.finalizeOutput(); + const result = new Uint8Array(outputLength); + + if (outputLength <= 64) { + const outWords = this.outWords; + compress( + output.inputCv, + 0, + output.blockWords, + 0, + outWords, + 0, + outputLength > 32, // full output if > 32 bytes + output.counter, + output.blockLen, + output.flags | ROOT, + ); + + if (IS_LITTLE_ENDIAN) { + const outBytes = new Uint8Array(outWords.buffer); + result.set(outBytes.subarray(0, outputLength)); + } else { + writeLittleEndianBytesPartial(outWords, 0, result, 0, outputLength); + } + } else { + // Multiple blocks - use XOF + const xof = this.finalizeXof(); + const full = xof.read(outputLength); + result.set(full); + } + + return result; + } + + /** + * Finalize and return an XOF reader for arbitrary-length output. + */ + finalizeXof(): XofReader { + const output = this.finalizeOutput(); + return new XofReader( + new Uint32Array(output.inputCv), + new Uint32Array(output.blockWords), + output.counter, + output.blockLen, + output.flags, + ); + } +} diff --git a/node_modules/@huggingface/blake3-jit/src/index.ts b/node_modules/@huggingface/blake3-jit/src/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..1e88d9c179c2e51cc927e308288a1d1544095e0c --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/src/index.ts @@ -0,0 +1,106 @@ +/** + * BLAKE3 - The fastest pure JavaScript implementation + * + * Features: + * - All 3 modes: hash, keyed (MAC), derive_key + * - XOF (eXtendable Output Function) support + * - Automatic WASM SIMD acceleration for large inputs + * - Zero dependencies + * - Tree-shakeable exports + * + * @example + * ```typescript + * import { hash, createKeyed, createDeriveKey } from 'blake3-jit'; + * + * // Simple hashing + * const digest = hash(new Uint8Array([1, 2, 3])); + * + * // Keyed hashing (MAC) + * const mac = createKeyed(key).update(data).finalize(); + * + * // Key derivation + * const derived = createDeriveKey("my context").update(material).finalize(64); + * ``` + */ + +// Core exports +export { Hasher, XofReader } from "./hasher.js"; +export { hash, hashInto, warmupSimd } from "./hash.js"; + +// Convenience imports +import { Hasher } from "./hasher.js"; + +/** + * Create a new keyed hasher (MAC). + * + * @param key - 32-byte key + * @returns A new Hasher configured for keyed hashing + * + * @example + * ```typescript + * const key = new Uint8Array(32); // Your 32-byte key + * crypto.getRandomValues(key); + * + * const mac = createKeyed(key) + * .update(message) + * .finalize(); + * ``` + */ +export function createKeyed(key: Uint8Array): Hasher { + return Hasher.newKeyed(key); +} + +/** + * Create a new key derivation hasher. + * + * @param context - Context string for domain separation + * @returns A new Hasher configured for key derivation + * + * @example + * ```typescript + * const derivedKey = createDeriveKey("my-app encryption key v1") + * .update(inputKeyMaterial) + * .finalize(32); + * ``` + */ +export function createDeriveKey(context: string): Hasher { + return Hasher.newDeriveKey(context); +} + +/** + * Create a new regular hasher for incremental hashing. + * + * @returns A new Hasher + * + * @example + * ```typescript + * const hasher = createHasher(); + * hasher.update(chunk1); + * hasher.update(chunk2); + * const digest = hasher.finalize(); + * ``` + */ +export function createHasher(): Hasher { + return new Hasher(); +} + +// Import for default export +import { hash, hashInto, warmupSimd } from "./hash.js"; + +// Pre-warm SIMD in browser environments (non-blocking) +// This avoids initialization latency on first large hash +if (typeof globalThis !== "undefined" && typeof globalThis.document !== "undefined") { + queueMicrotask(() => { + warmupSimd(); + }); +} + +// Default export for convenience +export default { + hash, + hashInto, + Hasher, + createHasher, + createKeyed, + createDeriveKey, +}; diff --git a/node_modules/@huggingface/blake3-jit/src/utils.ts b/node_modules/@huggingface/blake3-jit/src/utils.ts new file mode 100644 index 0000000000000000000000000000000000000000..b8ba08644cef119f8a8d483beb18abdaab5cbd04 --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/src/utils.ts @@ -0,0 +1,253 @@ +/** + * BLAKE3 Utility Functions + * + * Optimized for little-endian systems (most user-facing systems). + * BLAKE3 is little-endian friendly - on little-endian systems we can + * create Uint32Array views directly over input buffers. + */ + +/** + * Detect system endianness at module load time. + * On little-endian systems, the byte 0x01 will be at index 0. + */ +export const IS_LITTLE_ENDIAN = new Uint8Array(new Uint32Array([0x01020304]).buffer)[0] === 0x04; + +/** + * Read 16 little-endian 32-bit words from a byte array into a Uint32Array. + * This is only needed on big-endian systems. + * + * @param input - Source byte array + * @param offset - Starting byte offset in input + * @param words - Destination Uint32Array (must have at least 16 elements) + */ +export function readLittleEndianWordsFull( + input: Uint8Array, + offset: number, + words: Uint32Array, +): void { + for (let i = 0; i < 16; ++i, offset += 4) { + words[i] = + input[offset] | + (input[offset + 1] << 8) | + (input[offset + 2] << 16) | + (input[offset + 3] << 24); + } +} + +/** + * Read N little-endian 32-bit words from a byte array. + * Handles partial reads (for final blocks). + * + * @param input - Source byte array + * @param offset - Starting byte offset + * @param words - Destination Uint32Array + * @param wordCount - Number of words to read + */ +export function readLittleEndianWords( + input: Uint8Array, + offset: number, + words: Uint32Array, + wordCount: number, +): void { + for (let i = 0; i < wordCount; ++i, offset += 4) { + words[i] = + input[offset] | + (input[offset + 1] << 8) | + (input[offset + 2] << 16) | + (input[offset + 3] << 24); + } +} + +/** + * Read a partial block with zero padding. + * Used for the final block when input length is not a multiple of 64. + * + * @param input - Source byte array + * @param offset - Starting byte offset + * @param length - Number of bytes to read (< 64) + * @param words - Destination Uint32Array (must have 16 elements) + */ +export function readLittleEndianWordsPartial( + input: Uint8Array, + offset: number, + length: number, + words: Uint32Array, +): void { + // Zero out all words first + words.fill(0); + + // Read full words + const fullWords = length >>> 2; + let i = 0; + for (; i < fullWords; ++i, offset += 4) { + words[i] = + input[offset] | + (input[offset + 1] << 8) | + (input[offset + 2] << 16) | + (input[offset + 3] << 24); + } + + // Handle remaining bytes (0-3) + const remaining = length & 3; + if (remaining > 0) { + let word = input[offset]; + if (remaining > 1) word |= input[offset + 1] << 8; + if (remaining > 2) word |= input[offset + 2] << 16; + words[i] = word; + } +} + +/** + * Write 8 little-endian 32-bit words to a byte array. + * + * @param words - Source Uint32Array + * @param wordOffset - Starting word offset in source + * @param output - Destination byte array + * @param byteOffset - Starting byte offset in destination + */ +export function writeLittleEndianWords( + words: Uint32Array, + wordOffset: number, + output: Uint8Array, + byteOffset: number, +): void { + for (let i = 0; i < 8; ++i, byteOffset += 4) { + const w = words[wordOffset + i]; + output[byteOffset] = w & 0xff; + output[byteOffset + 1] = (w >>> 8) & 0xff; + output[byteOffset + 2] = (w >>> 16) & 0xff; + output[byteOffset + 3] = (w >>> 24) & 0xff; + } +} + +/** + * Write N bytes from 32-bit words to output. + * Used for variable-length output (XOF mode). + * + * @param words - Source Uint32Array + * @param wordOffset - Starting word offset + * @param output - Destination byte array + * @param byteOffset - Starting byte offset in destination + * @param byteCount - Number of bytes to write + */ +export function writeLittleEndianBytesPartial( + words: Uint32Array, + wordOffset: number, + output: Uint8Array, + byteOffset: number, + byteCount: number, +): void { + const fullWords = byteCount >>> 2; + let i = 0; + + // Write full words + for (; i < fullWords; ++i, byteOffset += 4) { + const w = words[wordOffset + i]; + output[byteOffset] = w & 0xff; + output[byteOffset + 1] = (w >>> 8) & 0xff; + output[byteOffset + 2] = (w >>> 16) & 0xff; + output[byteOffset + 3] = (w >>> 24) & 0xff; + } + + // Write remaining bytes + const remaining = byteCount & 3; + if (remaining > 0) { + const w = words[wordOffset + i]; + output[byteOffset] = w & 0xff; + if (remaining > 1) output[byteOffset + 1] = (w >>> 8) & 0xff; + if (remaining > 2) output[byteOffset + 2] = (w >>> 16) & 0xff; + } +} + +/** + * Encode a UTF-8 string to Uint8Array. + * Used for derive_key context strings. + */ +export function encodeUTF8(str: string): Uint8Array { + if (typeof TextEncoder !== "undefined") { + return new TextEncoder().encode(str); + } + // Fallback for older environments + const bytes: number[] = []; + for (let i = 0; i < str.length; i++) { + let c = str.charCodeAt(i); + if (c < 0x80) { + bytes.push(c); + } else if (c < 0x800) { + bytes.push(0xc0 | (c >> 6), 0x80 | (c & 0x3f)); + } else if (c < 0xd800 || c >= 0xe000) { + bytes.push(0xe0 | (c >> 12), 0x80 | ((c >> 6) & 0x3f), 0x80 | (c & 0x3f)); + } else { + // Surrogate pair + i++; + c = 0x10000 + (((c & 0x3ff) << 10) | (str.charCodeAt(i) & 0x3ff)); + bytes.push( + 0xf0 | (c >> 18), + 0x80 | ((c >> 12) & 0x3f), + 0x80 | ((c >> 6) & 0x3f), + 0x80 | (c & 0x3f), + ); + } + } + return new Uint8Array(bytes); +} + +/** + * De Bruijn lookup table for O(1) trailing zero count. + * The expression (n & -n) isolates the lowest set bit. + * Multiplying by the De Bruijn constant maps each power of 2 to a unique 5-bit index. + */ +const CTZ32_TABLE = new Uint8Array([ + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, + 18, 6, 11, 5, 10, 9, +]); + +/** + * Count trailing zero bits in a 32-bit number using De Bruijn multiplication. + * This is O(1) and branchless for non-zero inputs. + * + * For Merkle tree merge: ctz32(chunkCounter) tells us how many merges to do. + */ +export function ctz32(n: number): number { + if (n === 0) return 32; + // Use unsigned right shift to handle negative numbers correctly + return CTZ32_TABLE[(((n & -n) * 0x077cb531) >>> 27) & 31]; +} + +/** + * Count trailing zero bits in a 64-bit number. + * Used to determine how many parent nodes to compute after adding a chunk. + * + * Note: JavaScript bitwise ops work on 32-bit signed integers, + * so we need to handle 64-bit numbers carefully. + */ +export function countTrailingZeros(n: number): number { + if (n === 0) return 64; + + // For numbers that fit in 32 bits + const low = n | 0; + if (low !== 0) { + // Use Math.clz32 trick: ctz(x) = 31 - clz32(x & -x) for non-zero x + return 31 - Math.clz32(low & -low); + } + + // High 32 bits + const high = (n / 0x100000000) | 0; + if (high !== 0) { + return 32 + (31 - Math.clz32(high & -high)); + } + + return 64; +} + +/** + * Create a Uint32Array view of a Uint8Array. + * Only works correctly on little-endian systems when the offset is 4-byte aligned. + * + * @param arr - Source byte array + * @param byteOffset - Starting byte offset (must be 4-byte aligned) + * @param wordLength - Number of 32-bit words + */ +export function uint32View(arr: Uint8Array, byteOffset: number, wordLength: number): Uint32Array { + return new Uint32Array(arr.buffer, arr.byteOffset + byteOffset, wordLength); +} diff --git a/node_modules/@huggingface/blake3-jit/src/wasm-simd.ts b/node_modules/@huggingface/blake3-jit/src/wasm-simd.ts new file mode 100644 index 0000000000000000000000000000000000000000..9a8e3be6e33602b2ee7d3cbd63cdb68303a15ddb --- /dev/null +++ b/node_modules/@huggingface/blake3-jit/src/wasm-simd.ts @@ -0,0 +1,1087 @@ +/** + * BLAKE3 WASM SIMD - Runtime bytecode generation + * + * Generates WebAssembly SIMD bytecode at runtime to process 4 compress + * operations in parallel using 128-bit SIMD vectors (i32x4). + * + * Key insight: One i32x4.add instruction performs 4 parallel additions, + * giving us 4x throughput for the same number of instructions. + * + * Memory layout (all values are transposed for SIMD access): + * 0-511: 4 x 16 message words (m0_0,m0_1,m0_2,m0_3, m1_0,m1_1,m1_2,m1_3, ...) + * 512-639: 4 x 8 chaining values + * 640-767: 4 x 8 output values + * 768-783: 4 x counter low + * 784-799: 4 x counter high + * 800-815: 4 x block length + * 816-831: 4 x flags + */ + +// LEB128 encoding with minimum 2 bytes +// This fixes a V8 quirk where single-byte values 64-127 cause issues +// when followed by certain SIMD instructions +function toLebU32Min2(n: number): number[] { + // Always use at least 2 bytes + return [(n & 0x7f) | 0x80, (n >>> 7) & 0x7f]; +} + +// LEB128 encoding padded to exactly 5 bytes (for backpatching) +// Uses continuation bits for all but the last byte +function toLebU32Padded5(n: number): number[] { + return [ + (n & 0x7f) | 0x80, + ((n >>> 7) & 0x7f) | 0x80, + ((n >>> 14) & 0x7f) | 0x80, + ((n >>> 21) & 0x7f) | 0x80, + (n >>> 28) & 0x0f, // Last byte has no continuation bit + ]; +} + +// Signed LEB128 encoding for i32 constants (handles full 32-bit range) +// WASM i32.const uses signed LEB128 immediate +function toSignedLeb128_i32(n: number): number[] { + const bytes: number[] = []; + // Treat as signed 32-bit integer + let value = n | 0; + let more = true; + while (more) { + let byte = value & 0x7f; + // Arithmetic right shift preserves sign + value >>= 7; + // Check if we're done: + // - If value is 0 and sign bit of byte is clear, we're done + // - If value is -1 and sign bit of byte is set, we're done + if ((value === 0 && (byte & 0x40) === 0) || (value === -1 && (byte & 0x40) !== 0)) { + more = false; + } else { + byte |= 0x80; + } + bytes.push(byte); + } + return bytes; +} + +// Precomputed message access order for all 7 rounds +const MSG_ACCESS_ORDER = [ + // Round 1: 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + // Round 2: 2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8 + 2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8, + // Round 3: 3,4,10,12,13,2,7,14,6,5,9,0,11,15,8,1 + 3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1, + // Round 4: 10,7,12,9,14,3,13,15,4,0,11,2,5,8,1,6 + 10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6, + // Round 5: 12,13,9,11,15,10,14,8,7,2,5,3,0,1,6,4 + 12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4, + // Round 6: 9,14,11,5,8,12,15,1,13,3,0,10,2,6,4,7 + 9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7, + // Round 7: 11,15,5,0,1,9,8,6,14,10,2,12,3,4,7,13 + 11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13, +]; + +// BLAKE3 Constants (used in generated WASM code) +// CHUNK_START = 1, CHUNK_END = 2 are embedded directly in WASM bytecode + +/** + * Generate the WASM module bytecode with compress4x, compressChunks4x, and compressParent functions. + */ +function generateWasmBytes(): Uint8Array { + const code: number[] = []; + + // Helper to append bytes + function put(bytes: number[]): void { + code.push(...bytes); + } + + // WASM module header + put([0x00, 0x61, 0x73, 0x6d]); // Magic + put([0x01, 0x00, 0x00, 0x00]); // Version + + // Section 1: Types + put([0x01]); // Section ID + put([0x04]); // Section size + put([0x01]); // 1 type + put([0x60, 0x00, 0x00]); // func () -> () + + // Section 2: Imports (memory from JS) + put([0x02]); // Section ID + put([0x0b]); // Section size + put([0x01]); // 1 import + put([0x02, 0x6a, 0x73]); // "js" + put([0x03, 0x6d, 0x65, 0x6d]); // "mem" + put([0x02, 0x00, 0x01]); // memory min=1, no max + + // Section 3: Functions + put([0x03]); // Section ID + put([0x04]); // Section size (3 functions = 4 bytes) + put([0x03]); // 3 functions + put([0x00]); // Function 0: type index 0 + put([0x00]); // Function 1: type index 0 + put([0x00]); // Function 2: type index 0 + + // Section 7: Exports + // Size calculation: 1 (count) + (1+10+1+1) + (1+16+1+1) + (1+14+1+1) = 1 + 13 + 19 + 17 = 50 bytes + put([0x07]); // Section ID + put([0x32]); // Section size (50 bytes) + put([0x03]); // 3 exports + // "compress4x" -> func 0 + put([0x0a]); // name length + put([0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x34, 0x78]); // "compress4x" + put([0x00, 0x00]); // func index 0 + // "compressChunks4x" -> func 1 + put([0x10]); // name length (16) + put([ + 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x34, 0x78, + ]); // "compressChunks4x" + put([0x00, 0x01]); // func index 1 + // "compressParent" -> func 2 + put([0x0e]); // name length (14) + put([0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74]); // "compressParent" + put([0x00, 0x02]); // func index 2 + + // Section 10: Code + put([0x0a]); // Section ID + // Reserve 5 bytes for section size (LEB128 u32) + const sectionSizeOffset = code.length; + put([0x00, 0x00, 0x00, 0x00, 0x00]); + + put([0x03]); // 3 functions + + // === Function 0: compress4x === + // Reserve 5 bytes for function size + const funcSizeOffset = code.length; + put([0x00, 0x00, 0x00, 0x00, 0x00]); + + const funcBodyStart = code.length; + + // Local declarations: 32 v128 locals + // Variables $0-$15: message words (m0-m15) + // Variables $16-$31: state words (s0-s15) + put([0x01]); // 1 local declaration + put([0x20, 0x7b]); // 32 x v128 + + // ===== Function body ===== + + // Load message words from memory (offset 0-255) + // Each v128 is 16 bytes, so m[i] is at offset i*16 + // Note: we use toLebU32Min2 to avoid V8 quirk with single-byte values 64-127 + for (let i = 0; i < 16; i++) { + put([0x41, ...toLebU32Min2(i * 16)]); // i32.const offset (2+ byte LEB128) + put([0xfd, 0x00, 0x02, 0x00]); // v128.load align=4 offset=0 + put([0x21, i]); // local.set $i + } + + // Load chaining values (offset 512-639) + // cv[i] at offset 512 + i*16 + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(512 + i * 16)]); // i32.const offset + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 16 + i]); // local.set $(16+i) + } + + // Initialize state[8-15] from IV and parameters + // s8-s11 = IV[0-3] + const IV = [0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a]; + for (let i = 0; i < 4; i++) { + // Create v128 constant with all lanes set to IV[i] + const ivBytes = []; + for (let j = 0; j < 4; j++) { + ivBytes.push(IV[i] & 0xff); + ivBytes.push((IV[i] >>> 8) & 0xff); + ivBytes.push((IV[i] >>> 16) & 0xff); + ivBytes.push((IV[i] >>> 24) & 0xff); + } + put([0xfd, 0x0c, ...ivBytes]); // v128.const + put([0x21, 24 + i]); // local.set $(24+i) -> s8-s11 + } + + // s12 = counter_low (offset 768) + put([0x41, ...toLebU32Min2(768)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 28]); // local.set $28 -> s12 + + // s13 = counter_high (offset 784) + put([0x41, ...toLebU32Min2(784)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 29]); // local.set $29 -> s13 + + // s14 = block_len (offset 800) + put([0x41, ...toLebU32Min2(800)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 30]); // local.set $30 -> s14 + + // s15 = flags (offset 816) + put([0x41, ...toLebU32Min2(816)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 31]); // local.set $31 -> s15 + + // ===== 7 rounds of mixing ===== + + let msgIdx = 0; // Index into MSG_ACCESS_ORDER + + // Helper to generate G function (inlined) + // G(a, b, c, d) with two message words + function g(a: number, b: number, c: number, d: number): void { + const mx = MSG_ACCESS_ORDER[msgIdx++]; + const my = MSG_ACCESS_ORDER[msgIdx++]; + + // Variables: a,b,c,d are state indices (16-31), mx,my are message indices (0-15) + + // First half of G + // s[a] = s[a] + s[b] + m[mx] + put([0x20, 16 + a]); // local.get s[a] + put([0x20, 16 + b]); // local.get s[b] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x20, mx]); // local.get m[mx] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + a]); // local.set s[a] + + // s[d] = rotr(s[d] ^ s[a], 16) - using i8x16.shuffle (single instruction vs shift+or) + // ROTR16 pattern: [2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13] + put([0x20, 16 + d]); // local.get s[d] + put([0x20, 16 + a]); // local.get s[a] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + d]); // local.tee s[d] + put([0x20, 16 + d]); // local.get s[d] (second operand for shuffle) + put([0xfd, 0x0d, 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13]); // i8x16.shuffle ROTR16 + put([0x21, 16 + d]); // local.set s[d] + + // s[c] = s[c] + s[d] + put([0x20, 16 + c]); // local.get s[c] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + c]); // local.set s[c] + + // s[b] = (s[b] ^ s[c]) >>> 12 + put([0x20, 16 + b]); // local.get s[b] + put([0x20, 16 + c]); // local.get s[c] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + b]); // local.tee s[b] + put([0x41, 0x0c]); // i32.const 12 + put([0xfd, 0xad, 0x01]); // i32x4.shr_u (opcode 173 = 0xAD) + put([0x20, 16 + b]); // local.get s[b] + put([0x41, 0x14]); // i32.const 20 + put([0xfd, 0xab, 0x01]); // i32x4.shl + put([0xfd, 0x50]); // v128.or + put([0x21, 16 + b]); // local.set s[b] + + // Second half of G + // s[a] = s[a] + s[b] + m[my] + put([0x20, 16 + a]); // local.get s[a] + put([0x20, 16 + b]); // local.get s[b] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x20, my]); // local.get m[my] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + a]); // local.set s[a] + + // s[d] = rotr(s[d] ^ s[a], 8) - using i8x16.shuffle (single instruction vs shift+or) + // ROTR8 pattern: [1,2,3,0, 5,6,7,4, 9,10,11,8, 13,14,15,12] + put([0x20, 16 + d]); // local.get s[d] + put([0x20, 16 + a]); // local.get s[a] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + d]); // local.tee s[d] + put([0x20, 16 + d]); // local.get s[d] (second operand for shuffle) + put([0xfd, 0x0d, 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12]); // i8x16.shuffle ROTR8 + put([0x21, 16 + d]); // local.set s[d] + + // s[c] = s[c] + s[d] + put([0x20, 16 + c]); // local.get s[c] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + c]); // local.set s[c] + + // s[b] = (s[b] ^ s[c]) >>> 7 + put([0x20, 16 + b]); // local.get s[b] + put([0x20, 16 + c]); // local.get s[c] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + b]); // local.tee s[b] + put([0x41, 0x07]); // i32.const 7 + put([0xfd, 0xad, 0x01]); // i32x4.shr_u (opcode 173 = 0xAD) + put([0x20, 16 + b]); // local.get s[b] + put([0x41, 0x19]); // i32.const 25 + put([0xfd, 0xab, 0x01]); // i32x4.shl + put([0xfd, 0x50]); // v128.or + put([0x21, 16 + b]); // local.set s[b] + } + + // Generate all 7 rounds + for (let round = 0; round < 7; round++) { + // Column mixing + g(0, 4, 8, 12); + g(1, 5, 9, 13); + g(2, 6, 10, 14); + g(3, 7, 11, 15); + + // Diagonal mixing + g(0, 5, 10, 15); + g(1, 6, 11, 12); + g(2, 7, 8, 13); + g(3, 4, 9, 14); + } + + // ===== Final XOR and store output ===== + + // out[i] = s[i] ^ s[i+8] for i in 0..7 + // Store at offset 640-767 + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(640 + i * 16)]); // i32.const offset + put([0x20, 16 + i]); // local.get s[i] + put([0x20, 24 + i]); // local.get s[i+8] + put([0xfd, 0x51]); // v128.xor + put([0xfd, 0x0b, 0x02, 0x00]); // v128.store align=4 + } + + // End of function + put([0x0b]); // end + + // Fill in function 0 size using padded LEB128 + const funcBodySize = code.length - funcBodyStart; + const funcSizeBytes = toLebU32Padded5(funcBodySize); + for (let i = 0; i < 5; i++) { + code[funcSizeOffset + i] = funcSizeBytes[i]; + } + + // === Function 1: compressChunks4x === + // Reserve 5 bytes for function size + const func1SizeOffset = code.length; + put([0x00, 0x00, 0x00, 0x00, 0x00]); + + const func1BodyStart = code.length; + + // Generate the compressChunks4x function body + const compressChunksBody = generateCompressChunks4xBody(); + put(compressChunksBody); + + // Fill in function 1 size using padded LEB128 + const func1BodySize = code.length - func1BodyStart; + const func1SizeBytes = toLebU32Padded5(func1BodySize); + for (let i = 0; i < 5; i++) { + code[func1SizeOffset + i] = func1SizeBytes[i]; + } + + // === Function 2: compressParent === + // Reserve 5 bytes for function size + const func2SizeOffset = code.length; + put([0x00, 0x00, 0x00, 0x00, 0x00]); + + const func2BodyStart = code.length; + + // Generate the compressParent function body + const compressParentBody = generateCompressParentBody(); + put(compressParentBody); + + // Fill in function 2 size using padded LEB128 + const func2BodySize = code.length - func2BodyStart; + const func2SizeBytes = toLebU32Padded5(func2BodySize); + for (let i = 0; i < 5; i++) { + code[func2SizeOffset + i] = func2SizeBytes[i]; + } + + // Fill in section size using padded LEB128 + const sectionSize = code.length - sectionSizeOffset - 5; + const sectionSizeBytes = toLebU32Padded5(sectionSize); + for (let i = 0; i < 5; i++) { + code[sectionSizeOffset + i] = sectionSizeBytes[i]; + } + + return new Uint8Array(code); +} + +/** + * Generate compressChunks4x WASM function body. + * Processes all 16 blocks of 4 chunks in a single call. + */ +function generateCompressChunks4xBody(): number[] { + const code: number[] = []; + + function put(bytes: number[]): void { + code.push(...bytes); + } + + // Local declarations: 32 v128 locals + 1 i32 for position + // Locals $0-$15: message words (reloaded each iteration) + // Locals $16-$31: state words (s0-s15) + // Local $32: position counter (i32) + put([0x02]); // 2 local declarations + put([0x20, 0x7b]); // 32 x v128 + put([0x01, 0x7f]); // 1 x i32 + + const BATCH_BLOCK_WORDS = SIMD_MEMORY.BATCH_BLOCK_WORDS; + const BATCH_CV = SIMD_MEMORY.BATCH_CV; + const BATCH_COUNTER_LOW = SIMD_MEMORY.BATCH_COUNTER_LOW; + const BATCH_FLAGS_BASE = SIMD_MEMORY.BATCH_FLAGS_BASE; + const BATCH_OUTPUT = SIMD_MEMORY.BATCH_OUTPUT; + + // IV constants (same as compress4x) + const IV = [0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a]; + + // Load initial CVs from BATCH_CV into locals $16-$23 + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(BATCH_CV + i * 16)]); // i32.const offset + put([0xfd, 0x00, 0x02, 0x00]); // v128.load align=4 offset=0 + put([0x21, 16 + i]); // local.set $(16+i) -> s0-s7 + } + + // Initialize $32 (pos) = 0 + put([0x41, 0x00]); // i32.const 0 + put([0x21, 0x20]); // local.set $32 + + // block $done + put([0x02, 0x40]); // block void + + // loop $continue + put([0x03, 0x40]); // loop void + + // === Load message words for position $pos === + // offset = BATCH_BLOCK_WORDS + pos * 256 + word * 16 + for (let w = 0; w < 16; w++) { + put([0x20, 0x20]); // local.get $32 (pos) + put([0x41, ...toLebU32Min2(256)]); // i32.const 256 + put([0x6c]); // i32.mul + put([0x41, ...toLebU32Min2(BATCH_BLOCK_WORDS + w * 16)]); // i32.const base + word*16 + put([0x6a]); // i32.add + put([0xfd, 0x00, 0x02, 0x00]); // v128.load align=4 offset=0 + put([0x21, w]); // local.set $w + } + + // === Initialize state[8-15] === + // s8-s11 = IV[0-3] + for (let i = 0; i < 4; i++) { + const ivBytes = []; + for (let j = 0; j < 4; j++) { + ivBytes.push(IV[i] & 0xff); + ivBytes.push((IV[i] >>> 8) & 0xff); + ivBytes.push((IV[i] >>> 16) & 0xff); + ivBytes.push((IV[i] >>> 24) & 0xff); + } + put([0xfd, 0x0c, ...ivBytes]); // v128.const + put([0x21, 24 + i]); // local.set $(24+i) -> s8-s11 + } + + // s12 = counter_low (from BATCH_COUNTER_LOW) + put([0x41, ...toLebU32Min2(BATCH_COUNTER_LOW)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load + put([0x21, 28]); // local.set $28 -> s12 + + // s13 = 0 (counter high - assume fits in 32 bits) + put([0xfd, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); // v128.const 0 + put([0x21, 29]); // local.set $29 -> s13 + + // s14 = 64 (block_len = 64 for full blocks) + const blockLen64 = []; + for (let j = 0; j < 4; j++) { + blockLen64.push(64, 0, 0, 0); // 64 in little-endian + } + put([0xfd, 0x0c, ...blockLen64]); // v128.const [64,64,64,64] + put([0x21, 30]); // local.set $30 -> s14 + + // s15 = flags = base_flags | (pos == 0 ? 1 : 0) | (pos == 15 ? 2 : 0) + // First load base flags + put([0x41, ...toLebU32Min2(BATCH_FLAGS_BASE)]); // i32.const + put([0xfd, 0x00, 0x02, 0x00]); // v128.load base flags + + // Compute position-dependent bits + // CHUNK_START (1) if pos == 0 + put([0x20, 0x20]); // local.get $32 (pos) + put([0x45]); // i32.eqz -> 1 if pos==0, 0 otherwise + // CHUNK_END (2) if pos == 15 + put([0x20, 0x20]); // local.get $32 (pos) + put([0x41, 0x0f]); // i32.const 15 + put([0x46]); // i32.eq -> 1 if pos==15, 0 otherwise + put([0x41, 0x01]); // i32.const 1 (shift amount) + put([0x74]); // i32.shl -> 2 if pos==15, 0 otherwise + // OR the two bits together + put([0x72]); // i32.or -> combined position bits + // Splat to v128 and OR with base flags (stack: base_flags, bits) + put([0xfd, 0x11]); // i32x4.splat + put([0xfd, 0x50]); // v128.or + put([0x21, 31]); // local.set $31 -> s15 + + // === 7 rounds of mixing === + let msgIdx = 0; + + function g(a: number, b: number, c: number, d: number): void { + const mx = MSG_ACCESS_ORDER[msgIdx++]; + const my = MSG_ACCESS_ORDER[msgIdx++]; + + // First half of G: s[a] = s[a] + s[b] + m[mx] + put([0x20, 16 + a]); // local.get s[a] + put([0x20, 16 + b]); // local.get s[b] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x20, mx]); // local.get m[mx] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + a]); // local.set s[a] + + // s[d] = rotr(s[d] ^ s[a], 16) - byte shuffle + put([0x20, 16 + d]); // local.get s[d] + put([0x20, 16 + a]); // local.get s[a] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + d]); // local.tee s[d] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0x0d, 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13]); // i8x16.shuffle ROTR16 + put([0x21, 16 + d]); // local.set s[d] + + // s[c] = s[c] + s[d] + put([0x20, 16 + c]); // local.get s[c] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + c]); // local.set s[c] + + // s[b] = rotr(s[b] ^ s[c], 12) + put([0x20, 16 + b]); // local.get s[b] + put([0x20, 16 + c]); // local.get s[c] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + b]); // local.tee s[b] + put([0x41, 0x0c]); // i32.const 12 + put([0xfd, 0xad, 0x01]); // i32x4.shr_u (opcode 173 = 0xAD) + put([0x20, 16 + b]); // local.get s[b] + put([0x41, 0x14]); // i32.const 20 + put([0xfd, 0xab, 0x01]); // i32x4.shl + put([0xfd, 0x50]); // v128.or + put([0x21, 16 + b]); // local.set s[b] + + // Second half: s[a] = s[a] + s[b] + m[my] + put([0x20, 16 + a]); // local.get s[a] + put([0x20, 16 + b]); // local.get s[b] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x20, my]); // local.get m[my] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + a]); // local.set s[a] + + // s[d] = rotr(s[d] ^ s[a], 8) - byte shuffle + put([0x20, 16 + d]); // local.get s[d] + put([0x20, 16 + a]); // local.get s[a] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + d]); // local.tee s[d] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0x0d, 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12]); // i8x16.shuffle ROTR8 + put([0x21, 16 + d]); // local.set s[d] + + // s[c] = s[c] + s[d] + put([0x20, 16 + c]); // local.get s[c] + put([0x20, 16 + d]); // local.get s[d] + put([0xfd, 0xae, 0x01]); // i32x4.add + put([0x21, 16 + c]); // local.set s[c] + + // s[b] = rotr(s[b] ^ s[c], 7) + put([0x20, 16 + b]); // local.get s[b] + put([0x20, 16 + c]); // local.get s[c] + put([0xfd, 0x51]); // v128.xor + put([0x22, 16 + b]); // local.tee s[b] + put([0x41, 0x07]); // i32.const 7 + put([0xfd, 0xad, 0x01]); // i32x4.shr_u (opcode 173 = 0xAD) + put([0x20, 16 + b]); // local.get s[b] + put([0x41, 0x19]); // i32.const 25 + put([0xfd, 0xab, 0x01]); // i32x4.shl + put([0xfd, 0x50]); // v128.or + put([0x21, 16 + b]); // local.set s[b] + } + + // Generate all 7 rounds + for (let round = 0; round < 7; round++) { + // Column mixing + g(0, 4, 8, 12); + g(1, 5, 9, 13); + g(2, 6, 10, 14); + g(3, 7, 11, 15); + // Diagonal mixing + g(0, 5, 10, 15); + g(1, 6, 11, 12); + g(2, 7, 8, 13); + g(3, 4, 9, 14); + } + + // === Update CVs: cv[i] = s[i] ^ s[i+8] === + // Store back to state locals $16-$23 (the CV positions) + for (let i = 0; i < 8; i++) { + put([0x20, 16 + i]); // local.get s[i] + put([0x20, 24 + i]); // local.get s[i+8] + put([0xfd, 0x51]); // v128.xor + put([0x21, 16 + i]); // local.set $(16+i) - update CV + } + + // === Loop control: pos++, continue if pos < 16 === + put([0x20, 0x20]); // local.get $32 (pos) + put([0x41, 0x01]); // i32.const 1 + put([0x6a]); // i32.add + put([0x22, 0x20]); // local.tee $32 (pos) + put([0x41, 0x10]); // i32.const 16 + put([0x49]); // i32.lt_u + put([0x0d, 0x00]); // br_if 0 (continue loop) + + // end loop + put([0x0b]); // end + + // end block + put([0x0b]); // end + + // === Store final CVs to BATCH_OUTPUT === + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(BATCH_OUTPUT + i * 16)]); // i32.const offset + put([0x20, 16 + i]); // local.get $(16+i) - CV word + put([0xfd, 0x0b, 0x02, 0x00]); // v128.store align=4 + } + + // end function + put([0x0b]); // end + + return code; +} + +/** + * Generate compressParent WASM function body. + * Performs a single parent node compression using scalar i32 operations. + * Reads 16 words from PARENT_BLOCK, writes 8 words to CHUNK_CV. + * Uses IV, counter=0, blockLen=64, flags=PARENT(4). + */ +function generateCompressParentBody(): number[] { + const code: number[] = []; + + function put(bytes: number[]): void { + code.push(...bytes); + } + + // Local declarations: 32 i32 locals for state (s0-s15) and message (m0-m15) + put([0x01]); // 1 local declaration + put([0x20, 0x7f]); // 32 x i32 + + // Message word indices: 0-15, State indices: 16-31 + // Locals $0-$15: message words (m0-m15) + // Locals $16-$31: state words (s0-s15) + + const PARENT_BLOCK_OFFSET = SIMD_MEMORY.PARENT_BLOCK; + const CHUNK_CV_OFFSET = SIMD_MEMORY.CHUNK_CV; + + // BLAKE3 IV + const IV = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, + ]; + + // Load message words from PARENT_BLOCK (16 words at offset 7264) + for (let i = 0; i < 16; i++) { + put([0x41, ...toLebU32Min2(PARENT_BLOCK_OFFSET + i * 4)]); // i32.const offset + put([0x28, 0x02, 0x00]); // i32.load align=4 offset=0 + put([0x21, i]); // local.set $i (m0-m15) + } + + // Initialize state s0-s7 = IV[0-7] + for (let i = 0; i < 8; i++) { + put([0x41, ...toSignedLeb128_i32(IV[i])]); // i32.const IV[i] + put([0x21, 16 + i]); // local.set $(16+i) -> s0-s7 + } + + // Initialize state s8-s11 = IV[0-3] + for (let i = 0; i < 4; i++) { + put([0x41, ...toSignedLeb128_i32(IV[i])]); // i32.const IV[i] + put([0x21, 24 + i]); // local.set $(24+i) -> s8-s11 + } + + // s12 = counter_low = 0 + put([0x41, 0x00]); // i32.const 0 + put([0x21, 28]); // local.set $28 -> s12 + + // s13 = counter_high = 0 + put([0x41, 0x00]); // i32.const 0 + put([0x21, 29]); // local.set $29 -> s13 + + // s14 = block_len = 64 + // Note: 0x40 alone is -64 in signed LEB128 (bit 6 is sign bit) + // For 64, we need [0xC0, 0x00] to avoid sign extension + put([0x41, 0xc0, 0x00]); // i32.const 64 + put([0x21, 30]); // local.set $30 -> s14 + + // s15 = flags = PARENT = 4 + put([0x41, 0x04]); // i32.const 4 + put([0x21, 31]); // local.set $31 -> s15 + + // Helper to generate scalar G function (inlined) + // G(a, b, c, d, mx, my) where a,b,c,d are state indices 0-15, mx,my are message indices 0-15 + function g(a: number, b: number, c: number, d: number, mx: number, my: number): void { + const sa = 16 + a, + sb = 16 + b, + sc = 16 + c, + sd = 16 + d; + + // s[a] = (s[a] + s[b] + m[mx]) >>> 0 + put([0x20, sa]); // local.get s[a] + put([0x20, sb]); // local.get s[b] + put([0x6a]); // i32.add + put([0x20, mx]); // local.get m[mx] + put([0x6a]); // i32.add + put([0x21, sa]); // local.set s[a] + + // s[d] = rotr(s[d] ^ s[a], 16) + put([0x20, sd]); // local.get s[d] + put([0x20, sa]); // local.get s[a] + put([0x73]); // i32.xor + put([0x41, 0x10]); // i32.const 16 + put([0x78]); // i32.rotr + put([0x21, sd]); // local.set s[d] + + // s[c] = (s[c] + s[d]) >>> 0 + put([0x20, sc]); // local.get s[c] + put([0x20, sd]); // local.get s[d] + put([0x6a]); // i32.add + put([0x21, sc]); // local.set s[c] + + // s[b] = rotr(s[b] ^ s[c], 12) + put([0x20, sb]); // local.get s[b] + put([0x20, sc]); // local.get s[c] + put([0x73]); // i32.xor + put([0x41, 0x0c]); // i32.const 12 + put([0x78]); // i32.rotr + put([0x21, sb]); // local.set s[b] + + // s[a] = (s[a] + s[b] + m[my]) >>> 0 + put([0x20, sa]); // local.get s[a] + put([0x20, sb]); // local.get s[b] + put([0x6a]); // i32.add + put([0x20, my]); // local.get m[my] + put([0x6a]); // i32.add + put([0x21, sa]); // local.set s[a] + + // s[d] = rotr(s[d] ^ s[a], 8) + put([0x20, sd]); // local.get s[d] + put([0x20, sa]); // local.get s[a] + put([0x73]); // i32.xor + put([0x41, 0x08]); // i32.const 8 + put([0x78]); // i32.rotr + put([0x21, sd]); // local.set s[d] + + // s[c] = (s[c] + s[d]) >>> 0 + put([0x20, sc]); // local.get s[c] + put([0x20, sd]); // local.get s[d] + put([0x6a]); // i32.add + put([0x21, sc]); // local.set s[c] + + // s[b] = rotr(s[b] ^ s[c], 7) + put([0x20, sb]); // local.get s[b] + put([0x20, sc]); // local.get s[c] + put([0x73]); // i32.xor + put([0x41, 0x07]); // i32.const 7 + put([0x78]); // i32.rotr + put([0x21, sb]); // local.set s[b] + } + + // 7 rounds of mixing with permuted message schedule + let msgIdx = 0; + for (let round = 0; round < 7; round++) { + // Column mixing + g(0, 4, 8, 12, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(1, 5, 9, 13, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(2, 6, 10, 14, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(3, 7, 11, 15, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + + // Diagonal mixing + g(0, 5, 10, 15, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(1, 6, 11, 12, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(2, 7, 8, 13, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + g(3, 4, 9, 14, MSG_ACCESS_ORDER[msgIdx], MSG_ACCESS_ORDER[msgIdx + 1]); + msgIdx += 2; + } + + // Store output: out[i] = s[i] ^ s[i+8] for i in 0..7 + for (let i = 0; i < 8; i++) { + put([0x41, ...toLebU32Min2(CHUNK_CV_OFFSET + i * 4)]); // i32.const offset + put([0x20, 16 + i]); // local.get s[i] + put([0x20, 24 + i]); // local.get s[i+8] + put([0x73]); // i32.xor + put([0x36, 0x02, 0x00]); // i32.store align=4 offset=0 + } + + // end function + put([0x0b]); // end + + return code; +} + +// Cached WASM instance +let wasmInstance: WebAssembly.Instance | null = null; +let wasmMemory: WebAssembly.Memory | null = null; +let wasmCompress4x: (() => void) | null = null; +let wasmCompressChunks4x: (() => void) | null = null; +let wasmCompressParent: (() => void) | null = null; +let wasmMemoryView: Uint8Array | null = null; +let wasmMemoryView32: Uint32Array | null = null; + +/** + * Check if WASM SIMD is supported. + */ +export function isSimdSupported(): boolean { + try { + // Minimal WASM module with v128.const instruction to test SIMD support + const simdTest = new Uint8Array([ + 0x00, + 0x61, + 0x73, + 0x6d, // magic: \0asm + 0x01, + 0x00, + 0x00, + 0x00, // version: 1 + + // Type section (id=1): () -> v128 + 0x01, // section id = 1 (type) + 0x05, // section length = 5 + 0x01, // 1 type + 0x60, + 0x00, + 0x01, + 0x7b, // func () -> v128 + + // Function section (id=3) + 0x03, // section id = 3 (function) + 0x02, // section length = 2 + 0x01, // 1 function + 0x00, // type index 0 + + // Code section (id=10) with v128.const + 0x0a, // section id = 10 (code) + 0x16, // section length = 22 + 0x01, // 1 function body + 0x14, // body length = 20 + 0x00, // 0 locals + 0xfd, + 0x0c, // v128.const opcode + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x00, + 0x0b, // end + ]); + return WebAssembly.validate(simdTest); + } catch { + return false; + } +} + +/** + * Set up arena views over WASM memory. + * Called after WASM memory is allocated. + */ +function setupArenaViews(): void { + if (!wasmMemory) return; + + const buffer = wasmMemory.buffer; + // Create TypedArray views over WASM memory for arena buffers + // These views are backed by WASM memory, eliminating JS heap allocation + arenaCvStack = new Uint32Array(buffer, SIMD_MEMORY.CV_STACK, 64 * 8); // 64 levels × 8 words + arenaParentBlock = new Uint32Array(buffer, SIMD_MEMORY.PARENT_BLOCK, 16); // 16 words + arenaChunkCv = new Uint32Array(buffer, SIMD_MEMORY.CHUNK_CV, 8); // 8 words + arenaTempCvs = new Uint32Array(buffer, SIMD_MEMORY.TEMP_CVS, 32); // 4 × 8 words + + // Batch mode views + // 16 positions × 16 v128 words = 16 × 64 u32 words = 1024 words per position? No... + // In u32 terms: 16 positions × 16 words × 4 lanes = 1024 u32 values total + arenaBatchBlockWords = new Uint32Array(buffer, SIMD_MEMORY.BATCH_BLOCK_WORDS, 16 * 16 * 4); // 16 pos × 16 words × 4 lanes + arenaBatchCv = new Uint32Array(buffer, SIMD_MEMORY.BATCH_CV, 32); // 4 × 8 words + arenaBatchCounterLow = new Uint32Array(buffer, SIMD_MEMORY.BATCH_COUNTER_LOW, 4); // 4 words + arenaBatchFlagsBase = new Uint32Array(buffer, SIMD_MEMORY.BATCH_FLAGS_BASE, 4); // 4 words + arenaBatchOutput = new Uint32Array(buffer, SIMD_MEMORY.BATCH_OUTPUT, 32); // 4 × 8 words +} + +/** + * Initialize the WASM SIMD module synchronously. + * Call this once before using compress4x. + */ +// Cache generated WASM bytes to avoid regenerating on each init +let cachedWasmBytes: Uint8Array | null = null; + +export function initSimdSync(): boolean { + if (wasmInstance) return true; + + if (!isSimdSupported()) { + return false; + } + + try { + const wasmBytes = cachedWasmBytes || generateWasmBytes(); + cachedWasmBytes = wasmBytes; + wasmMemory = new WebAssembly.Memory({ initial: 1 }); + + const importObject = { + js: { mem: wasmMemory }, + }; + + const module = new WebAssembly.Module(wasmBytes.buffer as ArrayBuffer); + wasmInstance = new WebAssembly.Instance(module, importObject); + wasmCompress4x = wasmInstance.exports.compress4x as () => void; + wasmCompressChunks4x = wasmInstance.exports.compressChunks4x as () => void; + wasmCompressParent = wasmInstance.exports.compressParent as () => void; + wasmMemoryView = new Uint8Array(wasmMemory.buffer); + wasmMemoryView32 = new Uint32Array(wasmMemory.buffer); + + // Set up arena views for Merkle tree operations + setupArenaViews(); + + return true; + } catch (e) { + console.warn("Failed to initialize WASM SIMD:", e); + return false; + } +} + +/** + * Memory offsets for SIMD data layout + * + * WASM Arena Pattern: All working buffers live in WASM memory (64KB page) + * This eliminates JS heap allocations during hashing operations. + */ +export const SIMD_MEMORY = { + // SIMD compress4x working area (used by WASM code) - single block + BLOCK_WORDS: 0, // 4 x 16 words = 512 bytes (transposed layout) + CHAINING_VALUES: 512, // 4 x 8 words = 128 bytes + OUTPUT: 640, // 4 x 8 words = 128 bytes + COUNTER_LOW: 768, // 4 words = 16 bytes + COUNTER_HIGH: 784, // 4 words = 16 bytes + BLOCK_LEN: 800, // 4 words = 16 bytes + FLAGS: 816, // 4 words = 16 bytes + // End of single-block SIMD working area: 832 bytes + + // SIMD compressChunks4x working area - 16 blocks batched + // Each block position has 16 v128 values (one per message word) = 256 bytes + // 16 block positions = 16 × 256 = 4096 bytes + BATCH_BLOCK_WORDS: 832, // 16 positions × 256 bytes = 4096 bytes (transposed), ends at 4928 + BATCH_CV: 4928, // 4 × 8 words × 4 bytes = 128 bytes (working CVs), ends at 5056 + BATCH_COUNTER_LOW: 5056, // 4 words × 4 bytes = 16 bytes (per-chunk counters), ends at 5072 + BATCH_FLAGS_BASE: 5072, // 4 words × 4 bytes = 16 bytes (base flags, no START/END), ends at 5088 + BATCH_OUTPUT: 5088, // 4 × 8 words × 4 bytes = 128 bytes (final output), ends at 5216 + // End of batch working area: 5216 bytes + + // WASM Arena: JS working buffers (accessed via TypedArray views) + CV_STACK: 5216, // 64 levels × 8 words × 4 bytes = 2048 bytes, ends at 7264 + PARENT_BLOCK: 7264, // 16 words × 4 bytes = 64 bytes, ends at 7328 + CHUNK_CV: 7328, // 8 words × 4 bytes = 32 bytes, ends at 7360 + TEMP_CVS: 7360, // 4 × 8 words × 4 bytes = 128 bytes, ends at 7488 + // Total arena usage: ~7488 bytes (fits comfortably in 64KB page) +} as const; + +// Arena views - created once when SIMD initializes +let arenaCvStack: Uint32Array | null = null; +let arenaParentBlock: Uint32Array | null = null; +let arenaChunkCv: Uint32Array | null = null; +let arenaTempCvs: Uint32Array | null = null; + +// Batch mode arena views +let arenaBatchBlockWords: Uint32Array | null = null; +let arenaBatchCv: Uint32Array | null = null; +let arenaBatchCounterLow: Uint32Array | null = null; +let arenaBatchFlagsBase: Uint32Array | null = null; +let arenaBatchOutput: Uint32Array | null = null; + +/** + * Get the WASM memory views for writing input data. + */ +export function getSimdMemory(): { view: Uint8Array; view32: Uint32Array } | null { + if (!wasmMemoryView || !wasmMemoryView32) return null; + return { view: wasmMemoryView, view32: wasmMemoryView32 }; +} + +/** + * Get the arena buffers for Merkle tree operations. + * These TypedArray views are backed by WASM memory - zero JS heap allocation. + */ +export function getArenaBuffers(): { + cvStack: Uint32Array; + parentBlock: Uint32Array; + chunkCv: Uint32Array; + tempCvs: Uint32Array; +} | null { + if (!arenaCvStack || !arenaParentBlock || !arenaChunkCv || !arenaTempCvs) return null; + return { + cvStack: arenaCvStack, + parentBlock: arenaParentBlock, + chunkCv: arenaChunkCv, + tempCvs: arenaTempCvs, + }; +} + +/** + * Get the batch arena buffers for chunk-level batched operations. + * These TypedArray views are backed by WASM memory - zero JS heap allocation. + */ +export function getBatchArenaBuffers(): { + blockWords: Uint32Array; + cv: Uint32Array; + counterLow: Uint32Array; + flagsBase: Uint32Array; + output: Uint32Array; +} | null { + if ( + !arenaBatchBlockWords || + !arenaBatchCv || + !arenaBatchCounterLow || + !arenaBatchFlagsBase || + !arenaBatchOutput + ) + return null; + return { + blockWords: arenaBatchBlockWords, + cv: arenaBatchCv, + counterLow: arenaBatchCounterLow, + flagsBase: arenaBatchFlagsBase, + output: arenaBatchOutput, + }; +} + +/** + * Run the compress4x function. + * Data must already be set up in WASM memory. + */ +export function runCompress4x(): void { + if (!wasmCompress4x) { + throw new Error("WASM SIMD not initialized. Call initSimdSync() first."); + } + wasmCompress4x(); +} + +/** + * Run the compressChunks4x function. + * Processes 4 full chunks (16 blocks each) in a single WASM call. + * Data must already be set up in batch arena buffers. + */ +export function runCompressChunks4x(): void { + if (!wasmCompressChunks4x) { + throw new Error("WASM SIMD not initialized. Call initSimdSync() first."); + } + wasmCompressChunks4x(); +} + +/** + * Run the compressParent function. + * Compresses a parent node: reads 16 words from PARENT_BLOCK, writes 8 words to CHUNK_CV. + * Data must already be set up in arena buffers (PARENT_BLOCK at offset 7264). + * Output is written to CHUNK_CV at offset 7328. + */ +export function runCompressParent(): void { + if (!wasmCompressParent) { + throw new Error("WASM SIMD not initialized. Call initSimdSync() first."); + } + wasmCompressParent(); +} + +/** + * Check if SIMD is initialized and ready. + */ +export function isSimdReady(): boolean { + return wasmCompress4x !== null; +} diff --git a/node_modules/@huggingface/hub/LICENSE b/node_modules/@huggingface/hub/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..a3ce68eb3a1d6900bebd200d107c47769642a3c1 --- /dev/null +++ b/node_modules/@huggingface/hub/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Hugging Face + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/node_modules/@huggingface/hub/README.md b/node_modules/@huggingface/hub/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ddfee3e33820f1fd80e2e5ef23938ea62b5f330b --- /dev/null +++ b/node_modules/@huggingface/hub/README.md @@ -0,0 +1,228 @@ +# 🤗 Hugging Face Hub API + +Official utilities to use the Hugging Face Hub API. + +## Install + +```console +pnpm add @huggingface/hub + +npm add @huggingface/hub + +yarn add @huggingface/hub +``` + +### Deno + +```ts +// esm.sh +import { uploadFiles, listModels } from "https://esm.sh/@huggingface/hub" +// or npm: +import { uploadFiles, listModels } from "npm:@huggingface/hub" +``` + +Check out the [full documentation](https://huggingface.co/docs/huggingface.js/hub/README). + +## Usage + +For some of the calls, you need to create an account and generate an [access token](https://huggingface.co/settings/tokens). + +Learn how to find free models using the hub package in this [interactive tutorial](https://scrimba.com/scrim/c7BbVPcd?pl=pkVnrP7uP). + +```ts +import * as hub from "@huggingface/hub"; +import type { RepoDesignation } from "@huggingface/hub"; + +const repo: RepoDesignation = { type: "model", name: "myname/some-model" }; + +const {name: username} = await hub.whoAmI({accessToken: "hf_..."}); + +for await (const model of hub.listModels({search: {owner: username}, accessToken: "hf_..."})) { + console.log("My model:", model); +} + +const specificModel = await hub.modelInfo({name: "openai-community/gpt2"}); +await hub.checkRepoAccess({repo, accessToken: "hf_..."}); + +await hub.createRepo({ repo, accessToken: "hf_...", license: "mit" }); + +await hub.uploadFiles({ + repo, + accessToken: "hf_...", + files: [ + // path + blob content + { + path: "file.txt", + content: new Blob(["Hello World"]), + }, + // Local file URL + pathToFileURL("./pytorch-model.bin"), + // Local folder URL + pathToFileURL("./models"), + // Web URL + new URL("https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json"), + // Path + Web URL + { + path: "myfile.bin", + content: new URL("https://huggingface.co/bert-base-uncased/resolve/main/pytorch_model.bin") + } + // Can also work with native File in browsers + ], +}); + +// or + +for await (const progressEvent of await hub.uploadFilesWithProgress({ + repo, + accessToken: "hf_...", + files: [ + ... + ], +})) { + console.log(progressEvent); +} + +// Edit a file by adding prefix & suffix +await commit({ + repo, + accessToken: "hf_...", + operations: [{ + type: "edit", + originalContent: originalFile, + edits: [{ + start: 0, + end: 0, + content: new Blob(["prefix"]) + }, { + start: originalFile.length, + end: originalFile.length, + content: new Blob(["suffix"]) + }] + }] +}) + +await hub.deleteFile({repo, accessToken: "hf_...", path: "myfile.bin"}); + +await (await hub.downloadFile({ repo, path: "README.md" })).text(); + +for await (const fileInfo of hub.listFiles({repo})) { + console.log(fileInfo); +} + +await hub.deleteRepo({ repo, accessToken: "hf_..." }); +``` + +## CLI usage + +You can use `@huggingface/hub` in CLI mode to upload files and folders to your repo. + +```console +npx @huggingface/hub upload coyotte508/test-model . +npx @huggingface/hub upload datasets/coyotte508/test-dataset . +# Same thing +npx @huggingface/hub upload --repo-type dataset coyotte508/test-dataset . +# Upload new data with 0 history in a separate branch +npx @huggingface/hub branch create coyotte508/test-model release --empty +npx @huggingface/hub upload coyotte508/test-model . --revision release + +npx @huggingface/hub --help +npx @huggingface/hub upload --help +``` + +You can also install globally with `npm install -g @huggingface/hub`. Then you can do: + +```console +hfjs upload coyotte508/test-model . + +hfjs branch create --repo-type dataset coyotte508/test-dataset release --empty +hfjs upload --repo-type dataset coyotte508/test-dataset . --revision release + +hfjs --help +hfjs upload --help + +hfjs help jobs +``` + +## OAuth Login + +It's possible to login using OAuth (["Sign in with HF"](https://huggingface.co/docs/hub/oauth)). + +This will allow you get an access token to use some of the API, depending on the scopes set inside the Space or the OAuth App. + +```ts +import { oauthLoginUrl, oauthHandleRedirectIfPresent } from "@huggingface/hub"; + +const oauthResult = await oauthHandleRedirectIfPresent(); + +if (!oauthResult) { + // If the user is not logged in, redirect to the login page + window.location.href = await oauthLoginUrl(); +} + +// You can use oauthResult.accessToken, oauthResult.accessTokenExpiresAt and oauthResult.userInfo +console.log(oauthResult); +``` + +Checkout the demo: https://huggingface.co/spaces/huggingfacejs/client-side-oauth + +## Hugging face cache + +The `@huggingface/hub` package provide basic capabilities to scan the cache directory. Learn more about [Manage huggingface_hub cache-system](https://huggingface.co/docs/huggingface_hub/en/guides/manage-cache). + +### `scanCacheDir` + +You can get the list of cached repositories using the `scanCacheDir` function. + +```ts +import { scanCacheDir } from "@huggingface/hub"; + +const result = await scanCacheDir(); + +console.log(result); +``` +Note: this does not work in the browser + +### `downloadFileToCacheDir` + +You can cache a file of a repository using the `downloadFileToCacheDir` function. + +```ts +import { downloadFileToCacheDir } from "@huggingface/hub"; + +const file = await downloadFileToCacheDir({ + repo: 'foo/bar', + path: 'README.md' +}); + +console.log(file); +``` +Note: this does not work in the browser + +### `snapshotDownload` + +You can download an entire repository at a given revision in the cache directory using the `snapshotDownload` function. + +```ts +import { snapshotDownload } from "@huggingface/hub"; + +const directory = await snapshotDownload({ + repo: 'foo/bar', +}); + +console.log(directory); +``` +The code use internally the `downloadFileToCacheDir` function. + +Note: this does not work in the browser + +## Performance considerations + +When uploading large files, you may want to run the `commit` calls inside a worker, to offload the sha256 computations. + +Remote resources and local files should be passed as `URL` whenever it's possible so they can be lazy loaded in chunks to reduce RAM usage. Passing a `File` inside the browser's context is fine, because it natively behaves as a `Blob`. + +Under the hood, `@huggingface/hub` uses a lazy blob implementation to load the file. + +## Dependencies + +- `@huggingface/tasks` : Typings only diff --git a/node_modules/@huggingface/hub/dist/FileBlob-RUOT7DBI.mjs b/node_modules/@huggingface/hub/dist/FileBlob-RUOT7DBI.mjs new file mode 100644 index 0000000000000000000000000000000000000000..2b6b4adc748a63383770ad30c7cf35609c057f5f --- /dev/null +++ b/node_modules/@huggingface/hub/dist/FileBlob-RUOT7DBI.mjs @@ -0,0 +1,91 @@ +import "./chunk-FFYIGW52.mjs"; + +// src/utils/FileBlob.ts +import { createReadStream } from "fs"; +import { open, stat } from "fs/promises"; +import { Readable } from "stream"; +import { fileURLToPath } from "url"; +var FileBlob = class extends Blob { + /** + * Creates a new FileBlob on the provided file. + * + * @param path Path to the file to be lazy readed + */ + static async create(path) { + path = path instanceof URL ? fileURLToPath(path) : path; + const { size } = await stat(path); + const fileBlob = new FileBlob(path, 0, size); + return fileBlob; + } + path; + start; + end; + constructor(path, start, end) { + super(); + this.path = path; + this.start = start; + this.end = end; + } + /** + * Returns the size of the blob. + */ + get size() { + return this.end - this.start; + } + /** + * Returns a new instance of FileBlob that is a slice of the current one. + * + * The slice is inclusive of the start and exclusive of the end. + * + * The slice method does not supports negative start/end. + * + * @param start beginning of the slice + * @param end end of the slice + */ + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on FileBlob.slice"); + } + const slice = new FileBlob(this.path, this.start + start, Math.min(this.start + end, this.end)); + return slice; + } + /** + * Read the part of the file delimited by the FileBlob and returns it as an ArrayBuffer. + */ + async arrayBuffer() { + const slice = await this.execute((file) => file.read(Buffer.alloc(this.size), 0, this.size, this.start)); + return slice.buffer; + } + /** + * Read the part of the file delimited by the FileBlob and returns it as a string. + */ + async text() { + const buffer = await this.arrayBuffer(); + return buffer.toString("utf8"); + } + /** + * Returns a stream around the part of the file delimited by the FileBlob. + */ + stream() { + if (this.start === this.end) { + return new Blob([]).stream(); + } + return Readable.toWeb(createReadStream(this.path, { start: this.start, end: this.end - 1 })); + } + /** + * We are opening and closing the file for each action to prevent file descriptor leaks. + * + * It is an intended choice of developer experience over performances. + */ + async execute(action) { + const file = await open(this.path, "r"); + try { + return await action(file); + } finally { + await file.close(); + } + } +}; +export { + FileBlob +}; diff --git a/node_modules/@huggingface/hub/dist/browser/FileBlob-7MRLQ6TG.mjs b/node_modules/@huggingface/hub/dist/browser/FileBlob-7MRLQ6TG.mjs new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/node_modules/@huggingface/hub/dist/browser/FileBlob-YC2EPDW4.js b/node_modules/@huggingface/hub/dist/browser/FileBlob-YC2EPDW4.js new file mode 100644 index 0000000000000000000000000000000000000000..9a390c31f71bc7eae1522a280a2dc8f6723185bf --- /dev/null +++ b/node_modules/@huggingface/hub/dist/browser/FileBlob-YC2EPDW4.js @@ -0,0 +1 @@ +"use strict"; \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/browser/index.js b/node_modules/@huggingface/hub/dist/browser/index.js new file mode 100644 index 0000000000000000000000000000000000000000..0d884b77f131ff1f3149bfdce7caa5dbc1f482b5 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/browser/index.js @@ -0,0 +1,6001 @@ +"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { newObj[key] = obj[key]; } } } newObj.default = obj; return newObj; } } function _nullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return rhsFn(); } } async function _asyncNullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return await rhsFn(); } } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; } var _class; var _class2; var _class3;// src/consts.ts +var HUB_URL = "https://huggingface.co"; + +// src/error.ts +async function createApiError(response, opts) { + const error = new HubApiError(response.url, response.status, _nullishCoalesce(response.headers.get("X-Request-Id"), () => ( _optionalChain([opts, 'optionalAccess', _2 => _2.requestId])))); + error.message = `Api error with status ${error.statusCode}${_optionalChain([opts, 'optionalAccess', _3 => _3.message]) ? `. ${opts.message}` : ""}`; + const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : void 0].filter(Boolean).join(". "); + if (_optionalChain([response, 'access', _4 => _4.headers, 'access', _5 => _5.get, 'call', _6 => _6("Content-Type"), 'optionalAccess', _7 => _7.startsWith, 'call', _8 => _8("application/json")])) { + const json = await response.json(); + error.message = json.error || json.message || error.message; + if (json.error_description) { + error.message = error.message ? error.message + `: ${json.error_description}` : json.error_description; + } + error.data = json; + } else { + error.data = { message: await response.text() }; + } + error.message += `. ${trailer}`; + throw error; +} +var HubApiError = class extends Error { + + + + + constructor(url, statusCode, requestId, message) { + super(message); + this.statusCode = statusCode; + this.requestId = requestId; + this.url = url; + } +}; +var InvalidApiResponseFormatError = class extends Error { +}; + +// src/utils/checkCredentials.ts +function checkAccessToken(accessToken) { + if (!accessToken.startsWith("hf_")) { + throw new TypeError("Your access token must start with 'hf_'"); + } +} +function checkCredentials(params) { + if (params.accessToken) { + checkAccessToken(params.accessToken); + return params.accessToken; + } + if (_optionalChain([params, 'access', _9 => _9.credentials, 'optionalAccess', _10 => _10.accessToken])) { + checkAccessToken(params.credentials.accessToken); + return params.credentials.accessToken; + } +} + +// src/utils/toRepoId.ts +function toRepoId(repo) { + if (typeof repo !== "string") { + return repo; + } + if (repo.startsWith("model/") || repo.startsWith("models/")) { + throw new TypeError( + "A repo designation for a model should not start with 'models/', directly specify the model namespace / name" + ); + } + if (repo.startsWith("space/")) { + throw new TypeError("Spaces should start with 'spaces/', plural, not 'space/'"); + } + if (repo.startsWith("dataset/")) { + throw new TypeError("Datasets should start with 'datasets/', plural, not 'dataset/'"); + } + if (repo.startsWith("bucket/")) { + throw new TypeError("Buckets should start with 'buckets/', plural, not 'bucket/'"); + } + if (repo.startsWith("kernel/")) { + throw new TypeError("Kernels should start with 'kernels/', plural, not 'kernel/'"); + } + const slashes = repo.split("/").length - 1; + if (repo.startsWith("spaces/")) { + if (slashes !== 2) { + throw new TypeError("Space Id must include namespace and name of the space"); + } + return { + type: "space", + name: repo.slice("spaces/".length) + }; + } + if (repo.startsWith("datasets/")) { + if (slashes > 2) { + throw new TypeError("Too many slashes in repo designation: " + repo); + } + return { + type: "dataset", + name: repo.slice("datasets/".length) + }; + } + if (repo.startsWith("buckets/")) { + if (slashes !== 2) { + throw new TypeError("Bucket Id must include namespace and name of the bucket"); + } + return { + type: "bucket", + name: repo.slice("buckets/".length) + }; + } + if (repo.startsWith("kernels/")) { + if (slashes !== 2) { + throw new TypeError("Kernel Id must include namespace and name of the kernel"); + } + return { + type: "kernel", + name: repo.slice("kernels/".length) + }; + } + if (slashes > 1) { + throw new TypeError("Too many slashes in repo designation: " + repo); + } + return { + type: "model", + name: repo + }; +} + +// src/lib/check-repo-access.ts +async function checkRepoAccess(params) { + const accessToken = params && checkCredentials(params); + const repoId = toRepoId(params.repo); + const response = await (params.fetch || fetch)(`${_optionalChain([params, 'optionalAccess', _11 => _11.hubUrl]) || HUB_URL}/api/${repoId.type}s/${repoId.name}`, { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + }); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/utils/range.ts +function range(n, b) { + return b ? Array(b - n).fill(0).map((_, i) => n + i) : Array(n).fill(0).map((_, i) => i); +} + +// src/utils/chunk.ts +function chunk(arr, chunkSize) { + if (isNaN(chunkSize) || chunkSize < 1) { + throw new RangeError("Invalid chunk size: " + chunkSize); + } + if (!arr.length) { + return []; + } + if (arr.length <= chunkSize) { + return [arr]; + } + return range(Math.ceil(arr.length / chunkSize)).map((i) => { + return arr.slice(i * chunkSize, (i + 1) * chunkSize); + }); +} + +// src/utils/promisesQueue.ts +async function promisesQueue(factories, concurrency) { + const results = []; + const executing = /* @__PURE__ */ new Set(); + let index = 0; + for (const factory of factories) { + const closureIndex = index++; + const e = factory().then((r) => { + results[closureIndex] = r; + executing.delete(e); + }); + executing.add(e); + if (executing.size >= concurrency) { + await Promise.race(executing); + } + } + await Promise.all(executing); + return results; +} + +// src/utils/promisesQueueStreaming.ts +async function promisesQueueStreaming(factories, concurrency) { + const executing = []; + for await (const factory of factories) { + const e = factory().then(() => { + executing.splice(executing.indexOf(e), 1); + }); + executing.push(e); + if (executing.length >= concurrency) { + await Promise.race(executing); + } + } + await Promise.all(executing); +} + +// src/utils/eventToGenerator.ts +async function* eventToGenerator(cb) { + const promises = []; + function addPromise() { + let resolve2; + let reject; + const p = new Promise((res, rej) => { + resolve2 = res; + reject = rej; + }); + promises.push({ p, resolve: resolve2, reject }); + } + addPromise(); + const callbackRes = Promise.resolve().then( + () => cb( + (y) => { + addPromise(); + _optionalChain([promises, 'access', _12 => _12.at, 'call', _13 => _13(-2), 'optionalAccess', _14 => _14.resolve, 'call', _15 => _15({ done: false, value: y })]); + }, + (r) => { + addPromise(); + _optionalChain([promises, 'access', _16 => _16.at, 'call', _17 => _17(-2), 'optionalAccess', _18 => _18.resolve, 'call', _19 => _19({ done: true, value: r })]); + }, + (err) => _optionalChain([promises, 'access', _20 => _20.shift, 'call', _21 => _21(), 'optionalAccess', _22 => _22.reject, 'call', _23 => _23(err)]) + ) + ).catch((err) => _optionalChain([promises, 'access', _24 => _24.shift, 'call', _25 => _25(), 'optionalAccess', _26 => _26.reject, 'call', _27 => _27(err)])); + while (1) { + const p = promises[0]; + if (!p) { + throw new Error("Logic error in eventGenerator, promises should never be empty"); + } + const result = await p.p; + promises.shift(); + if (result.done) { + await callbackRes; + return result.value; + } + yield result.value; + } + throw new Error("Unreachable"); +} + +// src/utils/hexFromBytes.ts +function hexFromBytes(arr) { + if (globalThis.Buffer) { + return globalThis.Buffer.from(arr).toString("hex"); + } else { + const bin = []; + arr.forEach((byte) => { + bin.push(byte.toString(16).padStart(2, "0")); + }); + return bin.join(""); + } +} + +// src/utils/isBackend.ts +var isBrowser = typeof window !== "undefined" && typeof window.document !== "undefined"; +var isWebWorker = typeof self === "object" && self.constructor && self.constructor.name === "DedicatedWorkerGlobalScope"; +var isBackend = !isBrowser && !isWebWorker; + +// src/utils/isFrontend.ts +var isFrontend = !isBackend; + +// src/utils/sha256.ts +async function getWebWorkerCode() { + const sha256Module = await Promise.resolve().then(() => _interopRequireWildcard(require("./sha256-wrapper-6KQBPSEU.js"))); + return URL.createObjectURL(new Blob([sha256Module.createSHA256WorkerCode()])); +} +var pendingWorkers = []; +var runningWorkers = /* @__PURE__ */ new Set(); +var resolve; +var waitPromise = new Promise((r) => { + resolve = r; +}); +async function getWorker(poolSize) { + { + const worker2 = pendingWorkers.pop(); + if (worker2) { + runningWorkers.add(worker2); + return worker2; + } + } + if (!poolSize) { + const worker2 = new Worker(await getWebWorkerCode()); + runningWorkers.add(worker2); + return worker2; + } + if (poolSize <= 0) { + throw new TypeError("Invalid webworker pool size: " + poolSize); + } + while (runningWorkers.size >= poolSize) { + await waitPromise; + } + const worker = new Worker(await getWebWorkerCode()); + runningWorkers.add(worker); + return worker; +} +async function freeWorker(worker, poolSize) { + if (!poolSize) { + return destroyWorker(worker); + } + runningWorkers.delete(worker); + pendingWorkers.push(worker); + const r = resolve; + waitPromise = new Promise((r2) => { + resolve = r2; + }); + r(); +} +function destroyWorker(worker) { + runningWorkers.delete(worker); + worker.terminate(); + const r = resolve; + waitPromise = new Promise((r2) => { + resolve = r2; + }); + r(); +} +async function* sha256(buffer, opts) { + yield 0; + const maxCryptoSize = typeof _optionalChain([opts, 'optionalAccess', _28 => _28.useWebWorker]) === "object" && _optionalChain([opts, 'optionalAccess', _29 => _29.useWebWorker, 'access', _30 => _30.minSize]) !== void 0 ? opts.useWebWorker.minSize : 1e7; + if (buffer.size < maxCryptoSize && _optionalChain([globalThis, 'access', _31 => _31.crypto, 'optionalAccess', _32 => _32.subtle])) { + const res = hexFromBytes( + new Uint8Array( + await globalThis.crypto.subtle.digest("SHA-256", buffer instanceof Blob ? await buffer.arrayBuffer() : buffer) + ) + ); + yield 1; + return res; + } + if (isFrontend) { + if (_optionalChain([opts, 'optionalAccess', _33 => _33.useWebWorker])) { + try { + const poolSize = typeof _optionalChain([opts, 'optionalAccess', _34 => _34.useWebWorker]) === "object" ? opts.useWebWorker.poolSize : void 0; + const worker = await getWorker(poolSize); + let messageHandler; + let errorHandler; + const cleanup = () => { + worker.removeEventListener("message", messageHandler); + worker.removeEventListener("error", errorHandler); + }; + return yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + messageHandler = (event) => { + if (event.data.sha256) { + cleanup(); + freeWorker(worker, poolSize); + returnCallback(event.data.sha256); + } else if (event.data.progress) { + yieldCallback(event.data.progress); + try { + _optionalChain([opts, 'access', _35 => _35.abortSignal, 'optionalAccess', _36 => _36.throwIfAborted, 'call', _37 => _37()]); + } catch (err) { + cleanup(); + destroyWorker(worker); + rejectCallback(err); + } + } else { + cleanup(); + destroyWorker(worker); + rejectCallback(event); + } + }; + errorHandler = (event) => { + cleanup(); + destroyWorker(worker); + rejectCallback(event.error); + }; + if (_optionalChain([opts, 'optionalAccess', _38 => _38.abortSignal])) { + try { + _optionalChain([opts, 'access', _39 => _39.abortSignal, 'optionalAccess', _40 => _40.throwIfAborted, 'call', _41 => _41()]); + } catch (err) { + cleanup(); + destroyWorker(worker); + rejectCallback(_nullishCoalesce(opts.abortSignal.reason, () => ( new DOMException("Aborted", "AbortError")))); + return; + } + const abortListener = () => { + cleanup(); + destroyWorker(worker); + rejectCallback(_nullishCoalesce(_optionalChain([opts, 'access', _42 => _42.abortSignal, 'optionalAccess', _43 => _43.reason]), () => ( new DOMException("Aborted", "AbortError")))); + _optionalChain([opts, 'access', _44 => _44.abortSignal, 'optionalAccess', _45 => _45.removeEventListener, 'call', _46 => _46("abort", abortListener)]); + }; + opts.abortSignal.addEventListener("abort", abortListener); + } + worker.addEventListener("message", messageHandler); + worker.addEventListener("error", errorHandler); + worker.postMessage({ file: buffer }); + }); + } catch (err) { + console.warn("Failed to use web worker for sha256", err); + } + } + if (!wasmModule) { + wasmModule = await Promise.resolve().then(() => _interopRequireWildcard(require("./sha256-wrapper-6KQBPSEU.js"))); + } + const sha2562 = await wasmModule.createSHA256(); + sha2562.init(); + const reader = buffer.stream().getReader(); + const total = buffer.size; + let bytesDone = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + sha2562.update(value); + bytesDone += value.length; + yield bytesDone / total; + _optionalChain([opts, 'optionalAccess', _47 => _47.abortSignal, 'optionalAccess', _48 => _48.throwIfAborted, 'call', _49 => _49()]); + } + return sha2562.digest("hex"); + } + if (!cryptoModule) { + cryptoModule = await Promise.resolve().then(() => _interopRequireWildcard(require("./sha256-node-FT2Y3VXD.js"))); + } + return yield* cryptoModule.sha256Node(buffer, { abortSignal: _optionalChain([opts, 'optionalAccess', _50 => _50.abortSignal]) }); +} +var cryptoModule; +var wasmModule; + +// src/utils/WebBlob.ts +var WebBlob = class extends Blob { + static async create(url, opts) { + const customFetch = _nullishCoalesce(_optionalChain([opts, 'optionalAccess', _51 => _51.fetch]), () => ( fetch)); + const probe = await customFetch(url, { + headers: { + Range: "bytes=0-0", + ..._optionalChain([opts, 'optionalAccess', _52 => _52.accessToken]) && { Authorization: `Bearer ${opts.accessToken}` } + } + }); + if (!probe.ok) { + throw await createApiError(probe); + } + const contentType = probe.headers.get("content-type") || ""; + if (probe.status === 206) { + const totalSize = Number(_optionalChain([probe, 'access', _53 => _53.headers, 'access', _54 => _54.get, 'call', _55 => _55("content-range"), 'optionalAccess', _56 => _56.split, 'call', _57 => _57("/"), 'access', _58 => _58.pop, 'call', _59 => _59()])); + await _optionalChain([probe, 'access', _60 => _60.body, 'optionalAccess', _61 => _61.cancel, 'call', _62 => _62()]); + if (Number.isFinite(totalSize) && totalSize >= (_nullishCoalesce(_optionalChain([opts, 'optionalAccess', _63 => _63.cacheBelow]), () => ( 1e6)))) { + return new WebBlob(url, 0, totalSize, contentType, true, customFetch, _optionalChain([opts, 'optionalAccess', _64 => _64.accessToken])); + } + const full = await customFetch(url, { + ..._optionalChain([opts, 'optionalAccess', _65 => _65.accessToken]) && { headers: { Authorization: `Bearer ${opts.accessToken}` } } + }); + if (!full.ok) { + throw await createApiError(full); + } + return full.blob(); + } + return probe.blob(); + } + + + + + + + + constructor(url, start, end, contentType, full, customFetch, accessToken) { + super([]); + this.url = url; + this.start = start; + this.end = end; + this.contentType = contentType; + this.full = full; + this.fetch = customFetch; + this.accessToken = accessToken; + } + get size() { + return this.end - this.start; + } + get type() { + return this.contentType; + } + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on WebBlob.slice"); + } + const slice = new WebBlob( + this.url, + this.start + start, + Math.min(this.start + end, this.end), + this.contentType, + start === 0 && end === this.size ? this.full : false, + this.fetch, + this.accessToken + ); + return slice; + } + async arrayBuffer() { + const result = await this.fetchRange(); + return result.arrayBuffer(); + } + async text() { + const result = await this.fetchRange(); + return result.text(); + } + stream() { + const stream = new TransformStream(); + this.fetchRange().then((response) => _optionalChain([response, 'access', _66 => _66.body, 'optionalAccess', _67 => _67.pipeThrough, 'call', _68 => _68(stream)])).catch((error) => stream.writable.abort(error.message)); + return stream.readable; + } + fetchRange() { + const fetch2 = this.fetch; + if (this.full) { + return fetch2(this.url, { + ...this.accessToken && { + headers: { + Authorization: `Bearer ${this.accessToken}` + } + } + }).then((resp) => resp.ok ? resp : createApiError(resp)); + } + return fetch2(this.url, { + headers: { + Range: `bytes=${this.start}-${this.end - 1}`, + ...this.accessToken && { Authorization: `Bearer ${this.accessToken}` } + } + }).then((resp) => resp.ok ? resp : createApiError(resp)); + } +}; + +// src/utils/base64FromBytes.ts +function base64FromBytes(arr) { + if (globalThis.Buffer) { + return globalThis.Buffer.from(arr).toString("base64"); + } else { + const bin = []; + arr.forEach((byte) => { + bin.push(String.fromCharCode(byte)); + }); + return globalThis.btoa(bin.join("")); + } +} + +// src/utils/createBlobs.ts +async function createBlobs(url, destPath, opts) { + if (url.protocol === "http:" || url.protocol === "https:") { + const blob = await WebBlob.create(url, { fetch: _optionalChain([opts, 'optionalAccess', _69 => _69.fetch]), accessToken: _optionalChain([opts, 'optionalAccess', _70 => _70.accessToken]) }); + return [{ path: destPath, blob }]; + } + if (isFrontend) { + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); + } + if (url.protocol === "file:") { + const { FileBlob } = await Promise.resolve().then(() => _interopRequireWildcard(require("./FileBlob-YC2EPDW4.js"))); + const { subPaths } = await Promise.resolve().then(() => _interopRequireWildcard(require("./sub-paths-RH3O65LG.js"))); + const paths = await subPaths(url, _optionalChain([opts, 'optionalAccess', _71 => _71.maxFolderDepth])); + if (paths.length === 1 && paths[0].relativePath === ".") { + const blob = await FileBlob.create(url); + return [{ path: destPath, blob }]; + } + return Promise.all( + paths.map(async (path) => ({ + path: `${destPath}/${path.relativePath}`.replace(/\/[.]$/, "").replaceAll("//", "/").replace(/^[.]?\//, ""), + blob: await FileBlob.create(new URL(path.path)) + })) + ); + } + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); +} + +// src/utils/combineUint8Arrays.ts +function combineUint8Arrays(a, b) { + const aLength = a.length; + const combinedBytes = new Uint8Array(aLength + b.length); + combinedBytes.set(a); + combinedBytes.set(b, aLength); + return combinedBytes; +} + +// src/vendor/lz4js/util.ts +function hashU32(a) { + a = a | 0; + a = a + 2127912214 + (a << 12) | 0; + a = a ^ -949894596 ^ a >>> 19; + a = a + 374761393 + (a << 5) | 0; + a = a + -744332180 ^ a << 9; + a = a + -42973499 + (a << 3) | 0; + return a ^ -1252372727 ^ a >>> 16 | 0; +} +function readU64(b, n) { + let x = 0; + x |= b[n++] << 0; + x |= b[n++] << 8; + x |= b[n++] << 16; + x |= b[n++] << 24; + x |= b[n++] << 32; + x |= b[n++] << 40; + x |= b[n++] << 48; + x |= b[n++] << 56; + return x; +} +function readU32(b, n) { + let x = 0; + x |= b[n++] << 0; + x |= b[n++] << 8; + x |= b[n++] << 16; + x |= b[n++] << 24; + return x; +} +function writeU32(b, n, x) { + b[n++] = x >> 0 & 255; + b[n++] = x >> 8 & 255; + b[n++] = x >> 16 & 255; + b[n++] = x >> 24 & 255; +} +function imul(a, b) { + const ah = a >>> 16; + const al = a & 65535; + const bh = b >>> 16; + const bl = b & 65535; + return al * bl + (ah * bl + al * bh << 16) | 0; +} + +// src/vendor/lz4js/xxh32.ts +var prime1 = 2654435761; +var prime2 = 2246822519; +var prime3 = 3266489917; +var prime4 = 668265263; +var prime5 = 374761393; +function rotl32(x, r) { + x = x | 0; + r = r | 0; + return x >>> (32 - r | 0) | x << r | 0; +} +function rotmul32(h, r, m) { + h = h | 0; + r = r | 0; + m = m | 0; + return imul(h >>> (32 - r | 0) | h << r, m) | 0; +} +function shiftxor32(h, s) { + h = h | 0; + s = s | 0; + return h >>> s ^ h | 0; +} +function xxhapply(h, src, m0, s, m1) { + return rotmul32(imul(src, m0) + h, s, m1); +} +function xxh1(h, src, index) { + return rotmul32(h + imul(src[index], prime5), 11, prime1); +} +function xxh4(h, src, index) { + return xxhapply(h, readU32(src, index), prime3, 17, prime4); +} +function xxh16(h, src, index) { + return [ + xxhapply(h[0], readU32(src, index + 0), prime2, 13, prime1), + xxhapply(h[1], readU32(src, index + 4), prime2, 13, prime1), + xxhapply(h[2], readU32(src, index + 8), prime2, 13, prime1), + xxhapply(h[3], readU32(src, index + 12), prime2, 13, prime1) + ]; +} +function xxh32(seed, src, index, len) { + let h; + const l = len; + if (len >= 16) { + h = [seed + prime1 + prime2, seed + prime2, seed, seed - prime1]; + while (len >= 16) { + h = xxh16(h, src, index); + index += 16; + len -= 16; + } + h = rotl32(h[0], 1) + rotl32(h[1], 7) + rotl32(h[2], 12) + rotl32(h[3], 18) + l; + } else { + h = seed + prime5 + len >>> 0; + } + while (len >= 4) { + h = xxh4(h, src, index); + index += 4; + len -= 4; + } + while (len > 0) { + h = xxh1(h, src, index); + index++; + len--; + } + h = shiftxor32(imul(shiftxor32(imul(shiftxor32(h, 15), prime2), 13), prime3), 16); + return h >>> 0; +} +var hash = xxh32; + +// src/vendor/lz4js/index.ts +var minMatch = 4; +var matchSearchLimit = 12; +var minTrailingLitterals = 5; +var skipTrigger = 6; +var hashSize = 1 << 16; +var mlBits = 4; +var mlMask = (1 << mlBits) - 1; +var runBits = 4; +var runMask = (1 << runBits) - 1; +var blockBuf = makeBuffer(5 << 20); +var hashTable = makeHashTable(); +var magicNum = 407708164; +var fdContentChksum = 4; +var fdContentSize = 8; +var fdBlockChksum = 16; +var fdVersion = 64; +var fdVersionMask = 192; +var bsUncompressed = 2147483648; +var bsDefault = 7; +var bsShift = 4; +var bsMask = 7; +var bsMap = { + 4: 65536, + 5: 262144, + 6: 1048576, + 7: 4194304 +}; +function makeHashTable() { + try { + return new Uint32Array(hashSize); + } catch (error) { + const hashTable2 = new Array(hashSize); + for (let i = 0; i < hashSize; i++) { + hashTable2[i] = 0; + } + return hashTable2; + } +} +function clearHashTable(table) { + for (let i = 0; i < hashSize; i++) { + table[i] = 0; + } +} +function makeBuffer(size) { + return new Uint8Array(size); +} +function sliceArray(array, start, end) { + return array.slice(start, end); +} +function compressBound(n) { + return n + n / 255 + 16 | 0; +} +function decompressBound(src) { + let sIndex = 0; + if (readU32(src, sIndex) !== magicNum) { + throw new Error("invalid magic number"); + } + sIndex += 4; + const descriptor = src[sIndex++]; + if ((descriptor & fdVersionMask) !== fdVersion) { + throw new Error("incompatible descriptor version " + (descriptor & fdVersionMask)); + } + const useBlockSum = (descriptor & fdBlockChksum) !== 0; + const useContentSize = (descriptor & fdContentSize) !== 0; + const bsIdx = src[sIndex++] >> bsShift & bsMask; + if (bsMap[bsIdx] === void 0) { + throw new Error("invalid block size " + bsIdx); + } + const maxBlockSize = bsMap[bsIdx]; + if (useContentSize) { + return readU64(src, sIndex); + } + sIndex++; + let maxSize = 0; + while (true) { + let blockSize = readU32(src, sIndex); + sIndex += 4; + if (blockSize & bsUncompressed) { + blockSize &= ~bsUncompressed; + maxSize += blockSize; + } else if (blockSize > 0) { + maxSize += maxBlockSize; + } + if (blockSize === 0) { + return maxSize; + } + if (useBlockSum) { + sIndex += 4; + } + sIndex += blockSize; + } +} +function decompressBlock(src, dst, sIndex, sLength, dIndex) { + let mLength, mOffset, sEnd, n, i; + const hasCopyWithin = dst.copyWithin !== void 0 && dst.fill !== void 0; + sEnd = sIndex + sLength; + while (sIndex < sEnd) { + const token = src[sIndex++]; + let literalCount = token >> 4; + if (literalCount > 0) { + if (literalCount === 15) { + while (true) { + literalCount += src[sIndex]; + if (src[sIndex++] !== 255) { + break; + } + } + } + for (n = sIndex + literalCount; sIndex < n; ) { + dst[dIndex++] = src[sIndex++]; + } + } + if (sIndex >= sEnd) { + break; + } + mLength = token & 15; + mOffset = src[sIndex++] | src[sIndex++] << 8; + if (mLength === 15) { + while (true) { + mLength += src[sIndex]; + if (src[sIndex++] !== 255) { + break; + } + } + } + mLength += minMatch; + if (hasCopyWithin && mOffset === 1) { + dst.fill(dst[dIndex - 1] | 0, dIndex, dIndex + mLength); + dIndex += mLength; + } else if (hasCopyWithin && mOffset > mLength && mLength > 31) { + dst.copyWithin(dIndex, dIndex - mOffset, dIndex - mOffset + mLength); + dIndex += mLength; + } else { + for (i = dIndex - mOffset, n = i + mLength; i < n; ) { + dst[dIndex++] = dst[i++] | 0; + } + } + } + return dIndex; +} +function compressBlock(src, dst, sIndex, sLength, hashTable2) { + let mIndex, mAnchor, mLength, mOffset, mStep; + let literalCount, dIndex, sEnd, n; + dIndex = 0; + sEnd = sLength + sIndex; + mAnchor = sIndex; + let searchMatchCount = (1 << skipTrigger) + 3; + while (sIndex <= sEnd - matchSearchLimit) { + const seq = readU32(src, sIndex); + let hash2 = hashU32(seq) >>> 0; + hash2 = (hash2 >> 16 ^ hash2) >>> 0 & 65535; + mIndex = hashTable2[hash2] - 1; + hashTable2[hash2] = sIndex + 1; + if (mIndex < 0 || sIndex - mIndex >>> 16 > 0 || readU32(src, mIndex) !== seq) { + mStep = searchMatchCount++ >> skipTrigger; + sIndex += mStep; + continue; + } + searchMatchCount = (1 << skipTrigger) + 3; + literalCount = sIndex - mAnchor; + mOffset = sIndex - mIndex; + sIndex += minMatch; + mIndex += minMatch; + mLength = sIndex; + while (sIndex < sEnd - minTrailingLitterals && src[sIndex] === src[mIndex]) { + sIndex++; + mIndex++; + } + mLength = sIndex - mLength; + const token = mLength < mlMask ? mLength : mlMask; + if (literalCount >= runMask) { + dst[dIndex++] = (runMask << mlBits) + token; + for (n = literalCount - runMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } else { + dst[dIndex++] = (literalCount << mlBits) + token; + } + for (let i = 0; i < literalCount; i++) { + dst[dIndex++] = src[mAnchor + i]; + } + dst[dIndex++] = mOffset; + dst[dIndex++] = mOffset >> 8; + if (mLength >= mlMask) { + for (n = mLength - mlMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } + mAnchor = sIndex; + } + if (mAnchor === 0) { + return 0; + } + literalCount = sEnd - mAnchor; + if (literalCount >= runMask) { + dst[dIndex++] = runMask << mlBits; + for (n = literalCount - runMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } else { + dst[dIndex++] = literalCount << mlBits; + } + sIndex = mAnchor; + while (sIndex < sEnd) { + dst[dIndex++] = src[sIndex++]; + } + return dIndex; +} +function decompressFrame(src, dst) { + let useBlockSum, useContentSum, useContentSize, descriptor; + let sIndex = 0; + let dIndex = 0; + if (readU32(src, sIndex) !== magicNum) { + throw new Error("invalid magic number"); + } + sIndex += 4; + descriptor = src[sIndex++]; + if ((descriptor & fdVersionMask) !== fdVersion) { + throw new Error("incompatible descriptor version"); + } + useBlockSum = (descriptor & fdBlockChksum) !== 0; + useContentSum = (descriptor & fdContentChksum) !== 0; + useContentSize = (descriptor & fdContentSize) !== 0; + const bsIdx = src[sIndex++] >> bsShift & bsMask; + if (bsMap[bsIdx] === void 0) { + throw new Error("invalid block size"); + } + if (useContentSize) { + sIndex += 8; + } + sIndex++; + while (true) { + var compSize; + compSize = readU32(src, sIndex); + sIndex += 4; + if (compSize === 0) { + break; + } + if (useBlockSum) { + sIndex += 4; + } + if ((compSize & bsUncompressed) !== 0) { + compSize &= ~bsUncompressed; + for (let j = 0; j < compSize; j++) { + dst[dIndex++] = src[sIndex++]; + } + } else { + dIndex = decompressBlock(src, dst, sIndex, compSize, dIndex); + sIndex += compSize; + } + } + if (useContentSum) { + sIndex += 4; + } + return dIndex; +} +function compressFrame(src, dst) { + let dIndex = 0; + writeU32(dst, dIndex, magicNum); + dIndex += 4; + dst[dIndex++] = fdVersion; + dst[dIndex++] = bsDefault << bsShift; + dst[dIndex] = hash(0, dst, 4, dIndex - 4) >> 8; + dIndex++; + const maxBlockSize = bsMap[bsDefault]; + let remaining = src.length; + let sIndex = 0; + clearHashTable(hashTable); + while (remaining > 0) { + let compSize = 0; + const blockSize = remaining > maxBlockSize ? maxBlockSize : remaining; + compSize = compressBlock(src, blockBuf, sIndex, blockSize, hashTable); + if (compSize > blockSize || compSize === 0) { + writeU32(dst, dIndex, 2147483648 | blockSize); + dIndex += 4; + for (let z = sIndex + blockSize; sIndex < z; ) { + dst[dIndex++] = src[sIndex++]; + } + remaining -= blockSize; + } else { + writeU32(dst, dIndex, compSize); + dIndex += 4; + for (let j = 0; j < compSize; ) { + dst[dIndex++] = blockBuf[j++]; + } + sIndex += blockSize; + remaining -= blockSize; + } + } + writeU32(dst, dIndex, 0); + dIndex += 4; + return dIndex; +} +function decompress(src, maxSize) { + let dst, size; + if (maxSize === void 0) { + maxSize = decompressBound(src); + } + dst = makeBuffer(maxSize); + size = decompressFrame(src, dst); + if (size !== maxSize) { + dst = sliceArray(dst, 0, size); + } + return dst; +} +function compress(src, maxSize) { + let dst, size; + if (maxSize === void 0) { + maxSize = compressBound(src.length); + } + dst = makeBuffer(maxSize); + size = compressFrame(src, dst); + if (size !== maxSize) { + dst = sliceArray(dst, 0, size); + } + return dst; +} + +// src/utils/RangeList.ts +var RangeList = (_class = class {constructor() { _class.prototype.__init.call(this); } + __init() {this.ranges = []} + /** + * Add a range to the list. If it overlaps with existing ranges, + * it will split them and increment reference counts accordingly. + */ + add(start, end) { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + const overlappingRanges = []; + for (let i = 0; i < this.ranges.length; i++) { + const range2 = this.ranges[i]; + if (start < range2.end && end > range2.start) { + overlappingRanges.push({ index: i, range: range2 }); + } + if (range2.data !== null) { + throw new Error("Overlapping range already has data"); + } + } + if (overlappingRanges.length === 0) { + this.ranges.push({ start, end, refCount: 1, data: null }); + this.ranges.sort((a, b) => a.start - b.start); + return; + } + const newRanges = []; + let currentPos = start; + for (let i = 0; i < overlappingRanges.length; i++) { + const { range: range2 } = overlappingRanges[i]; + if (currentPos < range2.start) { + newRanges.push({ + start: currentPos, + end: range2.start, + refCount: 1, + data: null + }); + } else if (range2.start < currentPos) { + newRanges.push({ + start: range2.start, + end: currentPos, + refCount: range2.refCount, + data: null + }); + } + newRanges.push({ + start: Math.max(currentPos, range2.start), + end: Math.min(end, range2.end), + refCount: range2.refCount + 1, + data: null + }); + if (range2.end > end) { + newRanges.push({ + start: end, + end: range2.end, + refCount: range2.refCount, + data: null + }); + } + currentPos = Math.max(currentPos, range2.end); + } + if (currentPos < end) { + newRanges.push({ + start: currentPos, + end, + refCount: 1, + data: null + }); + } + const firstIndex = overlappingRanges[0].index; + const lastIndex = overlappingRanges[overlappingRanges.length - 1].index; + this.ranges.splice(firstIndex, lastIndex - firstIndex + 1, ...newRanges); + this.ranges.sort((a, b) => a.start - b.start); + } + /** + * Remove a range from the list. The range must start and end at existing boundaries. + */ + remove(start, end) { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + const affectedRanges = []; + for (let i = 0; i < this.ranges.length; i++) { + const range2 = this.ranges[i]; + if (start < range2.end && end > range2.start) { + affectedRanges.push({ index: i, range: range2 }); + } + } + if (affectedRanges.length === 0) { + throw new Error("No ranges found to remove"); + } + if (start !== affectedRanges[0].range.start || end !== affectedRanges[affectedRanges.length - 1].range.end) { + throw new Error("Range boundaries must match existing boundaries"); + } + for (let i = 0; i < affectedRanges.length; i++) { + const { range: range2 } = affectedRanges[i]; + range2.refCount--; + } + this.ranges = this.ranges.filter((range2) => range2.refCount > 0); + } + /** + * Get all ranges within the specified boundaries. + */ + getRanges(start, end) { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + return this.ranges.filter((range2) => start < range2.end && end > range2.start); + } + /** + * Get all ranges in the list + */ + getAllRanges() { + return [...this.ranges]; + } +}, _class); + +// src/utils/XetBlob.ts +var JWT_SAFETY_PERIOD = 6e4; +var JWT_CACHE_SIZE = 1e3; +var compressionSchemeLabels = { + [0 /* None */]: "None", + [1 /* LZ4 */]: "LZ4", + [2 /* ByteGroupingLZ4 */]: "ByteGroupingLZ4" +}; +var XET_CHUNK_HEADER_BYTES = 8; +var XetBlob = (_class2 = class extends Blob { + + + + + + __init2() {this.start = 0} + __init3() {this.end = 0} + __init4() {this.internalLogging = false} + + + constructor(params) { + super([]);_class2.prototype.__init2.call(this);_class2.prototype.__init3.call(this);_class2.prototype.__init4.call(this);; + this.fetch = _nullishCoalesce(params.fetch, () => ( fetch.bind(globalThis))); + this.accessToken = checkCredentials(params); + this.refreshUrl = params.refreshUrl; + this.end = params.size; + this.reconstructionUrl = params.reconstructionUrl; + this.hash = params.hash; + this.listener = params.listener; + this.internalLogging = _nullishCoalesce(params.internalLogging, () => ( false)); + if (params.readToken) { + const key = cacheKey({ refreshUrl: this.refreshUrl, initialAccessToken: this.accessToken }); + jwts.set(key, { + accessToken: params.readToken.accessToken, + expiresAt: new Date(params.readToken.exp * 1e3), + casUrl: params.readToken.casUrl + }); + } + } + get size() { + return this.end - this.start; + } + #clone() { + const blob = new XetBlob({ + fetch: this.fetch, + hash: this.hash, + refreshUrl: this.refreshUrl, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + reconstructionUrl: this.reconstructionUrl, + size: this.size + }); + blob.accessToken = this.accessToken; + blob.start = this.start; + blob.end = this.end; + blob.reconstructionInfo = this.reconstructionInfo; + blob.listener = this.listener; + blob.internalLogging = this.internalLogging; + return blob; + } + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on XetBlob.slice"); + } + const slice = this.#clone(); + slice.start = this.start + start; + slice.end = Math.min(this.start + end, this.end); + if (slice.start !== this.start || slice.end !== this.end) { + slice.reconstructionInfo = void 0; + } + return slice; + } + #reconstructionInfoPromise; + #loadReconstructionInfo() { + if (this.#reconstructionInfoPromise) { + return this.#reconstructionInfoPromise; + } + this.#reconstructionInfoPromise = (async () => { + const connParams = await getAccessToken(this.accessToken, this.fetch, this.refreshUrl); + const resp = await this.fetch(_nullishCoalesce(this.reconstructionUrl, () => ( `${connParams.casUrl}/v1/reconstructions/${this.hash}`)), { + headers: { + Authorization: `Bearer ${connParams.accessToken}`, + Range: `bytes=${this.start}-${this.end - 1}` + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } + this.reconstructionInfo = await resp.json(); + return this.reconstructionInfo; + })().finally(() => this.#reconstructionInfoPromise = void 0); + return this.#reconstructionInfoPromise; + } + async #fetch() { + if (this.size === 0) { + return new ReadableStream({ + start(controller) { + controller.close(); + } + }); + } + if (!this.reconstructionInfo) { + await this.#loadReconstructionInfo(); + } + const rangeLists = /* @__PURE__ */ new Map(); + if (!this.reconstructionInfo) { + throw new Error("Failed to load reconstruction info"); + } + for (const term of this.reconstructionInfo.terms) { + let rangeList = rangeLists.get(term.hash); + if (!rangeList) { + rangeList = new RangeList(); + rangeLists.set(term.hash, rangeList); + } + rangeList.add(term.range.start, term.range.end); + } + const listener = this.listener; + const log = this.internalLogging ? (...args) => console.log(...args) : () => { + }; + async function* readData(reconstructionInfo, customFetch, maxBytes, reloadReconstructionInfo) { + let totalBytesRead = 0; + let readBytesToSkip = reconstructionInfo.offset_into_first_range; + for (const term of reconstructionInfo.terms) { + if (totalBytesRead >= maxBytes) { + break; + } + const rangeList = rangeLists.get(term.hash); + if (!rangeList) { + throw new Error(`Failed to find range list for term ${term.hash}`); + } + { + const termRanges = rangeList.getRanges(term.range.start, term.range.end); + if (termRanges.every((range2) => range2.data)) { + log("all data available for term", term.hash, readBytesToSkip); + rangeLoop: + for (const range2 of termRanges) { + for (let chunk2 of range2.data) { + if (readBytesToSkip) { + const skipped = Math.min(readBytesToSkip, chunk2.byteLength); + chunk2 = chunk2.slice(skipped); + readBytesToSkip -= skipped; + if (!chunk2.byteLength) { + continue; + } + } + if (chunk2.byteLength > maxBytes - totalBytesRead) { + chunk2 = chunk2.slice(0, maxBytes - totalBytesRead); + } + totalBytesRead += chunk2.byteLength; + yield range2.refCount > 1 ? chunk2.slice() : chunk2; + _optionalChain([listener, 'optionalCall', _72 => _72({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } })]); + if (totalBytesRead >= maxBytes) { + break rangeLoop; + } + } + } + rangeList.remove(term.range.start, term.range.end); + continue; + } + } + let fetchInfo = reconstructionInfo.fetch_info[term.hash].find( + (info) => info.range.start <= term.range.start && info.range.end >= term.range.end + ); + if (!fetchInfo) { + throw new Error( + `Failed to find fetch info for term ${term.hash} and range ${term.range.start}-${term.range.end}` + ); + } + log("term", term); + log("fetchinfo", fetchInfo); + log("readBytesToSkip", readBytesToSkip); + let resp = await customFetch(fetchInfo.url, { + headers: { + Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}` + } + }); + if (resp.status === 403) { + reconstructionInfo = await reloadReconstructionInfo(); + fetchInfo = _optionalChain([reconstructionInfo, 'access', _73 => _73.fetch_info, 'access', _74 => _74[term.hash], 'optionalAccess', _75 => _75.find, 'call', _76 => _76( + (info) => info.range.start <= term.range.start && info.range.end >= term.range.end + )]); + if (!fetchInfo) { + throw new Error( + `Failed to find fetch info for term ${term.hash} and range ${term.range.start}-${term.range.end} after refresh` + ); + } + resp = await customFetch(fetchInfo.url, { + headers: { + Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}` + } + }); + } + if (!resp.ok) { + throw await createApiError(resp); + } + log( + "expected content length", + resp.headers.get("content-length"), + "range", + fetchInfo.url_range, + resp.headers.get("content-range") + ); + const reader = _optionalChain([resp, 'access', _77 => _77.body, 'optionalAccess', _78 => _78.getReader, 'call', _79 => _79()]); + if (!reader) { + throw new Error("Failed to get reader from response body"); + } + let done = false; + let chunkIndex = fetchInfo.range.start; + const ranges = rangeList.getRanges(fetchInfo.range.start, fetchInfo.range.end); + let leftoverBytes = void 0; + let totalFetchBytes = 0; + fetchData: + while (!done && totalBytesRead < maxBytes) { + const result = await reader.read(); + _optionalChain([listener, 'optionalCall', _80 => _80({ event: "read" })]); + done = result.done; + log("read", _optionalChain([result, 'access', _81 => _81.value, 'optionalAccess', _82 => _82.byteLength]), "bytes", "total read", totalBytesRead, "toSkip", readBytesToSkip); + if (!result.value) { + log("no data in result, cancelled", result); + continue; + } + totalFetchBytes += result.value.byteLength; + if (leftoverBytes) { + result.value = combineUint8Arrays(leftoverBytes, result.value); + leftoverBytes = void 0; + } + while (totalBytesRead < maxBytes && _optionalChain([result, 'access', _83 => _83.value, 'optionalAccess', _84 => _84.byteLength])) { + if (result.value.byteLength < 8) { + leftoverBytes = result.value; + continue fetchData; + } + const header = new DataView(result.value.buffer, result.value.byteOffset, XET_CHUNK_HEADER_BYTES); + const chunkHeader = { + version: header.getUint8(0), + compressed_length: header.getUint8(1) | header.getUint8(2) << 8 | header.getUint8(3) << 16, + compression_scheme: header.getUint8(4), + uncompressed_length: header.getUint8(5) | header.getUint8(6) << 8 | header.getUint8(7) << 16 + }; + log("chunk header", chunkHeader, "to skip", readBytesToSkip); + if (chunkHeader.version !== 0) { + throw new Error(`Unsupported chunk version ${chunkHeader.version}`); + } + if (chunkHeader.compression_scheme !== 0 /* None */ && chunkHeader.compression_scheme !== 1 /* LZ4 */ && chunkHeader.compression_scheme !== 2 /* ByteGroupingLZ4 */) { + throw new Error( + `Unsupported compression scheme ${_nullishCoalesce(compressionSchemeLabels[chunkHeader.compression_scheme], () => ( chunkHeader.compression_scheme))}` + ); + } + if (result.value.byteLength < chunkHeader.compressed_length + XET_CHUNK_HEADER_BYTES) { + leftoverBytes = result.value; + continue fetchData; + } + result.value = result.value.slice(XET_CHUNK_HEADER_BYTES); + let uncompressed = chunkHeader.compression_scheme === 1 /* LZ4 */ ? decompress(result.value.slice(0, chunkHeader.compressed_length), chunkHeader.uncompressed_length) : chunkHeader.compression_scheme === 2 /* ByteGroupingLZ4 */ ? bg4_regroup_bytes( + decompress( + result.value.slice(0, chunkHeader.compressed_length), + chunkHeader.uncompressed_length + ) + ) : result.value.slice(0, chunkHeader.compressed_length); + const range2 = ranges.find((range3) => chunkIndex >= range3.start && chunkIndex < range3.end); + const shouldYield = chunkIndex >= term.range.start && chunkIndex < term.range.end; + const minRefCountToStore = shouldYield ? 2 : 1; + let stored = false; + if (range2 && range2.refCount >= minRefCountToStore) { + range2.data ??= []; + range2.data.push(uncompressed); + stored = true; + } + if (shouldYield) { + if (readBytesToSkip) { + const skipped = Math.min(readBytesToSkip, uncompressed.byteLength); + uncompressed = uncompressed.slice(readBytesToSkip); + readBytesToSkip -= skipped; + } + if (uncompressed.byteLength > maxBytes - totalBytesRead) { + uncompressed = uncompressed.slice(0, maxBytes - totalBytesRead); + } + if (uncompressed.byteLength) { + log( + "yield", + uncompressed.byteLength, + "bytes", + result.value.byteLength, + "total read", + totalBytesRead, + stored + ); + totalBytesRead += uncompressed.byteLength; + yield stored ? uncompressed.slice() : uncompressed; + _optionalChain([listener, 'optionalCall', _85 => _85({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } })]); + } + } + chunkIndex++; + result.value = result.value.slice(chunkHeader.compressed_length); + } + } + if (done && totalBytesRead < maxBytes && totalFetchBytes < fetchInfo.url_range.end - fetchInfo.url_range.start + 1) { + log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes); + log("failed to fetch all data for term", term.hash); + throw new Error( + `Failed to fetch all data for term ${term.hash}, fetched ${totalFetchBytes} bytes out of ${fetchInfo.url_range.end - fetchInfo.url_range.start + 1}` + ); + } + log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes); + log("cancel reader"); + await reader.cancel(); + } + } + const iterator = readData( + this.reconstructionInfo, + this.fetch, + this.end - this.start, + this.#loadReconstructionInfo.bind(this) + ); + return new ReadableStream( + { + // todo: when Safari supports it, type controller as ReadableByteStreamController + async pull(controller) { + const result = await iterator.next(); + if (result.value) { + controller.enqueue(result.value); + } + if (result.done) { + controller.close(); + } + }, + type: "bytes" + // todo: when Safari supports it, add autoAllocateChunkSize param + }, + // todo : use ByteLengthQueuingStrategy when there's good support for it, currently in Node.js it fails due to size being a function + { + highWaterMark: 1e3 + // 1_000 chunks for ~1MB of RAM + } + ); + } + async arrayBuffer() { + const result = await this.#fetch(); + return new Response(result).arrayBuffer(); + } + async text() { + const result = await this.#fetch(); + return new Response(result).text(); + } + async response() { + const result = await this.#fetch(); + return new Response(result); + } + stream() { + const stream = new TransformStream(); + this.#fetch().then((response) => response.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message)); + return stream.readable; + } +}, _class2); +var jwtPromises = /* @__PURE__ */ new Map(); +var jwts = /* @__PURE__ */ new Map(); +function cacheKey(params) { + return JSON.stringify([params.refreshUrl, params.initialAccessToken]); +} +function bg4_regroup_bytes(bytes) { + const split = Math.floor(bytes.byteLength / 4); + const rem = bytes.byteLength % 4; + const g1_pos = split + (rem >= 1 ? 1 : 0); + const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); + const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); + const ret = new Uint8Array(bytes.byteLength); + for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + return ret; +} +function bg4_split_bytes(bytes) { + const ret = new Uint8Array(bytes.byteLength); + const split = Math.floor(bytes.byteLength / 4); + const rem = bytes.byteLength % 4; + const g1_pos = split + (rem >= 1 ? 1 : 0); + const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); + const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); + for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + return ret; +} +async function getAccessToken(initialAccessToken, customFetch, refreshUrl) { + const key = cacheKey({ refreshUrl, initialAccessToken }); + const jwt = jwts.get(key); + if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD)) { + return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; + } + const existingPromise = jwtPromises.get(key); + if (existingPromise) { + return existingPromise; + } + const promise = (async () => { + const resp = await customFetch(refreshUrl, { + headers: { + ...initialAccessToken ? { + Authorization: `Bearer ${initialAccessToken}` + } : {} + } + }); + if (!resp.ok) { + throw new Error(`Failed to get JWT token: ${resp.status} ${await resp.text()}`); + } + const json = await resp.json(); + const jwt2 = { + accessToken: json.accessToken, + expiresAt: new Date(json.exp * 1e3), + casUrl: json.casUrl + }; + jwtPromises.delete(key); + for (const [key2, value] of jwts.entries()) { + if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD)) { + jwts.delete(key2); + } else { + break; + } + } + if (jwts.size >= JWT_CACHE_SIZE) { + const keyToDelete = jwts.keys().next().value; + if (keyToDelete) { + jwts.delete(keyToDelete); + } + } + jwts.set(key, jwt2); + return { + accessToken: json.accessToken, + casUrl: json.casUrl + }; + })(); + jwtPromises.set(key, promise); + return promise; +} + +// src/utils/ChunkCache.ts +var CHUNK_CACHE_INITIAL_SIZE = 1e4; +var CHUNK_CACHE_GROW_FACTOR = 1.5; +var CHUNK_CACHE_MAX_SIZE = 1e6; +var ChunkCache = (_class3 = class { + __init5() {this.index = 0} + // Index >= 0 means local xorb, < 0 means remote xorb + + // Max 8K chunks per xorb, less than 64K uint16_t + + __init6() {this.map = /* @__PURE__ */ new Map()} + // hash -> chunkCacheIndex. Less overhead that way, empty object is 60+B and empty array is 40+B + __init7() {this.hmacs = /* @__PURE__ */ new Set()} + // todo : remove old hmacs + + constructor(maxSize = CHUNK_CACHE_MAX_SIZE) {;_class3.prototype.__init5.call(this);_class3.prototype.__init6.call(this);_class3.prototype.__init7.call(this); + if (maxSize < 1) { + throw new Error("maxSize must be at least 1"); + } + this.maxSize = maxSize; + this.xorbIndices = new Int32Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); + this.chunkIndices = new Uint16Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); + } + addChunkToCache(hash2, xorbIndex, chunkIndex, hmac2) { + if (this.map.has(hash2)) { + return; + } + if (this.map.values().next().value === this.index) { + this.map.delete(this.map.keys().next().value); + } + this.map.set(hash2, this.index); + if (hmac2 !== null) { + this.hmacs.add(hmac2); + } + if (this.index >= this.xorbIndices.length) { + const oldXorbIndices = this.xorbIndices; + const oldChunkIndices = this.chunkIndices; + this.xorbIndices = new Int32Array(Math.min(this.xorbIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); + this.chunkIndices = new Uint16Array(Math.min(this.chunkIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); + this.xorbIndices.set(oldXorbIndices); + this.chunkIndices.set(oldChunkIndices); + } + this.xorbIndices[this.index] = xorbIndex; + this.chunkIndices[this.index] = chunkIndex; + this.index = (this.index + 1) % this.maxSize; + } + getChunk(hash2, hmacFunction) { + let index = this.map.get(hash2); + if (index === void 0 && hmacFunction !== null) { + for (const hmac2 of this.hmacs) { + index = this.map.get(hmacFunction(hash2, hmac2)); + if (index !== void 0) { + break; + } + } + } + if (index === void 0) { + return void 0; + } + return { + xorbIndex: this.xorbIndices[index], + chunkIndex: this.chunkIndices[index] + }; + } + updateChunkIndex(hash2, chunkIndex) { + const index = this.map.get(hash2); + if (index === void 0) { + throw new Error(`Chunk not found in cache: ${hash2}`); + } + this.chunkIndices[index] = chunkIndex; + } + removeChunkFromCache(hash2) { + this.map.delete(hash2); + } +}, _class3); + +// src/utils/xetWriteToken.ts +var JWT_SAFETY_PERIOD2 = 6e4; +var JWT_CACHE_SIZE2 = 1e3; +var jwtPromises2 = /* @__PURE__ */ new Map(); +var jwts2 = /* @__PURE__ */ new Map(); +async function xetWriteToken(params) { + if (params.xetParams.expiresAt && params.xetParams.casUrl && params.xetParams.accessToken && params.xetParams.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD2)) { + return { accessToken: params.xetParams.accessToken, casUrl: params.xetParams.casUrl }; + } + const key = params.xetParams.refreshWriteTokenUrl; + const jwt = jwts2.get(key); + if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD2)) { + return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; + } + const existingPromise = jwtPromises2.get(key); + if (existingPromise) { + return existingPromise; + } + const promise = (async () => { + const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(params.xetParams.refreshWriteTokenUrl, { + headers: { + ...params.accessToken ? { + Authorization: `Bearer ${params.accessToken}` + } : {}, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } + const json = await resp.json(); + const jwt2 = { + accessToken: json.accessToken, + expiresAt: new Date(json.exp * 1e3), + casUrl: json.casUrl + }; + jwtPromises2.delete(key); + for (const [key2, value] of jwts2.entries()) { + if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD2)) { + jwts2.delete(key2); + } else { + break; + } + } + if (jwts2.size >= JWT_CACHE_SIZE2) { + const keyToDelete = jwts2.keys().next().value; + if (keyToDelete) { + jwts2.delete(keyToDelete); + } + } + jwts2.set(key, jwt2); + return { + accessToken: json.accessToken, + casUrl: json.casUrl + }; + })(); + jwtPromises2.set(key, promise); + return promise; +} + +// src/utils/shardParser.ts +var HASH_LENGTH = 32; +var XORB_HASH_BOOKEND = "ff".repeat(HASH_LENGTH); +function readHashFromArray(array, offset) { + let hash2 = ""; + for (let i = 0; i < HASH_LENGTH; i += 8) { + hash2 += `${array[offset + i + 7].toString(16).padStart(2, "0")}${array[offset + i + 6].toString(16).padStart(2, "0")}${array[offset + i + 5].toString(16).padStart(2, "0")}${array[offset + i + 4].toString(16).padStart(2, "0")}${array[offset + i + 3].toString(16).padStart(2, "0")}${array[offset + i + 2].toString(16).padStart(2, "0")}${array[offset + i + 1].toString(16).padStart(2, "0")}${array[offset + i].toString(16).padStart(2, "0")}`; + } + return hash2; +} +async function parseShardData(shardBlob) { + const shard = new Uint8Array(await shardBlob.arrayBuffer()); + const shardView = new DataView(shard.buffer); + const magicTag = shard.slice(0, SHARD_MAGIC_TAG.length); + if (!magicTag.every((byte, i) => byte === SHARD_MAGIC_TAG[i])) { + throw new Error("Invalid shard magic tag"); + } + const version = shardView.getBigUint64(SHARD_MAGIC_TAG.length, true); + if (version !== SHARD_HEADER_VERSION) { + throw new Error(`Invalid shard version: ${version}`); + } + const footerSize = Number(shardView.getBigUint64(SHARD_MAGIC_TAG.length + 8, true)); + const footerStart = shard.length - footerSize; + const footerVersion = shardView.getBigUint64(footerStart, true); + if (footerVersion !== SHARD_FOOTER_VERSION) { + throw new Error(`Invalid shard footer version: ${footerVersion}`); + } + const xorbInfoStart = Number(shardView.getBigUint64(footerStart + 16, true)); + const fileLookupStart = Number(shardView.getBigUint64(footerStart + 24, true)); + const hmacKey = readHashFromArray(shard, footerStart + 72); + const xorbs = []; + let offset = xorbInfoStart; + while (offset < fileLookupStart) { + const xorbHash2 = readHashFromArray(shard, offset); + offset += HASH_LENGTH; + if (xorbHash2 === XORB_HASH_BOOKEND) { + break; + } + offset += 4; + const chunkCount = shardView.getUint32(offset, true); + offset += 4; + offset += 4; + offset += 4; + const chunks = []; + for (let i = 0; i < chunkCount; i++) { + const chunkHash = readHashFromArray(shard, offset); + offset += HASH_LENGTH; + const startOffset = shardView.getUint32(offset, true); + offset += 4; + const length = shardView.getUint32(offset, true); + offset += 4; + offset += 8; + chunks.push({ + hash: chunkHash, + startOffset, + unpackedLength: length + }); + } + xorbs.push({ + hash: xorbHash2, + chunks + }); + } + return { + hmacKey, + xorbs + }; +} + +// src/utils/sum.ts +function sum(arr) { + return arr.reduce((a, b) => a + b, 0); +} + +// src/utils/SplicedBlob.ts +var SplicedBlob = class extends Blob { + + + constructor(originalBlob, spliceOperations) { + super(); + this.originalBlob = originalBlob; + this.spliceOperations = spliceOperations; + } + static create(originalBlob, operations) { + for (const op of operations) { + if (op.start < 0 || op.end < 0) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + if (op.start > originalBlob.size || op.end > originalBlob.size) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + if (op.start > op.end) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + } + const sortedOps = [...operations].sort((a, b) => a.start - b.start); + for (let i = 0; i < sortedOps.length - 1; i++) { + if (sortedOps[i].end > sortedOps[i + 1].start) { + throw new Error("Overlapping splice operations are not supported"); + } + } + return new SplicedBlob(originalBlob, sortedOps); + } + /** + * Returns the size of the spliced blob. + * Size = original size - total replaced size + total insert size + */ + get size() { + let totalReplacedSize = 0; + let totalInsertSize = 0; + for (const op of this.spliceOperations) { + totalReplacedSize += op.end - op.start; + totalInsertSize += op.insert.size; + } + return this.originalBlob.size - totalReplacedSize + totalInsertSize; + } + /** + * Returns the MIME type of the original blob. + */ + get type() { + return this.originalBlob.type; + } + /** + * Returns a new instance of SplicedBlob that is a slice of the current one. + * + * The slice is inclusive of the start and exclusive of the end. + * The slice method does not support negative start/end. + * + * @param start beginning of the slice + * @param end end of the slice + */ + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + throw new TypeError("Unsupported negative start/end on SplicedBlob.slice"); + } + start = Math.min(start, this.size); + end = Math.min(end, this.size); + if (start >= end) { + return new Blob([]); + } + const segments = this.segments; + const segmentBoundaries = [0]; + let cumulativeSize = 0; + for (const segment of segments) { + cumulativeSize += segment.size; + segmentBoundaries.push(cumulativeSize); + } + const resultSegments = []; + for (let i = 0; i < segments.length; i++) { + const segmentStart = segmentBoundaries[i]; + const segmentEnd = segmentBoundaries[i + 1]; + if (segmentEnd <= start) { + continue; + } + if (segmentStart >= end) { + break; + } + const sliceStart = Math.max(0, start - segmentStart); + const sliceEnd = Math.min(segments[i].size, end - segmentStart); + if (sliceStart < sliceEnd) { + resultSegments.push(segments[i].slice(sliceStart, sliceEnd)); + } + } + return new Blob(resultSegments); + } + get firstSpliceIndex() { + return _nullishCoalesce(_optionalChain([this, 'access', _86 => _86.spliceOperations, 'access', _87 => _87[0], 'optionalAccess', _88 => _88.start]), () => ( Infinity)); + } + /** + * Read the spliced blob content and returns it as an ArrayBuffer. + */ + async arrayBuffer() { + const segments = this.segments; + const buffers = await Promise.all(segments.map((segment) => segment.arrayBuffer())); + const totalSize = sum(buffers.map((buffer) => buffer.byteLength)); + const result = new Uint8Array(totalSize); + let offset = 0; + for (const buffer of buffers) { + result.set(new Uint8Array(buffer), offset); + offset += buffer.byteLength; + } + return result.buffer; + } + /** + * Read the spliced blob content and returns it as a string. + */ + async text() { + const buffer = await this.arrayBuffer(); + return new TextDecoder().decode(buffer); + } + /** + * Returns a stream around the spliced blob content. + */ + stream() { + const readable = new ReadableStream({ + start: async (controller) => { + try { + const segments = this.segments; + for (const segment of segments) { + const reader = segment.stream().getReader(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + controller.enqueue(value); + } + } finally { + reader.releaseLock(); + } + } + controller.close(); + } catch (error) { + controller.error(error); + } + } + }); + return readable; + } + /** + * Get all segments that make up the spliced blob. + * This includes original blob segments between splice operations and insert blobs. + */ + get segments() { + const segments = []; + let currentPosition = 0; + const sortedOps = [...this.spliceOperations].sort((a, b) => a.start - b.start); + for (const op of sortedOps) { + if (currentPosition < op.start) { + segments.push(this.originalBlob.slice(currentPosition, op.start)); + } + if (op.insert.size > 0) { + segments.push(op.insert); + } + currentPosition = op.end; + } + if (currentPosition < this.originalBlob.size) { + segments.push(this.originalBlob.slice(currentPosition)); + } + return segments; + } +}; + +// src/utils/createXorbs.ts + + + + + + + + + + +var _xetchunkwasm = require('@huggingface/xetchunk-wasm'); +var TARGET_CHUNK_SIZE = 64 * 1024; +var MAX_CHUNK_SIZE = 2 * TARGET_CHUNK_SIZE; +var XORB_SIZE = 64 * 1024 * 1024; +var MAX_XORB_CHUNKS = 8 * 1024; +var INTERVAL_BETWEEN_REMOTE_DEDUP = 4e6; +var PROCESSING_PROGRESS_RATIO = 0.1; +var UPLOADING_PROGRESS_RATIO = 1 - PROCESSING_PROGRESS_RATIO; +function computeXorbHashHex(chunks) { + const chunkObjs = chunks.map((c) => ({ hash: _xetchunkwasm.hexToBytes.call(void 0, c.hash), length: c.length })); + return _xetchunkwasm.hashToHex.call(void 0, _xetchunkwasm.xorbHash.call(void 0, chunkObjs)); +} +function computeHmacHex(hash2, key) { + return _xetchunkwasm.hashToHex.call(void 0, _xetchunkwasm.hmac.call(void 0, _xetchunkwasm.hexToBytes.call(void 0, hash2), _xetchunkwasm.hexToBytes.call(void 0, key))); +} +function computeVerificationHashHex(hashes) { + return _xetchunkwasm.hashToHex.call(void 0, _xetchunkwasm.verificationHash.call(void 0, hashes.map(_xetchunkwasm.hexToBytes))); +} +function computeFileHashHex(chunks) { + const chunkObjs = chunks.map((c) => ({ hash: _xetchunkwasm.hexToBytes.call(void 0, c.hash), length: c.length })); + return _xetchunkwasm.hashToHex.call(void 0, _xetchunkwasm.fileHash.call(void 0, chunkObjs)); +} +function addDataToChunker(data, chunker) { + return _xetchunkwasm.nextBlock.call(void 0, chunker, data).map((c) => ({ hash: _xetchunkwasm.hashToHex.call(void 0, c.hash), length: c.length, dedup: false })); +} +function finalizeChunker(chunker) { + const last = _xetchunkwasm.finalize.call(void 0, chunker); + if (!last) { + return []; + } + return [{ hash: _xetchunkwasm.hashToHex.call(void 0, last.hash), length: last.length, dedup: false }]; +} +var CurrentXorbInfo = class { + + + + + + + + + constructor() { + this.id = 0; + this.offset = 0; + this.chunks = []; + this.fileProcessedBytes = {}; + this.fileUploadedBytes = {}; + this.fileSize = {}; + this.data = new Uint8Array(XORB_SIZE); + this.immutableData = null; + } + event(computeXorbHash) { + const xorbChunksCleaned = this.chunks.map((chunk2) => ({ + hash: chunk2.hash, + length: chunk2.length + })); + return { + event: "xorb", + xorb: this.data.subarray(0, this.offset), + hash: computeXorbHash(xorbChunksCleaned), + chunks: xorbChunksCleaned, + id: this.id, + files: Object.entries(this.fileProcessedBytes).map(([path, processedBytes]) => ({ + path, + progress: processedBytes / this.fileSize[path], + lastSentProgress: ((_nullishCoalesce(this.fileUploadedBytes[path], () => ( 0))) + (processedBytes - (_nullishCoalesce(this.fileUploadedBytes[path], () => ( 0)))) * PROCESSING_PROGRESS_RATIO) / this.fileSize[path] + })) + }; + } +}; +async function* createXorbs(fileSources, params) { + const alreadyDoneFileSha256s = /* @__PURE__ */ new Set(); + let xorbId = 0; + const chunkCache = new ChunkCache(); + let xorb = new CurrentXorbInfo(); + const nextXorb = (currentFile) => { + const event = xorb.event(computeXorbHashHex); + xorbId++; + xorb = new CurrentXorbInfo(); + xorb.id = xorbId; + xorb.fileUploadedBytes = { + [currentFile.path]: currentFile.uploadedBytes + }; + xorb.fileSize[currentFile.path] = currentFile.size; + return event; + }; + const pendingFileEvents = []; + const remoteXorbHashes = [""]; + for await (const fileSource of fileSources) { + _optionalChain([params, 'access', _89 => _89.yieldCallback, 'optionalCall', _90 => _90({ + event: "fileProgress", + path: fileSource.path, + progress: 0 + })]); + if (fileSource.sha256 && alreadyDoneFileSha256s.has(fileSource.sha256)) { + _optionalChain([params, 'access', _91 => _91.yieldCallback, 'optionalCall', _92 => _92({ + event: "fileProgress", + path: fileSource.path, + progress: 1 + })]); + continue; + } + if (fileSource.sha256) { + alreadyDoneFileSha256s.add(fileSource.sha256); + } + const chunker = _xetchunkwasm.createChunker.call(void 0, TARGET_CHUNK_SIZE); + { + xorb.fileSize[fileSource.path] = fileSource.content.size; + if (fileSource.content instanceof SplicedBlob && fileSource.content.firstSpliceIndex < MAX_CHUNK_SIZE) { + await loadDedupInfoToCache( + fileSource.content.originalBlob.slice(0, MAX_CHUNK_SIZE), + remoteXorbHashes, + params, + chunkCache, + computeHmacHex, + { + maxChunks: 1, + isAtBeginning: true + } + ); + } + let bytesSinceRemoteDedup = Infinity; + let bytesSinceLastProgressEvent = 0; + let isFirstFileChunk = true; + const sourceChunks = []; + const reader = fileSource.content.stream().getReader(); + let processedBytes = 0; + let dedupedBytes = 0; + const fileChunks = []; + const chunkMetadata = []; + const addChunks = async function* (chunks) { + for (const chunk2 of chunks) { + if (isFirstFileChunk) { + chunk2.dedup = true; + isFirstFileChunk = false; + } + let chunkIndex = xorb.chunks.length; + let chunkXorbId = xorbId; + const chunkToCopy = removeChunkFromSourceData(sourceChunks, chunk2.length); + let cacheData = chunkCache.getChunk(chunk2.hash, computeHmacHex); + if (cacheData === void 0 && chunk2.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { + const token = await xetWriteToken(params); + bytesSinceRemoteDedup = 0; + const shardResp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(token.casUrl + "/v1/chunks/default/" + chunk2.hash, { + headers: { + Authorization: `Bearer ${token.accessToken}` + } + }); + if (shardResp.ok) { + const shard = await shardResp.blob(); + const shardData = await parseShardData(shard); + for (const xorb2 of shardData.xorbs) { + const remoteXorbId = -remoteXorbHashes.length; + remoteXorbHashes.push(xorb2.hash); + let i = 0; + for (const chunk3 of xorb2.chunks) { + chunkCache.addChunkToCache(chunk3.hash, remoteXorbId, i++, shardData.hmacKey); + } + } + cacheData = chunkCache.getChunk(chunk2.hash, computeHmacHex); + const oldDedupedBytes = dedupedBytes; + dedupedBytes = backtrackDedup(xorb, computeHmacHex, shardData, chunkCache, chunkMetadata, dedupedBytes); + if (dedupedBytes > oldDedupedBytes) { + xorb.fileUploadedBytes[fileSource.path] ??= 0; + xorb.fileUploadedBytes[fileSource.path] += dedupedBytes - oldDedupedBytes; + } + } + } + if (cacheData === void 0) { + if (!writeChunk(xorb, chunkToCopy, chunk2.hash)) { + yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); + chunkIndex = 0; + chunkXorbId = xorbId; + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } + pendingFileEvents.length = 0; + if (!writeChunk(xorb, chunkToCopy, chunk2.hash)) { + throw new Error("Failed to write chunk into xorb"); + } + } + chunkCache.addChunkToCache(chunk2.hash, xorbId, chunkIndex, null); + } else { + chunkXorbId = cacheData.xorbIndex; + chunkIndex = cacheData.chunkIndex; + dedupedBytes += chunk2.length; + xorb.fileUploadedBytes[fileSource.path] ??= 0; + xorb.fileUploadedBytes[fileSource.path] += chunk2.length; + } + bytesSinceRemoteDedup += chunk2.length; + bytesSinceLastProgressEvent += chunk2.length; + fileChunks.push({ hash: chunk2.hash, length: chunk2.length }); + chunkMetadata.push({ + xorbId: chunkXorbId, + chunkIndex, + length: chunk2.length + }); + xorb.fileProcessedBytes[fileSource.path] = processedBytes; + if (bytesSinceLastProgressEvent >= 1e6) { + bytesSinceLastProgressEvent = 0; + _optionalChain([params, 'access', _93 => _93.yieldCallback, 'optionalCall', _94 => _94({ + event: "fileProgress", + path: fileSource.path, + progress: ((_nullishCoalesce(xorb.fileUploadedBytes[fileSource.path], () => ( 0))) + (xorb.fileProcessedBytes[fileSource.path] - (_nullishCoalesce(xorb.fileUploadedBytes[fileSource.path], () => ( 0)))) * PROCESSING_PROGRESS_RATIO) / fileSource.content.size + })]); + } + if (xorb.chunks.length >= MAX_XORB_CHUNKS) { + yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } + pendingFileEvents.length = 0; + } + } + }; + while (true) { + const { done, value } = await reader.read(); + if (done) { + yield* addChunks(finalizeChunker(chunker)); + break; + } + processedBytes += value.length; + sourceChunks.push(value); + yield* addChunks(addDataToChunker(value, chunker)); + } + const fileRepresentation = buildFileRepresentation(chunkMetadata, fileChunks, computeVerificationHashHex); + xorb.immutableData = { + chunkIndex: xorb.chunks.length, + offset: xorb.offset + }; + const dedupRatio = fileSource.content.size > 0 ? dedupedBytes / fileSource.content.size : 0; + pendingFileEvents.push({ + event: "file", + path: fileSource.path, + hash: computeFileHashHex(fileChunks), + sha256: fileSource.sha256, + dedupRatio, + representation: fileRepresentation + }); + } + } + if (xorb.offset > 0) { + yield xorb.event(computeXorbHashHex); + } + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } +} +function backtrackDedup(xorb, computeHmac, shardData, chunkCache, chunkMetadata, dedupedBytes) { + const chunkIndexesToBacktrackFor = /* @__PURE__ */ new Map(); + for (let chunkToRecheckIndex = _nullishCoalesce(_optionalChain([xorb, 'access', _95 => _95.immutableData, 'optionalAccess', _96 => _96.chunkIndex]), () => ( 0)); chunkToRecheckIndex < xorb.chunks.length; chunkToRecheckIndex++) { + const chunk2 = xorb.chunks[chunkToRecheckIndex]; + const hmacHash = computeHmac(chunk2.hash, shardData.hmacKey); + const cacheData = chunkCache.getChunk(hmacHash, null); + if (cacheData !== void 0) { + chunkIndexesToBacktrackFor.set(chunkToRecheckIndex, { + xorbId: cacheData.xorbIndex, + chunkIndex: cacheData.chunkIndex + }); + chunkCache.removeChunkFromCache(chunk2.hash); + } + } + for (const metadata of chunkMetadata) { + if (metadata.xorbId === xorb.id && chunkIndexesToBacktrackFor.has(metadata.chunkIndex)) { + const backtrackData = chunkIndexesToBacktrackFor.get(metadata.chunkIndex); + if (backtrackData !== void 0) { + metadata.xorbId = backtrackData.xorbId; + metadata.chunkIndex = backtrackData.chunkIndex; + dedupedBytes += metadata.length; + } + } + } + const xorbRangesToErase = []; + for (let i = 0; i < xorb.chunks.length; i++) { + const chunk2 = xorb.chunks[i]; + if (chunkIndexesToBacktrackFor.has(i)) { + xorbRangesToErase.push({ + start: chunk2.offset, + end: i < xorb.chunks.length - 1 ? xorb.chunks[i + 1].offset : xorb.offset + }); + } + } + const xorbRangesToKeep = []; + let currentStart = 0; + for (let i = 0; i < xorbRangesToErase.length; i++) { + const range2 = xorbRangesToErase[i]; + if (currentStart !== range2.start) { + xorbRangesToKeep.push({ start: currentStart, end: range2.start }); + } + currentStart = range2.end; + } + if (currentStart !== xorb.offset) { + xorbRangesToKeep.push({ start: currentStart, end: xorb.offset }); + } + let currentOffset = 0; + for (const range2 of xorbRangesToKeep) { + if (range2.start !== currentOffset) { + xorb.data.set(xorb.data.subarray(range2.start, range2.end), currentOffset); + } + currentOffset += range2.end - range2.start; + } + const newXorbChunks = []; + const oldIndexToNewIndex = /* @__PURE__ */ new Map(); + let erasedOffset = 0; + for (let i = 0; i < xorb.chunks.length; i++) { + const chunk2 = xorb.chunks[i]; + if (chunkIndexesToBacktrackFor.has(i)) { + if (i < xorb.chunks.length - 1) { + erasedOffset += xorb.chunks[i + 1].offset - chunk2.offset; + } + } else { + newXorbChunks.push({ + hash: chunk2.hash, + length: chunk2.length, + offset: chunk2.offset - erasedOffset + }); + if (erasedOffset > 0) { + oldIndexToNewIndex.set(i, newXorbChunks.length - 1); + } + } + } + xorb.chunks = newXorbChunks; + xorb.offset = currentOffset; + for (const chunk2 of chunkMetadata) { + if (chunk2.xorbId === xorb.id) { + const newIndex = oldIndexToNewIndex.get(chunk2.chunkIndex); + if (newIndex !== void 0) { + const cached = chunkCache.getChunk(xorb.chunks[newIndex].hash, null); + if (cached !== void 0 && cached.xorbIndex === chunk2.xorbId && cached.chunkIndex === chunk2.chunkIndex) { + chunkCache.updateChunkIndex(xorb.chunks[newIndex].hash, newIndex); + } + chunk2.chunkIndex = newIndex; + } + } + } + return dedupedBytes; +} +function removeChunkFromSourceData(sourceChunks, chunkLength) { + if (chunkLength === sourceChunks[0].length) { + const chunkToCopy = sourceChunks[0]; + sourceChunks.shift(); + return chunkToCopy; + } else if (chunkLength < sourceChunks[0].length) { + const chunkToCopy = sourceChunks[0].subarray(0, chunkLength); + sourceChunks[0] = sourceChunks[0].subarray(chunkLength); + return chunkToCopy; + } else { + const chunkToCopy = new Uint8Array(chunkLength); + let copyOffset = 0; + let index = 0; + let toSlice = -1; + while (copyOffset < chunkLength) { + const nToCopy = Math.min(sourceChunks[index].length, chunkLength - copyOffset); + chunkToCopy.set(sourceChunks[index].subarray(0, nToCopy), copyOffset); + copyOffset += nToCopy; + if (nToCopy === sourceChunks[index].length) { + index++; + } else { + toSlice = nToCopy; + } + } + sourceChunks.splice(0, index); + if (toSlice !== -1) { + sourceChunks[0] = sourceChunks[0].subarray(toSlice); + } + return chunkToCopy; + } +} +function writeChunk(xorb, chunk2, hash2) { + const regularCompressedChunk = compress(chunk2); + const bgCompressedChunk = compress(bg4_split_bytes(chunk2)); + const compressedChunk = bgCompressedChunk.length < regularCompressedChunk.length ? bgCompressedChunk : regularCompressedChunk; + const chunkToWrite = compressedChunk.length < chunk2.length ? compressedChunk : chunk2; + if (xorb.offset + XET_CHUNK_HEADER_BYTES + chunkToWrite.length > XORB_SIZE) { + return false; + } + xorb.data[xorb.offset] = 0; + xorb.data[xorb.offset + 1] = chunkToWrite.length & 255; + xorb.data[xorb.offset + 2] = chunkToWrite.length >> 8 & 255; + xorb.data[xorb.offset + 3] = chunkToWrite.length >> 16 & 255; + xorb.data[xorb.offset + 4] = chunkToWrite.length < chunk2.length ? bgCompressedChunk.length < regularCompressedChunk.length ? 2 /* ByteGroupingLZ4 */ : 1 /* LZ4 */ : 0 /* None */; + xorb.data[xorb.offset + 5] = chunk2.length & 255; + xorb.data[xorb.offset + 6] = chunk2.length >> 8 & 255; + xorb.data[xorb.offset + 7] = chunk2.length >> 16 & 255; + xorb.data.set(chunkToWrite, xorb.offset + XET_CHUNK_HEADER_BYTES); + xorb.chunks.push({ hash: hash2, length: chunk2.length, offset: xorb.offset }); + xorb.offset += XET_CHUNK_HEADER_BYTES + chunkToWrite.length; + return true; +} +var buildFileRepresentation = (metadata, chunks, computeVerificationHash) => { + if (metadata.length === 0) { + return []; + } + const representation = []; + let currentRange = { + xorbId: metadata[0].xorbId, + indexStart: metadata[0].chunkIndex, + indexEnd: metadata[0].chunkIndex + 1, + length: metadata[0].length, + chunkHashStart: 0 + }; + for (let i = 1; i < metadata.length; i++) { + const chunk2 = metadata[i]; + if (currentRange.xorbId === chunk2.xorbId && currentRange.indexEnd === chunk2.chunkIndex) { + currentRange.indexEnd = chunk2.chunkIndex + 1; + currentRange.length += chunk2.length; + } else { + const rangeHash2 = computeVerificationHash(chunks.slice(currentRange.chunkHashStart, i).map((x) => x.hash)); + representation.push({ + xorbId: currentRange.xorbId, + indexStart: currentRange.indexStart, + indexEnd: currentRange.indexEnd, + length: currentRange.length, + rangeHash: rangeHash2 + }); + currentRange = { + xorbId: chunk2.xorbId, + indexStart: chunk2.chunkIndex, + indexEnd: chunk2.chunkIndex + 1, + length: chunk2.length, + chunkHashStart: i + }; + } + } + const rangeHash = computeVerificationHash(chunks.slice(currentRange.chunkHashStart).map((x) => x.hash)); + representation.push({ + xorbId: currentRange.xorbId, + indexStart: currentRange.indexStart, + indexEnd: currentRange.indexEnd, + length: currentRange.length, + rangeHash + }); + return representation; +}; +async function loadDedupInfoToCache(content, remoteXorbHashes, params, chunkCache, computeHmacHex2, opts) { + const chunker = _xetchunkwasm.createChunker.call(void 0, TARGET_CHUNK_SIZE); + const cache = chunkCache; + let dedupedBytes = 0; + let chunksProcessed = 0; + let totalBytes = 0; + let bytesSinceRemoteDedup = Infinity; + const sourceChunks = []; + const reader = content.stream().getReader(); + const processChunks = async (chunks) => { + for (const chunk2 of chunks) { + chunksProcessed++; + if (_optionalChain([opts, 'optionalAccess', _97 => _97.isAtBeginning]) && chunksProcessed === 1) { + chunk2.dedup = true; + } + totalBytes += chunk2.length; + removeChunkFromSourceData(sourceChunks, chunk2.length); + let cacheData = cache.getChunk(chunk2.hash, computeHmacHex2); + if (cacheData !== void 0) { + dedupedBytes += chunk2.length; + bytesSinceRemoteDedup += chunk2.length; + continue; + } + if (chunk2.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { + const token = await xetWriteToken(params); + bytesSinceRemoteDedup = 0; + const shardResp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(token.casUrl + "/v1/chunks/default/" + chunk2.hash, { + headers: { + Authorization: `Bearer ${token.accessToken}` + } + }); + if (shardResp.ok) { + const shard = await shardResp.blob(); + const shardData = await parseShardData(shard); + for (const xorb of shardData.xorbs) { + const remoteXorbId = -remoteXorbHashes.length; + remoteXorbHashes.push(xorb.hash); + let i = 0; + for (const xorbChunk of xorb.chunks) { + cache.addChunkToCache(xorbChunk.hash, remoteXorbId, i++, shardData.hmacKey); + } + } + cacheData = cache.getChunk(chunk2.hash, computeHmacHex2); + } + } + if (cacheData !== void 0) { + dedupedBytes += chunk2.length; + } + bytesSinceRemoteDedup += chunk2.length; + } + }; + while (true) { + if (_optionalChain([opts, 'optionalAccess', _98 => _98.end]) !== void 0 && totalBytes >= opts.end) { + break; + } + if (_optionalChain([opts, 'optionalAccess', _99 => _99.maxChunks]) !== void 0 && chunksProcessed >= opts.maxChunks) { + break; + } + const { done, value } = await reader.read(); + if (done) { + await processChunks(finalizeChunker(chunker)); + break; + } + sourceChunks.push(value); + await processChunks(addDataToChunker(value, chunker)); + } +} + +// src/utils/uploadShards.ts +var SHARD_MAX_SIZE = 64 * 1024 * 1024; +var SHARD_HEADER_SIZE = 48; +var SHARD_FOOTER_SIZE = 200; +var HASH_LENGTH2 = 32; +var XORB_FOOTER_LENGTH = 48; +var FILE_FOOTER_LENGTH = 48; +var SHARD_HEADER_VERSION = 2n; +var SHARD_FOOTER_VERSION = 1n; +var MDB_FILE_FLAG_WITH_VERIFICATION = 2147483648; +var MDB_FILE_FLAG_WITH_METADATA_EXT = 1073741824; +var SHARD_MAGIC_TAG = new Uint8Array([ + "H".charCodeAt(0), + "F".charCodeAt(0), + "R".charCodeAt(0), + "e".charCodeAt(0), + "p".charCodeAt(0), + "o".charCodeAt(0), + "M".charCodeAt(0), + "e".charCodeAt(0), + "t".charCodeAt(0), + "a".charCodeAt(0), + "D".charCodeAt(0), + "a".charCodeAt(0), + "t".charCodeAt(0), + "a".charCodeAt(0), + 0, + 85, + 105, + 103, + 69, + 106, + 123, + 129, + 87, + 131, + 165, + 189, + 217, + 92, + 205, + 209, + 74, + 169 +]); +async function* uploadShards(source, params) { + const xorbHashes = []; + const seenFileXetHashes = /* @__PURE__ */ new Set(); + const fileInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.25); + const xorbInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.75); + const xorbView = new DataView(xorbInfoSection.buffer); + let xorbViewOffset = 0; + const fileInfoView = new DataView(fileInfoSection.buffer); + let fileViewOffset = 0; + let xorbTotalSize = 0n; + let fileTotalSize = 0n; + let xorbTotalUnpackedSize = 0n; + for await (const output of createXorbs(source, params)) { + switch (output.event) { + case "xorb": { + xorbHashes.push(output.hash); + const xorbEntrySize = HASH_LENGTH2 + 4 + 4 + 4 + 4; + const chunksSize = output.chunks.length * (HASH_LENGTH2 + 4 + 4 + 8); + const totalXorbSize = xorbEntrySize + chunksSize; + if (xorbViewOffset + totalXorbSize > xorbInfoSection.length) { + if (xorbViewOffset > 0 || fileViewOffset > 0) { + await uploadShard(createShard(), params); + } + } + writeHashToArray(output.hash, xorbInfoSection, xorbViewOffset); + xorbViewOffset += HASH_LENGTH2; + xorbView.setUint32(xorbViewOffset, 0, true); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, output.chunks.length, true); + xorbViewOffset += 4; + const xorbUnpackedSize = sum(output.chunks.map((x) => x.length)); + xorbView.setUint32(xorbViewOffset, xorbUnpackedSize, true); + xorbTotalUnpackedSize += BigInt(xorbUnpackedSize); + xorbTotalSize += BigInt(output.xorb.byteLength); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, output.xorb.byteLength, true); + xorbViewOffset += 4; + let chunkBytes = 0; + for (const chunk2 of output.chunks) { + writeHashToArray(chunk2.hash, xorbInfoSection, xorbViewOffset); + xorbViewOffset += HASH_LENGTH2; + xorbView.setUint32(xorbViewOffset, chunkBytes, true); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, chunk2.length, true); + xorbViewOffset += 4; + xorbView.setBigUint64(xorbViewOffset, 0n, true); + xorbViewOffset += 8; + chunkBytes += chunk2.length; + } + for (const file of output.files) { + yield { + event: "fileProgress", + path: file.path, + progress: file.lastSentProgress + }; + } + await uploadXorb(output, params); + for (const file of output.files) { + yield { event: "fileProgress", path: file.path, progress: file.progress }; + } + break; + } + case "file": { + yield { + event: "file", + path: output.path, + xetHash: output.hash, + sha256: output.sha256, + dedupRatio: output.dedupRatio + }; + if (seenFileXetHashes.has(output.hash)) { + break; + } + seenFileXetHashes.add(output.hash); + const fileHeaderSize = HASH_LENGTH2 + 4 + 4 + 8; + const representationSize = output.representation.length * (HASH_LENGTH2 + 4 + 4 + 4 + 4); + const verificationSize = output.representation.length * (HASH_LENGTH2 + 16); + const fileSha256 = output.sha256; + const hasMetadataExt = fileSha256 !== void 0; + const metadataSize = hasMetadataExt ? HASH_LENGTH2 + 16 : 0; + const totalFileSize = fileHeaderSize + representationSize + verificationSize + metadataSize; + if (fileViewOffset + totalFileSize > fileInfoSection.length) { + if (xorbViewOffset > 0 || fileViewOffset > 0) { + await uploadShard(createShard(), params); + } + } + writeHashToArray(output.hash, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + fileInfoView.setUint32( + fileViewOffset, + MDB_FILE_FLAG_WITH_VERIFICATION + (hasMetadataExt ? MDB_FILE_FLAG_WITH_METADATA_EXT : 0), + true + ); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, output.representation.length, true); + fileViewOffset += 4; + fileInfoView.setBigUint64(fileViewOffset, 0n, true); + fileViewOffset += 8; + for (const repItem of output.representation) { + writeHashToArray( + typeof repItem.xorbId === "number" ? xorbHashes[repItem.xorbId] : repItem.xorbId, + fileInfoSection, + fileViewOffset + ); + fileViewOffset += HASH_LENGTH2; + fileInfoView.setUint32(fileViewOffset, 0, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.length, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.indexStart, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.indexEnd, true); + fileViewOffset += 4; + } + for (const repItem of output.representation) { + writeHashToArray(repItem.rangeHash, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + for (let i = 0; i < 16; i++) { + fileInfoSection[fileViewOffset + i] = 0; + } + fileViewOffset += 16; + } + if (hasMetadataExt) { + writeHashToArray(fileSha256, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + for (let i = 0; i < 16; i++) { + fileInfoSection[fileViewOffset + i] = 0; + } + fileViewOffset += 16; + } + break; + } + } + } + function createShard() { + const shard = new Uint8Array( + SHARD_HEADER_SIZE + SHARD_FOOTER_SIZE + xorbViewOffset + XORB_FOOTER_LENGTH + fileViewOffset + FILE_FOOTER_LENGTH + ); + const shardView = new DataView(shard.buffer); + let shardOffset = 0; + shard.set(SHARD_MAGIC_TAG, shardOffset); + shardOffset += SHARD_MAGIC_TAG.length; + shardView.setBigUint64(shardOffset, SHARD_HEADER_VERSION, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(SHARD_FOOTER_SIZE), true); + shardOffset += 8; + shard.set(fileInfoSection.slice(0, fileViewOffset), shardOffset); + shardOffset += fileViewOffset; + for (let i = 0; i < 32; i++) { + shard[shardOffset + i] = 255; + } + shardOffset += 32; + for (let i = 0; i < 16; i++) { + shard[shardOffset + i] = 0; + } + shardOffset += 16; + const xorbInfoOffset = shardOffset; + shard.set(xorbInfoSection.slice(0, xorbViewOffset), shardOffset); + shardOffset += xorbViewOffset; + for (let i = 0; i < 32; i++) { + shard[shardOffset + i] = 255; + } + shardOffset += 32; + for (let i = 0; i < 16; i++) { + shard[shardOffset + i] = 0; + } + shardOffset += 16; + const footerOffset = shardOffset; + shardView.setBigUint64(shardOffset, SHARD_FOOTER_VERSION, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(SHARD_HEADER_SIZE), true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(xorbInfoOffset), true); + shardOffset += 8; + for (let i = 0; i < 48; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 48; + for (let i = 0; i < 32; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 32; + shardView.setBigUint64(shardOffset, BigInt(Math.floor(Date.now() / 1e3)), true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, 0n, true); + shardOffset += 8; + for (let i = 0; i < 48; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 48; + shardView.setBigUint64(shardOffset, xorbTotalSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, fileTotalSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, xorbTotalUnpackedSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(footerOffset), true); + xorbViewOffset = 0; + fileViewOffset = 0; + xorbTotalSize = 0n; + xorbTotalUnpackedSize = 0n; + fileTotalSize = 0n; + return shard; + } + if (xorbViewOffset || fileViewOffset) { + await uploadShard(createShard(), params); + } +} +function writeHashToArray(hash2, array, offset) { + for (let i = 0; i < hash2.length; i += 16) { + array[offset + i / 2] = parseInt(hash2.substring(i + 2 * 7, i + 2 * 8), 16); + array[offset + i / 2 + 1] = parseInt(hash2.substring(i + 2 * 6, i + 2 * 7), 16); + array[offset + i / 2 + 2] = parseInt(hash2.substring(i + 2 * 5, i + 2 * 6), 16); + array[offset + i / 2 + 3] = parseInt(hash2.substring(i + 2 * 4, i + 2 * 5), 16); + array[offset + i / 2 + 4] = parseInt(hash2.substring(i + 2 * 3, i + 2 * 4), 16); + array[offset + i / 2 + 5] = parseInt(hash2.substring(i + 2 * 2, i + 2 * 3), 16); + array[offset + i / 2 + 6] = parseInt(hash2.substring(i + 2 * 1, i + 2 * 2), 16); + array[offset + i / 2 + 7] = parseInt(hash2.substring(i + 2 * 0, i + 2 * 1), 16); + } +} +async function uploadXorb(xorb, params) { + const token = await xetWriteToken(params); + const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${token.casUrl}/v1/xorbs/default/${xorb.hash}`, { + method: "POST", + body: xorb.xorb, + headers: { + Authorization: `Bearer ${token.accessToken}`, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + }, + ...{ + progressHint: { + progressCallback: (progress) => { + for (const file of xorb.files) { + _optionalChain([params, 'access', _100 => _100.yieldCallback, 'optionalCall', _101 => _101({ + event: "fileProgress", + path: file.path, + progress: file.lastSentProgress + (file.progress - file.lastSentProgress) * progress + })]); + } + } + } + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } +} +async function uploadShard(shard, params) { + const token = await xetWriteToken(params); + const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${token.casUrl}/v1/shards`, { + method: "POST", + body: shard, + headers: { + Authorization: `Bearer ${token.accessToken}`, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } +} + +// src/utils/splitAsyncGenerator.ts +function splitAsyncGenerator(source, n) { + if (n <= 0) { + return []; + } + const sleep = (ms) => new Promise((resolve2) => setTimeout(resolve2, ms)); + let takenIndex = null; + const generators = []; + let remaining = n; + for (let i = 0; i < n; i++) { + generators.push({ + next: async () => { + while (takenIndex !== null) { + await sleep(1); + } + takenIndex = i; + return source.next().then((r) => { + takenIndex = null; + return r; + }); + }, + return: async () => { + remaining--; + if (remaining === 0) { + return source.return(void 0); + } + return { + done: true, + value: void 0 + }; + }, + throw: async (error) => { + return source.throw(error); + }, + [Symbol.asyncIterator]: () => generators[i] + }); + } + return generators; +} + +// src/lib/commit.ts +var CONCURRENT_SHAS = 5; +var CONCURRENT_LFS_UPLOADS = 5; +var MULTIPART_PARALLEL_UPLOAD = 5; +function isFileOperation(op) { + const ret = op.operation === "addOrUpdate"; + if (ret && !(op.content instanceof Blob)) { + throw new TypeError("Precondition failed: op.content should be a Blob"); + } + return ret; +} +async function* commitIter(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + if (repoId.type === "bucket") { + return yield* commitIterBucket(params); + } + if (params.operations.some((op) => op.operation === "copy")) { + throw new Error("'copy' operations are only supported when the destination repo is a bucket"); + } + yield { event: "phase", phase: "preuploading" }; + let useXet = _nullishCoalesce(params.useXet, () => ( true)); + const lfsShas = /* @__PURE__ */ new Map(); + const abortController = new AbortController(); + const abortSignal = abortController.signal; + if (!abortSignal.throwIfAborted) { + abortSignal.throwIfAborted = () => { + if (abortSignal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + }; + } + if (params.abortSignal) { + params.abortSignal.addEventListener("abort", () => abortController.abort()); + } + try { + const allOperations = (await Promise.all( + params.operations.map(async (operation) => { + if (operation.operation === "edit") { + const splicedBlob = SplicedBlob.create( + operation.originalContent, + operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })) + ); + return { + operation: "addOrUpdate", + path: operation.path, + content: splicedBlob + }; + } + if (operation.operation !== "addOrUpdate") { + return operation; + } + if (!(operation.content instanceof URL)) { + return { ...operation, content: operation.content }; + } + const lazyBlobs = await createBlobs(operation.content, operation.path, { + fetch: params.fetch, + maxFolderDepth: params.maxFolderDepth + }); + _optionalChain([abortSignal, 'optionalAccess', _102 => _102.throwIfAborted, 'call', _103 => _103()]); + return lazyBlobs.map((blob) => ({ + ...operation, + content: blob.blob, + path: blob.path + })); + }) + )).flat(1); + const gitAttributes = _optionalChain([allOperations, 'access', _104 => _104.filter, 'call', _105 => _105(isFileOperation), 'access', _106 => _106.find, 'call', _107 => _107((op) => op.path === ".gitattributes"), 'optionalAccess', _108 => _108.content]); + for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { + const payload = { + gitAttributes: gitAttributes && await gitAttributes.text(), + files: await Promise.all( + operations.map(async (operation) => ({ + path: operation.path, + size: operation.content.size, + sample: base64FromBytes(new Uint8Array(await operation.content.slice(0, 512).arrayBuffer())) + })) + ) + }; + _optionalChain([abortSignal, 'optionalAccess', _109 => _109.throwIfAborted, 'call', _110 => _110()]); + const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( + `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/preupload/${encodeURIComponent( + _nullishCoalesce(params.branch, () => ( "main")) + )}` + (params.isPullRequest ? "?create_pr=1" : ""), + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/json" + }, + body: JSON.stringify(payload), + signal: abortSignal + } + ); + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + for (const file of json.files) { + if (file.uploadMode === "lfs") { + lfsShas.set(file.path, null); + } + } + } + yield { event: "phase", phase: "uploadingLargeFiles" }; + for (const operations of chunk( + allOperations.filter(isFileOperation).filter((op) => lfsShas.has(op.path)), + 100 + )) { + const shas = yield* eventToGenerator((yieldCallback, returnCallback, rejectCallack) => { + return promisesQueue( + operations.map((op) => async () => { + const iterator = sha256(op.content, { useWebWorker: params.useWebWorkers, abortSignal }); + let res2; + do { + res2 = await iterator.next(); + if (!res2.done) { + yieldCallback({ event: "fileProgress", path: op.path, progress: res2.value, state: "hashing" }); + } + } while (!res2.done); + const sha = res2.value; + lfsShas.set(op.path, res2.value); + return sha; + }), + CONCURRENT_SHAS + ).then(returnCallback, rejectCallack); + }); + _optionalChain([abortSignal, 'optionalAccess', _111 => _111.throwIfAborted, 'call', _112 => _112()]); + const payload = { + operation: "upload", + // multipart is a custom protocol for HF + transfers: ["basic", "multipart", ...useXet ? ["xet"] : []], + hash_algo: "sha_256", + ...!params.isPullRequest && { + ref: { + name: _nullishCoalesce(params.branch, () => ( "main")) + } + }, + objects: operations.map((op, i) => ({ + oid: shas[i], + size: op.content.size + })) + }; + const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( + `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/${repoId.type === "model" ? "" : repoId.type + "s/"}${repoId.name}.git/info/lfs/objects/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + Accept: "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json" + }, + body: JSON.stringify(payload), + signal: abortSignal + } + ); + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + const batchRequestId = res.headers.get("X-Request-Id") || void 0; + const shaToOperation = new Map(operations.map((op, i) => [shas[i], op])); + if (useXet && json.transfer !== "xet") { + useXet = false; + } + let xetParams = null; + if (useXet) { + for (const obj of json.objects) { + const op = shaToOperation.get(obj.oid); + if (!op) { + throw new InvalidApiResponseFormatError("Unrequested object ID in response"); + } + if (obj.error) { + const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${obj.error.message}${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; + throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); + } + if (!_optionalChain([obj, 'access', _113 => _113.actions, 'optionalAccess', _114 => _114.upload])) { + yield { + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }; + } else { + const headers = new Headers(obj.actions.upload.header); + xetParams = { + sessionId: _nullishCoalesce(headers.get("X-Xet-Session-Id"), () => ( void 0)), + casUrl: _nullishCoalesce(headers.get("X-Xet-Cas-Url"), () => ( void 0)), + accessToken: _nullishCoalesce(headers.get("X-Xet-Access-Token"), () => ( void 0)), + expiresAt: headers.get("X-Xet-Token-Expiration") ? new Date(parseInt(_nullishCoalesce(headers.get("X-Xet-Token-Expiration"), () => ( "0"))) * 1e3) : void 0, + refreshWriteTokenUrl: obj.actions.upload.href + }; + } + } + const source = async function* () { + for (const obj of json.objects) { + const op = shaToOperation.get(obj.oid); + if (!op || !_optionalChain([obj, 'access', _115 => _115.actions, 'optionalAccess', _116 => _116.upload])) { + continue; + } + _optionalChain([abortSignal, 'optionalAccess', _117 => _117.throwIfAborted, 'call', _118 => _118()]); + yield { content: op.content, path: op.path, sha256: obj.oid }; + } + }(); + if (xetParams) { + const fixedXetParams = xetParams; + const sources = splitAsyncGenerator(source, 5); + yield* eventToGenerator( + (yieldCallback, returnCallback, rejectCallback) => Promise.all( + sources.map(async function(source2) { + for await (const event of uploadShards(source2, { + fetch: params.fetch, + accessToken, + hubUrl: _nullishCoalesce(params.hubUrl, () => ( HUB_URL)), + repo: repoId, + xetParams: fixedXetParams, + // todo: maybe leave empty if PR? + rev: _nullishCoalesce(params.branch, () => ( "main")), + isPullRequest: params.isPullRequest, + yieldCallback: (event2) => yieldCallback({ ...event2, state: "uploading" }) + })) { + if (event.event === "file") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: 1, + state: "uploading" + }); + } else if (event.event === "fileProgress") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: event.progress, + state: "uploading" + }); + } + } + }) + ).then(() => returnCallback(void 0), rejectCallback) + ); + } else { + } + } else { + yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + return promisesQueueStreaming( + json.objects.map((obj) => async () => { + const op = shaToOperation.get(obj.oid); + if (!op) { + throw new InvalidApiResponseFormatError("Unrequested object ID in response"); + } + _optionalChain([abortSignal, 'optionalAccess', _119 => _119.throwIfAborted, 'call', _120 => _120()]); + if (obj.error) { + const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${obj.error.message}${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; + throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); + } + if (!_optionalChain([obj, 'access', _121 => _121.actions, 'optionalAccess', _122 => _122.upload])) { + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + return; + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 0, + state: "uploading" + }); + const content = op.content; + const header = obj.actions.upload.header; + if (_optionalChain([header, 'optionalAccess', _123 => _123.chunk_size])) { + const chunkSize = parseInt(header.chunk_size); + const completionUrl = obj.actions.upload.href; + const parts = Object.keys(header).filter((key) => /^[0-9]+$/.test(key)); + if (parts.length !== Math.ceil(content.size / chunkSize)) { + throw new Error("Invalid server response to upload large LFS file, wrong number of parts"); + } + const completeReq = { + oid: obj.oid, + parts: parts.map((part) => ({ + partNumber: +part, + etag: "" + })) + }; + const progressCallback = (progress) => yieldCallback({ event: "fileProgress", path: op.path, progress, state: "uploading" }); + await promisesQueueStreaming( + parts.map((part) => async () => { + _optionalChain([abortSignal, 'optionalAccess', _124 => _124.throwIfAborted, 'call', _125 => _125()]); + const index = parseInt(part) - 1; + const slice = content.slice(index * chunkSize, (index + 1) * chunkSize); + const res3 = await (_nullishCoalesce(params.fetch, () => ( fetch)))(header[part], { + method: "PUT", + /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ + body: slice instanceof WebBlob && isFrontend ? await slice.arrayBuffer() : slice, + signal: abortSignal, + ...{ + progressHint: { + path: op.path, + part: index, + numParts: parts.length, + progressCallback + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + }); + if (!res3.ok) { + throw await createApiError(res3, { + requestId: batchRequestId, + message: `Error while uploading part ${part} of ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + const eTag = res3.headers.get("ETag"); + if (!eTag) { + throw new Error("Cannot get ETag of part during multipart upload"); + } + completeReq.parts[Number(part) - 1].etag = eTag; + }), + MULTIPART_PARALLEL_UPLOAD + ); + _optionalChain([abortSignal, 'optionalAccess', _126 => _126.throwIfAborted, 'call', _127 => _127()]); + const res2 = await (_nullishCoalesce(params.fetch, () => ( fetch)))(completionUrl, { + method: "POST", + body: JSON.stringify(completeReq), + headers: { + Accept: "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json" + }, + signal: abortSignal + }); + if (!res2.ok) { + throw await createApiError(res2, { + requestId: batchRequestId, + message: `Error completing multipart upload of ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + } else { + const res2 = await (_nullishCoalesce(params.fetch, () => ( fetch)))(obj.actions.upload.href, { + method: "PUT", + headers: { + ...batchRequestId ? { "X-Request-Id": batchRequestId } : void 0 + }, + /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ + body: content instanceof WebBlob && isFrontend ? await content.arrayBuffer() : content, + signal: abortSignal, + ...{ + progressHint: { + path: op.path, + progressCallback: (progress) => yieldCallback({ + event: "fileProgress", + path: op.path, + progress, + state: "uploading" + }) + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + }); + if (!res2.ok) { + throw await createApiError(res2, { + requestId: batchRequestId, + message: `Error while uploading ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + } + }), + CONCURRENT_LFS_UPLOADS + ).then(returnCallback, rejectCallback); + }); + } + } + _optionalChain([abortSignal, 'optionalAccess', _128 => _128.throwIfAborted, 'call', _129 => _129()]); + yield { event: "phase", phase: "committing" }; + return yield* eventToGenerator( + async (yieldCallback, returnCallback, rejectCallback) => (_nullishCoalesce(params.fetch, () => ( fetch)))( + `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/commit/${encodeURIComponent( + _nullishCoalesce(params.branch, () => ( "main")) + )}` + (params.isPullRequest ? "?create_pr=1" : ""), + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: [ + { + key: "header", + value: { + summary: params.title, + description: params.description, + parentCommit: params.parentCommit + } + }, + ...await Promise.all( + allOperations.map((operation) => { + if (isFileOperation(operation)) { + const sha = lfsShas.get(operation.path); + if (sha) { + return { + key: "lfsFile", + value: { + path: operation.path, + algo: "sha256", + size: operation.content.size, + oid: sha + } + }; + } + } + return convertOperationToNdJson(operation); + }) + ) + ].map((x) => JSON.stringify(x)).join("\n"), + signal: abortSignal, + ...{ + progressHint: { + progressCallback: (progress) => { + for (const op of allOperations) { + if (isFileOperation(op) && !lfsShas.has(op.path)) { + yieldCallback({ + event: "fileProgress", + path: op.path, + progress, + state: "uploading" + }); + } + } + } + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + } + ).then(async (res) => { + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + returnCallback({ + pullRequestUrl: json.pullRequestUrl, + commit: { + oid: json.commitOid, + url: json.commitUrl + }, + hookOutput: json.hookOutput + }); + }).catch(rejectCallback) + ); + } catch (err) { + abortController.abort(); + throw err; + } +} +async function* commitIterBucket(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + if (params.useXet === false) { + throw new Error("useXet must be true or undefined for buckets"); + } + const abortController = new AbortController(); + const abortSignal = abortController.signal; + if (!abortSignal.throwIfAborted) { + abortSignal.throwIfAborted = () => { + if (abortSignal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + }; + } + if (params.abortSignal) { + params.abortSignal.addEventListener("abort", () => abortController.abort()); + } + try { + const allOperations = (await Promise.all( + params.operations.map(async (operation) => { + if (operation.operation === "edit") { + const splicedBlob = SplicedBlob.create( + operation.originalContent, + operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })) + ); + return { + operation: "addOrUpdate", + path: operation.path, + content: splicedBlob + }; + } + if (operation.operation !== "addOrUpdate") { + return operation; + } + if (!(operation.content instanceof URL)) { + return { ...operation, content: operation.content }; + } + const lazyBlobs = await createBlobs(operation.content, operation.path, { + fetch: params.fetch, + maxFolderDepth: params.maxFolderDepth + }); + _optionalChain([abortSignal, 'optionalAccess', _130 => _130.throwIfAborted, 'call', _131 => _131()]); + return lazyBlobs.map((blob) => ({ + ...operation, + content: blob.blob, + path: blob.path + })); + }) + )).flat(1); + yield { event: "phase", phase: "uploadingLargeFiles" }; + for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { + const xetHashes = /* @__PURE__ */ new Map(); + _optionalChain([abortSignal, 'optionalAccess', _132 => _132.throwIfAborted, 'call', _133 => _133()]); + const source = async function* () { + for (const operation of operations) { + _optionalChain([abortSignal, 'optionalAccess', _134 => _134.throwIfAborted, 'call', _135 => _135()]); + yield { content: operation.content, path: operation.path }; + } + }(); + const xetParams = { + sessionId: crypto.randomUUID(), + refreshWriteTokenUrl: `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/xet-write-token` + }; + const sources = splitAsyncGenerator(source, 5); + yield* eventToGenerator( + (yieldCallback, returnCallback, rejectCallback) => Promise.all( + sources.map(async function(source2) { + for await (const event of uploadShards(source2, { + fetch: params.fetch, + accessToken, + hubUrl: _nullishCoalesce(params.hubUrl, () => ( HUB_URL)), + repo: repoId, + xetParams, + rev: _nullishCoalesce(params.branch, () => ( "main")), + yieldCallback: (event2) => yieldCallback({ ...event2, state: "uploading" }) + })) { + if (event.event === "file") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: 1, + state: "uploading" + }); + xetHashes.set(event.path, event.xetHash); + } else if (event.event === "fileProgress") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: event.progress, + state: "uploading" + }); + } + } + }) + ).then(() => returnCallback(void 0), rejectCallback) + ); + const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))( + `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: [...xetHashes.entries()].map( + ([path, xetHash]) => JSON.stringify({ + type: "addFile", + path, + xetHash + }) + ).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok && resp.status !== 422) { + throw await createApiError(resp); + } + const json = await resp.json(); + for (const failed of json.failed) { + yield { + event: "fileProgress", + path: failed.path, + progress: 0, + state: "error" + }; + } + } + _optionalChain([abortSignal, 'optionalAccess', _136 => _136.throwIfAborted, 'call', _137 => _137()]); + const copyOperations = allOperations.filter( + (operation) => operation.operation === "copy" + ); + for (const copyChunk of chunk(copyOperations, 100)) { + _optionalChain([abortSignal, 'optionalAccess', _138 => _138.throwIfAborted, 'call', _139 => _139()]); + const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))( + `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: copyChunk.map((op) => { + const sourceRepoId = toRepoId(op.sourceRepo); + return JSON.stringify({ + type: "copyFile", + path: op.path, + xetHash: op.sourceXetHash, + sourceRepoType: sourceRepoId.type, + sourceRepoId: sourceRepoId.name + }); + }).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok && resp.status !== 422) { + throw await createApiError(resp); + } + const json = await resp.json(); + for (const failed of json.failed) { + yield { + event: "fileProgress", + path: failed.path, + progress: 0, + state: "error" + }; + } + } + _optionalChain([abortSignal, 'optionalAccess', _140 => _140.throwIfAborted, 'call', _141 => _141()]); + const deletedOperations = allOperations.filter((operation) => operation.operation === "delete"); + if (deletedOperations.length > 0) { + const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))( + `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: deletedOperations.map( + (operation) => JSON.stringify({ + type: "deleteFile", + path: operation.path + }) + ).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok) { + throw await createApiError(resp); + } + const json = await resp.json(); + if (json.failed.length > 0) { + const failedPaths = json.failed.slice(0, 5).map((f) => f.path); + throw new Error( + `Failed to delete ${json.failed.length} file(s): ${failedPaths.join(", ")}${json.failed.length > 5 ? "..." : ""}, request ID: ${resp.headers.get("X-Request-Id")}` + ); + } + } + _optionalChain([abortSignal, 'optionalAccess', _142 => _142.throwIfAborted, 'call', _143 => _143()]); + } catch (err) { + abortController.abort(); + throw err; + } +} +async function commit(params) { + const iterator = commitIter(params); + const failedPaths = []; + let failedCount = 0; + let res = await iterator.next(); + while (!res.done) { + if (res.value.event === "fileProgress" && res.value.state === "error") { + failedCount++; + if (failedPaths.length < 5) { + failedPaths.push(res.value.path); + } + } + res = await iterator.next(); + } + if (failedCount > 0) { + throw new Error( + `Failed to upload ${failedCount} file(s): ${failedPaths.join(", ")}${failedCount > 5 ? "..." : ""}` + ); + } + return res.value; +} +async function convertOperationToNdJson(operation) { + switch (operation.operation) { + case "addOrUpdate": { + return { + key: "file", + value: { + content: base64FromBytes(new Uint8Array(await operation.content.arrayBuffer())), + path: operation.path, + encoding: "base64" + } + }; + } + case "delete": { + return { + key: "deletedFile", + value: { + path: operation.path + } + }; + } + case "edit": { + throw new Error( + "Edit operations should be converted to addOrUpdate operations before reaching convertOperationToNdJson" + ); + } + default: + throw new TypeError("Unknown operation: " + operation.operation); + } +} + +// src/utils/formatBytes.ts +function formatBytes(bytes) { + if (!Number.isFinite(bytes) || bytes < 0) { + return `${bytes} B`; + } + const units = ["B", "kB", "MB", "GB", "TB", "PB"]; + let value = bytes; + let i = 0; + while (value >= 1e3 && i < units.length - 1) { + value /= 1e3; + i++; + } + const formatted = i === 0 ? value.toString() : value.toFixed(value >= 100 ? 0 : value >= 10 ? 1 : 2); + return `${formatted} ${units[i]}`; +} + +// src/utils/parseLinkHeader.ts +function parseLinkHeader(header) { + const regex = /<(https?:[/][/][^>]+)>;\s+rel="([^"]+)"/g; + return Object.fromEntries([...header.matchAll(regex)].map(([, url, rel]) => [rel, url])); +} + +// src/lib/file-download-info.ts +async function fileDownloadInfo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const hubUrl = _nullishCoalesce(params.hubUrl, () => ( HUB_URL)); + const revision = repoId.type === "bucket" ? void 0 : _nullishCoalesce(params.revision, () => ( "main")); + const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${params.raw ? "raw" : "resolve"}${revision ? `/${encodeURIComponent(revision)}` : ""}/${params.path}` + (params.noContentDisposition ? "?noContentDisposition=1" : ""); + const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { + method: "GET", + headers: { + ...accessToken && { + Authorization: `Bearer ${accessToken}` + }, + Range: "bytes=0-0", + Accept: "application/vnd.xet-fileinfo+json, */*" + } + }); + if (resp.status === 404 && resp.headers.get("X-Error-Code") === "EntryNotFound") { + return null; + } + if (!resp.ok) { + throw await createApiError(resp); + } + let size; + let xetInfo; + if (_optionalChain([resp, 'access', _144 => _144.headers, 'access', _145 => _145.get, 'call', _146 => _146("Content-Type"), 'optionalAccess', _147 => _147.includes, 'call', _148 => _148("application/vnd.xet-fileinfo+json")])) { + size = parseInt(_nullishCoalesce(resp.headers.get("X-Linked-Size"), () => ( "invalid"))); + if (isNaN(size)) { + throw new InvalidApiResponseFormatError("Invalid file size received in X-Linked-Size header"); + } + const hash2 = resp.headers.get("X-Xet-Hash"); + const links = parseLinkHeader(_nullishCoalesce(resp.headers.get("Link"), () => ( ""))); + const reconstructionUrl = (() => { + try { + return new URL(links["xet-reconstruction-info"]); + } catch (e2) { + return null; + } + })(); + const refreshUrl = (() => { + try { + return new URL(links["xet-auth"]); + } catch (e3) { + return null; + } + })(); + if (!hash2) { + throw new InvalidApiResponseFormatError("No hash received in X-Xet-Hash header"); + } + if (!reconstructionUrl || !refreshUrl) { + throw new InvalidApiResponseFormatError("No xet-reconstruction-info or xet-auth link header"); + } + xetInfo = { + hash: hash2, + refreshUrl, + reconstructionUrl + }; + } + if (size === void 0 || isNaN(size)) { + const contentRangeHeader = resp.headers.get("content-range"); + if (!contentRangeHeader) { + throw new InvalidApiResponseFormatError("Expected size information"); + } + const [, parsedSize] = contentRangeHeader.split("/"); + size = parseInt(parsedSize); + if (isNaN(size)) { + throw new InvalidApiResponseFormatError("Invalid file size received"); + } + } + const etag = _nullishCoalesce(_nullishCoalesce(resp.headers.get("X-Linked-ETag"), () => ( resp.headers.get("ETag"))), () => ( void 0)); + if (!etag) { + throw new InvalidApiResponseFormatError("Expected ETag"); + } + return { + etag, + size, + xet: xetInfo, + // Cannot use resp.url in case it's a S3 url and the user adds an Authorization header to it. + url: resp.url && (new URL(resp.url).origin === new URL(hubUrl).origin || _optionalChain([resp, 'access', _149 => _149.headers, 'access', _150 => _150.get, 'call', _151 => _151("X-Cache"), 'optionalAccess', _152 => _152.endsWith, 'call', _153 => _153(" cloudfront")])) ? resp.url : url + }; +} + +// src/lib/download-file.ts +async function downloadFile(params) { + const accessToken = checkCredentials(params); + const info = await _asyncNullishCoalesce(params.downloadInfo, async () => ( await fileDownloadInfo({ + accessToken, + repo: params.repo, + path: params.path, + revision: params.revision, + hubUrl: params.hubUrl, + fetch: params.fetch, + raw: params.raw + }))); + if (!info) { + return null; + } + if (info.xet && params.xet !== false) { + return new XetBlob({ + refreshUrl: info.xet.refreshUrl.href, + reconstructionUrl: info.xet.reconstructionUrl.href, + fetch: params.fetch, + accessToken, + size: info.size, + readToken: typeof params.xet === "object" ? params.xet.readToken : void 0 + }); + } + return new WebBlob(new URL(info.url), 0, info.size, "", true, _nullishCoalesce(params.fetch, () => ( fetch)), accessToken); +} + +// src/lib/list-files.ts +async function* listFiles(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const revision = repoId.type === "bucket" ? void 0 : params.revision || "main"; + let url = `${params.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}/tree${revision ? `/${revision}` : ""}${params.path ? "/" + params.path : ""}?recursive=${!!params.recursive}&expand=${!!params.expand}`; + while (url) { + const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + yield item; + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/paths-info.ts +async function pathsInfo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const hubUrl = _nullishCoalesce(params.hubUrl, () => ( HUB_URL)); + const revision = repoId.type === "bucket" ? void 0 : _nullishCoalesce(params.revision, () => ( "main")); + const url = `${hubUrl}/api/${repoId.type}s/${repoId.name}/paths-info${revision ? `/${encodeURIComponent(revision)}` : ""}`; + const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { + method: "POST", + headers: { + ...accessToken && { + Authorization: `Bearer ${accessToken}` + }, + Accept: "application/json", + "Content-Type": "application/json" + }, + body: JSON.stringify({ + paths: params.paths, + expand: params.expand + }) + }); + if (!resp.ok) { + throw await createApiError(resp); + } + const json = await resp.json(); + if (!Array.isArray(json)) { + throw new Error("malformed response: expected array"); + } + return json.map((item) => ({ + path: item.path, + lfs: item.lfs, + type: item.type, + oid: item.oid, + size: item.size, + xetHash: item.xetHash, + uploadedAt: item.uploadedAt, + securityFileStatus: item.securityFileStatus, + lastCommit: item.lastCommit ? { + date: new Date(item.lastCommit.date), + title: item.lastCommit.title, + id: item.lastCommit.id + } : void 0 + })); +} + +// src/lib/copy-files.ts +var DOWNLOAD_CONCURRENCY = 5; +var PATHS_INFO_BATCH_SIZE = 100; +var MAX_REPORTED_LFS_PATHS = 5; +function copyFile(params) { + return copyFiles({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + destination: params.destination.repo, + files: [ + { + source: params.source, + destinationPath: params.destination.path + } + ], + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); +} +function copyFileIter(params) { + return copyFilesIter({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + destination: params.destination.repo, + files: [ + { + source: params.source, + destinationPath: params.destination.path + } + ], + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); +} +async function copyFiles(params) { + const iterator = copyFilesIter(params); + while (true) { + const res = await iterator.next(); + if (res.done) { + return void 0; + } + } +} +async function* copyFilesIter(params) { + if (params.files.length === 0) { + return void 0; + } + const operations = yield* resolveCopyOperationsIter(params, params.files); + await commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.destination, + operations, + title: "", + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); + return void 0; +} +async function copyFolder(params) { + const iterator = copyFolderIter(params); + while (true) { + const res = await iterator.next(); + if (res.done) { + return void 0; + } + } +} +async function* copyFolderIter(params) { + const accessToken = checkCredentials(params); + const sourceRepoId = toRepoId(params.source.repo); + const sourcePath = (_nullishCoalesce(params.source.path, () => ( ""))).replace(/\/+$/, ""); + const destinationPrefix = (_nullishCoalesce(params.destination.path, () => ( ""))).replace(/\/+$/, ""); + const sourceRevision = sourceRepoId.type === "bucket" ? void 0 : _nullishCoalesce(params.source.revision, () => ( "main")); + const operations = []; + const pendingDownloads = []; + const lfsOffenders = []; + for await (const item of listFiles({ + repo: sourceRepoId, + path: sourcePath || void 0, + recursive: true, + revision: sourceRevision, + accessToken, + hubUrl: params.hubUrl, + fetch: params.fetch + })) { + if (item.type !== "file") { + continue; + } + const relPath = relativeUnderFolder(item.path, sourcePath); + const destPath = destinationPrefix ? `${destinationPrefix}/${relPath}` : relPath; + switch (classifySourceFile(item)) { + case "copy": + operations.push({ + operation: "copy", + path: destPath, + sourceXetHash: item.xetHash, + sourceRepo: sourceRepoId + }); + continue; + case "lfs": + lfsOffenders.push({ path: item.path, size: _nullishCoalesce(_optionalChain([item, 'access', _154 => _154.lfs, 'optionalAccess', _155 => _155.size]), () => ( item.size)) }); + continue; + case "download": + pendingDownloads.push({ + index: operations.length, + repoId: sourceRepoId, + revision: sourceRevision, + sourcePath: item.path + }); + operations.push({ + operation: "addOrUpdate", + path: destPath, + content: new Blob([]) + }); + continue; + } + } + if (lfsOffenders.length > 0) { + throwUnmigratedLfsError(sourceRepoId, lfsOffenders); + } + if (operations.length === 0) { + return void 0; + } + yield* downloadAndFillBlobsIter({ + pendingDownloads, + operations, + accessToken, + hubUrl: params.hubUrl, + fetch: params.fetch + }); + await commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.destination.repo, + operations, + title: "", + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); + return void 0; +} +async function* resolveCopyOperationsIter(shared, files) { + const accessToken = checkCredentials(shared); + const groups = /* @__PURE__ */ new Map(); + for (let i = 0; i < files.length; i++) { + const file = files[i]; + const repoId = toRepoId(file.source.repo); + const revision = repoId.type === "bucket" ? void 0 : _nullishCoalesce(file.source.revision, () => ( "main")); + const key = `${repoId.type}\0${repoId.name}\0${_nullishCoalesce(revision, () => ( ""))}`; + let group = groups.get(key); + if (!group) { + group = { repoId, revision, entries: [] }; + groups.set(key, group); + } + group.entries.push({ index: i, file }); + } + const operations = new Array(files.length); + const pendingDownloads = []; + for (const group of groups.values()) { + const paths = group.entries.map((e) => e.file.source.path); + const infos = []; + for (let offset = 0; offset < paths.length; offset += PATHS_INFO_BATCH_SIZE) { + const slice = paths.slice(offset, offset + PATHS_INFO_BATCH_SIZE); + const res = await pathsInfo({ + repo: group.repoId, + paths: slice, + revision: group.revision, + accessToken, + hubUrl: shared.hubUrl, + fetch: shared.fetch + }); + infos.push(...res); + } + const infoByPath = new Map(infos.map((i) => [i.path, i])); + const lfsOffenders = []; + for (const { index, file } of group.entries) { + const info = infoByPath.get(file.source.path); + if (!info) { + throw new Error(`Source file not found: '${file.source.path}' in ${group.repoId.type}s/${group.repoId.name}`); + } + if (info.type !== "file") { + throw new Error( + `Source path '${file.source.path}' in ${group.repoId.type}s/${group.repoId.name} is a folder; use copyFolder() instead.` + ); + } + switch (classifySourceFile(info)) { + case "copy": + operations[index] = { + operation: "copy", + path: file.destinationPath, + sourceXetHash: info.xetHash, + sourceRepo: group.repoId + }; + continue; + case "lfs": + lfsOffenders.push({ path: file.source.path, size: _nullishCoalesce(_optionalChain([info, 'access', _156 => _156.lfs, 'optionalAccess', _157 => _157.size]), () => ( info.size)) }); + continue; + case "download": + pendingDownloads.push({ + index, + repoId: group.repoId, + revision: group.revision, + sourcePath: file.source.path + }); + operations[index] = { + operation: "addOrUpdate", + path: file.destinationPath, + content: new Blob([]) + }; + continue; + } + } + if (lfsOffenders.length > 0) { + throwUnmigratedLfsError(group.repoId, lfsOffenders); + } + } + yield* downloadAndFillBlobsIter({ + pendingDownloads, + operations, + accessToken, + hubUrl: shared.hubUrl, + fetch: shared.fetch + }); + return operations; +} +function downloadAndFillBlobsIter(args) { + const total = args.pendingDownloads.length; + return eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + if (total === 0) { + returnCallback(); + return; + } + let downloaded = 0; + promisesQueue( + args.pendingDownloads.map(({ index, repoId, revision, sourcePath }) => async () => { + const blob = await downloadFile({ + repo: repoId, + path: sourcePath, + revision, + accessToken: args.accessToken, + hubUrl: args.hubUrl, + fetch: args.fetch + }); + if (!blob) { + throw new Error(`Failed to download '${sourcePath}' from ${repoId.type}s/${repoId.name}`); + } + const op = args.operations[index]; + if (op.operation !== "addOrUpdate") { + throw new Error("Internal: expected addOrUpdate placeholder operation"); + } + op.content = blob; + downloaded++; + yieldCallback({ event: "fileDownloaded", path: sourcePath, downloaded, total }); + }), + DOWNLOAD_CONCURRENCY + ).then( + () => returnCallback(), + (err) => rejectCallback(err) + ); + }); +} +function relativeUnderFolder(filePath, folderPath) { + if (!folderPath) { + return filePath; + } + if (filePath === folderPath) { + return _nullishCoalesce(filePath.split("/").pop(), () => ( filePath)); + } + if (filePath.startsWith(folderPath + "/")) { + return filePath.slice(folderPath.length + 1); + } + throw new Error(`Path '${filePath}' is not inside folder '${folderPath}'`); +} +function classifySourceFile(file) { + if (file.xetHash) { + return "copy"; + } + if (file.lfs) { + return "lfs"; + } + return "download"; +} +function throwUnmigratedLfsError(repoId, entries) { + const head = entries.slice(0, MAX_REPORTED_LFS_PATHS).map((e) => `'${e.path}' (${formatBytes(e.size)})`).join(", "); + const more = entries.length > MAX_REPORTED_LFS_PATHS ? ` (and ${entries.length - MAX_REPORTED_LFS_PATHS} more)` : ""; + throw new Error( + `Cannot copy ${entries.length} LFS file(s) from ${repoId.type}s/${repoId.name} that have not been migrated to xet: ${head}${more}. Migrate these files to xet before copying.` + ); +} + +// src/lib/count-commits.ts +async function countCommits(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const url = `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/commits/${_nullishCoalesce(params.revision, () => ( "main"))}?limit=1`; + const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }); + if (!res.ok) { + throw await createApiError(res); + } + return parseInt(_nullishCoalesce(res.headers.get("x-total-count"), () => ( "0")), 10); +} + +// src/lib/create-repo.ts +async function createRepo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const [namespace, repoName] = repoId.name.split("/"); + const visibility = _nullishCoalesce(params.visibility, () => ( (params.private !== void 0 ? params.private ? "private" : "public" : void 0))); + if (!namespace || !repoName) { + throw new TypeError( + `"${repoId.name}" is not a fully qualified repo name. It should be of the form "{namespace}/{repoName}".` + ); + } + const res = repoId.type === "bucket" ? await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/buckets/${namespace}/${repoName}`, { + method: "POST", + body: JSON.stringify({ + visibility, + resourceGroupId: params.resourceGroupId + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }) : await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/repos/create`, { + method: "POST", + body: JSON.stringify({ + name: repoName, + visibility, + organization: namespace, + resourceGroupId: params.resourceGroupId, + license: params.license, + ...repoId.type === "space" ? { + type: "space", + sdk: _nullishCoalesce(params.sdk, () => ( "static")) + } : { + type: repoId.type + }, + files: params.files ? await Promise.all( + params.files.map(async (file) => ({ + encoding: "base64", + path: file.path, + content: base64FromBytes( + new Uint8Array(file.content instanceof Blob ? await file.content.arrayBuffer() : file.content) + ) + })) + ) : void 0 + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const output = await res.json(); + return { repoUrl: output.url, id: output.id }; +} + +// src/lib/create-branch.ts +async function createBranch(params) { + const repoId = toRepoId(params.repo); + const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( + `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + }, + body: JSON.stringify({ + startingPoint: params.revision, + ...params.empty && { emptyBranch: true }, + overwrite: params.overwrite + }) + } + ); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/create-collection.ts +async function createCollection(params) { + const accessToken = checkCredentials(params); + const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/collections`, { + method: "POST", + body: JSON.stringify(params.collection), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const output = await res.json(); + return { slug: output.slug }; +} + +// src/utils/pick.ts +function pick(o, props) { + return Object.assign( + {}, + ...props.map((prop) => { + if (o[prop] !== void 0) { + return { [prop]: o[prop] }; + } + }) + ); +} + +// src/lib/list-datasets.ts +var DATASET_EXPAND_KEYS = [ + "private", + "downloads", + "gated", + "likes", + "lastModified" +]; +var DATASET_EXPANDABLE_KEYS = [ + "author", + "cardData", + "citation", + "createdAt", + "disabled", + "description", + "downloads", + "downloadsAllTime", + "gated", + "gitalyUid", + "lastModified", + "likes", + "paperswithcode_id", + "private", + // "siblings", + "sha", + "tags" +]; +async function* listDatasets(params) { + const accessToken = params && checkCredentials(params); + let totalToFetch = _nullishCoalesce(_optionalChain([params, 'optionalAccess', _158 => _158.limit]), () => ( Infinity)); + const search = new URLSearchParams([ + ...Object.entries({ + limit: String(Math.min(totalToFetch, 500)), + ..._optionalChain([params, 'optionalAccess', _159 => _159.search, 'optionalAccess', _160 => _160.owner]) ? { author: params.search.owner } : void 0, + ..._optionalChain([params, 'optionalAccess', _161 => _161.search, 'optionalAccess', _162 => _162.query]) ? { search: params.search.query } : void 0, + ..._optionalChain([params, 'optionalAccess', _163 => _163.sort]) ? { sort: params.sort } : void 0 + }), + ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _164 => _164.search, 'optionalAccess', _165 => _165.tags, 'optionalAccess', _166 => _166.map, 'call', _167 => _167((tag) => ["filter", tag])]), () => ( [])), + ...DATASET_EXPAND_KEYS.map((val) => ["expand", val]), + ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _168 => _168.additionalFields, 'optionalAccess', _169 => _169.map, 'call', _170 => _170((val) => ["expand", val])]), () => ( [])) + ]).toString(); + let url = `${_optionalChain([params, 'optionalAccess', _171 => _171.hubUrl]) || HUB_URL}/api/datasets` + (search ? "?" + search : ""); + while (url) { + const res = await (_nullishCoalesce(_optionalChain([params, 'optionalAccess', _172 => _172.fetch]), () => ( fetch)))(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + yield { + ..._optionalChain([params, 'optionalAccess', _173 => _173.additionalFields]) && pick(item, params.additionalFields), + id: item._id, + name: item.id, + private: item.private, + downloads: item.downloads, + likes: item.likes, + gated: item.gated, + updatedAt: new Date(item.lastModified) + }; + totalToFetch--; + if (totalToFetch <= 0) { + return; + } + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/dataset-info.ts +async function datasetInfo(params) { + const accessToken = params && checkCredentials(params); + const search = new URLSearchParams([ + ...DATASET_EXPAND_KEYS.map((val) => ["expand", val]), + ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _174 => _174.additionalFields, 'optionalAccess', _175 => _175.map, 'call', _176 => _176((val) => ["expand", val])]), () => ( [])) + ]).toString(); + const response = await (params.fetch || fetch)( + `${_optionalChain([params, 'optionalAccess', _177 => _177.hubUrl]) || HUB_URL}/api/datasets/${params.name}/revision/${encodeURIComponent( + _nullishCoalesce(params.revision, () => ( "HEAD")) + )}?${search.toString()}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + const data = await response.json(); + return { + ..._optionalChain([params, 'optionalAccess', _178 => _178.additionalFields]) && pick(data, params.additionalFields), + id: data._id, + name: data.id, + private: data.private, + downloads: data.downloads, + likes: data.likes, + gated: data.gated, + updatedAt: new Date(data.lastModified) + }; +} + +// src/lib/delete-branch.ts +async function deleteBranch(params) { + const repoId = toRepoId(params.repo); + const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( + `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, + { + method: "DELETE", + headers: { + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + } + } + ); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/delete-file.ts +function deleteFile(params) { + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: [ + { + operation: "delete", + path: params.path + } + ], + title: _nullishCoalesce(params.commitTitle, () => ( `Delete ${params.path}`)), + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch + }); +} + +// src/lib/delete-files.ts +function deleteFiles(params) { + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: params.paths.map((path) => ({ + operation: "delete", + path + })), + title: _nullishCoalesce(params.commitTitle, () => ( `Deletes ${params.paths.length} files`)), + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch + }); +} + +// src/lib/delete-repo.ts +async function deleteRepo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const [namespace, repoName] = repoId.name.split("/"); + const res = repoId.type === "bucket" ? await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/buckets/${namespace}/${repoName}`, { + method: "DELETE", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }) : await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/repos/delete`, { + method: "DELETE", + body: JSON.stringify({ + name: repoName, + organization: namespace, + type: repoId.type + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/delete-collection.ts +async function deleteCollection(params) { + if (!params.slug) { + throw new TypeError("slug is required"); + } + const accessToken = checkCredentials(params); + const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/collections/${params.slug}`, { + method: "DELETE", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/file-exists.ts +async function fileExists(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const hubUrl = _nullishCoalesce(params.hubUrl, () => ( HUB_URL)); + const revision = repoId.type === "bucket" ? void 0 : _nullishCoalesce(params.revision, () => ( "main")); + const endpoint = repoId.type === "bucket" ? "resolve" : "raw"; + const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${endpoint}${revision ? `/${encodeURIComponent(revision)}` : ""}/${params.path}`; + const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { + method: "HEAD", + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }); + if (resp.status === 404) { + return false; + } + if (!resp.ok) { + throw await createApiError(resp); + } + return true; +} + +// src/lib/jobs/cancel-job.ts +async function cancelJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/cancel`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/create-scheduled-job.ts +async function createScheduledJob(params) { + const accessToken = checkCredentials(params); + const { namespace, hubUrl, fetch: customFetch, ...rest } = params; + if (!rest.jobSpec.dockerImage && !rest.jobSpec.spaceId) { + throw new Error("Either dockerImage or spaceId must be provided in jobSpec"); + } + if (rest.jobSpec.dockerImage && rest.jobSpec.spaceId) { + throw new Error("Cannot provide both dockerImage and spaceId in jobSpec"); + } + const body = { + jobSpec: { + flavor: rest.jobSpec.flavor + }, + schedule: rest.schedule, + suspend: _nullishCoalesce(rest.suspend, () => ( false)), + concurrency: _nullishCoalesce(rest.concurrency, () => ( false)) + }; + if (rest.jobSpec.dockerImage) { + body.jobSpec.dockerImage = rest.jobSpec.dockerImage; + } + if (rest.jobSpec.spaceId) { + body.jobSpec.spaceId = rest.jobSpec.spaceId; + } + if (rest.jobSpec.command) { + body.jobSpec.command = rest.jobSpec.command; + } + body.jobSpec.environment = rest.jobSpec.environment || {}; + if (rest.jobSpec.secrets) { + body.jobSpec.secrets = rest.jobSpec.secrets; + } + if (rest.jobSpec.arch) { + body.jobSpec.arch = rest.jobSpec.arch; + } + if (rest.jobSpec.timeoutSeconds !== void 0) { + body.jobSpec.timeoutSeconds = rest.jobSpec.timeoutSeconds; + } + if (rest.jobSpec.attempts !== void 0) { + body.jobSpec.attempts = rest.jobSpec.attempts; + } + if (rest.jobSpec.labels) { + body.jobSpec.labels = rest.jobSpec.labels; + } + if (_optionalChain([rest, 'access', _179 => _179.jobSpec, 'access', _180 => _180.volumes, 'optionalAccess', _181 => _181.length])) { + body.jobSpec.volumes = rest.jobSpec.volumes.map(({ source, ...rest2 }) => { + const repoId = toRepoId(source); + return { type: repoId.type, source: repoId.name, ...rest2 }; + }); + } + const response = await (customFetch || fetch)(`${hubUrl || HUB_URL}/api/scheduled-jobs/${namespace}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }, + body: JSON.stringify(body) + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/delete-scheduled-job.ts +async function deleteScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}`, + { + method: "DELETE", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/lib/jobs/duplicate-job.ts +async function duplicateJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/duplicate`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/get-job.ts +async function getJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/get-scheduled-job.ts +async function getScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/list-job-hardware.ts +async function listJobHardware(params) { + const accessToken = checkCredentials(_nullishCoalesce(params, () => ( {}))); + const headers = {}; + if (accessToken) { + headers.Authorization = `Bearer ${accessToken}`; + } + const response = await (_optionalChain([params, 'optionalAccess', _182 => _182.fetch]) || fetch)(`${_optionalChain([params, 'optionalAccess', _183 => _183.hubUrl]) || HUB_URL}/api/jobs/hardware`, { + headers + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/list-jobs.ts +async function listJobs(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/list-scheduled-jobs.ts +async function listScheduledJobs(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}`, { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/resume-scheduled-job.ts +async function resumeScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/resume`, + { + method: "POST", + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/lib/jobs/run-job.ts +async function runJob(params) { + const accessToken = checkCredentials(params); + if (!params.dockerImage && !params.spaceId) { + throw new Error("Either dockerImage or spaceId must be provided"); + } + if (params.dockerImage && params.spaceId) { + throw new Error("Cannot provide both dockerImage and spaceId"); + } + const body = { + flavor: params.flavor, + environment: params.environment || {} + }; + if (params.dockerImage) { + body.dockerImage = params.dockerImage; + } + if (params.spaceId) { + body.spaceId = params.spaceId; + } + if (params.command) { + body.command = params.command; + } + if (params.arguments) { + body.arguments = params.arguments; + } + if (params.secrets) { + body.secrets = params.secrets; + } + if (params.arch) { + body.arch = params.arch; + } + if (params.timeoutSeconds !== void 0) { + body.timeoutSeconds = params.timeoutSeconds; + } + if (params.attempts !== void 0) { + body.attempts = params.attempts; + } + if (params.labels) { + body.labels = params.labels; + } + if (_optionalChain([params, 'access', _184 => _184.volumes, 'optionalAccess', _185 => _185.length])) { + body.volumes = params.volumes.map(({ source, ...rest }) => { + const repoId = toRepoId(source); + return { type: repoId.type, source: repoId.name, ...rest }; + }); + } + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }, + body: JSON.stringify(body) + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/run-scheduled-job.ts +async function runScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/run`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + if (response.status === 409) { + return null; + } + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/stream-job-events.ts +async function* streamJobEvents(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/events`, + { + headers: { + Accept: "text/event-stream", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + if (!response.body) { + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + } finally { + reader.releaseLock(); + } +} + +// src/lib/jobs/stream-job-logs.ts +async function* streamJobLogs(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/logs`, + { + headers: { + Accept: "text/event-stream", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + if (!response.body) { + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const data = JSON.parse(line.slice(6)); + yield { message: data.data, timestamp: new Date(data.timestamp) }; + } catch (e4) { + yield { message: line.slice(6), timestamp: /* @__PURE__ */ new Date() }; + } + } + } + } + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const data = JSON.parse(line.slice(6)); + yield { message: data.data, timestamp: new Date(data.timestamp) }; + } catch (e5) { + yield { message: line.slice(6), timestamp: /* @__PURE__ */ new Date() }; + } + } + } + } + } finally { + reader.releaseLock(); + } +} + +// src/lib/jobs/stream-job-metrics.ts +async function* streamJobMetrics(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/metrics`, + { + headers: { + Accept: "text/event-stream", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + if (!response.body) { + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + } finally { + reader.releaseLock(); + } +} + +// src/lib/jobs/suspend-scheduled-job.ts +async function suspendScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/suspend`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/lib/list-commits.ts +async function* listCommits(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + let url = `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/commits/${_nullishCoalesce(params.revision, () => ( "main"))}?limit=${_nullishCoalesce(params.batchSize, () => ( 100))}`; + while (url) { + const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }); + if (!res.ok) { + throw await createApiError(res); + } + const resJson = await res.json(); + for (const commit2 of resJson) { + yield { + oid: commit2.id, + title: commit2.title, + message: commit2.message, + authors: commit2.authors.map((author) => ({ + username: author.user, + avatarUrl: author.avatar + })), + date: new Date(commit2.date) + }; + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/utils/normalizeInferenceProviderMapping.ts +function normalizeInferenceProviderMapping(hfModelId, inferenceProviderMapping) { + if (!inferenceProviderMapping) { + return []; + } + if (Array.isArray(inferenceProviderMapping)) { + return inferenceProviderMapping.map((entry) => ({ + ...entry, + hfModelId + })); + } + return Object.entries(inferenceProviderMapping).map(([provider, mapping]) => ({ + provider, + hfModelId, + providerId: mapping.providerId, + status: mapping.status, + task: mapping.task + })); +} + +// src/lib/list-models.ts +var MODEL_EXPAND_KEYS = [ + "pipeline_tag", + "private", + "gated", + "downloads", + "likes", + "lastModified" +]; +var MODEL_EXPANDABLE_KEYS = [ + "author", + "cardData", + "config", + "createdAt", + "disabled", + "downloads", + "downloadsAllTime", + "gated", + "gitalyUid", + "inferenceProviderMapping", + "lastModified", + "library_name", + "likes", + "model-index", + "pipeline_tag", + "private", + "safetensors", + "sha", + "spaces", + "tags", + "transformersInfo" +]; +var MODEL_DERIVED_FIELD_TO_API_KEY = { + filePaths: "siblings" +}; +async function* listModels(params) { + const accessToken = params && checkCredentials(params); + let totalToFetch = _nullishCoalesce(_optionalChain([params, 'optionalAccess', _186 => _186.limit]), () => ( Infinity)); + const additionalExpandKeys = _nullishCoalesce(_optionalChain([params, 'optionalAccess', _187 => _187.additionalFields, 'optionalAccess', _188 => _188.map, 'call', _189 => _189( + (field) => _nullishCoalesce(MODEL_DERIVED_FIELD_TO_API_KEY[field], () => ( field)) + )]), () => ( [])); + const search = new URLSearchParams([ + ...Object.entries({ + limit: String(Math.min(totalToFetch, 500)), + ..._optionalChain([params, 'optionalAccess', _190 => _190.search, 'optionalAccess', _191 => _191.owner]) ? { author: params.search.owner } : void 0, + ..._optionalChain([params, 'optionalAccess', _192 => _192.search, 'optionalAccess', _193 => _193.task]) ? { pipeline_tag: params.search.task } : void 0, + ..._optionalChain([params, 'optionalAccess', _194 => _194.search, 'optionalAccess', _195 => _195.query]) ? { search: params.search.query } : void 0, + ..._optionalChain([params, 'optionalAccess', _196 => _196.search, 'optionalAccess', _197 => _197.inferenceProviders]) ? { inference_provider: params.search.inferenceProviders.join(",") } : void 0, + ..._optionalChain([params, 'optionalAccess', _198 => _198.search, 'optionalAccess', _199 => _199.apps]) ? { apps: params.search.apps.join(",") } : void 0, + ..._optionalChain([params, 'optionalAccess', _200 => _200.sort]) ? { sort: params.sort } : void 0 + }), + ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _201 => _201.search, 'optionalAccess', _202 => _202.tags, 'optionalAccess', _203 => _203.map, 'call', _204 => _204((tag) => ["filter", tag])]), () => ( [])), + ...MODEL_EXPAND_KEYS.map((val) => ["expand", val]), + ...additionalExpandKeys.map((val) => ["expand", val]) + ]).toString(); + let url = `${_optionalChain([params, 'optionalAccess', _205 => _205.hubUrl]) || HUB_URL}/api/models?${search}`; + while (url) { + const res = await (_nullishCoalesce(_optionalChain([params, 'optionalAccess', _206 => _206.fetch]), () => ( fetch)))(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + const additional = {}; + if (_optionalChain([params, 'optionalAccess', _207 => _207.additionalFields])) { + for (const field of params.additionalFields) { + if (field === "filePaths") { + additional.filePaths = (_nullishCoalesce(item.siblings, () => ( []))).map((s) => s.rfilename); + } else if (field === "inferenceProviderMapping" && item.inferenceProviderMapping) { + additional.inferenceProviderMapping = normalizeInferenceProviderMapping( + item.id, + item.inferenceProviderMapping + ); + } else { + additional[field] = item[field]; + } + } + } + yield { + ...additional, + id: item._id, + name: item.id, + private: item.private, + task: item.pipeline_tag, + downloads: item.downloads, + gated: item.gated, + likes: item.likes, + updatedAt: new Date(item.lastModified) + }; + totalToFetch--; + if (totalToFetch <= 0) { + return; + } + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/list-spaces.ts +var SPACE_EXPAND_KEYS = [ + "sdk", + "likes", + "private", + "lastModified" +]; +var SPACE_EXPANDABLE_KEYS = [ + "author", + "cardData", + "datasets", + "disabled", + "gitalyUid", + "lastModified", + "createdAt", + "likes", + "private", + "runtime", + "sdk", + // "siblings", + "sha", + "subdomain", + "tags", + "models" +]; +async function* listSpaces(params) { + const accessToken = params && checkCredentials(params); + const search = new URLSearchParams([ + ...Object.entries({ + limit: "500", + ..._optionalChain([params, 'optionalAccess', _208 => _208.search, 'optionalAccess', _209 => _209.owner]) ? { author: params.search.owner } : void 0, + ..._optionalChain([params, 'optionalAccess', _210 => _210.search, 'optionalAccess', _211 => _211.query]) ? { search: params.search.query } : void 0, + ..._optionalChain([params, 'optionalAccess', _212 => _212.sort]) ? { sort: params.sort } : void 0 + }), + ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _213 => _213.search, 'optionalAccess', _214 => _214.tags, 'optionalAccess', _215 => _215.map, 'call', _216 => _216((tag) => ["filter", tag])]), () => ( [])), + ...[...SPACE_EXPAND_KEYS, ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _217 => _217.additionalFields]), () => ( []))].map( + (val) => ["expand", val] + ) + ]).toString(); + let url = `${_optionalChain([params, 'optionalAccess', _218 => _218.hubUrl]) || HUB_URL}/api/spaces?${search}`; + while (url) { + const res = await (_nullishCoalesce(_optionalChain([params, 'optionalAccess', _219 => _219.fetch]), () => ( fetch)))(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + yield { + ..._optionalChain([params, 'optionalAccess', _220 => _220.additionalFields]) && pick(item, params.additionalFields), + id: item._id, + name: item.id, + sdk: item.sdk, + likes: item.likes, + private: item.private, + updatedAt: new Date(item.lastModified) + }; + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/list-collections.ts +async function* listCollections(params) { + const accessToken = params && checkCredentials(params); + const searchParams = new URLSearchParams(); + let totalToFetch = _nullishCoalesce(_optionalChain([params, 'optionalAccess', _221 => _221.limit]), () => ( Infinity)); + searchParams.append("limit", String(Math.min(totalToFetch, 100))); + if (_optionalChain([params, 'optionalAccess', _222 => _222.sort])) { + searchParams.append("sort", params.sort); + } + if (_optionalChain([params, 'optionalAccess', _223 => _223.search, 'optionalAccess', _224 => _224.owner])) { + for (const owner of params.search.owner) { + searchParams.append("owner", owner); + } + } + if (_optionalChain([params, 'optionalAccess', _225 => _225.search, 'optionalAccess', _226 => _226.item])) { + for (const item of params.search.item) { + searchParams.append("item", item); + } + } + if (_optionalChain([params, 'optionalAccess', _227 => _227.search, 'optionalAccess', _228 => _228.q])) { + searchParams.append("q", params.search.q); + } + let url = `${_optionalChain([params, 'optionalAccess', _229 => _229.hubUrl]) || HUB_URL}/api/collections?${searchParams}`; + while (url) { + const res = await (_nullishCoalesce(_optionalChain([params, 'optionalAccess', _230 => _230.fetch]), () => ( fetch)))(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const collections = await res.json(); + for (const collection of collections) { + yield collection; + totalToFetch--; + if (totalToFetch <= 0) { + return; + } + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/model-info.ts +async function modelInfo(params) { + const accessToken = params && checkCredentials(params); + const additionalExpandKeys = _nullishCoalesce(_optionalChain([params, 'optionalAccess', _231 => _231.additionalFields, 'optionalAccess', _232 => _232.map, 'call', _233 => _233( + (field) => _nullishCoalesce(MODEL_DERIVED_FIELD_TO_API_KEY[field], () => ( field)) + )]), () => ( [])); + const search = new URLSearchParams([ + ...MODEL_EXPAND_KEYS.map((val) => ["expand", val]), + ...additionalExpandKeys.map((val) => ["expand", val]) + ]).toString(); + const response = await (params.fetch || fetch)( + `${_optionalChain([params, 'optionalAccess', _234 => _234.hubUrl]) || HUB_URL}/api/models/${params.name}/revision/${encodeURIComponent( + _nullishCoalesce(params.revision, () => ( "HEAD")) + )}?${search.toString()}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + const data = await response.json(); + const additional = {}; + if (_optionalChain([params, 'optionalAccess', _235 => _235.additionalFields])) { + for (const field of params.additionalFields) { + if (field === "filePaths") { + additional.filePaths = (_nullishCoalesce(data.siblings, () => ( []))).map((s) => s.rfilename); + } else if (field === "inferenceProviderMapping" && data.inferenceProviderMapping) { + additional.inferenceProviderMapping = normalizeInferenceProviderMapping(data.id, data.inferenceProviderMapping); + } else { + additional[field] = data[field]; + } + } + } + return { + ...additional, + id: data._id, + name: data.id, + private: data.private, + task: data.pipeline_tag, + downloads: data.downloads, + gated: data.gated, + likes: data.likes, + updatedAt: new Date(data.lastModified) + }; +} + +// src/lib/oauth-handle-redirect.ts +async function oauthHandleRedirect(opts) { + if (typeof window === "undefined" && !_optionalChain([opts, 'optionalAccess', _236 => _236.redirectedUrl])) { + throw new Error("oauthHandleRedirect is only available in the browser, unless you provide redirectedUrl"); + } + if (typeof localStorage === "undefined" && (!_optionalChain([opts, 'optionalAccess', _237 => _237.nonce]) || !_optionalChain([opts, 'optionalAccess', _238 => _238.codeVerifier]))) { + throw new Error( + "oauthHandleRedirect requires localStorage to be available, unless you provide nonce and codeVerifier" + ); + } + const redirectedUrl = _nullishCoalesce(_optionalChain([opts, 'optionalAccess', _239 => _239.redirectedUrl]), () => ( window.location.href)); + const searchParams = (() => { + try { + return new URL(redirectedUrl).searchParams; + } catch (err) { + throw new Error("Failed to parse redirected URL: " + redirectedUrl); + } + })(); + const [error, errorDescription] = [searchParams.get("error"), searchParams.get("error_description")]; + if (error) { + throw new Error(`${error}: ${errorDescription}`); + } + const code = searchParams.get("code"); + const nonce = _nullishCoalesce(_optionalChain([opts, 'optionalAccess', _240 => _240.nonce]), () => ( localStorage.getItem("huggingface.co:oauth:nonce"))); + if (!code) { + throw new Error("Missing oauth code from query parameters in redirected URL: " + redirectedUrl); + } + if (!nonce) { + throw new Error("Missing oauth nonce from localStorage"); + } + const codeVerifier = _nullishCoalesce(_optionalChain([opts, 'optionalAccess', _241 => _241.codeVerifier]), () => ( localStorage.getItem("huggingface.co:oauth:code_verifier"))); + if (!codeVerifier) { + throw new Error("Missing oauth code_verifier from localStorage"); + } + const state = searchParams.get("state"); + if (!state) { + throw new Error("Missing oauth state from query parameters in redirected URL"); + } + let parsedState; + try { + parsedState = JSON.parse(state); + } catch (e6) { + throw new Error("Invalid oauth state in redirected URL, unable to parse JSON: " + state); + } + if (parsedState.nonce !== nonce) { + throw new Error("Invalid oauth state in redirected URL"); + } + const hubUrl = _optionalChain([opts, 'optionalAccess', _242 => _242.hubUrl]) || HUB_URL; + const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; + const openidConfigRes = await fetch(openidConfigUrl, { + headers: { + Accept: "application/json" + } + }); + if (!openidConfigRes.ok) { + throw await createApiError(openidConfigRes); + } + const openidConfig = await openidConfigRes.json(); + const tokenRes = await fetch(openidConfig.token_endpoint, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded" + }, + body: new URLSearchParams({ + grant_type: "authorization_code", + code, + redirect_uri: parsedState.redirectUri, + code_verifier: codeVerifier + }).toString() + }); + if (!_optionalChain([opts, 'optionalAccess', _243 => _243.codeVerifier])) { + localStorage.removeItem("huggingface.co:oauth:code_verifier"); + } + if (!_optionalChain([opts, 'optionalAccess', _244 => _244.nonce])) { + localStorage.removeItem("huggingface.co:oauth:nonce"); + } + if (!tokenRes.ok) { + throw await createApiError(tokenRes); + } + const token = await tokenRes.json(); + const accessTokenExpiresAt = new Date(Date.now() + token.expires_in * 1e3); + const userInfoRes = await fetch(openidConfig.userinfo_endpoint, { + headers: { + Authorization: `Bearer ${token.access_token}` + } + }); + if (!userInfoRes.ok) { + throw await createApiError(userInfoRes); + } + const userInfo = await userInfoRes.json(); + return { + accessToken: token.access_token, + accessTokenExpiresAt, + userInfo, + state: parsedState.state, + scope: token.scope + }; +} +async function oauthHandleRedirectIfPresent(opts) { + if (typeof window === "undefined" && !_optionalChain([opts, 'optionalAccess', _245 => _245.redirectedUrl])) { + throw new Error("oauthHandleRedirect is only available in the browser, unless you provide redirectedUrl"); + } + if (typeof localStorage === "undefined" && (!_optionalChain([opts, 'optionalAccess', _246 => _246.nonce]) || !_optionalChain([opts, 'optionalAccess', _247 => _247.codeVerifier]))) { + throw new Error( + "oauthHandleRedirect requires localStorage to be available, unless you provide nonce and codeVerifier" + ); + } + const searchParams = new URLSearchParams(_nullishCoalesce(_optionalChain([opts, 'optionalAccess', _248 => _248.redirectedUrl]), () => ( window.location.search))); + if (searchParams.has("error")) { + return oauthHandleRedirect(opts); + } + if (searchParams.has("code")) { + if (!localStorage.getItem("huggingface.co:oauth:nonce")) { + console.warn( + "Missing oauth nonce from localStorage. This can happen when the user refreshes the page after logging in, without changing the URL." + ); + return false; + } + return oauthHandleRedirect(opts); + } + return false; +} + +// src/lib/oauth-login-url.ts +async function oauthLoginUrl(opts) { + if (typeof window === "undefined" && (!_optionalChain([opts, 'optionalAccess', _249 => _249.redirectUrl]) || !_optionalChain([opts, 'optionalAccess', _250 => _250.clientId]))) { + throw new Error("oauthLogin is only available in the browser, unless you provide clientId and redirectUrl"); + } + if (typeof localStorage === "undefined" && !_optionalChain([opts, 'optionalAccess', _251 => _251.localStorage])) { + throw new Error( + "oauthLogin requires localStorage to be available in the context, unless you provide a localStorage empty object as argument" + ); + } + const hubUrl = _optionalChain([opts, 'optionalAccess', _252 => _252.hubUrl]) || HUB_URL; + const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; + const openidConfigRes = await fetch(openidConfigUrl, { + headers: { + Accept: "application/json" + } + }); + if (!openidConfigRes.ok) { + throw await createApiError(openidConfigRes); + } + const opendidConfig = await openidConfigRes.json(); + const newNonce = globalThis.crypto.randomUUID(); + const newCodeVerifier = globalThis.crypto.randomUUID() + globalThis.crypto.randomUUID(); + if (_optionalChain([opts, 'optionalAccess', _253 => _253.localStorage])) { + if (opts.localStorage.codeVerifier !== void 0 && opts.localStorage.codeVerifier !== null) { + throw new Error( + "localStorage.codeVerifier must be initially set to null or undefined, and will be filled by oauthLoginUrl" + ); + } + if (opts.localStorage.nonce !== void 0 && opts.localStorage.nonce !== null) { + throw new Error( + "localStorage.nonce must be initially set to null or undefined, and will be filled by oauthLoginUrl" + ); + } + opts.localStorage.codeVerifier = newCodeVerifier; + opts.localStorage.nonce = newNonce; + } else { + localStorage.setItem("huggingface.co:oauth:nonce", newNonce); + localStorage.setItem("huggingface.co:oauth:code_verifier", newCodeVerifier); + } + const redirectUri = _optionalChain([opts, 'optionalAccess', _254 => _254.redirectUrl]) || (typeof window !== "undefined" ? window.location.href : void 0); + if (!redirectUri) { + throw new Error("Missing redirectUrl"); + } + const state = JSON.stringify({ + nonce: newNonce, + redirectUri, + state: _optionalChain([opts, 'optionalAccess', _255 => _255.state]) + }); + const variables = ( + // @ts-expect-error window.huggingface is defined inside static Spaces. + typeof window !== "undefined" ? _nullishCoalesce(_optionalChain([window, 'access', _256 => _256.huggingface, 'optionalAccess', _257 => _257.variables]), () => ( null)) : null + ); + const clientId = _optionalChain([opts, 'optionalAccess', _258 => _258.clientId]) || _optionalChain([variables, 'optionalAccess', _259 => _259.OAUTH_CLIENT_ID]); + if (!clientId) { + if (variables) { + throw new Error("Missing clientId, please add hf_oauth: true to the README.md's metadata in your static Space"); + } + throw new Error("Missing clientId"); + } + const challenge = base64FromBytes( + new Uint8Array(await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(newCodeVerifier))) + ).replace(/[+]/g, "-").replace(/[/]/g, "_").replace(/=/g, ""); + return `${opendidConfig.authorization_endpoint}?${new URLSearchParams({ + client_id: clientId, + scope: _optionalChain([opts, 'optionalAccess', _260 => _260.scopes]) || _optionalChain([variables, 'optionalAccess', _261 => _261.OAUTH_SCOPES]) || "openid profile", + response_type: "code", + redirect_uri: redirectUri, + state, + code_challenge: challenge, + code_challenge_method: "S256" + }).toString()}`; +} + +// src/utils/typedInclude.ts +function typedInclude(arr, v) { + return arr.includes(v); +} + +// src/utils/omit.ts +function omit(o, props) { + const propsArr = Array.isArray(props) ? props : [props]; + const letsKeep = Object.keys(o).filter((prop) => !typedInclude(propsArr, prop)); + return pick(o, letsKeep); +} + +// src/utils/typedEntries.ts +function typedEntries(obj) { + return Object.entries(obj); +} + +// src/lib/parse-safetensors-metadata.ts +var SAFETENSORS_FILE = "model.safetensors"; +var SAFETENSORS_INDEX_FILE = "model.safetensors.index.json"; +var RE_SAFETENSORS_FILE = /\.safetensors$/; +var RE_SAFETENSORS_INDEX_FILE = /\.safetensors\.index\.json$/; +var RE_SAFETENSORS_SHARD_FILE = /^(?(?.*?)[_-])(?\d{5,6})-of-(?\d{5,6})\.safetensors$/; +function parseSafetensorsShardFilename(filename) { + const match = RE_SAFETENSORS_SHARD_FILE.exec(filename); + if (match && match.groups) { + return { + prefix: match.groups["prefix"], + basePrefix: match.groups["basePrefix"], + shard: match.groups["shard"], + total: match.groups["total"] + }; + } + return null; +} +var PARALLEL_DOWNLOADS = 20; +var MAX_HEADER_LENGTH = 25e6; +var MAX_CONFIG_LENGTH = 1e7; +var MAX_SHARD_COUNT = 1e4; +var GPTQ_QWEIGHT_SUFFIX = "qweight"; +var GPTQ_AWQ_AUXILIARY_SUFFIXES = ["qzeros", "g_idx", "scales"]; +var SafetensorParseError = class extends Error { +}; +async function fetchModelConfig(params) { + try { + const configBlob = await downloadFile({ + ...params, + path: "config.json" + }); + if (!configBlob) { + return null; + } + const config = JSON.parse(await configBlob.slice(0, MAX_CONFIG_LENGTH).text()); + return config; + } catch (error) { + return null; + } +} +async function parseSingleFile(path, params) { + const blob = await downloadFile({ ...params, path }); + if (!blob) { + throw new SafetensorParseError(`Failed to parse file ${path}: failed to fetch safetensors header length.`); + } + const bufLengthOfHeaderLE = await blob.slice(0, 8).arrayBuffer(); + const lengthOfHeader = new DataView(bufLengthOfHeaderLE).getBigUint64(0, true); + if (lengthOfHeader <= 0) { + throw new SafetensorParseError(`Failed to parse file ${path}: safetensors header is malformed.`); + } + if (lengthOfHeader > MAX_HEADER_LENGTH) { + throw new SafetensorParseError( + `Failed to parse file ${path}: safetensor header is too big. Maximum supported size is ${MAX_HEADER_LENGTH} bytes.` + ); + } + try { + const header = JSON.parse(await blob.slice(8, 8 + Number(lengthOfHeader)).text()); + return header; + } catch (err) { + throw new SafetensorParseError(`Failed to parse file ${path}: safetensors header is not valid JSON.`); + } +} +async function parseShardedIndex(path, params) { + const indexBlob = await downloadFile({ + ...params, + path + }); + if (!indexBlob) { + throw new SafetensorParseError(`Failed to parse file ${path}: failed to fetch safetensors index.`); + } + try { + const index = JSON.parse(await indexBlob.slice(0, MAX_HEADER_LENGTH).text()); + return index; + } catch (error) { + throw new SafetensorParseError(`Failed to parse file ${path}: not a valid JSON.`); + } +} +async function fetchAllHeaders(path, index, params) { + const pathPrefix = path.slice(0, path.lastIndexOf("/") + 1); + const filenames = [...new Set(Object.values(index.weight_map))]; + if (filenames.length > MAX_SHARD_COUNT) { + throw new SafetensorParseError( + `Too many shard files (${filenames.length}). Maximum supported is ${MAX_SHARD_COUNT}.` + ); + } + for (const filename of filenames) { + if (filename.includes("..") || filename.startsWith("/") || filename.includes("://")) { + throw new SafetensorParseError(`Unsafe shard filename in weight_map: "${filename}"`); + } + } + const shardedMap = Object.fromEntries( + await promisesQueue( + filenames.map( + (filename) => async () => [filename, await parseSingleFile(pathPrefix + filename, params)] + ), + PARALLEL_DOWNLOADS + ) + ); + return shardedMap; +} +function parseTotalParameters(value) { + if (!value) { + return void 0; + } + if (typeof value === "number") { + return value; + } + return parseInt(value); +} +async function parseSafetensorsMetadata(params) { + const repoId = toRepoId(params.repo); + if (repoId.type !== "model") { + throw new TypeError("Only model repos should contain safetensors files."); + } + const modelConfig = params.computeParametersCount ? await fetchModelConfig(params) : null; + const quantConfig = _nullishCoalesce(_optionalChain([modelConfig, 'optionalAccess', _262 => _262.quantization_config]), () => ( _optionalChain([modelConfig, 'optionalAccess', _263 => _263.text_config, 'optionalAccess', _264 => _264.quantization_config]))); + if (params.path && RE_SAFETENSORS_FILE.test(params.path) || await fileExists({ ...params, path: SAFETENSORS_FILE })) { + const header = await parseSingleFile(_nullishCoalesce(params.path, () => ( SAFETENSORS_FILE)), params); + const paramStats = params.computeParametersCount ? { + parameterCount: computeNumOfParamsByDtypeSingleFile(header, quantConfig), + /// shortcut: get param count directly from metadata + parameterTotal: parseTotalParameters(_optionalChain([header, 'access', _265 => _265.__metadata__, 'optionalAccess', _266 => _266.total_parameters])) + } : void 0; + return { + sharded: false, + header, + ...paramStats, + filepaths: [_nullishCoalesce(params.path, () => ( SAFETENSORS_FILE))] + }; + } else if (params.path && RE_SAFETENSORS_INDEX_FILE.test(params.path) || await fileExists({ ...params, path: SAFETENSORS_INDEX_FILE })) { + const path = _nullishCoalesce(params.path, () => ( SAFETENSORS_INDEX_FILE)); + const index = await parseShardedIndex(path, params); + const shardedMap = await fetchAllHeaders(path, index, params); + const pathPrefix = path.slice(0, path.lastIndexOf("/") + 1); + const paramStats = params.computeParametersCount ? { + parameterCount: computeNumOfParamsByDtypeSharded(shardedMap, quantConfig), + /// shortcut: get param count directly from metadata + parameterTotal: parseTotalParameters(_optionalChain([index, 'access', _267 => _267.metadata, 'optionalAccess', _268 => _268.total_parameters])) + } : void 0; + return { + sharded: true, + index, + headers: shardedMap, + ...paramStats, + filepaths: [path, ...Object.keys(shardedMap).map((filename) => pathPrefix + filename)] + }; + } else { + throw new Error("model id does not seem to contain safetensors weights"); + } +} +function globMatch(pattern, str) { + const parts = pattern.split("*"); + if (parts.length === 1) { + return pattern === str; + } + if (!str.startsWith(parts[0])) { + return false; + } + let pos = parts[0].length; + const lastPart = parts[parts.length - 1]; + if (!str.endsWith(lastPart)) { + return false; + } + const end = str.length - lastPart.length; + for (let i = 1; i < parts.length - 1; i++) { + const idx = str.indexOf(parts[i], pos); + if (idx === -1 || idx + parts[i].length > end) { + return false; + } + pos = idx + parts[i].length; + } + return pos <= end; +} +function isQuantizedTensor(tensorName, quantConfig) { + if (!quantConfig) { + return false; + } + const patterns = quantConfig.modules_to_not_convert; + if (!_optionalChain([patterns, 'optionalAccess', _269 => _269.length])) { + return true; + } + return !patterns.some( + (pattern) => pattern.includes("*") ? globMatch(pattern, tensorName) : tensorName.includes(pattern) + ); +} +function matchesCompressedTensorsTarget(target, moduleName) { + if (!target.startsWith("re:")) { + return target === moduleName; + } + let pattern = target.slice(3); + if (pattern.startsWith("^")) { + pattern = pattern.slice(1); + } + if (pattern.endsWith("$")) { + pattern = pattern.slice(0, -1); + } else { + pattern += ".*"; + } + const glob = pattern.replaceAll(".*", "*").replaceAll("\\.", "."); + if (/[\\+?()[\]{}|^$]/.test(glob)) { + return false; + } + return globMatch(glob, moduleName); +} +function getQuantizationMultiplier(tensorName, dtype, quantConfig) { + if (!quantConfig || !isQuantizedTensor(tensorName, quantConfig)) { + return 1; + } + const quantMethod = _optionalChain([quantConfig, 'access', _270 => _270.quant_method, 'optionalAccess', _271 => _271.toLowerCase, 'call', _272 => _272()]); + switch (quantMethod) { + case "mxfp4": + if (dtype === "U8" && tensorName.includes("_blocks")) { + return 2; + } + return 1; + case "gptq": + case "awq": + if (getTensorSuffix(tensorName) === GPTQ_QWEIGHT_SUFFIX) { + const bits = quantConfig.bits && quantConfig.bits > 0 ? quantConfig.bits : 4; + return Math.max(1, Math.floor(32 / bits)); + } + if (quantConfig.bits === 4 && dtype === "U8") { + return 2; + } + if (quantConfig.bits === 2 && dtype === "U8") { + return 4; + } + return 1; + case "compressed-tensors": + if (dtype === "I32") { + const groups = Object.values(_nullishCoalesce(quantConfig.config_groups, () => ( {}))); + const suffixIndex = tensorName.lastIndexOf(".weight"); + const moduleName = suffixIndex === -1 ? tensorName : tensorName.slice(0, suffixIndex); + const group = groups.find( + (g) => _optionalChain([g, 'access', _273 => _273.targets, 'optionalAccess', _274 => _274.some, 'call', _275 => _275((target) => matchesCompressedTensorsTarget(target, moduleName))]) + ); + if (group) { + if ((_nullishCoalesce(group.format, () => ( quantConfig.format))) !== "pack-quantized") { + return 1; + } + const numBits = _nullishCoalesce(_optionalChain([group, 'access', _276 => _276.weights, 'optionalAccess', _277 => _277.num_bits]), () => ( 4)); + return Math.max(1, Math.floor(32 / numBits)); + } + if (quantConfig.format === "pack-quantized") { + const numBits = _nullishCoalesce(_optionalChain([groups, 'access', _278 => _278.find, 'call', _279 => _279((g) => _optionalChain([g, 'access', _280 => _280.weights, 'optionalAccess', _281 => _281.num_bits])), 'optionalAccess', _282 => _282.weights, 'optionalAccess', _283 => _283.num_bits]), () => ( 4)); + return Math.max(1, Math.floor(32 / numBits)); + } + } + return 1; + case "bitsandbytes": + if (quantConfig.load_in_4bit && dtype === "U8") { + return 2; + } + return 1; + default: + if (dtype === "U8" && (quantConfig.load_in_4bit || quantConfig.bits === 4)) { + return 2; + } + return 1; + } +} +function computeNumOfParamsByDtypeSingleFile(header, quantConfig) { + const counter = {}; + const tensors = omit(header, "__metadata__"); + for (const [tensorName, v] of typedEntries(tensors)) { + if (shouldSkipTensor(tensorName, quantConfig)) { + continue; + } + if (v.shape.length === 0) { + continue; + } + const elements = v.shape.reduce((a, b) => a * b); + if (!Number.isFinite(elements)) { + continue; + } + const multiplier = quantConfig ? getQuantizationMultiplier(tensorName, v.dtype, quantConfig) : 1; + if (multiplier === 0) { + continue; + } + counter[v.dtype] = (_nullishCoalesce(counter[v.dtype], () => ( 0))) + elements * multiplier; + } + return counter; +} +function computeNumOfParamsByDtypeSharded(shardedMap, quantConfig) { + const counter = {}; + for (const header of Object.values(shardedMap)) { + for (const [k, v] of typedEntries(computeNumOfParamsByDtypeSingleFile(header, quantConfig))) { + counter[k] = (_nullishCoalesce(counter[k], () => ( 0))) + (_nullishCoalesce(v, () => ( 0))); + } + } + return counter; +} +function getTensorSuffix(tensorName) { + const lastDotIndex = tensorName.lastIndexOf("."); + return lastDotIndex === -1 ? tensorName : tensorName.slice(lastDotIndex + 1); +} +function shouldSkipTensor(tensorName, quantConfig) { + if (!quantConfig) { + return false; + } + const quantMethod = _optionalChain([quantConfig, 'access', _284 => _284.quant_method, 'optionalAccess', _285 => _285.toLowerCase, 'call', _286 => _286()]); + if (quantMethod !== "gptq" && quantMethod !== "awq") { + return false; + } + if (!isQuantizedTensor(tensorName, quantConfig)) { + return false; + } + const suffix = getTensorSuffix(tensorName); + return suffix !== GPTQ_QWEIGHT_SUFFIX && GPTQ_AWQ_AUXILIARY_SUFFIXES.includes(suffix); +} + +// src/lib/repo-exists.ts +async function repoExists(params) { + const repoId = toRepoId(params.repo); + const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( + `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}?expand[]=likes`, + { + method: "GET", + headers: { + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + } + } + ); + if (res.status === 404 || res.status === 401) { + return false; + } + if (!res.ok) { + throw await createApiError(res); + } + return true; +} + +// src/lib/space-info.ts +async function spaceInfo(params) { + const accessToken = params && checkCredentials(params); + const search = new URLSearchParams([ + ...SPACE_EXPAND_KEYS.map((val) => ["expand", val]), + ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _287 => _287.additionalFields, 'optionalAccess', _288 => _288.map, 'call', _289 => _289((val) => ["expand", val])]), () => ( [])) + ]).toString(); + const response = await (params.fetch || fetch)( + `${_optionalChain([params, 'optionalAccess', _290 => _290.hubUrl]) || HUB_URL}/api/spaces/${params.name}/revision/${encodeURIComponent( + _nullishCoalesce(params.revision, () => ( "HEAD")) + )}?${search.toString()}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + const data = await response.json(); + return { + ..._optionalChain([params, 'optionalAccess', _291 => _291.additionalFields]) && pick(data, params.additionalFields), + id: data._id, + name: data.id, + sdk: data.sdk, + likes: data.likes, + private: data.private, + updatedAt: new Date(data.lastModified) + }; +} + +// src/lib/upload-file.ts +function uploadFile(params) { + const path = params.file instanceof URL ? _nullishCoalesce(params.file.pathname.split("/").at(-1), () => ( "file")) : "path" in params.file ? params.file.path : params.file.name; + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: [ + { + operation: "addOrUpdate", + path, + content: "content" in params.file ? params.file.content : params.file + } + ], + title: _nullishCoalesce(params.commitTitle, () => ( `Add ${path}`)), + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet + }); +} + +// src/lib/upload-files.ts +function uploadFiles(params) { + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: params.files.map((file) => ({ + operation: "addOrUpdate", + path: file instanceof URL ? _nullishCoalesce(file.pathname.split("/").at(-1), () => ( "file")) : "path" in file ? file.path : file.name, + content: "content" in file ? file.content : file + })), + title: _nullishCoalesce(params.commitTitle, () => ( `Add ${params.files.length} files`)), + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet + }); +} + +// src/lib/upload-files-with-progress.ts +var multipartUploadTracking = /* @__PURE__ */ new WeakMap(); +async function* uploadFilesWithProgress(params) { + return yield* commitIter({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: params.files.map((file) => ({ + operation: "addOrUpdate", + path: file instanceof URL ? _nullishCoalesce(file.pathname.split("/").at(-1), () => ( "file")) : "path" in file ? file.path : file.name, + content: "content" in file ? file.content : file + })), + title: _nullishCoalesce(params.commitTitle, () => ( `Add ${params.files.length} files`)), + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet, + fetch: async (input, init) => { + if (!init) { + return fetch(input); + } + if (!typedInclude(["PUT", "POST"], init.method) || !("progressHint" in init) || !init.progressHint || typeof XMLHttpRequest === "undefined" || typeof input !== "string" || !(init.body instanceof ArrayBuffer) && !(init.body instanceof Blob) && !(init.body instanceof File) && typeof init.body !== "string") { + return fetch(input, init); + } + const progressHint = init.progressHint; + const progressCallback = progressHint.progressCallback; + const xhr = new XMLHttpRequest(); + xhr.upload.addEventListener("progress", (event) => { + if (event.lengthComputable) { + if (progressHint.part !== void 0) { + let tracking = multipartUploadTracking.get(progressCallback); + if (!tracking) { + tracking = { numParts: progressHint.numParts, partsProgress: {} }; + multipartUploadTracking.set(progressCallback, tracking); + } + tracking.partsProgress[progressHint.part] = event.loaded / event.total; + let totalProgress = 0; + for (const partProgress of Object.values(tracking.partsProgress)) { + totalProgress += partProgress; + } + if (totalProgress === tracking.numParts) { + progressCallback(0.9999999999); + } else { + progressCallback(totalProgress / tracking.numParts); + } + } else { + if (event.loaded === event.total) { + progressCallback(0.9999999999); + } else { + progressCallback(event.loaded / event.total); + } + } + } + }); + xhr.open(init.method, input, true); + if (init.headers) { + const headers = new Headers(init.headers); + headers.forEach((value, key) => { + xhr.setRequestHeader(key, value); + }); + } + _optionalChain([init, 'access', _292 => _292.signal, 'optionalAccess', _293 => _293.throwIfAborted, 'call', _294 => _294()]); + xhr.send(init.body); + return new Promise((resolve2, reject) => { + xhr.addEventListener("load", () => { + resolve2( + new Response(xhr.responseText, { + status: xhr.status, + statusText: xhr.statusText, + headers: Object.fromEntries( + xhr.getAllResponseHeaders().trim().split("\n").map((header) => [ + header.slice(0, header.indexOf(":")), + header.slice(header.indexOf(":") + 1).trim() + ]) + ) + }) + ); + }); + xhr.addEventListener("error", () => { + reject(new Error(xhr.statusText)); + }); + if (init.signal) { + init.signal.addEventListener("abort", () => { + xhr.abort(); + try { + _optionalChain([init, 'access', _295 => _295.signal, 'optionalAccess', _296 => _296.throwIfAborted, 'call', _297 => _297()]); + } catch (err) { + reject(err); + } + }); + } + }); + } + }); +} + +// src/lib/who-am-i.ts +async function whoAmI(params) { + const accessToken = checkCredentials(params); + const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/whoami-v2`, { + headers: { + Authorization: `Bearer ${accessToken}` + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const response = await res.json(); + if (typeof _optionalChain([response, 'access', _298 => _298.auth, 'access', _299 => _299.accessToken, 'optionalAccess', _300 => _300.createdAt]) === "string") { + response.auth.accessToken.createdAt = new Date(response.auth.accessToken.createdAt); + } + return response; +} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +exports.DATASET_EXPANDABLE_KEYS = DATASET_EXPANDABLE_KEYS; exports.DATASET_EXPAND_KEYS = DATASET_EXPAND_KEYS; exports.HUB_URL = HUB_URL; exports.HubApiError = HubApiError; exports.InvalidApiResponseFormatError = InvalidApiResponseFormatError; exports.MODEL_DERIVED_FIELD_TO_API_KEY = MODEL_DERIVED_FIELD_TO_API_KEY; exports.MODEL_EXPANDABLE_KEYS = MODEL_EXPANDABLE_KEYS; exports.MODEL_EXPAND_KEYS = MODEL_EXPAND_KEYS; exports.RE_SAFETENSORS_FILE = RE_SAFETENSORS_FILE; exports.RE_SAFETENSORS_INDEX_FILE = RE_SAFETENSORS_INDEX_FILE; exports.RE_SAFETENSORS_SHARD_FILE = RE_SAFETENSORS_SHARD_FILE; exports.SAFETENSORS_FILE = SAFETENSORS_FILE; exports.SAFETENSORS_INDEX_FILE = SAFETENSORS_INDEX_FILE; exports.SPACE_EXPANDABLE_KEYS = SPACE_EXPANDABLE_KEYS; exports.SPACE_EXPAND_KEYS = SPACE_EXPAND_KEYS; exports.__internal_XetBlob = XetBlob; exports.__internal_sha256 = sha256; exports.cancelJob = cancelJob; exports.checkRepoAccess = checkRepoAccess; exports.commit = commit; exports.commitIter = commitIter; exports.commitIterBucket = commitIterBucket; exports.copyFile = copyFile; exports.copyFileIter = copyFileIter; exports.copyFiles = copyFiles; exports.copyFilesIter = copyFilesIter; exports.copyFolder = copyFolder; exports.copyFolderIter = copyFolderIter; exports.countCommits = countCommits; exports.createBranch = createBranch; exports.createCollection = createCollection; exports.createRepo = createRepo; exports.createScheduledJob = createScheduledJob; exports.datasetInfo = datasetInfo; exports.deleteBranch = deleteBranch; exports.deleteCollection = deleteCollection; exports.deleteFile = deleteFile; exports.deleteFiles = deleteFiles; exports.deleteRepo = deleteRepo; exports.deleteScheduledJob = deleteScheduledJob; exports.downloadFile = downloadFile; exports.duplicateJob = duplicateJob; exports.fileDownloadInfo = fileDownloadInfo; exports.fileExists = fileExists; exports.getJob = getJob; exports.getScheduledJob = getScheduledJob; exports.globMatch = globMatch; exports.isQuantizedTensor = isQuantizedTensor; exports.listCollections = listCollections; exports.listCommits = listCommits; exports.listDatasets = listDatasets; exports.listFiles = listFiles; exports.listJobHardware = listJobHardware; exports.listJobs = listJobs; exports.listModels = listModels; exports.listScheduledJobs = listScheduledJobs; exports.listSpaces = listSpaces; exports.matchesCompressedTensorsTarget = matchesCompressedTensorsTarget; exports.modelInfo = modelInfo; exports.oauthHandleRedirect = oauthHandleRedirect; exports.oauthHandleRedirectIfPresent = oauthHandleRedirectIfPresent; exports.oauthLoginUrl = oauthLoginUrl; exports.parseSafetensorsMetadata = parseSafetensorsMetadata; exports.parseSafetensorsShardFilename = parseSafetensorsShardFilename; exports.pathsInfo = pathsInfo; exports.relativeUnderFolder = relativeUnderFolder; exports.repoExists = repoExists; exports.resumeScheduledJob = resumeScheduledJob; exports.runJob = runJob; exports.runScheduledJob = runScheduledJob; exports.spaceInfo = spaceInfo; exports.streamJobEvents = streamJobEvents; exports.streamJobLogs = streamJobLogs; exports.streamJobMetrics = streamJobMetrics; exports.suspendScheduledJob = suspendScheduledJob; exports.uploadFile = uploadFile; exports.uploadFiles = uploadFiles; exports.uploadFilesWithProgress = uploadFilesWithProgress; exports.whoAmI = whoAmI; diff --git a/node_modules/@huggingface/hub/dist/browser/index.mjs b/node_modules/@huggingface/hub/dist/browser/index.mjs new file mode 100644 index 0000000000000000000000000000000000000000..a0b1aa67fbb3e7970c0dc7ca7a2a8ef401bde058 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/browser/index.mjs @@ -0,0 +1,6001 @@ +// src/consts.ts +var HUB_URL = "https://huggingface.co"; + +// src/error.ts +async function createApiError(response, opts) { + const error = new HubApiError(response.url, response.status, response.headers.get("X-Request-Id") ?? opts?.requestId); + error.message = `Api error with status ${error.statusCode}${opts?.message ? `. ${opts.message}` : ""}`; + const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : void 0].filter(Boolean).join(". "); + if (response.headers.get("Content-Type")?.startsWith("application/json")) { + const json = await response.json(); + error.message = json.error || json.message || error.message; + if (json.error_description) { + error.message = error.message ? error.message + `: ${json.error_description}` : json.error_description; + } + error.data = json; + } else { + error.data = { message: await response.text() }; + } + error.message += `. ${trailer}`; + throw error; +} +var HubApiError = class extends Error { + statusCode; + url; + requestId; + data; + constructor(url, statusCode, requestId, message) { + super(message); + this.statusCode = statusCode; + this.requestId = requestId; + this.url = url; + } +}; +var InvalidApiResponseFormatError = class extends Error { +}; + +// src/utils/checkCredentials.ts +function checkAccessToken(accessToken) { + if (!accessToken.startsWith("hf_")) { + throw new TypeError("Your access token must start with 'hf_'"); + } +} +function checkCredentials(params) { + if (params.accessToken) { + checkAccessToken(params.accessToken); + return params.accessToken; + } + if (params.credentials?.accessToken) { + checkAccessToken(params.credentials.accessToken); + return params.credentials.accessToken; + } +} + +// src/utils/toRepoId.ts +function toRepoId(repo) { + if (typeof repo !== "string") { + return repo; + } + if (repo.startsWith("model/") || repo.startsWith("models/")) { + throw new TypeError( + "A repo designation for a model should not start with 'models/', directly specify the model namespace / name" + ); + } + if (repo.startsWith("space/")) { + throw new TypeError("Spaces should start with 'spaces/', plural, not 'space/'"); + } + if (repo.startsWith("dataset/")) { + throw new TypeError("Datasets should start with 'datasets/', plural, not 'dataset/'"); + } + if (repo.startsWith("bucket/")) { + throw new TypeError("Buckets should start with 'buckets/', plural, not 'bucket/'"); + } + if (repo.startsWith("kernel/")) { + throw new TypeError("Kernels should start with 'kernels/', plural, not 'kernel/'"); + } + const slashes = repo.split("/").length - 1; + if (repo.startsWith("spaces/")) { + if (slashes !== 2) { + throw new TypeError("Space Id must include namespace and name of the space"); + } + return { + type: "space", + name: repo.slice("spaces/".length) + }; + } + if (repo.startsWith("datasets/")) { + if (slashes > 2) { + throw new TypeError("Too many slashes in repo designation: " + repo); + } + return { + type: "dataset", + name: repo.slice("datasets/".length) + }; + } + if (repo.startsWith("buckets/")) { + if (slashes !== 2) { + throw new TypeError("Bucket Id must include namespace and name of the bucket"); + } + return { + type: "bucket", + name: repo.slice("buckets/".length) + }; + } + if (repo.startsWith("kernels/")) { + if (slashes !== 2) { + throw new TypeError("Kernel Id must include namespace and name of the kernel"); + } + return { + type: "kernel", + name: repo.slice("kernels/".length) + }; + } + if (slashes > 1) { + throw new TypeError("Too many slashes in repo designation: " + repo); + } + return { + type: "model", + name: repo + }; +} + +// src/lib/check-repo-access.ts +async function checkRepoAccess(params) { + const accessToken = params && checkCredentials(params); + const repoId = toRepoId(params.repo); + const response = await (params.fetch || fetch)(`${params?.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}`, { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + }); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/utils/range.ts +function range(n, b) { + return b ? Array(b - n).fill(0).map((_, i) => n + i) : Array(n).fill(0).map((_, i) => i); +} + +// src/utils/chunk.ts +function chunk(arr, chunkSize) { + if (isNaN(chunkSize) || chunkSize < 1) { + throw new RangeError("Invalid chunk size: " + chunkSize); + } + if (!arr.length) { + return []; + } + if (arr.length <= chunkSize) { + return [arr]; + } + return range(Math.ceil(arr.length / chunkSize)).map((i) => { + return arr.slice(i * chunkSize, (i + 1) * chunkSize); + }); +} + +// src/utils/promisesQueue.ts +async function promisesQueue(factories, concurrency) { + const results = []; + const executing = /* @__PURE__ */ new Set(); + let index = 0; + for (const factory of factories) { + const closureIndex = index++; + const e = factory().then((r) => { + results[closureIndex] = r; + executing.delete(e); + }); + executing.add(e); + if (executing.size >= concurrency) { + await Promise.race(executing); + } + } + await Promise.all(executing); + return results; +} + +// src/utils/promisesQueueStreaming.ts +async function promisesQueueStreaming(factories, concurrency) { + const executing = []; + for await (const factory of factories) { + const e = factory().then(() => { + executing.splice(executing.indexOf(e), 1); + }); + executing.push(e); + if (executing.length >= concurrency) { + await Promise.race(executing); + } + } + await Promise.all(executing); +} + +// src/utils/eventToGenerator.ts +async function* eventToGenerator(cb) { + const promises = []; + function addPromise() { + let resolve2; + let reject; + const p = new Promise((res, rej) => { + resolve2 = res; + reject = rej; + }); + promises.push({ p, resolve: resolve2, reject }); + } + addPromise(); + const callbackRes = Promise.resolve().then( + () => cb( + (y) => { + addPromise(); + promises.at(-2)?.resolve({ done: false, value: y }); + }, + (r) => { + addPromise(); + promises.at(-2)?.resolve({ done: true, value: r }); + }, + (err) => promises.shift()?.reject(err) + ) + ).catch((err) => promises.shift()?.reject(err)); + while (1) { + const p = promises[0]; + if (!p) { + throw new Error("Logic error in eventGenerator, promises should never be empty"); + } + const result = await p.p; + promises.shift(); + if (result.done) { + await callbackRes; + return result.value; + } + yield result.value; + } + throw new Error("Unreachable"); +} + +// src/utils/hexFromBytes.ts +function hexFromBytes(arr) { + if (globalThis.Buffer) { + return globalThis.Buffer.from(arr).toString("hex"); + } else { + const bin = []; + arr.forEach((byte) => { + bin.push(byte.toString(16).padStart(2, "0")); + }); + return bin.join(""); + } +} + +// src/utils/isBackend.ts +var isBrowser = typeof window !== "undefined" && typeof window.document !== "undefined"; +var isWebWorker = typeof self === "object" && self.constructor && self.constructor.name === "DedicatedWorkerGlobalScope"; +var isBackend = !isBrowser && !isWebWorker; + +// src/utils/isFrontend.ts +var isFrontend = !isBackend; + +// src/utils/sha256.ts +async function getWebWorkerCode() { + const sha256Module = await import("./sha256-wrapper-DYTB3MXW.mjs"); + return URL.createObjectURL(new Blob([sha256Module.createSHA256WorkerCode()])); +} +var pendingWorkers = []; +var runningWorkers = /* @__PURE__ */ new Set(); +var resolve; +var waitPromise = new Promise((r) => { + resolve = r; +}); +async function getWorker(poolSize) { + { + const worker2 = pendingWorkers.pop(); + if (worker2) { + runningWorkers.add(worker2); + return worker2; + } + } + if (!poolSize) { + const worker2 = new Worker(await getWebWorkerCode()); + runningWorkers.add(worker2); + return worker2; + } + if (poolSize <= 0) { + throw new TypeError("Invalid webworker pool size: " + poolSize); + } + while (runningWorkers.size >= poolSize) { + await waitPromise; + } + const worker = new Worker(await getWebWorkerCode()); + runningWorkers.add(worker); + return worker; +} +async function freeWorker(worker, poolSize) { + if (!poolSize) { + return destroyWorker(worker); + } + runningWorkers.delete(worker); + pendingWorkers.push(worker); + const r = resolve; + waitPromise = new Promise((r2) => { + resolve = r2; + }); + r(); +} +function destroyWorker(worker) { + runningWorkers.delete(worker); + worker.terminate(); + const r = resolve; + waitPromise = new Promise((r2) => { + resolve = r2; + }); + r(); +} +async function* sha256(buffer, opts) { + yield 0; + const maxCryptoSize = typeof opts?.useWebWorker === "object" && opts?.useWebWorker.minSize !== void 0 ? opts.useWebWorker.minSize : 1e7; + if (buffer.size < maxCryptoSize && globalThis.crypto?.subtle) { + const res = hexFromBytes( + new Uint8Array( + await globalThis.crypto.subtle.digest("SHA-256", buffer instanceof Blob ? await buffer.arrayBuffer() : buffer) + ) + ); + yield 1; + return res; + } + if (isFrontend) { + if (opts?.useWebWorker) { + try { + const poolSize = typeof opts?.useWebWorker === "object" ? opts.useWebWorker.poolSize : void 0; + const worker = await getWorker(poolSize); + let messageHandler; + let errorHandler; + const cleanup = () => { + worker.removeEventListener("message", messageHandler); + worker.removeEventListener("error", errorHandler); + }; + return yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + messageHandler = (event) => { + if (event.data.sha256) { + cleanup(); + freeWorker(worker, poolSize); + returnCallback(event.data.sha256); + } else if (event.data.progress) { + yieldCallback(event.data.progress); + try { + opts.abortSignal?.throwIfAborted(); + } catch (err) { + cleanup(); + destroyWorker(worker); + rejectCallback(err); + } + } else { + cleanup(); + destroyWorker(worker); + rejectCallback(event); + } + }; + errorHandler = (event) => { + cleanup(); + destroyWorker(worker); + rejectCallback(event.error); + }; + if (opts?.abortSignal) { + try { + opts.abortSignal?.throwIfAborted(); + } catch (err) { + cleanup(); + destroyWorker(worker); + rejectCallback(opts.abortSignal.reason ?? new DOMException("Aborted", "AbortError")); + return; + } + const abortListener = () => { + cleanup(); + destroyWorker(worker); + rejectCallback(opts.abortSignal?.reason ?? new DOMException("Aborted", "AbortError")); + opts.abortSignal?.removeEventListener("abort", abortListener); + }; + opts.abortSignal.addEventListener("abort", abortListener); + } + worker.addEventListener("message", messageHandler); + worker.addEventListener("error", errorHandler); + worker.postMessage({ file: buffer }); + }); + } catch (err) { + console.warn("Failed to use web worker for sha256", err); + } + } + if (!wasmModule) { + wasmModule = await import("./sha256-wrapper-DYTB3MXW.mjs"); + } + const sha2562 = await wasmModule.createSHA256(); + sha2562.init(); + const reader = buffer.stream().getReader(); + const total = buffer.size; + let bytesDone = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + sha2562.update(value); + bytesDone += value.length; + yield bytesDone / total; + opts?.abortSignal?.throwIfAborted(); + } + return sha2562.digest("hex"); + } + if (!cryptoModule) { + cryptoModule = await import("./sha256-node-TNZ2WHTI.mjs"); + } + return yield* cryptoModule.sha256Node(buffer, { abortSignal: opts?.abortSignal }); +} +var cryptoModule; +var wasmModule; + +// src/utils/WebBlob.ts +var WebBlob = class extends Blob { + static async create(url, opts) { + const customFetch = opts?.fetch ?? fetch; + const probe = await customFetch(url, { + headers: { + Range: "bytes=0-0", + ...opts?.accessToken && { Authorization: `Bearer ${opts.accessToken}` } + } + }); + if (!probe.ok) { + throw await createApiError(probe); + } + const contentType = probe.headers.get("content-type") || ""; + if (probe.status === 206) { + const totalSize = Number(probe.headers.get("content-range")?.split("/").pop()); + await probe.body?.cancel(); + if (Number.isFinite(totalSize) && totalSize >= (opts?.cacheBelow ?? 1e6)) { + return new WebBlob(url, 0, totalSize, contentType, true, customFetch, opts?.accessToken); + } + const full = await customFetch(url, { + ...opts?.accessToken && { headers: { Authorization: `Bearer ${opts.accessToken}` } } + }); + if (!full.ok) { + throw await createApiError(full); + } + return full.blob(); + } + return probe.blob(); + } + url; + start; + end; + contentType; + full; + fetch; + accessToken; + constructor(url, start, end, contentType, full, customFetch, accessToken) { + super([]); + this.url = url; + this.start = start; + this.end = end; + this.contentType = contentType; + this.full = full; + this.fetch = customFetch; + this.accessToken = accessToken; + } + get size() { + return this.end - this.start; + } + get type() { + return this.contentType; + } + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on WebBlob.slice"); + } + const slice = new WebBlob( + this.url, + this.start + start, + Math.min(this.start + end, this.end), + this.contentType, + start === 0 && end === this.size ? this.full : false, + this.fetch, + this.accessToken + ); + return slice; + } + async arrayBuffer() { + const result = await this.fetchRange(); + return result.arrayBuffer(); + } + async text() { + const result = await this.fetchRange(); + return result.text(); + } + stream() { + const stream = new TransformStream(); + this.fetchRange().then((response) => response.body?.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message)); + return stream.readable; + } + fetchRange() { + const fetch2 = this.fetch; + if (this.full) { + return fetch2(this.url, { + ...this.accessToken && { + headers: { + Authorization: `Bearer ${this.accessToken}` + } + } + }).then((resp) => resp.ok ? resp : createApiError(resp)); + } + return fetch2(this.url, { + headers: { + Range: `bytes=${this.start}-${this.end - 1}`, + ...this.accessToken && { Authorization: `Bearer ${this.accessToken}` } + } + }).then((resp) => resp.ok ? resp : createApiError(resp)); + } +}; + +// src/utils/base64FromBytes.ts +function base64FromBytes(arr) { + if (globalThis.Buffer) { + return globalThis.Buffer.from(arr).toString("base64"); + } else { + const bin = []; + arr.forEach((byte) => { + bin.push(String.fromCharCode(byte)); + }); + return globalThis.btoa(bin.join("")); + } +} + +// src/utils/createBlobs.ts +async function createBlobs(url, destPath, opts) { + if (url.protocol === "http:" || url.protocol === "https:") { + const blob = await WebBlob.create(url, { fetch: opts?.fetch, accessToken: opts?.accessToken }); + return [{ path: destPath, blob }]; + } + if (isFrontend) { + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); + } + if (url.protocol === "file:") { + const { FileBlob } = await import("./FileBlob-7MRLQ6TG.mjs"); + const { subPaths } = await import("./sub-paths-F6TP7MGR.mjs"); + const paths = await subPaths(url, opts?.maxFolderDepth); + if (paths.length === 1 && paths[0].relativePath === ".") { + const blob = await FileBlob.create(url); + return [{ path: destPath, blob }]; + } + return Promise.all( + paths.map(async (path) => ({ + path: `${destPath}/${path.relativePath}`.replace(/\/[.]$/, "").replaceAll("//", "/").replace(/^[.]?\//, ""), + blob: await FileBlob.create(new URL(path.path)) + })) + ); + } + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); +} + +// src/utils/combineUint8Arrays.ts +function combineUint8Arrays(a, b) { + const aLength = a.length; + const combinedBytes = new Uint8Array(aLength + b.length); + combinedBytes.set(a); + combinedBytes.set(b, aLength); + return combinedBytes; +} + +// src/vendor/lz4js/util.ts +function hashU32(a) { + a = a | 0; + a = a + 2127912214 + (a << 12) | 0; + a = a ^ -949894596 ^ a >>> 19; + a = a + 374761393 + (a << 5) | 0; + a = a + -744332180 ^ a << 9; + a = a + -42973499 + (a << 3) | 0; + return a ^ -1252372727 ^ a >>> 16 | 0; +} +function readU64(b, n) { + let x = 0; + x |= b[n++] << 0; + x |= b[n++] << 8; + x |= b[n++] << 16; + x |= b[n++] << 24; + x |= b[n++] << 32; + x |= b[n++] << 40; + x |= b[n++] << 48; + x |= b[n++] << 56; + return x; +} +function readU32(b, n) { + let x = 0; + x |= b[n++] << 0; + x |= b[n++] << 8; + x |= b[n++] << 16; + x |= b[n++] << 24; + return x; +} +function writeU32(b, n, x) { + b[n++] = x >> 0 & 255; + b[n++] = x >> 8 & 255; + b[n++] = x >> 16 & 255; + b[n++] = x >> 24 & 255; +} +function imul(a, b) { + const ah = a >>> 16; + const al = a & 65535; + const bh = b >>> 16; + const bl = b & 65535; + return al * bl + (ah * bl + al * bh << 16) | 0; +} + +// src/vendor/lz4js/xxh32.ts +var prime1 = 2654435761; +var prime2 = 2246822519; +var prime3 = 3266489917; +var prime4 = 668265263; +var prime5 = 374761393; +function rotl32(x, r) { + x = x | 0; + r = r | 0; + return x >>> (32 - r | 0) | x << r | 0; +} +function rotmul32(h, r, m) { + h = h | 0; + r = r | 0; + m = m | 0; + return imul(h >>> (32 - r | 0) | h << r, m) | 0; +} +function shiftxor32(h, s) { + h = h | 0; + s = s | 0; + return h >>> s ^ h | 0; +} +function xxhapply(h, src, m0, s, m1) { + return rotmul32(imul(src, m0) + h, s, m1); +} +function xxh1(h, src, index) { + return rotmul32(h + imul(src[index], prime5), 11, prime1); +} +function xxh4(h, src, index) { + return xxhapply(h, readU32(src, index), prime3, 17, prime4); +} +function xxh16(h, src, index) { + return [ + xxhapply(h[0], readU32(src, index + 0), prime2, 13, prime1), + xxhapply(h[1], readU32(src, index + 4), prime2, 13, prime1), + xxhapply(h[2], readU32(src, index + 8), prime2, 13, prime1), + xxhapply(h[3], readU32(src, index + 12), prime2, 13, prime1) + ]; +} +function xxh32(seed, src, index, len) { + let h; + const l = len; + if (len >= 16) { + h = [seed + prime1 + prime2, seed + prime2, seed, seed - prime1]; + while (len >= 16) { + h = xxh16(h, src, index); + index += 16; + len -= 16; + } + h = rotl32(h[0], 1) + rotl32(h[1], 7) + rotl32(h[2], 12) + rotl32(h[3], 18) + l; + } else { + h = seed + prime5 + len >>> 0; + } + while (len >= 4) { + h = xxh4(h, src, index); + index += 4; + len -= 4; + } + while (len > 0) { + h = xxh1(h, src, index); + index++; + len--; + } + h = shiftxor32(imul(shiftxor32(imul(shiftxor32(h, 15), prime2), 13), prime3), 16); + return h >>> 0; +} +var hash = xxh32; + +// src/vendor/lz4js/index.ts +var minMatch = 4; +var matchSearchLimit = 12; +var minTrailingLitterals = 5; +var skipTrigger = 6; +var hashSize = 1 << 16; +var mlBits = 4; +var mlMask = (1 << mlBits) - 1; +var runBits = 4; +var runMask = (1 << runBits) - 1; +var blockBuf = makeBuffer(5 << 20); +var hashTable = makeHashTable(); +var magicNum = 407708164; +var fdContentChksum = 4; +var fdContentSize = 8; +var fdBlockChksum = 16; +var fdVersion = 64; +var fdVersionMask = 192; +var bsUncompressed = 2147483648; +var bsDefault = 7; +var bsShift = 4; +var bsMask = 7; +var bsMap = { + 4: 65536, + 5: 262144, + 6: 1048576, + 7: 4194304 +}; +function makeHashTable() { + try { + return new Uint32Array(hashSize); + } catch (error) { + const hashTable2 = new Array(hashSize); + for (let i = 0; i < hashSize; i++) { + hashTable2[i] = 0; + } + return hashTable2; + } +} +function clearHashTable(table) { + for (let i = 0; i < hashSize; i++) { + table[i] = 0; + } +} +function makeBuffer(size) { + return new Uint8Array(size); +} +function sliceArray(array, start, end) { + return array.slice(start, end); +} +function compressBound(n) { + return n + n / 255 + 16 | 0; +} +function decompressBound(src) { + let sIndex = 0; + if (readU32(src, sIndex) !== magicNum) { + throw new Error("invalid magic number"); + } + sIndex += 4; + const descriptor = src[sIndex++]; + if ((descriptor & fdVersionMask) !== fdVersion) { + throw new Error("incompatible descriptor version " + (descriptor & fdVersionMask)); + } + const useBlockSum = (descriptor & fdBlockChksum) !== 0; + const useContentSize = (descriptor & fdContentSize) !== 0; + const bsIdx = src[sIndex++] >> bsShift & bsMask; + if (bsMap[bsIdx] === void 0) { + throw new Error("invalid block size " + bsIdx); + } + const maxBlockSize = bsMap[bsIdx]; + if (useContentSize) { + return readU64(src, sIndex); + } + sIndex++; + let maxSize = 0; + while (true) { + let blockSize = readU32(src, sIndex); + sIndex += 4; + if (blockSize & bsUncompressed) { + blockSize &= ~bsUncompressed; + maxSize += blockSize; + } else if (blockSize > 0) { + maxSize += maxBlockSize; + } + if (blockSize === 0) { + return maxSize; + } + if (useBlockSum) { + sIndex += 4; + } + sIndex += blockSize; + } +} +function decompressBlock(src, dst, sIndex, sLength, dIndex) { + let mLength, mOffset, sEnd, n, i; + const hasCopyWithin = dst.copyWithin !== void 0 && dst.fill !== void 0; + sEnd = sIndex + sLength; + while (sIndex < sEnd) { + const token = src[sIndex++]; + let literalCount = token >> 4; + if (literalCount > 0) { + if (literalCount === 15) { + while (true) { + literalCount += src[sIndex]; + if (src[sIndex++] !== 255) { + break; + } + } + } + for (n = sIndex + literalCount; sIndex < n; ) { + dst[dIndex++] = src[sIndex++]; + } + } + if (sIndex >= sEnd) { + break; + } + mLength = token & 15; + mOffset = src[sIndex++] | src[sIndex++] << 8; + if (mLength === 15) { + while (true) { + mLength += src[sIndex]; + if (src[sIndex++] !== 255) { + break; + } + } + } + mLength += minMatch; + if (hasCopyWithin && mOffset === 1) { + dst.fill(dst[dIndex - 1] | 0, dIndex, dIndex + mLength); + dIndex += mLength; + } else if (hasCopyWithin && mOffset > mLength && mLength > 31) { + dst.copyWithin(dIndex, dIndex - mOffset, dIndex - mOffset + mLength); + dIndex += mLength; + } else { + for (i = dIndex - mOffset, n = i + mLength; i < n; ) { + dst[dIndex++] = dst[i++] | 0; + } + } + } + return dIndex; +} +function compressBlock(src, dst, sIndex, sLength, hashTable2) { + let mIndex, mAnchor, mLength, mOffset, mStep; + let literalCount, dIndex, sEnd, n; + dIndex = 0; + sEnd = sLength + sIndex; + mAnchor = sIndex; + let searchMatchCount = (1 << skipTrigger) + 3; + while (sIndex <= sEnd - matchSearchLimit) { + const seq = readU32(src, sIndex); + let hash2 = hashU32(seq) >>> 0; + hash2 = (hash2 >> 16 ^ hash2) >>> 0 & 65535; + mIndex = hashTable2[hash2] - 1; + hashTable2[hash2] = sIndex + 1; + if (mIndex < 0 || sIndex - mIndex >>> 16 > 0 || readU32(src, mIndex) !== seq) { + mStep = searchMatchCount++ >> skipTrigger; + sIndex += mStep; + continue; + } + searchMatchCount = (1 << skipTrigger) + 3; + literalCount = sIndex - mAnchor; + mOffset = sIndex - mIndex; + sIndex += minMatch; + mIndex += minMatch; + mLength = sIndex; + while (sIndex < sEnd - minTrailingLitterals && src[sIndex] === src[mIndex]) { + sIndex++; + mIndex++; + } + mLength = sIndex - mLength; + const token = mLength < mlMask ? mLength : mlMask; + if (literalCount >= runMask) { + dst[dIndex++] = (runMask << mlBits) + token; + for (n = literalCount - runMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } else { + dst[dIndex++] = (literalCount << mlBits) + token; + } + for (let i = 0; i < literalCount; i++) { + dst[dIndex++] = src[mAnchor + i]; + } + dst[dIndex++] = mOffset; + dst[dIndex++] = mOffset >> 8; + if (mLength >= mlMask) { + for (n = mLength - mlMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } + mAnchor = sIndex; + } + if (mAnchor === 0) { + return 0; + } + literalCount = sEnd - mAnchor; + if (literalCount >= runMask) { + dst[dIndex++] = runMask << mlBits; + for (n = literalCount - runMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } else { + dst[dIndex++] = literalCount << mlBits; + } + sIndex = mAnchor; + while (sIndex < sEnd) { + dst[dIndex++] = src[sIndex++]; + } + return dIndex; +} +function decompressFrame(src, dst) { + let useBlockSum, useContentSum, useContentSize, descriptor; + let sIndex = 0; + let dIndex = 0; + if (readU32(src, sIndex) !== magicNum) { + throw new Error("invalid magic number"); + } + sIndex += 4; + descriptor = src[sIndex++]; + if ((descriptor & fdVersionMask) !== fdVersion) { + throw new Error("incompatible descriptor version"); + } + useBlockSum = (descriptor & fdBlockChksum) !== 0; + useContentSum = (descriptor & fdContentChksum) !== 0; + useContentSize = (descriptor & fdContentSize) !== 0; + const bsIdx = src[sIndex++] >> bsShift & bsMask; + if (bsMap[bsIdx] === void 0) { + throw new Error("invalid block size"); + } + if (useContentSize) { + sIndex += 8; + } + sIndex++; + while (true) { + var compSize; + compSize = readU32(src, sIndex); + sIndex += 4; + if (compSize === 0) { + break; + } + if (useBlockSum) { + sIndex += 4; + } + if ((compSize & bsUncompressed) !== 0) { + compSize &= ~bsUncompressed; + for (let j = 0; j < compSize; j++) { + dst[dIndex++] = src[sIndex++]; + } + } else { + dIndex = decompressBlock(src, dst, sIndex, compSize, dIndex); + sIndex += compSize; + } + } + if (useContentSum) { + sIndex += 4; + } + return dIndex; +} +function compressFrame(src, dst) { + let dIndex = 0; + writeU32(dst, dIndex, magicNum); + dIndex += 4; + dst[dIndex++] = fdVersion; + dst[dIndex++] = bsDefault << bsShift; + dst[dIndex] = hash(0, dst, 4, dIndex - 4) >> 8; + dIndex++; + const maxBlockSize = bsMap[bsDefault]; + let remaining = src.length; + let sIndex = 0; + clearHashTable(hashTable); + while (remaining > 0) { + let compSize = 0; + const blockSize = remaining > maxBlockSize ? maxBlockSize : remaining; + compSize = compressBlock(src, blockBuf, sIndex, blockSize, hashTable); + if (compSize > blockSize || compSize === 0) { + writeU32(dst, dIndex, 2147483648 | blockSize); + dIndex += 4; + for (let z = sIndex + blockSize; sIndex < z; ) { + dst[dIndex++] = src[sIndex++]; + } + remaining -= blockSize; + } else { + writeU32(dst, dIndex, compSize); + dIndex += 4; + for (let j = 0; j < compSize; ) { + dst[dIndex++] = blockBuf[j++]; + } + sIndex += blockSize; + remaining -= blockSize; + } + } + writeU32(dst, dIndex, 0); + dIndex += 4; + return dIndex; +} +function decompress(src, maxSize) { + let dst, size; + if (maxSize === void 0) { + maxSize = decompressBound(src); + } + dst = makeBuffer(maxSize); + size = decompressFrame(src, dst); + if (size !== maxSize) { + dst = sliceArray(dst, 0, size); + } + return dst; +} +function compress(src, maxSize) { + let dst, size; + if (maxSize === void 0) { + maxSize = compressBound(src.length); + } + dst = makeBuffer(maxSize); + size = compressFrame(src, dst); + if (size !== maxSize) { + dst = sliceArray(dst, 0, size); + } + return dst; +} + +// src/utils/RangeList.ts +var RangeList = class { + ranges = []; + /** + * Add a range to the list. If it overlaps with existing ranges, + * it will split them and increment reference counts accordingly. + */ + add(start, end) { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + const overlappingRanges = []; + for (let i = 0; i < this.ranges.length; i++) { + const range2 = this.ranges[i]; + if (start < range2.end && end > range2.start) { + overlappingRanges.push({ index: i, range: range2 }); + } + if (range2.data !== null) { + throw new Error("Overlapping range already has data"); + } + } + if (overlappingRanges.length === 0) { + this.ranges.push({ start, end, refCount: 1, data: null }); + this.ranges.sort((a, b) => a.start - b.start); + return; + } + const newRanges = []; + let currentPos = start; + for (let i = 0; i < overlappingRanges.length; i++) { + const { range: range2 } = overlappingRanges[i]; + if (currentPos < range2.start) { + newRanges.push({ + start: currentPos, + end: range2.start, + refCount: 1, + data: null + }); + } else if (range2.start < currentPos) { + newRanges.push({ + start: range2.start, + end: currentPos, + refCount: range2.refCount, + data: null + }); + } + newRanges.push({ + start: Math.max(currentPos, range2.start), + end: Math.min(end, range2.end), + refCount: range2.refCount + 1, + data: null + }); + if (range2.end > end) { + newRanges.push({ + start: end, + end: range2.end, + refCount: range2.refCount, + data: null + }); + } + currentPos = Math.max(currentPos, range2.end); + } + if (currentPos < end) { + newRanges.push({ + start: currentPos, + end, + refCount: 1, + data: null + }); + } + const firstIndex = overlappingRanges[0].index; + const lastIndex = overlappingRanges[overlappingRanges.length - 1].index; + this.ranges.splice(firstIndex, lastIndex - firstIndex + 1, ...newRanges); + this.ranges.sort((a, b) => a.start - b.start); + } + /** + * Remove a range from the list. The range must start and end at existing boundaries. + */ + remove(start, end) { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + const affectedRanges = []; + for (let i = 0; i < this.ranges.length; i++) { + const range2 = this.ranges[i]; + if (start < range2.end && end > range2.start) { + affectedRanges.push({ index: i, range: range2 }); + } + } + if (affectedRanges.length === 0) { + throw new Error("No ranges found to remove"); + } + if (start !== affectedRanges[0].range.start || end !== affectedRanges[affectedRanges.length - 1].range.end) { + throw new Error("Range boundaries must match existing boundaries"); + } + for (let i = 0; i < affectedRanges.length; i++) { + const { range: range2 } = affectedRanges[i]; + range2.refCount--; + } + this.ranges = this.ranges.filter((range2) => range2.refCount > 0); + } + /** + * Get all ranges within the specified boundaries. + */ + getRanges(start, end) { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + return this.ranges.filter((range2) => start < range2.end && end > range2.start); + } + /** + * Get all ranges in the list + */ + getAllRanges() { + return [...this.ranges]; + } +}; + +// src/utils/XetBlob.ts +var JWT_SAFETY_PERIOD = 6e4; +var JWT_CACHE_SIZE = 1e3; +var compressionSchemeLabels = { + [0 /* None */]: "None", + [1 /* LZ4 */]: "LZ4", + [2 /* ByteGroupingLZ4 */]: "ByteGroupingLZ4" +}; +var XET_CHUNK_HEADER_BYTES = 8; +var XetBlob = class extends Blob { + fetch; + accessToken; + refreshUrl; + reconstructionUrl; + hash; + start = 0; + end = 0; + internalLogging = false; + reconstructionInfo; + listener; + constructor(params) { + super([]); + this.fetch = params.fetch ?? fetch.bind(globalThis); + this.accessToken = checkCredentials(params); + this.refreshUrl = params.refreshUrl; + this.end = params.size; + this.reconstructionUrl = params.reconstructionUrl; + this.hash = params.hash; + this.listener = params.listener; + this.internalLogging = params.internalLogging ?? false; + if (params.readToken) { + const key = cacheKey({ refreshUrl: this.refreshUrl, initialAccessToken: this.accessToken }); + jwts.set(key, { + accessToken: params.readToken.accessToken, + expiresAt: new Date(params.readToken.exp * 1e3), + casUrl: params.readToken.casUrl + }); + } + } + get size() { + return this.end - this.start; + } + #clone() { + const blob = new XetBlob({ + fetch: this.fetch, + hash: this.hash, + refreshUrl: this.refreshUrl, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + reconstructionUrl: this.reconstructionUrl, + size: this.size + }); + blob.accessToken = this.accessToken; + blob.start = this.start; + blob.end = this.end; + blob.reconstructionInfo = this.reconstructionInfo; + blob.listener = this.listener; + blob.internalLogging = this.internalLogging; + return blob; + } + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on XetBlob.slice"); + } + const slice = this.#clone(); + slice.start = this.start + start; + slice.end = Math.min(this.start + end, this.end); + if (slice.start !== this.start || slice.end !== this.end) { + slice.reconstructionInfo = void 0; + } + return slice; + } + #reconstructionInfoPromise; + #loadReconstructionInfo() { + if (this.#reconstructionInfoPromise) { + return this.#reconstructionInfoPromise; + } + this.#reconstructionInfoPromise = (async () => { + const connParams = await getAccessToken(this.accessToken, this.fetch, this.refreshUrl); + const resp = await this.fetch(this.reconstructionUrl ?? `${connParams.casUrl}/v1/reconstructions/${this.hash}`, { + headers: { + Authorization: `Bearer ${connParams.accessToken}`, + Range: `bytes=${this.start}-${this.end - 1}` + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } + this.reconstructionInfo = await resp.json(); + return this.reconstructionInfo; + })().finally(() => this.#reconstructionInfoPromise = void 0); + return this.#reconstructionInfoPromise; + } + async #fetch() { + if (this.size === 0) { + return new ReadableStream({ + start(controller) { + controller.close(); + } + }); + } + if (!this.reconstructionInfo) { + await this.#loadReconstructionInfo(); + } + const rangeLists = /* @__PURE__ */ new Map(); + if (!this.reconstructionInfo) { + throw new Error("Failed to load reconstruction info"); + } + for (const term of this.reconstructionInfo.terms) { + let rangeList = rangeLists.get(term.hash); + if (!rangeList) { + rangeList = new RangeList(); + rangeLists.set(term.hash, rangeList); + } + rangeList.add(term.range.start, term.range.end); + } + const listener = this.listener; + const log = this.internalLogging ? (...args) => console.log(...args) : () => { + }; + async function* readData(reconstructionInfo, customFetch, maxBytes, reloadReconstructionInfo) { + let totalBytesRead = 0; + let readBytesToSkip = reconstructionInfo.offset_into_first_range; + for (const term of reconstructionInfo.terms) { + if (totalBytesRead >= maxBytes) { + break; + } + const rangeList = rangeLists.get(term.hash); + if (!rangeList) { + throw new Error(`Failed to find range list for term ${term.hash}`); + } + { + const termRanges = rangeList.getRanges(term.range.start, term.range.end); + if (termRanges.every((range2) => range2.data)) { + log("all data available for term", term.hash, readBytesToSkip); + rangeLoop: + for (const range2 of termRanges) { + for (let chunk2 of range2.data) { + if (readBytesToSkip) { + const skipped = Math.min(readBytesToSkip, chunk2.byteLength); + chunk2 = chunk2.slice(skipped); + readBytesToSkip -= skipped; + if (!chunk2.byteLength) { + continue; + } + } + if (chunk2.byteLength > maxBytes - totalBytesRead) { + chunk2 = chunk2.slice(0, maxBytes - totalBytesRead); + } + totalBytesRead += chunk2.byteLength; + yield range2.refCount > 1 ? chunk2.slice() : chunk2; + listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } }); + if (totalBytesRead >= maxBytes) { + break rangeLoop; + } + } + } + rangeList.remove(term.range.start, term.range.end); + continue; + } + } + let fetchInfo = reconstructionInfo.fetch_info[term.hash].find( + (info) => info.range.start <= term.range.start && info.range.end >= term.range.end + ); + if (!fetchInfo) { + throw new Error( + `Failed to find fetch info for term ${term.hash} and range ${term.range.start}-${term.range.end}` + ); + } + log("term", term); + log("fetchinfo", fetchInfo); + log("readBytesToSkip", readBytesToSkip); + let resp = await customFetch(fetchInfo.url, { + headers: { + Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}` + } + }); + if (resp.status === 403) { + reconstructionInfo = await reloadReconstructionInfo(); + fetchInfo = reconstructionInfo.fetch_info[term.hash]?.find( + (info) => info.range.start <= term.range.start && info.range.end >= term.range.end + ); + if (!fetchInfo) { + throw new Error( + `Failed to find fetch info for term ${term.hash} and range ${term.range.start}-${term.range.end} after refresh` + ); + } + resp = await customFetch(fetchInfo.url, { + headers: { + Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}` + } + }); + } + if (!resp.ok) { + throw await createApiError(resp); + } + log( + "expected content length", + resp.headers.get("content-length"), + "range", + fetchInfo.url_range, + resp.headers.get("content-range") + ); + const reader = resp.body?.getReader(); + if (!reader) { + throw new Error("Failed to get reader from response body"); + } + let done = false; + let chunkIndex = fetchInfo.range.start; + const ranges = rangeList.getRanges(fetchInfo.range.start, fetchInfo.range.end); + let leftoverBytes = void 0; + let totalFetchBytes = 0; + fetchData: + while (!done && totalBytesRead < maxBytes) { + const result = await reader.read(); + listener?.({ event: "read" }); + done = result.done; + log("read", result.value?.byteLength, "bytes", "total read", totalBytesRead, "toSkip", readBytesToSkip); + if (!result.value) { + log("no data in result, cancelled", result); + continue; + } + totalFetchBytes += result.value.byteLength; + if (leftoverBytes) { + result.value = combineUint8Arrays(leftoverBytes, result.value); + leftoverBytes = void 0; + } + while (totalBytesRead < maxBytes && result.value?.byteLength) { + if (result.value.byteLength < 8) { + leftoverBytes = result.value; + continue fetchData; + } + const header = new DataView(result.value.buffer, result.value.byteOffset, XET_CHUNK_HEADER_BYTES); + const chunkHeader = { + version: header.getUint8(0), + compressed_length: header.getUint8(1) | header.getUint8(2) << 8 | header.getUint8(3) << 16, + compression_scheme: header.getUint8(4), + uncompressed_length: header.getUint8(5) | header.getUint8(6) << 8 | header.getUint8(7) << 16 + }; + log("chunk header", chunkHeader, "to skip", readBytesToSkip); + if (chunkHeader.version !== 0) { + throw new Error(`Unsupported chunk version ${chunkHeader.version}`); + } + if (chunkHeader.compression_scheme !== 0 /* None */ && chunkHeader.compression_scheme !== 1 /* LZ4 */ && chunkHeader.compression_scheme !== 2 /* ByteGroupingLZ4 */) { + throw new Error( + `Unsupported compression scheme ${compressionSchemeLabels[chunkHeader.compression_scheme] ?? chunkHeader.compression_scheme}` + ); + } + if (result.value.byteLength < chunkHeader.compressed_length + XET_CHUNK_HEADER_BYTES) { + leftoverBytes = result.value; + continue fetchData; + } + result.value = result.value.slice(XET_CHUNK_HEADER_BYTES); + let uncompressed = chunkHeader.compression_scheme === 1 /* LZ4 */ ? decompress(result.value.slice(0, chunkHeader.compressed_length), chunkHeader.uncompressed_length) : chunkHeader.compression_scheme === 2 /* ByteGroupingLZ4 */ ? bg4_regroup_bytes( + decompress( + result.value.slice(0, chunkHeader.compressed_length), + chunkHeader.uncompressed_length + ) + ) : result.value.slice(0, chunkHeader.compressed_length); + const range2 = ranges.find((range3) => chunkIndex >= range3.start && chunkIndex < range3.end); + const shouldYield = chunkIndex >= term.range.start && chunkIndex < term.range.end; + const minRefCountToStore = shouldYield ? 2 : 1; + let stored = false; + if (range2 && range2.refCount >= minRefCountToStore) { + range2.data ??= []; + range2.data.push(uncompressed); + stored = true; + } + if (shouldYield) { + if (readBytesToSkip) { + const skipped = Math.min(readBytesToSkip, uncompressed.byteLength); + uncompressed = uncompressed.slice(readBytesToSkip); + readBytesToSkip -= skipped; + } + if (uncompressed.byteLength > maxBytes - totalBytesRead) { + uncompressed = uncompressed.slice(0, maxBytes - totalBytesRead); + } + if (uncompressed.byteLength) { + log( + "yield", + uncompressed.byteLength, + "bytes", + result.value.byteLength, + "total read", + totalBytesRead, + stored + ); + totalBytesRead += uncompressed.byteLength; + yield stored ? uncompressed.slice() : uncompressed; + listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } }); + } + } + chunkIndex++; + result.value = result.value.slice(chunkHeader.compressed_length); + } + } + if (done && totalBytesRead < maxBytes && totalFetchBytes < fetchInfo.url_range.end - fetchInfo.url_range.start + 1) { + log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes); + log("failed to fetch all data for term", term.hash); + throw new Error( + `Failed to fetch all data for term ${term.hash}, fetched ${totalFetchBytes} bytes out of ${fetchInfo.url_range.end - fetchInfo.url_range.start + 1}` + ); + } + log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes); + log("cancel reader"); + await reader.cancel(); + } + } + const iterator = readData( + this.reconstructionInfo, + this.fetch, + this.end - this.start, + this.#loadReconstructionInfo.bind(this) + ); + return new ReadableStream( + { + // todo: when Safari supports it, type controller as ReadableByteStreamController + async pull(controller) { + const result = await iterator.next(); + if (result.value) { + controller.enqueue(result.value); + } + if (result.done) { + controller.close(); + } + }, + type: "bytes" + // todo: when Safari supports it, add autoAllocateChunkSize param + }, + // todo : use ByteLengthQueuingStrategy when there's good support for it, currently in Node.js it fails due to size being a function + { + highWaterMark: 1e3 + // 1_000 chunks for ~1MB of RAM + } + ); + } + async arrayBuffer() { + const result = await this.#fetch(); + return new Response(result).arrayBuffer(); + } + async text() { + const result = await this.#fetch(); + return new Response(result).text(); + } + async response() { + const result = await this.#fetch(); + return new Response(result); + } + stream() { + const stream = new TransformStream(); + this.#fetch().then((response) => response.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message)); + return stream.readable; + } +}; +var jwtPromises = /* @__PURE__ */ new Map(); +var jwts = /* @__PURE__ */ new Map(); +function cacheKey(params) { + return JSON.stringify([params.refreshUrl, params.initialAccessToken]); +} +function bg4_regroup_bytes(bytes) { + const split = Math.floor(bytes.byteLength / 4); + const rem = bytes.byteLength % 4; + const g1_pos = split + (rem >= 1 ? 1 : 0); + const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); + const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); + const ret = new Uint8Array(bytes.byteLength); + for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + return ret; +} +function bg4_split_bytes(bytes) { + const ret = new Uint8Array(bytes.byteLength); + const split = Math.floor(bytes.byteLength / 4); + const rem = bytes.byteLength % 4; + const g1_pos = split + (rem >= 1 ? 1 : 0); + const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); + const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); + for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + return ret; +} +async function getAccessToken(initialAccessToken, customFetch, refreshUrl) { + const key = cacheKey({ refreshUrl, initialAccessToken }); + const jwt = jwts.get(key); + if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD)) { + return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; + } + const existingPromise = jwtPromises.get(key); + if (existingPromise) { + return existingPromise; + } + const promise = (async () => { + const resp = await customFetch(refreshUrl, { + headers: { + ...initialAccessToken ? { + Authorization: `Bearer ${initialAccessToken}` + } : {} + } + }); + if (!resp.ok) { + throw new Error(`Failed to get JWT token: ${resp.status} ${await resp.text()}`); + } + const json = await resp.json(); + const jwt2 = { + accessToken: json.accessToken, + expiresAt: new Date(json.exp * 1e3), + casUrl: json.casUrl + }; + jwtPromises.delete(key); + for (const [key2, value] of jwts.entries()) { + if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD)) { + jwts.delete(key2); + } else { + break; + } + } + if (jwts.size >= JWT_CACHE_SIZE) { + const keyToDelete = jwts.keys().next().value; + if (keyToDelete) { + jwts.delete(keyToDelete); + } + } + jwts.set(key, jwt2); + return { + accessToken: json.accessToken, + casUrl: json.casUrl + }; + })(); + jwtPromises.set(key, promise); + return promise; +} + +// src/utils/ChunkCache.ts +var CHUNK_CACHE_INITIAL_SIZE = 1e4; +var CHUNK_CACHE_GROW_FACTOR = 1.5; +var CHUNK_CACHE_MAX_SIZE = 1e6; +var ChunkCache = class { + index = 0; + // Index >= 0 means local xorb, < 0 means remote xorb + xorbIndices; + // Max 8K chunks per xorb, less than 64K uint16_t + chunkIndices; + map = /* @__PURE__ */ new Map(); + // hash -> chunkCacheIndex. Less overhead that way, empty object is 60+B and empty array is 40+B + hmacs = /* @__PURE__ */ new Set(); + // todo : remove old hmacs + maxSize; + constructor(maxSize = CHUNK_CACHE_MAX_SIZE) { + if (maxSize < 1) { + throw new Error("maxSize must be at least 1"); + } + this.maxSize = maxSize; + this.xorbIndices = new Int32Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); + this.chunkIndices = new Uint16Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); + } + addChunkToCache(hash2, xorbIndex, chunkIndex, hmac2) { + if (this.map.has(hash2)) { + return; + } + if (this.map.values().next().value === this.index) { + this.map.delete(this.map.keys().next().value); + } + this.map.set(hash2, this.index); + if (hmac2 !== null) { + this.hmacs.add(hmac2); + } + if (this.index >= this.xorbIndices.length) { + const oldXorbIndices = this.xorbIndices; + const oldChunkIndices = this.chunkIndices; + this.xorbIndices = new Int32Array(Math.min(this.xorbIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); + this.chunkIndices = new Uint16Array(Math.min(this.chunkIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); + this.xorbIndices.set(oldXorbIndices); + this.chunkIndices.set(oldChunkIndices); + } + this.xorbIndices[this.index] = xorbIndex; + this.chunkIndices[this.index] = chunkIndex; + this.index = (this.index + 1) % this.maxSize; + } + getChunk(hash2, hmacFunction) { + let index = this.map.get(hash2); + if (index === void 0 && hmacFunction !== null) { + for (const hmac2 of this.hmacs) { + index = this.map.get(hmacFunction(hash2, hmac2)); + if (index !== void 0) { + break; + } + } + } + if (index === void 0) { + return void 0; + } + return { + xorbIndex: this.xorbIndices[index], + chunkIndex: this.chunkIndices[index] + }; + } + updateChunkIndex(hash2, chunkIndex) { + const index = this.map.get(hash2); + if (index === void 0) { + throw new Error(`Chunk not found in cache: ${hash2}`); + } + this.chunkIndices[index] = chunkIndex; + } + removeChunkFromCache(hash2) { + this.map.delete(hash2); + } +}; + +// src/utils/xetWriteToken.ts +var JWT_SAFETY_PERIOD2 = 6e4; +var JWT_CACHE_SIZE2 = 1e3; +var jwtPromises2 = /* @__PURE__ */ new Map(); +var jwts2 = /* @__PURE__ */ new Map(); +async function xetWriteToken(params) { + if (params.xetParams.expiresAt && params.xetParams.casUrl && params.xetParams.accessToken && params.xetParams.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD2)) { + return { accessToken: params.xetParams.accessToken, casUrl: params.xetParams.casUrl }; + } + const key = params.xetParams.refreshWriteTokenUrl; + const jwt = jwts2.get(key); + if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD2)) { + return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; + } + const existingPromise = jwtPromises2.get(key); + if (existingPromise) { + return existingPromise; + } + const promise = (async () => { + const resp = await (params.fetch ?? fetch)(params.xetParams.refreshWriteTokenUrl, { + headers: { + ...params.accessToken ? { + Authorization: `Bearer ${params.accessToken}` + } : {}, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } + const json = await resp.json(); + const jwt2 = { + accessToken: json.accessToken, + expiresAt: new Date(json.exp * 1e3), + casUrl: json.casUrl + }; + jwtPromises2.delete(key); + for (const [key2, value] of jwts2.entries()) { + if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD2)) { + jwts2.delete(key2); + } else { + break; + } + } + if (jwts2.size >= JWT_CACHE_SIZE2) { + const keyToDelete = jwts2.keys().next().value; + if (keyToDelete) { + jwts2.delete(keyToDelete); + } + } + jwts2.set(key, jwt2); + return { + accessToken: json.accessToken, + casUrl: json.casUrl + }; + })(); + jwtPromises2.set(key, promise); + return promise; +} + +// src/utils/shardParser.ts +var HASH_LENGTH = 32; +var XORB_HASH_BOOKEND = "ff".repeat(HASH_LENGTH); +function readHashFromArray(array, offset) { + let hash2 = ""; + for (let i = 0; i < HASH_LENGTH; i += 8) { + hash2 += `${array[offset + i + 7].toString(16).padStart(2, "0")}${array[offset + i + 6].toString(16).padStart(2, "0")}${array[offset + i + 5].toString(16).padStart(2, "0")}${array[offset + i + 4].toString(16).padStart(2, "0")}${array[offset + i + 3].toString(16).padStart(2, "0")}${array[offset + i + 2].toString(16).padStart(2, "0")}${array[offset + i + 1].toString(16).padStart(2, "0")}${array[offset + i].toString(16).padStart(2, "0")}`; + } + return hash2; +} +async function parseShardData(shardBlob) { + const shard = new Uint8Array(await shardBlob.arrayBuffer()); + const shardView = new DataView(shard.buffer); + const magicTag = shard.slice(0, SHARD_MAGIC_TAG.length); + if (!magicTag.every((byte, i) => byte === SHARD_MAGIC_TAG[i])) { + throw new Error("Invalid shard magic tag"); + } + const version = shardView.getBigUint64(SHARD_MAGIC_TAG.length, true); + if (version !== SHARD_HEADER_VERSION) { + throw new Error(`Invalid shard version: ${version}`); + } + const footerSize = Number(shardView.getBigUint64(SHARD_MAGIC_TAG.length + 8, true)); + const footerStart = shard.length - footerSize; + const footerVersion = shardView.getBigUint64(footerStart, true); + if (footerVersion !== SHARD_FOOTER_VERSION) { + throw new Error(`Invalid shard footer version: ${footerVersion}`); + } + const xorbInfoStart = Number(shardView.getBigUint64(footerStart + 16, true)); + const fileLookupStart = Number(shardView.getBigUint64(footerStart + 24, true)); + const hmacKey = readHashFromArray(shard, footerStart + 72); + const xorbs = []; + let offset = xorbInfoStart; + while (offset < fileLookupStart) { + const xorbHash2 = readHashFromArray(shard, offset); + offset += HASH_LENGTH; + if (xorbHash2 === XORB_HASH_BOOKEND) { + break; + } + offset += 4; + const chunkCount = shardView.getUint32(offset, true); + offset += 4; + offset += 4; + offset += 4; + const chunks = []; + for (let i = 0; i < chunkCount; i++) { + const chunkHash = readHashFromArray(shard, offset); + offset += HASH_LENGTH; + const startOffset = shardView.getUint32(offset, true); + offset += 4; + const length = shardView.getUint32(offset, true); + offset += 4; + offset += 8; + chunks.push({ + hash: chunkHash, + startOffset, + unpackedLength: length + }); + } + xorbs.push({ + hash: xorbHash2, + chunks + }); + } + return { + hmacKey, + xorbs + }; +} + +// src/utils/sum.ts +function sum(arr) { + return arr.reduce((a, b) => a + b, 0); +} + +// src/utils/SplicedBlob.ts +var SplicedBlob = class extends Blob { + originalBlob; + spliceOperations; + constructor(originalBlob, spliceOperations) { + super(); + this.originalBlob = originalBlob; + this.spliceOperations = spliceOperations; + } + static create(originalBlob, operations) { + for (const op of operations) { + if (op.start < 0 || op.end < 0) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + if (op.start > originalBlob.size || op.end > originalBlob.size) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + if (op.start > op.end) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + } + const sortedOps = [...operations].sort((a, b) => a.start - b.start); + for (let i = 0; i < sortedOps.length - 1; i++) { + if (sortedOps[i].end > sortedOps[i + 1].start) { + throw new Error("Overlapping splice operations are not supported"); + } + } + return new SplicedBlob(originalBlob, sortedOps); + } + /** + * Returns the size of the spliced blob. + * Size = original size - total replaced size + total insert size + */ + get size() { + let totalReplacedSize = 0; + let totalInsertSize = 0; + for (const op of this.spliceOperations) { + totalReplacedSize += op.end - op.start; + totalInsertSize += op.insert.size; + } + return this.originalBlob.size - totalReplacedSize + totalInsertSize; + } + /** + * Returns the MIME type of the original blob. + */ + get type() { + return this.originalBlob.type; + } + /** + * Returns a new instance of SplicedBlob that is a slice of the current one. + * + * The slice is inclusive of the start and exclusive of the end. + * The slice method does not support negative start/end. + * + * @param start beginning of the slice + * @param end end of the slice + */ + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + throw new TypeError("Unsupported negative start/end on SplicedBlob.slice"); + } + start = Math.min(start, this.size); + end = Math.min(end, this.size); + if (start >= end) { + return new Blob([]); + } + const segments = this.segments; + const segmentBoundaries = [0]; + let cumulativeSize = 0; + for (const segment of segments) { + cumulativeSize += segment.size; + segmentBoundaries.push(cumulativeSize); + } + const resultSegments = []; + for (let i = 0; i < segments.length; i++) { + const segmentStart = segmentBoundaries[i]; + const segmentEnd = segmentBoundaries[i + 1]; + if (segmentEnd <= start) { + continue; + } + if (segmentStart >= end) { + break; + } + const sliceStart = Math.max(0, start - segmentStart); + const sliceEnd = Math.min(segments[i].size, end - segmentStart); + if (sliceStart < sliceEnd) { + resultSegments.push(segments[i].slice(sliceStart, sliceEnd)); + } + } + return new Blob(resultSegments); + } + get firstSpliceIndex() { + return this.spliceOperations[0]?.start ?? Infinity; + } + /** + * Read the spliced blob content and returns it as an ArrayBuffer. + */ + async arrayBuffer() { + const segments = this.segments; + const buffers = await Promise.all(segments.map((segment) => segment.arrayBuffer())); + const totalSize = sum(buffers.map((buffer) => buffer.byteLength)); + const result = new Uint8Array(totalSize); + let offset = 0; + for (const buffer of buffers) { + result.set(new Uint8Array(buffer), offset); + offset += buffer.byteLength; + } + return result.buffer; + } + /** + * Read the spliced blob content and returns it as a string. + */ + async text() { + const buffer = await this.arrayBuffer(); + return new TextDecoder().decode(buffer); + } + /** + * Returns a stream around the spliced blob content. + */ + stream() { + const readable = new ReadableStream({ + start: async (controller) => { + try { + const segments = this.segments; + for (const segment of segments) { + const reader = segment.stream().getReader(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + controller.enqueue(value); + } + } finally { + reader.releaseLock(); + } + } + controller.close(); + } catch (error) { + controller.error(error); + } + } + }); + return readable; + } + /** + * Get all segments that make up the spliced blob. + * This includes original blob segments between splice operations and insert blobs. + */ + get segments() { + const segments = []; + let currentPosition = 0; + const sortedOps = [...this.spliceOperations].sort((a, b) => a.start - b.start); + for (const op of sortedOps) { + if (currentPosition < op.start) { + segments.push(this.originalBlob.slice(currentPosition, op.start)); + } + if (op.insert.size > 0) { + segments.push(op.insert); + } + currentPosition = op.end; + } + if (currentPosition < this.originalBlob.size) { + segments.push(this.originalBlob.slice(currentPosition)); + } + return segments; + } +}; + +// src/utils/createXorbs.ts +import { + createChunker, + nextBlock, + finalize, + hashToHex, + hexToBytes, + xorbHash, + fileHash, + hmac, + verificationHash +} from "@huggingface/xetchunk-wasm"; +var TARGET_CHUNK_SIZE = 64 * 1024; +var MAX_CHUNK_SIZE = 2 * TARGET_CHUNK_SIZE; +var XORB_SIZE = 64 * 1024 * 1024; +var MAX_XORB_CHUNKS = 8 * 1024; +var INTERVAL_BETWEEN_REMOTE_DEDUP = 4e6; +var PROCESSING_PROGRESS_RATIO = 0.1; +var UPLOADING_PROGRESS_RATIO = 1 - PROCESSING_PROGRESS_RATIO; +function computeXorbHashHex(chunks) { + const chunkObjs = chunks.map((c) => ({ hash: hexToBytes(c.hash), length: c.length })); + return hashToHex(xorbHash(chunkObjs)); +} +function computeHmacHex(hash2, key) { + return hashToHex(hmac(hexToBytes(hash2), hexToBytes(key))); +} +function computeVerificationHashHex(hashes) { + return hashToHex(verificationHash(hashes.map(hexToBytes))); +} +function computeFileHashHex(chunks) { + const chunkObjs = chunks.map((c) => ({ hash: hexToBytes(c.hash), length: c.length })); + return hashToHex(fileHash(chunkObjs)); +} +function addDataToChunker(data, chunker) { + return nextBlock(chunker, data).map((c) => ({ hash: hashToHex(c.hash), length: c.length, dedup: false })); +} +function finalizeChunker(chunker) { + const last = finalize(chunker); + if (!last) { + return []; + } + return [{ hash: hashToHex(last.hash), length: last.length, dedup: false }]; +} +var CurrentXorbInfo = class { + id; + offset; + chunks; + fileProcessedBytes; + fileUploadedBytes; + fileSize; + data; + immutableData; + constructor() { + this.id = 0; + this.offset = 0; + this.chunks = []; + this.fileProcessedBytes = {}; + this.fileUploadedBytes = {}; + this.fileSize = {}; + this.data = new Uint8Array(XORB_SIZE); + this.immutableData = null; + } + event(computeXorbHash) { + const xorbChunksCleaned = this.chunks.map((chunk2) => ({ + hash: chunk2.hash, + length: chunk2.length + })); + return { + event: "xorb", + xorb: this.data.subarray(0, this.offset), + hash: computeXorbHash(xorbChunksCleaned), + chunks: xorbChunksCleaned, + id: this.id, + files: Object.entries(this.fileProcessedBytes).map(([path, processedBytes]) => ({ + path, + progress: processedBytes / this.fileSize[path], + lastSentProgress: ((this.fileUploadedBytes[path] ?? 0) + (processedBytes - (this.fileUploadedBytes[path] ?? 0)) * PROCESSING_PROGRESS_RATIO) / this.fileSize[path] + })) + }; + } +}; +async function* createXorbs(fileSources, params) { + const alreadyDoneFileSha256s = /* @__PURE__ */ new Set(); + let xorbId = 0; + const chunkCache = new ChunkCache(); + let xorb = new CurrentXorbInfo(); + const nextXorb = (currentFile) => { + const event = xorb.event(computeXorbHashHex); + xorbId++; + xorb = new CurrentXorbInfo(); + xorb.id = xorbId; + xorb.fileUploadedBytes = { + [currentFile.path]: currentFile.uploadedBytes + }; + xorb.fileSize[currentFile.path] = currentFile.size; + return event; + }; + const pendingFileEvents = []; + const remoteXorbHashes = [""]; + for await (const fileSource of fileSources) { + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: 0 + }); + if (fileSource.sha256 && alreadyDoneFileSha256s.has(fileSource.sha256)) { + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: 1 + }); + continue; + } + if (fileSource.sha256) { + alreadyDoneFileSha256s.add(fileSource.sha256); + } + const chunker = createChunker(TARGET_CHUNK_SIZE); + { + xorb.fileSize[fileSource.path] = fileSource.content.size; + if (fileSource.content instanceof SplicedBlob && fileSource.content.firstSpliceIndex < MAX_CHUNK_SIZE) { + await loadDedupInfoToCache( + fileSource.content.originalBlob.slice(0, MAX_CHUNK_SIZE), + remoteXorbHashes, + params, + chunkCache, + computeHmacHex, + { + maxChunks: 1, + isAtBeginning: true + } + ); + } + let bytesSinceRemoteDedup = Infinity; + let bytesSinceLastProgressEvent = 0; + let isFirstFileChunk = true; + const sourceChunks = []; + const reader = fileSource.content.stream().getReader(); + let processedBytes = 0; + let dedupedBytes = 0; + const fileChunks = []; + const chunkMetadata = []; + const addChunks = async function* (chunks) { + for (const chunk2 of chunks) { + if (isFirstFileChunk) { + chunk2.dedup = true; + isFirstFileChunk = false; + } + let chunkIndex = xorb.chunks.length; + let chunkXorbId = xorbId; + const chunkToCopy = removeChunkFromSourceData(sourceChunks, chunk2.length); + let cacheData = chunkCache.getChunk(chunk2.hash, computeHmacHex); + if (cacheData === void 0 && chunk2.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { + const token = await xetWriteToken(params); + bytesSinceRemoteDedup = 0; + const shardResp = await (params.fetch ?? fetch)(token.casUrl + "/v1/chunks/default/" + chunk2.hash, { + headers: { + Authorization: `Bearer ${token.accessToken}` + } + }); + if (shardResp.ok) { + const shard = await shardResp.blob(); + const shardData = await parseShardData(shard); + for (const xorb2 of shardData.xorbs) { + const remoteXorbId = -remoteXorbHashes.length; + remoteXorbHashes.push(xorb2.hash); + let i = 0; + for (const chunk3 of xorb2.chunks) { + chunkCache.addChunkToCache(chunk3.hash, remoteXorbId, i++, shardData.hmacKey); + } + } + cacheData = chunkCache.getChunk(chunk2.hash, computeHmacHex); + const oldDedupedBytes = dedupedBytes; + dedupedBytes = backtrackDedup(xorb, computeHmacHex, shardData, chunkCache, chunkMetadata, dedupedBytes); + if (dedupedBytes > oldDedupedBytes) { + xorb.fileUploadedBytes[fileSource.path] ??= 0; + xorb.fileUploadedBytes[fileSource.path] += dedupedBytes - oldDedupedBytes; + } + } + } + if (cacheData === void 0) { + if (!writeChunk(xorb, chunkToCopy, chunk2.hash)) { + yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); + chunkIndex = 0; + chunkXorbId = xorbId; + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } + pendingFileEvents.length = 0; + if (!writeChunk(xorb, chunkToCopy, chunk2.hash)) { + throw new Error("Failed to write chunk into xorb"); + } + } + chunkCache.addChunkToCache(chunk2.hash, xorbId, chunkIndex, null); + } else { + chunkXorbId = cacheData.xorbIndex; + chunkIndex = cacheData.chunkIndex; + dedupedBytes += chunk2.length; + xorb.fileUploadedBytes[fileSource.path] ??= 0; + xorb.fileUploadedBytes[fileSource.path] += chunk2.length; + } + bytesSinceRemoteDedup += chunk2.length; + bytesSinceLastProgressEvent += chunk2.length; + fileChunks.push({ hash: chunk2.hash, length: chunk2.length }); + chunkMetadata.push({ + xorbId: chunkXorbId, + chunkIndex, + length: chunk2.length + }); + xorb.fileProcessedBytes[fileSource.path] = processedBytes; + if (bytesSinceLastProgressEvent >= 1e6) { + bytesSinceLastProgressEvent = 0; + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: ((xorb.fileUploadedBytes[fileSource.path] ?? 0) + (xorb.fileProcessedBytes[fileSource.path] - (xorb.fileUploadedBytes[fileSource.path] ?? 0)) * PROCESSING_PROGRESS_RATIO) / fileSource.content.size + }); + } + if (xorb.chunks.length >= MAX_XORB_CHUNKS) { + yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } + pendingFileEvents.length = 0; + } + } + }; + while (true) { + const { done, value } = await reader.read(); + if (done) { + yield* addChunks(finalizeChunker(chunker)); + break; + } + processedBytes += value.length; + sourceChunks.push(value); + yield* addChunks(addDataToChunker(value, chunker)); + } + const fileRepresentation = buildFileRepresentation(chunkMetadata, fileChunks, computeVerificationHashHex); + xorb.immutableData = { + chunkIndex: xorb.chunks.length, + offset: xorb.offset + }; + const dedupRatio = fileSource.content.size > 0 ? dedupedBytes / fileSource.content.size : 0; + pendingFileEvents.push({ + event: "file", + path: fileSource.path, + hash: computeFileHashHex(fileChunks), + sha256: fileSource.sha256, + dedupRatio, + representation: fileRepresentation + }); + } + } + if (xorb.offset > 0) { + yield xorb.event(computeXorbHashHex); + } + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } +} +function backtrackDedup(xorb, computeHmac, shardData, chunkCache, chunkMetadata, dedupedBytes) { + const chunkIndexesToBacktrackFor = /* @__PURE__ */ new Map(); + for (let chunkToRecheckIndex = xorb.immutableData?.chunkIndex ?? 0; chunkToRecheckIndex < xorb.chunks.length; chunkToRecheckIndex++) { + const chunk2 = xorb.chunks[chunkToRecheckIndex]; + const hmacHash = computeHmac(chunk2.hash, shardData.hmacKey); + const cacheData = chunkCache.getChunk(hmacHash, null); + if (cacheData !== void 0) { + chunkIndexesToBacktrackFor.set(chunkToRecheckIndex, { + xorbId: cacheData.xorbIndex, + chunkIndex: cacheData.chunkIndex + }); + chunkCache.removeChunkFromCache(chunk2.hash); + } + } + for (const metadata of chunkMetadata) { + if (metadata.xorbId === xorb.id && chunkIndexesToBacktrackFor.has(metadata.chunkIndex)) { + const backtrackData = chunkIndexesToBacktrackFor.get(metadata.chunkIndex); + if (backtrackData !== void 0) { + metadata.xorbId = backtrackData.xorbId; + metadata.chunkIndex = backtrackData.chunkIndex; + dedupedBytes += metadata.length; + } + } + } + const xorbRangesToErase = []; + for (let i = 0; i < xorb.chunks.length; i++) { + const chunk2 = xorb.chunks[i]; + if (chunkIndexesToBacktrackFor.has(i)) { + xorbRangesToErase.push({ + start: chunk2.offset, + end: i < xorb.chunks.length - 1 ? xorb.chunks[i + 1].offset : xorb.offset + }); + } + } + const xorbRangesToKeep = []; + let currentStart = 0; + for (let i = 0; i < xorbRangesToErase.length; i++) { + const range2 = xorbRangesToErase[i]; + if (currentStart !== range2.start) { + xorbRangesToKeep.push({ start: currentStart, end: range2.start }); + } + currentStart = range2.end; + } + if (currentStart !== xorb.offset) { + xorbRangesToKeep.push({ start: currentStart, end: xorb.offset }); + } + let currentOffset = 0; + for (const range2 of xorbRangesToKeep) { + if (range2.start !== currentOffset) { + xorb.data.set(xorb.data.subarray(range2.start, range2.end), currentOffset); + } + currentOffset += range2.end - range2.start; + } + const newXorbChunks = []; + const oldIndexToNewIndex = /* @__PURE__ */ new Map(); + let erasedOffset = 0; + for (let i = 0; i < xorb.chunks.length; i++) { + const chunk2 = xorb.chunks[i]; + if (chunkIndexesToBacktrackFor.has(i)) { + if (i < xorb.chunks.length - 1) { + erasedOffset += xorb.chunks[i + 1].offset - chunk2.offset; + } + } else { + newXorbChunks.push({ + hash: chunk2.hash, + length: chunk2.length, + offset: chunk2.offset - erasedOffset + }); + if (erasedOffset > 0) { + oldIndexToNewIndex.set(i, newXorbChunks.length - 1); + } + } + } + xorb.chunks = newXorbChunks; + xorb.offset = currentOffset; + for (const chunk2 of chunkMetadata) { + if (chunk2.xorbId === xorb.id) { + const newIndex = oldIndexToNewIndex.get(chunk2.chunkIndex); + if (newIndex !== void 0) { + const cached = chunkCache.getChunk(xorb.chunks[newIndex].hash, null); + if (cached !== void 0 && cached.xorbIndex === chunk2.xorbId && cached.chunkIndex === chunk2.chunkIndex) { + chunkCache.updateChunkIndex(xorb.chunks[newIndex].hash, newIndex); + } + chunk2.chunkIndex = newIndex; + } + } + } + return dedupedBytes; +} +function removeChunkFromSourceData(sourceChunks, chunkLength) { + if (chunkLength === sourceChunks[0].length) { + const chunkToCopy = sourceChunks[0]; + sourceChunks.shift(); + return chunkToCopy; + } else if (chunkLength < sourceChunks[0].length) { + const chunkToCopy = sourceChunks[0].subarray(0, chunkLength); + sourceChunks[0] = sourceChunks[0].subarray(chunkLength); + return chunkToCopy; + } else { + const chunkToCopy = new Uint8Array(chunkLength); + let copyOffset = 0; + let index = 0; + let toSlice = -1; + while (copyOffset < chunkLength) { + const nToCopy = Math.min(sourceChunks[index].length, chunkLength - copyOffset); + chunkToCopy.set(sourceChunks[index].subarray(0, nToCopy), copyOffset); + copyOffset += nToCopy; + if (nToCopy === sourceChunks[index].length) { + index++; + } else { + toSlice = nToCopy; + } + } + sourceChunks.splice(0, index); + if (toSlice !== -1) { + sourceChunks[0] = sourceChunks[0].subarray(toSlice); + } + return chunkToCopy; + } +} +function writeChunk(xorb, chunk2, hash2) { + const regularCompressedChunk = compress(chunk2); + const bgCompressedChunk = compress(bg4_split_bytes(chunk2)); + const compressedChunk = bgCompressedChunk.length < regularCompressedChunk.length ? bgCompressedChunk : regularCompressedChunk; + const chunkToWrite = compressedChunk.length < chunk2.length ? compressedChunk : chunk2; + if (xorb.offset + XET_CHUNK_HEADER_BYTES + chunkToWrite.length > XORB_SIZE) { + return false; + } + xorb.data[xorb.offset] = 0; + xorb.data[xorb.offset + 1] = chunkToWrite.length & 255; + xorb.data[xorb.offset + 2] = chunkToWrite.length >> 8 & 255; + xorb.data[xorb.offset + 3] = chunkToWrite.length >> 16 & 255; + xorb.data[xorb.offset + 4] = chunkToWrite.length < chunk2.length ? bgCompressedChunk.length < regularCompressedChunk.length ? 2 /* ByteGroupingLZ4 */ : 1 /* LZ4 */ : 0 /* None */; + xorb.data[xorb.offset + 5] = chunk2.length & 255; + xorb.data[xorb.offset + 6] = chunk2.length >> 8 & 255; + xorb.data[xorb.offset + 7] = chunk2.length >> 16 & 255; + xorb.data.set(chunkToWrite, xorb.offset + XET_CHUNK_HEADER_BYTES); + xorb.chunks.push({ hash: hash2, length: chunk2.length, offset: xorb.offset }); + xorb.offset += XET_CHUNK_HEADER_BYTES + chunkToWrite.length; + return true; +} +var buildFileRepresentation = (metadata, chunks, computeVerificationHash) => { + if (metadata.length === 0) { + return []; + } + const representation = []; + let currentRange = { + xorbId: metadata[0].xorbId, + indexStart: metadata[0].chunkIndex, + indexEnd: metadata[0].chunkIndex + 1, + length: metadata[0].length, + chunkHashStart: 0 + }; + for (let i = 1; i < metadata.length; i++) { + const chunk2 = metadata[i]; + if (currentRange.xorbId === chunk2.xorbId && currentRange.indexEnd === chunk2.chunkIndex) { + currentRange.indexEnd = chunk2.chunkIndex + 1; + currentRange.length += chunk2.length; + } else { + const rangeHash2 = computeVerificationHash(chunks.slice(currentRange.chunkHashStart, i).map((x) => x.hash)); + representation.push({ + xorbId: currentRange.xorbId, + indexStart: currentRange.indexStart, + indexEnd: currentRange.indexEnd, + length: currentRange.length, + rangeHash: rangeHash2 + }); + currentRange = { + xorbId: chunk2.xorbId, + indexStart: chunk2.chunkIndex, + indexEnd: chunk2.chunkIndex + 1, + length: chunk2.length, + chunkHashStart: i + }; + } + } + const rangeHash = computeVerificationHash(chunks.slice(currentRange.chunkHashStart).map((x) => x.hash)); + representation.push({ + xorbId: currentRange.xorbId, + indexStart: currentRange.indexStart, + indexEnd: currentRange.indexEnd, + length: currentRange.length, + rangeHash + }); + return representation; +}; +async function loadDedupInfoToCache(content, remoteXorbHashes, params, chunkCache, computeHmacHex2, opts) { + const chunker = createChunker(TARGET_CHUNK_SIZE); + const cache = chunkCache; + let dedupedBytes = 0; + let chunksProcessed = 0; + let totalBytes = 0; + let bytesSinceRemoteDedup = Infinity; + const sourceChunks = []; + const reader = content.stream().getReader(); + const processChunks = async (chunks) => { + for (const chunk2 of chunks) { + chunksProcessed++; + if (opts?.isAtBeginning && chunksProcessed === 1) { + chunk2.dedup = true; + } + totalBytes += chunk2.length; + removeChunkFromSourceData(sourceChunks, chunk2.length); + let cacheData = cache.getChunk(chunk2.hash, computeHmacHex2); + if (cacheData !== void 0) { + dedupedBytes += chunk2.length; + bytesSinceRemoteDedup += chunk2.length; + continue; + } + if (chunk2.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { + const token = await xetWriteToken(params); + bytesSinceRemoteDedup = 0; + const shardResp = await (params.fetch ?? fetch)(token.casUrl + "/v1/chunks/default/" + chunk2.hash, { + headers: { + Authorization: `Bearer ${token.accessToken}` + } + }); + if (shardResp.ok) { + const shard = await shardResp.blob(); + const shardData = await parseShardData(shard); + for (const xorb of shardData.xorbs) { + const remoteXorbId = -remoteXorbHashes.length; + remoteXorbHashes.push(xorb.hash); + let i = 0; + for (const xorbChunk of xorb.chunks) { + cache.addChunkToCache(xorbChunk.hash, remoteXorbId, i++, shardData.hmacKey); + } + } + cacheData = cache.getChunk(chunk2.hash, computeHmacHex2); + } + } + if (cacheData !== void 0) { + dedupedBytes += chunk2.length; + } + bytesSinceRemoteDedup += chunk2.length; + } + }; + while (true) { + if (opts?.end !== void 0 && totalBytes >= opts.end) { + break; + } + if (opts?.maxChunks !== void 0 && chunksProcessed >= opts.maxChunks) { + break; + } + const { done, value } = await reader.read(); + if (done) { + await processChunks(finalizeChunker(chunker)); + break; + } + sourceChunks.push(value); + await processChunks(addDataToChunker(value, chunker)); + } +} + +// src/utils/uploadShards.ts +var SHARD_MAX_SIZE = 64 * 1024 * 1024; +var SHARD_HEADER_SIZE = 48; +var SHARD_FOOTER_SIZE = 200; +var HASH_LENGTH2 = 32; +var XORB_FOOTER_LENGTH = 48; +var FILE_FOOTER_LENGTH = 48; +var SHARD_HEADER_VERSION = 2n; +var SHARD_FOOTER_VERSION = 1n; +var MDB_FILE_FLAG_WITH_VERIFICATION = 2147483648; +var MDB_FILE_FLAG_WITH_METADATA_EXT = 1073741824; +var SHARD_MAGIC_TAG = new Uint8Array([ + "H".charCodeAt(0), + "F".charCodeAt(0), + "R".charCodeAt(0), + "e".charCodeAt(0), + "p".charCodeAt(0), + "o".charCodeAt(0), + "M".charCodeAt(0), + "e".charCodeAt(0), + "t".charCodeAt(0), + "a".charCodeAt(0), + "D".charCodeAt(0), + "a".charCodeAt(0), + "t".charCodeAt(0), + "a".charCodeAt(0), + 0, + 85, + 105, + 103, + 69, + 106, + 123, + 129, + 87, + 131, + 165, + 189, + 217, + 92, + 205, + 209, + 74, + 169 +]); +async function* uploadShards(source, params) { + const xorbHashes = []; + const seenFileXetHashes = /* @__PURE__ */ new Set(); + const fileInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.25); + const xorbInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.75); + const xorbView = new DataView(xorbInfoSection.buffer); + let xorbViewOffset = 0; + const fileInfoView = new DataView(fileInfoSection.buffer); + let fileViewOffset = 0; + let xorbTotalSize = 0n; + let fileTotalSize = 0n; + let xorbTotalUnpackedSize = 0n; + for await (const output of createXorbs(source, params)) { + switch (output.event) { + case "xorb": { + xorbHashes.push(output.hash); + const xorbEntrySize = HASH_LENGTH2 + 4 + 4 + 4 + 4; + const chunksSize = output.chunks.length * (HASH_LENGTH2 + 4 + 4 + 8); + const totalXorbSize = xorbEntrySize + chunksSize; + if (xorbViewOffset + totalXorbSize > xorbInfoSection.length) { + if (xorbViewOffset > 0 || fileViewOffset > 0) { + await uploadShard(createShard(), params); + } + } + writeHashToArray(output.hash, xorbInfoSection, xorbViewOffset); + xorbViewOffset += HASH_LENGTH2; + xorbView.setUint32(xorbViewOffset, 0, true); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, output.chunks.length, true); + xorbViewOffset += 4; + const xorbUnpackedSize = sum(output.chunks.map((x) => x.length)); + xorbView.setUint32(xorbViewOffset, xorbUnpackedSize, true); + xorbTotalUnpackedSize += BigInt(xorbUnpackedSize); + xorbTotalSize += BigInt(output.xorb.byteLength); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, output.xorb.byteLength, true); + xorbViewOffset += 4; + let chunkBytes = 0; + for (const chunk2 of output.chunks) { + writeHashToArray(chunk2.hash, xorbInfoSection, xorbViewOffset); + xorbViewOffset += HASH_LENGTH2; + xorbView.setUint32(xorbViewOffset, chunkBytes, true); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, chunk2.length, true); + xorbViewOffset += 4; + xorbView.setBigUint64(xorbViewOffset, 0n, true); + xorbViewOffset += 8; + chunkBytes += chunk2.length; + } + for (const file of output.files) { + yield { + event: "fileProgress", + path: file.path, + progress: file.lastSentProgress + }; + } + await uploadXorb(output, params); + for (const file of output.files) { + yield { event: "fileProgress", path: file.path, progress: file.progress }; + } + break; + } + case "file": { + yield { + event: "file", + path: output.path, + xetHash: output.hash, + sha256: output.sha256, + dedupRatio: output.dedupRatio + }; + if (seenFileXetHashes.has(output.hash)) { + break; + } + seenFileXetHashes.add(output.hash); + const fileHeaderSize = HASH_LENGTH2 + 4 + 4 + 8; + const representationSize = output.representation.length * (HASH_LENGTH2 + 4 + 4 + 4 + 4); + const verificationSize = output.representation.length * (HASH_LENGTH2 + 16); + const fileSha256 = output.sha256; + const hasMetadataExt = fileSha256 !== void 0; + const metadataSize = hasMetadataExt ? HASH_LENGTH2 + 16 : 0; + const totalFileSize = fileHeaderSize + representationSize + verificationSize + metadataSize; + if (fileViewOffset + totalFileSize > fileInfoSection.length) { + if (xorbViewOffset > 0 || fileViewOffset > 0) { + await uploadShard(createShard(), params); + } + } + writeHashToArray(output.hash, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + fileInfoView.setUint32( + fileViewOffset, + MDB_FILE_FLAG_WITH_VERIFICATION + (hasMetadataExt ? MDB_FILE_FLAG_WITH_METADATA_EXT : 0), + true + ); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, output.representation.length, true); + fileViewOffset += 4; + fileInfoView.setBigUint64(fileViewOffset, 0n, true); + fileViewOffset += 8; + for (const repItem of output.representation) { + writeHashToArray( + typeof repItem.xorbId === "number" ? xorbHashes[repItem.xorbId] : repItem.xorbId, + fileInfoSection, + fileViewOffset + ); + fileViewOffset += HASH_LENGTH2; + fileInfoView.setUint32(fileViewOffset, 0, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.length, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.indexStart, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.indexEnd, true); + fileViewOffset += 4; + } + for (const repItem of output.representation) { + writeHashToArray(repItem.rangeHash, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + for (let i = 0; i < 16; i++) { + fileInfoSection[fileViewOffset + i] = 0; + } + fileViewOffset += 16; + } + if (hasMetadataExt) { + writeHashToArray(fileSha256, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + for (let i = 0; i < 16; i++) { + fileInfoSection[fileViewOffset + i] = 0; + } + fileViewOffset += 16; + } + break; + } + } + } + function createShard() { + const shard = new Uint8Array( + SHARD_HEADER_SIZE + SHARD_FOOTER_SIZE + xorbViewOffset + XORB_FOOTER_LENGTH + fileViewOffset + FILE_FOOTER_LENGTH + ); + const shardView = new DataView(shard.buffer); + let shardOffset = 0; + shard.set(SHARD_MAGIC_TAG, shardOffset); + shardOffset += SHARD_MAGIC_TAG.length; + shardView.setBigUint64(shardOffset, SHARD_HEADER_VERSION, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(SHARD_FOOTER_SIZE), true); + shardOffset += 8; + shard.set(fileInfoSection.slice(0, fileViewOffset), shardOffset); + shardOffset += fileViewOffset; + for (let i = 0; i < 32; i++) { + shard[shardOffset + i] = 255; + } + shardOffset += 32; + for (let i = 0; i < 16; i++) { + shard[shardOffset + i] = 0; + } + shardOffset += 16; + const xorbInfoOffset = shardOffset; + shard.set(xorbInfoSection.slice(0, xorbViewOffset), shardOffset); + shardOffset += xorbViewOffset; + for (let i = 0; i < 32; i++) { + shard[shardOffset + i] = 255; + } + shardOffset += 32; + for (let i = 0; i < 16; i++) { + shard[shardOffset + i] = 0; + } + shardOffset += 16; + const footerOffset = shardOffset; + shardView.setBigUint64(shardOffset, SHARD_FOOTER_VERSION, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(SHARD_HEADER_SIZE), true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(xorbInfoOffset), true); + shardOffset += 8; + for (let i = 0; i < 48; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 48; + for (let i = 0; i < 32; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 32; + shardView.setBigUint64(shardOffset, BigInt(Math.floor(Date.now() / 1e3)), true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, 0n, true); + shardOffset += 8; + for (let i = 0; i < 48; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 48; + shardView.setBigUint64(shardOffset, xorbTotalSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, fileTotalSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, xorbTotalUnpackedSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(footerOffset), true); + xorbViewOffset = 0; + fileViewOffset = 0; + xorbTotalSize = 0n; + xorbTotalUnpackedSize = 0n; + fileTotalSize = 0n; + return shard; + } + if (xorbViewOffset || fileViewOffset) { + await uploadShard(createShard(), params); + } +} +function writeHashToArray(hash2, array, offset) { + for (let i = 0; i < hash2.length; i += 16) { + array[offset + i / 2] = parseInt(hash2.substring(i + 2 * 7, i + 2 * 8), 16); + array[offset + i / 2 + 1] = parseInt(hash2.substring(i + 2 * 6, i + 2 * 7), 16); + array[offset + i / 2 + 2] = parseInt(hash2.substring(i + 2 * 5, i + 2 * 6), 16); + array[offset + i / 2 + 3] = parseInt(hash2.substring(i + 2 * 4, i + 2 * 5), 16); + array[offset + i / 2 + 4] = parseInt(hash2.substring(i + 2 * 3, i + 2 * 4), 16); + array[offset + i / 2 + 5] = parseInt(hash2.substring(i + 2 * 2, i + 2 * 3), 16); + array[offset + i / 2 + 6] = parseInt(hash2.substring(i + 2 * 1, i + 2 * 2), 16); + array[offset + i / 2 + 7] = parseInt(hash2.substring(i + 2 * 0, i + 2 * 1), 16); + } +} +async function uploadXorb(xorb, params) { + const token = await xetWriteToken(params); + const resp = await (params.fetch ?? fetch)(`${token.casUrl}/v1/xorbs/default/${xorb.hash}`, { + method: "POST", + body: xorb.xorb, + headers: { + Authorization: `Bearer ${token.accessToken}`, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + }, + ...{ + progressHint: { + progressCallback: (progress) => { + for (const file of xorb.files) { + params.yieldCallback?.({ + event: "fileProgress", + path: file.path, + progress: file.lastSentProgress + (file.progress - file.lastSentProgress) * progress + }); + } + } + } + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } +} +async function uploadShard(shard, params) { + const token = await xetWriteToken(params); + const resp = await (params.fetch ?? fetch)(`${token.casUrl}/v1/shards`, { + method: "POST", + body: shard, + headers: { + Authorization: `Bearer ${token.accessToken}`, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } +} + +// src/utils/splitAsyncGenerator.ts +function splitAsyncGenerator(source, n) { + if (n <= 0) { + return []; + } + const sleep = (ms) => new Promise((resolve2) => setTimeout(resolve2, ms)); + let takenIndex = null; + const generators = []; + let remaining = n; + for (let i = 0; i < n; i++) { + generators.push({ + next: async () => { + while (takenIndex !== null) { + await sleep(1); + } + takenIndex = i; + return source.next().then((r) => { + takenIndex = null; + return r; + }); + }, + return: async () => { + remaining--; + if (remaining === 0) { + return source.return(void 0); + } + return { + done: true, + value: void 0 + }; + }, + throw: async (error) => { + return source.throw(error); + }, + [Symbol.asyncIterator]: () => generators[i] + }); + } + return generators; +} + +// src/lib/commit.ts +var CONCURRENT_SHAS = 5; +var CONCURRENT_LFS_UPLOADS = 5; +var MULTIPART_PARALLEL_UPLOAD = 5; +function isFileOperation(op) { + const ret = op.operation === "addOrUpdate"; + if (ret && !(op.content instanceof Blob)) { + throw new TypeError("Precondition failed: op.content should be a Blob"); + } + return ret; +} +async function* commitIter(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + if (repoId.type === "bucket") { + return yield* commitIterBucket(params); + } + if (params.operations.some((op) => op.operation === "copy")) { + throw new Error("'copy' operations are only supported when the destination repo is a bucket"); + } + yield { event: "phase", phase: "preuploading" }; + let useXet = params.useXet ?? true; + const lfsShas = /* @__PURE__ */ new Map(); + const abortController = new AbortController(); + const abortSignal = abortController.signal; + if (!abortSignal.throwIfAborted) { + abortSignal.throwIfAborted = () => { + if (abortSignal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + }; + } + if (params.abortSignal) { + params.abortSignal.addEventListener("abort", () => abortController.abort()); + } + try { + const allOperations = (await Promise.all( + params.operations.map(async (operation) => { + if (operation.operation === "edit") { + const splicedBlob = SplicedBlob.create( + operation.originalContent, + operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })) + ); + return { + operation: "addOrUpdate", + path: operation.path, + content: splicedBlob + }; + } + if (operation.operation !== "addOrUpdate") { + return operation; + } + if (!(operation.content instanceof URL)) { + return { ...operation, content: operation.content }; + } + const lazyBlobs = await createBlobs(operation.content, operation.path, { + fetch: params.fetch, + maxFolderDepth: params.maxFolderDepth + }); + abortSignal?.throwIfAborted(); + return lazyBlobs.map((blob) => ({ + ...operation, + content: blob.blob, + path: blob.path + })); + }) + )).flat(1); + const gitAttributes = allOperations.filter(isFileOperation).find((op) => op.path === ".gitattributes")?.content; + for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { + const payload = { + gitAttributes: gitAttributes && await gitAttributes.text(), + files: await Promise.all( + operations.map(async (operation) => ({ + path: operation.path, + size: operation.content.size, + sample: base64FromBytes(new Uint8Array(await operation.content.slice(0, 512).arrayBuffer())) + })) + ) + }; + abortSignal?.throwIfAborted(); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/preupload/${encodeURIComponent( + params.branch ?? "main" + )}` + (params.isPullRequest ? "?create_pr=1" : ""), + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/json" + }, + body: JSON.stringify(payload), + signal: abortSignal + } + ); + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + for (const file of json.files) { + if (file.uploadMode === "lfs") { + lfsShas.set(file.path, null); + } + } + } + yield { event: "phase", phase: "uploadingLargeFiles" }; + for (const operations of chunk( + allOperations.filter(isFileOperation).filter((op) => lfsShas.has(op.path)), + 100 + )) { + const shas = yield* eventToGenerator((yieldCallback, returnCallback, rejectCallack) => { + return promisesQueue( + operations.map((op) => async () => { + const iterator = sha256(op.content, { useWebWorker: params.useWebWorkers, abortSignal }); + let res2; + do { + res2 = await iterator.next(); + if (!res2.done) { + yieldCallback({ event: "fileProgress", path: op.path, progress: res2.value, state: "hashing" }); + } + } while (!res2.done); + const sha = res2.value; + lfsShas.set(op.path, res2.value); + return sha; + }), + CONCURRENT_SHAS + ).then(returnCallback, rejectCallack); + }); + abortSignal?.throwIfAborted(); + const payload = { + operation: "upload", + // multipart is a custom protocol for HF + transfers: ["basic", "multipart", ...useXet ? ["xet"] : []], + hash_algo: "sha_256", + ...!params.isPullRequest && { + ref: { + name: params.branch ?? "main" + } + }, + objects: operations.map((op, i) => ({ + oid: shas[i], + size: op.content.size + })) + }; + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/${repoId.type === "model" ? "" : repoId.type + "s/"}${repoId.name}.git/info/lfs/objects/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + Accept: "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json" + }, + body: JSON.stringify(payload), + signal: abortSignal + } + ); + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + const batchRequestId = res.headers.get("X-Request-Id") || void 0; + const shaToOperation = new Map(operations.map((op, i) => [shas[i], op])); + if (useXet && json.transfer !== "xet") { + useXet = false; + } + let xetParams = null; + if (useXet) { + for (const obj of json.objects) { + const op = shaToOperation.get(obj.oid); + if (!op) { + throw new InvalidApiResponseFormatError("Unrequested object ID in response"); + } + if (obj.error) { + const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${obj.error.message}${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; + throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); + } + if (!obj.actions?.upload) { + yield { + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }; + } else { + const headers = new Headers(obj.actions.upload.header); + xetParams = { + sessionId: headers.get("X-Xet-Session-Id") ?? void 0, + casUrl: headers.get("X-Xet-Cas-Url") ?? void 0, + accessToken: headers.get("X-Xet-Access-Token") ?? void 0, + expiresAt: headers.get("X-Xet-Token-Expiration") ? new Date(parseInt(headers.get("X-Xet-Token-Expiration") ?? "0") * 1e3) : void 0, + refreshWriteTokenUrl: obj.actions.upload.href + }; + } + } + const source = async function* () { + for (const obj of json.objects) { + const op = shaToOperation.get(obj.oid); + if (!op || !obj.actions?.upload) { + continue; + } + abortSignal?.throwIfAborted(); + yield { content: op.content, path: op.path, sha256: obj.oid }; + } + }(); + if (xetParams) { + const fixedXetParams = xetParams; + const sources = splitAsyncGenerator(source, 5); + yield* eventToGenerator( + (yieldCallback, returnCallback, rejectCallback) => Promise.all( + sources.map(async function(source2) { + for await (const event of uploadShards(source2, { + fetch: params.fetch, + accessToken, + hubUrl: params.hubUrl ?? HUB_URL, + repo: repoId, + xetParams: fixedXetParams, + // todo: maybe leave empty if PR? + rev: params.branch ?? "main", + isPullRequest: params.isPullRequest, + yieldCallback: (event2) => yieldCallback({ ...event2, state: "uploading" }) + })) { + if (event.event === "file") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: 1, + state: "uploading" + }); + } else if (event.event === "fileProgress") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: event.progress, + state: "uploading" + }); + } + } + }) + ).then(() => returnCallback(void 0), rejectCallback) + ); + } else { + } + } else { + yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + return promisesQueueStreaming( + json.objects.map((obj) => async () => { + const op = shaToOperation.get(obj.oid); + if (!op) { + throw new InvalidApiResponseFormatError("Unrequested object ID in response"); + } + abortSignal?.throwIfAborted(); + if (obj.error) { + const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${obj.error.message}${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; + throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); + } + if (!obj.actions?.upload) { + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + return; + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 0, + state: "uploading" + }); + const content = op.content; + const header = obj.actions.upload.header; + if (header?.chunk_size) { + const chunkSize = parseInt(header.chunk_size); + const completionUrl = obj.actions.upload.href; + const parts = Object.keys(header).filter((key) => /^[0-9]+$/.test(key)); + if (parts.length !== Math.ceil(content.size / chunkSize)) { + throw new Error("Invalid server response to upload large LFS file, wrong number of parts"); + } + const completeReq = { + oid: obj.oid, + parts: parts.map((part) => ({ + partNumber: +part, + etag: "" + })) + }; + const progressCallback = (progress) => yieldCallback({ event: "fileProgress", path: op.path, progress, state: "uploading" }); + await promisesQueueStreaming( + parts.map((part) => async () => { + abortSignal?.throwIfAborted(); + const index = parseInt(part) - 1; + const slice = content.slice(index * chunkSize, (index + 1) * chunkSize); + const res3 = await (params.fetch ?? fetch)(header[part], { + method: "PUT", + /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ + body: slice instanceof WebBlob && isFrontend ? await slice.arrayBuffer() : slice, + signal: abortSignal, + ...{ + progressHint: { + path: op.path, + part: index, + numParts: parts.length, + progressCallback + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + }); + if (!res3.ok) { + throw await createApiError(res3, { + requestId: batchRequestId, + message: `Error while uploading part ${part} of ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + const eTag = res3.headers.get("ETag"); + if (!eTag) { + throw new Error("Cannot get ETag of part during multipart upload"); + } + completeReq.parts[Number(part) - 1].etag = eTag; + }), + MULTIPART_PARALLEL_UPLOAD + ); + abortSignal?.throwIfAborted(); + const res2 = await (params.fetch ?? fetch)(completionUrl, { + method: "POST", + body: JSON.stringify(completeReq), + headers: { + Accept: "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json" + }, + signal: abortSignal + }); + if (!res2.ok) { + throw await createApiError(res2, { + requestId: batchRequestId, + message: `Error completing multipart upload of ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + } else { + const res2 = await (params.fetch ?? fetch)(obj.actions.upload.href, { + method: "PUT", + headers: { + ...batchRequestId ? { "X-Request-Id": batchRequestId } : void 0 + }, + /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ + body: content instanceof WebBlob && isFrontend ? await content.arrayBuffer() : content, + signal: abortSignal, + ...{ + progressHint: { + path: op.path, + progressCallback: (progress) => yieldCallback({ + event: "fileProgress", + path: op.path, + progress, + state: "uploading" + }) + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + }); + if (!res2.ok) { + throw await createApiError(res2, { + requestId: batchRequestId, + message: `Error while uploading ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + } + }), + CONCURRENT_LFS_UPLOADS + ).then(returnCallback, rejectCallback); + }); + } + } + abortSignal?.throwIfAborted(); + yield { event: "phase", phase: "committing" }; + return yield* eventToGenerator( + async (yieldCallback, returnCallback, rejectCallback) => (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commit/${encodeURIComponent( + params.branch ?? "main" + )}` + (params.isPullRequest ? "?create_pr=1" : ""), + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: [ + { + key: "header", + value: { + summary: params.title, + description: params.description, + parentCommit: params.parentCommit + } + }, + ...await Promise.all( + allOperations.map((operation) => { + if (isFileOperation(operation)) { + const sha = lfsShas.get(operation.path); + if (sha) { + return { + key: "lfsFile", + value: { + path: operation.path, + algo: "sha256", + size: operation.content.size, + oid: sha + } + }; + } + } + return convertOperationToNdJson(operation); + }) + ) + ].map((x) => JSON.stringify(x)).join("\n"), + signal: abortSignal, + ...{ + progressHint: { + progressCallback: (progress) => { + for (const op of allOperations) { + if (isFileOperation(op) && !lfsShas.has(op.path)) { + yieldCallback({ + event: "fileProgress", + path: op.path, + progress, + state: "uploading" + }); + } + } + } + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + } + ).then(async (res) => { + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + returnCallback({ + pullRequestUrl: json.pullRequestUrl, + commit: { + oid: json.commitOid, + url: json.commitUrl + }, + hookOutput: json.hookOutput + }); + }).catch(rejectCallback) + ); + } catch (err) { + abortController.abort(); + throw err; + } +} +async function* commitIterBucket(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + if (params.useXet === false) { + throw new Error("useXet must be true or undefined for buckets"); + } + const abortController = new AbortController(); + const abortSignal = abortController.signal; + if (!abortSignal.throwIfAborted) { + abortSignal.throwIfAborted = () => { + if (abortSignal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + }; + } + if (params.abortSignal) { + params.abortSignal.addEventListener("abort", () => abortController.abort()); + } + try { + const allOperations = (await Promise.all( + params.operations.map(async (operation) => { + if (operation.operation === "edit") { + const splicedBlob = SplicedBlob.create( + operation.originalContent, + operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })) + ); + return { + operation: "addOrUpdate", + path: operation.path, + content: splicedBlob + }; + } + if (operation.operation !== "addOrUpdate") { + return operation; + } + if (!(operation.content instanceof URL)) { + return { ...operation, content: operation.content }; + } + const lazyBlobs = await createBlobs(operation.content, operation.path, { + fetch: params.fetch, + maxFolderDepth: params.maxFolderDepth + }); + abortSignal?.throwIfAborted(); + return lazyBlobs.map((blob) => ({ + ...operation, + content: blob.blob, + path: blob.path + })); + }) + )).flat(1); + yield { event: "phase", phase: "uploadingLargeFiles" }; + for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { + const xetHashes = /* @__PURE__ */ new Map(); + abortSignal?.throwIfAborted(); + const source = async function* () { + for (const operation of operations) { + abortSignal?.throwIfAborted(); + yield { content: operation.content, path: operation.path }; + } + }(); + const xetParams = { + sessionId: crypto.randomUUID(), + refreshWriteTokenUrl: `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/xet-write-token` + }; + const sources = splitAsyncGenerator(source, 5); + yield* eventToGenerator( + (yieldCallback, returnCallback, rejectCallback) => Promise.all( + sources.map(async function(source2) { + for await (const event of uploadShards(source2, { + fetch: params.fetch, + accessToken, + hubUrl: params.hubUrl ?? HUB_URL, + repo: repoId, + xetParams, + rev: params.branch ?? "main", + yieldCallback: (event2) => yieldCallback({ ...event2, state: "uploading" }) + })) { + if (event.event === "file") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: 1, + state: "uploading" + }); + xetHashes.set(event.path, event.xetHash); + } else if (event.event === "fileProgress") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: event.progress, + state: "uploading" + }); + } + } + }) + ).then(() => returnCallback(void 0), rejectCallback) + ); + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: [...xetHashes.entries()].map( + ([path, xetHash]) => JSON.stringify({ + type: "addFile", + path, + xetHash + }) + ).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok && resp.status !== 422) { + throw await createApiError(resp); + } + const json = await resp.json(); + for (const failed of json.failed) { + yield { + event: "fileProgress", + path: failed.path, + progress: 0, + state: "error" + }; + } + } + abortSignal?.throwIfAborted(); + const copyOperations = allOperations.filter( + (operation) => operation.operation === "copy" + ); + for (const copyChunk of chunk(copyOperations, 100)) { + abortSignal?.throwIfAborted(); + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: copyChunk.map((op) => { + const sourceRepoId = toRepoId(op.sourceRepo); + return JSON.stringify({ + type: "copyFile", + path: op.path, + xetHash: op.sourceXetHash, + sourceRepoType: sourceRepoId.type, + sourceRepoId: sourceRepoId.name + }); + }).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok && resp.status !== 422) { + throw await createApiError(resp); + } + const json = await resp.json(); + for (const failed of json.failed) { + yield { + event: "fileProgress", + path: failed.path, + progress: 0, + state: "error" + }; + } + } + abortSignal?.throwIfAborted(); + const deletedOperations = allOperations.filter((operation) => operation.operation === "delete"); + if (deletedOperations.length > 0) { + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: deletedOperations.map( + (operation) => JSON.stringify({ + type: "deleteFile", + path: operation.path + }) + ).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok) { + throw await createApiError(resp); + } + const json = await resp.json(); + if (json.failed.length > 0) { + const failedPaths = json.failed.slice(0, 5).map((f) => f.path); + throw new Error( + `Failed to delete ${json.failed.length} file(s): ${failedPaths.join(", ")}${json.failed.length > 5 ? "..." : ""}, request ID: ${resp.headers.get("X-Request-Id")}` + ); + } + } + abortSignal?.throwIfAborted(); + } catch (err) { + abortController.abort(); + throw err; + } +} +async function commit(params) { + const iterator = commitIter(params); + const failedPaths = []; + let failedCount = 0; + let res = await iterator.next(); + while (!res.done) { + if (res.value.event === "fileProgress" && res.value.state === "error") { + failedCount++; + if (failedPaths.length < 5) { + failedPaths.push(res.value.path); + } + } + res = await iterator.next(); + } + if (failedCount > 0) { + throw new Error( + `Failed to upload ${failedCount} file(s): ${failedPaths.join(", ")}${failedCount > 5 ? "..." : ""}` + ); + } + return res.value; +} +async function convertOperationToNdJson(operation) { + switch (operation.operation) { + case "addOrUpdate": { + return { + key: "file", + value: { + content: base64FromBytes(new Uint8Array(await operation.content.arrayBuffer())), + path: operation.path, + encoding: "base64" + } + }; + } + case "delete": { + return { + key: "deletedFile", + value: { + path: operation.path + } + }; + } + case "edit": { + throw new Error( + "Edit operations should be converted to addOrUpdate operations before reaching convertOperationToNdJson" + ); + } + default: + throw new TypeError("Unknown operation: " + operation.operation); + } +} + +// src/utils/formatBytes.ts +function formatBytes(bytes) { + if (!Number.isFinite(bytes) || bytes < 0) { + return `${bytes} B`; + } + const units = ["B", "kB", "MB", "GB", "TB", "PB"]; + let value = bytes; + let i = 0; + while (value >= 1e3 && i < units.length - 1) { + value /= 1e3; + i++; + } + const formatted = i === 0 ? value.toString() : value.toFixed(value >= 100 ? 0 : value >= 10 ? 1 : 2); + return `${formatted} ${units[i]}`; +} + +// src/utils/parseLinkHeader.ts +function parseLinkHeader(header) { + const regex = /<(https?:[/][/][^>]+)>;\s+rel="([^"]+)"/g; + return Object.fromEntries([...header.matchAll(regex)].map(([, url, rel]) => [rel, url])); +} + +// src/lib/file-download-info.ts +async function fileDownloadInfo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const hubUrl = params.hubUrl ?? HUB_URL; + const revision = repoId.type === "bucket" ? void 0 : params.revision ?? "main"; + const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${params.raw ? "raw" : "resolve"}${revision ? `/${encodeURIComponent(revision)}` : ""}/${params.path}` + (params.noContentDisposition ? "?noContentDisposition=1" : ""); + const resp = await (params.fetch ?? fetch)(url, { + method: "GET", + headers: { + ...accessToken && { + Authorization: `Bearer ${accessToken}` + }, + Range: "bytes=0-0", + Accept: "application/vnd.xet-fileinfo+json, */*" + } + }); + if (resp.status === 404 && resp.headers.get("X-Error-Code") === "EntryNotFound") { + return null; + } + if (!resp.ok) { + throw await createApiError(resp); + } + let size; + let xetInfo; + if (resp.headers.get("Content-Type")?.includes("application/vnd.xet-fileinfo+json")) { + size = parseInt(resp.headers.get("X-Linked-Size") ?? "invalid"); + if (isNaN(size)) { + throw new InvalidApiResponseFormatError("Invalid file size received in X-Linked-Size header"); + } + const hash2 = resp.headers.get("X-Xet-Hash"); + const links = parseLinkHeader(resp.headers.get("Link") ?? ""); + const reconstructionUrl = (() => { + try { + return new URL(links["xet-reconstruction-info"]); + } catch { + return null; + } + })(); + const refreshUrl = (() => { + try { + return new URL(links["xet-auth"]); + } catch { + return null; + } + })(); + if (!hash2) { + throw new InvalidApiResponseFormatError("No hash received in X-Xet-Hash header"); + } + if (!reconstructionUrl || !refreshUrl) { + throw new InvalidApiResponseFormatError("No xet-reconstruction-info or xet-auth link header"); + } + xetInfo = { + hash: hash2, + refreshUrl, + reconstructionUrl + }; + } + if (size === void 0 || isNaN(size)) { + const contentRangeHeader = resp.headers.get("content-range"); + if (!contentRangeHeader) { + throw new InvalidApiResponseFormatError("Expected size information"); + } + const [, parsedSize] = contentRangeHeader.split("/"); + size = parseInt(parsedSize); + if (isNaN(size)) { + throw new InvalidApiResponseFormatError("Invalid file size received"); + } + } + const etag = resp.headers.get("X-Linked-ETag") ?? resp.headers.get("ETag") ?? void 0; + if (!etag) { + throw new InvalidApiResponseFormatError("Expected ETag"); + } + return { + etag, + size, + xet: xetInfo, + // Cannot use resp.url in case it's a S3 url and the user adds an Authorization header to it. + url: resp.url && (new URL(resp.url).origin === new URL(hubUrl).origin || resp.headers.get("X-Cache")?.endsWith(" cloudfront")) ? resp.url : url + }; +} + +// src/lib/download-file.ts +async function downloadFile(params) { + const accessToken = checkCredentials(params); + const info = params.downloadInfo ?? await fileDownloadInfo({ + accessToken, + repo: params.repo, + path: params.path, + revision: params.revision, + hubUrl: params.hubUrl, + fetch: params.fetch, + raw: params.raw + }); + if (!info) { + return null; + } + if (info.xet && params.xet !== false) { + return new XetBlob({ + refreshUrl: info.xet.refreshUrl.href, + reconstructionUrl: info.xet.reconstructionUrl.href, + fetch: params.fetch, + accessToken, + size: info.size, + readToken: typeof params.xet === "object" ? params.xet.readToken : void 0 + }); + } + return new WebBlob(new URL(info.url), 0, info.size, "", true, params.fetch ?? fetch, accessToken); +} + +// src/lib/list-files.ts +async function* listFiles(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const revision = repoId.type === "bucket" ? void 0 : params.revision || "main"; + let url = `${params.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}/tree${revision ? `/${revision}` : ""}${params.path ? "/" + params.path : ""}?recursive=${!!params.recursive}&expand=${!!params.expand}`; + while (url) { + const res = await (params.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + yield item; + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/paths-info.ts +async function pathsInfo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const hubUrl = params.hubUrl ?? HUB_URL; + const revision = repoId.type === "bucket" ? void 0 : params.revision ?? "main"; + const url = `${hubUrl}/api/${repoId.type}s/${repoId.name}/paths-info${revision ? `/${encodeURIComponent(revision)}` : ""}`; + const resp = await (params.fetch ?? fetch)(url, { + method: "POST", + headers: { + ...accessToken && { + Authorization: `Bearer ${accessToken}` + }, + Accept: "application/json", + "Content-Type": "application/json" + }, + body: JSON.stringify({ + paths: params.paths, + expand: params.expand + }) + }); + if (!resp.ok) { + throw await createApiError(resp); + } + const json = await resp.json(); + if (!Array.isArray(json)) { + throw new Error("malformed response: expected array"); + } + return json.map((item) => ({ + path: item.path, + lfs: item.lfs, + type: item.type, + oid: item.oid, + size: item.size, + xetHash: item.xetHash, + uploadedAt: item.uploadedAt, + securityFileStatus: item.securityFileStatus, + lastCommit: item.lastCommit ? { + date: new Date(item.lastCommit.date), + title: item.lastCommit.title, + id: item.lastCommit.id + } : void 0 + })); +} + +// src/lib/copy-files.ts +var DOWNLOAD_CONCURRENCY = 5; +var PATHS_INFO_BATCH_SIZE = 100; +var MAX_REPORTED_LFS_PATHS = 5; +function copyFile(params) { + return copyFiles({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + destination: params.destination.repo, + files: [ + { + source: params.source, + destinationPath: params.destination.path + } + ], + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); +} +function copyFileIter(params) { + return copyFilesIter({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + destination: params.destination.repo, + files: [ + { + source: params.source, + destinationPath: params.destination.path + } + ], + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); +} +async function copyFiles(params) { + const iterator = copyFilesIter(params); + while (true) { + const res = await iterator.next(); + if (res.done) { + return void 0; + } + } +} +async function* copyFilesIter(params) { + if (params.files.length === 0) { + return void 0; + } + const operations = yield* resolveCopyOperationsIter(params, params.files); + await commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.destination, + operations, + title: "", + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); + return void 0; +} +async function copyFolder(params) { + const iterator = copyFolderIter(params); + while (true) { + const res = await iterator.next(); + if (res.done) { + return void 0; + } + } +} +async function* copyFolderIter(params) { + const accessToken = checkCredentials(params); + const sourceRepoId = toRepoId(params.source.repo); + const sourcePath = (params.source.path ?? "").replace(/\/+$/, ""); + const destinationPrefix = (params.destination.path ?? "").replace(/\/+$/, ""); + const sourceRevision = sourceRepoId.type === "bucket" ? void 0 : params.source.revision ?? "main"; + const operations = []; + const pendingDownloads = []; + const lfsOffenders = []; + for await (const item of listFiles({ + repo: sourceRepoId, + path: sourcePath || void 0, + recursive: true, + revision: sourceRevision, + accessToken, + hubUrl: params.hubUrl, + fetch: params.fetch + })) { + if (item.type !== "file") { + continue; + } + const relPath = relativeUnderFolder(item.path, sourcePath); + const destPath = destinationPrefix ? `${destinationPrefix}/${relPath}` : relPath; + switch (classifySourceFile(item)) { + case "copy": + operations.push({ + operation: "copy", + path: destPath, + sourceXetHash: item.xetHash, + sourceRepo: sourceRepoId + }); + continue; + case "lfs": + lfsOffenders.push({ path: item.path, size: item.lfs?.size ?? item.size }); + continue; + case "download": + pendingDownloads.push({ + index: operations.length, + repoId: sourceRepoId, + revision: sourceRevision, + sourcePath: item.path + }); + operations.push({ + operation: "addOrUpdate", + path: destPath, + content: new Blob([]) + }); + continue; + } + } + if (lfsOffenders.length > 0) { + throwUnmigratedLfsError(sourceRepoId, lfsOffenders); + } + if (operations.length === 0) { + return void 0; + } + yield* downloadAndFillBlobsIter({ + pendingDownloads, + operations, + accessToken, + hubUrl: params.hubUrl, + fetch: params.fetch + }); + await commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.destination.repo, + operations, + title: "", + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); + return void 0; +} +async function* resolveCopyOperationsIter(shared, files) { + const accessToken = checkCredentials(shared); + const groups = /* @__PURE__ */ new Map(); + for (let i = 0; i < files.length; i++) { + const file = files[i]; + const repoId = toRepoId(file.source.repo); + const revision = repoId.type === "bucket" ? void 0 : file.source.revision ?? "main"; + const key = `${repoId.type}\0${repoId.name}\0${revision ?? ""}`; + let group = groups.get(key); + if (!group) { + group = { repoId, revision, entries: [] }; + groups.set(key, group); + } + group.entries.push({ index: i, file }); + } + const operations = new Array(files.length); + const pendingDownloads = []; + for (const group of groups.values()) { + const paths = group.entries.map((e) => e.file.source.path); + const infos = []; + for (let offset = 0; offset < paths.length; offset += PATHS_INFO_BATCH_SIZE) { + const slice = paths.slice(offset, offset + PATHS_INFO_BATCH_SIZE); + const res = await pathsInfo({ + repo: group.repoId, + paths: slice, + revision: group.revision, + accessToken, + hubUrl: shared.hubUrl, + fetch: shared.fetch + }); + infos.push(...res); + } + const infoByPath = new Map(infos.map((i) => [i.path, i])); + const lfsOffenders = []; + for (const { index, file } of group.entries) { + const info = infoByPath.get(file.source.path); + if (!info) { + throw new Error(`Source file not found: '${file.source.path}' in ${group.repoId.type}s/${group.repoId.name}`); + } + if (info.type !== "file") { + throw new Error( + `Source path '${file.source.path}' in ${group.repoId.type}s/${group.repoId.name} is a folder; use copyFolder() instead.` + ); + } + switch (classifySourceFile(info)) { + case "copy": + operations[index] = { + operation: "copy", + path: file.destinationPath, + sourceXetHash: info.xetHash, + sourceRepo: group.repoId + }; + continue; + case "lfs": + lfsOffenders.push({ path: file.source.path, size: info.lfs?.size ?? info.size }); + continue; + case "download": + pendingDownloads.push({ + index, + repoId: group.repoId, + revision: group.revision, + sourcePath: file.source.path + }); + operations[index] = { + operation: "addOrUpdate", + path: file.destinationPath, + content: new Blob([]) + }; + continue; + } + } + if (lfsOffenders.length > 0) { + throwUnmigratedLfsError(group.repoId, lfsOffenders); + } + } + yield* downloadAndFillBlobsIter({ + pendingDownloads, + operations, + accessToken, + hubUrl: shared.hubUrl, + fetch: shared.fetch + }); + return operations; +} +function downloadAndFillBlobsIter(args) { + const total = args.pendingDownloads.length; + return eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + if (total === 0) { + returnCallback(); + return; + } + let downloaded = 0; + promisesQueue( + args.pendingDownloads.map(({ index, repoId, revision, sourcePath }) => async () => { + const blob = await downloadFile({ + repo: repoId, + path: sourcePath, + revision, + accessToken: args.accessToken, + hubUrl: args.hubUrl, + fetch: args.fetch + }); + if (!blob) { + throw new Error(`Failed to download '${sourcePath}' from ${repoId.type}s/${repoId.name}`); + } + const op = args.operations[index]; + if (op.operation !== "addOrUpdate") { + throw new Error("Internal: expected addOrUpdate placeholder operation"); + } + op.content = blob; + downloaded++; + yieldCallback({ event: "fileDownloaded", path: sourcePath, downloaded, total }); + }), + DOWNLOAD_CONCURRENCY + ).then( + () => returnCallback(), + (err) => rejectCallback(err) + ); + }); +} +function relativeUnderFolder(filePath, folderPath) { + if (!folderPath) { + return filePath; + } + if (filePath === folderPath) { + return filePath.split("/").pop() ?? filePath; + } + if (filePath.startsWith(folderPath + "/")) { + return filePath.slice(folderPath.length + 1); + } + throw new Error(`Path '${filePath}' is not inside folder '${folderPath}'`); +} +function classifySourceFile(file) { + if (file.xetHash) { + return "copy"; + } + if (file.lfs) { + return "lfs"; + } + return "download"; +} +function throwUnmigratedLfsError(repoId, entries) { + const head = entries.slice(0, MAX_REPORTED_LFS_PATHS).map((e) => `'${e.path}' (${formatBytes(e.size)})`).join(", "); + const more = entries.length > MAX_REPORTED_LFS_PATHS ? ` (and ${entries.length - MAX_REPORTED_LFS_PATHS} more)` : ""; + throw new Error( + `Cannot copy ${entries.length} LFS file(s) from ${repoId.type}s/${repoId.name} that have not been migrated to xet: ${head}${more}. Migrate these files to xet before copying.` + ); +} + +// src/lib/count-commits.ts +async function countCommits(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const url = `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commits/${params.revision ?? "main"}?limit=1`; + const res = await (params.fetch ?? fetch)(url, { + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }); + if (!res.ok) { + throw await createApiError(res); + } + return parseInt(res.headers.get("x-total-count") ?? "0", 10); +} + +// src/lib/create-repo.ts +async function createRepo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const [namespace, repoName] = repoId.name.split("/"); + const visibility = params.visibility ?? (params.private !== void 0 ? params.private ? "private" : "public" : void 0); + if (!namespace || !repoName) { + throw new TypeError( + `"${repoId.name}" is not a fully qualified repo name. It should be of the form "{namespace}/{repoName}".` + ); + } + const res = repoId.type === "bucket" ? await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/buckets/${namespace}/${repoName}`, { + method: "POST", + body: JSON.stringify({ + visibility, + resourceGroupId: params.resourceGroupId + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }) : await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/repos/create`, { + method: "POST", + body: JSON.stringify({ + name: repoName, + visibility, + organization: namespace, + resourceGroupId: params.resourceGroupId, + license: params.license, + ...repoId.type === "space" ? { + type: "space", + sdk: params.sdk ?? "static" + } : { + type: repoId.type + }, + files: params.files ? await Promise.all( + params.files.map(async (file) => ({ + encoding: "base64", + path: file.path, + content: base64FromBytes( + new Uint8Array(file.content instanceof Blob ? await file.content.arrayBuffer() : file.content) + ) + })) + ) : void 0 + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const output = await res.json(); + return { repoUrl: output.url, id: output.id }; +} + +// src/lib/create-branch.ts +async function createBranch(params) { + const repoId = toRepoId(params.repo); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + }, + body: JSON.stringify({ + startingPoint: params.revision, + ...params.empty && { emptyBranch: true }, + overwrite: params.overwrite + }) + } + ); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/create-collection.ts +async function createCollection(params) { + const accessToken = checkCredentials(params); + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/collections`, { + method: "POST", + body: JSON.stringify(params.collection), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const output = await res.json(); + return { slug: output.slug }; +} + +// src/utils/pick.ts +function pick(o, props) { + return Object.assign( + {}, + ...props.map((prop) => { + if (o[prop] !== void 0) { + return { [prop]: o[prop] }; + } + }) + ); +} + +// src/lib/list-datasets.ts +var DATASET_EXPAND_KEYS = [ + "private", + "downloads", + "gated", + "likes", + "lastModified" +]; +var DATASET_EXPANDABLE_KEYS = [ + "author", + "cardData", + "citation", + "createdAt", + "disabled", + "description", + "downloads", + "downloadsAllTime", + "gated", + "gitalyUid", + "lastModified", + "likes", + "paperswithcode_id", + "private", + // "siblings", + "sha", + "tags" +]; +async function* listDatasets(params) { + const accessToken = params && checkCredentials(params); + let totalToFetch = params?.limit ?? Infinity; + const search = new URLSearchParams([ + ...Object.entries({ + limit: String(Math.min(totalToFetch, 500)), + ...params?.search?.owner ? { author: params.search.owner } : void 0, + ...params?.search?.query ? { search: params.search.query } : void 0, + ...params?.sort ? { sort: params.sort } : void 0 + }), + ...params?.search?.tags?.map((tag) => ["filter", tag]) ?? [], + ...DATASET_EXPAND_KEYS.map((val) => ["expand", val]), + ...params?.additionalFields?.map((val) => ["expand", val]) ?? [] + ]).toString(); + let url = `${params?.hubUrl || HUB_URL}/api/datasets` + (search ? "?" + search : ""); + while (url) { + const res = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + yield { + ...params?.additionalFields && pick(item, params.additionalFields), + id: item._id, + name: item.id, + private: item.private, + downloads: item.downloads, + likes: item.likes, + gated: item.gated, + updatedAt: new Date(item.lastModified) + }; + totalToFetch--; + if (totalToFetch <= 0) { + return; + } + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/dataset-info.ts +async function datasetInfo(params) { + const accessToken = params && checkCredentials(params); + const search = new URLSearchParams([ + ...DATASET_EXPAND_KEYS.map((val) => ["expand", val]), + ...params?.additionalFields?.map((val) => ["expand", val]) ?? [] + ]).toString(); + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/datasets/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD" + )}?${search.toString()}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + const data = await response.json(); + return { + ...params?.additionalFields && pick(data, params.additionalFields), + id: data._id, + name: data.id, + private: data.private, + downloads: data.downloads, + likes: data.likes, + gated: data.gated, + updatedAt: new Date(data.lastModified) + }; +} + +// src/lib/delete-branch.ts +async function deleteBranch(params) { + const repoId = toRepoId(params.repo); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, + { + method: "DELETE", + headers: { + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + } + } + ); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/delete-file.ts +function deleteFile(params) { + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: [ + { + operation: "delete", + path: params.path + } + ], + title: params.commitTitle ?? `Delete ${params.path}`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch + }); +} + +// src/lib/delete-files.ts +function deleteFiles(params) { + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: params.paths.map((path) => ({ + operation: "delete", + path + })), + title: params.commitTitle ?? `Deletes ${params.paths.length} files`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch + }); +} + +// src/lib/delete-repo.ts +async function deleteRepo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const [namespace, repoName] = repoId.name.split("/"); + const res = repoId.type === "bucket" ? await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/buckets/${namespace}/${repoName}`, { + method: "DELETE", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }) : await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/repos/delete`, { + method: "DELETE", + body: JSON.stringify({ + name: repoName, + organization: namespace, + type: repoId.type + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/delete-collection.ts +async function deleteCollection(params) { + if (!params.slug) { + throw new TypeError("slug is required"); + } + const accessToken = checkCredentials(params); + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/collections/${params.slug}`, { + method: "DELETE", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/file-exists.ts +async function fileExists(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const hubUrl = params.hubUrl ?? HUB_URL; + const revision = repoId.type === "bucket" ? void 0 : params.revision ?? "main"; + const endpoint = repoId.type === "bucket" ? "resolve" : "raw"; + const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${endpoint}${revision ? `/${encodeURIComponent(revision)}` : ""}/${params.path}`; + const resp = await (params.fetch ?? fetch)(url, { + method: "HEAD", + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }); + if (resp.status === 404) { + return false; + } + if (!resp.ok) { + throw await createApiError(resp); + } + return true; +} + +// src/lib/jobs/cancel-job.ts +async function cancelJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/cancel`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/create-scheduled-job.ts +async function createScheduledJob(params) { + const accessToken = checkCredentials(params); + const { namespace, hubUrl, fetch: customFetch, ...rest } = params; + if (!rest.jobSpec.dockerImage && !rest.jobSpec.spaceId) { + throw new Error("Either dockerImage or spaceId must be provided in jobSpec"); + } + if (rest.jobSpec.dockerImage && rest.jobSpec.spaceId) { + throw new Error("Cannot provide both dockerImage and spaceId in jobSpec"); + } + const body = { + jobSpec: { + flavor: rest.jobSpec.flavor + }, + schedule: rest.schedule, + suspend: rest.suspend ?? false, + concurrency: rest.concurrency ?? false + }; + if (rest.jobSpec.dockerImage) { + body.jobSpec.dockerImage = rest.jobSpec.dockerImage; + } + if (rest.jobSpec.spaceId) { + body.jobSpec.spaceId = rest.jobSpec.spaceId; + } + if (rest.jobSpec.command) { + body.jobSpec.command = rest.jobSpec.command; + } + body.jobSpec.environment = rest.jobSpec.environment || {}; + if (rest.jobSpec.secrets) { + body.jobSpec.secrets = rest.jobSpec.secrets; + } + if (rest.jobSpec.arch) { + body.jobSpec.arch = rest.jobSpec.arch; + } + if (rest.jobSpec.timeoutSeconds !== void 0) { + body.jobSpec.timeoutSeconds = rest.jobSpec.timeoutSeconds; + } + if (rest.jobSpec.attempts !== void 0) { + body.jobSpec.attempts = rest.jobSpec.attempts; + } + if (rest.jobSpec.labels) { + body.jobSpec.labels = rest.jobSpec.labels; + } + if (rest.jobSpec.volumes?.length) { + body.jobSpec.volumes = rest.jobSpec.volumes.map(({ source, ...rest2 }) => { + const repoId = toRepoId(source); + return { type: repoId.type, source: repoId.name, ...rest2 }; + }); + } + const response = await (customFetch || fetch)(`${hubUrl || HUB_URL}/api/scheduled-jobs/${namespace}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }, + body: JSON.stringify(body) + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/delete-scheduled-job.ts +async function deleteScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}`, + { + method: "DELETE", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/lib/jobs/duplicate-job.ts +async function duplicateJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/duplicate`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/get-job.ts +async function getJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/get-scheduled-job.ts +async function getScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/list-job-hardware.ts +async function listJobHardware(params) { + const accessToken = checkCredentials(params ?? {}); + const headers = {}; + if (accessToken) { + headers.Authorization = `Bearer ${accessToken}`; + } + const response = await (params?.fetch || fetch)(`${params?.hubUrl || HUB_URL}/api/jobs/hardware`, { + headers + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/list-jobs.ts +async function listJobs(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/list-scheduled-jobs.ts +async function listScheduledJobs(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}`, { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/resume-scheduled-job.ts +async function resumeScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/resume`, + { + method: "POST", + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/lib/jobs/run-job.ts +async function runJob(params) { + const accessToken = checkCredentials(params); + if (!params.dockerImage && !params.spaceId) { + throw new Error("Either dockerImage or spaceId must be provided"); + } + if (params.dockerImage && params.spaceId) { + throw new Error("Cannot provide both dockerImage and spaceId"); + } + const body = { + flavor: params.flavor, + environment: params.environment || {} + }; + if (params.dockerImage) { + body.dockerImage = params.dockerImage; + } + if (params.spaceId) { + body.spaceId = params.spaceId; + } + if (params.command) { + body.command = params.command; + } + if (params.arguments) { + body.arguments = params.arguments; + } + if (params.secrets) { + body.secrets = params.secrets; + } + if (params.arch) { + body.arch = params.arch; + } + if (params.timeoutSeconds !== void 0) { + body.timeoutSeconds = params.timeoutSeconds; + } + if (params.attempts !== void 0) { + body.attempts = params.attempts; + } + if (params.labels) { + body.labels = params.labels; + } + if (params.volumes?.length) { + body.volumes = params.volumes.map(({ source, ...rest }) => { + const repoId = toRepoId(source); + return { type: repoId.type, source: repoId.name, ...rest }; + }); + } + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }, + body: JSON.stringify(body) + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/run-scheduled-job.ts +async function runScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/run`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + if (response.status === 409) { + return null; + } + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/stream-job-events.ts +async function* streamJobEvents(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/events`, + { + headers: { + Accept: "text/event-stream", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + if (!response.body) { + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + } finally { + reader.releaseLock(); + } +} + +// src/lib/jobs/stream-job-logs.ts +async function* streamJobLogs(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/logs`, + { + headers: { + Accept: "text/event-stream", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + if (!response.body) { + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const data = JSON.parse(line.slice(6)); + yield { message: data.data, timestamp: new Date(data.timestamp) }; + } catch { + yield { message: line.slice(6), timestamp: /* @__PURE__ */ new Date() }; + } + } + } + } + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const data = JSON.parse(line.slice(6)); + yield { message: data.data, timestamp: new Date(data.timestamp) }; + } catch { + yield { message: line.slice(6), timestamp: /* @__PURE__ */ new Date() }; + } + } + } + } + } finally { + reader.releaseLock(); + } +} + +// src/lib/jobs/stream-job-metrics.ts +async function* streamJobMetrics(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/metrics`, + { + headers: { + Accept: "text/event-stream", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + if (!response.body) { + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + } finally { + reader.releaseLock(); + } +} + +// src/lib/jobs/suspend-scheduled-job.ts +async function suspendScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/suspend`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/lib/list-commits.ts +async function* listCommits(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + let url = `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commits/${params.revision ?? "main"}?limit=${params.batchSize ?? 100}`; + while (url) { + const res = await (params.fetch ?? fetch)(url, { + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }); + if (!res.ok) { + throw await createApiError(res); + } + const resJson = await res.json(); + for (const commit2 of resJson) { + yield { + oid: commit2.id, + title: commit2.title, + message: commit2.message, + authors: commit2.authors.map((author) => ({ + username: author.user, + avatarUrl: author.avatar + })), + date: new Date(commit2.date) + }; + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/utils/normalizeInferenceProviderMapping.ts +function normalizeInferenceProviderMapping(hfModelId, inferenceProviderMapping) { + if (!inferenceProviderMapping) { + return []; + } + if (Array.isArray(inferenceProviderMapping)) { + return inferenceProviderMapping.map((entry) => ({ + ...entry, + hfModelId + })); + } + return Object.entries(inferenceProviderMapping).map(([provider, mapping]) => ({ + provider, + hfModelId, + providerId: mapping.providerId, + status: mapping.status, + task: mapping.task + })); +} + +// src/lib/list-models.ts +var MODEL_EXPAND_KEYS = [ + "pipeline_tag", + "private", + "gated", + "downloads", + "likes", + "lastModified" +]; +var MODEL_EXPANDABLE_KEYS = [ + "author", + "cardData", + "config", + "createdAt", + "disabled", + "downloads", + "downloadsAllTime", + "gated", + "gitalyUid", + "inferenceProviderMapping", + "lastModified", + "library_name", + "likes", + "model-index", + "pipeline_tag", + "private", + "safetensors", + "sha", + "spaces", + "tags", + "transformersInfo" +]; +var MODEL_DERIVED_FIELD_TO_API_KEY = { + filePaths: "siblings" +}; +async function* listModels(params) { + const accessToken = params && checkCredentials(params); + let totalToFetch = params?.limit ?? Infinity; + const additionalExpandKeys = params?.additionalFields?.map( + (field) => MODEL_DERIVED_FIELD_TO_API_KEY[field] ?? field + ) ?? []; + const search = new URLSearchParams([ + ...Object.entries({ + limit: String(Math.min(totalToFetch, 500)), + ...params?.search?.owner ? { author: params.search.owner } : void 0, + ...params?.search?.task ? { pipeline_tag: params.search.task } : void 0, + ...params?.search?.query ? { search: params.search.query } : void 0, + ...params?.search?.inferenceProviders ? { inference_provider: params.search.inferenceProviders.join(",") } : void 0, + ...params?.search?.apps ? { apps: params.search.apps.join(",") } : void 0, + ...params?.sort ? { sort: params.sort } : void 0 + }), + ...params?.search?.tags?.map((tag) => ["filter", tag]) ?? [], + ...MODEL_EXPAND_KEYS.map((val) => ["expand", val]), + ...additionalExpandKeys.map((val) => ["expand", val]) + ]).toString(); + let url = `${params?.hubUrl || HUB_URL}/api/models?${search}`; + while (url) { + const res = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + const additional = {}; + if (params?.additionalFields) { + for (const field of params.additionalFields) { + if (field === "filePaths") { + additional.filePaths = (item.siblings ?? []).map((s) => s.rfilename); + } else if (field === "inferenceProviderMapping" && item.inferenceProviderMapping) { + additional.inferenceProviderMapping = normalizeInferenceProviderMapping( + item.id, + item.inferenceProviderMapping + ); + } else { + additional[field] = item[field]; + } + } + } + yield { + ...additional, + id: item._id, + name: item.id, + private: item.private, + task: item.pipeline_tag, + downloads: item.downloads, + gated: item.gated, + likes: item.likes, + updatedAt: new Date(item.lastModified) + }; + totalToFetch--; + if (totalToFetch <= 0) { + return; + } + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/list-spaces.ts +var SPACE_EXPAND_KEYS = [ + "sdk", + "likes", + "private", + "lastModified" +]; +var SPACE_EXPANDABLE_KEYS = [ + "author", + "cardData", + "datasets", + "disabled", + "gitalyUid", + "lastModified", + "createdAt", + "likes", + "private", + "runtime", + "sdk", + // "siblings", + "sha", + "subdomain", + "tags", + "models" +]; +async function* listSpaces(params) { + const accessToken = params && checkCredentials(params); + const search = new URLSearchParams([ + ...Object.entries({ + limit: "500", + ...params?.search?.owner ? { author: params.search.owner } : void 0, + ...params?.search?.query ? { search: params.search.query } : void 0, + ...params?.sort ? { sort: params.sort } : void 0 + }), + ...params?.search?.tags?.map((tag) => ["filter", tag]) ?? [], + ...[...SPACE_EXPAND_KEYS, ...params?.additionalFields ?? []].map( + (val) => ["expand", val] + ) + ]).toString(); + let url = `${params?.hubUrl || HUB_URL}/api/spaces?${search}`; + while (url) { + const res = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + yield { + ...params?.additionalFields && pick(item, params.additionalFields), + id: item._id, + name: item.id, + sdk: item.sdk, + likes: item.likes, + private: item.private, + updatedAt: new Date(item.lastModified) + }; + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/list-collections.ts +async function* listCollections(params) { + const accessToken = params && checkCredentials(params); + const searchParams = new URLSearchParams(); + let totalToFetch = params?.limit ?? Infinity; + searchParams.append("limit", String(Math.min(totalToFetch, 100))); + if (params?.sort) { + searchParams.append("sort", params.sort); + } + if (params?.search?.owner) { + for (const owner of params.search.owner) { + searchParams.append("owner", owner); + } + } + if (params?.search?.item) { + for (const item of params.search.item) { + searchParams.append("item", item); + } + } + if (params?.search?.q) { + searchParams.append("q", params.search.q); + } + let url = `${params?.hubUrl || HUB_URL}/api/collections?${searchParams}`; + while (url) { + const res = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const collections = await res.json(); + for (const collection of collections) { + yield collection; + totalToFetch--; + if (totalToFetch <= 0) { + return; + } + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/model-info.ts +async function modelInfo(params) { + const accessToken = params && checkCredentials(params); + const additionalExpandKeys = params?.additionalFields?.map( + (field) => MODEL_DERIVED_FIELD_TO_API_KEY[field] ?? field + ) ?? []; + const search = new URLSearchParams([ + ...MODEL_EXPAND_KEYS.map((val) => ["expand", val]), + ...additionalExpandKeys.map((val) => ["expand", val]) + ]).toString(); + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/models/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD" + )}?${search.toString()}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + const data = await response.json(); + const additional = {}; + if (params?.additionalFields) { + for (const field of params.additionalFields) { + if (field === "filePaths") { + additional.filePaths = (data.siblings ?? []).map((s) => s.rfilename); + } else if (field === "inferenceProviderMapping" && data.inferenceProviderMapping) { + additional.inferenceProviderMapping = normalizeInferenceProviderMapping(data.id, data.inferenceProviderMapping); + } else { + additional[field] = data[field]; + } + } + } + return { + ...additional, + id: data._id, + name: data.id, + private: data.private, + task: data.pipeline_tag, + downloads: data.downloads, + gated: data.gated, + likes: data.likes, + updatedAt: new Date(data.lastModified) + }; +} + +// src/lib/oauth-handle-redirect.ts +async function oauthHandleRedirect(opts) { + if (typeof window === "undefined" && !opts?.redirectedUrl) { + throw new Error("oauthHandleRedirect is only available in the browser, unless you provide redirectedUrl"); + } + if (typeof localStorage === "undefined" && (!opts?.nonce || !opts?.codeVerifier)) { + throw new Error( + "oauthHandleRedirect requires localStorage to be available, unless you provide nonce and codeVerifier" + ); + } + const redirectedUrl = opts?.redirectedUrl ?? window.location.href; + const searchParams = (() => { + try { + return new URL(redirectedUrl).searchParams; + } catch (err) { + throw new Error("Failed to parse redirected URL: " + redirectedUrl); + } + })(); + const [error, errorDescription] = [searchParams.get("error"), searchParams.get("error_description")]; + if (error) { + throw new Error(`${error}: ${errorDescription}`); + } + const code = searchParams.get("code"); + const nonce = opts?.nonce ?? localStorage.getItem("huggingface.co:oauth:nonce"); + if (!code) { + throw new Error("Missing oauth code from query parameters in redirected URL: " + redirectedUrl); + } + if (!nonce) { + throw new Error("Missing oauth nonce from localStorage"); + } + const codeVerifier = opts?.codeVerifier ?? localStorage.getItem("huggingface.co:oauth:code_verifier"); + if (!codeVerifier) { + throw new Error("Missing oauth code_verifier from localStorage"); + } + const state = searchParams.get("state"); + if (!state) { + throw new Error("Missing oauth state from query parameters in redirected URL"); + } + let parsedState; + try { + parsedState = JSON.parse(state); + } catch { + throw new Error("Invalid oauth state in redirected URL, unable to parse JSON: " + state); + } + if (parsedState.nonce !== nonce) { + throw new Error("Invalid oauth state in redirected URL"); + } + const hubUrl = opts?.hubUrl || HUB_URL; + const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; + const openidConfigRes = await fetch(openidConfigUrl, { + headers: { + Accept: "application/json" + } + }); + if (!openidConfigRes.ok) { + throw await createApiError(openidConfigRes); + } + const openidConfig = await openidConfigRes.json(); + const tokenRes = await fetch(openidConfig.token_endpoint, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded" + }, + body: new URLSearchParams({ + grant_type: "authorization_code", + code, + redirect_uri: parsedState.redirectUri, + code_verifier: codeVerifier + }).toString() + }); + if (!opts?.codeVerifier) { + localStorage.removeItem("huggingface.co:oauth:code_verifier"); + } + if (!opts?.nonce) { + localStorage.removeItem("huggingface.co:oauth:nonce"); + } + if (!tokenRes.ok) { + throw await createApiError(tokenRes); + } + const token = await tokenRes.json(); + const accessTokenExpiresAt = new Date(Date.now() + token.expires_in * 1e3); + const userInfoRes = await fetch(openidConfig.userinfo_endpoint, { + headers: { + Authorization: `Bearer ${token.access_token}` + } + }); + if (!userInfoRes.ok) { + throw await createApiError(userInfoRes); + } + const userInfo = await userInfoRes.json(); + return { + accessToken: token.access_token, + accessTokenExpiresAt, + userInfo, + state: parsedState.state, + scope: token.scope + }; +} +async function oauthHandleRedirectIfPresent(opts) { + if (typeof window === "undefined" && !opts?.redirectedUrl) { + throw new Error("oauthHandleRedirect is only available in the browser, unless you provide redirectedUrl"); + } + if (typeof localStorage === "undefined" && (!opts?.nonce || !opts?.codeVerifier)) { + throw new Error( + "oauthHandleRedirect requires localStorage to be available, unless you provide nonce and codeVerifier" + ); + } + const searchParams = new URLSearchParams(opts?.redirectedUrl ?? window.location.search); + if (searchParams.has("error")) { + return oauthHandleRedirect(opts); + } + if (searchParams.has("code")) { + if (!localStorage.getItem("huggingface.co:oauth:nonce")) { + console.warn( + "Missing oauth nonce from localStorage. This can happen when the user refreshes the page after logging in, without changing the URL." + ); + return false; + } + return oauthHandleRedirect(opts); + } + return false; +} + +// src/lib/oauth-login-url.ts +async function oauthLoginUrl(opts) { + if (typeof window === "undefined" && (!opts?.redirectUrl || !opts?.clientId)) { + throw new Error("oauthLogin is only available in the browser, unless you provide clientId and redirectUrl"); + } + if (typeof localStorage === "undefined" && !opts?.localStorage) { + throw new Error( + "oauthLogin requires localStorage to be available in the context, unless you provide a localStorage empty object as argument" + ); + } + const hubUrl = opts?.hubUrl || HUB_URL; + const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; + const openidConfigRes = await fetch(openidConfigUrl, { + headers: { + Accept: "application/json" + } + }); + if (!openidConfigRes.ok) { + throw await createApiError(openidConfigRes); + } + const opendidConfig = await openidConfigRes.json(); + const newNonce = globalThis.crypto.randomUUID(); + const newCodeVerifier = globalThis.crypto.randomUUID() + globalThis.crypto.randomUUID(); + if (opts?.localStorage) { + if (opts.localStorage.codeVerifier !== void 0 && opts.localStorage.codeVerifier !== null) { + throw new Error( + "localStorage.codeVerifier must be initially set to null or undefined, and will be filled by oauthLoginUrl" + ); + } + if (opts.localStorage.nonce !== void 0 && opts.localStorage.nonce !== null) { + throw new Error( + "localStorage.nonce must be initially set to null or undefined, and will be filled by oauthLoginUrl" + ); + } + opts.localStorage.codeVerifier = newCodeVerifier; + opts.localStorage.nonce = newNonce; + } else { + localStorage.setItem("huggingface.co:oauth:nonce", newNonce); + localStorage.setItem("huggingface.co:oauth:code_verifier", newCodeVerifier); + } + const redirectUri = opts?.redirectUrl || (typeof window !== "undefined" ? window.location.href : void 0); + if (!redirectUri) { + throw new Error("Missing redirectUrl"); + } + const state = JSON.stringify({ + nonce: newNonce, + redirectUri, + state: opts?.state + }); + const variables = ( + // @ts-expect-error window.huggingface is defined inside static Spaces. + typeof window !== "undefined" ? window.huggingface?.variables ?? null : null + ); + const clientId = opts?.clientId || variables?.OAUTH_CLIENT_ID; + if (!clientId) { + if (variables) { + throw new Error("Missing clientId, please add hf_oauth: true to the README.md's metadata in your static Space"); + } + throw new Error("Missing clientId"); + } + const challenge = base64FromBytes( + new Uint8Array(await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(newCodeVerifier))) + ).replace(/[+]/g, "-").replace(/[/]/g, "_").replace(/=/g, ""); + return `${opendidConfig.authorization_endpoint}?${new URLSearchParams({ + client_id: clientId, + scope: opts?.scopes || variables?.OAUTH_SCOPES || "openid profile", + response_type: "code", + redirect_uri: redirectUri, + state, + code_challenge: challenge, + code_challenge_method: "S256" + }).toString()}`; +} + +// src/utils/typedInclude.ts +function typedInclude(arr, v) { + return arr.includes(v); +} + +// src/utils/omit.ts +function omit(o, props) { + const propsArr = Array.isArray(props) ? props : [props]; + const letsKeep = Object.keys(o).filter((prop) => !typedInclude(propsArr, prop)); + return pick(o, letsKeep); +} + +// src/utils/typedEntries.ts +function typedEntries(obj) { + return Object.entries(obj); +} + +// src/lib/parse-safetensors-metadata.ts +var SAFETENSORS_FILE = "model.safetensors"; +var SAFETENSORS_INDEX_FILE = "model.safetensors.index.json"; +var RE_SAFETENSORS_FILE = /\.safetensors$/; +var RE_SAFETENSORS_INDEX_FILE = /\.safetensors\.index\.json$/; +var RE_SAFETENSORS_SHARD_FILE = /^(?(?.*?)[_-])(?\d{5,6})-of-(?\d{5,6})\.safetensors$/; +function parseSafetensorsShardFilename(filename) { + const match = RE_SAFETENSORS_SHARD_FILE.exec(filename); + if (match && match.groups) { + return { + prefix: match.groups["prefix"], + basePrefix: match.groups["basePrefix"], + shard: match.groups["shard"], + total: match.groups["total"] + }; + } + return null; +} +var PARALLEL_DOWNLOADS = 20; +var MAX_HEADER_LENGTH = 25e6; +var MAX_CONFIG_LENGTH = 1e7; +var MAX_SHARD_COUNT = 1e4; +var GPTQ_QWEIGHT_SUFFIX = "qweight"; +var GPTQ_AWQ_AUXILIARY_SUFFIXES = ["qzeros", "g_idx", "scales"]; +var SafetensorParseError = class extends Error { +}; +async function fetchModelConfig(params) { + try { + const configBlob = await downloadFile({ + ...params, + path: "config.json" + }); + if (!configBlob) { + return null; + } + const config = JSON.parse(await configBlob.slice(0, MAX_CONFIG_LENGTH).text()); + return config; + } catch (error) { + return null; + } +} +async function parseSingleFile(path, params) { + const blob = await downloadFile({ ...params, path }); + if (!blob) { + throw new SafetensorParseError(`Failed to parse file ${path}: failed to fetch safetensors header length.`); + } + const bufLengthOfHeaderLE = await blob.slice(0, 8).arrayBuffer(); + const lengthOfHeader = new DataView(bufLengthOfHeaderLE).getBigUint64(0, true); + if (lengthOfHeader <= 0) { + throw new SafetensorParseError(`Failed to parse file ${path}: safetensors header is malformed.`); + } + if (lengthOfHeader > MAX_HEADER_LENGTH) { + throw new SafetensorParseError( + `Failed to parse file ${path}: safetensor header is too big. Maximum supported size is ${MAX_HEADER_LENGTH} bytes.` + ); + } + try { + const header = JSON.parse(await blob.slice(8, 8 + Number(lengthOfHeader)).text()); + return header; + } catch (err) { + throw new SafetensorParseError(`Failed to parse file ${path}: safetensors header is not valid JSON.`); + } +} +async function parseShardedIndex(path, params) { + const indexBlob = await downloadFile({ + ...params, + path + }); + if (!indexBlob) { + throw new SafetensorParseError(`Failed to parse file ${path}: failed to fetch safetensors index.`); + } + try { + const index = JSON.parse(await indexBlob.slice(0, MAX_HEADER_LENGTH).text()); + return index; + } catch (error) { + throw new SafetensorParseError(`Failed to parse file ${path}: not a valid JSON.`); + } +} +async function fetchAllHeaders(path, index, params) { + const pathPrefix = path.slice(0, path.lastIndexOf("/") + 1); + const filenames = [...new Set(Object.values(index.weight_map))]; + if (filenames.length > MAX_SHARD_COUNT) { + throw new SafetensorParseError( + `Too many shard files (${filenames.length}). Maximum supported is ${MAX_SHARD_COUNT}.` + ); + } + for (const filename of filenames) { + if (filename.includes("..") || filename.startsWith("/") || filename.includes("://")) { + throw new SafetensorParseError(`Unsafe shard filename in weight_map: "${filename}"`); + } + } + const shardedMap = Object.fromEntries( + await promisesQueue( + filenames.map( + (filename) => async () => [filename, await parseSingleFile(pathPrefix + filename, params)] + ), + PARALLEL_DOWNLOADS + ) + ); + return shardedMap; +} +function parseTotalParameters(value) { + if (!value) { + return void 0; + } + if (typeof value === "number") { + return value; + } + return parseInt(value); +} +async function parseSafetensorsMetadata(params) { + const repoId = toRepoId(params.repo); + if (repoId.type !== "model") { + throw new TypeError("Only model repos should contain safetensors files."); + } + const modelConfig = params.computeParametersCount ? await fetchModelConfig(params) : null; + const quantConfig = modelConfig?.quantization_config ?? modelConfig?.text_config?.quantization_config; + if (params.path && RE_SAFETENSORS_FILE.test(params.path) || await fileExists({ ...params, path: SAFETENSORS_FILE })) { + const header = await parseSingleFile(params.path ?? SAFETENSORS_FILE, params); + const paramStats = params.computeParametersCount ? { + parameterCount: computeNumOfParamsByDtypeSingleFile(header, quantConfig), + /// shortcut: get param count directly from metadata + parameterTotal: parseTotalParameters(header.__metadata__?.total_parameters) + } : void 0; + return { + sharded: false, + header, + ...paramStats, + filepaths: [params.path ?? SAFETENSORS_FILE] + }; + } else if (params.path && RE_SAFETENSORS_INDEX_FILE.test(params.path) || await fileExists({ ...params, path: SAFETENSORS_INDEX_FILE })) { + const path = params.path ?? SAFETENSORS_INDEX_FILE; + const index = await parseShardedIndex(path, params); + const shardedMap = await fetchAllHeaders(path, index, params); + const pathPrefix = path.slice(0, path.lastIndexOf("/") + 1); + const paramStats = params.computeParametersCount ? { + parameterCount: computeNumOfParamsByDtypeSharded(shardedMap, quantConfig), + /// shortcut: get param count directly from metadata + parameterTotal: parseTotalParameters(index.metadata?.total_parameters) + } : void 0; + return { + sharded: true, + index, + headers: shardedMap, + ...paramStats, + filepaths: [path, ...Object.keys(shardedMap).map((filename) => pathPrefix + filename)] + }; + } else { + throw new Error("model id does not seem to contain safetensors weights"); + } +} +function globMatch(pattern, str) { + const parts = pattern.split("*"); + if (parts.length === 1) { + return pattern === str; + } + if (!str.startsWith(parts[0])) { + return false; + } + let pos = parts[0].length; + const lastPart = parts[parts.length - 1]; + if (!str.endsWith(lastPart)) { + return false; + } + const end = str.length - lastPart.length; + for (let i = 1; i < parts.length - 1; i++) { + const idx = str.indexOf(parts[i], pos); + if (idx === -1 || idx + parts[i].length > end) { + return false; + } + pos = idx + parts[i].length; + } + return pos <= end; +} +function isQuantizedTensor(tensorName, quantConfig) { + if (!quantConfig) { + return false; + } + const patterns = quantConfig.modules_to_not_convert; + if (!patterns?.length) { + return true; + } + return !patterns.some( + (pattern) => pattern.includes("*") ? globMatch(pattern, tensorName) : tensorName.includes(pattern) + ); +} +function matchesCompressedTensorsTarget(target, moduleName) { + if (!target.startsWith("re:")) { + return target === moduleName; + } + let pattern = target.slice(3); + if (pattern.startsWith("^")) { + pattern = pattern.slice(1); + } + if (pattern.endsWith("$")) { + pattern = pattern.slice(0, -1); + } else { + pattern += ".*"; + } + const glob = pattern.replaceAll(".*", "*").replaceAll("\\.", "."); + if (/[\\+?()[\]{}|^$]/.test(glob)) { + return false; + } + return globMatch(glob, moduleName); +} +function getQuantizationMultiplier(tensorName, dtype, quantConfig) { + if (!quantConfig || !isQuantizedTensor(tensorName, quantConfig)) { + return 1; + } + const quantMethod = quantConfig.quant_method?.toLowerCase(); + switch (quantMethod) { + case "mxfp4": + if (dtype === "U8" && tensorName.includes("_blocks")) { + return 2; + } + return 1; + case "gptq": + case "awq": + if (getTensorSuffix(tensorName) === GPTQ_QWEIGHT_SUFFIX) { + const bits = quantConfig.bits && quantConfig.bits > 0 ? quantConfig.bits : 4; + return Math.max(1, Math.floor(32 / bits)); + } + if (quantConfig.bits === 4 && dtype === "U8") { + return 2; + } + if (quantConfig.bits === 2 && dtype === "U8") { + return 4; + } + return 1; + case "compressed-tensors": + if (dtype === "I32") { + const groups = Object.values(quantConfig.config_groups ?? {}); + const suffixIndex = tensorName.lastIndexOf(".weight"); + const moduleName = suffixIndex === -1 ? tensorName : tensorName.slice(0, suffixIndex); + const group = groups.find( + (g) => g.targets?.some((target) => matchesCompressedTensorsTarget(target, moduleName)) + ); + if (group) { + if ((group.format ?? quantConfig.format) !== "pack-quantized") { + return 1; + } + const numBits = group.weights?.num_bits ?? 4; + return Math.max(1, Math.floor(32 / numBits)); + } + if (quantConfig.format === "pack-quantized") { + const numBits = groups.find((g) => g.weights?.num_bits)?.weights?.num_bits ?? 4; + return Math.max(1, Math.floor(32 / numBits)); + } + } + return 1; + case "bitsandbytes": + if (quantConfig.load_in_4bit && dtype === "U8") { + return 2; + } + return 1; + default: + if (dtype === "U8" && (quantConfig.load_in_4bit || quantConfig.bits === 4)) { + return 2; + } + return 1; + } +} +function computeNumOfParamsByDtypeSingleFile(header, quantConfig) { + const counter = {}; + const tensors = omit(header, "__metadata__"); + for (const [tensorName, v] of typedEntries(tensors)) { + if (shouldSkipTensor(tensorName, quantConfig)) { + continue; + } + if (v.shape.length === 0) { + continue; + } + const elements = v.shape.reduce((a, b) => a * b); + if (!Number.isFinite(elements)) { + continue; + } + const multiplier = quantConfig ? getQuantizationMultiplier(tensorName, v.dtype, quantConfig) : 1; + if (multiplier === 0) { + continue; + } + counter[v.dtype] = (counter[v.dtype] ?? 0) + elements * multiplier; + } + return counter; +} +function computeNumOfParamsByDtypeSharded(shardedMap, quantConfig) { + const counter = {}; + for (const header of Object.values(shardedMap)) { + for (const [k, v] of typedEntries(computeNumOfParamsByDtypeSingleFile(header, quantConfig))) { + counter[k] = (counter[k] ?? 0) + (v ?? 0); + } + } + return counter; +} +function getTensorSuffix(tensorName) { + const lastDotIndex = tensorName.lastIndexOf("."); + return lastDotIndex === -1 ? tensorName : tensorName.slice(lastDotIndex + 1); +} +function shouldSkipTensor(tensorName, quantConfig) { + if (!quantConfig) { + return false; + } + const quantMethod = quantConfig.quant_method?.toLowerCase(); + if (quantMethod !== "gptq" && quantMethod !== "awq") { + return false; + } + if (!isQuantizedTensor(tensorName, quantConfig)) { + return false; + } + const suffix = getTensorSuffix(tensorName); + return suffix !== GPTQ_QWEIGHT_SUFFIX && GPTQ_AWQ_AUXILIARY_SUFFIXES.includes(suffix); +} + +// src/lib/repo-exists.ts +async function repoExists(params) { + const repoId = toRepoId(params.repo); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}?expand[]=likes`, + { + method: "GET", + headers: { + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + } + } + ); + if (res.status === 404 || res.status === 401) { + return false; + } + if (!res.ok) { + throw await createApiError(res); + } + return true; +} + +// src/lib/space-info.ts +async function spaceInfo(params) { + const accessToken = params && checkCredentials(params); + const search = new URLSearchParams([ + ...SPACE_EXPAND_KEYS.map((val) => ["expand", val]), + ...params?.additionalFields?.map((val) => ["expand", val]) ?? [] + ]).toString(); + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/spaces/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD" + )}?${search.toString()}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + const data = await response.json(); + return { + ...params?.additionalFields && pick(data, params.additionalFields), + id: data._id, + name: data.id, + sdk: data.sdk, + likes: data.likes, + private: data.private, + updatedAt: new Date(data.lastModified) + }; +} + +// src/lib/upload-file.ts +function uploadFile(params) { + const path = params.file instanceof URL ? params.file.pathname.split("/").at(-1) ?? "file" : "path" in params.file ? params.file.path : params.file.name; + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: [ + { + operation: "addOrUpdate", + path, + content: "content" in params.file ? params.file.content : params.file + } + ], + title: params.commitTitle ?? `Add ${path}`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet + }); +} + +// src/lib/upload-files.ts +function uploadFiles(params) { + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: params.files.map((file) => ({ + operation: "addOrUpdate", + path: file instanceof URL ? file.pathname.split("/").at(-1) ?? "file" : "path" in file ? file.path : file.name, + content: "content" in file ? file.content : file + })), + title: params.commitTitle ?? `Add ${params.files.length} files`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet + }); +} + +// src/lib/upload-files-with-progress.ts +var multipartUploadTracking = /* @__PURE__ */ new WeakMap(); +async function* uploadFilesWithProgress(params) { + return yield* commitIter({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: params.files.map((file) => ({ + operation: "addOrUpdate", + path: file instanceof URL ? file.pathname.split("/").at(-1) ?? "file" : "path" in file ? file.path : file.name, + content: "content" in file ? file.content : file + })), + title: params.commitTitle ?? `Add ${params.files.length} files`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet, + fetch: async (input, init) => { + if (!init) { + return fetch(input); + } + if (!typedInclude(["PUT", "POST"], init.method) || !("progressHint" in init) || !init.progressHint || typeof XMLHttpRequest === "undefined" || typeof input !== "string" || !(init.body instanceof ArrayBuffer) && !(init.body instanceof Blob) && !(init.body instanceof File) && typeof init.body !== "string") { + return fetch(input, init); + } + const progressHint = init.progressHint; + const progressCallback = progressHint.progressCallback; + const xhr = new XMLHttpRequest(); + xhr.upload.addEventListener("progress", (event) => { + if (event.lengthComputable) { + if (progressHint.part !== void 0) { + let tracking = multipartUploadTracking.get(progressCallback); + if (!tracking) { + tracking = { numParts: progressHint.numParts, partsProgress: {} }; + multipartUploadTracking.set(progressCallback, tracking); + } + tracking.partsProgress[progressHint.part] = event.loaded / event.total; + let totalProgress = 0; + for (const partProgress of Object.values(tracking.partsProgress)) { + totalProgress += partProgress; + } + if (totalProgress === tracking.numParts) { + progressCallback(0.9999999999); + } else { + progressCallback(totalProgress / tracking.numParts); + } + } else { + if (event.loaded === event.total) { + progressCallback(0.9999999999); + } else { + progressCallback(event.loaded / event.total); + } + } + } + }); + xhr.open(init.method, input, true); + if (init.headers) { + const headers = new Headers(init.headers); + headers.forEach((value, key) => { + xhr.setRequestHeader(key, value); + }); + } + init.signal?.throwIfAborted(); + xhr.send(init.body); + return new Promise((resolve2, reject) => { + xhr.addEventListener("load", () => { + resolve2( + new Response(xhr.responseText, { + status: xhr.status, + statusText: xhr.statusText, + headers: Object.fromEntries( + xhr.getAllResponseHeaders().trim().split("\n").map((header) => [ + header.slice(0, header.indexOf(":")), + header.slice(header.indexOf(":") + 1).trim() + ]) + ) + }) + ); + }); + xhr.addEventListener("error", () => { + reject(new Error(xhr.statusText)); + }); + if (init.signal) { + init.signal.addEventListener("abort", () => { + xhr.abort(); + try { + init.signal?.throwIfAborted(); + } catch (err) { + reject(err); + } + }); + } + }); + } + }); +} + +// src/lib/who-am-i.ts +async function whoAmI(params) { + const accessToken = checkCredentials(params); + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/whoami-v2`, { + headers: { + Authorization: `Bearer ${accessToken}` + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const response = await res.json(); + if (typeof response.auth.accessToken?.createdAt === "string") { + response.auth.accessToken.createdAt = new Date(response.auth.accessToken.createdAt); + } + return response; +} +export { + DATASET_EXPANDABLE_KEYS, + DATASET_EXPAND_KEYS, + HUB_URL, + HubApiError, + InvalidApiResponseFormatError, + MODEL_DERIVED_FIELD_TO_API_KEY, + MODEL_EXPANDABLE_KEYS, + MODEL_EXPAND_KEYS, + RE_SAFETENSORS_FILE, + RE_SAFETENSORS_INDEX_FILE, + RE_SAFETENSORS_SHARD_FILE, + SAFETENSORS_FILE, + SAFETENSORS_INDEX_FILE, + SPACE_EXPANDABLE_KEYS, + SPACE_EXPAND_KEYS, + XetBlob as __internal_XetBlob, + sha256 as __internal_sha256, + cancelJob, + checkRepoAccess, + commit, + commitIter, + commitIterBucket, + copyFile, + copyFileIter, + copyFiles, + copyFilesIter, + copyFolder, + copyFolderIter, + countCommits, + createBranch, + createCollection, + createRepo, + createScheduledJob, + datasetInfo, + deleteBranch, + deleteCollection, + deleteFile, + deleteFiles, + deleteRepo, + deleteScheduledJob, + downloadFile, + duplicateJob, + fileDownloadInfo, + fileExists, + getJob, + getScheduledJob, + globMatch, + isQuantizedTensor, + listCollections, + listCommits, + listDatasets, + listFiles, + listJobHardware, + listJobs, + listModels, + listScheduledJobs, + listSpaces, + matchesCompressedTensorsTarget, + modelInfo, + oauthHandleRedirect, + oauthHandleRedirectIfPresent, + oauthLoginUrl, + parseSafetensorsMetadata, + parseSafetensorsShardFilename, + pathsInfo, + relativeUnderFolder, + repoExists, + resumeScheduledJob, + runJob, + runScheduledJob, + spaceInfo, + streamJobEvents, + streamJobLogs, + streamJobMetrics, + suspendScheduledJob, + uploadFile, + uploadFiles, + uploadFilesWithProgress, + whoAmI +}; diff --git a/node_modules/@huggingface/hub/dist/browser/sha256-node-FT2Y3VXD.js b/node_modules/@huggingface/hub/dist/browser/sha256-node-FT2Y3VXD.js new file mode 100644 index 0000000000000000000000000000000000000000..9a390c31f71bc7eae1522a280a2dc8f6723185bf --- /dev/null +++ b/node_modules/@huggingface/hub/dist/browser/sha256-node-FT2Y3VXD.js @@ -0,0 +1 @@ +"use strict"; \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/browser/sha256-node-TNZ2WHTI.mjs b/node_modules/@huggingface/hub/dist/browser/sha256-node-TNZ2WHTI.mjs new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/node_modules/@huggingface/hub/dist/browser/sha256-wrapper-6KQBPSEU.js b/node_modules/@huggingface/hub/dist/browser/sha256-wrapper-6KQBPSEU.js new file mode 100644 index 0000000000000000000000000000000000000000..8c7a39d9ea7b4ce585242aaa31c68b433dc4f372 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/browser/sha256-wrapper-6KQBPSEU.js @@ -0,0 +1,458 @@ +"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }// src/vendor/hash-wasm/sha256.js +var Module = (() => { + var _unused = import.meta.url; + return function(moduleArg = {}) { + var Module2 = moduleArg; + var readyPromiseResolve, readyPromiseReject; + Module2["ready"] = new Promise((resolve, reject) => { + readyPromiseResolve = resolve; + readyPromiseReject = reject; + }); + var moduleOverrides = Object.assign({}, Module2); + var arguments_ = []; + var thisProgram = "./this.program"; + var quit_ = (status, toThrow) => { + throw toThrow; + }; + var ENVIRONMENT_IS_WEB = typeof window == "object"; + var ENVIRONMENT_IS_WORKER = typeof importScripts == "function"; + var ENVIRONMENT_IS_NODE = typeof process == "object" && typeof process.versions == "object" && typeof process.versions.node == "string"; + var ENVIRONMENT_IS_SHELL = !ENVIRONMENT_IS_WEB && !ENVIRONMENT_IS_NODE && !ENVIRONMENT_IS_WORKER; + var scriptDirectory = ""; + function locateFile(path) { + if (Module2["locateFile"]) { + return Module2["locateFile"](path, scriptDirectory); + } + return scriptDirectory + path; + } + var read_, readAsync, readBinary; + if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { + if (ENVIRONMENT_IS_WORKER) { + scriptDirectory = self.location.href; + } else if (typeof document != "undefined" && document.currentScript) { + scriptDirectory = document.currentScript.src; + } + if (false) { + scriptDirectory = false; + } + if (scriptDirectory.startsWith("blob:")) { + scriptDirectory = ""; + } else { + scriptDirectory = scriptDirectory.substr(0, scriptDirectory.replace(/[?#].*/, "").lastIndexOf("/") + 1); + } + { + read_ = (url) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, false); + xhr.send(null); + return xhr.responseText; + }; + if (ENVIRONMENT_IS_WORKER) { + readBinary = (url) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, false); + xhr.responseType = "arraybuffer"; + xhr.send(null); + return new Uint8Array( + /** @type{!ArrayBuffer} */ + xhr.response + ); + }; + } + readAsync = (url, onload, onerror) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, true); + xhr.responseType = "arraybuffer"; + xhr.onload = () => { + if (xhr.status == 200 || xhr.status == 0 && xhr.response) { + onload(xhr.response); + return; + } + onerror(); + }; + xhr.onerror = onerror; + xhr.send(null); + }; + } + } else { + } + var out = Module2["print"] || console.log.bind(console); + var err = Module2["printErr"] || console.error.bind(console); + Object.assign(Module2, moduleOverrides); + moduleOverrides = null; + if (Module2["arguments"]) + arguments_ = Module2["arguments"]; + if (Module2["thisProgram"]) + thisProgram = Module2["thisProgram"]; + if (Module2["quit"]) + quit_ = Module2["quit"]; + var wasmBinary; + if (Module2["wasmBinary"]) + wasmBinary = Module2["wasmBinary"]; + if (typeof WebAssembly != "object") { + abort("no native wasm support detected"); + } + function intArrayFromBase64(s) { + var decoded = atob(s); + var bytes = new Uint8Array(decoded.length); + for (var i = 0; i < decoded.length; ++i) { + bytes[i] = decoded.charCodeAt(i); + } + return bytes; + } + function tryParseAsDataURI(filename) { + if (!isDataURI(filename)) { + return; + } + return intArrayFromBase64(filename.slice(dataURIPrefix.length)); + } + var wasmMemory; + var ABORT = false; + var EXITSTATUS; + function assert(condition, text) { + if (!condition) { + abort(text); + } + } + var HEAP, HEAP8, HEAPU8, HEAP16, HEAPU16, HEAP32, HEAPU32, HEAPF32, HEAPF64; + function updateMemoryViews() { + var b = wasmMemory.buffer; + Module2["HEAP8"] = HEAP8 = new Int8Array(b); + Module2["HEAP16"] = HEAP16 = new Int16Array(b); + Module2["HEAPU8"] = HEAPU8 = new Uint8Array(b); + Module2["HEAPU16"] = HEAPU16 = new Uint16Array(b); + Module2["HEAP32"] = HEAP32 = new Int32Array(b); + Module2["HEAPU32"] = HEAPU32 = new Uint32Array(b); + Module2["HEAPF32"] = HEAPF32 = new Float32Array(b); + Module2["HEAPF64"] = HEAPF64 = new Float64Array(b); + } + var __ATPRERUN__ = []; + var __ATINIT__ = []; + var __ATEXIT__ = []; + var __ATPOSTRUN__ = []; + var runtimeInitialized = false; + function preRun() { + if (Module2["preRun"]) { + if (typeof Module2["preRun"] == "function") + Module2["preRun"] = [Module2["preRun"]]; + while (Module2["preRun"].length) { + addOnPreRun(Module2["preRun"].shift()); + } + } + callRuntimeCallbacks(__ATPRERUN__); + } + function initRuntime() { + runtimeInitialized = true; + callRuntimeCallbacks(__ATINIT__); + } + function postRun() { + if (Module2["postRun"]) { + if (typeof Module2["postRun"] == "function") + Module2["postRun"] = [Module2["postRun"]]; + while (Module2["postRun"].length) { + addOnPostRun(Module2["postRun"].shift()); + } + } + callRuntimeCallbacks(__ATPOSTRUN__); + } + function addOnPreRun(cb) { + __ATPRERUN__.unshift(cb); + } + function addOnInit(cb) { + __ATINIT__.unshift(cb); + } + function addOnExit(cb) { + } + function addOnPostRun(cb) { + __ATPOSTRUN__.unshift(cb); + } + var runDependencies = 0; + var runDependencyWatcher = null; + var dependenciesFulfilled = null; + function getUniqueRunDependency(id) { + return id; + } + function addRunDependency(id) { + runDependencies++; + _optionalChain([Module2, 'access', _ => _["monitorRunDependencies"], 'optionalCall', _2 => _2(runDependencies)]); + } + function removeRunDependency(id) { + runDependencies--; + _optionalChain([Module2, 'access', _3 => _3["monitorRunDependencies"], 'optionalCall', _4 => _4(runDependencies)]); + if (runDependencies == 0) { + if (runDependencyWatcher !== null) { + clearInterval(runDependencyWatcher); + runDependencyWatcher = null; + } + if (dependenciesFulfilled) { + var callback = dependenciesFulfilled; + dependenciesFulfilled = null; + callback(); + } + } + } + function abort(what) { + _optionalChain([Module2, 'access', _5 => _5["onAbort"], 'optionalCall', _6 => _6(what)]); + what = "Aborted(" + what + ")"; + err(what); + ABORT = true; + EXITSTATUS = 1; + what += ". Build with -sASSERTIONS for more info."; + var e = new WebAssembly.RuntimeError(what); + readyPromiseReject(e); + throw e; + } + var dataURIPrefix = "data:application/octet-stream;base64,"; + var isDataURI = (filename) => filename.startsWith(dataURIPrefix); + var isFileURI = (filename) => filename.startsWith("file://"); + var wasmBinaryFile; + wasmBinaryFile = "data:application/octet-stream;base64,AGFzbQEAAAABHQZgAX8AYAABf2AAAGABfwF/YAJ/fwBgA39/fwF/Aw0MAgAEAgMBBQABAQADBAUBcAEBAQUGAQGAAoACBg4CfwFB8IuEBAt/AUEACweYAQoGbWVtb3J5AgARX193YXNtX2NhbGxfY3RvcnMAAAtIYXNoX1VwZGF0ZQABCkhhc2hfRmluYWwAAwlIYXNoX0luaXQABAxHZXRCdWZmZXJQdHIABRlfX2luZGlyZWN0X2Z1bmN0aW9uX3RhYmxlAQAJc3RhY2tTYXZlAAkMc3RhY2tSZXN0b3JlAAoKc3RhY2tBbGxvYwALCossDAIAC+4CAgV/AX5BACgCwAoiASABKQNAIgYgAK18NwNAAkACQAJAIAanQT9xIgINAEGACyEBIAAhAgwBC0HAACACayEDAkAgAEUNACADIAAgAyAASRshBCABIAJqIQVBACEBA0AgBSABIgFqQYALIAFqLQAAOgAAIAFBAWoiAiEBIAIgBEcNAAsLAkACQCAAIANJIgRFDQBBgAshASAAIQIMAQtBACgCwAoiAUHIAGogARACQYALIANqIQEgACADayECCyABIQEgAiECIAQNAQsgASEBAkACQCACIgJBwABPDQAgASEFIAIhAAwBCyACIQIgASEEA0BBACgCwApByABqIAQiBBACIAJBQGoiASECIARBwABqIgUhBCAFIQUgASEAIAFBP0sNAAsLIAUhBSAAIgBFDQBBACEBQQAhAgNAQQAoAsAKIAEiAWogBSABai0AADoAACACQQFqIgJB/wFxIgQhASACIQIgACAESw0ACwsLqCEBK38gACgCCCICIAAoAgQiAyAAKAIAIgRzcSADIARxcyAEQR53IARBE3dzIARBCndzaiAAKAIQIgVBGncgBUEVd3MgBUEHd3MgACgCHCIGaiAAKAIYIgcgACgCFCIIcyAFcSAHc2ogASgCACIJQRh0IAlBgP4DcUEIdHIgCUEIdkGA/gNxIAlBGHZyciIKakGY36iUBGoiC2oiCSAEcyADcSAJIARxcyAJQR53IAlBE3dzIAlBCndzaiAHIAEoAgQiDEEYdCAMQYD+A3FBCHRyIAxBCHZBgP4DcSAMQRh2cnIiDWogCyAAKAIMIg5qIg8gCCAFc3EgCHNqIA9BGncgD0EVd3MgD0EHd3NqQZGJ3YkHaiIQaiIMIAlzIARxIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAggASgCCCILQRh0IAtBgP4DcUEIdHIgC0EIdkGA/gNxIAtBGHZyciIRaiAQIAJqIhIgDyAFc3EgBXNqIBJBGncgEkEVd3MgEkEHd3NqQc/3g657aiITaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAUgASgCDCIQQRh0IBBBgP4DcUEIdHIgEEEIdkGA/gNxIBBBGHZyciIUaiATIANqIhMgEiAPc3EgD3NqIBNBGncgE0EVd3MgE0EHd3NqQaW3181+aiIVaiIQIAtzIAxxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIA8gASgCECIWQRh0IBZBgP4DcUEIdHIgFkEIdkGA/gNxIBZBGHZyciIXaiAVIARqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQduE28oDaiIYaiIPIBBzIAtxIA8gEHFzIA9BHncgD0ETd3MgD0EKd3NqIAEoAhQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGSASaiAYIAlqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQfGjxM8FaiIYaiIJIA9zIBBxIAkgD3FzIAlBHncgCUETd3MgCUEKd3NqIAEoAhgiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGiATaiAYIAxqIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQaSF/pF5aiIYaiIMIAlzIA9xIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAEoAhwiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGyAWaiAYIAtqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQdW98dh6aiIYaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAEoAiAiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHCASaiAYIBBqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQZjVnsB9aiIYaiIQIAtzIAxxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAiQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHSATaiAYIA9qIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQYG2jZQBaiIYaiIPIBBzIAtxIA8gEHFzIA9BHncgD0ETd3MgD0EKd3NqIAEoAigiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHiAWaiAYIAlqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQb6LxqECaiIYaiIJIA9zIBBxIAkgD3FzIAlBHncgCUETd3MgCUEKd3NqIAEoAiwiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHyASaiAYIAxqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQcP7sagFaiIYaiIMIAlzIA9xIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAEoAjAiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiICATaiAYIAtqIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQfS6+ZUHaiIYaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAEoAjQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiISAWaiAYIBBqIhAgEyASc3EgEnNqIBBBGncgEEEVd3MgEEEHd3NqQf7j+oZ4aiIYaiIWIAtzIAxxIBYgC3FzIBZBHncgFkETd3MgFkEKd3NqIAEoAjgiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiIiASaiAYIA9qIg8gECATc3EgE3NqIA9BGncgD0EVd3MgD0EHd3NqQaeN8N55aiIVaiISIBZzIAtxIBIgFnFzIBJBHncgEkETd3MgEkEKd3NqIAEoAjwiAUEYdCABQYD+A3FBCHRyIAFBCHZBgP4DcSABQRh2cnIiIyATaiAVIAlqIgEgDyAQc3EgEHNqIAFBGncgAUEVd3MgAUEHd3NqQfTi74x8aiIJaiEVIBIhGCAWISQgCyElIAkgDGohJiABIScgDyEoIBAhKSAjISMgIiEiICEhISAgISAgHyEfIB4hHiAdIR0gHCEcIBshGyAaIRogGSEZIBchFyAUIRQgESERIA0hECAKIQxBgAkhAUEQISoDQCAVIgkgGCIKcyAkIitxIAkgCnFzIAlBHncgCUETd3MgCUEKd3NqIBAiEEEZdyAQQQ53cyAQQQN2cyAMaiAdIh1qICIiFkEPdyAWQQ13cyAWQQp2c2oiDCApaiAmIhIgJyIPICgiE3NxIBNzaiASQRp3IBJBFXdzIBJBB3dzaiABIgEoAgBqIiRqIgsgCXMgCnEgCyAJcXMgC0EedyALQRN3cyALQQp3c2ogESIYQRl3IBhBDndzIBhBA3ZzIBBqIB4iHmogIyIVQQ93IBVBDXdzIBVBCnZzaiINIBNqIAEoAgRqICQgJWoiEyASIA9zcSAPc2ogE0EadyATQRV3cyATQQd3c2oiJWoiECALcyAJcSAQIAtxcyAQQR53IBBBE3dzIBBBCndzaiAUIiRBGXcgJEEOd3MgJEEDdnMgGGogHyIfaiAMQQ93IAxBDXdzIAxBCnZzaiIRIA9qIAEoAghqICUgK2oiGCATIBJzcSASc2ogGEEadyAYQRV3cyAYQQd3c2oiJWoiDyAQcyALcSAPIBBxcyAPQR53IA9BE3dzIA9BCndzaiAXIhdBGXcgF0EOd3MgF0EDdnMgJGogICIgaiANQQ93IA1BDXdzIA1BCnZzaiIUIBJqIAEoAgxqICUgCmoiCiAYIBNzcSATc2ogCkEadyAKQRV3cyAKQQd3c2oiJWoiEiAPcyAQcSASIA9xcyASQR53IBJBE3dzIBJBCndzaiATIBkiJEEZdyAkQQ53cyAkQQN2cyAXaiAhIiFqIBFBD3cgEUENd3MgEUEKdnNqIhdqIAEoAhBqICUgCWoiEyAKIBhzcSAYc2ogE0EadyATQRV3cyATQQd3c2oiJWoiCSAScyAPcSAJIBJxcyAJQR53IAlBE3dzIAlBCndzaiABKAIUIBoiGkEZdyAaQQ53cyAaQQN2cyAkaiAWaiAUQQ93IBRBDXdzIBRBCnZzaiIZaiAYaiAlIAtqIhggEyAKc3EgCnNqIBhBGncgGEEVd3MgGEEHd3NqIiVqIgsgCXMgEnEgCyAJcXMgC0EedyALQRN3cyALQQp3c2ogASgCGCAbIiRBGXcgJEEOd3MgJEEDdnMgGmogFWogF0EPdyAXQQ13cyAXQQp2c2oiGmogCmogJSAQaiIKIBggE3NxIBNzaiAKQRp3IApBFXdzIApBB3dzaiIlaiIQIAtzIAlxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAhwgHCIcQRl3IBxBDndzIBxBA3ZzICRqIAxqIBlBD3cgGUENd3MgGUEKdnNqIhtqIBNqICUgD2oiJCAKIBhzcSAYc2ogJEEadyAkQRV3cyAkQQd3c2oiE2oiDyAQcyALcSAPIBBxcyAPQR53IA9BE3dzIA9BCndzaiABKAIgIB1BGXcgHUEOd3MgHUEDdnMgHGogDWogGkEPdyAaQQ13cyAaQQp2c2oiHGogGGogEyASaiIYICQgCnNxIApzaiAYQRp3IBhBFXdzIBhBB3dzaiITaiISIA9zIBBxIBIgD3FzIBJBHncgEkETd3MgEkEKd3NqIAEoAiQgHkEZdyAeQQ53cyAeQQN2cyAdaiARaiAbQQ93IBtBDXdzIBtBCnZzaiIdaiAKaiATIAlqIgkgGCAkc3EgJHNqIAlBGncgCUEVd3MgCUEHd3NqIgpqIhMgEnMgD3EgEyAScXMgE0EedyATQRN3cyATQQp3c2ogASgCKCAfQRl3IB9BDndzIB9BA3ZzIB5qIBRqIBxBD3cgHEENd3MgHEEKdnNqIh5qICRqIAogC2oiCiAJIBhzcSAYc2ogCkEadyAKQRV3cyAKQQd3c2oiJGoiCyATcyAScSALIBNxcyALQR53IAtBE3dzIAtBCndzaiABKAIsICBBGXcgIEEOd3MgIEEDdnMgH2ogF2ogHUEPdyAdQQ13cyAdQQp2c2oiH2ogGGogJCAQaiIYIAogCXNxIAlzaiAYQRp3IBhBFXdzIBhBB3dzaiIkaiIQIAtzIBNxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAjAgIUEZdyAhQQ53cyAhQQN2cyAgaiAZaiAeQQ93IB5BDXdzIB5BCnZzaiIgaiAJaiAkIA9qIiQgGCAKc3EgCnNqICRBGncgJEEVd3MgJEEHd3NqIg9qIgkgEHMgC3EgCSAQcXMgCUEedyAJQRN3cyAJQQp3c2ogASgCNCAWQRl3IBZBDndzIBZBA3ZzICFqIBpqIB9BD3cgH0ENd3MgH0EKdnNqIiFqIApqIA8gEmoiDyAkIBhzcSAYc2ogD0EadyAPQRV3cyAPQQd3c2oiCmoiEiAJcyAQcSASIAlxcyASQR53IBJBE3dzIBJBCndzaiABKAI4IBVBGXcgFUEOd3MgFUEDdnMgFmogG2ogIEEPdyAgQQ13cyAgQQp2c2oiImogGGogCiATaiITIA8gJHNxICRzaiATQRp3IBNBFXdzIBNBB3dzaiIYaiIWIBJzIAlxIBYgEnFzIBZBHncgFkETd3MgFkEKd3NqIAEoAjwgDEEZdyAMQQ53cyAMQQN2cyAVaiAcaiAhQQ93ICFBDXdzICFBCnZzaiIKaiAkaiAYIAtqIgsgEyAPc3EgD3NqIAtBGncgC0EVd3MgC0EHd3NqIiZqIishFSAWIRggEiEkIAkhJSAmIBBqIiwhJiALIScgEyEoIA8hKSAKISMgIiEiICEhISAgISAgHyEfIB4hHiAdIR0gHCEcIBshGyAaIRogGSEZIBchFyAUIRQgESERIA0hECAMIQwgAUHAAGohASAqIgpBEGohKiAKQTBJDQALIAAgDyAGajYCHCAAIBMgB2o2AhggACALIAhqNgIUIAAgLCAFajYCECAAIAkgDmo2AgwgACASIAJqNgIIIAAgFiADajYCBCAAICsgBGo2AgAL1AMDBX8BfgF7QQAoAsAKIgAgACgCQCIBQQJ2QQ9xIgJBAnRqIgMgAygCAEF/IAFBA3QiAXRBf3NxQYABIAF0czYCAAJAAkAgAkEOTw0AIAJBAWohAAwBCwJAIAJBDkcNACAAQQA2AjwLIABByABqIAAQAkEAIQALAkAgACIAQQ1LDQBBACgCwAogAEECdCIAakEAQTggAGsQBhoLQQAoAsAKIgAgACkDQCIFpyICQRt0IAJBC3RBgID8B3FyIAJBBXZBgP4DcSACQQN0QRh2cnI2AjwgACAFQh2IpyICQRh0IAJBgP4DcUEIdHIgAkEIdkGA/gNxIAJBGHZycjYCOCAAQcgAaiAAEAJBACgCwApBPGohAUEAIQADQCABQQcgACIAa0ECdGoiAiAC/QACACAG/Q0MDQ4PCAkKCwQFBgcAAQIDIAb9DQMCAQAHBgUECwoJCA8ODQwgBv0NDA0ODwgJCgsEBQYHAAECA/0LAgAgAEEEaiICIQAgAkEIRw0ACwJAQQAoAsAKIgMoAmhFDQAgA0HIAGohBEEAIQBBACECA0BBgAsgACIAaiAEIABqLQAAOgAAIAJBAWoiAkH/AXEiASEAIAIhAiADKAJoIAFLDQALCwtxAQJ/QQAoAsAKIgFCADcDQCABQcgAaiECAkAgAEHgAUcNACABQRw2AmggAkEQakEA/QAEsAj9CwIAIAJBAP0ABKAI/QsCAEEADwsgAUEgNgJoIAJBEGpBAP0ABJAI/QsCACACQQD9AASACP0LAgBBAAsFAEGACwvyAgIDfwF+AkAgAkUNACAAIAE6AAAgACACaiIDQX9qIAE6AAAgAkEDSQ0AIAAgAToAAiAAIAE6AAEgA0F9aiABOgAAIANBfmogAToAACACQQdJDQAgACABOgADIANBfGogAToAACACQQlJDQAgAEEAIABrQQNxIgRqIgMgAUH/AXFBgYKECGwiATYCACADIAIgBGtBfHEiBGoiAkF8aiABNgIAIARBCUkNACADIAE2AgggAyABNgIEIAJBeGogATYCACACQXRqIAE2AgAgBEEZSQ0AIAMgATYCGCADIAE2AhQgAyABNgIQIAMgATYCDCACQXBqIAE2AgAgAkFsaiABNgIAIAJBaGogATYCACACQWRqIAE2AgAgBCADQQRxQRhyIgVrIgJBIEkNACABrUKBgICAEH4hBiADIAVqIQEDQCABIAY3AxggASAGNwMQIAEgBjcDCCABIAY3AwAgAUEgaiEBIAJBYGoiAkEfSw0ACwsgAAsGACAAJAELBAAjAQsEACMACwYAIAAkAAsSAQJ/IwAgAGtBcHEiASQAIAELC9ICAgBBgAgLwAJn5glqha5nu3Lzbjw69U+lf1IOUYxoBZur2YMfGc3gW9ieBcEH1Xw2F91wMDlZDvcxC8D/ERVYaKeP+WSkT/q+mC+KQpFEN3HP+8C1pdu16VvCVjnxEfFZpII/ktVeHKuYqgfYAVuDEr6FMSTDfQxVdF2+cv6x3oCnBtybdPGbwcFpm+SGR77vxp3BD8yhDCRvLOktqoR0StypsFzaiPl2UlE+mG3GMajIJwOwx39Zv/ML4MZHkafVUWPKBmcpKRSFCrcnOCEbLvxtLE0TDThTVHMKZbsKanYuycKBhSxykqHov6JLZhqocItLwqNRbMcZ6JLRJAaZ1oU1DvRwoGoQFsGkGQhsNx5Md0gntbywNLMMHDlKqthOT8qcW/NvLmjugo90b2OleBR4yIQIAseM+v++kOtsUKT3o/m+8nhxxgBBwAoLBIAFgAA="; + if (!isDataURI(wasmBinaryFile)) { + wasmBinaryFile = locateFile(wasmBinaryFile); + } + function getBinarySync(file) { + if (file == wasmBinaryFile && wasmBinary) { + return new Uint8Array(wasmBinary); + } + var binary = tryParseAsDataURI(file); + if (binary) { + return binary; + } + if (readBinary) { + return readBinary(file); + } + throw "both async and sync fetching of the wasm failed"; + } + function getBinaryPromise(binaryFile) { + return Promise.resolve().then(() => getBinarySync(binaryFile)); + } + function instantiateArrayBuffer(binaryFile, imports, receiver) { + return getBinaryPromise(binaryFile).then((binary) => { + return WebAssembly.instantiate(binary, imports); + }).then(receiver, (reason) => { + err(`failed to asynchronously prepare wasm: ${reason}`); + abort(reason); + }); + } + function instantiateAsync(binary, binaryFile, imports, callback) { + return instantiateArrayBuffer(binaryFile, imports, callback); + } + function createWasm() { + var info = { + "env": wasmImports, + "wasi_snapshot_preview1": wasmImports + }; + function receiveInstance(instance, module) { + wasmExports = instance.exports; + wasmMemory = wasmExports["memory"]; + updateMemoryViews(); + addOnInit(wasmExports["__wasm_call_ctors"]); + removeRunDependency("wasm-instantiate"); + return wasmExports; + } + addRunDependency("wasm-instantiate"); + function receiveInstantiationResult(result) { + receiveInstance(result["instance"]); + } + if (Module2["instantiateWasm"]) { + try { + return Module2["instantiateWasm"](info, receiveInstance); + } catch (e) { + err(`Module.instantiateWasm callback failed with error: ${e}`); + readyPromiseReject(e); + } + } + instantiateAsync(wasmBinary, wasmBinaryFile, info, receiveInstantiationResult).catch(readyPromiseReject); + return {}; + } + var tempDouble; + var tempI64; + function ExitStatus(status) { + this.name = "ExitStatus"; + this.message = `Program terminated with exit(${status})`; + this.status = status; + } + var callRuntimeCallbacks = (callbacks) => { + while (callbacks.length > 0) { + callbacks.shift()(Module2); + } + }; + function getValue(ptr, type = "i8") { + if (type.endsWith("*")) + type = "*"; + switch (type) { + case "i1": + return HEAP8[ptr]; + case "i8": + return HEAP8[ptr]; + case "i16": + return HEAP16[ptr >> 1]; + case "i32": + return HEAP32[ptr >> 2]; + case "i64": + abort("to do getValue(i64) use WASM_BIGINT"); + case "float": + return HEAPF32[ptr >> 2]; + case "double": + return HEAPF64[ptr >> 3]; + case "*": + return HEAPU32[ptr >> 2]; + default: + abort(`invalid type for getValue: ${type}`); + } + } + var noExitRuntime = Module2["noExitRuntime"] || true; + function setValue(ptr, value, type = "i8") { + if (type.endsWith("*")) + type = "*"; + switch (type) { + case "i1": + HEAP8[ptr] = value; + break; + case "i8": + HEAP8[ptr] = value; + break; + case "i16": + HEAP16[ptr >> 1] = value; + break; + case "i32": + HEAP32[ptr >> 2] = value; + break; + case "i64": + abort("to do setValue(i64) use WASM_BIGINT"); + case "float": + HEAPF32[ptr >> 2] = value; + break; + case "double": + HEAPF64[ptr >> 3] = value; + break; + case "*": + HEAPU32[ptr >> 2] = value; + break; + default: + abort(`invalid type for setValue: ${type}`); + } + } + var wasmImports = {}; + var wasmExports = createWasm(); + var ___wasm_call_ctors = () => (___wasm_call_ctors = wasmExports["__wasm_call_ctors"])(); + var _Hash_Update = Module2["_Hash_Update"] = (a0) => (_Hash_Update = Module2["_Hash_Update"] = wasmExports["Hash_Update"])(a0); + var _Hash_Final = Module2["_Hash_Final"] = () => (_Hash_Final = Module2["_Hash_Final"] = wasmExports["Hash_Final"])(); + var _Hash_Init = Module2["_Hash_Init"] = (a0) => (_Hash_Init = Module2["_Hash_Init"] = wasmExports["Hash_Init"])(a0); + var _GetBufferPtr = Module2["_GetBufferPtr"] = () => (_GetBufferPtr = Module2["_GetBufferPtr"] = wasmExports["GetBufferPtr"])(); + var stackSave = () => (stackSave = wasmExports["stackSave"])(); + var stackRestore = (a0) => (stackRestore = wasmExports["stackRestore"])(a0); + var stackAlloc = (a0) => (stackAlloc = wasmExports["stackAlloc"])(a0); + var calledRun; + dependenciesFulfilled = function runCaller() { + if (!calledRun) + run(); + if (!calledRun) + dependenciesFulfilled = runCaller; + }; + function run() { + if (runDependencies > 0) { + return; + } + preRun(); + if (runDependencies > 0) { + return; + } + function doRun() { + if (calledRun) + return; + calledRun = true; + Module2["calledRun"] = true; + if (ABORT) + return; + initRuntime(); + readyPromiseResolve(Module2); + if (Module2["onRuntimeInitialized"]) + Module2["onRuntimeInitialized"](); + postRun(); + } + if (Module2["setStatus"]) { + Module2["setStatus"]("Running..."); + setTimeout(function() { + setTimeout(function() { + Module2["setStatus"](""); + }, 1); + doRun(); + }, 1); + } else { + doRun(); + } + } + if (Module2["preInit"]) { + if (typeof Module2["preInit"] == "function") + Module2["preInit"] = [Module2["preInit"]]; + while (Module2["preInit"].length > 0) { + Module2["preInit"].pop()(); + } + } + run(); + return moduleArg.ready; + }; +})(); +var sha256_default = Module; + +// src/vendor/hash-wasm/sha256-wrapper.ts +async function createSHA256(isInsideWorker = false) { + const BUFFER_MAX_SIZE = 8 * 1024 * 1024; + const wasm = isInsideWorker ? ( + // @ts-expect-error WasmModule will be populated inside self object + await self["SHA256WasmModule"]() + ) : await sha256_default(); + const heap = wasm.HEAPU8.subarray(wasm._GetBufferPtr()); + return { + init() { + wasm._Hash_Init(256); + }, + update(data) { + let byteUsed = 0; + while (byteUsed < data.byteLength) { + const bytesLeft = data.byteLength - byteUsed; + const length = Math.min(bytesLeft, BUFFER_MAX_SIZE); + heap.set(data.subarray(byteUsed, byteUsed + length)); + wasm._Hash_Update(length); + byteUsed += length; + } + }, + digest(method) { + if (method !== "hex") { + throw new Error("Only digest hex is supported"); + } + wasm._Hash_Final(); + const result = Array.from(heap.slice(0, 32)); + return result.map((b) => b.toString(16).padStart(2, "0")).join(""); + } + }; +} +function createSHA256WorkerCode() { + return ` + self.addEventListener('message', async (event) => { + const { file } = event.data; + const sha256 = await self.createSHA256(true); + sha256.init(); + const reader = file.stream().getReader(); + const total = file.size; + let bytesDone = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + sha256.update(value); + bytesDone += value.length; + postMessage({ progress: bytesDone / total }); + } + postMessage({ sha256: sha256.digest('hex') }); + }); + self.SHA256WasmModule = ${sha256_default.toString()}; + self.createSHA256 = ${createSHA256.toString()}; + `; +} + + + +exports.createSHA256 = createSHA256; exports.createSHA256WorkerCode = createSHA256WorkerCode; diff --git a/node_modules/@huggingface/hub/dist/browser/sha256-wrapper-DYTB3MXW.mjs b/node_modules/@huggingface/hub/dist/browser/sha256-wrapper-DYTB3MXW.mjs new file mode 100644 index 0000000000000000000000000000000000000000..920134a3c8fed4778a4c4e7108e054bed9c16f5a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/browser/sha256-wrapper-DYTB3MXW.mjs @@ -0,0 +1,458 @@ +// src/vendor/hash-wasm/sha256.js +var Module = (() => { + var _unused = import.meta.url; + return function(moduleArg = {}) { + var Module2 = moduleArg; + var readyPromiseResolve, readyPromiseReject; + Module2["ready"] = new Promise((resolve, reject) => { + readyPromiseResolve = resolve; + readyPromiseReject = reject; + }); + var moduleOverrides = Object.assign({}, Module2); + var arguments_ = []; + var thisProgram = "./this.program"; + var quit_ = (status, toThrow) => { + throw toThrow; + }; + var ENVIRONMENT_IS_WEB = typeof window == "object"; + var ENVIRONMENT_IS_WORKER = typeof importScripts == "function"; + var ENVIRONMENT_IS_NODE = typeof process == "object" && typeof process.versions == "object" && typeof process.versions.node == "string"; + var ENVIRONMENT_IS_SHELL = !ENVIRONMENT_IS_WEB && !ENVIRONMENT_IS_NODE && !ENVIRONMENT_IS_WORKER; + var scriptDirectory = ""; + function locateFile(path) { + if (Module2["locateFile"]) { + return Module2["locateFile"](path, scriptDirectory); + } + return scriptDirectory + path; + } + var read_, readAsync, readBinary; + if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { + if (ENVIRONMENT_IS_WORKER) { + scriptDirectory = self.location.href; + } else if (typeof document != "undefined" && document.currentScript) { + scriptDirectory = document.currentScript.src; + } + if (false) { + scriptDirectory = false; + } + if (scriptDirectory.startsWith("blob:")) { + scriptDirectory = ""; + } else { + scriptDirectory = scriptDirectory.substr(0, scriptDirectory.replace(/[?#].*/, "").lastIndexOf("/") + 1); + } + { + read_ = (url) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, false); + xhr.send(null); + return xhr.responseText; + }; + if (ENVIRONMENT_IS_WORKER) { + readBinary = (url) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, false); + xhr.responseType = "arraybuffer"; + xhr.send(null); + return new Uint8Array( + /** @type{!ArrayBuffer} */ + xhr.response + ); + }; + } + readAsync = (url, onload, onerror) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, true); + xhr.responseType = "arraybuffer"; + xhr.onload = () => { + if (xhr.status == 200 || xhr.status == 0 && xhr.response) { + onload(xhr.response); + return; + } + onerror(); + }; + xhr.onerror = onerror; + xhr.send(null); + }; + } + } else { + } + var out = Module2["print"] || console.log.bind(console); + var err = Module2["printErr"] || console.error.bind(console); + Object.assign(Module2, moduleOverrides); + moduleOverrides = null; + if (Module2["arguments"]) + arguments_ = Module2["arguments"]; + if (Module2["thisProgram"]) + thisProgram = Module2["thisProgram"]; + if (Module2["quit"]) + quit_ = Module2["quit"]; + var wasmBinary; + if (Module2["wasmBinary"]) + wasmBinary = Module2["wasmBinary"]; + if (typeof WebAssembly != "object") { + abort("no native wasm support detected"); + } + function intArrayFromBase64(s) { + var decoded = atob(s); + var bytes = new Uint8Array(decoded.length); + for (var i = 0; i < decoded.length; ++i) { + bytes[i] = decoded.charCodeAt(i); + } + return bytes; + } + function tryParseAsDataURI(filename) { + if (!isDataURI(filename)) { + return; + } + return intArrayFromBase64(filename.slice(dataURIPrefix.length)); + } + var wasmMemory; + var ABORT = false; + var EXITSTATUS; + function assert(condition, text) { + if (!condition) { + abort(text); + } + } + var HEAP, HEAP8, HEAPU8, HEAP16, HEAPU16, HEAP32, HEAPU32, HEAPF32, HEAPF64; + function updateMemoryViews() { + var b = wasmMemory.buffer; + Module2["HEAP8"] = HEAP8 = new Int8Array(b); + Module2["HEAP16"] = HEAP16 = new Int16Array(b); + Module2["HEAPU8"] = HEAPU8 = new Uint8Array(b); + Module2["HEAPU16"] = HEAPU16 = new Uint16Array(b); + Module2["HEAP32"] = HEAP32 = new Int32Array(b); + Module2["HEAPU32"] = HEAPU32 = new Uint32Array(b); + Module2["HEAPF32"] = HEAPF32 = new Float32Array(b); + Module2["HEAPF64"] = HEAPF64 = new Float64Array(b); + } + var __ATPRERUN__ = []; + var __ATINIT__ = []; + var __ATEXIT__ = []; + var __ATPOSTRUN__ = []; + var runtimeInitialized = false; + function preRun() { + if (Module2["preRun"]) { + if (typeof Module2["preRun"] == "function") + Module2["preRun"] = [Module2["preRun"]]; + while (Module2["preRun"].length) { + addOnPreRun(Module2["preRun"].shift()); + } + } + callRuntimeCallbacks(__ATPRERUN__); + } + function initRuntime() { + runtimeInitialized = true; + callRuntimeCallbacks(__ATINIT__); + } + function postRun() { + if (Module2["postRun"]) { + if (typeof Module2["postRun"] == "function") + Module2["postRun"] = [Module2["postRun"]]; + while (Module2["postRun"].length) { + addOnPostRun(Module2["postRun"].shift()); + } + } + callRuntimeCallbacks(__ATPOSTRUN__); + } + function addOnPreRun(cb) { + __ATPRERUN__.unshift(cb); + } + function addOnInit(cb) { + __ATINIT__.unshift(cb); + } + function addOnExit(cb) { + } + function addOnPostRun(cb) { + __ATPOSTRUN__.unshift(cb); + } + var runDependencies = 0; + var runDependencyWatcher = null; + var dependenciesFulfilled = null; + function getUniqueRunDependency(id) { + return id; + } + function addRunDependency(id) { + runDependencies++; + Module2["monitorRunDependencies"]?.(runDependencies); + } + function removeRunDependency(id) { + runDependencies--; + Module2["monitorRunDependencies"]?.(runDependencies); + if (runDependencies == 0) { + if (runDependencyWatcher !== null) { + clearInterval(runDependencyWatcher); + runDependencyWatcher = null; + } + if (dependenciesFulfilled) { + var callback = dependenciesFulfilled; + dependenciesFulfilled = null; + callback(); + } + } + } + function abort(what) { + Module2["onAbort"]?.(what); + what = "Aborted(" + what + ")"; + err(what); + ABORT = true; + EXITSTATUS = 1; + what += ". Build with -sASSERTIONS for more info."; + var e = new WebAssembly.RuntimeError(what); + readyPromiseReject(e); + throw e; + } + var dataURIPrefix = "data:application/octet-stream;base64,"; + var isDataURI = (filename) => filename.startsWith(dataURIPrefix); + var isFileURI = (filename) => filename.startsWith("file://"); + var wasmBinaryFile; + wasmBinaryFile = "data:application/octet-stream;base64,AGFzbQEAAAABHQZgAX8AYAABf2AAAGABfwF/YAJ/fwBgA39/fwF/Aw0MAgAEAgMBBQABAQADBAUBcAEBAQUGAQGAAoACBg4CfwFB8IuEBAt/AUEACweYAQoGbWVtb3J5AgARX193YXNtX2NhbGxfY3RvcnMAAAtIYXNoX1VwZGF0ZQABCkhhc2hfRmluYWwAAwlIYXNoX0luaXQABAxHZXRCdWZmZXJQdHIABRlfX2luZGlyZWN0X2Z1bmN0aW9uX3RhYmxlAQAJc3RhY2tTYXZlAAkMc3RhY2tSZXN0b3JlAAoKc3RhY2tBbGxvYwALCossDAIAC+4CAgV/AX5BACgCwAoiASABKQNAIgYgAK18NwNAAkACQAJAIAanQT9xIgINAEGACyEBIAAhAgwBC0HAACACayEDAkAgAEUNACADIAAgAyAASRshBCABIAJqIQVBACEBA0AgBSABIgFqQYALIAFqLQAAOgAAIAFBAWoiAiEBIAIgBEcNAAsLAkACQCAAIANJIgRFDQBBgAshASAAIQIMAQtBACgCwAoiAUHIAGogARACQYALIANqIQEgACADayECCyABIQEgAiECIAQNAQsgASEBAkACQCACIgJBwABPDQAgASEFIAIhAAwBCyACIQIgASEEA0BBACgCwApByABqIAQiBBACIAJBQGoiASECIARBwABqIgUhBCAFIQUgASEAIAFBP0sNAAsLIAUhBSAAIgBFDQBBACEBQQAhAgNAQQAoAsAKIAEiAWogBSABai0AADoAACACQQFqIgJB/wFxIgQhASACIQIgACAESw0ACwsLqCEBK38gACgCCCICIAAoAgQiAyAAKAIAIgRzcSADIARxcyAEQR53IARBE3dzIARBCndzaiAAKAIQIgVBGncgBUEVd3MgBUEHd3MgACgCHCIGaiAAKAIYIgcgACgCFCIIcyAFcSAHc2ogASgCACIJQRh0IAlBgP4DcUEIdHIgCUEIdkGA/gNxIAlBGHZyciIKakGY36iUBGoiC2oiCSAEcyADcSAJIARxcyAJQR53IAlBE3dzIAlBCndzaiAHIAEoAgQiDEEYdCAMQYD+A3FBCHRyIAxBCHZBgP4DcSAMQRh2cnIiDWogCyAAKAIMIg5qIg8gCCAFc3EgCHNqIA9BGncgD0EVd3MgD0EHd3NqQZGJ3YkHaiIQaiIMIAlzIARxIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAggASgCCCILQRh0IAtBgP4DcUEIdHIgC0EIdkGA/gNxIAtBGHZyciIRaiAQIAJqIhIgDyAFc3EgBXNqIBJBGncgEkEVd3MgEkEHd3NqQc/3g657aiITaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAUgASgCDCIQQRh0IBBBgP4DcUEIdHIgEEEIdkGA/gNxIBBBGHZyciIUaiATIANqIhMgEiAPc3EgD3NqIBNBGncgE0EVd3MgE0EHd3NqQaW3181+aiIVaiIQIAtzIAxxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIA8gASgCECIWQRh0IBZBgP4DcUEIdHIgFkEIdkGA/gNxIBZBGHZyciIXaiAVIARqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQduE28oDaiIYaiIPIBBzIAtxIA8gEHFzIA9BHncgD0ETd3MgD0EKd3NqIAEoAhQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGSASaiAYIAlqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQfGjxM8FaiIYaiIJIA9zIBBxIAkgD3FzIAlBHncgCUETd3MgCUEKd3NqIAEoAhgiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGiATaiAYIAxqIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQaSF/pF5aiIYaiIMIAlzIA9xIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAEoAhwiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGyAWaiAYIAtqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQdW98dh6aiIYaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAEoAiAiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHCASaiAYIBBqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQZjVnsB9aiIYaiIQIAtzIAxxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAiQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHSATaiAYIA9qIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQYG2jZQBaiIYaiIPIBBzIAtxIA8gEHFzIA9BHncgD0ETd3MgD0EKd3NqIAEoAigiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHiAWaiAYIAlqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQb6LxqECaiIYaiIJIA9zIBBxIAkgD3FzIAlBHncgCUETd3MgCUEKd3NqIAEoAiwiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHyASaiAYIAxqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQcP7sagFaiIYaiIMIAlzIA9xIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAEoAjAiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiICATaiAYIAtqIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQfS6+ZUHaiIYaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAEoAjQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiISAWaiAYIBBqIhAgEyASc3EgEnNqIBBBGncgEEEVd3MgEEEHd3NqQf7j+oZ4aiIYaiIWIAtzIAxxIBYgC3FzIBZBHncgFkETd3MgFkEKd3NqIAEoAjgiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiIiASaiAYIA9qIg8gECATc3EgE3NqIA9BGncgD0EVd3MgD0EHd3NqQaeN8N55aiIVaiISIBZzIAtxIBIgFnFzIBJBHncgEkETd3MgEkEKd3NqIAEoAjwiAUEYdCABQYD+A3FBCHRyIAFBCHZBgP4DcSABQRh2cnIiIyATaiAVIAlqIgEgDyAQc3EgEHNqIAFBGncgAUEVd3MgAUEHd3NqQfTi74x8aiIJaiEVIBIhGCAWISQgCyElIAkgDGohJiABIScgDyEoIBAhKSAjISMgIiEiICEhISAgISAgHyEfIB4hHiAdIR0gHCEcIBshGyAaIRogGSEZIBchFyAUIRQgESERIA0hECAKIQxBgAkhAUEQISoDQCAVIgkgGCIKcyAkIitxIAkgCnFzIAlBHncgCUETd3MgCUEKd3NqIBAiEEEZdyAQQQ53cyAQQQN2cyAMaiAdIh1qICIiFkEPdyAWQQ13cyAWQQp2c2oiDCApaiAmIhIgJyIPICgiE3NxIBNzaiASQRp3IBJBFXdzIBJBB3dzaiABIgEoAgBqIiRqIgsgCXMgCnEgCyAJcXMgC0EedyALQRN3cyALQQp3c2ogESIYQRl3IBhBDndzIBhBA3ZzIBBqIB4iHmogIyIVQQ93IBVBDXdzIBVBCnZzaiINIBNqIAEoAgRqICQgJWoiEyASIA9zcSAPc2ogE0EadyATQRV3cyATQQd3c2oiJWoiECALcyAJcSAQIAtxcyAQQR53IBBBE3dzIBBBCndzaiAUIiRBGXcgJEEOd3MgJEEDdnMgGGogHyIfaiAMQQ93IAxBDXdzIAxBCnZzaiIRIA9qIAEoAghqICUgK2oiGCATIBJzcSASc2ogGEEadyAYQRV3cyAYQQd3c2oiJWoiDyAQcyALcSAPIBBxcyAPQR53IA9BE3dzIA9BCndzaiAXIhdBGXcgF0EOd3MgF0EDdnMgJGogICIgaiANQQ93IA1BDXdzIA1BCnZzaiIUIBJqIAEoAgxqICUgCmoiCiAYIBNzcSATc2ogCkEadyAKQRV3cyAKQQd3c2oiJWoiEiAPcyAQcSASIA9xcyASQR53IBJBE3dzIBJBCndzaiATIBkiJEEZdyAkQQ53cyAkQQN2cyAXaiAhIiFqIBFBD3cgEUENd3MgEUEKdnNqIhdqIAEoAhBqICUgCWoiEyAKIBhzcSAYc2ogE0EadyATQRV3cyATQQd3c2oiJWoiCSAScyAPcSAJIBJxcyAJQR53IAlBE3dzIAlBCndzaiABKAIUIBoiGkEZdyAaQQ53cyAaQQN2cyAkaiAWaiAUQQ93IBRBDXdzIBRBCnZzaiIZaiAYaiAlIAtqIhggEyAKc3EgCnNqIBhBGncgGEEVd3MgGEEHd3NqIiVqIgsgCXMgEnEgCyAJcXMgC0EedyALQRN3cyALQQp3c2ogASgCGCAbIiRBGXcgJEEOd3MgJEEDdnMgGmogFWogF0EPdyAXQQ13cyAXQQp2c2oiGmogCmogJSAQaiIKIBggE3NxIBNzaiAKQRp3IApBFXdzIApBB3dzaiIlaiIQIAtzIAlxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAhwgHCIcQRl3IBxBDndzIBxBA3ZzICRqIAxqIBlBD3cgGUENd3MgGUEKdnNqIhtqIBNqICUgD2oiJCAKIBhzcSAYc2ogJEEadyAkQRV3cyAkQQd3c2oiE2oiDyAQcyALcSAPIBBxcyAPQR53IA9BE3dzIA9BCndzaiABKAIgIB1BGXcgHUEOd3MgHUEDdnMgHGogDWogGkEPdyAaQQ13cyAaQQp2c2oiHGogGGogEyASaiIYICQgCnNxIApzaiAYQRp3IBhBFXdzIBhBB3dzaiITaiISIA9zIBBxIBIgD3FzIBJBHncgEkETd3MgEkEKd3NqIAEoAiQgHkEZdyAeQQ53cyAeQQN2cyAdaiARaiAbQQ93IBtBDXdzIBtBCnZzaiIdaiAKaiATIAlqIgkgGCAkc3EgJHNqIAlBGncgCUEVd3MgCUEHd3NqIgpqIhMgEnMgD3EgEyAScXMgE0EedyATQRN3cyATQQp3c2ogASgCKCAfQRl3IB9BDndzIB9BA3ZzIB5qIBRqIBxBD3cgHEENd3MgHEEKdnNqIh5qICRqIAogC2oiCiAJIBhzcSAYc2ogCkEadyAKQRV3cyAKQQd3c2oiJGoiCyATcyAScSALIBNxcyALQR53IAtBE3dzIAtBCndzaiABKAIsICBBGXcgIEEOd3MgIEEDdnMgH2ogF2ogHUEPdyAdQQ13cyAdQQp2c2oiH2ogGGogJCAQaiIYIAogCXNxIAlzaiAYQRp3IBhBFXdzIBhBB3dzaiIkaiIQIAtzIBNxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAjAgIUEZdyAhQQ53cyAhQQN2cyAgaiAZaiAeQQ93IB5BDXdzIB5BCnZzaiIgaiAJaiAkIA9qIiQgGCAKc3EgCnNqICRBGncgJEEVd3MgJEEHd3NqIg9qIgkgEHMgC3EgCSAQcXMgCUEedyAJQRN3cyAJQQp3c2ogASgCNCAWQRl3IBZBDndzIBZBA3ZzICFqIBpqIB9BD3cgH0ENd3MgH0EKdnNqIiFqIApqIA8gEmoiDyAkIBhzcSAYc2ogD0EadyAPQRV3cyAPQQd3c2oiCmoiEiAJcyAQcSASIAlxcyASQR53IBJBE3dzIBJBCndzaiABKAI4IBVBGXcgFUEOd3MgFUEDdnMgFmogG2ogIEEPdyAgQQ13cyAgQQp2c2oiImogGGogCiATaiITIA8gJHNxICRzaiATQRp3IBNBFXdzIBNBB3dzaiIYaiIWIBJzIAlxIBYgEnFzIBZBHncgFkETd3MgFkEKd3NqIAEoAjwgDEEZdyAMQQ53cyAMQQN2cyAVaiAcaiAhQQ93ICFBDXdzICFBCnZzaiIKaiAkaiAYIAtqIgsgEyAPc3EgD3NqIAtBGncgC0EVd3MgC0EHd3NqIiZqIishFSAWIRggEiEkIAkhJSAmIBBqIiwhJiALIScgEyEoIA8hKSAKISMgIiEiICEhISAgISAgHyEfIB4hHiAdIR0gHCEcIBshGyAaIRogGSEZIBchFyAUIRQgESERIA0hECAMIQwgAUHAAGohASAqIgpBEGohKiAKQTBJDQALIAAgDyAGajYCHCAAIBMgB2o2AhggACALIAhqNgIUIAAgLCAFajYCECAAIAkgDmo2AgwgACASIAJqNgIIIAAgFiADajYCBCAAICsgBGo2AgAL1AMDBX8BfgF7QQAoAsAKIgAgACgCQCIBQQJ2QQ9xIgJBAnRqIgMgAygCAEF/IAFBA3QiAXRBf3NxQYABIAF0czYCAAJAAkAgAkEOTw0AIAJBAWohAAwBCwJAIAJBDkcNACAAQQA2AjwLIABByABqIAAQAkEAIQALAkAgACIAQQ1LDQBBACgCwAogAEECdCIAakEAQTggAGsQBhoLQQAoAsAKIgAgACkDQCIFpyICQRt0IAJBC3RBgID8B3FyIAJBBXZBgP4DcSACQQN0QRh2cnI2AjwgACAFQh2IpyICQRh0IAJBgP4DcUEIdHIgAkEIdkGA/gNxIAJBGHZycjYCOCAAQcgAaiAAEAJBACgCwApBPGohAUEAIQADQCABQQcgACIAa0ECdGoiAiAC/QACACAG/Q0MDQ4PCAkKCwQFBgcAAQIDIAb9DQMCAQAHBgUECwoJCA8ODQwgBv0NDA0ODwgJCgsEBQYHAAECA/0LAgAgAEEEaiICIQAgAkEIRw0ACwJAQQAoAsAKIgMoAmhFDQAgA0HIAGohBEEAIQBBACECA0BBgAsgACIAaiAEIABqLQAAOgAAIAJBAWoiAkH/AXEiASEAIAIhAiADKAJoIAFLDQALCwtxAQJ/QQAoAsAKIgFCADcDQCABQcgAaiECAkAgAEHgAUcNACABQRw2AmggAkEQakEA/QAEsAj9CwIAIAJBAP0ABKAI/QsCAEEADwsgAUEgNgJoIAJBEGpBAP0ABJAI/QsCACACQQD9AASACP0LAgBBAAsFAEGACwvyAgIDfwF+AkAgAkUNACAAIAE6AAAgACACaiIDQX9qIAE6AAAgAkEDSQ0AIAAgAToAAiAAIAE6AAEgA0F9aiABOgAAIANBfmogAToAACACQQdJDQAgACABOgADIANBfGogAToAACACQQlJDQAgAEEAIABrQQNxIgRqIgMgAUH/AXFBgYKECGwiATYCACADIAIgBGtBfHEiBGoiAkF8aiABNgIAIARBCUkNACADIAE2AgggAyABNgIEIAJBeGogATYCACACQXRqIAE2AgAgBEEZSQ0AIAMgATYCGCADIAE2AhQgAyABNgIQIAMgATYCDCACQXBqIAE2AgAgAkFsaiABNgIAIAJBaGogATYCACACQWRqIAE2AgAgBCADQQRxQRhyIgVrIgJBIEkNACABrUKBgICAEH4hBiADIAVqIQEDQCABIAY3AxggASAGNwMQIAEgBjcDCCABIAY3AwAgAUEgaiEBIAJBYGoiAkEfSw0ACwsgAAsGACAAJAELBAAjAQsEACMACwYAIAAkAAsSAQJ/IwAgAGtBcHEiASQAIAELC9ICAgBBgAgLwAJn5glqha5nu3Lzbjw69U+lf1IOUYxoBZur2YMfGc3gW9ieBcEH1Xw2F91wMDlZDvcxC8D/ERVYaKeP+WSkT/q+mC+KQpFEN3HP+8C1pdu16VvCVjnxEfFZpII/ktVeHKuYqgfYAVuDEr6FMSTDfQxVdF2+cv6x3oCnBtybdPGbwcFpm+SGR77vxp3BD8yhDCRvLOktqoR0StypsFzaiPl2UlE+mG3GMajIJwOwx39Zv/ML4MZHkafVUWPKBmcpKRSFCrcnOCEbLvxtLE0TDThTVHMKZbsKanYuycKBhSxykqHov6JLZhqocItLwqNRbMcZ6JLRJAaZ1oU1DvRwoGoQFsGkGQhsNx5Md0gntbywNLMMHDlKqthOT8qcW/NvLmjugo90b2OleBR4yIQIAseM+v++kOtsUKT3o/m+8nhxxgBBwAoLBIAFgAA="; + if (!isDataURI(wasmBinaryFile)) { + wasmBinaryFile = locateFile(wasmBinaryFile); + } + function getBinarySync(file) { + if (file == wasmBinaryFile && wasmBinary) { + return new Uint8Array(wasmBinary); + } + var binary = tryParseAsDataURI(file); + if (binary) { + return binary; + } + if (readBinary) { + return readBinary(file); + } + throw "both async and sync fetching of the wasm failed"; + } + function getBinaryPromise(binaryFile) { + return Promise.resolve().then(() => getBinarySync(binaryFile)); + } + function instantiateArrayBuffer(binaryFile, imports, receiver) { + return getBinaryPromise(binaryFile).then((binary) => { + return WebAssembly.instantiate(binary, imports); + }).then(receiver, (reason) => { + err(`failed to asynchronously prepare wasm: ${reason}`); + abort(reason); + }); + } + function instantiateAsync(binary, binaryFile, imports, callback) { + return instantiateArrayBuffer(binaryFile, imports, callback); + } + function createWasm() { + var info = { + "env": wasmImports, + "wasi_snapshot_preview1": wasmImports + }; + function receiveInstance(instance, module) { + wasmExports = instance.exports; + wasmMemory = wasmExports["memory"]; + updateMemoryViews(); + addOnInit(wasmExports["__wasm_call_ctors"]); + removeRunDependency("wasm-instantiate"); + return wasmExports; + } + addRunDependency("wasm-instantiate"); + function receiveInstantiationResult(result) { + receiveInstance(result["instance"]); + } + if (Module2["instantiateWasm"]) { + try { + return Module2["instantiateWasm"](info, receiveInstance); + } catch (e) { + err(`Module.instantiateWasm callback failed with error: ${e}`); + readyPromiseReject(e); + } + } + instantiateAsync(wasmBinary, wasmBinaryFile, info, receiveInstantiationResult).catch(readyPromiseReject); + return {}; + } + var tempDouble; + var tempI64; + function ExitStatus(status) { + this.name = "ExitStatus"; + this.message = `Program terminated with exit(${status})`; + this.status = status; + } + var callRuntimeCallbacks = (callbacks) => { + while (callbacks.length > 0) { + callbacks.shift()(Module2); + } + }; + function getValue(ptr, type = "i8") { + if (type.endsWith("*")) + type = "*"; + switch (type) { + case "i1": + return HEAP8[ptr]; + case "i8": + return HEAP8[ptr]; + case "i16": + return HEAP16[ptr >> 1]; + case "i32": + return HEAP32[ptr >> 2]; + case "i64": + abort("to do getValue(i64) use WASM_BIGINT"); + case "float": + return HEAPF32[ptr >> 2]; + case "double": + return HEAPF64[ptr >> 3]; + case "*": + return HEAPU32[ptr >> 2]; + default: + abort(`invalid type for getValue: ${type}`); + } + } + var noExitRuntime = Module2["noExitRuntime"] || true; + function setValue(ptr, value, type = "i8") { + if (type.endsWith("*")) + type = "*"; + switch (type) { + case "i1": + HEAP8[ptr] = value; + break; + case "i8": + HEAP8[ptr] = value; + break; + case "i16": + HEAP16[ptr >> 1] = value; + break; + case "i32": + HEAP32[ptr >> 2] = value; + break; + case "i64": + abort("to do setValue(i64) use WASM_BIGINT"); + case "float": + HEAPF32[ptr >> 2] = value; + break; + case "double": + HEAPF64[ptr >> 3] = value; + break; + case "*": + HEAPU32[ptr >> 2] = value; + break; + default: + abort(`invalid type for setValue: ${type}`); + } + } + var wasmImports = {}; + var wasmExports = createWasm(); + var ___wasm_call_ctors = () => (___wasm_call_ctors = wasmExports["__wasm_call_ctors"])(); + var _Hash_Update = Module2["_Hash_Update"] = (a0) => (_Hash_Update = Module2["_Hash_Update"] = wasmExports["Hash_Update"])(a0); + var _Hash_Final = Module2["_Hash_Final"] = () => (_Hash_Final = Module2["_Hash_Final"] = wasmExports["Hash_Final"])(); + var _Hash_Init = Module2["_Hash_Init"] = (a0) => (_Hash_Init = Module2["_Hash_Init"] = wasmExports["Hash_Init"])(a0); + var _GetBufferPtr = Module2["_GetBufferPtr"] = () => (_GetBufferPtr = Module2["_GetBufferPtr"] = wasmExports["GetBufferPtr"])(); + var stackSave = () => (stackSave = wasmExports["stackSave"])(); + var stackRestore = (a0) => (stackRestore = wasmExports["stackRestore"])(a0); + var stackAlloc = (a0) => (stackAlloc = wasmExports["stackAlloc"])(a0); + var calledRun; + dependenciesFulfilled = function runCaller() { + if (!calledRun) + run(); + if (!calledRun) + dependenciesFulfilled = runCaller; + }; + function run() { + if (runDependencies > 0) { + return; + } + preRun(); + if (runDependencies > 0) { + return; + } + function doRun() { + if (calledRun) + return; + calledRun = true; + Module2["calledRun"] = true; + if (ABORT) + return; + initRuntime(); + readyPromiseResolve(Module2); + if (Module2["onRuntimeInitialized"]) + Module2["onRuntimeInitialized"](); + postRun(); + } + if (Module2["setStatus"]) { + Module2["setStatus"]("Running..."); + setTimeout(function() { + setTimeout(function() { + Module2["setStatus"](""); + }, 1); + doRun(); + }, 1); + } else { + doRun(); + } + } + if (Module2["preInit"]) { + if (typeof Module2["preInit"] == "function") + Module2["preInit"] = [Module2["preInit"]]; + while (Module2["preInit"].length > 0) { + Module2["preInit"].pop()(); + } + } + run(); + return moduleArg.ready; + }; +})(); +var sha256_default = Module; + +// src/vendor/hash-wasm/sha256-wrapper.ts +async function createSHA256(isInsideWorker = false) { + const BUFFER_MAX_SIZE = 8 * 1024 * 1024; + const wasm = isInsideWorker ? ( + // @ts-expect-error WasmModule will be populated inside self object + await self["SHA256WasmModule"]() + ) : await sha256_default(); + const heap = wasm.HEAPU8.subarray(wasm._GetBufferPtr()); + return { + init() { + wasm._Hash_Init(256); + }, + update(data) { + let byteUsed = 0; + while (byteUsed < data.byteLength) { + const bytesLeft = data.byteLength - byteUsed; + const length = Math.min(bytesLeft, BUFFER_MAX_SIZE); + heap.set(data.subarray(byteUsed, byteUsed + length)); + wasm._Hash_Update(length); + byteUsed += length; + } + }, + digest(method) { + if (method !== "hex") { + throw new Error("Only digest hex is supported"); + } + wasm._Hash_Final(); + const result = Array.from(heap.slice(0, 32)); + return result.map((b) => b.toString(16).padStart(2, "0")).join(""); + } + }; +} +function createSHA256WorkerCode() { + return ` + self.addEventListener('message', async (event) => { + const { file } = event.data; + const sha256 = await self.createSHA256(true); + sha256.init(); + const reader = file.stream().getReader(); + const total = file.size; + let bytesDone = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + sha256.update(value); + bytesDone += value.length; + postMessage({ progress: bytesDone / total }); + } + postMessage({ sha256: sha256.digest('hex') }); + }); + self.SHA256WasmModule = ${sha256_default.toString()}; + self.createSHA256 = ${createSHA256.toString()}; + `; +} +export { + createSHA256, + createSHA256WorkerCode +}; diff --git a/node_modules/@huggingface/hub/dist/browser/sub-paths-F6TP7MGR.mjs b/node_modules/@huggingface/hub/dist/browser/sub-paths-F6TP7MGR.mjs new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/node_modules/@huggingface/hub/dist/browser/sub-paths-RH3O65LG.js b/node_modules/@huggingface/hub/dist/browser/sub-paths-RH3O65LG.js new file mode 100644 index 0000000000000000000000000000000000000000..9a390c31f71bc7eae1522a280a2dc8f6723185bf --- /dev/null +++ b/node_modules/@huggingface/hub/dist/browser/sub-paths-RH3O65LG.js @@ -0,0 +1 @@ +"use strict"; \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/chunk-FFYIGW52.mjs b/node_modules/@huggingface/hub/dist/chunk-FFYIGW52.mjs new file mode 100644 index 0000000000000000000000000000000000000000..f2b17d00160926ef1834bbe46f0a62fe88800d05 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/chunk-FFYIGW52.mjs @@ -0,0 +1,16 @@ +var __getOwnPropNames = Object.getOwnPropertyNames; +var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, { + get: (a, b) => (typeof require !== "undefined" ? require : a)[b] +}) : x)(function(x) { + if (typeof require !== "undefined") + return require.apply(this, arguments); + throw new Error('Dynamic require of "' + x + '" is not supported'); +}); +var __commonJS = (cb, mod) => function __require2() { + return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports; +}; + +export { + __require, + __commonJS +}; diff --git a/node_modules/@huggingface/hub/dist/chunk-OPQ3EOKY.mjs b/node_modules/@huggingface/hub/dist/chunk-OPQ3EOKY.mjs new file mode 100644 index 0000000000000000000000000000000000000000..73dcc4969cb2cb95343a4c641056057eb6990a14 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/chunk-OPQ3EOKY.mjs @@ -0,0 +1,6395 @@ +// src/lib/cache-management.ts +import { homedir } from "os"; +import { join, basename } from "path"; +import { stat, readdir, readFile, realpath, lstat } from "fs/promises"; +function getDefaultHome() { + return join(homedir(), ".cache"); +} +function getDefaultCachePath() { + return join(process.env["HF_HOME"] ?? join(process.env["XDG_CACHE_HOME"] ?? getDefaultHome(), "huggingface"), "hub"); +} +function getHuggingFaceHubCache() { + return process.env["HUGGINGFACE_HUB_CACHE"] ?? getDefaultCachePath(); +} +function getHFHubCachePath() { + return process.env["HF_HUB_CACHE"] ?? getHuggingFaceHubCache(); +} +var FILES_TO_IGNORE = [".DS_Store"]; +var REPO_ID_SEPARATOR = "--"; +function getRepoFolderName({ name, type }) { + const parts = [`${type}s`, ...name.split("/")]; + return parts.join(REPO_ID_SEPARATOR); +} +async function scanCacheDir(cacheDir = void 0) { + if (!cacheDir) { + cacheDir = getHFHubCachePath(); + } + const s = await stat(cacheDir); + if (!s.isDirectory()) { + throw new Error( + `Scan cache expects a directory but found a file: ${cacheDir}. Please use \`cacheDir\` argument or set \`HF_HUB_CACHE\` environment variable.` + ); + } + const repos = []; + const warnings = []; + const directories = await readdir(cacheDir); + for (const repo of directories) { + if (repo === ".locks") { + continue; + } + const absolute = join(cacheDir, repo); + const s2 = await stat(absolute); + if (!s2.isDirectory()) { + continue; + } + try { + const cached = await scanCachedRepo(absolute); + repos.push(cached); + } catch (err) { + warnings.push(err); + } + } + return { + repos, + size: [...repos.values()].reduce((sum2, repo) => sum2 + repo.size, 0), + warnings + }; +} +async function scanCachedRepo(repoPath) { + const name = basename(repoPath); + if (!name.includes(REPO_ID_SEPARATOR)) { + throw new Error(`Repo path is not a valid HuggingFace cache directory: ${name}`); + } + const [type, ...remaining] = name.split(REPO_ID_SEPARATOR); + const repoType = parseRepoType(type); + const repoId = remaining.join("/"); + const snapshotsPath = join(repoPath, "snapshots"); + const refsPath = join(repoPath, "refs"); + const snapshotStat = await stat(snapshotsPath); + if (!snapshotStat.isDirectory()) { + throw new Error(`Snapshots dir doesn't exist in cached repo ${snapshotsPath}`); + } + const refsByHash = /* @__PURE__ */ new Map(); + const refsStat = await stat(refsPath); + if (refsStat.isDirectory()) { + await scanRefsDir(refsPath, refsByHash); + } + const cachedRevisions = []; + const blobStats = /* @__PURE__ */ new Map(); + const snapshotDirs = await readdir(snapshotsPath); + for (const dir of snapshotDirs) { + if (FILES_TO_IGNORE.includes(dir)) { + continue; + } + const revisionPath = join(snapshotsPath, dir); + const revisionStat = await stat(revisionPath); + if (!revisionStat.isDirectory()) { + throw new Error(`Snapshots folder corrupted. Found a file: ${revisionPath}`); + } + const cachedFiles = []; + await scanSnapshotDir(revisionPath, cachedFiles, blobStats); + const revisionLastModified = cachedFiles.length > 0 ? Math.max(...[...cachedFiles].map((file) => file.blob.lastModifiedAt.getTime())) : revisionStat.mtimeMs; + cachedRevisions.push({ + commitOid: dir, + files: cachedFiles, + refs: refsByHash.get(dir) || [], + size: [...cachedFiles].reduce((sum2, file) => sum2 + file.blob.size, 0), + path: revisionPath, + lastModifiedAt: new Date(revisionLastModified) + }); + refsByHash.delete(dir); + } + if (refsByHash.size > 0) { + throw new Error( + `Reference(s) refer to missing commit hashes: ${JSON.stringify(Object.fromEntries(refsByHash))} (${repoPath})` + ); + } + const repoStats = await stat(repoPath); + const repoLastAccessed = blobStats.size > 0 ? Math.max(...[...blobStats.values()].map((stat3) => stat3.atimeMs)) : repoStats.atimeMs; + const repoLastModified = blobStats.size > 0 ? Math.max(...[...blobStats.values()].map((stat3) => stat3.mtimeMs)) : repoStats.mtimeMs; + return { + id: { + name: repoId, + type: repoType + }, + path: repoPath, + filesCount: blobStats.size, + revisions: cachedRevisions, + size: [...blobStats.values()].reduce((sum2, stat3) => sum2 + stat3.size, 0), + lastAccessedAt: new Date(repoLastAccessed), + lastModifiedAt: new Date(repoLastModified) + }; +} +async function scanRefsDir(refsPath, refsByHash) { + const refFiles = await readdir(refsPath, { withFileTypes: true }); + for (const refFile of refFiles) { + const refFilePath = join(refsPath, refFile.name); + if (refFile.isDirectory()) { + continue; + } + const commitHash = await readFile(refFilePath, "utf-8"); + const refName = refFile.name; + if (!refsByHash.has(commitHash)) { + refsByHash.set(commitHash, []); + } + refsByHash.get(commitHash)?.push(refName); + } +} +async function scanSnapshotDir(revisionPath, cachedFiles, blobStats) { + const files = await readdir(revisionPath, { withFileTypes: true }); + for (const file of files) { + if (file.isDirectory()) { + continue; + } + const filePath = join(revisionPath, file.name); + const blobPath = await realpath(filePath); + const blobStat = await getBlobStat(blobPath, blobStats); + cachedFiles.push({ + path: filePath, + blob: { + path: blobPath, + size: blobStat.size, + lastAccessedAt: new Date(blobStat.atimeMs), + lastModifiedAt: new Date(blobStat.mtimeMs) + } + }); + } +} +async function getBlobStat(blobPath, blobStats) { + const blob = blobStats.get(blobPath); + if (!blob) { + const statResult = await lstat(blobPath); + blobStats.set(blobPath, statResult); + return statResult; + } + return blob; +} +function parseRepoType(type) { + switch (type) { + case "models": + return "model"; + case "datasets": + return "dataset"; + case "spaces": + return "space"; + case "buckets": + return "bucket"; + case "kernels": + return "kernel"; + default: + throw new TypeError(`Invalid repo type: ${type}`); + } +} + +// src/consts.ts +var HUB_URL = "https://huggingface.co"; + +// src/error.ts +async function createApiError(response, opts) { + const error = new HubApiError(response.url, response.status, response.headers.get("X-Request-Id") ?? opts?.requestId); + error.message = `Api error with status ${error.statusCode}${opts?.message ? `. ${opts.message}` : ""}`; + const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : void 0].filter(Boolean).join(". "); + if (response.headers.get("Content-Type")?.startsWith("application/json")) { + const json = await response.json(); + error.message = json.error || json.message || error.message; + if (json.error_description) { + error.message = error.message ? error.message + `: ${json.error_description}` : json.error_description; + } + error.data = json; + } else { + error.data = { message: await response.text() }; + } + error.message += `. ${trailer}`; + throw error; +} +var HubApiError = class extends Error { + statusCode; + url; + requestId; + data; + constructor(url, statusCode, requestId, message) { + super(message); + this.statusCode = statusCode; + this.requestId = requestId; + this.url = url; + } +}; +var InvalidApiResponseFormatError = class extends Error { +}; + +// src/utils/checkCredentials.ts +function checkAccessToken(accessToken) { + if (!accessToken.startsWith("hf_")) { + throw new TypeError("Your access token must start with 'hf_'"); + } +} +function checkCredentials(params) { + if (params.accessToken) { + checkAccessToken(params.accessToken); + return params.accessToken; + } + if (params.credentials?.accessToken) { + checkAccessToken(params.credentials.accessToken); + return params.credentials.accessToken; + } +} + +// src/utils/toRepoId.ts +function toRepoId(repo) { + if (typeof repo !== "string") { + return repo; + } + if (repo.startsWith("model/") || repo.startsWith("models/")) { + throw new TypeError( + "A repo designation for a model should not start with 'models/', directly specify the model namespace / name" + ); + } + if (repo.startsWith("space/")) { + throw new TypeError("Spaces should start with 'spaces/', plural, not 'space/'"); + } + if (repo.startsWith("dataset/")) { + throw new TypeError("Datasets should start with 'datasets/', plural, not 'dataset/'"); + } + if (repo.startsWith("bucket/")) { + throw new TypeError("Buckets should start with 'buckets/', plural, not 'bucket/'"); + } + if (repo.startsWith("kernel/")) { + throw new TypeError("Kernels should start with 'kernels/', plural, not 'kernel/'"); + } + const slashes = repo.split("/").length - 1; + if (repo.startsWith("spaces/")) { + if (slashes !== 2) { + throw new TypeError("Space Id must include namespace and name of the space"); + } + return { + type: "space", + name: repo.slice("spaces/".length) + }; + } + if (repo.startsWith("datasets/")) { + if (slashes > 2) { + throw new TypeError("Too many slashes in repo designation: " + repo); + } + return { + type: "dataset", + name: repo.slice("datasets/".length) + }; + } + if (repo.startsWith("buckets/")) { + if (slashes !== 2) { + throw new TypeError("Bucket Id must include namespace and name of the bucket"); + } + return { + type: "bucket", + name: repo.slice("buckets/".length) + }; + } + if (repo.startsWith("kernels/")) { + if (slashes !== 2) { + throw new TypeError("Kernel Id must include namespace and name of the kernel"); + } + return { + type: "kernel", + name: repo.slice("kernels/".length) + }; + } + if (slashes > 1) { + throw new TypeError("Too many slashes in repo designation: " + repo); + } + return { + type: "model", + name: repo + }; +} + +// src/lib/check-repo-access.ts +async function checkRepoAccess(params) { + const accessToken = params && checkCredentials(params); + const repoId = toRepoId(params.repo); + const response = await (params.fetch || fetch)(`${params?.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}`, { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + }); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/utils/eventToGenerator.ts +async function* eventToGenerator(cb) { + const promises = []; + function addPromise() { + let resolve3; + let reject; + const p = new Promise((res, rej) => { + resolve3 = res; + reject = rej; + }); + promises.push({ p, resolve: resolve3, reject }); + } + addPromise(); + const callbackRes = Promise.resolve().then( + () => cb( + (y) => { + addPromise(); + promises.at(-2)?.resolve({ done: false, value: y }); + }, + (r) => { + addPromise(); + promises.at(-2)?.resolve({ done: true, value: r }); + }, + (err) => promises.shift()?.reject(err) + ) + ).catch((err) => promises.shift()?.reject(err)); + while (1) { + const p = promises[0]; + if (!p) { + throw new Error("Logic error in eventGenerator, promises should never be empty"); + } + const result = await p.p; + promises.shift(); + if (result.done) { + await callbackRes; + return result.value; + } + yield result.value; + } + throw new Error("Unreachable"); +} + +// src/utils/hexFromBytes.ts +function hexFromBytes(arr) { + if (globalThis.Buffer) { + return globalThis.Buffer.from(arr).toString("hex"); + } else { + const bin = []; + arr.forEach((byte) => { + bin.push(byte.toString(16).padStart(2, "0")); + }); + return bin.join(""); + } +} + +// src/utils/isBackend.ts +var isBrowser = typeof window !== "undefined" && typeof window.document !== "undefined"; +var isWebWorker = typeof self === "object" && self.constructor && self.constructor.name === "DedicatedWorkerGlobalScope"; +var isBackend = !isBrowser && !isWebWorker; + +// src/utils/isFrontend.ts +var isFrontend = !isBackend; + +// src/utils/sha256.ts +async function getWebWorkerCode() { + const sha256Module = await import("./sha256-wrapper-ITDNMKRK.mjs"); + return URL.createObjectURL(new Blob([sha256Module.createSHA256WorkerCode()])); +} +var pendingWorkers = []; +var runningWorkers = /* @__PURE__ */ new Set(); +var resolve; +var waitPromise = new Promise((r) => { + resolve = r; +}); +async function getWorker(poolSize) { + { + const worker2 = pendingWorkers.pop(); + if (worker2) { + runningWorkers.add(worker2); + return worker2; + } + } + if (!poolSize) { + const worker2 = new Worker(await getWebWorkerCode()); + runningWorkers.add(worker2); + return worker2; + } + if (poolSize <= 0) { + throw new TypeError("Invalid webworker pool size: " + poolSize); + } + while (runningWorkers.size >= poolSize) { + await waitPromise; + } + const worker = new Worker(await getWebWorkerCode()); + runningWorkers.add(worker); + return worker; +} +async function freeWorker(worker, poolSize) { + if (!poolSize) { + return destroyWorker(worker); + } + runningWorkers.delete(worker); + pendingWorkers.push(worker); + const r = resolve; + waitPromise = new Promise((r2) => { + resolve = r2; + }); + r(); +} +function destroyWorker(worker) { + runningWorkers.delete(worker); + worker.terminate(); + const r = resolve; + waitPromise = new Promise((r2) => { + resolve = r2; + }); + r(); +} +async function* sha256(buffer, opts) { + yield 0; + const maxCryptoSize = typeof opts?.useWebWorker === "object" && opts?.useWebWorker.minSize !== void 0 ? opts.useWebWorker.minSize : 1e7; + if (buffer.size < maxCryptoSize && globalThis.crypto?.subtle) { + const res = hexFromBytes( + new Uint8Array( + await globalThis.crypto.subtle.digest("SHA-256", buffer instanceof Blob ? await buffer.arrayBuffer() : buffer) + ) + ); + yield 1; + return res; + } + if (isFrontend) { + if (opts?.useWebWorker) { + try { + const poolSize = typeof opts?.useWebWorker === "object" ? opts.useWebWorker.poolSize : void 0; + const worker = await getWorker(poolSize); + let messageHandler; + let errorHandler; + const cleanup = () => { + worker.removeEventListener("message", messageHandler); + worker.removeEventListener("error", errorHandler); + }; + return yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + messageHandler = (event) => { + if (event.data.sha256) { + cleanup(); + freeWorker(worker, poolSize); + returnCallback(event.data.sha256); + } else if (event.data.progress) { + yieldCallback(event.data.progress); + try { + opts.abortSignal?.throwIfAborted(); + } catch (err) { + cleanup(); + destroyWorker(worker); + rejectCallback(err); + } + } else { + cleanup(); + destroyWorker(worker); + rejectCallback(event); + } + }; + errorHandler = (event) => { + cleanup(); + destroyWorker(worker); + rejectCallback(event.error); + }; + if (opts?.abortSignal) { + try { + opts.abortSignal?.throwIfAborted(); + } catch (err) { + cleanup(); + destroyWorker(worker); + rejectCallback(opts.abortSignal.reason ?? new DOMException("Aborted", "AbortError")); + return; + } + const abortListener = () => { + cleanup(); + destroyWorker(worker); + rejectCallback(opts.abortSignal?.reason ?? new DOMException("Aborted", "AbortError")); + opts.abortSignal?.removeEventListener("abort", abortListener); + }; + opts.abortSignal.addEventListener("abort", abortListener); + } + worker.addEventListener("message", messageHandler); + worker.addEventListener("error", errorHandler); + worker.postMessage({ file: buffer }); + }); + } catch (err) { + console.warn("Failed to use web worker for sha256", err); + } + } + if (!wasmModule) { + wasmModule = await import("./sha256-wrapper-ITDNMKRK.mjs"); + } + const sha2562 = await wasmModule.createSHA256(); + sha2562.init(); + const reader = buffer.stream().getReader(); + const total = buffer.size; + let bytesDone = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + sha2562.update(value); + bytesDone += value.length; + yield bytesDone / total; + opts?.abortSignal?.throwIfAborted(); + } + return sha2562.digest("hex"); + } + if (!cryptoModule) { + cryptoModule = await import("./sha256-node-ZPWO3OWR.mjs"); + } + return yield* cryptoModule.sha256Node(buffer, { abortSignal: opts?.abortSignal }); +} +var cryptoModule; +var wasmModule; + +// src/utils/combineUint8Arrays.ts +function combineUint8Arrays(a, b) { + const aLength = a.length; + const combinedBytes = new Uint8Array(aLength + b.length); + combinedBytes.set(a); + combinedBytes.set(b, aLength); + return combinedBytes; +} + +// src/vendor/lz4js/util.ts +function hashU32(a) { + a = a | 0; + a = a + 2127912214 + (a << 12) | 0; + a = a ^ -949894596 ^ a >>> 19; + a = a + 374761393 + (a << 5) | 0; + a = a + -744332180 ^ a << 9; + a = a + -42973499 + (a << 3) | 0; + return a ^ -1252372727 ^ a >>> 16 | 0; +} +function readU64(b, n) { + let x = 0; + x |= b[n++] << 0; + x |= b[n++] << 8; + x |= b[n++] << 16; + x |= b[n++] << 24; + x |= b[n++] << 32; + x |= b[n++] << 40; + x |= b[n++] << 48; + x |= b[n++] << 56; + return x; +} +function readU32(b, n) { + let x = 0; + x |= b[n++] << 0; + x |= b[n++] << 8; + x |= b[n++] << 16; + x |= b[n++] << 24; + return x; +} +function writeU32(b, n, x) { + b[n++] = x >> 0 & 255; + b[n++] = x >> 8 & 255; + b[n++] = x >> 16 & 255; + b[n++] = x >> 24 & 255; +} +function imul(a, b) { + const ah = a >>> 16; + const al = a & 65535; + const bh = b >>> 16; + const bl = b & 65535; + return al * bl + (ah * bl + al * bh << 16) | 0; +} + +// src/vendor/lz4js/xxh32.ts +var prime1 = 2654435761; +var prime2 = 2246822519; +var prime3 = 3266489917; +var prime4 = 668265263; +var prime5 = 374761393; +function rotl32(x, r) { + x = x | 0; + r = r | 0; + return x >>> (32 - r | 0) | x << r | 0; +} +function rotmul32(h, r, m) { + h = h | 0; + r = r | 0; + m = m | 0; + return imul(h >>> (32 - r | 0) | h << r, m) | 0; +} +function shiftxor32(h, s) { + h = h | 0; + s = s | 0; + return h >>> s ^ h | 0; +} +function xxhapply(h, src, m0, s, m1) { + return rotmul32(imul(src, m0) + h, s, m1); +} +function xxh1(h, src, index) { + return rotmul32(h + imul(src[index], prime5), 11, prime1); +} +function xxh4(h, src, index) { + return xxhapply(h, readU32(src, index), prime3, 17, prime4); +} +function xxh16(h, src, index) { + return [ + xxhapply(h[0], readU32(src, index + 0), prime2, 13, prime1), + xxhapply(h[1], readU32(src, index + 4), prime2, 13, prime1), + xxhapply(h[2], readU32(src, index + 8), prime2, 13, prime1), + xxhapply(h[3], readU32(src, index + 12), prime2, 13, prime1) + ]; +} +function xxh32(seed, src, index, len) { + let h; + const l = len; + if (len >= 16) { + h = [seed + prime1 + prime2, seed + prime2, seed, seed - prime1]; + while (len >= 16) { + h = xxh16(h, src, index); + index += 16; + len -= 16; + } + h = rotl32(h[0], 1) + rotl32(h[1], 7) + rotl32(h[2], 12) + rotl32(h[3], 18) + l; + } else { + h = seed + prime5 + len >>> 0; + } + while (len >= 4) { + h = xxh4(h, src, index); + index += 4; + len -= 4; + } + while (len > 0) { + h = xxh1(h, src, index); + index++; + len--; + } + h = shiftxor32(imul(shiftxor32(imul(shiftxor32(h, 15), prime2), 13), prime3), 16); + return h >>> 0; +} +var hash = xxh32; + +// src/vendor/lz4js/index.ts +var minMatch = 4; +var matchSearchLimit = 12; +var minTrailingLitterals = 5; +var skipTrigger = 6; +var hashSize = 1 << 16; +var mlBits = 4; +var mlMask = (1 << mlBits) - 1; +var runBits = 4; +var runMask = (1 << runBits) - 1; +var blockBuf = makeBuffer(5 << 20); +var hashTable = makeHashTable(); +var magicNum = 407708164; +var fdContentChksum = 4; +var fdContentSize = 8; +var fdBlockChksum = 16; +var fdVersion = 64; +var fdVersionMask = 192; +var bsUncompressed = 2147483648; +var bsDefault = 7; +var bsShift = 4; +var bsMask = 7; +var bsMap = { + 4: 65536, + 5: 262144, + 6: 1048576, + 7: 4194304 +}; +function makeHashTable() { + try { + return new Uint32Array(hashSize); + } catch (error) { + const hashTable2 = new Array(hashSize); + for (let i = 0; i < hashSize; i++) { + hashTable2[i] = 0; + } + return hashTable2; + } +} +function clearHashTable(table) { + for (let i = 0; i < hashSize; i++) { + table[i] = 0; + } +} +function makeBuffer(size) { + return new Uint8Array(size); +} +function sliceArray(array, start, end) { + return array.slice(start, end); +} +function compressBound(n) { + return n + n / 255 + 16 | 0; +} +function decompressBound(src) { + let sIndex = 0; + if (readU32(src, sIndex) !== magicNum) { + throw new Error("invalid magic number"); + } + sIndex += 4; + const descriptor = src[sIndex++]; + if ((descriptor & fdVersionMask) !== fdVersion) { + throw new Error("incompatible descriptor version " + (descriptor & fdVersionMask)); + } + const useBlockSum = (descriptor & fdBlockChksum) !== 0; + const useContentSize = (descriptor & fdContentSize) !== 0; + const bsIdx = src[sIndex++] >> bsShift & bsMask; + if (bsMap[bsIdx] === void 0) { + throw new Error("invalid block size " + bsIdx); + } + const maxBlockSize = bsMap[bsIdx]; + if (useContentSize) { + return readU64(src, sIndex); + } + sIndex++; + let maxSize = 0; + while (true) { + let blockSize = readU32(src, sIndex); + sIndex += 4; + if (blockSize & bsUncompressed) { + blockSize &= ~bsUncompressed; + maxSize += blockSize; + } else if (blockSize > 0) { + maxSize += maxBlockSize; + } + if (blockSize === 0) { + return maxSize; + } + if (useBlockSum) { + sIndex += 4; + } + sIndex += blockSize; + } +} +function decompressBlock(src, dst, sIndex, sLength, dIndex) { + let mLength, mOffset, sEnd, n, i; + const hasCopyWithin = dst.copyWithin !== void 0 && dst.fill !== void 0; + sEnd = sIndex + sLength; + while (sIndex < sEnd) { + const token = src[sIndex++]; + let literalCount = token >> 4; + if (literalCount > 0) { + if (literalCount === 15) { + while (true) { + literalCount += src[sIndex]; + if (src[sIndex++] !== 255) { + break; + } + } + } + for (n = sIndex + literalCount; sIndex < n; ) { + dst[dIndex++] = src[sIndex++]; + } + } + if (sIndex >= sEnd) { + break; + } + mLength = token & 15; + mOffset = src[sIndex++] | src[sIndex++] << 8; + if (mLength === 15) { + while (true) { + mLength += src[sIndex]; + if (src[sIndex++] !== 255) { + break; + } + } + } + mLength += minMatch; + if (hasCopyWithin && mOffset === 1) { + dst.fill(dst[dIndex - 1] | 0, dIndex, dIndex + mLength); + dIndex += mLength; + } else if (hasCopyWithin && mOffset > mLength && mLength > 31) { + dst.copyWithin(dIndex, dIndex - mOffset, dIndex - mOffset + mLength); + dIndex += mLength; + } else { + for (i = dIndex - mOffset, n = i + mLength; i < n; ) { + dst[dIndex++] = dst[i++] | 0; + } + } + } + return dIndex; +} +function compressBlock(src, dst, sIndex, sLength, hashTable2) { + let mIndex, mAnchor, mLength, mOffset, mStep; + let literalCount, dIndex, sEnd, n; + dIndex = 0; + sEnd = sLength + sIndex; + mAnchor = sIndex; + let searchMatchCount = (1 << skipTrigger) + 3; + while (sIndex <= sEnd - matchSearchLimit) { + const seq = readU32(src, sIndex); + let hash2 = hashU32(seq) >>> 0; + hash2 = (hash2 >> 16 ^ hash2) >>> 0 & 65535; + mIndex = hashTable2[hash2] - 1; + hashTable2[hash2] = sIndex + 1; + if (mIndex < 0 || sIndex - mIndex >>> 16 > 0 || readU32(src, mIndex) !== seq) { + mStep = searchMatchCount++ >> skipTrigger; + sIndex += mStep; + continue; + } + searchMatchCount = (1 << skipTrigger) + 3; + literalCount = sIndex - mAnchor; + mOffset = sIndex - mIndex; + sIndex += minMatch; + mIndex += minMatch; + mLength = sIndex; + while (sIndex < sEnd - minTrailingLitterals && src[sIndex] === src[mIndex]) { + sIndex++; + mIndex++; + } + mLength = sIndex - mLength; + const token = mLength < mlMask ? mLength : mlMask; + if (literalCount >= runMask) { + dst[dIndex++] = (runMask << mlBits) + token; + for (n = literalCount - runMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } else { + dst[dIndex++] = (literalCount << mlBits) + token; + } + for (let i = 0; i < literalCount; i++) { + dst[dIndex++] = src[mAnchor + i]; + } + dst[dIndex++] = mOffset; + dst[dIndex++] = mOffset >> 8; + if (mLength >= mlMask) { + for (n = mLength - mlMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } + mAnchor = sIndex; + } + if (mAnchor === 0) { + return 0; + } + literalCount = sEnd - mAnchor; + if (literalCount >= runMask) { + dst[dIndex++] = runMask << mlBits; + for (n = literalCount - runMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } else { + dst[dIndex++] = literalCount << mlBits; + } + sIndex = mAnchor; + while (sIndex < sEnd) { + dst[dIndex++] = src[sIndex++]; + } + return dIndex; +} +function decompressFrame(src, dst) { + let useBlockSum, useContentSum, useContentSize, descriptor; + let sIndex = 0; + let dIndex = 0; + if (readU32(src, sIndex) !== magicNum) { + throw new Error("invalid magic number"); + } + sIndex += 4; + descriptor = src[sIndex++]; + if ((descriptor & fdVersionMask) !== fdVersion) { + throw new Error("incompatible descriptor version"); + } + useBlockSum = (descriptor & fdBlockChksum) !== 0; + useContentSum = (descriptor & fdContentChksum) !== 0; + useContentSize = (descriptor & fdContentSize) !== 0; + const bsIdx = src[sIndex++] >> bsShift & bsMask; + if (bsMap[bsIdx] === void 0) { + throw new Error("invalid block size"); + } + if (useContentSize) { + sIndex += 8; + } + sIndex++; + while (true) { + var compSize; + compSize = readU32(src, sIndex); + sIndex += 4; + if (compSize === 0) { + break; + } + if (useBlockSum) { + sIndex += 4; + } + if ((compSize & bsUncompressed) !== 0) { + compSize &= ~bsUncompressed; + for (let j = 0; j < compSize; j++) { + dst[dIndex++] = src[sIndex++]; + } + } else { + dIndex = decompressBlock(src, dst, sIndex, compSize, dIndex); + sIndex += compSize; + } + } + if (useContentSum) { + sIndex += 4; + } + return dIndex; +} +function compressFrame(src, dst) { + let dIndex = 0; + writeU32(dst, dIndex, magicNum); + dIndex += 4; + dst[dIndex++] = fdVersion; + dst[dIndex++] = bsDefault << bsShift; + dst[dIndex] = hash(0, dst, 4, dIndex - 4) >> 8; + dIndex++; + const maxBlockSize = bsMap[bsDefault]; + let remaining = src.length; + let sIndex = 0; + clearHashTable(hashTable); + while (remaining > 0) { + let compSize = 0; + const blockSize = remaining > maxBlockSize ? maxBlockSize : remaining; + compSize = compressBlock(src, blockBuf, sIndex, blockSize, hashTable); + if (compSize > blockSize || compSize === 0) { + writeU32(dst, dIndex, 2147483648 | blockSize); + dIndex += 4; + for (let z = sIndex + blockSize; sIndex < z; ) { + dst[dIndex++] = src[sIndex++]; + } + remaining -= blockSize; + } else { + writeU32(dst, dIndex, compSize); + dIndex += 4; + for (let j = 0; j < compSize; ) { + dst[dIndex++] = blockBuf[j++]; + } + sIndex += blockSize; + remaining -= blockSize; + } + } + writeU32(dst, dIndex, 0); + dIndex += 4; + return dIndex; +} +function decompress(src, maxSize) { + let dst, size; + if (maxSize === void 0) { + maxSize = decompressBound(src); + } + dst = makeBuffer(maxSize); + size = decompressFrame(src, dst); + if (size !== maxSize) { + dst = sliceArray(dst, 0, size); + } + return dst; +} +function compress(src, maxSize) { + let dst, size; + if (maxSize === void 0) { + maxSize = compressBound(src.length); + } + dst = makeBuffer(maxSize); + size = compressFrame(src, dst); + if (size !== maxSize) { + dst = sliceArray(dst, 0, size); + } + return dst; +} + +// src/utils/RangeList.ts +var RangeList = class { + ranges = []; + /** + * Add a range to the list. If it overlaps with existing ranges, + * it will split them and increment reference counts accordingly. + */ + add(start, end) { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + const overlappingRanges = []; + for (let i = 0; i < this.ranges.length; i++) { + const range2 = this.ranges[i]; + if (start < range2.end && end > range2.start) { + overlappingRanges.push({ index: i, range: range2 }); + } + if (range2.data !== null) { + throw new Error("Overlapping range already has data"); + } + } + if (overlappingRanges.length === 0) { + this.ranges.push({ start, end, refCount: 1, data: null }); + this.ranges.sort((a, b) => a.start - b.start); + return; + } + const newRanges = []; + let currentPos = start; + for (let i = 0; i < overlappingRanges.length; i++) { + const { range: range2 } = overlappingRanges[i]; + if (currentPos < range2.start) { + newRanges.push({ + start: currentPos, + end: range2.start, + refCount: 1, + data: null + }); + } else if (range2.start < currentPos) { + newRanges.push({ + start: range2.start, + end: currentPos, + refCount: range2.refCount, + data: null + }); + } + newRanges.push({ + start: Math.max(currentPos, range2.start), + end: Math.min(end, range2.end), + refCount: range2.refCount + 1, + data: null + }); + if (range2.end > end) { + newRanges.push({ + start: end, + end: range2.end, + refCount: range2.refCount, + data: null + }); + } + currentPos = Math.max(currentPos, range2.end); + } + if (currentPos < end) { + newRanges.push({ + start: currentPos, + end, + refCount: 1, + data: null + }); + } + const firstIndex = overlappingRanges[0].index; + const lastIndex = overlappingRanges[overlappingRanges.length - 1].index; + this.ranges.splice(firstIndex, lastIndex - firstIndex + 1, ...newRanges); + this.ranges.sort((a, b) => a.start - b.start); + } + /** + * Remove a range from the list. The range must start and end at existing boundaries. + */ + remove(start, end) { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + const affectedRanges = []; + for (let i = 0; i < this.ranges.length; i++) { + const range2 = this.ranges[i]; + if (start < range2.end && end > range2.start) { + affectedRanges.push({ index: i, range: range2 }); + } + } + if (affectedRanges.length === 0) { + throw new Error("No ranges found to remove"); + } + if (start !== affectedRanges[0].range.start || end !== affectedRanges[affectedRanges.length - 1].range.end) { + throw new Error("Range boundaries must match existing boundaries"); + } + for (let i = 0; i < affectedRanges.length; i++) { + const { range: range2 } = affectedRanges[i]; + range2.refCount--; + } + this.ranges = this.ranges.filter((range2) => range2.refCount > 0); + } + /** + * Get all ranges within the specified boundaries. + */ + getRanges(start, end) { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + return this.ranges.filter((range2) => start < range2.end && end > range2.start); + } + /** + * Get all ranges in the list + */ + getAllRanges() { + return [...this.ranges]; + } +}; + +// src/utils/XetBlob.ts +var JWT_SAFETY_PERIOD = 6e4; +var JWT_CACHE_SIZE = 1e3; +var compressionSchemeLabels = { + [0 /* None */]: "None", + [1 /* LZ4 */]: "LZ4", + [2 /* ByteGroupingLZ4 */]: "ByteGroupingLZ4" +}; +var XET_CHUNK_HEADER_BYTES = 8; +var XetBlob = class extends Blob { + fetch; + accessToken; + refreshUrl; + reconstructionUrl; + hash; + start = 0; + end = 0; + internalLogging = false; + reconstructionInfo; + listener; + constructor(params) { + super([]); + this.fetch = params.fetch ?? fetch.bind(globalThis); + this.accessToken = checkCredentials(params); + this.refreshUrl = params.refreshUrl; + this.end = params.size; + this.reconstructionUrl = params.reconstructionUrl; + this.hash = params.hash; + this.listener = params.listener; + this.internalLogging = params.internalLogging ?? false; + if (params.readToken) { + const key = cacheKey({ refreshUrl: this.refreshUrl, initialAccessToken: this.accessToken }); + jwts.set(key, { + accessToken: params.readToken.accessToken, + expiresAt: new Date(params.readToken.exp * 1e3), + casUrl: params.readToken.casUrl + }); + } + } + get size() { + return this.end - this.start; + } + #clone() { + const blob = new XetBlob({ + fetch: this.fetch, + hash: this.hash, + refreshUrl: this.refreshUrl, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + reconstructionUrl: this.reconstructionUrl, + size: this.size + }); + blob.accessToken = this.accessToken; + blob.start = this.start; + blob.end = this.end; + blob.reconstructionInfo = this.reconstructionInfo; + blob.listener = this.listener; + blob.internalLogging = this.internalLogging; + return blob; + } + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on XetBlob.slice"); + } + const slice = this.#clone(); + slice.start = this.start + start; + slice.end = Math.min(this.start + end, this.end); + if (slice.start !== this.start || slice.end !== this.end) { + slice.reconstructionInfo = void 0; + } + return slice; + } + #reconstructionInfoPromise; + #loadReconstructionInfo() { + if (this.#reconstructionInfoPromise) { + return this.#reconstructionInfoPromise; + } + this.#reconstructionInfoPromise = (async () => { + const connParams = await getAccessToken(this.accessToken, this.fetch, this.refreshUrl); + const resp = await this.fetch(this.reconstructionUrl ?? `${connParams.casUrl}/v1/reconstructions/${this.hash}`, { + headers: { + Authorization: `Bearer ${connParams.accessToken}`, + Range: `bytes=${this.start}-${this.end - 1}` + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } + this.reconstructionInfo = await resp.json(); + return this.reconstructionInfo; + })().finally(() => this.#reconstructionInfoPromise = void 0); + return this.#reconstructionInfoPromise; + } + async #fetch() { + if (this.size === 0) { + return new ReadableStream({ + start(controller) { + controller.close(); + } + }); + } + if (!this.reconstructionInfo) { + await this.#loadReconstructionInfo(); + } + const rangeLists = /* @__PURE__ */ new Map(); + if (!this.reconstructionInfo) { + throw new Error("Failed to load reconstruction info"); + } + for (const term of this.reconstructionInfo.terms) { + let rangeList = rangeLists.get(term.hash); + if (!rangeList) { + rangeList = new RangeList(); + rangeLists.set(term.hash, rangeList); + } + rangeList.add(term.range.start, term.range.end); + } + const listener = this.listener; + const log = this.internalLogging ? (...args) => console.log(...args) : () => { + }; + async function* readData(reconstructionInfo, customFetch, maxBytes, reloadReconstructionInfo) { + let totalBytesRead = 0; + let readBytesToSkip = reconstructionInfo.offset_into_first_range; + for (const term of reconstructionInfo.terms) { + if (totalBytesRead >= maxBytes) { + break; + } + const rangeList = rangeLists.get(term.hash); + if (!rangeList) { + throw new Error(`Failed to find range list for term ${term.hash}`); + } + { + const termRanges = rangeList.getRanges(term.range.start, term.range.end); + if (termRanges.every((range2) => range2.data)) { + log("all data available for term", term.hash, readBytesToSkip); + rangeLoop: + for (const range2 of termRanges) { + for (let chunk2 of range2.data) { + if (readBytesToSkip) { + const skipped = Math.min(readBytesToSkip, chunk2.byteLength); + chunk2 = chunk2.slice(skipped); + readBytesToSkip -= skipped; + if (!chunk2.byteLength) { + continue; + } + } + if (chunk2.byteLength > maxBytes - totalBytesRead) { + chunk2 = chunk2.slice(0, maxBytes - totalBytesRead); + } + totalBytesRead += chunk2.byteLength; + yield range2.refCount > 1 ? chunk2.slice() : chunk2; + listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } }); + if (totalBytesRead >= maxBytes) { + break rangeLoop; + } + } + } + rangeList.remove(term.range.start, term.range.end); + continue; + } + } + let fetchInfo = reconstructionInfo.fetch_info[term.hash].find( + (info) => info.range.start <= term.range.start && info.range.end >= term.range.end + ); + if (!fetchInfo) { + throw new Error( + `Failed to find fetch info for term ${term.hash} and range ${term.range.start}-${term.range.end}` + ); + } + log("term", term); + log("fetchinfo", fetchInfo); + log("readBytesToSkip", readBytesToSkip); + let resp = await customFetch(fetchInfo.url, { + headers: { + Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}` + } + }); + if (resp.status === 403) { + reconstructionInfo = await reloadReconstructionInfo(); + fetchInfo = reconstructionInfo.fetch_info[term.hash]?.find( + (info) => info.range.start <= term.range.start && info.range.end >= term.range.end + ); + if (!fetchInfo) { + throw new Error( + `Failed to find fetch info for term ${term.hash} and range ${term.range.start}-${term.range.end} after refresh` + ); + } + resp = await customFetch(fetchInfo.url, { + headers: { + Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}` + } + }); + } + if (!resp.ok) { + throw await createApiError(resp); + } + log( + "expected content length", + resp.headers.get("content-length"), + "range", + fetchInfo.url_range, + resp.headers.get("content-range") + ); + const reader = resp.body?.getReader(); + if (!reader) { + throw new Error("Failed to get reader from response body"); + } + let done = false; + let chunkIndex = fetchInfo.range.start; + const ranges = rangeList.getRanges(fetchInfo.range.start, fetchInfo.range.end); + let leftoverBytes = void 0; + let totalFetchBytes = 0; + fetchData: + while (!done && totalBytesRead < maxBytes) { + const result = await reader.read(); + listener?.({ event: "read" }); + done = result.done; + log("read", result.value?.byteLength, "bytes", "total read", totalBytesRead, "toSkip", readBytesToSkip); + if (!result.value) { + log("no data in result, cancelled", result); + continue; + } + totalFetchBytes += result.value.byteLength; + if (leftoverBytes) { + result.value = combineUint8Arrays(leftoverBytes, result.value); + leftoverBytes = void 0; + } + while (totalBytesRead < maxBytes && result.value?.byteLength) { + if (result.value.byteLength < 8) { + leftoverBytes = result.value; + continue fetchData; + } + const header = new DataView(result.value.buffer, result.value.byteOffset, XET_CHUNK_HEADER_BYTES); + const chunkHeader = { + version: header.getUint8(0), + compressed_length: header.getUint8(1) | header.getUint8(2) << 8 | header.getUint8(3) << 16, + compression_scheme: header.getUint8(4), + uncompressed_length: header.getUint8(5) | header.getUint8(6) << 8 | header.getUint8(7) << 16 + }; + log("chunk header", chunkHeader, "to skip", readBytesToSkip); + if (chunkHeader.version !== 0) { + throw new Error(`Unsupported chunk version ${chunkHeader.version}`); + } + if (chunkHeader.compression_scheme !== 0 /* None */ && chunkHeader.compression_scheme !== 1 /* LZ4 */ && chunkHeader.compression_scheme !== 2 /* ByteGroupingLZ4 */) { + throw new Error( + `Unsupported compression scheme ${compressionSchemeLabels[chunkHeader.compression_scheme] ?? chunkHeader.compression_scheme}` + ); + } + if (result.value.byteLength < chunkHeader.compressed_length + XET_CHUNK_HEADER_BYTES) { + leftoverBytes = result.value; + continue fetchData; + } + result.value = result.value.slice(XET_CHUNK_HEADER_BYTES); + let uncompressed = chunkHeader.compression_scheme === 1 /* LZ4 */ ? decompress(result.value.slice(0, chunkHeader.compressed_length), chunkHeader.uncompressed_length) : chunkHeader.compression_scheme === 2 /* ByteGroupingLZ4 */ ? bg4_regroup_bytes( + decompress( + result.value.slice(0, chunkHeader.compressed_length), + chunkHeader.uncompressed_length + ) + ) : result.value.slice(0, chunkHeader.compressed_length); + const range2 = ranges.find((range3) => chunkIndex >= range3.start && chunkIndex < range3.end); + const shouldYield = chunkIndex >= term.range.start && chunkIndex < term.range.end; + const minRefCountToStore = shouldYield ? 2 : 1; + let stored = false; + if (range2 && range2.refCount >= minRefCountToStore) { + range2.data ??= []; + range2.data.push(uncompressed); + stored = true; + } + if (shouldYield) { + if (readBytesToSkip) { + const skipped = Math.min(readBytesToSkip, uncompressed.byteLength); + uncompressed = uncompressed.slice(readBytesToSkip); + readBytesToSkip -= skipped; + } + if (uncompressed.byteLength > maxBytes - totalBytesRead) { + uncompressed = uncompressed.slice(0, maxBytes - totalBytesRead); + } + if (uncompressed.byteLength) { + log( + "yield", + uncompressed.byteLength, + "bytes", + result.value.byteLength, + "total read", + totalBytesRead, + stored + ); + totalBytesRead += uncompressed.byteLength; + yield stored ? uncompressed.slice() : uncompressed; + listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } }); + } + } + chunkIndex++; + result.value = result.value.slice(chunkHeader.compressed_length); + } + } + if (done && totalBytesRead < maxBytes && totalFetchBytes < fetchInfo.url_range.end - fetchInfo.url_range.start + 1) { + log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes); + log("failed to fetch all data for term", term.hash); + throw new Error( + `Failed to fetch all data for term ${term.hash}, fetched ${totalFetchBytes} bytes out of ${fetchInfo.url_range.end - fetchInfo.url_range.start + 1}` + ); + } + log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes); + log("cancel reader"); + await reader.cancel(); + } + } + const iterator = readData( + this.reconstructionInfo, + this.fetch, + this.end - this.start, + this.#loadReconstructionInfo.bind(this) + ); + return new ReadableStream( + { + // todo: when Safari supports it, type controller as ReadableByteStreamController + async pull(controller) { + const result = await iterator.next(); + if (result.value) { + controller.enqueue(result.value); + } + if (result.done) { + controller.close(); + } + }, + type: "bytes" + // todo: when Safari supports it, add autoAllocateChunkSize param + }, + // todo : use ByteLengthQueuingStrategy when there's good support for it, currently in Node.js it fails due to size being a function + { + highWaterMark: 1e3 + // 1_000 chunks for ~1MB of RAM + } + ); + } + async arrayBuffer() { + const result = await this.#fetch(); + return new Response(result).arrayBuffer(); + } + async text() { + const result = await this.#fetch(); + return new Response(result).text(); + } + async response() { + const result = await this.#fetch(); + return new Response(result); + } + stream() { + const stream = new TransformStream(); + this.#fetch().then((response) => response.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message)); + return stream.readable; + } +}; +var jwtPromises = /* @__PURE__ */ new Map(); +var jwts = /* @__PURE__ */ new Map(); +function cacheKey(params) { + return JSON.stringify([params.refreshUrl, params.initialAccessToken]); +} +function bg4_regroup_bytes(bytes) { + const split = Math.floor(bytes.byteLength / 4); + const rem = bytes.byteLength % 4; + const g1_pos = split + (rem >= 1 ? 1 : 0); + const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); + const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); + const ret = new Uint8Array(bytes.byteLength); + for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + return ret; +} +function bg4_split_bytes(bytes) { + const ret = new Uint8Array(bytes.byteLength); + const split = Math.floor(bytes.byteLength / 4); + const rem = bytes.byteLength % 4; + const g1_pos = split + (rem >= 1 ? 1 : 0); + const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); + const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); + for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + return ret; +} +async function getAccessToken(initialAccessToken, customFetch, refreshUrl) { + const key = cacheKey({ refreshUrl, initialAccessToken }); + const jwt = jwts.get(key); + if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD)) { + return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; + } + const existingPromise = jwtPromises.get(key); + if (existingPromise) { + return existingPromise; + } + const promise = (async () => { + const resp = await customFetch(refreshUrl, { + headers: { + ...initialAccessToken ? { + Authorization: `Bearer ${initialAccessToken}` + } : {} + } + }); + if (!resp.ok) { + throw new Error(`Failed to get JWT token: ${resp.status} ${await resp.text()}`); + } + const json = await resp.json(); + const jwt2 = { + accessToken: json.accessToken, + expiresAt: new Date(json.exp * 1e3), + casUrl: json.casUrl + }; + jwtPromises.delete(key); + for (const [key2, value] of jwts.entries()) { + if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD)) { + jwts.delete(key2); + } else { + break; + } + } + if (jwts.size >= JWT_CACHE_SIZE) { + const keyToDelete = jwts.keys().next().value; + if (keyToDelete) { + jwts.delete(keyToDelete); + } + } + jwts.set(key, jwt2); + return { + accessToken: json.accessToken, + casUrl: json.casUrl + }; + })(); + jwtPromises.set(key, promise); + return promise; +} + +// src/utils/range.ts +function range(n, b) { + return b ? Array(b - n).fill(0).map((_, i) => n + i) : Array(n).fill(0).map((_, i) => i); +} + +// src/utils/chunk.ts +function chunk(arr, chunkSize) { + if (isNaN(chunkSize) || chunkSize < 1) { + throw new RangeError("Invalid chunk size: " + chunkSize); + } + if (!arr.length) { + return []; + } + if (arr.length <= chunkSize) { + return [arr]; + } + return range(Math.ceil(arr.length / chunkSize)).map((i) => { + return arr.slice(i * chunkSize, (i + 1) * chunkSize); + }); +} + +// src/utils/promisesQueue.ts +async function promisesQueue(factories, concurrency) { + const results = []; + const executing = /* @__PURE__ */ new Set(); + let index = 0; + for (const factory of factories) { + const closureIndex = index++; + const e = factory().then((r) => { + results[closureIndex] = r; + executing.delete(e); + }); + executing.add(e); + if (executing.size >= concurrency) { + await Promise.race(executing); + } + } + await Promise.all(executing); + return results; +} + +// src/utils/promisesQueueStreaming.ts +async function promisesQueueStreaming(factories, concurrency) { + const executing = []; + for await (const factory of factories) { + const e = factory().then(() => { + executing.splice(executing.indexOf(e), 1); + }); + executing.push(e); + if (executing.length >= concurrency) { + await Promise.race(executing); + } + } + await Promise.all(executing); +} + +// src/utils/WebBlob.ts +var WebBlob = class extends Blob { + static async create(url, opts) { + const customFetch = opts?.fetch ?? fetch; + const probe = await customFetch(url, { + headers: { + Range: "bytes=0-0", + ...opts?.accessToken && { Authorization: `Bearer ${opts.accessToken}` } + } + }); + if (!probe.ok) { + throw await createApiError(probe); + } + const contentType = probe.headers.get("content-type") || ""; + if (probe.status === 206) { + const totalSize = Number(probe.headers.get("content-range")?.split("/").pop()); + await probe.body?.cancel(); + if (Number.isFinite(totalSize) && totalSize >= (opts?.cacheBelow ?? 1e6)) { + return new WebBlob(url, 0, totalSize, contentType, true, customFetch, opts?.accessToken); + } + const full = await customFetch(url, { + ...opts?.accessToken && { headers: { Authorization: `Bearer ${opts.accessToken}` } } + }); + if (!full.ok) { + throw await createApiError(full); + } + return full.blob(); + } + return probe.blob(); + } + url; + start; + end; + contentType; + full; + fetch; + accessToken; + constructor(url, start, end, contentType, full, customFetch, accessToken) { + super([]); + this.url = url; + this.start = start; + this.end = end; + this.contentType = contentType; + this.full = full; + this.fetch = customFetch; + this.accessToken = accessToken; + } + get size() { + return this.end - this.start; + } + get type() { + return this.contentType; + } + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on WebBlob.slice"); + } + const slice = new WebBlob( + this.url, + this.start + start, + Math.min(this.start + end, this.end), + this.contentType, + start === 0 && end === this.size ? this.full : false, + this.fetch, + this.accessToken + ); + return slice; + } + async arrayBuffer() { + const result = await this.fetchRange(); + return result.arrayBuffer(); + } + async text() { + const result = await this.fetchRange(); + return result.text(); + } + stream() { + const stream = new TransformStream(); + this.fetchRange().then((response) => response.body?.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message)); + return stream.readable; + } + fetchRange() { + const fetch2 = this.fetch; + if (this.full) { + return fetch2(this.url, { + ...this.accessToken && { + headers: { + Authorization: `Bearer ${this.accessToken}` + } + } + }).then((resp) => resp.ok ? resp : createApiError(resp)); + } + return fetch2(this.url, { + headers: { + Range: `bytes=${this.start}-${this.end - 1}`, + ...this.accessToken && { Authorization: `Bearer ${this.accessToken}` } + } + }).then((resp) => resp.ok ? resp : createApiError(resp)); + } +}; + +// src/utils/base64FromBytes.ts +function base64FromBytes(arr) { + if (globalThis.Buffer) { + return globalThis.Buffer.from(arr).toString("base64"); + } else { + const bin = []; + arr.forEach((byte) => { + bin.push(String.fromCharCode(byte)); + }); + return globalThis.btoa(bin.join("")); + } +} + +// src/utils/createBlobs.ts +async function createBlobs(url, destPath, opts) { + if (url.protocol === "http:" || url.protocol === "https:") { + const blob = await WebBlob.create(url, { fetch: opts?.fetch, accessToken: opts?.accessToken }); + return [{ path: destPath, blob }]; + } + if (isFrontend) { + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); + } + if (url.protocol === "file:") { + const { FileBlob } = await import("./FileBlob-RUOT7DBI.mjs"); + const { subPaths } = await import("./sub-paths-HFKHI55E.mjs"); + const paths = await subPaths(url, opts?.maxFolderDepth); + if (paths.length === 1 && paths[0].relativePath === ".") { + const blob = await FileBlob.create(url); + return [{ path: destPath, blob }]; + } + return Promise.all( + paths.map(async (path2) => ({ + path: `${destPath}/${path2.relativePath}`.replace(/\/[.]$/, "").replaceAll("//", "/").replace(/^[.]?\//, ""), + blob: await FileBlob.create(new URL(path2.path)) + })) + ); + } + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); +} + +// src/utils/ChunkCache.ts +var CHUNK_CACHE_INITIAL_SIZE = 1e4; +var CHUNK_CACHE_GROW_FACTOR = 1.5; +var CHUNK_CACHE_MAX_SIZE = 1e6; +var ChunkCache = class { + index = 0; + // Index >= 0 means local xorb, < 0 means remote xorb + xorbIndices; + // Max 8K chunks per xorb, less than 64K uint16_t + chunkIndices; + map = /* @__PURE__ */ new Map(); + // hash -> chunkCacheIndex. Less overhead that way, empty object is 60+B and empty array is 40+B + hmacs = /* @__PURE__ */ new Set(); + // todo : remove old hmacs + maxSize; + constructor(maxSize = CHUNK_CACHE_MAX_SIZE) { + if (maxSize < 1) { + throw new Error("maxSize must be at least 1"); + } + this.maxSize = maxSize; + this.xorbIndices = new Int32Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); + this.chunkIndices = new Uint16Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); + } + addChunkToCache(hash2, xorbIndex, chunkIndex, hmac2) { + if (this.map.has(hash2)) { + return; + } + if (this.map.values().next().value === this.index) { + this.map.delete(this.map.keys().next().value); + } + this.map.set(hash2, this.index); + if (hmac2 !== null) { + this.hmacs.add(hmac2); + } + if (this.index >= this.xorbIndices.length) { + const oldXorbIndices = this.xorbIndices; + const oldChunkIndices = this.chunkIndices; + this.xorbIndices = new Int32Array(Math.min(this.xorbIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); + this.chunkIndices = new Uint16Array(Math.min(this.chunkIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); + this.xorbIndices.set(oldXorbIndices); + this.chunkIndices.set(oldChunkIndices); + } + this.xorbIndices[this.index] = xorbIndex; + this.chunkIndices[this.index] = chunkIndex; + this.index = (this.index + 1) % this.maxSize; + } + getChunk(hash2, hmacFunction) { + let index = this.map.get(hash2); + if (index === void 0 && hmacFunction !== null) { + for (const hmac2 of this.hmacs) { + index = this.map.get(hmacFunction(hash2, hmac2)); + if (index !== void 0) { + break; + } + } + } + if (index === void 0) { + return void 0; + } + return { + xorbIndex: this.xorbIndices[index], + chunkIndex: this.chunkIndices[index] + }; + } + updateChunkIndex(hash2, chunkIndex) { + const index = this.map.get(hash2); + if (index === void 0) { + throw new Error(`Chunk not found in cache: ${hash2}`); + } + this.chunkIndices[index] = chunkIndex; + } + removeChunkFromCache(hash2) { + this.map.delete(hash2); + } +}; + +// src/utils/xetWriteToken.ts +var JWT_SAFETY_PERIOD2 = 6e4; +var JWT_CACHE_SIZE2 = 1e3; +var jwtPromises2 = /* @__PURE__ */ new Map(); +var jwts2 = /* @__PURE__ */ new Map(); +async function xetWriteToken(params) { + if (params.xetParams.expiresAt && params.xetParams.casUrl && params.xetParams.accessToken && params.xetParams.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD2)) { + return { accessToken: params.xetParams.accessToken, casUrl: params.xetParams.casUrl }; + } + const key = params.xetParams.refreshWriteTokenUrl; + const jwt = jwts2.get(key); + if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD2)) { + return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; + } + const existingPromise = jwtPromises2.get(key); + if (existingPromise) { + return existingPromise; + } + const promise = (async () => { + const resp = await (params.fetch ?? fetch)(params.xetParams.refreshWriteTokenUrl, { + headers: { + ...params.accessToken ? { + Authorization: `Bearer ${params.accessToken}` + } : {}, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } + const json = await resp.json(); + const jwt2 = { + accessToken: json.accessToken, + expiresAt: new Date(json.exp * 1e3), + casUrl: json.casUrl + }; + jwtPromises2.delete(key); + for (const [key2, value] of jwts2.entries()) { + if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD2)) { + jwts2.delete(key2); + } else { + break; + } + } + if (jwts2.size >= JWT_CACHE_SIZE2) { + const keyToDelete = jwts2.keys().next().value; + if (keyToDelete) { + jwts2.delete(keyToDelete); + } + } + jwts2.set(key, jwt2); + return { + accessToken: json.accessToken, + casUrl: json.casUrl + }; + })(); + jwtPromises2.set(key, promise); + return promise; +} + +// src/utils/shardParser.ts +var HASH_LENGTH = 32; +var XORB_HASH_BOOKEND = "ff".repeat(HASH_LENGTH); +function readHashFromArray(array, offset) { + let hash2 = ""; + for (let i = 0; i < HASH_LENGTH; i += 8) { + hash2 += `${array[offset + i + 7].toString(16).padStart(2, "0")}${array[offset + i + 6].toString(16).padStart(2, "0")}${array[offset + i + 5].toString(16).padStart(2, "0")}${array[offset + i + 4].toString(16).padStart(2, "0")}${array[offset + i + 3].toString(16).padStart(2, "0")}${array[offset + i + 2].toString(16).padStart(2, "0")}${array[offset + i + 1].toString(16).padStart(2, "0")}${array[offset + i].toString(16).padStart(2, "0")}`; + } + return hash2; +} +async function parseShardData(shardBlob) { + const shard = new Uint8Array(await shardBlob.arrayBuffer()); + const shardView = new DataView(shard.buffer); + const magicTag = shard.slice(0, SHARD_MAGIC_TAG.length); + if (!magicTag.every((byte, i) => byte === SHARD_MAGIC_TAG[i])) { + throw new Error("Invalid shard magic tag"); + } + const version = shardView.getBigUint64(SHARD_MAGIC_TAG.length, true); + if (version !== SHARD_HEADER_VERSION) { + throw new Error(`Invalid shard version: ${version}`); + } + const footerSize = Number(shardView.getBigUint64(SHARD_MAGIC_TAG.length + 8, true)); + const footerStart = shard.length - footerSize; + const footerVersion = shardView.getBigUint64(footerStart, true); + if (footerVersion !== SHARD_FOOTER_VERSION) { + throw new Error(`Invalid shard footer version: ${footerVersion}`); + } + const xorbInfoStart = Number(shardView.getBigUint64(footerStart + 16, true)); + const fileLookupStart = Number(shardView.getBigUint64(footerStart + 24, true)); + const hmacKey = readHashFromArray(shard, footerStart + 72); + const xorbs = []; + let offset = xorbInfoStart; + while (offset < fileLookupStart) { + const xorbHash2 = readHashFromArray(shard, offset); + offset += HASH_LENGTH; + if (xorbHash2 === XORB_HASH_BOOKEND) { + break; + } + offset += 4; + const chunkCount = shardView.getUint32(offset, true); + offset += 4; + offset += 4; + offset += 4; + const chunks = []; + for (let i = 0; i < chunkCount; i++) { + const chunkHash = readHashFromArray(shard, offset); + offset += HASH_LENGTH; + const startOffset = shardView.getUint32(offset, true); + offset += 4; + const length = shardView.getUint32(offset, true); + offset += 4; + offset += 8; + chunks.push({ + hash: chunkHash, + startOffset, + unpackedLength: length + }); + } + xorbs.push({ + hash: xorbHash2, + chunks + }); + } + return { + hmacKey, + xorbs + }; +} + +// src/utils/sum.ts +function sum(arr) { + return arr.reduce((a, b) => a + b, 0); +} + +// src/utils/SplicedBlob.ts +var SplicedBlob = class extends Blob { + originalBlob; + spliceOperations; + constructor(originalBlob, spliceOperations) { + super(); + this.originalBlob = originalBlob; + this.spliceOperations = spliceOperations; + } + static create(originalBlob, operations) { + for (const op of operations) { + if (op.start < 0 || op.end < 0) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + if (op.start > originalBlob.size || op.end > originalBlob.size) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + if (op.start > op.end) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + } + const sortedOps = [...operations].sort((a, b) => a.start - b.start); + for (let i = 0; i < sortedOps.length - 1; i++) { + if (sortedOps[i].end > sortedOps[i + 1].start) { + throw new Error("Overlapping splice operations are not supported"); + } + } + return new SplicedBlob(originalBlob, sortedOps); + } + /** + * Returns the size of the spliced blob. + * Size = original size - total replaced size + total insert size + */ + get size() { + let totalReplacedSize = 0; + let totalInsertSize = 0; + for (const op of this.spliceOperations) { + totalReplacedSize += op.end - op.start; + totalInsertSize += op.insert.size; + } + return this.originalBlob.size - totalReplacedSize + totalInsertSize; + } + /** + * Returns the MIME type of the original blob. + */ + get type() { + return this.originalBlob.type; + } + /** + * Returns a new instance of SplicedBlob that is a slice of the current one. + * + * The slice is inclusive of the start and exclusive of the end. + * The slice method does not support negative start/end. + * + * @param start beginning of the slice + * @param end end of the slice + */ + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + throw new TypeError("Unsupported negative start/end on SplicedBlob.slice"); + } + start = Math.min(start, this.size); + end = Math.min(end, this.size); + if (start >= end) { + return new Blob([]); + } + const segments = this.segments; + const segmentBoundaries = [0]; + let cumulativeSize = 0; + for (const segment of segments) { + cumulativeSize += segment.size; + segmentBoundaries.push(cumulativeSize); + } + const resultSegments = []; + for (let i = 0; i < segments.length; i++) { + const segmentStart = segmentBoundaries[i]; + const segmentEnd = segmentBoundaries[i + 1]; + if (segmentEnd <= start) { + continue; + } + if (segmentStart >= end) { + break; + } + const sliceStart = Math.max(0, start - segmentStart); + const sliceEnd = Math.min(segments[i].size, end - segmentStart); + if (sliceStart < sliceEnd) { + resultSegments.push(segments[i].slice(sliceStart, sliceEnd)); + } + } + return new Blob(resultSegments); + } + get firstSpliceIndex() { + return this.spliceOperations[0]?.start ?? Infinity; + } + /** + * Read the spliced blob content and returns it as an ArrayBuffer. + */ + async arrayBuffer() { + const segments = this.segments; + const buffers = await Promise.all(segments.map((segment) => segment.arrayBuffer())); + const totalSize = sum(buffers.map((buffer) => buffer.byteLength)); + const result = new Uint8Array(totalSize); + let offset = 0; + for (const buffer of buffers) { + result.set(new Uint8Array(buffer), offset); + offset += buffer.byteLength; + } + return result.buffer; + } + /** + * Read the spliced blob content and returns it as a string. + */ + async text() { + const buffer = await this.arrayBuffer(); + return new TextDecoder().decode(buffer); + } + /** + * Returns a stream around the spliced blob content. + */ + stream() { + const readable = new ReadableStream({ + start: async (controller) => { + try { + const segments = this.segments; + for (const segment of segments) { + const reader = segment.stream().getReader(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + controller.enqueue(value); + } + } finally { + reader.releaseLock(); + } + } + controller.close(); + } catch (error) { + controller.error(error); + } + } + }); + return readable; + } + /** + * Get all segments that make up the spliced blob. + * This includes original blob segments between splice operations and insert blobs. + */ + get segments() { + const segments = []; + let currentPosition = 0; + const sortedOps = [...this.spliceOperations].sort((a, b) => a.start - b.start); + for (const op of sortedOps) { + if (currentPosition < op.start) { + segments.push(this.originalBlob.slice(currentPosition, op.start)); + } + if (op.insert.size > 0) { + segments.push(op.insert); + } + currentPosition = op.end; + } + if (currentPosition < this.originalBlob.size) { + segments.push(this.originalBlob.slice(currentPosition)); + } + return segments; + } +}; + +// src/utils/createXorbs.ts +import { + createChunker, + nextBlock, + finalize, + hashToHex, + hexToBytes, + xorbHash, + fileHash, + hmac, + verificationHash +} from "@huggingface/xetchunk-wasm"; +var TARGET_CHUNK_SIZE = 64 * 1024; +var MAX_CHUNK_SIZE = 2 * TARGET_CHUNK_SIZE; +var XORB_SIZE = 64 * 1024 * 1024; +var MAX_XORB_CHUNKS = 8 * 1024; +var INTERVAL_BETWEEN_REMOTE_DEDUP = 4e6; +var PROCESSING_PROGRESS_RATIO = 0.1; +var UPLOADING_PROGRESS_RATIO = 1 - PROCESSING_PROGRESS_RATIO; +function computeXorbHashHex(chunks) { + const chunkObjs = chunks.map((c) => ({ hash: hexToBytes(c.hash), length: c.length })); + return hashToHex(xorbHash(chunkObjs)); +} +function computeHmacHex(hash2, key) { + return hashToHex(hmac(hexToBytes(hash2), hexToBytes(key))); +} +function computeVerificationHashHex(hashes) { + return hashToHex(verificationHash(hashes.map(hexToBytes))); +} +function computeFileHashHex(chunks) { + const chunkObjs = chunks.map((c) => ({ hash: hexToBytes(c.hash), length: c.length })); + return hashToHex(fileHash(chunkObjs)); +} +function addDataToChunker(data, chunker) { + return nextBlock(chunker, data).map((c) => ({ hash: hashToHex(c.hash), length: c.length, dedup: false })); +} +function finalizeChunker(chunker) { + const last = finalize(chunker); + if (!last) { + return []; + } + return [{ hash: hashToHex(last.hash), length: last.length, dedup: false }]; +} +var CurrentXorbInfo = class { + id; + offset; + chunks; + fileProcessedBytes; + fileUploadedBytes; + fileSize; + data; + immutableData; + constructor() { + this.id = 0; + this.offset = 0; + this.chunks = []; + this.fileProcessedBytes = {}; + this.fileUploadedBytes = {}; + this.fileSize = {}; + this.data = new Uint8Array(XORB_SIZE); + this.immutableData = null; + } + event(computeXorbHash) { + const xorbChunksCleaned = this.chunks.map((chunk2) => ({ + hash: chunk2.hash, + length: chunk2.length + })); + return { + event: "xorb", + xorb: this.data.subarray(0, this.offset), + hash: computeXorbHash(xorbChunksCleaned), + chunks: xorbChunksCleaned, + id: this.id, + files: Object.entries(this.fileProcessedBytes).map(([path2, processedBytes]) => ({ + path: path2, + progress: processedBytes / this.fileSize[path2], + lastSentProgress: ((this.fileUploadedBytes[path2] ?? 0) + (processedBytes - (this.fileUploadedBytes[path2] ?? 0)) * PROCESSING_PROGRESS_RATIO) / this.fileSize[path2] + })) + }; + } +}; +async function* createXorbs(fileSources, params) { + const alreadyDoneFileSha256s = /* @__PURE__ */ new Set(); + let xorbId = 0; + const chunkCache = new ChunkCache(); + let xorb = new CurrentXorbInfo(); + const nextXorb = (currentFile) => { + const event = xorb.event(computeXorbHashHex); + xorbId++; + xorb = new CurrentXorbInfo(); + xorb.id = xorbId; + xorb.fileUploadedBytes = { + [currentFile.path]: currentFile.uploadedBytes + }; + xorb.fileSize[currentFile.path] = currentFile.size; + return event; + }; + const pendingFileEvents = []; + const remoteXorbHashes = [""]; + for await (const fileSource of fileSources) { + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: 0 + }); + if (fileSource.sha256 && alreadyDoneFileSha256s.has(fileSource.sha256)) { + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: 1 + }); + continue; + } + if (fileSource.sha256) { + alreadyDoneFileSha256s.add(fileSource.sha256); + } + const chunker = createChunker(TARGET_CHUNK_SIZE); + { + xorb.fileSize[fileSource.path] = fileSource.content.size; + if (fileSource.content instanceof SplicedBlob && fileSource.content.firstSpliceIndex < MAX_CHUNK_SIZE) { + await loadDedupInfoToCache( + fileSource.content.originalBlob.slice(0, MAX_CHUNK_SIZE), + remoteXorbHashes, + params, + chunkCache, + computeHmacHex, + { + maxChunks: 1, + isAtBeginning: true + } + ); + } + let bytesSinceRemoteDedup = Infinity; + let bytesSinceLastProgressEvent = 0; + let isFirstFileChunk = true; + const sourceChunks = []; + const reader = fileSource.content.stream().getReader(); + let processedBytes = 0; + let dedupedBytes = 0; + const fileChunks = []; + const chunkMetadata = []; + const addChunks = async function* (chunks) { + for (const chunk2 of chunks) { + if (isFirstFileChunk) { + chunk2.dedup = true; + isFirstFileChunk = false; + } + let chunkIndex = xorb.chunks.length; + let chunkXorbId = xorbId; + const chunkToCopy = removeChunkFromSourceData(sourceChunks, chunk2.length); + let cacheData = chunkCache.getChunk(chunk2.hash, computeHmacHex); + if (cacheData === void 0 && chunk2.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { + const token = await xetWriteToken(params); + bytesSinceRemoteDedup = 0; + const shardResp = await (params.fetch ?? fetch)(token.casUrl + "/v1/chunks/default/" + chunk2.hash, { + headers: { + Authorization: `Bearer ${token.accessToken}` + } + }); + if (shardResp.ok) { + const shard = await shardResp.blob(); + const shardData = await parseShardData(shard); + for (const xorb2 of shardData.xorbs) { + const remoteXorbId = -remoteXorbHashes.length; + remoteXorbHashes.push(xorb2.hash); + let i = 0; + for (const chunk3 of xorb2.chunks) { + chunkCache.addChunkToCache(chunk3.hash, remoteXorbId, i++, shardData.hmacKey); + } + } + cacheData = chunkCache.getChunk(chunk2.hash, computeHmacHex); + const oldDedupedBytes = dedupedBytes; + dedupedBytes = backtrackDedup(xorb, computeHmacHex, shardData, chunkCache, chunkMetadata, dedupedBytes); + if (dedupedBytes > oldDedupedBytes) { + xorb.fileUploadedBytes[fileSource.path] ??= 0; + xorb.fileUploadedBytes[fileSource.path] += dedupedBytes - oldDedupedBytes; + } + } + } + if (cacheData === void 0) { + if (!writeChunk(xorb, chunkToCopy, chunk2.hash)) { + yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); + chunkIndex = 0; + chunkXorbId = xorbId; + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } + pendingFileEvents.length = 0; + if (!writeChunk(xorb, chunkToCopy, chunk2.hash)) { + throw new Error("Failed to write chunk into xorb"); + } + } + chunkCache.addChunkToCache(chunk2.hash, xorbId, chunkIndex, null); + } else { + chunkXorbId = cacheData.xorbIndex; + chunkIndex = cacheData.chunkIndex; + dedupedBytes += chunk2.length; + xorb.fileUploadedBytes[fileSource.path] ??= 0; + xorb.fileUploadedBytes[fileSource.path] += chunk2.length; + } + bytesSinceRemoteDedup += chunk2.length; + bytesSinceLastProgressEvent += chunk2.length; + fileChunks.push({ hash: chunk2.hash, length: chunk2.length }); + chunkMetadata.push({ + xorbId: chunkXorbId, + chunkIndex, + length: chunk2.length + }); + xorb.fileProcessedBytes[fileSource.path] = processedBytes; + if (bytesSinceLastProgressEvent >= 1e6) { + bytesSinceLastProgressEvent = 0; + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: ((xorb.fileUploadedBytes[fileSource.path] ?? 0) + (xorb.fileProcessedBytes[fileSource.path] - (xorb.fileUploadedBytes[fileSource.path] ?? 0)) * PROCESSING_PROGRESS_RATIO) / fileSource.content.size + }); + } + if (xorb.chunks.length >= MAX_XORB_CHUNKS) { + yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } + pendingFileEvents.length = 0; + } + } + }; + while (true) { + const { done, value } = await reader.read(); + if (done) { + yield* addChunks(finalizeChunker(chunker)); + break; + } + processedBytes += value.length; + sourceChunks.push(value); + yield* addChunks(addDataToChunker(value, chunker)); + } + const fileRepresentation = buildFileRepresentation(chunkMetadata, fileChunks, computeVerificationHashHex); + xorb.immutableData = { + chunkIndex: xorb.chunks.length, + offset: xorb.offset + }; + const dedupRatio = fileSource.content.size > 0 ? dedupedBytes / fileSource.content.size : 0; + pendingFileEvents.push({ + event: "file", + path: fileSource.path, + hash: computeFileHashHex(fileChunks), + sha256: fileSource.sha256, + dedupRatio, + representation: fileRepresentation + }); + } + } + if (xorb.offset > 0) { + yield xorb.event(computeXorbHashHex); + } + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } +} +function backtrackDedup(xorb, computeHmac, shardData, chunkCache, chunkMetadata, dedupedBytes) { + const chunkIndexesToBacktrackFor = /* @__PURE__ */ new Map(); + for (let chunkToRecheckIndex = xorb.immutableData?.chunkIndex ?? 0; chunkToRecheckIndex < xorb.chunks.length; chunkToRecheckIndex++) { + const chunk2 = xorb.chunks[chunkToRecheckIndex]; + const hmacHash = computeHmac(chunk2.hash, shardData.hmacKey); + const cacheData = chunkCache.getChunk(hmacHash, null); + if (cacheData !== void 0) { + chunkIndexesToBacktrackFor.set(chunkToRecheckIndex, { + xorbId: cacheData.xorbIndex, + chunkIndex: cacheData.chunkIndex + }); + chunkCache.removeChunkFromCache(chunk2.hash); + } + } + for (const metadata of chunkMetadata) { + if (metadata.xorbId === xorb.id && chunkIndexesToBacktrackFor.has(metadata.chunkIndex)) { + const backtrackData = chunkIndexesToBacktrackFor.get(metadata.chunkIndex); + if (backtrackData !== void 0) { + metadata.xorbId = backtrackData.xorbId; + metadata.chunkIndex = backtrackData.chunkIndex; + dedupedBytes += metadata.length; + } + } + } + const xorbRangesToErase = []; + for (let i = 0; i < xorb.chunks.length; i++) { + const chunk2 = xorb.chunks[i]; + if (chunkIndexesToBacktrackFor.has(i)) { + xorbRangesToErase.push({ + start: chunk2.offset, + end: i < xorb.chunks.length - 1 ? xorb.chunks[i + 1].offset : xorb.offset + }); + } + } + const xorbRangesToKeep = []; + let currentStart = 0; + for (let i = 0; i < xorbRangesToErase.length; i++) { + const range2 = xorbRangesToErase[i]; + if (currentStart !== range2.start) { + xorbRangesToKeep.push({ start: currentStart, end: range2.start }); + } + currentStart = range2.end; + } + if (currentStart !== xorb.offset) { + xorbRangesToKeep.push({ start: currentStart, end: xorb.offset }); + } + let currentOffset = 0; + for (const range2 of xorbRangesToKeep) { + if (range2.start !== currentOffset) { + xorb.data.set(xorb.data.subarray(range2.start, range2.end), currentOffset); + } + currentOffset += range2.end - range2.start; + } + const newXorbChunks = []; + const oldIndexToNewIndex = /* @__PURE__ */ new Map(); + let erasedOffset = 0; + for (let i = 0; i < xorb.chunks.length; i++) { + const chunk2 = xorb.chunks[i]; + if (chunkIndexesToBacktrackFor.has(i)) { + if (i < xorb.chunks.length - 1) { + erasedOffset += xorb.chunks[i + 1].offset - chunk2.offset; + } + } else { + newXorbChunks.push({ + hash: chunk2.hash, + length: chunk2.length, + offset: chunk2.offset - erasedOffset + }); + if (erasedOffset > 0) { + oldIndexToNewIndex.set(i, newXorbChunks.length - 1); + } + } + } + xorb.chunks = newXorbChunks; + xorb.offset = currentOffset; + for (const chunk2 of chunkMetadata) { + if (chunk2.xorbId === xorb.id) { + const newIndex = oldIndexToNewIndex.get(chunk2.chunkIndex); + if (newIndex !== void 0) { + const cached = chunkCache.getChunk(xorb.chunks[newIndex].hash, null); + if (cached !== void 0 && cached.xorbIndex === chunk2.xorbId && cached.chunkIndex === chunk2.chunkIndex) { + chunkCache.updateChunkIndex(xorb.chunks[newIndex].hash, newIndex); + } + chunk2.chunkIndex = newIndex; + } + } + } + return dedupedBytes; +} +function removeChunkFromSourceData(sourceChunks, chunkLength) { + if (chunkLength === sourceChunks[0].length) { + const chunkToCopy = sourceChunks[0]; + sourceChunks.shift(); + return chunkToCopy; + } else if (chunkLength < sourceChunks[0].length) { + const chunkToCopy = sourceChunks[0].subarray(0, chunkLength); + sourceChunks[0] = sourceChunks[0].subarray(chunkLength); + return chunkToCopy; + } else { + const chunkToCopy = new Uint8Array(chunkLength); + let copyOffset = 0; + let index = 0; + let toSlice = -1; + while (copyOffset < chunkLength) { + const nToCopy = Math.min(sourceChunks[index].length, chunkLength - copyOffset); + chunkToCopy.set(sourceChunks[index].subarray(0, nToCopy), copyOffset); + copyOffset += nToCopy; + if (nToCopy === sourceChunks[index].length) { + index++; + } else { + toSlice = nToCopy; + } + } + sourceChunks.splice(0, index); + if (toSlice !== -1) { + sourceChunks[0] = sourceChunks[0].subarray(toSlice); + } + return chunkToCopy; + } +} +function writeChunk(xorb, chunk2, hash2) { + const regularCompressedChunk = compress(chunk2); + const bgCompressedChunk = compress(bg4_split_bytes(chunk2)); + const compressedChunk = bgCompressedChunk.length < regularCompressedChunk.length ? bgCompressedChunk : regularCompressedChunk; + const chunkToWrite = compressedChunk.length < chunk2.length ? compressedChunk : chunk2; + if (xorb.offset + XET_CHUNK_HEADER_BYTES + chunkToWrite.length > XORB_SIZE) { + return false; + } + xorb.data[xorb.offset] = 0; + xorb.data[xorb.offset + 1] = chunkToWrite.length & 255; + xorb.data[xorb.offset + 2] = chunkToWrite.length >> 8 & 255; + xorb.data[xorb.offset + 3] = chunkToWrite.length >> 16 & 255; + xorb.data[xorb.offset + 4] = chunkToWrite.length < chunk2.length ? bgCompressedChunk.length < regularCompressedChunk.length ? 2 /* ByteGroupingLZ4 */ : 1 /* LZ4 */ : 0 /* None */; + xorb.data[xorb.offset + 5] = chunk2.length & 255; + xorb.data[xorb.offset + 6] = chunk2.length >> 8 & 255; + xorb.data[xorb.offset + 7] = chunk2.length >> 16 & 255; + xorb.data.set(chunkToWrite, xorb.offset + XET_CHUNK_HEADER_BYTES); + xorb.chunks.push({ hash: hash2, length: chunk2.length, offset: xorb.offset }); + xorb.offset += XET_CHUNK_HEADER_BYTES + chunkToWrite.length; + return true; +} +var buildFileRepresentation = (metadata, chunks, computeVerificationHash) => { + if (metadata.length === 0) { + return []; + } + const representation = []; + let currentRange = { + xorbId: metadata[0].xorbId, + indexStart: metadata[0].chunkIndex, + indexEnd: metadata[0].chunkIndex + 1, + length: metadata[0].length, + chunkHashStart: 0 + }; + for (let i = 1; i < metadata.length; i++) { + const chunk2 = metadata[i]; + if (currentRange.xorbId === chunk2.xorbId && currentRange.indexEnd === chunk2.chunkIndex) { + currentRange.indexEnd = chunk2.chunkIndex + 1; + currentRange.length += chunk2.length; + } else { + const rangeHash2 = computeVerificationHash(chunks.slice(currentRange.chunkHashStart, i).map((x) => x.hash)); + representation.push({ + xorbId: currentRange.xorbId, + indexStart: currentRange.indexStart, + indexEnd: currentRange.indexEnd, + length: currentRange.length, + rangeHash: rangeHash2 + }); + currentRange = { + xorbId: chunk2.xorbId, + indexStart: chunk2.chunkIndex, + indexEnd: chunk2.chunkIndex + 1, + length: chunk2.length, + chunkHashStart: i + }; + } + } + const rangeHash = computeVerificationHash(chunks.slice(currentRange.chunkHashStart).map((x) => x.hash)); + representation.push({ + xorbId: currentRange.xorbId, + indexStart: currentRange.indexStart, + indexEnd: currentRange.indexEnd, + length: currentRange.length, + rangeHash + }); + return representation; +}; +async function loadDedupInfoToCache(content, remoteXorbHashes, params, chunkCache, computeHmacHex2, opts) { + const chunker = createChunker(TARGET_CHUNK_SIZE); + const cache = chunkCache; + let dedupedBytes = 0; + let chunksProcessed = 0; + let totalBytes = 0; + let bytesSinceRemoteDedup = Infinity; + const sourceChunks = []; + const reader = content.stream().getReader(); + const processChunks = async (chunks) => { + for (const chunk2 of chunks) { + chunksProcessed++; + if (opts?.isAtBeginning && chunksProcessed === 1) { + chunk2.dedup = true; + } + totalBytes += chunk2.length; + removeChunkFromSourceData(sourceChunks, chunk2.length); + let cacheData = cache.getChunk(chunk2.hash, computeHmacHex2); + if (cacheData !== void 0) { + dedupedBytes += chunk2.length; + bytesSinceRemoteDedup += chunk2.length; + continue; + } + if (chunk2.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { + const token = await xetWriteToken(params); + bytesSinceRemoteDedup = 0; + const shardResp = await (params.fetch ?? fetch)(token.casUrl + "/v1/chunks/default/" + chunk2.hash, { + headers: { + Authorization: `Bearer ${token.accessToken}` + } + }); + if (shardResp.ok) { + const shard = await shardResp.blob(); + const shardData = await parseShardData(shard); + for (const xorb of shardData.xorbs) { + const remoteXorbId = -remoteXorbHashes.length; + remoteXorbHashes.push(xorb.hash); + let i = 0; + for (const xorbChunk of xorb.chunks) { + cache.addChunkToCache(xorbChunk.hash, remoteXorbId, i++, shardData.hmacKey); + } + } + cacheData = cache.getChunk(chunk2.hash, computeHmacHex2); + } + } + if (cacheData !== void 0) { + dedupedBytes += chunk2.length; + } + bytesSinceRemoteDedup += chunk2.length; + } + }; + while (true) { + if (opts?.end !== void 0 && totalBytes >= opts.end) { + break; + } + if (opts?.maxChunks !== void 0 && chunksProcessed >= opts.maxChunks) { + break; + } + const { done, value } = await reader.read(); + if (done) { + await processChunks(finalizeChunker(chunker)); + break; + } + sourceChunks.push(value); + await processChunks(addDataToChunker(value, chunker)); + } +} + +// src/utils/uploadShards.ts +var SHARD_MAX_SIZE = 64 * 1024 * 1024; +var SHARD_HEADER_SIZE = 48; +var SHARD_FOOTER_SIZE = 200; +var HASH_LENGTH2 = 32; +var XORB_FOOTER_LENGTH = 48; +var FILE_FOOTER_LENGTH = 48; +var SHARD_HEADER_VERSION = 2n; +var SHARD_FOOTER_VERSION = 1n; +var MDB_FILE_FLAG_WITH_VERIFICATION = 2147483648; +var MDB_FILE_FLAG_WITH_METADATA_EXT = 1073741824; +var SHARD_MAGIC_TAG = new Uint8Array([ + "H".charCodeAt(0), + "F".charCodeAt(0), + "R".charCodeAt(0), + "e".charCodeAt(0), + "p".charCodeAt(0), + "o".charCodeAt(0), + "M".charCodeAt(0), + "e".charCodeAt(0), + "t".charCodeAt(0), + "a".charCodeAt(0), + "D".charCodeAt(0), + "a".charCodeAt(0), + "t".charCodeAt(0), + "a".charCodeAt(0), + 0, + 85, + 105, + 103, + 69, + 106, + 123, + 129, + 87, + 131, + 165, + 189, + 217, + 92, + 205, + 209, + 74, + 169 +]); +async function* uploadShards(source, params) { + const xorbHashes = []; + const seenFileXetHashes = /* @__PURE__ */ new Set(); + const fileInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.25); + const xorbInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.75); + const xorbView = new DataView(xorbInfoSection.buffer); + let xorbViewOffset = 0; + const fileInfoView = new DataView(fileInfoSection.buffer); + let fileViewOffset = 0; + let xorbTotalSize = 0n; + let fileTotalSize = 0n; + let xorbTotalUnpackedSize = 0n; + for await (const output of createXorbs(source, params)) { + switch (output.event) { + case "xorb": { + xorbHashes.push(output.hash); + const xorbEntrySize = HASH_LENGTH2 + 4 + 4 + 4 + 4; + const chunksSize = output.chunks.length * (HASH_LENGTH2 + 4 + 4 + 8); + const totalXorbSize = xorbEntrySize + chunksSize; + if (xorbViewOffset + totalXorbSize > xorbInfoSection.length) { + if (xorbViewOffset > 0 || fileViewOffset > 0) { + await uploadShard(createShard(), params); + } + } + writeHashToArray(output.hash, xorbInfoSection, xorbViewOffset); + xorbViewOffset += HASH_LENGTH2; + xorbView.setUint32(xorbViewOffset, 0, true); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, output.chunks.length, true); + xorbViewOffset += 4; + const xorbUnpackedSize = sum(output.chunks.map((x) => x.length)); + xorbView.setUint32(xorbViewOffset, xorbUnpackedSize, true); + xorbTotalUnpackedSize += BigInt(xorbUnpackedSize); + xorbTotalSize += BigInt(output.xorb.byteLength); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, output.xorb.byteLength, true); + xorbViewOffset += 4; + let chunkBytes = 0; + for (const chunk2 of output.chunks) { + writeHashToArray(chunk2.hash, xorbInfoSection, xorbViewOffset); + xorbViewOffset += HASH_LENGTH2; + xorbView.setUint32(xorbViewOffset, chunkBytes, true); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, chunk2.length, true); + xorbViewOffset += 4; + xorbView.setBigUint64(xorbViewOffset, 0n, true); + xorbViewOffset += 8; + chunkBytes += chunk2.length; + } + for (const file of output.files) { + yield { + event: "fileProgress", + path: file.path, + progress: file.lastSentProgress + }; + } + await uploadXorb(output, params); + for (const file of output.files) { + yield { event: "fileProgress", path: file.path, progress: file.progress }; + } + break; + } + case "file": { + yield { + event: "file", + path: output.path, + xetHash: output.hash, + sha256: output.sha256, + dedupRatio: output.dedupRatio + }; + if (seenFileXetHashes.has(output.hash)) { + break; + } + seenFileXetHashes.add(output.hash); + const fileHeaderSize = HASH_LENGTH2 + 4 + 4 + 8; + const representationSize = output.representation.length * (HASH_LENGTH2 + 4 + 4 + 4 + 4); + const verificationSize = output.representation.length * (HASH_LENGTH2 + 16); + const fileSha256 = output.sha256; + const hasMetadataExt = fileSha256 !== void 0; + const metadataSize = hasMetadataExt ? HASH_LENGTH2 + 16 : 0; + const totalFileSize = fileHeaderSize + representationSize + verificationSize + metadataSize; + if (fileViewOffset + totalFileSize > fileInfoSection.length) { + if (xorbViewOffset > 0 || fileViewOffset > 0) { + await uploadShard(createShard(), params); + } + } + writeHashToArray(output.hash, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + fileInfoView.setUint32( + fileViewOffset, + MDB_FILE_FLAG_WITH_VERIFICATION + (hasMetadataExt ? MDB_FILE_FLAG_WITH_METADATA_EXT : 0), + true + ); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, output.representation.length, true); + fileViewOffset += 4; + fileInfoView.setBigUint64(fileViewOffset, 0n, true); + fileViewOffset += 8; + for (const repItem of output.representation) { + writeHashToArray( + typeof repItem.xorbId === "number" ? xorbHashes[repItem.xorbId] : repItem.xorbId, + fileInfoSection, + fileViewOffset + ); + fileViewOffset += HASH_LENGTH2; + fileInfoView.setUint32(fileViewOffset, 0, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.length, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.indexStart, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.indexEnd, true); + fileViewOffset += 4; + } + for (const repItem of output.representation) { + writeHashToArray(repItem.rangeHash, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + for (let i = 0; i < 16; i++) { + fileInfoSection[fileViewOffset + i] = 0; + } + fileViewOffset += 16; + } + if (hasMetadataExt) { + writeHashToArray(fileSha256, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + for (let i = 0; i < 16; i++) { + fileInfoSection[fileViewOffset + i] = 0; + } + fileViewOffset += 16; + } + break; + } + } + } + function createShard() { + const shard = new Uint8Array( + SHARD_HEADER_SIZE + SHARD_FOOTER_SIZE + xorbViewOffset + XORB_FOOTER_LENGTH + fileViewOffset + FILE_FOOTER_LENGTH + ); + const shardView = new DataView(shard.buffer); + let shardOffset = 0; + shard.set(SHARD_MAGIC_TAG, shardOffset); + shardOffset += SHARD_MAGIC_TAG.length; + shardView.setBigUint64(shardOffset, SHARD_HEADER_VERSION, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(SHARD_FOOTER_SIZE), true); + shardOffset += 8; + shard.set(fileInfoSection.slice(0, fileViewOffset), shardOffset); + shardOffset += fileViewOffset; + for (let i = 0; i < 32; i++) { + shard[shardOffset + i] = 255; + } + shardOffset += 32; + for (let i = 0; i < 16; i++) { + shard[shardOffset + i] = 0; + } + shardOffset += 16; + const xorbInfoOffset = shardOffset; + shard.set(xorbInfoSection.slice(0, xorbViewOffset), shardOffset); + shardOffset += xorbViewOffset; + for (let i = 0; i < 32; i++) { + shard[shardOffset + i] = 255; + } + shardOffset += 32; + for (let i = 0; i < 16; i++) { + shard[shardOffset + i] = 0; + } + shardOffset += 16; + const footerOffset = shardOffset; + shardView.setBigUint64(shardOffset, SHARD_FOOTER_VERSION, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(SHARD_HEADER_SIZE), true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(xorbInfoOffset), true); + shardOffset += 8; + for (let i = 0; i < 48; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 48; + for (let i = 0; i < 32; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 32; + shardView.setBigUint64(shardOffset, BigInt(Math.floor(Date.now() / 1e3)), true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, 0n, true); + shardOffset += 8; + for (let i = 0; i < 48; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 48; + shardView.setBigUint64(shardOffset, xorbTotalSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, fileTotalSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, xorbTotalUnpackedSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(footerOffset), true); + xorbViewOffset = 0; + fileViewOffset = 0; + xorbTotalSize = 0n; + xorbTotalUnpackedSize = 0n; + fileTotalSize = 0n; + return shard; + } + if (xorbViewOffset || fileViewOffset) { + await uploadShard(createShard(), params); + } +} +function writeHashToArray(hash2, array, offset) { + for (let i = 0; i < hash2.length; i += 16) { + array[offset + i / 2] = parseInt(hash2.substring(i + 2 * 7, i + 2 * 8), 16); + array[offset + i / 2 + 1] = parseInt(hash2.substring(i + 2 * 6, i + 2 * 7), 16); + array[offset + i / 2 + 2] = parseInt(hash2.substring(i + 2 * 5, i + 2 * 6), 16); + array[offset + i / 2 + 3] = parseInt(hash2.substring(i + 2 * 4, i + 2 * 5), 16); + array[offset + i / 2 + 4] = parseInt(hash2.substring(i + 2 * 3, i + 2 * 4), 16); + array[offset + i / 2 + 5] = parseInt(hash2.substring(i + 2 * 2, i + 2 * 3), 16); + array[offset + i / 2 + 6] = parseInt(hash2.substring(i + 2 * 1, i + 2 * 2), 16); + array[offset + i / 2 + 7] = parseInt(hash2.substring(i + 2 * 0, i + 2 * 1), 16); + } +} +async function uploadXorb(xorb, params) { + const token = await xetWriteToken(params); + const resp = await (params.fetch ?? fetch)(`${token.casUrl}/v1/xorbs/default/${xorb.hash}`, { + method: "POST", + body: xorb.xorb, + headers: { + Authorization: `Bearer ${token.accessToken}`, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + }, + ...{ + progressHint: { + progressCallback: (progress) => { + for (const file of xorb.files) { + params.yieldCallback?.({ + event: "fileProgress", + path: file.path, + progress: file.lastSentProgress + (file.progress - file.lastSentProgress) * progress + }); + } + } + } + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } +} +async function uploadShard(shard, params) { + const token = await xetWriteToken(params); + const resp = await (params.fetch ?? fetch)(`${token.casUrl}/v1/shards`, { + method: "POST", + body: shard, + headers: { + Authorization: `Bearer ${token.accessToken}`, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } +} + +// src/utils/splitAsyncGenerator.ts +function splitAsyncGenerator(source, n) { + if (n <= 0) { + return []; + } + const sleep = (ms) => new Promise((resolve3) => setTimeout(resolve3, ms)); + let takenIndex = null; + const generators = []; + let remaining = n; + for (let i = 0; i < n; i++) { + generators.push({ + next: async () => { + while (takenIndex !== null) { + await sleep(1); + } + takenIndex = i; + return source.next().then((r) => { + takenIndex = null; + return r; + }); + }, + return: async () => { + remaining--; + if (remaining === 0) { + return source.return(void 0); + } + return { + done: true, + value: void 0 + }; + }, + throw: async (error) => { + return source.throw(error); + }, + [Symbol.asyncIterator]: () => generators[i] + }); + } + return generators; +} + +// src/lib/commit.ts +var CONCURRENT_SHAS = 5; +var CONCURRENT_LFS_UPLOADS = 5; +var MULTIPART_PARALLEL_UPLOAD = 5; +function isFileOperation(op) { + const ret = op.operation === "addOrUpdate"; + if (ret && !(op.content instanceof Blob)) { + throw new TypeError("Precondition failed: op.content should be a Blob"); + } + return ret; +} +async function* commitIter(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + if (repoId.type === "bucket") { + return yield* commitIterBucket(params); + } + if (params.operations.some((op) => op.operation === "copy")) { + throw new Error("'copy' operations are only supported when the destination repo is a bucket"); + } + yield { event: "phase", phase: "preuploading" }; + let useXet = params.useXet ?? true; + const lfsShas = /* @__PURE__ */ new Map(); + const abortController = new AbortController(); + const abortSignal = abortController.signal; + if (!abortSignal.throwIfAborted) { + abortSignal.throwIfAborted = () => { + if (abortSignal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + }; + } + if (params.abortSignal) { + params.abortSignal.addEventListener("abort", () => abortController.abort()); + } + try { + const allOperations = (await Promise.all( + params.operations.map(async (operation) => { + if (operation.operation === "edit") { + const splicedBlob = SplicedBlob.create( + operation.originalContent, + operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })) + ); + return { + operation: "addOrUpdate", + path: operation.path, + content: splicedBlob + }; + } + if (operation.operation !== "addOrUpdate") { + return operation; + } + if (!(operation.content instanceof URL)) { + return { ...operation, content: operation.content }; + } + const lazyBlobs = await createBlobs(operation.content, operation.path, { + fetch: params.fetch, + maxFolderDepth: params.maxFolderDepth + }); + abortSignal?.throwIfAborted(); + return lazyBlobs.map((blob) => ({ + ...operation, + content: blob.blob, + path: blob.path + })); + }) + )).flat(1); + const gitAttributes = allOperations.filter(isFileOperation).find((op) => op.path === ".gitattributes")?.content; + for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { + const payload = { + gitAttributes: gitAttributes && await gitAttributes.text(), + files: await Promise.all( + operations.map(async (operation) => ({ + path: operation.path, + size: operation.content.size, + sample: base64FromBytes(new Uint8Array(await operation.content.slice(0, 512).arrayBuffer())) + })) + ) + }; + abortSignal?.throwIfAborted(); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/preupload/${encodeURIComponent( + params.branch ?? "main" + )}` + (params.isPullRequest ? "?create_pr=1" : ""), + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/json" + }, + body: JSON.stringify(payload), + signal: abortSignal + } + ); + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + for (const file of json.files) { + if (file.uploadMode === "lfs") { + lfsShas.set(file.path, null); + } + } + } + yield { event: "phase", phase: "uploadingLargeFiles" }; + for (const operations of chunk( + allOperations.filter(isFileOperation).filter((op) => lfsShas.has(op.path)), + 100 + )) { + const shas = yield* eventToGenerator((yieldCallback, returnCallback, rejectCallack) => { + return promisesQueue( + operations.map((op) => async () => { + const iterator = sha256(op.content, { useWebWorker: params.useWebWorkers, abortSignal }); + let res2; + do { + res2 = await iterator.next(); + if (!res2.done) { + yieldCallback({ event: "fileProgress", path: op.path, progress: res2.value, state: "hashing" }); + } + } while (!res2.done); + const sha = res2.value; + lfsShas.set(op.path, res2.value); + return sha; + }), + CONCURRENT_SHAS + ).then(returnCallback, rejectCallack); + }); + abortSignal?.throwIfAborted(); + const payload = { + operation: "upload", + // multipart is a custom protocol for HF + transfers: ["basic", "multipart", ...useXet ? ["xet"] : []], + hash_algo: "sha_256", + ...!params.isPullRequest && { + ref: { + name: params.branch ?? "main" + } + }, + objects: operations.map((op, i) => ({ + oid: shas[i], + size: op.content.size + })) + }; + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/${repoId.type === "model" ? "" : repoId.type + "s/"}${repoId.name}.git/info/lfs/objects/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + Accept: "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json" + }, + body: JSON.stringify(payload), + signal: abortSignal + } + ); + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + const batchRequestId = res.headers.get("X-Request-Id") || void 0; + const shaToOperation = new Map(operations.map((op, i) => [shas[i], op])); + if (useXet && json.transfer !== "xet") { + useXet = false; + } + let xetParams = null; + if (useXet) { + for (const obj of json.objects) { + const op = shaToOperation.get(obj.oid); + if (!op) { + throw new InvalidApiResponseFormatError("Unrequested object ID in response"); + } + if (obj.error) { + const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${obj.error.message}${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; + throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); + } + if (!obj.actions?.upload) { + yield { + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }; + } else { + const headers = new Headers(obj.actions.upload.header); + xetParams = { + sessionId: headers.get("X-Xet-Session-Id") ?? void 0, + casUrl: headers.get("X-Xet-Cas-Url") ?? void 0, + accessToken: headers.get("X-Xet-Access-Token") ?? void 0, + expiresAt: headers.get("X-Xet-Token-Expiration") ? new Date(parseInt(headers.get("X-Xet-Token-Expiration") ?? "0") * 1e3) : void 0, + refreshWriteTokenUrl: obj.actions.upload.href + }; + } + } + const source = async function* () { + for (const obj of json.objects) { + const op = shaToOperation.get(obj.oid); + if (!op || !obj.actions?.upload) { + continue; + } + abortSignal?.throwIfAborted(); + yield { content: op.content, path: op.path, sha256: obj.oid }; + } + }(); + if (xetParams) { + const fixedXetParams = xetParams; + const sources = splitAsyncGenerator(source, 5); + yield* eventToGenerator( + (yieldCallback, returnCallback, rejectCallback) => Promise.all( + sources.map(async function(source2) { + for await (const event of uploadShards(source2, { + fetch: params.fetch, + accessToken, + hubUrl: params.hubUrl ?? HUB_URL, + repo: repoId, + xetParams: fixedXetParams, + // todo: maybe leave empty if PR? + rev: params.branch ?? "main", + isPullRequest: params.isPullRequest, + yieldCallback: (event2) => yieldCallback({ ...event2, state: "uploading" }) + })) { + if (event.event === "file") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: 1, + state: "uploading" + }); + } else if (event.event === "fileProgress") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: event.progress, + state: "uploading" + }); + } + } + }) + ).then(() => returnCallback(void 0), rejectCallback) + ); + } else { + } + } else { + yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + return promisesQueueStreaming( + json.objects.map((obj) => async () => { + const op = shaToOperation.get(obj.oid); + if (!op) { + throw new InvalidApiResponseFormatError("Unrequested object ID in response"); + } + abortSignal?.throwIfAborted(); + if (obj.error) { + const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${obj.error.message}${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; + throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); + } + if (!obj.actions?.upload) { + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + return; + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 0, + state: "uploading" + }); + const content = op.content; + const header = obj.actions.upload.header; + if (header?.chunk_size) { + const chunkSize = parseInt(header.chunk_size); + const completionUrl = obj.actions.upload.href; + const parts = Object.keys(header).filter((key) => /^[0-9]+$/.test(key)); + if (parts.length !== Math.ceil(content.size / chunkSize)) { + throw new Error("Invalid server response to upload large LFS file, wrong number of parts"); + } + const completeReq = { + oid: obj.oid, + parts: parts.map((part) => ({ + partNumber: +part, + etag: "" + })) + }; + const progressCallback = (progress) => yieldCallback({ event: "fileProgress", path: op.path, progress, state: "uploading" }); + await promisesQueueStreaming( + parts.map((part) => async () => { + abortSignal?.throwIfAborted(); + const index = parseInt(part) - 1; + const slice = content.slice(index * chunkSize, (index + 1) * chunkSize); + const res3 = await (params.fetch ?? fetch)(header[part], { + method: "PUT", + /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ + body: slice instanceof WebBlob && isFrontend ? await slice.arrayBuffer() : slice, + signal: abortSignal, + ...{ + progressHint: { + path: op.path, + part: index, + numParts: parts.length, + progressCallback + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + }); + if (!res3.ok) { + throw await createApiError(res3, { + requestId: batchRequestId, + message: `Error while uploading part ${part} of ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + const eTag = res3.headers.get("ETag"); + if (!eTag) { + throw new Error("Cannot get ETag of part during multipart upload"); + } + completeReq.parts[Number(part) - 1].etag = eTag; + }), + MULTIPART_PARALLEL_UPLOAD + ); + abortSignal?.throwIfAborted(); + const res2 = await (params.fetch ?? fetch)(completionUrl, { + method: "POST", + body: JSON.stringify(completeReq), + headers: { + Accept: "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json" + }, + signal: abortSignal + }); + if (!res2.ok) { + throw await createApiError(res2, { + requestId: batchRequestId, + message: `Error completing multipart upload of ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + } else { + const res2 = await (params.fetch ?? fetch)(obj.actions.upload.href, { + method: "PUT", + headers: { + ...batchRequestId ? { "X-Request-Id": batchRequestId } : void 0 + }, + /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ + body: content instanceof WebBlob && isFrontend ? await content.arrayBuffer() : content, + signal: abortSignal, + ...{ + progressHint: { + path: op.path, + progressCallback: (progress) => yieldCallback({ + event: "fileProgress", + path: op.path, + progress, + state: "uploading" + }) + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + }); + if (!res2.ok) { + throw await createApiError(res2, { + requestId: batchRequestId, + message: `Error while uploading ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + } + }), + CONCURRENT_LFS_UPLOADS + ).then(returnCallback, rejectCallback); + }); + } + } + abortSignal?.throwIfAborted(); + yield { event: "phase", phase: "committing" }; + return yield* eventToGenerator( + async (yieldCallback, returnCallback, rejectCallback) => (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commit/${encodeURIComponent( + params.branch ?? "main" + )}` + (params.isPullRequest ? "?create_pr=1" : ""), + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: [ + { + key: "header", + value: { + summary: params.title, + description: params.description, + parentCommit: params.parentCommit + } + }, + ...await Promise.all( + allOperations.map((operation) => { + if (isFileOperation(operation)) { + const sha = lfsShas.get(operation.path); + if (sha) { + return { + key: "lfsFile", + value: { + path: operation.path, + algo: "sha256", + size: operation.content.size, + oid: sha + } + }; + } + } + return convertOperationToNdJson(operation); + }) + ) + ].map((x) => JSON.stringify(x)).join("\n"), + signal: abortSignal, + ...{ + progressHint: { + progressCallback: (progress) => { + for (const op of allOperations) { + if (isFileOperation(op) && !lfsShas.has(op.path)) { + yieldCallback({ + event: "fileProgress", + path: op.path, + progress, + state: "uploading" + }); + } + } + } + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + } + ).then(async (res) => { + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + returnCallback({ + pullRequestUrl: json.pullRequestUrl, + commit: { + oid: json.commitOid, + url: json.commitUrl + }, + hookOutput: json.hookOutput + }); + }).catch(rejectCallback) + ); + } catch (err) { + abortController.abort(); + throw err; + } +} +async function* commitIterBucket(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + if (params.useXet === false) { + throw new Error("useXet must be true or undefined for buckets"); + } + const abortController = new AbortController(); + const abortSignal = abortController.signal; + if (!abortSignal.throwIfAborted) { + abortSignal.throwIfAborted = () => { + if (abortSignal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + }; + } + if (params.abortSignal) { + params.abortSignal.addEventListener("abort", () => abortController.abort()); + } + try { + const allOperations = (await Promise.all( + params.operations.map(async (operation) => { + if (operation.operation === "edit") { + const splicedBlob = SplicedBlob.create( + operation.originalContent, + operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })) + ); + return { + operation: "addOrUpdate", + path: operation.path, + content: splicedBlob + }; + } + if (operation.operation !== "addOrUpdate") { + return operation; + } + if (!(operation.content instanceof URL)) { + return { ...operation, content: operation.content }; + } + const lazyBlobs = await createBlobs(operation.content, operation.path, { + fetch: params.fetch, + maxFolderDepth: params.maxFolderDepth + }); + abortSignal?.throwIfAborted(); + return lazyBlobs.map((blob) => ({ + ...operation, + content: blob.blob, + path: blob.path + })); + }) + )).flat(1); + yield { event: "phase", phase: "uploadingLargeFiles" }; + for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { + const xetHashes = /* @__PURE__ */ new Map(); + abortSignal?.throwIfAborted(); + const source = async function* () { + for (const operation of operations) { + abortSignal?.throwIfAborted(); + yield { content: operation.content, path: operation.path }; + } + }(); + const xetParams = { + sessionId: crypto.randomUUID(), + refreshWriteTokenUrl: `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/xet-write-token` + }; + const sources = splitAsyncGenerator(source, 5); + yield* eventToGenerator( + (yieldCallback, returnCallback, rejectCallback) => Promise.all( + sources.map(async function(source2) { + for await (const event of uploadShards(source2, { + fetch: params.fetch, + accessToken, + hubUrl: params.hubUrl ?? HUB_URL, + repo: repoId, + xetParams, + rev: params.branch ?? "main", + yieldCallback: (event2) => yieldCallback({ ...event2, state: "uploading" }) + })) { + if (event.event === "file") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: 1, + state: "uploading" + }); + xetHashes.set(event.path, event.xetHash); + } else if (event.event === "fileProgress") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: event.progress, + state: "uploading" + }); + } + } + }) + ).then(() => returnCallback(void 0), rejectCallback) + ); + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: [...xetHashes.entries()].map( + ([path2, xetHash]) => JSON.stringify({ + type: "addFile", + path: path2, + xetHash + }) + ).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok && resp.status !== 422) { + throw await createApiError(resp); + } + const json = await resp.json(); + for (const failed of json.failed) { + yield { + event: "fileProgress", + path: failed.path, + progress: 0, + state: "error" + }; + } + } + abortSignal?.throwIfAborted(); + const copyOperations = allOperations.filter( + (operation) => operation.operation === "copy" + ); + for (const copyChunk of chunk(copyOperations, 100)) { + abortSignal?.throwIfAborted(); + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: copyChunk.map((op) => { + const sourceRepoId = toRepoId(op.sourceRepo); + return JSON.stringify({ + type: "copyFile", + path: op.path, + xetHash: op.sourceXetHash, + sourceRepoType: sourceRepoId.type, + sourceRepoId: sourceRepoId.name + }); + }).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok && resp.status !== 422) { + throw await createApiError(resp); + } + const json = await resp.json(); + for (const failed of json.failed) { + yield { + event: "fileProgress", + path: failed.path, + progress: 0, + state: "error" + }; + } + } + abortSignal?.throwIfAborted(); + const deletedOperations = allOperations.filter((operation) => operation.operation === "delete"); + if (deletedOperations.length > 0) { + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: deletedOperations.map( + (operation) => JSON.stringify({ + type: "deleteFile", + path: operation.path + }) + ).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok) { + throw await createApiError(resp); + } + const json = await resp.json(); + if (json.failed.length > 0) { + const failedPaths = json.failed.slice(0, 5).map((f) => f.path); + throw new Error( + `Failed to delete ${json.failed.length} file(s): ${failedPaths.join(", ")}${json.failed.length > 5 ? "..." : ""}, request ID: ${resp.headers.get("X-Request-Id")}` + ); + } + } + abortSignal?.throwIfAborted(); + } catch (err) { + abortController.abort(); + throw err; + } +} +async function commit(params) { + const iterator = commitIter(params); + const failedPaths = []; + let failedCount = 0; + let res = await iterator.next(); + while (!res.done) { + if (res.value.event === "fileProgress" && res.value.state === "error") { + failedCount++; + if (failedPaths.length < 5) { + failedPaths.push(res.value.path); + } + } + res = await iterator.next(); + } + if (failedCount > 0) { + throw new Error( + `Failed to upload ${failedCount} file(s): ${failedPaths.join(", ")}${failedCount > 5 ? "..." : ""}` + ); + } + return res.value; +} +async function convertOperationToNdJson(operation) { + switch (operation.operation) { + case "addOrUpdate": { + return { + key: "file", + value: { + content: base64FromBytes(new Uint8Array(await operation.content.arrayBuffer())), + path: operation.path, + encoding: "base64" + } + }; + } + case "delete": { + return { + key: "deletedFile", + value: { + path: operation.path + } + }; + } + case "edit": { + throw new Error( + "Edit operations should be converted to addOrUpdate operations before reaching convertOperationToNdJson" + ); + } + default: + throw new TypeError("Unknown operation: " + operation.operation); + } +} + +// src/utils/parseLinkHeader.ts +function parseLinkHeader(header) { + const regex = /<(https?:[/][/][^>]+)>;\s+rel="([^"]+)"/g; + return Object.fromEntries([...header.matchAll(regex)].map(([, url, rel]) => [rel, url])); +} + +// src/lib/file-download-info.ts +async function fileDownloadInfo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const hubUrl = params.hubUrl ?? HUB_URL; + const revision = repoId.type === "bucket" ? void 0 : params.revision ?? "main"; + const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${params.raw ? "raw" : "resolve"}${revision ? `/${encodeURIComponent(revision)}` : ""}/${params.path}` + (params.noContentDisposition ? "?noContentDisposition=1" : ""); + const resp = await (params.fetch ?? fetch)(url, { + method: "GET", + headers: { + ...accessToken && { + Authorization: `Bearer ${accessToken}` + }, + Range: "bytes=0-0", + Accept: "application/vnd.xet-fileinfo+json, */*" + } + }); + if (resp.status === 404 && resp.headers.get("X-Error-Code") === "EntryNotFound") { + return null; + } + if (!resp.ok) { + throw await createApiError(resp); + } + let size; + let xetInfo; + if (resp.headers.get("Content-Type")?.includes("application/vnd.xet-fileinfo+json")) { + size = parseInt(resp.headers.get("X-Linked-Size") ?? "invalid"); + if (isNaN(size)) { + throw new InvalidApiResponseFormatError("Invalid file size received in X-Linked-Size header"); + } + const hash2 = resp.headers.get("X-Xet-Hash"); + const links = parseLinkHeader(resp.headers.get("Link") ?? ""); + const reconstructionUrl = (() => { + try { + return new URL(links["xet-reconstruction-info"]); + } catch { + return null; + } + })(); + const refreshUrl = (() => { + try { + return new URL(links["xet-auth"]); + } catch { + return null; + } + })(); + if (!hash2) { + throw new InvalidApiResponseFormatError("No hash received in X-Xet-Hash header"); + } + if (!reconstructionUrl || !refreshUrl) { + throw new InvalidApiResponseFormatError("No xet-reconstruction-info or xet-auth link header"); + } + xetInfo = { + hash: hash2, + refreshUrl, + reconstructionUrl + }; + } + if (size === void 0 || isNaN(size)) { + const contentRangeHeader = resp.headers.get("content-range"); + if (!contentRangeHeader) { + throw new InvalidApiResponseFormatError("Expected size information"); + } + const [, parsedSize] = contentRangeHeader.split("/"); + size = parseInt(parsedSize); + if (isNaN(size)) { + throw new InvalidApiResponseFormatError("Invalid file size received"); + } + } + const etag = resp.headers.get("X-Linked-ETag") ?? resp.headers.get("ETag") ?? void 0; + if (!etag) { + throw new InvalidApiResponseFormatError("Expected ETag"); + } + return { + etag, + size, + xet: xetInfo, + // Cannot use resp.url in case it's a S3 url and the user adds an Authorization header to it. + url: resp.url && (new URL(resp.url).origin === new URL(hubUrl).origin || resp.headers.get("X-Cache")?.endsWith(" cloudfront")) ? resp.url : url + }; +} + +// src/lib/download-file.ts +async function downloadFile(params) { + const accessToken = checkCredentials(params); + const info = params.downloadInfo ?? await fileDownloadInfo({ + accessToken, + repo: params.repo, + path: params.path, + revision: params.revision, + hubUrl: params.hubUrl, + fetch: params.fetch, + raw: params.raw + }); + if (!info) { + return null; + } + if (info.xet && params.xet !== false) { + return new XetBlob({ + refreshUrl: info.xet.refreshUrl.href, + reconstructionUrl: info.xet.reconstructionUrl.href, + fetch: params.fetch, + accessToken, + size: info.size, + readToken: typeof params.xet === "object" ? params.xet.readToken : void 0 + }); + } + return new WebBlob(new URL(info.url), 0, info.size, "", true, params.fetch ?? fetch, accessToken); +} + +// src/lib/list-files.ts +async function* listFiles(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const revision = repoId.type === "bucket" ? void 0 : params.revision || "main"; + let url = `${params.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}/tree${revision ? `/${revision}` : ""}${params.path ? "/" + params.path : ""}?recursive=${!!params.recursive}&expand=${!!params.expand}`; + while (url) { + const res = await (params.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + yield item; + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/paths-info.ts +async function pathsInfo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const hubUrl = params.hubUrl ?? HUB_URL; + const revision = repoId.type === "bucket" ? void 0 : params.revision ?? "main"; + const url = `${hubUrl}/api/${repoId.type}s/${repoId.name}/paths-info${revision ? `/${encodeURIComponent(revision)}` : ""}`; + const resp = await (params.fetch ?? fetch)(url, { + method: "POST", + headers: { + ...accessToken && { + Authorization: `Bearer ${accessToken}` + }, + Accept: "application/json", + "Content-Type": "application/json" + }, + body: JSON.stringify({ + paths: params.paths, + expand: params.expand + }) + }); + if (!resp.ok) { + throw await createApiError(resp); + } + const json = await resp.json(); + if (!Array.isArray(json)) { + throw new Error("malformed response: expected array"); + } + return json.map((item) => ({ + path: item.path, + lfs: item.lfs, + type: item.type, + oid: item.oid, + size: item.size, + xetHash: item.xetHash, + uploadedAt: item.uploadedAt, + securityFileStatus: item.securityFileStatus, + lastCommit: item.lastCommit ? { + date: new Date(item.lastCommit.date), + title: item.lastCommit.title, + id: item.lastCommit.id + } : void 0 + })); +} + +// src/utils/formatBytes.ts +function formatBytes(bytes) { + if (!Number.isFinite(bytes) || bytes < 0) { + return `${bytes} B`; + } + const units = ["B", "kB", "MB", "GB", "TB", "PB"]; + let value = bytes; + let i = 0; + while (value >= 1e3 && i < units.length - 1) { + value /= 1e3; + i++; + } + const formatted = i === 0 ? value.toString() : value.toFixed(value >= 100 ? 0 : value >= 10 ? 1 : 2); + return `${formatted} ${units[i]}`; +} + +// src/lib/copy-files.ts +var DOWNLOAD_CONCURRENCY = 5; +var PATHS_INFO_BATCH_SIZE = 100; +var MAX_REPORTED_LFS_PATHS = 5; +function copyFile(params) { + return copyFiles({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + destination: params.destination.repo, + files: [ + { + source: params.source, + destinationPath: params.destination.path + } + ], + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); +} +function copyFileIter(params) { + return copyFilesIter({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + destination: params.destination.repo, + files: [ + { + source: params.source, + destinationPath: params.destination.path + } + ], + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); +} +async function copyFiles(params) { + const iterator = copyFilesIter(params); + while (true) { + const res = await iterator.next(); + if (res.done) { + return void 0; + } + } +} +async function* copyFilesIter(params) { + if (params.files.length === 0) { + return void 0; + } + const operations = yield* resolveCopyOperationsIter(params, params.files); + await commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.destination, + operations, + title: "", + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); + return void 0; +} +async function copyFolder(params) { + const iterator = copyFolderIter(params); + while (true) { + const res = await iterator.next(); + if (res.done) { + return void 0; + } + } +} +async function* copyFolderIter(params) { + const accessToken = checkCredentials(params); + const sourceRepoId = toRepoId(params.source.repo); + const sourcePath = (params.source.path ?? "").replace(/\/+$/, ""); + const destinationPrefix = (params.destination.path ?? "").replace(/\/+$/, ""); + const sourceRevision = sourceRepoId.type === "bucket" ? void 0 : params.source.revision ?? "main"; + const operations = []; + const pendingDownloads = []; + const lfsOffenders = []; + for await (const item of listFiles({ + repo: sourceRepoId, + path: sourcePath || void 0, + recursive: true, + revision: sourceRevision, + accessToken, + hubUrl: params.hubUrl, + fetch: params.fetch + })) { + if (item.type !== "file") { + continue; + } + const relPath = relativeUnderFolder(item.path, sourcePath); + const destPath = destinationPrefix ? `${destinationPrefix}/${relPath}` : relPath; + switch (classifySourceFile(item)) { + case "copy": + operations.push({ + operation: "copy", + path: destPath, + sourceXetHash: item.xetHash, + sourceRepo: sourceRepoId + }); + continue; + case "lfs": + lfsOffenders.push({ path: item.path, size: item.lfs?.size ?? item.size }); + continue; + case "download": + pendingDownloads.push({ + index: operations.length, + repoId: sourceRepoId, + revision: sourceRevision, + sourcePath: item.path + }); + operations.push({ + operation: "addOrUpdate", + path: destPath, + content: new Blob([]) + }); + continue; + } + } + if (lfsOffenders.length > 0) { + throwUnmigratedLfsError(sourceRepoId, lfsOffenders); + } + if (operations.length === 0) { + return void 0; + } + yield* downloadAndFillBlobsIter({ + pendingDownloads, + operations, + accessToken, + hubUrl: params.hubUrl, + fetch: params.fetch + }); + await commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.destination.repo, + operations, + title: "", + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); + return void 0; +} +async function* resolveCopyOperationsIter(shared, files) { + const accessToken = checkCredentials(shared); + const groups = /* @__PURE__ */ new Map(); + for (let i = 0; i < files.length; i++) { + const file = files[i]; + const repoId = toRepoId(file.source.repo); + const revision = repoId.type === "bucket" ? void 0 : file.source.revision ?? "main"; + const key = `${repoId.type}\0${repoId.name}\0${revision ?? ""}`; + let group = groups.get(key); + if (!group) { + group = { repoId, revision, entries: [] }; + groups.set(key, group); + } + group.entries.push({ index: i, file }); + } + const operations = new Array(files.length); + const pendingDownloads = []; + for (const group of groups.values()) { + const paths = group.entries.map((e) => e.file.source.path); + const infos = []; + for (let offset = 0; offset < paths.length; offset += PATHS_INFO_BATCH_SIZE) { + const slice = paths.slice(offset, offset + PATHS_INFO_BATCH_SIZE); + const res = await pathsInfo({ + repo: group.repoId, + paths: slice, + revision: group.revision, + accessToken, + hubUrl: shared.hubUrl, + fetch: shared.fetch + }); + infos.push(...res); + } + const infoByPath = new Map(infos.map((i) => [i.path, i])); + const lfsOffenders = []; + for (const { index, file } of group.entries) { + const info = infoByPath.get(file.source.path); + if (!info) { + throw new Error(`Source file not found: '${file.source.path}' in ${group.repoId.type}s/${group.repoId.name}`); + } + if (info.type !== "file") { + throw new Error( + `Source path '${file.source.path}' in ${group.repoId.type}s/${group.repoId.name} is a folder; use copyFolder() instead.` + ); + } + switch (classifySourceFile(info)) { + case "copy": + operations[index] = { + operation: "copy", + path: file.destinationPath, + sourceXetHash: info.xetHash, + sourceRepo: group.repoId + }; + continue; + case "lfs": + lfsOffenders.push({ path: file.source.path, size: info.lfs?.size ?? info.size }); + continue; + case "download": + pendingDownloads.push({ + index, + repoId: group.repoId, + revision: group.revision, + sourcePath: file.source.path + }); + operations[index] = { + operation: "addOrUpdate", + path: file.destinationPath, + content: new Blob([]) + }; + continue; + } + } + if (lfsOffenders.length > 0) { + throwUnmigratedLfsError(group.repoId, lfsOffenders); + } + } + yield* downloadAndFillBlobsIter({ + pendingDownloads, + operations, + accessToken, + hubUrl: shared.hubUrl, + fetch: shared.fetch + }); + return operations; +} +function downloadAndFillBlobsIter(args) { + const total = args.pendingDownloads.length; + return eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + if (total === 0) { + returnCallback(); + return; + } + let downloaded = 0; + promisesQueue( + args.pendingDownloads.map(({ index, repoId, revision, sourcePath }) => async () => { + const blob = await downloadFile({ + repo: repoId, + path: sourcePath, + revision, + accessToken: args.accessToken, + hubUrl: args.hubUrl, + fetch: args.fetch + }); + if (!blob) { + throw new Error(`Failed to download '${sourcePath}' from ${repoId.type}s/${repoId.name}`); + } + const op = args.operations[index]; + if (op.operation !== "addOrUpdate") { + throw new Error("Internal: expected addOrUpdate placeholder operation"); + } + op.content = blob; + downloaded++; + yieldCallback({ event: "fileDownloaded", path: sourcePath, downloaded, total }); + }), + DOWNLOAD_CONCURRENCY + ).then( + () => returnCallback(), + (err) => rejectCallback(err) + ); + }); +} +function relativeUnderFolder(filePath, folderPath) { + if (!folderPath) { + return filePath; + } + if (filePath === folderPath) { + return filePath.split("/").pop() ?? filePath; + } + if (filePath.startsWith(folderPath + "/")) { + return filePath.slice(folderPath.length + 1); + } + throw new Error(`Path '${filePath}' is not inside folder '${folderPath}'`); +} +function classifySourceFile(file) { + if (file.xetHash) { + return "copy"; + } + if (file.lfs) { + return "lfs"; + } + return "download"; +} +function throwUnmigratedLfsError(repoId, entries) { + const head = entries.slice(0, MAX_REPORTED_LFS_PATHS).map((e) => `'${e.path}' (${formatBytes(e.size)})`).join(", "); + const more = entries.length > MAX_REPORTED_LFS_PATHS ? ` (and ${entries.length - MAX_REPORTED_LFS_PATHS} more)` : ""; + throw new Error( + `Cannot copy ${entries.length} LFS file(s) from ${repoId.type}s/${repoId.name} that have not been migrated to xet: ${head}${more}. Migrate these files to xet before copying.` + ); +} + +// src/lib/count-commits.ts +async function countCommits(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const url = `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commits/${params.revision ?? "main"}?limit=1`; + const res = await (params.fetch ?? fetch)(url, { + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }); + if (!res.ok) { + throw await createApiError(res); + } + return parseInt(res.headers.get("x-total-count") ?? "0", 10); +} + +// src/lib/create-repo.ts +async function createRepo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const [namespace, repoName] = repoId.name.split("/"); + const visibility = params.visibility ?? (params.private !== void 0 ? params.private ? "private" : "public" : void 0); + if (!namespace || !repoName) { + throw new TypeError( + `"${repoId.name}" is not a fully qualified repo name. It should be of the form "{namespace}/{repoName}".` + ); + } + const res = repoId.type === "bucket" ? await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/buckets/${namespace}/${repoName}`, { + method: "POST", + body: JSON.stringify({ + visibility, + resourceGroupId: params.resourceGroupId + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }) : await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/repos/create`, { + method: "POST", + body: JSON.stringify({ + name: repoName, + visibility, + organization: namespace, + resourceGroupId: params.resourceGroupId, + license: params.license, + ...repoId.type === "space" ? { + type: "space", + sdk: params.sdk ?? "static" + } : { + type: repoId.type + }, + files: params.files ? await Promise.all( + params.files.map(async (file) => ({ + encoding: "base64", + path: file.path, + content: base64FromBytes( + new Uint8Array(file.content instanceof Blob ? await file.content.arrayBuffer() : file.content) + ) + })) + ) : void 0 + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const output = await res.json(); + return { repoUrl: output.url, id: output.id }; +} + +// src/lib/create-branch.ts +async function createBranch(params) { + const repoId = toRepoId(params.repo); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + }, + body: JSON.stringify({ + startingPoint: params.revision, + ...params.empty && { emptyBranch: true }, + overwrite: params.overwrite + }) + } + ); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/create-collection.ts +async function createCollection(params) { + const accessToken = checkCredentials(params); + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/collections`, { + method: "POST", + body: JSON.stringify(params.collection), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const output = await res.json(); + return { slug: output.slug }; +} + +// src/utils/pick.ts +function pick(o, props) { + return Object.assign( + {}, + ...props.map((prop) => { + if (o[prop] !== void 0) { + return { [prop]: o[prop] }; + } + }) + ); +} + +// src/lib/list-datasets.ts +var DATASET_EXPAND_KEYS = [ + "private", + "downloads", + "gated", + "likes", + "lastModified" +]; +var DATASET_EXPANDABLE_KEYS = [ + "author", + "cardData", + "citation", + "createdAt", + "disabled", + "description", + "downloads", + "downloadsAllTime", + "gated", + "gitalyUid", + "lastModified", + "likes", + "paperswithcode_id", + "private", + // "siblings", + "sha", + "tags" +]; +async function* listDatasets(params) { + const accessToken = params && checkCredentials(params); + let totalToFetch = params?.limit ?? Infinity; + const search = new URLSearchParams([ + ...Object.entries({ + limit: String(Math.min(totalToFetch, 500)), + ...params?.search?.owner ? { author: params.search.owner } : void 0, + ...params?.search?.query ? { search: params.search.query } : void 0, + ...params?.sort ? { sort: params.sort } : void 0 + }), + ...params?.search?.tags?.map((tag) => ["filter", tag]) ?? [], + ...DATASET_EXPAND_KEYS.map((val) => ["expand", val]), + ...params?.additionalFields?.map((val) => ["expand", val]) ?? [] + ]).toString(); + let url = `${params?.hubUrl || HUB_URL}/api/datasets` + (search ? "?" + search : ""); + while (url) { + const res = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + yield { + ...params?.additionalFields && pick(item, params.additionalFields), + id: item._id, + name: item.id, + private: item.private, + downloads: item.downloads, + likes: item.likes, + gated: item.gated, + updatedAt: new Date(item.lastModified) + }; + totalToFetch--; + if (totalToFetch <= 0) { + return; + } + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/dataset-info.ts +async function datasetInfo(params) { + const accessToken = params && checkCredentials(params); + const search = new URLSearchParams([ + ...DATASET_EXPAND_KEYS.map((val) => ["expand", val]), + ...params?.additionalFields?.map((val) => ["expand", val]) ?? [] + ]).toString(); + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/datasets/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD" + )}?${search.toString()}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + const data = await response.json(); + return { + ...params?.additionalFields && pick(data, params.additionalFields), + id: data._id, + name: data.id, + private: data.private, + downloads: data.downloads, + likes: data.likes, + gated: data.gated, + updatedAt: new Date(data.lastModified) + }; +} + +// src/lib/delete-branch.ts +async function deleteBranch(params) { + const repoId = toRepoId(params.repo); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, + { + method: "DELETE", + headers: { + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + } + } + ); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/delete-file.ts +function deleteFile(params) { + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: [ + { + operation: "delete", + path: params.path + } + ], + title: params.commitTitle ?? `Delete ${params.path}`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch + }); +} + +// src/lib/delete-files.ts +function deleteFiles(params) { + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: params.paths.map((path2) => ({ + operation: "delete", + path: path2 + })), + title: params.commitTitle ?? `Deletes ${params.paths.length} files`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch + }); +} + +// src/lib/delete-repo.ts +async function deleteRepo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const [namespace, repoName] = repoId.name.split("/"); + const res = repoId.type === "bucket" ? await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/buckets/${namespace}/${repoName}`, { + method: "DELETE", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }) : await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/repos/delete`, { + method: "DELETE", + body: JSON.stringify({ + name: repoName, + organization: namespace, + type: repoId.type + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/delete-collection.ts +async function deleteCollection(params) { + if (!params.slug) { + throw new TypeError("slug is required"); + } + const accessToken = checkCredentials(params); + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/collections/${params.slug}`, { + method: "DELETE", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/download-file-to-cache-dir.ts +import { dirname as dirname2, join as join2 } from "path"; +import { rename, lstat as lstat2, mkdir, stat as stat2 } from "fs/promises"; + +// src/utils/symlink.ts +import * as fs from "fs/promises"; +import * as path from "path"; +import * as os from "os"; +function expandUser(path2) { + if (path2.startsWith("~")) { + return path2.replace("~", os.homedir()); + } + return path2; +} +async function createSymlink(params) { + const abs_src = path.resolve(expandUser(params.sourcePath)); + const abs_dst = path.resolve(expandUser(params.finalPath)); + try { + await fs.rm(abs_dst); + } catch { + } + try { + await fs.symlink(path.relative(path.dirname(abs_dst), abs_src), abs_dst); + } catch { + console.info(`Symlink not supported. Copying file from ${abs_src} to ${abs_dst}`); + await fs.copyFile(abs_src, abs_dst); + } +} + +// src/lib/download-file-to-cache-dir.ts +import { Readable } from "stream"; +import { pipeline } from "stream/promises"; +import { createWriteStream } from "fs"; +var REGEX_COMMIT_HASH = new RegExp("^[0-9a-f]{40}$"); +function getFilePointer(storageFolder, revision, relativeFilename) { + const snapshotPath = join2(storageFolder, "snapshots"); + return join2(snapshotPath, revision, relativeFilename); +} +async function exists(path2, followSymlinks) { + try { + if (followSymlinks) { + await stat2(path2); + } else { + await lstat2(path2); + } + return true; + } catch (err) { + return false; + } +} +async function downloadFileToCacheDir(params) { + const repoId = toRepoId(params.repo); + if (repoId.type === "bucket") { + throw new Error("downloadFileToCacheDir is not supported for bucket repos."); + } + const revision = params.revision ?? "main"; + const cacheDir = params.cacheDir ?? getHFHubCachePath(); + const storageFolder = join2(cacheDir, getRepoFolderName(repoId)); + let commitHash; + if (revision && REGEX_COMMIT_HASH.test(revision)) { + commitHash = revision; + const pointerPath2 = getFilePointer(storageFolder, revision, params.path); + if (await exists(pointerPath2, true)) { + return pointerPath2; + } + } + const pathsInformation = await pathsInfo({ + ...params, + paths: [params.path], + revision, + expand: true + }); + if (!pathsInformation || pathsInformation.length !== 1) { + throw new Error(`cannot get path info for ${params.path}`); + } + const info = pathsInformation[0]; + let etag; + if (info.lfs) { + etag = info.lfs.oid; + } else if (info.xetHash) { + etag = info.xetHash; + } else if (info.oid) { + etag = info.oid; + } else { + throw new Error(`cannot determine etag for ${params.path}`); + } + const snapshotId = commitHash ?? info.lastCommit?.id ?? etag; + const pointerPath = getFilePointer(storageFolder, snapshotId, params.path); + const blobPath = join2(storageFolder, "blobs", etag); + if (await exists(pointerPath, true)) { + return pointerPath; + } + await mkdir(dirname2(blobPath), { recursive: true }); + await mkdir(dirname2(pointerPath), { recursive: true }); + if (await exists(blobPath)) { + await createSymlink({ sourcePath: blobPath, finalPath: pointerPath }); + return pointerPath; + } + const incomplete = `${blobPath}.incomplete`; + console.debug(`Downloading ${params.path} to ${incomplete}`); + const blob = await downloadFile({ + ...params, + revision + }); + if (!blob) { + throw new Error(`invalid response for file ${params.path}`); + } + await pipeline(Readable.fromWeb(blob.stream()), createWriteStream(incomplete)); + await rename(incomplete, blobPath); + await createSymlink({ sourcePath: blobPath, finalPath: pointerPath }); + return pointerPath; +} + +// src/lib/file-exists.ts +async function fileExists(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const hubUrl = params.hubUrl ?? HUB_URL; + const revision = repoId.type === "bucket" ? void 0 : params.revision ?? "main"; + const endpoint = repoId.type === "bucket" ? "resolve" : "raw"; + const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${endpoint}${revision ? `/${encodeURIComponent(revision)}` : ""}/${params.path}`; + const resp = await (params.fetch ?? fetch)(url, { + method: "HEAD", + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }); + if (resp.status === 404) { + return false; + } + if (!resp.ok) { + throw await createApiError(resp); + } + return true; +} + +// src/lib/jobs/cancel-job.ts +async function cancelJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/cancel`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/create-scheduled-job.ts +async function createScheduledJob(params) { + const accessToken = checkCredentials(params); + const { namespace, hubUrl, fetch: customFetch, ...rest } = params; + if (!rest.jobSpec.dockerImage && !rest.jobSpec.spaceId) { + throw new Error("Either dockerImage or spaceId must be provided in jobSpec"); + } + if (rest.jobSpec.dockerImage && rest.jobSpec.spaceId) { + throw new Error("Cannot provide both dockerImage and spaceId in jobSpec"); + } + const body = { + jobSpec: { + flavor: rest.jobSpec.flavor + }, + schedule: rest.schedule, + suspend: rest.suspend ?? false, + concurrency: rest.concurrency ?? false + }; + if (rest.jobSpec.dockerImage) { + body.jobSpec.dockerImage = rest.jobSpec.dockerImage; + } + if (rest.jobSpec.spaceId) { + body.jobSpec.spaceId = rest.jobSpec.spaceId; + } + if (rest.jobSpec.command) { + body.jobSpec.command = rest.jobSpec.command; + } + body.jobSpec.environment = rest.jobSpec.environment || {}; + if (rest.jobSpec.secrets) { + body.jobSpec.secrets = rest.jobSpec.secrets; + } + if (rest.jobSpec.arch) { + body.jobSpec.arch = rest.jobSpec.arch; + } + if (rest.jobSpec.timeoutSeconds !== void 0) { + body.jobSpec.timeoutSeconds = rest.jobSpec.timeoutSeconds; + } + if (rest.jobSpec.attempts !== void 0) { + body.jobSpec.attempts = rest.jobSpec.attempts; + } + if (rest.jobSpec.labels) { + body.jobSpec.labels = rest.jobSpec.labels; + } + if (rest.jobSpec.volumes?.length) { + body.jobSpec.volumes = rest.jobSpec.volumes.map(({ source, ...rest2 }) => { + const repoId = toRepoId(source); + return { type: repoId.type, source: repoId.name, ...rest2 }; + }); + } + const response = await (customFetch || fetch)(`${hubUrl || HUB_URL}/api/scheduled-jobs/${namespace}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }, + body: JSON.stringify(body) + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/delete-scheduled-job.ts +async function deleteScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}`, + { + method: "DELETE", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/lib/jobs/duplicate-job.ts +async function duplicateJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/duplicate`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/get-job.ts +async function getJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/get-scheduled-job.ts +async function getScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/list-job-hardware.ts +async function listJobHardware(params) { + const accessToken = checkCredentials(params ?? {}); + const headers = {}; + if (accessToken) { + headers.Authorization = `Bearer ${accessToken}`; + } + const response = await (params?.fetch || fetch)(`${params?.hubUrl || HUB_URL}/api/jobs/hardware`, { + headers + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/list-jobs.ts +async function listJobs(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/list-scheduled-jobs.ts +async function listScheduledJobs(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}`, { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/resume-scheduled-job.ts +async function resumeScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/resume`, + { + method: "POST", + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/lib/jobs/run-job.ts +async function runJob(params) { + const accessToken = checkCredentials(params); + if (!params.dockerImage && !params.spaceId) { + throw new Error("Either dockerImage or spaceId must be provided"); + } + if (params.dockerImage && params.spaceId) { + throw new Error("Cannot provide both dockerImage and spaceId"); + } + const body = { + flavor: params.flavor, + environment: params.environment || {} + }; + if (params.dockerImage) { + body.dockerImage = params.dockerImage; + } + if (params.spaceId) { + body.spaceId = params.spaceId; + } + if (params.command) { + body.command = params.command; + } + if (params.arguments) { + body.arguments = params.arguments; + } + if (params.secrets) { + body.secrets = params.secrets; + } + if (params.arch) { + body.arch = params.arch; + } + if (params.timeoutSeconds !== void 0) { + body.timeoutSeconds = params.timeoutSeconds; + } + if (params.attempts !== void 0) { + body.attempts = params.attempts; + } + if (params.labels) { + body.labels = params.labels; + } + if (params.volumes?.length) { + body.volumes = params.volumes.map(({ source, ...rest }) => { + const repoId = toRepoId(source); + return { type: repoId.type, source: repoId.name, ...rest }; + }); + } + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }, + body: JSON.stringify(body) + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/run-scheduled-job.ts +async function runScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/run`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + if (response.status === 409) { + return null; + } + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/stream-job-events.ts +async function* streamJobEvents(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/events`, + { + headers: { + Accept: "text/event-stream", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + if (!response.body) { + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + } finally { + reader.releaseLock(); + } +} + +// src/lib/jobs/stream-job-logs.ts +async function* streamJobLogs(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/logs`, + { + headers: { + Accept: "text/event-stream", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + if (!response.body) { + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const data = JSON.parse(line.slice(6)); + yield { message: data.data, timestamp: new Date(data.timestamp) }; + } catch { + yield { message: line.slice(6), timestamp: /* @__PURE__ */ new Date() }; + } + } + } + } + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const data = JSON.parse(line.slice(6)); + yield { message: data.data, timestamp: new Date(data.timestamp) }; + } catch { + yield { message: line.slice(6), timestamp: /* @__PURE__ */ new Date() }; + } + } + } + } + } finally { + reader.releaseLock(); + } +} + +// src/lib/jobs/stream-job-metrics.ts +async function* streamJobMetrics(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/metrics`, + { + headers: { + Accept: "text/event-stream", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + if (!response.body) { + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + } finally { + reader.releaseLock(); + } +} + +// src/lib/jobs/suspend-scheduled-job.ts +async function suspendScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/suspend`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/lib/list-commits.ts +async function* listCommits(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + let url = `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commits/${params.revision ?? "main"}?limit=${params.batchSize ?? 100}`; + while (url) { + const res = await (params.fetch ?? fetch)(url, { + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }); + if (!res.ok) { + throw await createApiError(res); + } + const resJson = await res.json(); + for (const commit2 of resJson) { + yield { + oid: commit2.id, + title: commit2.title, + message: commit2.message, + authors: commit2.authors.map((author) => ({ + username: author.user, + avatarUrl: author.avatar + })), + date: new Date(commit2.date) + }; + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/utils/normalizeInferenceProviderMapping.ts +function normalizeInferenceProviderMapping(hfModelId, inferenceProviderMapping) { + if (!inferenceProviderMapping) { + return []; + } + if (Array.isArray(inferenceProviderMapping)) { + return inferenceProviderMapping.map((entry) => ({ + ...entry, + hfModelId + })); + } + return Object.entries(inferenceProviderMapping).map(([provider, mapping]) => ({ + provider, + hfModelId, + providerId: mapping.providerId, + status: mapping.status, + task: mapping.task + })); +} + +// src/lib/list-models.ts +var MODEL_EXPAND_KEYS = [ + "pipeline_tag", + "private", + "gated", + "downloads", + "likes", + "lastModified" +]; +var MODEL_EXPANDABLE_KEYS = [ + "author", + "cardData", + "config", + "createdAt", + "disabled", + "downloads", + "downloadsAllTime", + "gated", + "gitalyUid", + "inferenceProviderMapping", + "lastModified", + "library_name", + "likes", + "model-index", + "pipeline_tag", + "private", + "safetensors", + "sha", + "spaces", + "tags", + "transformersInfo" +]; +var MODEL_DERIVED_FIELD_TO_API_KEY = { + filePaths: "siblings" +}; +async function* listModels(params) { + const accessToken = params && checkCredentials(params); + let totalToFetch = params?.limit ?? Infinity; + const additionalExpandKeys = params?.additionalFields?.map( + (field) => MODEL_DERIVED_FIELD_TO_API_KEY[field] ?? field + ) ?? []; + const search = new URLSearchParams([ + ...Object.entries({ + limit: String(Math.min(totalToFetch, 500)), + ...params?.search?.owner ? { author: params.search.owner } : void 0, + ...params?.search?.task ? { pipeline_tag: params.search.task } : void 0, + ...params?.search?.query ? { search: params.search.query } : void 0, + ...params?.search?.inferenceProviders ? { inference_provider: params.search.inferenceProviders.join(",") } : void 0, + ...params?.search?.apps ? { apps: params.search.apps.join(",") } : void 0, + ...params?.sort ? { sort: params.sort } : void 0 + }), + ...params?.search?.tags?.map((tag) => ["filter", tag]) ?? [], + ...MODEL_EXPAND_KEYS.map((val) => ["expand", val]), + ...additionalExpandKeys.map((val) => ["expand", val]) + ]).toString(); + let url = `${params?.hubUrl || HUB_URL}/api/models?${search}`; + while (url) { + const res = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + const additional = {}; + if (params?.additionalFields) { + for (const field of params.additionalFields) { + if (field === "filePaths") { + additional.filePaths = (item.siblings ?? []).map((s) => s.rfilename); + } else if (field === "inferenceProviderMapping" && item.inferenceProviderMapping) { + additional.inferenceProviderMapping = normalizeInferenceProviderMapping( + item.id, + item.inferenceProviderMapping + ); + } else { + additional[field] = item[field]; + } + } + } + yield { + ...additional, + id: item._id, + name: item.id, + private: item.private, + task: item.pipeline_tag, + downloads: item.downloads, + gated: item.gated, + likes: item.likes, + updatedAt: new Date(item.lastModified) + }; + totalToFetch--; + if (totalToFetch <= 0) { + return; + } + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/list-spaces.ts +var SPACE_EXPAND_KEYS = [ + "sdk", + "likes", + "private", + "lastModified" +]; +var SPACE_EXPANDABLE_KEYS = [ + "author", + "cardData", + "datasets", + "disabled", + "gitalyUid", + "lastModified", + "createdAt", + "likes", + "private", + "runtime", + "sdk", + // "siblings", + "sha", + "subdomain", + "tags", + "models" +]; +async function* listSpaces(params) { + const accessToken = params && checkCredentials(params); + const search = new URLSearchParams([ + ...Object.entries({ + limit: "500", + ...params?.search?.owner ? { author: params.search.owner } : void 0, + ...params?.search?.query ? { search: params.search.query } : void 0, + ...params?.sort ? { sort: params.sort } : void 0 + }), + ...params?.search?.tags?.map((tag) => ["filter", tag]) ?? [], + ...[...SPACE_EXPAND_KEYS, ...params?.additionalFields ?? []].map( + (val) => ["expand", val] + ) + ]).toString(); + let url = `${params?.hubUrl || HUB_URL}/api/spaces?${search}`; + while (url) { + const res = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + yield { + ...params?.additionalFields && pick(item, params.additionalFields), + id: item._id, + name: item.id, + sdk: item.sdk, + likes: item.likes, + private: item.private, + updatedAt: new Date(item.lastModified) + }; + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/list-collections.ts +async function* listCollections(params) { + const accessToken = params && checkCredentials(params); + const searchParams = new URLSearchParams(); + let totalToFetch = params?.limit ?? Infinity; + searchParams.append("limit", String(Math.min(totalToFetch, 100))); + if (params?.sort) { + searchParams.append("sort", params.sort); + } + if (params?.search?.owner) { + for (const owner of params.search.owner) { + searchParams.append("owner", owner); + } + } + if (params?.search?.item) { + for (const item of params.search.item) { + searchParams.append("item", item); + } + } + if (params?.search?.q) { + searchParams.append("q", params.search.q); + } + let url = `${params?.hubUrl || HUB_URL}/api/collections?${searchParams}`; + while (url) { + const res = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const collections = await res.json(); + for (const collection of collections) { + yield collection; + totalToFetch--; + if (totalToFetch <= 0) { + return; + } + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/model-info.ts +async function modelInfo(params) { + const accessToken = params && checkCredentials(params); + const additionalExpandKeys = params?.additionalFields?.map( + (field) => MODEL_DERIVED_FIELD_TO_API_KEY[field] ?? field + ) ?? []; + const search = new URLSearchParams([ + ...MODEL_EXPAND_KEYS.map((val) => ["expand", val]), + ...additionalExpandKeys.map((val) => ["expand", val]) + ]).toString(); + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/models/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD" + )}?${search.toString()}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + const data = await response.json(); + const additional = {}; + if (params?.additionalFields) { + for (const field of params.additionalFields) { + if (field === "filePaths") { + additional.filePaths = (data.siblings ?? []).map((s) => s.rfilename); + } else if (field === "inferenceProviderMapping" && data.inferenceProviderMapping) { + additional.inferenceProviderMapping = normalizeInferenceProviderMapping(data.id, data.inferenceProviderMapping); + } else { + additional[field] = data[field]; + } + } + } + return { + ...additional, + id: data._id, + name: data.id, + private: data.private, + task: data.pipeline_tag, + downloads: data.downloads, + gated: data.gated, + likes: data.likes, + updatedAt: new Date(data.lastModified) + }; +} + +// src/lib/oauth-handle-redirect.ts +async function oauthHandleRedirect(opts) { + if (typeof window === "undefined" && !opts?.redirectedUrl) { + throw new Error("oauthHandleRedirect is only available in the browser, unless you provide redirectedUrl"); + } + if (typeof localStorage === "undefined" && (!opts?.nonce || !opts?.codeVerifier)) { + throw new Error( + "oauthHandleRedirect requires localStorage to be available, unless you provide nonce and codeVerifier" + ); + } + const redirectedUrl = opts?.redirectedUrl ?? window.location.href; + const searchParams = (() => { + try { + return new URL(redirectedUrl).searchParams; + } catch (err) { + throw new Error("Failed to parse redirected URL: " + redirectedUrl); + } + })(); + const [error, errorDescription] = [searchParams.get("error"), searchParams.get("error_description")]; + if (error) { + throw new Error(`${error}: ${errorDescription}`); + } + const code = searchParams.get("code"); + const nonce = opts?.nonce ?? localStorage.getItem("huggingface.co:oauth:nonce"); + if (!code) { + throw new Error("Missing oauth code from query parameters in redirected URL: " + redirectedUrl); + } + if (!nonce) { + throw new Error("Missing oauth nonce from localStorage"); + } + const codeVerifier = opts?.codeVerifier ?? localStorage.getItem("huggingface.co:oauth:code_verifier"); + if (!codeVerifier) { + throw new Error("Missing oauth code_verifier from localStorage"); + } + const state = searchParams.get("state"); + if (!state) { + throw new Error("Missing oauth state from query parameters in redirected URL"); + } + let parsedState; + try { + parsedState = JSON.parse(state); + } catch { + throw new Error("Invalid oauth state in redirected URL, unable to parse JSON: " + state); + } + if (parsedState.nonce !== nonce) { + throw new Error("Invalid oauth state in redirected URL"); + } + const hubUrl = opts?.hubUrl || HUB_URL; + const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; + const openidConfigRes = await fetch(openidConfigUrl, { + headers: { + Accept: "application/json" + } + }); + if (!openidConfigRes.ok) { + throw await createApiError(openidConfigRes); + } + const openidConfig = await openidConfigRes.json(); + const tokenRes = await fetch(openidConfig.token_endpoint, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded" + }, + body: new URLSearchParams({ + grant_type: "authorization_code", + code, + redirect_uri: parsedState.redirectUri, + code_verifier: codeVerifier + }).toString() + }); + if (!opts?.codeVerifier) { + localStorage.removeItem("huggingface.co:oauth:code_verifier"); + } + if (!opts?.nonce) { + localStorage.removeItem("huggingface.co:oauth:nonce"); + } + if (!tokenRes.ok) { + throw await createApiError(tokenRes); + } + const token = await tokenRes.json(); + const accessTokenExpiresAt = new Date(Date.now() + token.expires_in * 1e3); + const userInfoRes = await fetch(openidConfig.userinfo_endpoint, { + headers: { + Authorization: `Bearer ${token.access_token}` + } + }); + if (!userInfoRes.ok) { + throw await createApiError(userInfoRes); + } + const userInfo = await userInfoRes.json(); + return { + accessToken: token.access_token, + accessTokenExpiresAt, + userInfo, + state: parsedState.state, + scope: token.scope + }; +} +async function oauthHandleRedirectIfPresent(opts) { + if (typeof window === "undefined" && !opts?.redirectedUrl) { + throw new Error("oauthHandleRedirect is only available in the browser, unless you provide redirectedUrl"); + } + if (typeof localStorage === "undefined" && (!opts?.nonce || !opts?.codeVerifier)) { + throw new Error( + "oauthHandleRedirect requires localStorage to be available, unless you provide nonce and codeVerifier" + ); + } + const searchParams = new URLSearchParams(opts?.redirectedUrl ?? window.location.search); + if (searchParams.has("error")) { + return oauthHandleRedirect(opts); + } + if (searchParams.has("code")) { + if (!localStorage.getItem("huggingface.co:oauth:nonce")) { + console.warn( + "Missing oauth nonce from localStorage. This can happen when the user refreshes the page after logging in, without changing the URL." + ); + return false; + } + return oauthHandleRedirect(opts); + } + return false; +} + +// src/lib/oauth-login-url.ts +async function oauthLoginUrl(opts) { + if (typeof window === "undefined" && (!opts?.redirectUrl || !opts?.clientId)) { + throw new Error("oauthLogin is only available in the browser, unless you provide clientId and redirectUrl"); + } + if (typeof localStorage === "undefined" && !opts?.localStorage) { + throw new Error( + "oauthLogin requires localStorage to be available in the context, unless you provide a localStorage empty object as argument" + ); + } + const hubUrl = opts?.hubUrl || HUB_URL; + const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; + const openidConfigRes = await fetch(openidConfigUrl, { + headers: { + Accept: "application/json" + } + }); + if (!openidConfigRes.ok) { + throw await createApiError(openidConfigRes); + } + const opendidConfig = await openidConfigRes.json(); + const newNonce = globalThis.crypto.randomUUID(); + const newCodeVerifier = globalThis.crypto.randomUUID() + globalThis.crypto.randomUUID(); + if (opts?.localStorage) { + if (opts.localStorage.codeVerifier !== void 0 && opts.localStorage.codeVerifier !== null) { + throw new Error( + "localStorage.codeVerifier must be initially set to null or undefined, and will be filled by oauthLoginUrl" + ); + } + if (opts.localStorage.nonce !== void 0 && opts.localStorage.nonce !== null) { + throw new Error( + "localStorage.nonce must be initially set to null or undefined, and will be filled by oauthLoginUrl" + ); + } + opts.localStorage.codeVerifier = newCodeVerifier; + opts.localStorage.nonce = newNonce; + } else { + localStorage.setItem("huggingface.co:oauth:nonce", newNonce); + localStorage.setItem("huggingface.co:oauth:code_verifier", newCodeVerifier); + } + const redirectUri = opts?.redirectUrl || (typeof window !== "undefined" ? window.location.href : void 0); + if (!redirectUri) { + throw new Error("Missing redirectUrl"); + } + const state = JSON.stringify({ + nonce: newNonce, + redirectUri, + state: opts?.state + }); + const variables = ( + // @ts-expect-error window.huggingface is defined inside static Spaces. + typeof window !== "undefined" ? window.huggingface?.variables ?? null : null + ); + const clientId = opts?.clientId || variables?.OAUTH_CLIENT_ID; + if (!clientId) { + if (variables) { + throw new Error("Missing clientId, please add hf_oauth: true to the README.md's metadata in your static Space"); + } + throw new Error("Missing clientId"); + } + const challenge = base64FromBytes( + new Uint8Array(await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(newCodeVerifier))) + ).replace(/[+]/g, "-").replace(/[/]/g, "_").replace(/=/g, ""); + return `${opendidConfig.authorization_endpoint}?${new URLSearchParams({ + client_id: clientId, + scope: opts?.scopes || variables?.OAUTH_SCOPES || "openid profile", + response_type: "code", + redirect_uri: redirectUri, + state, + code_challenge: challenge, + code_challenge_method: "S256" + }).toString()}`; +} + +// src/utils/typedEntries.ts +function typedEntries(obj) { + return Object.entries(obj); +} + +// src/utils/typedInclude.ts +function typedInclude(arr, v) { + return arr.includes(v); +} + +// src/utils/omit.ts +function omit(o, props) { + const propsArr = Array.isArray(props) ? props : [props]; + const letsKeep = Object.keys(o).filter((prop) => !typedInclude(propsArr, prop)); + return pick(o, letsKeep); +} + +// src/lib/parse-safetensors-metadata.ts +var SAFETENSORS_FILE = "model.safetensors"; +var SAFETENSORS_INDEX_FILE = "model.safetensors.index.json"; +var RE_SAFETENSORS_FILE = /\.safetensors$/; +var RE_SAFETENSORS_INDEX_FILE = /\.safetensors\.index\.json$/; +var RE_SAFETENSORS_SHARD_FILE = /^(?(?.*?)[_-])(?\d{5,6})-of-(?\d{5,6})\.safetensors$/; +function parseSafetensorsShardFilename(filename) { + const match = RE_SAFETENSORS_SHARD_FILE.exec(filename); + if (match && match.groups) { + return { + prefix: match.groups["prefix"], + basePrefix: match.groups["basePrefix"], + shard: match.groups["shard"], + total: match.groups["total"] + }; + } + return null; +} +var PARALLEL_DOWNLOADS = 20; +var MAX_HEADER_LENGTH = 25e6; +var MAX_CONFIG_LENGTH = 1e7; +var MAX_SHARD_COUNT = 1e4; +var GPTQ_QWEIGHT_SUFFIX = "qweight"; +var GPTQ_AWQ_AUXILIARY_SUFFIXES = ["qzeros", "g_idx", "scales"]; +var SafetensorParseError = class extends Error { +}; +async function fetchModelConfig(params) { + try { + const configBlob = await downloadFile({ + ...params, + path: "config.json" + }); + if (!configBlob) { + return null; + } + const config = JSON.parse(await configBlob.slice(0, MAX_CONFIG_LENGTH).text()); + return config; + } catch (error) { + return null; + } +} +async function parseSingleFile(path2, params) { + const blob = await downloadFile({ ...params, path: path2 }); + if (!blob) { + throw new SafetensorParseError(`Failed to parse file ${path2}: failed to fetch safetensors header length.`); + } + const bufLengthOfHeaderLE = await blob.slice(0, 8).arrayBuffer(); + const lengthOfHeader = new DataView(bufLengthOfHeaderLE).getBigUint64(0, true); + if (lengthOfHeader <= 0) { + throw new SafetensorParseError(`Failed to parse file ${path2}: safetensors header is malformed.`); + } + if (lengthOfHeader > MAX_HEADER_LENGTH) { + throw new SafetensorParseError( + `Failed to parse file ${path2}: safetensor header is too big. Maximum supported size is ${MAX_HEADER_LENGTH} bytes.` + ); + } + try { + const header = JSON.parse(await blob.slice(8, 8 + Number(lengthOfHeader)).text()); + return header; + } catch (err) { + throw new SafetensorParseError(`Failed to parse file ${path2}: safetensors header is not valid JSON.`); + } +} +async function parseShardedIndex(path2, params) { + const indexBlob = await downloadFile({ + ...params, + path: path2 + }); + if (!indexBlob) { + throw new SafetensorParseError(`Failed to parse file ${path2}: failed to fetch safetensors index.`); + } + try { + const index = JSON.parse(await indexBlob.slice(0, MAX_HEADER_LENGTH).text()); + return index; + } catch (error) { + throw new SafetensorParseError(`Failed to parse file ${path2}: not a valid JSON.`); + } +} +async function fetchAllHeaders(path2, index, params) { + const pathPrefix = path2.slice(0, path2.lastIndexOf("/") + 1); + const filenames = [...new Set(Object.values(index.weight_map))]; + if (filenames.length > MAX_SHARD_COUNT) { + throw new SafetensorParseError( + `Too many shard files (${filenames.length}). Maximum supported is ${MAX_SHARD_COUNT}.` + ); + } + for (const filename of filenames) { + if (filename.includes("..") || filename.startsWith("/") || filename.includes("://")) { + throw new SafetensorParseError(`Unsafe shard filename in weight_map: "${filename}"`); + } + } + const shardedMap = Object.fromEntries( + await promisesQueue( + filenames.map( + (filename) => async () => [filename, await parseSingleFile(pathPrefix + filename, params)] + ), + PARALLEL_DOWNLOADS + ) + ); + return shardedMap; +} +function parseTotalParameters(value) { + if (!value) { + return void 0; + } + if (typeof value === "number") { + return value; + } + return parseInt(value); +} +async function parseSafetensorsMetadata(params) { + const repoId = toRepoId(params.repo); + if (repoId.type !== "model") { + throw new TypeError("Only model repos should contain safetensors files."); + } + const modelConfig = params.computeParametersCount ? await fetchModelConfig(params) : null; + const quantConfig = modelConfig?.quantization_config ?? modelConfig?.text_config?.quantization_config; + if (params.path && RE_SAFETENSORS_FILE.test(params.path) || await fileExists({ ...params, path: SAFETENSORS_FILE })) { + const header = await parseSingleFile(params.path ?? SAFETENSORS_FILE, params); + const paramStats = params.computeParametersCount ? { + parameterCount: computeNumOfParamsByDtypeSingleFile(header, quantConfig), + /// shortcut: get param count directly from metadata + parameterTotal: parseTotalParameters(header.__metadata__?.total_parameters) + } : void 0; + return { + sharded: false, + header, + ...paramStats, + filepaths: [params.path ?? SAFETENSORS_FILE] + }; + } else if (params.path && RE_SAFETENSORS_INDEX_FILE.test(params.path) || await fileExists({ ...params, path: SAFETENSORS_INDEX_FILE })) { + const path2 = params.path ?? SAFETENSORS_INDEX_FILE; + const index = await parseShardedIndex(path2, params); + const shardedMap = await fetchAllHeaders(path2, index, params); + const pathPrefix = path2.slice(0, path2.lastIndexOf("/") + 1); + const paramStats = params.computeParametersCount ? { + parameterCount: computeNumOfParamsByDtypeSharded(shardedMap, quantConfig), + /// shortcut: get param count directly from metadata + parameterTotal: parseTotalParameters(index.metadata?.total_parameters) + } : void 0; + return { + sharded: true, + index, + headers: shardedMap, + ...paramStats, + filepaths: [path2, ...Object.keys(shardedMap).map((filename) => pathPrefix + filename)] + }; + } else { + throw new Error("model id does not seem to contain safetensors weights"); + } +} +function globMatch(pattern, str) { + const parts = pattern.split("*"); + if (parts.length === 1) { + return pattern === str; + } + if (!str.startsWith(parts[0])) { + return false; + } + let pos = parts[0].length; + const lastPart = parts[parts.length - 1]; + if (!str.endsWith(lastPart)) { + return false; + } + const end = str.length - lastPart.length; + for (let i = 1; i < parts.length - 1; i++) { + const idx = str.indexOf(parts[i], pos); + if (idx === -1 || idx + parts[i].length > end) { + return false; + } + pos = idx + parts[i].length; + } + return pos <= end; +} +function isQuantizedTensor(tensorName, quantConfig) { + if (!quantConfig) { + return false; + } + const patterns = quantConfig.modules_to_not_convert; + if (!patterns?.length) { + return true; + } + return !patterns.some( + (pattern) => pattern.includes("*") ? globMatch(pattern, tensorName) : tensorName.includes(pattern) + ); +} +function matchesCompressedTensorsTarget(target, moduleName) { + if (!target.startsWith("re:")) { + return target === moduleName; + } + let pattern = target.slice(3); + if (pattern.startsWith("^")) { + pattern = pattern.slice(1); + } + if (pattern.endsWith("$")) { + pattern = pattern.slice(0, -1); + } else { + pattern += ".*"; + } + const glob = pattern.replaceAll(".*", "*").replaceAll("\\.", "."); + if (/[\\+?()[\]{}|^$]/.test(glob)) { + return false; + } + return globMatch(glob, moduleName); +} +function getQuantizationMultiplier(tensorName, dtype, quantConfig) { + if (!quantConfig || !isQuantizedTensor(tensorName, quantConfig)) { + return 1; + } + const quantMethod = quantConfig.quant_method?.toLowerCase(); + switch (quantMethod) { + case "mxfp4": + if (dtype === "U8" && tensorName.includes("_blocks")) { + return 2; + } + return 1; + case "gptq": + case "awq": + if (getTensorSuffix(tensorName) === GPTQ_QWEIGHT_SUFFIX) { + const bits = quantConfig.bits && quantConfig.bits > 0 ? quantConfig.bits : 4; + return Math.max(1, Math.floor(32 / bits)); + } + if (quantConfig.bits === 4 && dtype === "U8") { + return 2; + } + if (quantConfig.bits === 2 && dtype === "U8") { + return 4; + } + return 1; + case "compressed-tensors": + if (dtype === "I32") { + const groups = Object.values(quantConfig.config_groups ?? {}); + const suffixIndex = tensorName.lastIndexOf(".weight"); + const moduleName = suffixIndex === -1 ? tensorName : tensorName.slice(0, suffixIndex); + const group = groups.find( + (g) => g.targets?.some((target) => matchesCompressedTensorsTarget(target, moduleName)) + ); + if (group) { + if ((group.format ?? quantConfig.format) !== "pack-quantized") { + return 1; + } + const numBits = group.weights?.num_bits ?? 4; + return Math.max(1, Math.floor(32 / numBits)); + } + if (quantConfig.format === "pack-quantized") { + const numBits = groups.find((g) => g.weights?.num_bits)?.weights?.num_bits ?? 4; + return Math.max(1, Math.floor(32 / numBits)); + } + } + return 1; + case "bitsandbytes": + if (quantConfig.load_in_4bit && dtype === "U8") { + return 2; + } + return 1; + default: + if (dtype === "U8" && (quantConfig.load_in_4bit || quantConfig.bits === 4)) { + return 2; + } + return 1; + } +} +function computeNumOfParamsByDtypeSingleFile(header, quantConfig) { + const counter = {}; + const tensors = omit(header, "__metadata__"); + for (const [tensorName, v] of typedEntries(tensors)) { + if (shouldSkipTensor(tensorName, quantConfig)) { + continue; + } + if (v.shape.length === 0) { + continue; + } + const elements = v.shape.reduce((a, b) => a * b); + if (!Number.isFinite(elements)) { + continue; + } + const multiplier = quantConfig ? getQuantizationMultiplier(tensorName, v.dtype, quantConfig) : 1; + if (multiplier === 0) { + continue; + } + counter[v.dtype] = (counter[v.dtype] ?? 0) + elements * multiplier; + } + return counter; +} +function computeNumOfParamsByDtypeSharded(shardedMap, quantConfig) { + const counter = {}; + for (const header of Object.values(shardedMap)) { + for (const [k, v] of typedEntries(computeNumOfParamsByDtypeSingleFile(header, quantConfig))) { + counter[k] = (counter[k] ?? 0) + (v ?? 0); + } + } + return counter; +} +function getTensorSuffix(tensorName) { + const lastDotIndex = tensorName.lastIndexOf("."); + return lastDotIndex === -1 ? tensorName : tensorName.slice(lastDotIndex + 1); +} +function shouldSkipTensor(tensorName, quantConfig) { + if (!quantConfig) { + return false; + } + const quantMethod = quantConfig.quant_method?.toLowerCase(); + if (quantMethod !== "gptq" && quantMethod !== "awq") { + return false; + } + if (!isQuantizedTensor(tensorName, quantConfig)) { + return false; + } + const suffix = getTensorSuffix(tensorName); + return suffix !== GPTQ_QWEIGHT_SUFFIX && GPTQ_AWQ_AUXILIARY_SUFFIXES.includes(suffix); +} + +// src/lib/repo-exists.ts +async function repoExists(params) { + const repoId = toRepoId(params.repo); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}?expand[]=likes`, + { + method: "GET", + headers: { + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + } + } + ); + if (res.status === 404 || res.status === 401) { + return false; + } + if (!res.ok) { + throw await createApiError(res); + } + return true; +} + +// src/lib/space-info.ts +async function spaceInfo(params) { + const accessToken = params && checkCredentials(params); + const search = new URLSearchParams([ + ...SPACE_EXPAND_KEYS.map((val) => ["expand", val]), + ...params?.additionalFields?.map((val) => ["expand", val]) ?? [] + ]).toString(); + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/spaces/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD" + )}?${search.toString()}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + const data = await response.json(); + return { + ...params?.additionalFields && pick(data, params.additionalFields), + id: data._id, + name: data.id, + sdk: data.sdk, + likes: data.likes, + private: data.private, + updatedAt: new Date(data.lastModified) + }; +} + +// src/lib/snapshot-download.ts +import { join as join3, dirname as dirname3 } from "path"; +import { mkdir as mkdir2, writeFile } from "fs/promises"; +var DEFAULT_REVISION = "main"; +async function snapshotDownload(params) { + let cacheDir; + if (params.cacheDir) { + cacheDir = params.cacheDir; + } else { + cacheDir = getHFHubCachePath(); + } + let revision; + if (params.revision) { + revision = params.revision; + } else { + revision = DEFAULT_REVISION; + } + const repoId = toRepoId(params.repo); + const storageFolder = join3(cacheDir, getRepoFolderName(repoId)); + let repoInfo; + switch (repoId.type) { + case "space": + repoInfo = await spaceInfo({ + ...params, + name: repoId.name, + additionalFields: ["sha"], + revision + }); + break; + case "dataset": + repoInfo = await datasetInfo({ + ...params, + name: repoId.name, + additionalFields: ["sha"], + revision + }); + break; + case "model": + repoInfo = await modelInfo({ + ...params, + name: repoId.name, + additionalFields: ["sha"], + revision + }); + break; + default: + throw new Error( + `Unsupported repository type: ${repoId.type}. snapshotDownload is not supported for bucket repos.` + ); + } + const commitHash = repoInfo.sha; + if (revision !== commitHash) { + const refPath = join3(storageFolder, "refs", revision); + await mkdir2(dirname3(refPath), { recursive: true }); + await writeFile(refPath, commitHash); + } + const snapshotFolder = join3(storageFolder, "snapshots", commitHash); + const cursor = listFiles({ + ...params, + repo: params.repo, + recursive: true, + revision: commitHash + }); + for await (const entry of cursor) { + switch (entry.type) { + case "file": + await downloadFileToCacheDir({ + ...params, + path: entry.path, + revision: commitHash, + cacheDir + }); + break; + case "directory": + await mkdir2(join3(snapshotFolder, entry.path), { recursive: true }); + break; + default: + throw new Error(`unknown entry type: ${entry.type}`); + } + } + return snapshotFolder; +} + +// src/lib/upload-file.ts +function uploadFile(params) { + const path2 = params.file instanceof URL ? params.file.pathname.split("/").at(-1) ?? "file" : "path" in params.file ? params.file.path : params.file.name; + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: [ + { + operation: "addOrUpdate", + path: path2, + content: "content" in params.file ? params.file.content : params.file + } + ], + title: params.commitTitle ?? `Add ${path2}`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet + }); +} + +// src/lib/upload-files.ts +function uploadFiles(params) { + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: params.files.map((file) => ({ + operation: "addOrUpdate", + path: file instanceof URL ? file.pathname.split("/").at(-1) ?? "file" : "path" in file ? file.path : file.name, + content: "content" in file ? file.content : file + })), + title: params.commitTitle ?? `Add ${params.files.length} files`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet + }); +} + +// src/lib/upload-files-with-progress.ts +var multipartUploadTracking = /* @__PURE__ */ new WeakMap(); +async function* uploadFilesWithProgress(params) { + return yield* commitIter({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: params.files.map((file) => ({ + operation: "addOrUpdate", + path: file instanceof URL ? file.pathname.split("/").at(-1) ?? "file" : "path" in file ? file.path : file.name, + content: "content" in file ? file.content : file + })), + title: params.commitTitle ?? `Add ${params.files.length} files`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet, + fetch: async (input, init) => { + if (!init) { + return fetch(input); + } + if (!typedInclude(["PUT", "POST"], init.method) || !("progressHint" in init) || !init.progressHint || typeof XMLHttpRequest === "undefined" || typeof input !== "string" || !(init.body instanceof ArrayBuffer) && !(init.body instanceof Blob) && !(init.body instanceof File) && typeof init.body !== "string") { + return fetch(input, init); + } + const progressHint = init.progressHint; + const progressCallback = progressHint.progressCallback; + const xhr = new XMLHttpRequest(); + xhr.upload.addEventListener("progress", (event) => { + if (event.lengthComputable) { + if (progressHint.part !== void 0) { + let tracking = multipartUploadTracking.get(progressCallback); + if (!tracking) { + tracking = { numParts: progressHint.numParts, partsProgress: {} }; + multipartUploadTracking.set(progressCallback, tracking); + } + tracking.partsProgress[progressHint.part] = event.loaded / event.total; + let totalProgress = 0; + for (const partProgress of Object.values(tracking.partsProgress)) { + totalProgress += partProgress; + } + if (totalProgress === tracking.numParts) { + progressCallback(0.9999999999); + } else { + progressCallback(totalProgress / tracking.numParts); + } + } else { + if (event.loaded === event.total) { + progressCallback(0.9999999999); + } else { + progressCallback(event.loaded / event.total); + } + } + } + }); + xhr.open(init.method, input, true); + if (init.headers) { + const headers = new Headers(init.headers); + headers.forEach((value, key) => { + xhr.setRequestHeader(key, value); + }); + } + init.signal?.throwIfAborted(); + xhr.send(init.body); + return new Promise((resolve3, reject) => { + xhr.addEventListener("load", () => { + resolve3( + new Response(xhr.responseText, { + status: xhr.status, + statusText: xhr.statusText, + headers: Object.fromEntries( + xhr.getAllResponseHeaders().trim().split("\n").map((header) => [ + header.slice(0, header.indexOf(":")), + header.slice(header.indexOf(":") + 1).trim() + ]) + ) + }) + ); + }); + xhr.addEventListener("error", () => { + reject(new Error(xhr.statusText)); + }); + if (init.signal) { + init.signal.addEventListener("abort", () => { + xhr.abort(); + try { + init.signal?.throwIfAborted(); + } catch (err) { + reject(err); + } + }); + } + }); + } + }); +} + +// src/lib/who-am-i.ts +async function whoAmI(params) { + const accessToken = checkCredentials(params); + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/whoami-v2`, { + headers: { + Authorization: `Bearer ${accessToken}` + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const response = await res.json(); + if (typeof response.auth.accessToken?.createdAt === "string") { + response.auth.accessToken.createdAt = new Date(response.auth.accessToken.createdAt); + } + return response; +} + +export { + getHFHubCachePath, + REPO_ID_SEPARATOR, + getRepoFolderName, + scanCacheDir, + scanCachedRepo, + scanRefsDir, + scanSnapshotDir, + getBlobStat, + parseRepoType, + HUB_URL, + HubApiError, + InvalidApiResponseFormatError, + checkRepoAccess, + sha256, + XetBlob, + commitIter, + commitIterBucket, + commit, + fileDownloadInfo, + downloadFile, + listFiles, + pathsInfo, + copyFile, + copyFileIter, + copyFiles, + copyFilesIter, + copyFolder, + copyFolderIter, + relativeUnderFolder, + countCommits, + createRepo, + createBranch, + createCollection, + DATASET_EXPAND_KEYS, + DATASET_EXPANDABLE_KEYS, + listDatasets, + datasetInfo, + deleteBranch, + deleteFile, + deleteFiles, + deleteRepo, + deleteCollection, + REGEX_COMMIT_HASH, + downloadFileToCacheDir, + fileExists, + cancelJob, + createScheduledJob, + deleteScheduledJob, + duplicateJob, + getJob, + getScheduledJob, + listJobHardware, + listJobs, + listScheduledJobs, + resumeScheduledJob, + runJob, + runScheduledJob, + streamJobEvents, + streamJobLogs, + streamJobMetrics, + suspendScheduledJob, + listCommits, + MODEL_EXPAND_KEYS, + MODEL_EXPANDABLE_KEYS, + MODEL_DERIVED_FIELD_TO_API_KEY, + listModels, + SPACE_EXPAND_KEYS, + SPACE_EXPANDABLE_KEYS, + listSpaces, + listCollections, + modelInfo, + oauthHandleRedirect, + oauthHandleRedirectIfPresent, + oauthLoginUrl, + typedEntries, + SAFETENSORS_FILE, + SAFETENSORS_INDEX_FILE, + RE_SAFETENSORS_FILE, + RE_SAFETENSORS_INDEX_FILE, + RE_SAFETENSORS_SHARD_FILE, + parseSafetensorsShardFilename, + parseSafetensorsMetadata, + globMatch, + isQuantizedTensor, + matchesCompressedTensorsTarget, + repoExists, + spaceInfo, + DEFAULT_REVISION, + snapshotDownload, + uploadFile, + uploadFiles, + uploadFilesWithProgress, + whoAmI +}; diff --git a/node_modules/@huggingface/hub/dist/cli-progress-XP5T6RZP.mjs b/node_modules/@huggingface/hub/dist/cli-progress-XP5T6RZP.mjs new file mode 100644 index 0000000000000000000000000000000000000000..0aaa0fb32e3376ba7bf64f3cfe1c7cc7b94a588d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/cli-progress-XP5T6RZP.mjs @@ -0,0 +1,873 @@ +import { + __commonJS, + __require +} from "./chunk-FFYIGW52.mjs"; + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/eta.js +var require_eta = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/eta.js"(exports, module) { + var ETA = class { + constructor(length, initTime, initValue) { + this.etaBufferLength = length || 100; + this.valueBuffer = [initValue]; + this.timeBuffer = [initTime]; + this.eta = "0"; + } + // add new values to calculation buffer + update(time, value, total) { + this.valueBuffer.push(value); + this.timeBuffer.push(time); + this.calculate(total - value); + } + // fetch estimated time + getTime() { + return this.eta; + } + // eta calculation - request number of remaining events + calculate(remaining) { + const currentBufferSize = this.valueBuffer.length; + const buffer = Math.min(this.etaBufferLength, currentBufferSize); + const v_diff = this.valueBuffer[currentBufferSize - 1] - this.valueBuffer[currentBufferSize - buffer]; + const t_diff = this.timeBuffer[currentBufferSize - 1] - this.timeBuffer[currentBufferSize - buffer]; + const vt_rate = v_diff / t_diff; + this.valueBuffer = this.valueBuffer.slice(-this.etaBufferLength); + this.timeBuffer = this.timeBuffer.slice(-this.etaBufferLength); + const eta = Math.ceil(remaining / vt_rate / 1e3); + if (isNaN(eta)) { + this.eta = "NULL"; + } else if (!isFinite(eta)) { + this.eta = "INF"; + } else if (eta > 1e7) { + this.eta = "INF"; + } else if (eta < 0) { + this.eta = 0; + } else { + this.eta = eta; + } + } + }; + module.exports = ETA; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/terminal.js +var require_terminal = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/terminal.js"(exports, module) { + var _readline = __require("readline"); + var Terminal = class { + constructor(outputStream) { + this.stream = outputStream; + this.linewrap = true; + this.dy = 0; + } + // save cursor position + settings + cursorSave() { + if (!this.stream.isTTY) { + return; + } + this.stream.write("\x1B7"); + } + // restore last cursor position + settings + cursorRestore() { + if (!this.stream.isTTY) { + return; + } + this.stream.write("\x1B8"); + } + // show/hide cursor + cursor(enabled) { + if (!this.stream.isTTY) { + return; + } + if (enabled) { + this.stream.write("\x1B[?25h"); + } else { + this.stream.write("\x1B[?25l"); + } + } + // change cursor positionn + cursorTo(x = null, y = null) { + if (!this.stream.isTTY) { + return; + } + _readline.cursorTo(this.stream, x, y); + } + // change relative cursor position + cursorRelative(dx = null, dy = null) { + if (!this.stream.isTTY) { + return; + } + this.dy = this.dy + dy; + _readline.moveCursor(this.stream, dx, dy); + } + // relative reset + cursorRelativeReset() { + if (!this.stream.isTTY) { + return; + } + _readline.moveCursor(this.stream, 0, -this.dy); + _readline.cursorTo(this.stream, 0, null); + this.dy = 0; + } + // clear to the right from cursor + clearRight() { + if (!this.stream.isTTY) { + return; + } + _readline.clearLine(this.stream, 1); + } + // clear the full line + clearLine() { + if (!this.stream.isTTY) { + return; + } + _readline.clearLine(this.stream, 0); + } + // clear everyting beyond the current line + clearBottom() { + if (!this.stream.isTTY) { + return; + } + _readline.clearScreenDown(this.stream); + } + // add new line; increment counter + newline() { + this.stream.write("\n"); + this.dy++; + } + // write content to output stream + // @TODO use string-width to strip length + write(s, rawWrite = false) { + if (this.linewrap === true && rawWrite === false) { + this.stream.write(s.substr(0, this.getWidth())); + } else { + this.stream.write(s); + } + } + // control line wrapping + lineWrapping(enabled) { + if (!this.stream.isTTY) { + return; + } + this.linewrap = enabled; + if (enabled) { + this.stream.write("\x1B[?7h"); + } else { + this.stream.write("\x1B[?7l"); + } + } + // tty environment ? + isTTY() { + return this.stream.isTTY === true; + } + // get terminal width + getWidth() { + return this.stream.columns || (this.stream.isTTY ? 80 : 200); + } + }; + module.exports = Terminal; + } +}); + +// node_modules/.pnpm/ansi-regex@5.0.1/node_modules/ansi-regex/index.js +var require_ansi_regex = __commonJS({ + "node_modules/.pnpm/ansi-regex@5.0.1/node_modules/ansi-regex/index.js"(exports, module) { + "use strict"; + module.exports = ({ onlyFirst = false } = {}) => { + const pattern = [ + "[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)", + "(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))" + ].join("|"); + return new RegExp(pattern, onlyFirst ? void 0 : "g"); + }; + } +}); + +// node_modules/.pnpm/strip-ansi@6.0.1/node_modules/strip-ansi/index.js +var require_strip_ansi = __commonJS({ + "node_modules/.pnpm/strip-ansi@6.0.1/node_modules/strip-ansi/index.js"(exports, module) { + "use strict"; + var ansiRegex = require_ansi_regex(); + module.exports = (string) => typeof string === "string" ? string.replace(ansiRegex(), "") : string; + } +}); + +// node_modules/.pnpm/is-fullwidth-code-point@3.0.0/node_modules/is-fullwidth-code-point/index.js +var require_is_fullwidth_code_point = __commonJS({ + "node_modules/.pnpm/is-fullwidth-code-point@3.0.0/node_modules/is-fullwidth-code-point/index.js"(exports, module) { + "use strict"; + var isFullwidthCodePoint = (codePoint) => { + if (Number.isNaN(codePoint)) { + return false; + } + if (codePoint >= 4352 && (codePoint <= 4447 || // Hangul Jamo + codePoint === 9001 || // LEFT-POINTING ANGLE BRACKET + codePoint === 9002 || // RIGHT-POINTING ANGLE BRACKET + // CJK Radicals Supplement .. Enclosed CJK Letters and Months + 11904 <= codePoint && codePoint <= 12871 && codePoint !== 12351 || // Enclosed CJK Letters and Months .. CJK Unified Ideographs Extension A + 12880 <= codePoint && codePoint <= 19903 || // CJK Unified Ideographs .. Yi Radicals + 19968 <= codePoint && codePoint <= 42182 || // Hangul Jamo Extended-A + 43360 <= codePoint && codePoint <= 43388 || // Hangul Syllables + 44032 <= codePoint && codePoint <= 55203 || // CJK Compatibility Ideographs + 63744 <= codePoint && codePoint <= 64255 || // Vertical Forms + 65040 <= codePoint && codePoint <= 65049 || // CJK Compatibility Forms .. Small Form Variants + 65072 <= codePoint && codePoint <= 65131 || // Halfwidth and Fullwidth Forms + 65281 <= codePoint && codePoint <= 65376 || 65504 <= codePoint && codePoint <= 65510 || // Kana Supplement + 110592 <= codePoint && codePoint <= 110593 || // Enclosed Ideographic Supplement + 127488 <= codePoint && codePoint <= 127569 || // CJK Unified Ideographs Extension B .. Tertiary Ideographic Plane + 131072 <= codePoint && codePoint <= 262141)) { + return true; + } + return false; + }; + module.exports = isFullwidthCodePoint; + module.exports.default = isFullwidthCodePoint; + } +}); + +// node_modules/.pnpm/emoji-regex@8.0.0/node_modules/emoji-regex/index.js +var require_emoji_regex = __commonJS({ + "node_modules/.pnpm/emoji-regex@8.0.0/node_modules/emoji-regex/index.js"(exports, module) { + "use strict"; + module.exports = function() { + return /\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62(?:\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67|\uDB40\uDC73\uDB40\uDC63\uDB40\uDC74|\uDB40\uDC77\uDB40\uDC6C\uDB40\uDC73)\uDB40\uDC7F|\uD83D\uDC68(?:\uD83C\uDFFC\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68\uD83C\uDFFB|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFF\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFE])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFE\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFD])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFD\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB\uDFFC])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\u200D(?:\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D)?\uD83D\uDC68|(?:\uD83D[\uDC68\uDC69])\u200D(?:\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67]))|\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67])|(?:\uD83D[\uDC68\uDC69])\u200D(?:\uD83D[\uDC66\uDC67])|[\u2695\u2696\u2708]\uFE0F|\uD83D[\uDC66\uDC67]|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|(?:\uD83C\uDFFB\u200D[\u2695\u2696\u2708]|\uD83C\uDFFF\u200D[\u2695\u2696\u2708]|\uD83C\uDFFE\u200D[\u2695\u2696\u2708]|\uD83C\uDFFD\u200D[\u2695\u2696\u2708]|\uD83C\uDFFC\u200D[\u2695\u2696\u2708])\uFE0F|\uD83C\uDFFB\u200D(?:\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C[\uDFFB-\uDFFF])|(?:\uD83E\uDDD1\uD83C\uDFFB\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFC\u200D\uD83E\uDD1D\u200D\uD83D\uDC69)\uD83C\uDFFB|\uD83E\uDDD1(?:\uD83C\uDFFF\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1(?:\uD83C[\uDFFB-\uDFFF])|\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1)|(?:\uD83E\uDDD1\uD83C\uDFFE\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFF\u200D\uD83E\uDD1D\u200D(?:\uD83D[\uDC68\uDC69]))(?:\uD83C[\uDFFB-\uDFFE])|(?:\uD83E\uDDD1\uD83C\uDFFC\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFD\u200D\uD83E\uDD1D\u200D\uD83D\uDC69)(?:\uD83C[\uDFFB\uDFFC])|\uD83D\uDC69(?:\uD83C\uDFFE\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFD\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFC\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB\uDFFD-\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFB\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFC-\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFD\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB\uDFFC\uDFFE\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\u200D(?:\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D(?:\uD83D[\uDC68\uDC69])|\uD83D[\uDC68\uDC69])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFF\u200D(?:\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD]))|\uD83D\uDC69\u200D\uD83D\uDC69\u200D(?:\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67]))|(?:\uD83E\uDDD1\uD83C\uDFFD\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFE\u200D\uD83E\uDD1D\u200D\uD83D\uDC69)(?:\uD83C[\uDFFB-\uDFFD])|\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC69\u200D\uD83D\uDC69\u200D(?:\uD83D[\uDC66\uDC67])|(?:\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8|\uD83D\uDC69(?:\uD83C\uDFFF\u200D[\u2695\u2696\u2708]|\uD83C\uDFFE\u200D[\u2695\u2696\u2708]|\uD83C\uDFFC\u200D[\u2695\u2696\u2708]|\uD83C\uDFFB\u200D[\u2695\u2696\u2708]|\uD83C\uDFFD\u200D[\u2695\u2696\u2708]|\u200D[\u2695\u2696\u2708])|(?:(?:\u26F9|\uD83C[\uDFCB\uDFCC]|\uD83D\uDD75)\uFE0F|\uD83D\uDC6F|\uD83E[\uDD3C\uDDDE\uDDDF])\u200D[\u2640\u2642]|(?:\u26F9|\uD83C[\uDFCB\uDFCC]|\uD83D\uDD75)(?:\uD83C[\uDFFB-\uDFFF])\u200D[\u2640\u2642]|(?:\uD83C[\uDFC3\uDFC4\uDFCA]|\uD83D[\uDC6E\uDC71\uDC73\uDC77\uDC81\uDC82\uDC86\uDC87\uDE45-\uDE47\uDE4B\uDE4D\uDE4E\uDEA3\uDEB4-\uDEB6]|\uD83E[\uDD26\uDD37-\uDD39\uDD3D\uDD3E\uDDB8\uDDB9\uDDCD-\uDDCF\uDDD6-\uDDDD])(?:(?:\uD83C[\uDFFB-\uDFFF])\u200D[\u2640\u2642]|\u200D[\u2640\u2642])|\uD83C\uDFF4\u200D\u2620)\uFE0F|\uD83D\uDC69\u200D\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67])|\uD83C\uDFF3\uFE0F\u200D\uD83C\uDF08|\uD83D\uDC15\u200D\uD83E\uDDBA|\uD83D\uDC69\u200D\uD83D\uDC66|\uD83D\uDC69\u200D\uD83D\uDC67|\uD83C\uDDFD\uD83C\uDDF0|\uD83C\uDDF4\uD83C\uDDF2|\uD83C\uDDF6\uD83C\uDDE6|[#\*0-9]\uFE0F\u20E3|\uD83C\uDDE7(?:\uD83C[\uDDE6\uDDE7\uDDE9-\uDDEF\uDDF1-\uDDF4\uDDF6-\uDDF9\uDDFB\uDDFC\uDDFE\uDDFF])|\uD83C\uDDF9(?:\uD83C[\uDDE6\uDDE8\uDDE9\uDDEB-\uDDED\uDDEF-\uDDF4\uDDF7\uDDF9\uDDFB\uDDFC\uDDFF])|\uD83C\uDDEA(?:\uD83C[\uDDE6\uDDE8\uDDEA\uDDEC\uDDED\uDDF7-\uDDFA])|\uD83E\uDDD1(?:\uD83C[\uDFFB-\uDFFF])|\uD83C\uDDF7(?:\uD83C[\uDDEA\uDDF4\uDDF8\uDDFA\uDDFC])|\uD83D\uDC69(?:\uD83C[\uDFFB-\uDFFF])|\uD83C\uDDF2(?:\uD83C[\uDDE6\uDDE8-\uDDED\uDDF0-\uDDFF])|\uD83C\uDDE6(?:\uD83C[\uDDE8-\uDDEC\uDDEE\uDDF1\uDDF2\uDDF4\uDDF6-\uDDFA\uDDFC\uDDFD\uDDFF])|\uD83C\uDDF0(?:\uD83C[\uDDEA\uDDEC-\uDDEE\uDDF2\uDDF3\uDDF5\uDDF7\uDDFC\uDDFE\uDDFF])|\uD83C\uDDED(?:\uD83C[\uDDF0\uDDF2\uDDF3\uDDF7\uDDF9\uDDFA])|\uD83C\uDDE9(?:\uD83C[\uDDEA\uDDEC\uDDEF\uDDF0\uDDF2\uDDF4\uDDFF])|\uD83C\uDDFE(?:\uD83C[\uDDEA\uDDF9])|\uD83C\uDDEC(?:\uD83C[\uDDE6\uDDE7\uDDE9-\uDDEE\uDDF1-\uDDF3\uDDF5-\uDDFA\uDDFC\uDDFE])|\uD83C\uDDF8(?:\uD83C[\uDDE6-\uDDEA\uDDEC-\uDDF4\uDDF7-\uDDF9\uDDFB\uDDFD-\uDDFF])|\uD83C\uDDEB(?:\uD83C[\uDDEE-\uDDF0\uDDF2\uDDF4\uDDF7])|\uD83C\uDDF5(?:\uD83C[\uDDE6\uDDEA-\uDDED\uDDF0-\uDDF3\uDDF7-\uDDF9\uDDFC\uDDFE])|\uD83C\uDDFB(?:\uD83C[\uDDE6\uDDE8\uDDEA\uDDEC\uDDEE\uDDF3\uDDFA])|\uD83C\uDDF3(?:\uD83C[\uDDE6\uDDE8\uDDEA-\uDDEC\uDDEE\uDDF1\uDDF4\uDDF5\uDDF7\uDDFA\uDDFF])|\uD83C\uDDE8(?:\uD83C[\uDDE6\uDDE8\uDDE9\uDDEB-\uDDEE\uDDF0-\uDDF5\uDDF7\uDDFA-\uDDFF])|\uD83C\uDDF1(?:\uD83C[\uDDE6-\uDDE8\uDDEE\uDDF0\uDDF7-\uDDFB\uDDFE])|\uD83C\uDDFF(?:\uD83C[\uDDE6\uDDF2\uDDFC])|\uD83C\uDDFC(?:\uD83C[\uDDEB\uDDF8])|\uD83C\uDDFA(?:\uD83C[\uDDE6\uDDEC\uDDF2\uDDF3\uDDF8\uDDFE\uDDFF])|\uD83C\uDDEE(?:\uD83C[\uDDE8-\uDDEA\uDDF1-\uDDF4\uDDF6-\uDDF9])|\uD83C\uDDEF(?:\uD83C[\uDDEA\uDDF2\uDDF4\uDDF5])|(?:\uD83C[\uDFC3\uDFC4\uDFCA]|\uD83D[\uDC6E\uDC71\uDC73\uDC77\uDC81\uDC82\uDC86\uDC87\uDE45-\uDE47\uDE4B\uDE4D\uDE4E\uDEA3\uDEB4-\uDEB6]|\uD83E[\uDD26\uDD37-\uDD39\uDD3D\uDD3E\uDDB8\uDDB9\uDDCD-\uDDCF\uDDD6-\uDDDD])(?:\uD83C[\uDFFB-\uDFFF])|(?:\u26F9|\uD83C[\uDFCB\uDFCC]|\uD83D\uDD75)(?:\uD83C[\uDFFB-\uDFFF])|(?:[\u261D\u270A-\u270D]|\uD83C[\uDF85\uDFC2\uDFC7]|\uD83D[\uDC42\uDC43\uDC46-\uDC50\uDC66\uDC67\uDC6B-\uDC6D\uDC70\uDC72\uDC74-\uDC76\uDC78\uDC7C\uDC83\uDC85\uDCAA\uDD74\uDD7A\uDD90\uDD95\uDD96\uDE4C\uDE4F\uDEC0\uDECC]|\uD83E[\uDD0F\uDD18-\uDD1C\uDD1E\uDD1F\uDD30-\uDD36\uDDB5\uDDB6\uDDBB\uDDD2-\uDDD5])(?:\uD83C[\uDFFB-\uDFFF])|(?:[\u231A\u231B\u23E9-\u23EC\u23F0\u23F3\u25FD\u25FE\u2614\u2615\u2648-\u2653\u267F\u2693\u26A1\u26AA\u26AB\u26BD\u26BE\u26C4\u26C5\u26CE\u26D4\u26EA\u26F2\u26F3\u26F5\u26FA\u26FD\u2705\u270A\u270B\u2728\u274C\u274E\u2753-\u2755\u2757\u2795-\u2797\u27B0\u27BF\u2B1B\u2B1C\u2B50\u2B55]|\uD83C[\uDC04\uDCCF\uDD8E\uDD91-\uDD9A\uDDE6-\uDDFF\uDE01\uDE1A\uDE2F\uDE32-\uDE36\uDE38-\uDE3A\uDE50\uDE51\uDF00-\uDF20\uDF2D-\uDF35\uDF37-\uDF7C\uDF7E-\uDF93\uDFA0-\uDFCA\uDFCF-\uDFD3\uDFE0-\uDFF0\uDFF4\uDFF8-\uDFFF]|\uD83D[\uDC00-\uDC3E\uDC40\uDC42-\uDCFC\uDCFF-\uDD3D\uDD4B-\uDD4E\uDD50-\uDD67\uDD7A\uDD95\uDD96\uDDA4\uDDFB-\uDE4F\uDE80-\uDEC5\uDECC\uDED0-\uDED2\uDED5\uDEEB\uDEEC\uDEF4-\uDEFA\uDFE0-\uDFEB]|\uD83E[\uDD0D-\uDD3A\uDD3C-\uDD45\uDD47-\uDD71\uDD73-\uDD76\uDD7A-\uDDA2\uDDA5-\uDDAA\uDDAE-\uDDCA\uDDCD-\uDDFF\uDE70-\uDE73\uDE78-\uDE7A\uDE80-\uDE82\uDE90-\uDE95])|(?:[#\*0-9\xA9\xAE\u203C\u2049\u2122\u2139\u2194-\u2199\u21A9\u21AA\u231A\u231B\u2328\u23CF\u23E9-\u23F3\u23F8-\u23FA\u24C2\u25AA\u25AB\u25B6\u25C0\u25FB-\u25FE\u2600-\u2604\u260E\u2611\u2614\u2615\u2618\u261D\u2620\u2622\u2623\u2626\u262A\u262E\u262F\u2638-\u263A\u2640\u2642\u2648-\u2653\u265F\u2660\u2663\u2665\u2666\u2668\u267B\u267E\u267F\u2692-\u2697\u2699\u269B\u269C\u26A0\u26A1\u26AA\u26AB\u26B0\u26B1\u26BD\u26BE\u26C4\u26C5\u26C8\u26CE\u26CF\u26D1\u26D3\u26D4\u26E9\u26EA\u26F0-\u26F5\u26F7-\u26FA\u26FD\u2702\u2705\u2708-\u270D\u270F\u2712\u2714\u2716\u271D\u2721\u2728\u2733\u2734\u2744\u2747\u274C\u274E\u2753-\u2755\u2757\u2763\u2764\u2795-\u2797\u27A1\u27B0\u27BF\u2934\u2935\u2B05-\u2B07\u2B1B\u2B1C\u2B50\u2B55\u3030\u303D\u3297\u3299]|\uD83C[\uDC04\uDCCF\uDD70\uDD71\uDD7E\uDD7F\uDD8E\uDD91-\uDD9A\uDDE6-\uDDFF\uDE01\uDE02\uDE1A\uDE2F\uDE32-\uDE3A\uDE50\uDE51\uDF00-\uDF21\uDF24-\uDF93\uDF96\uDF97\uDF99-\uDF9B\uDF9E-\uDFF0\uDFF3-\uDFF5\uDFF7-\uDFFF]|\uD83D[\uDC00-\uDCFD\uDCFF-\uDD3D\uDD49-\uDD4E\uDD50-\uDD67\uDD6F\uDD70\uDD73-\uDD7A\uDD87\uDD8A-\uDD8D\uDD90\uDD95\uDD96\uDDA4\uDDA5\uDDA8\uDDB1\uDDB2\uDDBC\uDDC2-\uDDC4\uDDD1-\uDDD3\uDDDC-\uDDDE\uDDE1\uDDE3\uDDE8\uDDEF\uDDF3\uDDFA-\uDE4F\uDE80-\uDEC5\uDECB-\uDED2\uDED5\uDEE0-\uDEE5\uDEE9\uDEEB\uDEEC\uDEF0\uDEF3-\uDEFA\uDFE0-\uDFEB]|\uD83E[\uDD0D-\uDD3A\uDD3C-\uDD45\uDD47-\uDD71\uDD73-\uDD76\uDD7A-\uDDA2\uDDA5-\uDDAA\uDDAE-\uDDCA\uDDCD-\uDDFF\uDE70-\uDE73\uDE78-\uDE7A\uDE80-\uDE82\uDE90-\uDE95])\uFE0F|(?:[\u261D\u26F9\u270A-\u270D]|\uD83C[\uDF85\uDFC2-\uDFC4\uDFC7\uDFCA-\uDFCC]|\uD83D[\uDC42\uDC43\uDC46-\uDC50\uDC66-\uDC78\uDC7C\uDC81-\uDC83\uDC85-\uDC87\uDC8F\uDC91\uDCAA\uDD74\uDD75\uDD7A\uDD90\uDD95\uDD96\uDE45-\uDE47\uDE4B-\uDE4F\uDEA3\uDEB4-\uDEB6\uDEC0\uDECC]|\uD83E[\uDD0F\uDD18-\uDD1F\uDD26\uDD30-\uDD39\uDD3C-\uDD3E\uDDB5\uDDB6\uDDB8\uDDB9\uDDBB\uDDCD-\uDDCF\uDDD1-\uDDDD])/g; + }; + } +}); + +// node_modules/.pnpm/string-width@4.2.3/node_modules/string-width/index.js +var require_string_width = __commonJS({ + "node_modules/.pnpm/string-width@4.2.3/node_modules/string-width/index.js"(exports, module) { + "use strict"; + var stripAnsi = require_strip_ansi(); + var isFullwidthCodePoint = require_is_fullwidth_code_point(); + var emojiRegex = require_emoji_regex(); + var stringWidth = (string) => { + if (typeof string !== "string" || string.length === 0) { + return 0; + } + string = stripAnsi(string); + if (string.length === 0) { + return 0; + } + string = string.replace(emojiRegex(), " "); + let width = 0; + for (let i = 0; i < string.length; i++) { + const code = string.codePointAt(i); + if (code <= 31 || code >= 127 && code <= 159) { + continue; + } + if (code >= 768 && code <= 879) { + continue; + } + if (code > 65535) { + i++; + } + width += isFullwidthCodePoint(code) ? 2 : 1; + } + return width; + }; + module.exports = stringWidth; + module.exports.default = stringWidth; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/format-value.js +var require_format_value = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/format-value.js"(exports, module) { + module.exports = function formatValue(v, options, type) { + if (options.autopadding !== true) { + return v; + } + function autopadding(value, length) { + return (options.autopaddingChar + value).slice(-length); + } + switch (type) { + case "percentage": + return autopadding(v, 3); + default: + return v; + } + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/format-bar.js +var require_format_bar = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/format-bar.js"(exports, module) { + module.exports = function formatBar(progress, options) { + const completeSize = Math.round(progress * options.barsize); + const incompleteSize = options.barsize - completeSize; + return options.barCompleteString.substr(0, completeSize) + options.barGlue + options.barIncompleteString.substr(0, incompleteSize); + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/format-time.js +var require_format_time = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/format-time.js"(exports, module) { + module.exports = function formatTime(t, options, roundToMultipleOf) { + function round(input) { + if (roundToMultipleOf) { + return roundToMultipleOf * Math.round(input / roundToMultipleOf); + } else { + return input; + } + } + function autopadding(v) { + return (options.autopaddingChar + v).slice(-2); + } + if (t > 3600) { + return autopadding(Math.floor(t / 3600)) + "h" + autopadding(round(t % 3600 / 60)) + "m"; + } else if (t > 60) { + return autopadding(Math.floor(t / 60)) + "m" + autopadding(round(t % 60)) + "s"; + } else if (t > 10) { + return autopadding(round(t)) + "s"; + } else { + return autopadding(t) + "s"; + } + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/formatter.js +var require_formatter = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/formatter.js"(exports, module) { + var _stringWidth = require_string_width(); + var _defaultFormatValue = require_format_value(); + var _defaultFormatBar = require_format_bar(); + var _defaultFormatTime = require_format_time(); + module.exports = function defaultFormatter(options, params, payload) { + let s = options.format; + const formatTime = options.formatTime || _defaultFormatTime; + const formatValue = options.formatValue || _defaultFormatValue; + const formatBar = options.formatBar || _defaultFormatBar; + const percentage = Math.floor(params.progress * 100) + ""; + const stopTime = params.stopTime || Date.now(); + const elapsedTime = Math.round((stopTime - params.startTime) / 1e3); + const context = Object.assign({}, payload, { + bar: formatBar(params.progress, options), + percentage: formatValue(percentage, options, "percentage"), + total: formatValue(params.total, options, "total"), + value: formatValue(params.value, options, "value"), + eta: formatValue(params.eta, options, "eta"), + eta_formatted: formatTime(params.eta, options, 5), + duration: formatValue(elapsedTime, options, "duration"), + duration_formatted: formatTime(elapsedTime, options, 1) + }); + s = s.replace(/\{(\w+)\}/g, function(match, key) { + if (typeof context[key] !== "undefined") { + return context[key]; + } + return match; + }); + const fullMargin = Math.max(0, params.maxWidth - _stringWidth(s) - 2); + const halfMargin = Math.floor(fullMargin / 2); + switch (options.align) { + case "right": + s = fullMargin > 0 ? " ".repeat(fullMargin) + s : s; + break; + case "center": + s = halfMargin > 0 ? " ".repeat(halfMargin) + s : s; + break; + case "left": + default: + break; + } + return s; + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/options.js +var require_options = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/options.js"(exports, module) { + function mergeOption(v, defaultValue) { + if (typeof v === "undefined" || v === null) { + return defaultValue; + } else { + return v; + } + } + module.exports = { + // set global options + parse: function parse(rawOptions, preset) { + const options = {}; + const opt = Object.assign({}, preset, rawOptions); + options.throttleTime = 1e3 / mergeOption(opt.fps, 10); + options.stream = mergeOption(opt.stream, process.stderr); + options.terminal = mergeOption(opt.terminal, null); + options.clearOnComplete = mergeOption(opt.clearOnComplete, false); + options.stopOnComplete = mergeOption(opt.stopOnComplete, false); + options.barsize = mergeOption(opt.barsize, 40); + options.align = mergeOption(opt.align, "left"); + options.hideCursor = mergeOption(opt.hideCursor, false); + options.linewrap = mergeOption(opt.linewrap, false); + options.barGlue = mergeOption(opt.barGlue, ""); + options.barCompleteChar = mergeOption(opt.barCompleteChar, "="); + options.barIncompleteChar = mergeOption(opt.barIncompleteChar, "-"); + options.format = mergeOption(opt.format, "progress [{bar}] {percentage}% | ETA: {eta}s | {value}/{total}"); + options.formatTime = mergeOption(opt.formatTime, null); + options.formatValue = mergeOption(opt.formatValue, null); + options.formatBar = mergeOption(opt.formatBar, null); + options.etaBufferLength = mergeOption(opt.etaBuffer, 10); + options.etaAsynchronousUpdate = mergeOption(opt.etaAsynchronousUpdate, false); + options.progressCalculationRelative = mergeOption(opt.progressCalculationRelative, false); + options.synchronousUpdate = mergeOption(opt.synchronousUpdate, true); + options.noTTYOutput = mergeOption(opt.noTTYOutput, false); + options.notTTYSchedule = mergeOption(opt.notTTYSchedule, 2e3); + options.emptyOnZero = mergeOption(opt.emptyOnZero, false); + options.forceRedraw = mergeOption(opt.forceRedraw, false); + options.autopadding = mergeOption(opt.autopadding, false); + options.gracefulExit = mergeOption(opt.gracefulExit, false); + return options; + }, + // derived options: instance specific, has to be created for every bar element + assignDerivedOptions: function assignDerivedOptions(options) { + options.barCompleteString = options.barCompleteChar.repeat(options.barsize + 1); + options.barIncompleteString = options.barIncompleteChar.repeat(options.barsize + 1); + options.autopaddingChar = options.autopadding ? mergeOption(options.autopaddingChar, " ") : ""; + return options; + } + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/generic-bar.js +var require_generic_bar = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/generic-bar.js"(exports, module) { + var _ETA = require_eta(); + var _Terminal = require_terminal(); + var _formatter = require_formatter(); + var _options = require_options(); + var _EventEmitter = __require("events"); + module.exports = class GenericBar extends _EventEmitter { + constructor(options) { + super(); + this.options = _options.assignDerivedOptions(options); + this.terminal = this.options.terminal ? this.options.terminal : new _Terminal(this.options.stream); + this.value = 0; + this.startValue = 0; + this.total = 100; + this.lastDrawnString = null; + this.startTime = null; + this.stopTime = null; + this.lastRedraw = Date.now(); + this.eta = new _ETA(this.options.etaBufferLength, 0, 0); + this.payload = {}; + this.isActive = false; + this.formatter = typeof this.options.format === "function" ? this.options.format : _formatter; + } + // internal render function + render(forceRendering = false) { + const params = { + progress: this.getProgress(), + eta: this.eta.getTime(), + startTime: this.startTime, + stopTime: this.stopTime, + total: this.total, + value: this.value, + maxWidth: this.terminal.getWidth() + }; + if (this.options.etaAsynchronousUpdate) { + this.updateETA(); + } + const s = this.formatter(this.options, params, this.payload); + const forceRedraw = forceRendering || this.options.forceRedraw || this.options.noTTYOutput && !this.terminal.isTTY(); + if (forceRedraw || this.lastDrawnString != s) { + this.emit("redraw-pre"); + this.terminal.cursorTo(0, null); + this.terminal.write(s); + this.terminal.clearRight(); + this.lastDrawnString = s; + this.lastRedraw = Date.now(); + this.emit("redraw-post"); + } + } + // start the progress bar + start(total, startValue, payload) { + this.value = startValue || 0; + this.total = typeof total !== "undefined" && total >= 0 ? total : 100; + this.startValue = startValue || 0; + this.payload = payload || {}; + this.startTime = Date.now(); + this.stopTime = null; + this.lastDrawnString = ""; + this.eta = new _ETA(this.options.etaBufferLength, this.startTime, this.value); + this.isActive = true; + this.emit("start", total, startValue); + } + // stop the bar + stop() { + this.isActive = false; + this.stopTime = Date.now(); + this.emit("stop", this.total, this.value); + } + // update the bar value + // update(value, payload) + // update(payload) + update(arg0, arg1 = {}) { + if (typeof arg0 === "number") { + this.value = arg0; + this.eta.update(Date.now(), arg0, this.total); + } + const payloadData = (typeof arg0 === "object" ? arg0 : arg1) || {}; + this.emit("update", this.total, this.value); + for (const key in payloadData) { + this.payload[key] = payloadData[key]; + } + if (this.value >= this.getTotal() && this.options.stopOnComplete) { + this.stop(); + } + } + // calculate the actual progress value + getProgress() { + let progress = this.value / this.total; + if (this.options.progressCalculationRelative) { + progress = (this.value - this.startValue) / (this.total - this.startValue); + } + if (isNaN(progress)) { + progress = this.options && this.options.emptyOnZero ? 0 : 1; + } + progress = Math.min(Math.max(progress, 0), 1); + return progress; + } + // update the bar value + // increment(delta, payload) + // increment(payload) + increment(arg0 = 1, arg1 = {}) { + if (typeof arg0 === "object") { + this.update(this.value + 1, arg0); + } else { + this.update(this.value + arg0, arg1); + } + } + // get the total (limit) value + getTotal() { + return this.total; + } + // set the total (limit) value + setTotal(total) { + if (typeof total !== "undefined" && total >= 0) { + this.total = total; + } + } + // force eta calculation update (long running processes) + updateETA() { + this.eta.update(Date.now(), this.value, this.total); + } + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/single-bar.js +var require_single_bar = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/single-bar.js"(exports, module) { + var _GenericBar = require_generic_bar(); + var _options = require_options(); + module.exports = class SingleBar extends _GenericBar { + constructor(options, preset) { + super(_options.parse(options, preset)); + this.timer = null; + if (this.options.noTTYOutput && this.terminal.isTTY() === false) { + this.options.synchronousUpdate = false; + } + this.schedulingRate = this.terminal.isTTY() ? this.options.throttleTime : this.options.notTTYSchedule; + this.sigintCallback = null; + } + // internal render function + render() { + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + super.render(); + if (this.options.noTTYOutput && this.terminal.isTTY() === false) { + this.terminal.newline(); + } + this.timer = setTimeout(this.render.bind(this), this.schedulingRate); + } + update(current, payload) { + if (!this.timer) { + return; + } + super.update(current, payload); + if (this.options.synchronousUpdate && this.lastRedraw + this.options.throttleTime * 2 < Date.now()) { + this.render(); + } + } + // start the progress bar + start(total, startValue, payload) { + if (this.options.noTTYOutput === false && this.terminal.isTTY() === false) { + return; + } + if (this.sigintCallback === null && this.options.gracefulExit) { + this.sigintCallback = this.stop.bind(this); + process.once("SIGINT", this.sigintCallback); + process.once("SIGTERM", this.sigintCallback); + } + this.terminal.cursorSave(); + if (this.options.hideCursor === true) { + this.terminal.cursor(false); + } + if (this.options.linewrap === false) { + this.terminal.lineWrapping(false); + } + super.start(total, startValue, payload); + this.render(); + } + // stop the bar + stop() { + if (!this.timer) { + return; + } + if (this.sigintCallback) { + process.removeListener("SIGINT", this.sigintCallback); + process.removeListener("SIGTERM", this.sigintCallback); + this.sigintCallback = null; + } + this.render(); + super.stop(); + clearTimeout(this.timer); + this.timer = null; + if (this.options.hideCursor === true) { + this.terminal.cursor(true); + } + if (this.options.linewrap === false) { + this.terminal.lineWrapping(true); + } + this.terminal.cursorRestore(); + if (this.options.clearOnComplete) { + this.terminal.cursorTo(0, null); + this.terminal.clearLine(); + } else { + this.terminal.newline(); + } + } + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/multi-bar.js +var require_multi_bar = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/multi-bar.js"(exports, module) { + var _Terminal = require_terminal(); + var _BarElement = require_generic_bar(); + var _options = require_options(); + var _EventEmitter = __require("events"); + module.exports = class MultiBar extends _EventEmitter { + constructor(options, preset) { + super(); + this.bars = []; + this.options = _options.parse(options, preset); + this.options.synchronousUpdate = false; + this.terminal = this.options.terminal ? this.options.terminal : new _Terminal(this.options.stream); + this.timer = null; + this.isActive = false; + this.schedulingRate = this.terminal.isTTY() ? this.options.throttleTime : this.options.notTTYSchedule; + this.loggingBuffer = []; + this.sigintCallback = null; + } + // add a new bar to the stack + create(total, startValue, payload, barOptions = {}) { + const bar = new _BarElement(Object.assign( + {}, + // global options + this.options, + // terminal instance + { + terminal: this.terminal + }, + // overrides + barOptions + )); + this.bars.push(bar); + if (this.options.noTTYOutput === false && this.terminal.isTTY() === false) { + return bar; + } + if (this.sigintCallback === null && this.options.gracefulExit) { + this.sigintCallback = this.stop.bind(this); + process.once("SIGINT", this.sigintCallback); + process.once("SIGTERM", this.sigintCallback); + } + if (!this.isActive) { + if (this.options.hideCursor === true) { + this.terminal.cursor(false); + } + if (this.options.linewrap === false) { + this.terminal.lineWrapping(false); + } + this.timer = setTimeout(this.update.bind(this), this.schedulingRate); + } + this.isActive = true; + bar.start(total, startValue, payload); + this.emit("start"); + return bar; + } + // remove a bar from the stack + remove(bar) { + const index = this.bars.indexOf(bar); + if (index < 0) { + return false; + } + this.bars.splice(index, 1); + this.update(); + this.terminal.newline(); + this.terminal.clearBottom(); + return true; + } + // internal update routine + update() { + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + this.emit("update-pre"); + this.terminal.cursorRelativeReset(); + this.emit("redraw-pre"); + if (this.loggingBuffer.length > 0) { + this.terminal.clearLine(); + while (this.loggingBuffer.length > 0) { + this.terminal.write(this.loggingBuffer.shift(), true); + } + } + for (let i = 0; i < this.bars.length; i++) { + if (i > 0) { + this.terminal.newline(); + } + this.bars[i].render(); + } + this.emit("redraw-post"); + if (this.options.noTTYOutput && this.terminal.isTTY() === false) { + this.terminal.newline(); + this.terminal.newline(); + } + this.timer = setTimeout(this.update.bind(this), this.schedulingRate); + this.emit("update-post"); + if (this.options.stopOnComplete && !this.bars.find((bar) => bar.isActive)) { + this.stop(); + } + } + stop() { + clearTimeout(this.timer); + this.timer = null; + if (this.sigintCallback) { + process.removeListener("SIGINT", this.sigintCallback); + process.removeListener("SIGTERM", this.sigintCallback); + this.sigintCallback = null; + } + this.isActive = false; + if (this.options.hideCursor === true) { + this.terminal.cursor(true); + } + if (this.options.linewrap === false) { + this.terminal.lineWrapping(true); + } + this.terminal.cursorRelativeReset(); + this.emit("stop-pre-clear"); + if (this.options.clearOnComplete) { + this.terminal.clearBottom(); + } else { + for (let i = 0; i < this.bars.length; i++) { + if (i > 0) { + this.terminal.newline(); + } + this.bars[i].render(); + this.bars[i].stop(); + } + this.terminal.newline(); + } + this.emit("stop"); + } + log(s) { + this.loggingBuffer.push(s); + } + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/legacy.js +var require_legacy = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/legacy.js"(exports, module) { + module.exports = { + format: "progress [{bar}] {percentage}% | ETA: {eta}s | {value}/{total}", + barCompleteChar: "=", + barIncompleteChar: "-" + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/shades-classic.js +var require_shades_classic = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/shades-classic.js"(exports, module) { + module.exports = { + format: " {bar} {percentage}% | ETA: {eta}s | {value}/{total}", + barCompleteChar: "\u2588", + barIncompleteChar: "\u2591" + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/shades-grey.js +var require_shades_grey = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/shades-grey.js"(exports, module) { + module.exports = { + format: " \x1B[90m{bar}\x1B[0m {percentage}% | ETA: {eta}s | {value}/{total}", + barCompleteChar: "\u2588", + barIncompleteChar: "\u2591" + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/rect.js +var require_rect = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/rect.js"(exports, module) { + module.exports = { + format: " {bar}\u25A0 {percentage}% | ETA: {eta}s | {value}/{total}", + barCompleteChar: "\u25A0", + barIncompleteChar: " " + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/index.js +var require_presets = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/index.js"(exports, module) { + var _legacy = require_legacy(); + var _shades_classic = require_shades_classic(); + var _shades_grey = require_shades_grey(); + var _rect = require_rect(); + module.exports = { + legacy: _legacy, + shades_classic: _shades_classic, + shades_grey: _shades_grey, + rect: _rect + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/cli-progress.js +var require_cli_progress = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/cli-progress.js"(exports, module) { + var _SingleBar = require_single_bar(); + var _MultiBar = require_multi_bar(); + var _Presets = require_presets(); + var _Formatter = require_formatter(); + var _defaultFormatValue = require_format_value(); + var _defaultFormatBar = require_format_bar(); + var _defaultFormatTime = require_format_time(); + module.exports = { + Bar: _SingleBar, + SingleBar: _SingleBar, + MultiBar: _MultiBar, + Presets: _Presets, + Format: { + Formatter: _Formatter, + BarFormat: _defaultFormatBar, + ValueFormat: _defaultFormatValue, + TimeFormat: _defaultFormatTime + } + }; + } +}); +export default require_cli_progress(); diff --git a/node_modules/@huggingface/hub/dist/cli.d.ts b/node_modules/@huggingface/hub/dist/cli.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..7f0c963fd8778db09b8b443e4024f8d3524c4387 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/cli.d.ts @@ -0,0 +1,3 @@ +#! /usr/bin/env node +export {}; +//# sourceMappingURL=cli.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/cli.d.ts.map b/node_modules/@huggingface/hub/dist/cli.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..99ff4773a76bee649ee7cf9618578a0809ed8ee8 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/cli.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../cli.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/cli.js b/node_modules/@huggingface/hub/dist/cli.js new file mode 100755 index 0000000000000000000000000000000000000000..4af3b80747a51642967b9bb4a7f463e790a677dc --- /dev/null +++ b/node_modules/@huggingface/hub/dist/cli.js @@ -0,0 +1,6098 @@ +#! /usr/bin/env node +"use strict"; +var __create = Object.create; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __getProtoOf = Object.getPrototypeOf; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __esm = (fn, res) => function __init() { + return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res; +}; +var __commonJS = (cb, mod) => function __require() { + return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports; +}; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( + // If the importer is in node compatibility mode or this is not an ESM + // file that has been converted to a CommonJS file using a Babel- + // compatible transform (i.e. "__esModule" has not been set), then set + // "default" to the CommonJS "module.exports" for node compatibility. + isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, + mod +)); + +// src/vendor/hash-wasm/sha256.js +var import_meta, Module, sha256_default; +var init_sha256 = __esm({ + "src/vendor/hash-wasm/sha256.js"() { + "use strict"; + import_meta = {}; + Module = (() => { + var _unused = import_meta.url; + return function(moduleArg = {}) { + var Module2 = moduleArg; + var readyPromiseResolve, readyPromiseReject; + Module2["ready"] = new Promise((resolve3, reject) => { + readyPromiseResolve = resolve3; + readyPromiseReject = reject; + }); + var moduleOverrides = Object.assign({}, Module2); + var arguments_ = []; + var thisProgram = "./this.program"; + var quit_ = (status, toThrow) => { + throw toThrow; + }; + var ENVIRONMENT_IS_WEB = typeof window == "object"; + var ENVIRONMENT_IS_WORKER = typeof importScripts == "function"; + var ENVIRONMENT_IS_NODE = typeof process == "object" && typeof process.versions == "object" && typeof process.versions.node == "string"; + var ENVIRONMENT_IS_SHELL = !ENVIRONMENT_IS_WEB && !ENVIRONMENT_IS_NODE && !ENVIRONMENT_IS_WORKER; + var scriptDirectory = ""; + function locateFile(path2) { + if (Module2["locateFile"]) { + return Module2["locateFile"](path2, scriptDirectory); + } + return scriptDirectory + path2; + } + var read_, readAsync, readBinary; + if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { + if (ENVIRONMENT_IS_WORKER) { + scriptDirectory = self.location.href; + } else if (typeof document != "undefined" && document.currentScript) { + scriptDirectory = document.currentScript.src; + } + if (false) { + scriptDirectory = false; + } + if (scriptDirectory.startsWith("blob:")) { + scriptDirectory = ""; + } else { + scriptDirectory = scriptDirectory.substr(0, scriptDirectory.replace(/[?#].*/, "").lastIndexOf("/") + 1); + } + { + read_ = (url) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, false); + xhr.send(null); + return xhr.responseText; + }; + if (ENVIRONMENT_IS_WORKER) { + readBinary = (url) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, false); + xhr.responseType = "arraybuffer"; + xhr.send(null); + return new Uint8Array( + /** @type{!ArrayBuffer} */ + xhr.response + ); + }; + } + readAsync = (url, onload, onerror) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, true); + xhr.responseType = "arraybuffer"; + xhr.onload = () => { + if (xhr.status == 200 || xhr.status == 0 && xhr.response) { + onload(xhr.response); + return; + } + onerror(); + }; + xhr.onerror = onerror; + xhr.send(null); + }; + } + } else { + } + var out = Module2["print"] || console.log.bind(console); + var err = Module2["printErr"] || console.error.bind(console); + Object.assign(Module2, moduleOverrides); + moduleOverrides = null; + if (Module2["arguments"]) + arguments_ = Module2["arguments"]; + if (Module2["thisProgram"]) + thisProgram = Module2["thisProgram"]; + if (Module2["quit"]) + quit_ = Module2["quit"]; + var wasmBinary; + if (Module2["wasmBinary"]) + wasmBinary = Module2["wasmBinary"]; + if (typeof WebAssembly != "object") { + abort("no native wasm support detected"); + } + function intArrayFromBase64(s) { + var decoded = atob(s); + var bytes = new Uint8Array(decoded.length); + for (var i = 0; i < decoded.length; ++i) { + bytes[i] = decoded.charCodeAt(i); + } + return bytes; + } + function tryParseAsDataURI(filename) { + if (!isDataURI(filename)) { + return; + } + return intArrayFromBase64(filename.slice(dataURIPrefix.length)); + } + var wasmMemory; + var ABORT = false; + var EXITSTATUS; + function assert(condition, text) { + if (!condition) { + abort(text); + } + } + var HEAP, HEAP8, HEAPU8, HEAP16, HEAPU16, HEAP32, HEAPU32, HEAPF32, HEAPF64; + function updateMemoryViews() { + var b = wasmMemory.buffer; + Module2["HEAP8"] = HEAP8 = new Int8Array(b); + Module2["HEAP16"] = HEAP16 = new Int16Array(b); + Module2["HEAPU8"] = HEAPU8 = new Uint8Array(b); + Module2["HEAPU16"] = HEAPU16 = new Uint16Array(b); + Module2["HEAP32"] = HEAP32 = new Int32Array(b); + Module2["HEAPU32"] = HEAPU32 = new Uint32Array(b); + Module2["HEAPF32"] = HEAPF32 = new Float32Array(b); + Module2["HEAPF64"] = HEAPF64 = new Float64Array(b); + } + var __ATPRERUN__ = []; + var __ATINIT__ = []; + var __ATEXIT__ = []; + var __ATPOSTRUN__ = []; + var runtimeInitialized = false; + function preRun() { + if (Module2["preRun"]) { + if (typeof Module2["preRun"] == "function") + Module2["preRun"] = [Module2["preRun"]]; + while (Module2["preRun"].length) { + addOnPreRun(Module2["preRun"].shift()); + } + } + callRuntimeCallbacks(__ATPRERUN__); + } + function initRuntime() { + runtimeInitialized = true; + callRuntimeCallbacks(__ATINIT__); + } + function postRun() { + if (Module2["postRun"]) { + if (typeof Module2["postRun"] == "function") + Module2["postRun"] = [Module2["postRun"]]; + while (Module2["postRun"].length) { + addOnPostRun(Module2["postRun"].shift()); + } + } + callRuntimeCallbacks(__ATPOSTRUN__); + } + function addOnPreRun(cb) { + __ATPRERUN__.unshift(cb); + } + function addOnInit(cb) { + __ATINIT__.unshift(cb); + } + function addOnExit(cb) { + } + function addOnPostRun(cb) { + __ATPOSTRUN__.unshift(cb); + } + var runDependencies = 0; + var runDependencyWatcher = null; + var dependenciesFulfilled = null; + function getUniqueRunDependency(id) { + return id; + } + function addRunDependency(id) { + runDependencies++; + Module2["monitorRunDependencies"]?.(runDependencies); + } + function removeRunDependency(id) { + runDependencies--; + Module2["monitorRunDependencies"]?.(runDependencies); + if (runDependencies == 0) { + if (runDependencyWatcher !== null) { + clearInterval(runDependencyWatcher); + runDependencyWatcher = null; + } + if (dependenciesFulfilled) { + var callback = dependenciesFulfilled; + dependenciesFulfilled = null; + callback(); + } + } + } + function abort(what) { + Module2["onAbort"]?.(what); + what = "Aborted(" + what + ")"; + err(what); + ABORT = true; + EXITSTATUS = 1; + what += ". Build with -sASSERTIONS for more info."; + var e = new WebAssembly.RuntimeError(what); + readyPromiseReject(e); + throw e; + } + var dataURIPrefix = "data:application/octet-stream;base64,"; + var isDataURI = (filename) => filename.startsWith(dataURIPrefix); + var isFileURI = (filename) => filename.startsWith("file://"); + var wasmBinaryFile; + wasmBinaryFile = "data:application/octet-stream;base64,AGFzbQEAAAABHQZgAX8AYAABf2AAAGABfwF/YAJ/fwBgA39/fwF/Aw0MAgAEAgMBBQABAQADBAUBcAEBAQUGAQGAAoACBg4CfwFB8IuEBAt/AUEACweYAQoGbWVtb3J5AgARX193YXNtX2NhbGxfY3RvcnMAAAtIYXNoX1VwZGF0ZQABCkhhc2hfRmluYWwAAwlIYXNoX0luaXQABAxHZXRCdWZmZXJQdHIABRlfX2luZGlyZWN0X2Z1bmN0aW9uX3RhYmxlAQAJc3RhY2tTYXZlAAkMc3RhY2tSZXN0b3JlAAoKc3RhY2tBbGxvYwALCossDAIAC+4CAgV/AX5BACgCwAoiASABKQNAIgYgAK18NwNAAkACQAJAIAanQT9xIgINAEGACyEBIAAhAgwBC0HAACACayEDAkAgAEUNACADIAAgAyAASRshBCABIAJqIQVBACEBA0AgBSABIgFqQYALIAFqLQAAOgAAIAFBAWoiAiEBIAIgBEcNAAsLAkACQCAAIANJIgRFDQBBgAshASAAIQIMAQtBACgCwAoiAUHIAGogARACQYALIANqIQEgACADayECCyABIQEgAiECIAQNAQsgASEBAkACQCACIgJBwABPDQAgASEFIAIhAAwBCyACIQIgASEEA0BBACgCwApByABqIAQiBBACIAJBQGoiASECIARBwABqIgUhBCAFIQUgASEAIAFBP0sNAAsLIAUhBSAAIgBFDQBBACEBQQAhAgNAQQAoAsAKIAEiAWogBSABai0AADoAACACQQFqIgJB/wFxIgQhASACIQIgACAESw0ACwsLqCEBK38gACgCCCICIAAoAgQiAyAAKAIAIgRzcSADIARxcyAEQR53IARBE3dzIARBCndzaiAAKAIQIgVBGncgBUEVd3MgBUEHd3MgACgCHCIGaiAAKAIYIgcgACgCFCIIcyAFcSAHc2ogASgCACIJQRh0IAlBgP4DcUEIdHIgCUEIdkGA/gNxIAlBGHZyciIKakGY36iUBGoiC2oiCSAEcyADcSAJIARxcyAJQR53IAlBE3dzIAlBCndzaiAHIAEoAgQiDEEYdCAMQYD+A3FBCHRyIAxBCHZBgP4DcSAMQRh2cnIiDWogCyAAKAIMIg5qIg8gCCAFc3EgCHNqIA9BGncgD0EVd3MgD0EHd3NqQZGJ3YkHaiIQaiIMIAlzIARxIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAggASgCCCILQRh0IAtBgP4DcUEIdHIgC0EIdkGA/gNxIAtBGHZyciIRaiAQIAJqIhIgDyAFc3EgBXNqIBJBGncgEkEVd3MgEkEHd3NqQc/3g657aiITaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAUgASgCDCIQQRh0IBBBgP4DcUEIdHIgEEEIdkGA/gNxIBBBGHZyciIUaiATIANqIhMgEiAPc3EgD3NqIBNBGncgE0EVd3MgE0EHd3NqQaW3181+aiIVaiIQIAtzIAxxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIA8gASgCECIWQRh0IBZBgP4DcUEIdHIgFkEIdkGA/gNxIBZBGHZyciIXaiAVIARqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQduE28oDaiIYaiIPIBBzIAtxIA8gEHFzIA9BHncgD0ETd3MgD0EKd3NqIAEoAhQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGSASaiAYIAlqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQfGjxM8FaiIYaiIJIA9zIBBxIAkgD3FzIAlBHncgCUETd3MgCUEKd3NqIAEoAhgiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGiATaiAYIAxqIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQaSF/pF5aiIYaiIMIAlzIA9xIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAEoAhwiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGyAWaiAYIAtqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQdW98dh6aiIYaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAEoAiAiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHCASaiAYIBBqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQZjVnsB9aiIYaiIQIAtzIAxxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAiQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHSATaiAYIA9qIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQYG2jZQBaiIYaiIPIBBzIAtxIA8gEHFzIA9BHncgD0ETd3MgD0EKd3NqIAEoAigiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHiAWaiAYIAlqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQb6LxqECaiIYaiIJIA9zIBBxIAkgD3FzIAlBHncgCUETd3MgCUEKd3NqIAEoAiwiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHyASaiAYIAxqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQcP7sagFaiIYaiIMIAlzIA9xIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAEoAjAiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiICATaiAYIAtqIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQfS6+ZUHaiIYaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAEoAjQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiISAWaiAYIBBqIhAgEyASc3EgEnNqIBBBGncgEEEVd3MgEEEHd3NqQf7j+oZ4aiIYaiIWIAtzIAxxIBYgC3FzIBZBHncgFkETd3MgFkEKd3NqIAEoAjgiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiIiASaiAYIA9qIg8gECATc3EgE3NqIA9BGncgD0EVd3MgD0EHd3NqQaeN8N55aiIVaiISIBZzIAtxIBIgFnFzIBJBHncgEkETd3MgEkEKd3NqIAEoAjwiAUEYdCABQYD+A3FBCHRyIAFBCHZBgP4DcSABQRh2cnIiIyATaiAVIAlqIgEgDyAQc3EgEHNqIAFBGncgAUEVd3MgAUEHd3NqQfTi74x8aiIJaiEVIBIhGCAWISQgCyElIAkgDGohJiABIScgDyEoIBAhKSAjISMgIiEiICEhISAgISAgHyEfIB4hHiAdIR0gHCEcIBshGyAaIRogGSEZIBchFyAUIRQgESERIA0hECAKIQxBgAkhAUEQISoDQCAVIgkgGCIKcyAkIitxIAkgCnFzIAlBHncgCUETd3MgCUEKd3NqIBAiEEEZdyAQQQ53cyAQQQN2cyAMaiAdIh1qICIiFkEPdyAWQQ13cyAWQQp2c2oiDCApaiAmIhIgJyIPICgiE3NxIBNzaiASQRp3IBJBFXdzIBJBB3dzaiABIgEoAgBqIiRqIgsgCXMgCnEgCyAJcXMgC0EedyALQRN3cyALQQp3c2ogESIYQRl3IBhBDndzIBhBA3ZzIBBqIB4iHmogIyIVQQ93IBVBDXdzIBVBCnZzaiINIBNqIAEoAgRqICQgJWoiEyASIA9zcSAPc2ogE0EadyATQRV3cyATQQd3c2oiJWoiECALcyAJcSAQIAtxcyAQQR53IBBBE3dzIBBBCndzaiAUIiRBGXcgJEEOd3MgJEEDdnMgGGogHyIfaiAMQQ93IAxBDXdzIAxBCnZzaiIRIA9qIAEoAghqICUgK2oiGCATIBJzcSASc2ogGEEadyAYQRV3cyAYQQd3c2oiJWoiDyAQcyALcSAPIBBxcyAPQR53IA9BE3dzIA9BCndzaiAXIhdBGXcgF0EOd3MgF0EDdnMgJGogICIgaiANQQ93IA1BDXdzIA1BCnZzaiIUIBJqIAEoAgxqICUgCmoiCiAYIBNzcSATc2ogCkEadyAKQRV3cyAKQQd3c2oiJWoiEiAPcyAQcSASIA9xcyASQR53IBJBE3dzIBJBCndzaiATIBkiJEEZdyAkQQ53cyAkQQN2cyAXaiAhIiFqIBFBD3cgEUENd3MgEUEKdnNqIhdqIAEoAhBqICUgCWoiEyAKIBhzcSAYc2ogE0EadyATQRV3cyATQQd3c2oiJWoiCSAScyAPcSAJIBJxcyAJQR53IAlBE3dzIAlBCndzaiABKAIUIBoiGkEZdyAaQQ53cyAaQQN2cyAkaiAWaiAUQQ93IBRBDXdzIBRBCnZzaiIZaiAYaiAlIAtqIhggEyAKc3EgCnNqIBhBGncgGEEVd3MgGEEHd3NqIiVqIgsgCXMgEnEgCyAJcXMgC0EedyALQRN3cyALQQp3c2ogASgCGCAbIiRBGXcgJEEOd3MgJEEDdnMgGmogFWogF0EPdyAXQQ13cyAXQQp2c2oiGmogCmogJSAQaiIKIBggE3NxIBNzaiAKQRp3IApBFXdzIApBB3dzaiIlaiIQIAtzIAlxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAhwgHCIcQRl3IBxBDndzIBxBA3ZzICRqIAxqIBlBD3cgGUENd3MgGUEKdnNqIhtqIBNqICUgD2oiJCAKIBhzcSAYc2ogJEEadyAkQRV3cyAkQQd3c2oiE2oiDyAQcyALcSAPIBBxcyAPQR53IA9BE3dzIA9BCndzaiABKAIgIB1BGXcgHUEOd3MgHUEDdnMgHGogDWogGkEPdyAaQQ13cyAaQQp2c2oiHGogGGogEyASaiIYICQgCnNxIApzaiAYQRp3IBhBFXdzIBhBB3dzaiITaiISIA9zIBBxIBIgD3FzIBJBHncgEkETd3MgEkEKd3NqIAEoAiQgHkEZdyAeQQ53cyAeQQN2cyAdaiARaiAbQQ93IBtBDXdzIBtBCnZzaiIdaiAKaiATIAlqIgkgGCAkc3EgJHNqIAlBGncgCUEVd3MgCUEHd3NqIgpqIhMgEnMgD3EgEyAScXMgE0EedyATQRN3cyATQQp3c2ogASgCKCAfQRl3IB9BDndzIB9BA3ZzIB5qIBRqIBxBD3cgHEENd3MgHEEKdnNqIh5qICRqIAogC2oiCiAJIBhzcSAYc2ogCkEadyAKQRV3cyAKQQd3c2oiJGoiCyATcyAScSALIBNxcyALQR53IAtBE3dzIAtBCndzaiABKAIsICBBGXcgIEEOd3MgIEEDdnMgH2ogF2ogHUEPdyAdQQ13cyAdQQp2c2oiH2ogGGogJCAQaiIYIAogCXNxIAlzaiAYQRp3IBhBFXdzIBhBB3dzaiIkaiIQIAtzIBNxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAjAgIUEZdyAhQQ53cyAhQQN2cyAgaiAZaiAeQQ93IB5BDXdzIB5BCnZzaiIgaiAJaiAkIA9qIiQgGCAKc3EgCnNqICRBGncgJEEVd3MgJEEHd3NqIg9qIgkgEHMgC3EgCSAQcXMgCUEedyAJQRN3cyAJQQp3c2ogASgCNCAWQRl3IBZBDndzIBZBA3ZzICFqIBpqIB9BD3cgH0ENd3MgH0EKdnNqIiFqIApqIA8gEmoiDyAkIBhzcSAYc2ogD0EadyAPQRV3cyAPQQd3c2oiCmoiEiAJcyAQcSASIAlxcyASQR53IBJBE3dzIBJBCndzaiABKAI4IBVBGXcgFUEOd3MgFUEDdnMgFmogG2ogIEEPdyAgQQ13cyAgQQp2c2oiImogGGogCiATaiITIA8gJHNxICRzaiATQRp3IBNBFXdzIBNBB3dzaiIYaiIWIBJzIAlxIBYgEnFzIBZBHncgFkETd3MgFkEKd3NqIAEoAjwgDEEZdyAMQQ53cyAMQQN2cyAVaiAcaiAhQQ93ICFBDXdzICFBCnZzaiIKaiAkaiAYIAtqIgsgEyAPc3EgD3NqIAtBGncgC0EVd3MgC0EHd3NqIiZqIishFSAWIRggEiEkIAkhJSAmIBBqIiwhJiALIScgEyEoIA8hKSAKISMgIiEiICEhISAgISAgHyEfIB4hHiAdIR0gHCEcIBshGyAaIRogGSEZIBchFyAUIRQgESERIA0hECAMIQwgAUHAAGohASAqIgpBEGohKiAKQTBJDQALIAAgDyAGajYCHCAAIBMgB2o2AhggACALIAhqNgIUIAAgLCAFajYCECAAIAkgDmo2AgwgACASIAJqNgIIIAAgFiADajYCBCAAICsgBGo2AgAL1AMDBX8BfgF7QQAoAsAKIgAgACgCQCIBQQJ2QQ9xIgJBAnRqIgMgAygCAEF/IAFBA3QiAXRBf3NxQYABIAF0czYCAAJAAkAgAkEOTw0AIAJBAWohAAwBCwJAIAJBDkcNACAAQQA2AjwLIABByABqIAAQAkEAIQALAkAgACIAQQ1LDQBBACgCwAogAEECdCIAakEAQTggAGsQBhoLQQAoAsAKIgAgACkDQCIFpyICQRt0IAJBC3RBgID8B3FyIAJBBXZBgP4DcSACQQN0QRh2cnI2AjwgACAFQh2IpyICQRh0IAJBgP4DcUEIdHIgAkEIdkGA/gNxIAJBGHZycjYCOCAAQcgAaiAAEAJBACgCwApBPGohAUEAIQADQCABQQcgACIAa0ECdGoiAiAC/QACACAG/Q0MDQ4PCAkKCwQFBgcAAQIDIAb9DQMCAQAHBgUECwoJCA8ODQwgBv0NDA0ODwgJCgsEBQYHAAECA/0LAgAgAEEEaiICIQAgAkEIRw0ACwJAQQAoAsAKIgMoAmhFDQAgA0HIAGohBEEAIQBBACECA0BBgAsgACIAaiAEIABqLQAAOgAAIAJBAWoiAkH/AXEiASEAIAIhAiADKAJoIAFLDQALCwtxAQJ/QQAoAsAKIgFCADcDQCABQcgAaiECAkAgAEHgAUcNACABQRw2AmggAkEQakEA/QAEsAj9CwIAIAJBAP0ABKAI/QsCAEEADwsgAUEgNgJoIAJBEGpBAP0ABJAI/QsCACACQQD9AASACP0LAgBBAAsFAEGACwvyAgIDfwF+AkAgAkUNACAAIAE6AAAgACACaiIDQX9qIAE6AAAgAkEDSQ0AIAAgAToAAiAAIAE6AAEgA0F9aiABOgAAIANBfmogAToAACACQQdJDQAgACABOgADIANBfGogAToAACACQQlJDQAgAEEAIABrQQNxIgRqIgMgAUH/AXFBgYKECGwiATYCACADIAIgBGtBfHEiBGoiAkF8aiABNgIAIARBCUkNACADIAE2AgggAyABNgIEIAJBeGogATYCACACQXRqIAE2AgAgBEEZSQ0AIAMgATYCGCADIAE2AhQgAyABNgIQIAMgATYCDCACQXBqIAE2AgAgAkFsaiABNgIAIAJBaGogATYCACACQWRqIAE2AgAgBCADQQRxQRhyIgVrIgJBIEkNACABrUKBgICAEH4hBiADIAVqIQEDQCABIAY3AxggASAGNwMQIAEgBjcDCCABIAY3AwAgAUEgaiEBIAJBYGoiAkEfSw0ACwsgAAsGACAAJAELBAAjAQsEACMACwYAIAAkAAsSAQJ/IwAgAGtBcHEiASQAIAELC9ICAgBBgAgLwAJn5glqha5nu3Lzbjw69U+lf1IOUYxoBZur2YMfGc3gW9ieBcEH1Xw2F91wMDlZDvcxC8D/ERVYaKeP+WSkT/q+mC+KQpFEN3HP+8C1pdu16VvCVjnxEfFZpII/ktVeHKuYqgfYAVuDEr6FMSTDfQxVdF2+cv6x3oCnBtybdPGbwcFpm+SGR77vxp3BD8yhDCRvLOktqoR0StypsFzaiPl2UlE+mG3GMajIJwOwx39Zv/ML4MZHkafVUWPKBmcpKRSFCrcnOCEbLvxtLE0TDThTVHMKZbsKanYuycKBhSxykqHov6JLZhqocItLwqNRbMcZ6JLRJAaZ1oU1DvRwoGoQFsGkGQhsNx5Md0gntbywNLMMHDlKqthOT8qcW/NvLmjugo90b2OleBR4yIQIAseM+v++kOtsUKT3o/m+8nhxxgBBwAoLBIAFgAA="; + if (!isDataURI(wasmBinaryFile)) { + wasmBinaryFile = locateFile(wasmBinaryFile); + } + function getBinarySync(file) { + if (file == wasmBinaryFile && wasmBinary) { + return new Uint8Array(wasmBinary); + } + var binary = tryParseAsDataURI(file); + if (binary) { + return binary; + } + if (readBinary) { + return readBinary(file); + } + throw "both async and sync fetching of the wasm failed"; + } + function getBinaryPromise(binaryFile) { + return Promise.resolve().then(() => getBinarySync(binaryFile)); + } + function instantiateArrayBuffer(binaryFile, imports, receiver) { + return getBinaryPromise(binaryFile).then((binary) => { + return WebAssembly.instantiate(binary, imports); + }).then(receiver, (reason) => { + err(`failed to asynchronously prepare wasm: ${reason}`); + abort(reason); + }); + } + function instantiateAsync(binary, binaryFile, imports, callback) { + return instantiateArrayBuffer(binaryFile, imports, callback); + } + function createWasm() { + var info = { + "env": wasmImports, + "wasi_snapshot_preview1": wasmImports + }; + function receiveInstance(instance, module2) { + wasmExports = instance.exports; + wasmMemory = wasmExports["memory"]; + updateMemoryViews(); + addOnInit(wasmExports["__wasm_call_ctors"]); + removeRunDependency("wasm-instantiate"); + return wasmExports; + } + addRunDependency("wasm-instantiate"); + function receiveInstantiationResult(result) { + receiveInstance(result["instance"]); + } + if (Module2["instantiateWasm"]) { + try { + return Module2["instantiateWasm"](info, receiveInstance); + } catch (e) { + err(`Module.instantiateWasm callback failed with error: ${e}`); + readyPromiseReject(e); + } + } + instantiateAsync(wasmBinary, wasmBinaryFile, info, receiveInstantiationResult).catch(readyPromiseReject); + return {}; + } + var tempDouble; + var tempI64; + function ExitStatus(status) { + this.name = "ExitStatus"; + this.message = `Program terminated with exit(${status})`; + this.status = status; + } + var callRuntimeCallbacks = (callbacks) => { + while (callbacks.length > 0) { + callbacks.shift()(Module2); + } + }; + function getValue(ptr, type = "i8") { + if (type.endsWith("*")) + type = "*"; + switch (type) { + case "i1": + return HEAP8[ptr]; + case "i8": + return HEAP8[ptr]; + case "i16": + return HEAP16[ptr >> 1]; + case "i32": + return HEAP32[ptr >> 2]; + case "i64": + abort("to do getValue(i64) use WASM_BIGINT"); + case "float": + return HEAPF32[ptr >> 2]; + case "double": + return HEAPF64[ptr >> 3]; + case "*": + return HEAPU32[ptr >> 2]; + default: + abort(`invalid type for getValue: ${type}`); + } + } + var noExitRuntime = Module2["noExitRuntime"] || true; + function setValue(ptr, value, type = "i8") { + if (type.endsWith("*")) + type = "*"; + switch (type) { + case "i1": + HEAP8[ptr] = value; + break; + case "i8": + HEAP8[ptr] = value; + break; + case "i16": + HEAP16[ptr >> 1] = value; + break; + case "i32": + HEAP32[ptr >> 2] = value; + break; + case "i64": + abort("to do setValue(i64) use WASM_BIGINT"); + case "float": + HEAPF32[ptr >> 2] = value; + break; + case "double": + HEAPF64[ptr >> 3] = value; + break; + case "*": + HEAPU32[ptr >> 2] = value; + break; + default: + abort(`invalid type for setValue: ${type}`); + } + } + var wasmImports = {}; + var wasmExports = createWasm(); + var ___wasm_call_ctors = () => (___wasm_call_ctors = wasmExports["__wasm_call_ctors"])(); + var _Hash_Update = Module2["_Hash_Update"] = (a0) => (_Hash_Update = Module2["_Hash_Update"] = wasmExports["Hash_Update"])(a0); + var _Hash_Final = Module2["_Hash_Final"] = () => (_Hash_Final = Module2["_Hash_Final"] = wasmExports["Hash_Final"])(); + var _Hash_Init = Module2["_Hash_Init"] = (a0) => (_Hash_Init = Module2["_Hash_Init"] = wasmExports["Hash_Init"])(a0); + var _GetBufferPtr = Module2["_GetBufferPtr"] = () => (_GetBufferPtr = Module2["_GetBufferPtr"] = wasmExports["GetBufferPtr"])(); + var stackSave = () => (stackSave = wasmExports["stackSave"])(); + var stackRestore = (a0) => (stackRestore = wasmExports["stackRestore"])(a0); + var stackAlloc = (a0) => (stackAlloc = wasmExports["stackAlloc"])(a0); + var calledRun; + dependenciesFulfilled = function runCaller() { + if (!calledRun) + run2(); + if (!calledRun) + dependenciesFulfilled = runCaller; + }; + function run2() { + if (runDependencies > 0) { + return; + } + preRun(); + if (runDependencies > 0) { + return; + } + function doRun() { + if (calledRun) + return; + calledRun = true; + Module2["calledRun"] = true; + if (ABORT) + return; + initRuntime(); + readyPromiseResolve(Module2); + if (Module2["onRuntimeInitialized"]) + Module2["onRuntimeInitialized"](); + postRun(); + } + if (Module2["setStatus"]) { + Module2["setStatus"]("Running..."); + setTimeout(function() { + setTimeout(function() { + Module2["setStatus"](""); + }, 1); + doRun(); + }, 1); + } else { + doRun(); + } + } + if (Module2["preInit"]) { + if (typeof Module2["preInit"] == "function") + Module2["preInit"] = [Module2["preInit"]]; + while (Module2["preInit"].length > 0) { + Module2["preInit"].pop()(); + } + } + run2(); + return moduleArg.ready; + }; + })(); + sha256_default = Module; + } +}); + +// src/vendor/hash-wasm/sha256-wrapper.ts +var sha256_wrapper_exports = {}; +__export(sha256_wrapper_exports, { + createSHA256: () => createSHA256, + createSHA256WorkerCode: () => createSHA256WorkerCode +}); +async function createSHA256(isInsideWorker = false) { + const BUFFER_MAX_SIZE = 8 * 1024 * 1024; + const wasm = isInsideWorker ? ( + // @ts-expect-error WasmModule will be populated inside self object + await self["SHA256WasmModule"]() + ) : await sha256_default(); + const heap = wasm.HEAPU8.subarray(wasm._GetBufferPtr()); + return { + init() { + wasm._Hash_Init(256); + }, + update(data) { + let byteUsed = 0; + while (byteUsed < data.byteLength) { + const bytesLeft = data.byteLength - byteUsed; + const length = Math.min(bytesLeft, BUFFER_MAX_SIZE); + heap.set(data.subarray(byteUsed, byteUsed + length)); + wasm._Hash_Update(length); + byteUsed += length; + } + }, + digest(method) { + if (method !== "hex") { + throw new Error("Only digest hex is supported"); + } + wasm._Hash_Final(); + const result = Array.from(heap.slice(0, 32)); + return result.map((b) => b.toString(16).padStart(2, "0")).join(""); + } + }; +} +function createSHA256WorkerCode() { + return ` + self.addEventListener('message', async (event) => { + const { file } = event.data; + const sha256 = await self.createSHA256(true); + sha256.init(); + const reader = file.stream().getReader(); + const total = file.size; + let bytesDone = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + sha256.update(value); + bytesDone += value.length; + postMessage({ progress: bytesDone / total }); + } + postMessage({ sha256: sha256.digest('hex') }); + }); + self.SHA256WasmModule = ${sha256_default.toString()}; + self.createSHA256 = ${createSHA256.toString()}; + `; +} +var init_sha256_wrapper = __esm({ + "src/vendor/hash-wasm/sha256-wrapper.ts"() { + "use strict"; + init_sha256(); + } +}); + +// src/utils/sha256-node.ts +var sha256_node_exports = {}; +__export(sha256_node_exports, { + sha256Node: () => sha256Node +}); +async function* sha256Node(buffer, opts) { + const sha256Stream = (0, import_node_crypto.createHash)("sha256"); + const size = buffer instanceof Blob ? buffer.size : buffer.byteLength; + let done = 0; + const readable = buffer instanceof Blob ? import_node_stream.Readable.fromWeb(buffer.stream()) : import_node_stream.Readable.from(Buffer.from(buffer)); + for await (const buffer2 of readable) { + sha256Stream.update(buffer2); + done += buffer2.length; + yield done / size; + opts?.abortSignal?.throwIfAborted(); + } + return sha256Stream.digest("hex"); +} +var import_node_stream, import_node_crypto; +var init_sha256_node = __esm({ + "src/utils/sha256-node.ts"() { + "use strict"; + import_node_stream = require("stream"); + import_node_crypto = require("crypto"); + } +}); + +// src/utils/FileBlob.ts +var FileBlob_exports = {}; +__export(FileBlob_exports, { + FileBlob: () => FileBlob +}); +var import_node_fs, import_promises2, import_node_stream2, import_node_url, FileBlob; +var init_FileBlob = __esm({ + "src/utils/FileBlob.ts"() { + "use strict"; + import_node_fs = require("fs"); + import_promises2 = require("fs/promises"); + import_node_stream2 = require("stream"); + import_node_url = require("url"); + FileBlob = class extends Blob { + /** + * Creates a new FileBlob on the provided file. + * + * @param path Path to the file to be lazy readed + */ + static async create(path2) { + path2 = path2 instanceof URL ? (0, import_node_url.fileURLToPath)(path2) : path2; + const { size } = await (0, import_promises2.stat)(path2); + const fileBlob = new FileBlob(path2, 0, size); + return fileBlob; + } + path; + start; + end; + constructor(path2, start, end) { + super(); + this.path = path2; + this.start = start; + this.end = end; + } + /** + * Returns the size of the blob. + */ + get size() { + return this.end - this.start; + } + /** + * Returns a new instance of FileBlob that is a slice of the current one. + * + * The slice is inclusive of the start and exclusive of the end. + * + * The slice method does not supports negative start/end. + * + * @param start beginning of the slice + * @param end end of the slice + */ + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on FileBlob.slice"); + } + const slice = new FileBlob(this.path, this.start + start, Math.min(this.start + end, this.end)); + return slice; + } + /** + * Read the part of the file delimited by the FileBlob and returns it as an ArrayBuffer. + */ + async arrayBuffer() { + const slice = await this.execute((file) => file.read(Buffer.alloc(this.size), 0, this.size, this.start)); + return slice.buffer; + } + /** + * Read the part of the file delimited by the FileBlob and returns it as a string. + */ + async text() { + const buffer = await this.arrayBuffer(); + return buffer.toString("utf8"); + } + /** + * Returns a stream around the part of the file delimited by the FileBlob. + */ + stream() { + if (this.start === this.end) { + return new Blob([]).stream(); + } + return import_node_stream2.Readable.toWeb((0, import_node_fs.createReadStream)(this.path, { start: this.start, end: this.end - 1 })); + } + /** + * We are opening and closing the file for each action to prevent file descriptor leaks. + * + * It is an intended choice of developer experience over performances. + */ + async execute(action) { + const file = await (0, import_promises2.open)(this.path, "r"); + try { + return await action(file); + } finally { + await file.close(); + } + } + }; + } +}); + +// src/utils/sub-paths.ts +var sub_paths_exports = {}; +__export(sub_paths_exports, { + subPaths: () => subPaths +}); +async function subPaths(path2, maxDepth = 10) { + const state = await (0, import_promises3.stat)(path2); + if (!state.isDirectory()) { + return [{ path: path2, relativePath: "." }]; + } + const files = await (0, import_promises3.readdir)(path2, { withFileTypes: true }); + const ret = []; + for (const file of files) { + const filePath = (0, import_node_url2.pathToFileURL)((0, import_node_url2.fileURLToPath)(path2) + "/" + file.name); + if (file.isDirectory()) { + ret.push( + ...(await subPaths(filePath, maxDepth - 1)).map((subPath) => ({ + ...subPath, + relativePath: `${file.name}/${subPath.relativePath}` + })) + ); + } else { + ret.push({ path: filePath, relativePath: file.name }); + } + } + return ret; +} +var import_promises3, import_node_url2; +var init_sub_paths = __esm({ + "src/utils/sub-paths.ts"() { + "use strict"; + import_promises3 = require("fs/promises"); + import_node_url2 = require("url"); + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/eta.js +var require_eta = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/eta.js"(exports, module2) { + var ETA = class { + constructor(length, initTime, initValue) { + this.etaBufferLength = length || 100; + this.valueBuffer = [initValue]; + this.timeBuffer = [initTime]; + this.eta = "0"; + } + // add new values to calculation buffer + update(time, value, total) { + this.valueBuffer.push(value); + this.timeBuffer.push(time); + this.calculate(total - value); + } + // fetch estimated time + getTime() { + return this.eta; + } + // eta calculation - request number of remaining events + calculate(remaining) { + const currentBufferSize = this.valueBuffer.length; + const buffer = Math.min(this.etaBufferLength, currentBufferSize); + const v_diff = this.valueBuffer[currentBufferSize - 1] - this.valueBuffer[currentBufferSize - buffer]; + const t_diff = this.timeBuffer[currentBufferSize - 1] - this.timeBuffer[currentBufferSize - buffer]; + const vt_rate = v_diff / t_diff; + this.valueBuffer = this.valueBuffer.slice(-this.etaBufferLength); + this.timeBuffer = this.timeBuffer.slice(-this.etaBufferLength); + const eta = Math.ceil(remaining / vt_rate / 1e3); + if (isNaN(eta)) { + this.eta = "NULL"; + } else if (!isFinite(eta)) { + this.eta = "INF"; + } else if (eta > 1e7) { + this.eta = "INF"; + } else if (eta < 0) { + this.eta = 0; + } else { + this.eta = eta; + } + } + }; + module2.exports = ETA; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/terminal.js +var require_terminal = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/terminal.js"(exports, module2) { + var _readline = require("readline"); + var Terminal = class { + constructor(outputStream) { + this.stream = outputStream; + this.linewrap = true; + this.dy = 0; + } + // save cursor position + settings + cursorSave() { + if (!this.stream.isTTY) { + return; + } + this.stream.write("\x1B7"); + } + // restore last cursor position + settings + cursorRestore() { + if (!this.stream.isTTY) { + return; + } + this.stream.write("\x1B8"); + } + // show/hide cursor + cursor(enabled) { + if (!this.stream.isTTY) { + return; + } + if (enabled) { + this.stream.write("\x1B[?25h"); + } else { + this.stream.write("\x1B[?25l"); + } + } + // change cursor positionn + cursorTo(x = null, y = null) { + if (!this.stream.isTTY) { + return; + } + _readline.cursorTo(this.stream, x, y); + } + // change relative cursor position + cursorRelative(dx = null, dy = null) { + if (!this.stream.isTTY) { + return; + } + this.dy = this.dy + dy; + _readline.moveCursor(this.stream, dx, dy); + } + // relative reset + cursorRelativeReset() { + if (!this.stream.isTTY) { + return; + } + _readline.moveCursor(this.stream, 0, -this.dy); + _readline.cursorTo(this.stream, 0, null); + this.dy = 0; + } + // clear to the right from cursor + clearRight() { + if (!this.stream.isTTY) { + return; + } + _readline.clearLine(this.stream, 1); + } + // clear the full line + clearLine() { + if (!this.stream.isTTY) { + return; + } + _readline.clearLine(this.stream, 0); + } + // clear everyting beyond the current line + clearBottom() { + if (!this.stream.isTTY) { + return; + } + _readline.clearScreenDown(this.stream); + } + // add new line; increment counter + newline() { + this.stream.write("\n"); + this.dy++; + } + // write content to output stream + // @TODO use string-width to strip length + write(s, rawWrite = false) { + if (this.linewrap === true && rawWrite === false) { + this.stream.write(s.substr(0, this.getWidth())); + } else { + this.stream.write(s); + } + } + // control line wrapping + lineWrapping(enabled) { + if (!this.stream.isTTY) { + return; + } + this.linewrap = enabled; + if (enabled) { + this.stream.write("\x1B[?7h"); + } else { + this.stream.write("\x1B[?7l"); + } + } + // tty environment ? + isTTY() { + return this.stream.isTTY === true; + } + // get terminal width + getWidth() { + return this.stream.columns || (this.stream.isTTY ? 80 : 200); + } + }; + module2.exports = Terminal; + } +}); + +// node_modules/.pnpm/ansi-regex@5.0.1/node_modules/ansi-regex/index.js +var require_ansi_regex = __commonJS({ + "node_modules/.pnpm/ansi-regex@5.0.1/node_modules/ansi-regex/index.js"(exports, module2) { + "use strict"; + module2.exports = ({ onlyFirst = false } = {}) => { + const pattern = [ + "[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)", + "(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))" + ].join("|"); + return new RegExp(pattern, onlyFirst ? void 0 : "g"); + }; + } +}); + +// node_modules/.pnpm/strip-ansi@6.0.1/node_modules/strip-ansi/index.js +var require_strip_ansi = __commonJS({ + "node_modules/.pnpm/strip-ansi@6.0.1/node_modules/strip-ansi/index.js"(exports, module2) { + "use strict"; + var ansiRegex = require_ansi_regex(); + module2.exports = (string) => typeof string === "string" ? string.replace(ansiRegex(), "") : string; + } +}); + +// node_modules/.pnpm/is-fullwidth-code-point@3.0.0/node_modules/is-fullwidth-code-point/index.js +var require_is_fullwidth_code_point = __commonJS({ + "node_modules/.pnpm/is-fullwidth-code-point@3.0.0/node_modules/is-fullwidth-code-point/index.js"(exports, module2) { + "use strict"; + var isFullwidthCodePoint = (codePoint) => { + if (Number.isNaN(codePoint)) { + return false; + } + if (codePoint >= 4352 && (codePoint <= 4447 || // Hangul Jamo + codePoint === 9001 || // LEFT-POINTING ANGLE BRACKET + codePoint === 9002 || // RIGHT-POINTING ANGLE BRACKET + // CJK Radicals Supplement .. Enclosed CJK Letters and Months + 11904 <= codePoint && codePoint <= 12871 && codePoint !== 12351 || // Enclosed CJK Letters and Months .. CJK Unified Ideographs Extension A + 12880 <= codePoint && codePoint <= 19903 || // CJK Unified Ideographs .. Yi Radicals + 19968 <= codePoint && codePoint <= 42182 || // Hangul Jamo Extended-A + 43360 <= codePoint && codePoint <= 43388 || // Hangul Syllables + 44032 <= codePoint && codePoint <= 55203 || // CJK Compatibility Ideographs + 63744 <= codePoint && codePoint <= 64255 || // Vertical Forms + 65040 <= codePoint && codePoint <= 65049 || // CJK Compatibility Forms .. Small Form Variants + 65072 <= codePoint && codePoint <= 65131 || // Halfwidth and Fullwidth Forms + 65281 <= codePoint && codePoint <= 65376 || 65504 <= codePoint && codePoint <= 65510 || // Kana Supplement + 110592 <= codePoint && codePoint <= 110593 || // Enclosed Ideographic Supplement + 127488 <= codePoint && codePoint <= 127569 || // CJK Unified Ideographs Extension B .. Tertiary Ideographic Plane + 131072 <= codePoint && codePoint <= 262141)) { + return true; + } + return false; + }; + module2.exports = isFullwidthCodePoint; + module2.exports.default = isFullwidthCodePoint; + } +}); + +// node_modules/.pnpm/emoji-regex@8.0.0/node_modules/emoji-regex/index.js +var require_emoji_regex = __commonJS({ + "node_modules/.pnpm/emoji-regex@8.0.0/node_modules/emoji-regex/index.js"(exports, module2) { + "use strict"; + module2.exports = function() { + return /\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62(?:\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67|\uDB40\uDC73\uDB40\uDC63\uDB40\uDC74|\uDB40\uDC77\uDB40\uDC6C\uDB40\uDC73)\uDB40\uDC7F|\uD83D\uDC68(?:\uD83C\uDFFC\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68\uD83C\uDFFB|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFF\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFE])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFE\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFD])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFD\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB\uDFFC])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\u200D(?:\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D)?\uD83D\uDC68|(?:\uD83D[\uDC68\uDC69])\u200D(?:\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67]))|\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67])|(?:\uD83D[\uDC68\uDC69])\u200D(?:\uD83D[\uDC66\uDC67])|[\u2695\u2696\u2708]\uFE0F|\uD83D[\uDC66\uDC67]|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|(?:\uD83C\uDFFB\u200D[\u2695\u2696\u2708]|\uD83C\uDFFF\u200D[\u2695\u2696\u2708]|\uD83C\uDFFE\u200D[\u2695\u2696\u2708]|\uD83C\uDFFD\u200D[\u2695\u2696\u2708]|\uD83C\uDFFC\u200D[\u2695\u2696\u2708])\uFE0F|\uD83C\uDFFB\u200D(?:\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C[\uDFFB-\uDFFF])|(?:\uD83E\uDDD1\uD83C\uDFFB\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFC\u200D\uD83E\uDD1D\u200D\uD83D\uDC69)\uD83C\uDFFB|\uD83E\uDDD1(?:\uD83C\uDFFF\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1(?:\uD83C[\uDFFB-\uDFFF])|\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1)|(?:\uD83E\uDDD1\uD83C\uDFFE\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFF\u200D\uD83E\uDD1D\u200D(?:\uD83D[\uDC68\uDC69]))(?:\uD83C[\uDFFB-\uDFFE])|(?:\uD83E\uDDD1\uD83C\uDFFC\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFD\u200D\uD83E\uDD1D\u200D\uD83D\uDC69)(?:\uD83C[\uDFFB\uDFFC])|\uD83D\uDC69(?:\uD83C\uDFFE\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFD\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFC\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB\uDFFD-\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFB\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFC-\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFD\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB\uDFFC\uDFFE\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\u200D(?:\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D(?:\uD83D[\uDC68\uDC69])|\uD83D[\uDC68\uDC69])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFF\u200D(?:\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD]))|\uD83D\uDC69\u200D\uD83D\uDC69\u200D(?:\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67]))|(?:\uD83E\uDDD1\uD83C\uDFFD\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFE\u200D\uD83E\uDD1D\u200D\uD83D\uDC69)(?:\uD83C[\uDFFB-\uDFFD])|\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC69\u200D\uD83D\uDC69\u200D(?:\uD83D[\uDC66\uDC67])|(?:\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8|\uD83D\uDC69(?:\uD83C\uDFFF\u200D[\u2695\u2696\u2708]|\uD83C\uDFFE\u200D[\u2695\u2696\u2708]|\uD83C\uDFFC\u200D[\u2695\u2696\u2708]|\uD83C\uDFFB\u200D[\u2695\u2696\u2708]|\uD83C\uDFFD\u200D[\u2695\u2696\u2708]|\u200D[\u2695\u2696\u2708])|(?:(?:\u26F9|\uD83C[\uDFCB\uDFCC]|\uD83D\uDD75)\uFE0F|\uD83D\uDC6F|\uD83E[\uDD3C\uDDDE\uDDDF])\u200D[\u2640\u2642]|(?:\u26F9|\uD83C[\uDFCB\uDFCC]|\uD83D\uDD75)(?:\uD83C[\uDFFB-\uDFFF])\u200D[\u2640\u2642]|(?:\uD83C[\uDFC3\uDFC4\uDFCA]|\uD83D[\uDC6E\uDC71\uDC73\uDC77\uDC81\uDC82\uDC86\uDC87\uDE45-\uDE47\uDE4B\uDE4D\uDE4E\uDEA3\uDEB4-\uDEB6]|\uD83E[\uDD26\uDD37-\uDD39\uDD3D\uDD3E\uDDB8\uDDB9\uDDCD-\uDDCF\uDDD6-\uDDDD])(?:(?:\uD83C[\uDFFB-\uDFFF])\u200D[\u2640\u2642]|\u200D[\u2640\u2642])|\uD83C\uDFF4\u200D\u2620)\uFE0F|\uD83D\uDC69\u200D\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67])|\uD83C\uDFF3\uFE0F\u200D\uD83C\uDF08|\uD83D\uDC15\u200D\uD83E\uDDBA|\uD83D\uDC69\u200D\uD83D\uDC66|\uD83D\uDC69\u200D\uD83D\uDC67|\uD83C\uDDFD\uD83C\uDDF0|\uD83C\uDDF4\uD83C\uDDF2|\uD83C\uDDF6\uD83C\uDDE6|[#\*0-9]\uFE0F\u20E3|\uD83C\uDDE7(?:\uD83C[\uDDE6\uDDE7\uDDE9-\uDDEF\uDDF1-\uDDF4\uDDF6-\uDDF9\uDDFB\uDDFC\uDDFE\uDDFF])|\uD83C\uDDF9(?:\uD83C[\uDDE6\uDDE8\uDDE9\uDDEB-\uDDED\uDDEF-\uDDF4\uDDF7\uDDF9\uDDFB\uDDFC\uDDFF])|\uD83C\uDDEA(?:\uD83C[\uDDE6\uDDE8\uDDEA\uDDEC\uDDED\uDDF7-\uDDFA])|\uD83E\uDDD1(?:\uD83C[\uDFFB-\uDFFF])|\uD83C\uDDF7(?:\uD83C[\uDDEA\uDDF4\uDDF8\uDDFA\uDDFC])|\uD83D\uDC69(?:\uD83C[\uDFFB-\uDFFF])|\uD83C\uDDF2(?:\uD83C[\uDDE6\uDDE8-\uDDED\uDDF0-\uDDFF])|\uD83C\uDDE6(?:\uD83C[\uDDE8-\uDDEC\uDDEE\uDDF1\uDDF2\uDDF4\uDDF6-\uDDFA\uDDFC\uDDFD\uDDFF])|\uD83C\uDDF0(?:\uD83C[\uDDEA\uDDEC-\uDDEE\uDDF2\uDDF3\uDDF5\uDDF7\uDDFC\uDDFE\uDDFF])|\uD83C\uDDED(?:\uD83C[\uDDF0\uDDF2\uDDF3\uDDF7\uDDF9\uDDFA])|\uD83C\uDDE9(?:\uD83C[\uDDEA\uDDEC\uDDEF\uDDF0\uDDF2\uDDF4\uDDFF])|\uD83C\uDDFE(?:\uD83C[\uDDEA\uDDF9])|\uD83C\uDDEC(?:\uD83C[\uDDE6\uDDE7\uDDE9-\uDDEE\uDDF1-\uDDF3\uDDF5-\uDDFA\uDDFC\uDDFE])|\uD83C\uDDF8(?:\uD83C[\uDDE6-\uDDEA\uDDEC-\uDDF4\uDDF7-\uDDF9\uDDFB\uDDFD-\uDDFF])|\uD83C\uDDEB(?:\uD83C[\uDDEE-\uDDF0\uDDF2\uDDF4\uDDF7])|\uD83C\uDDF5(?:\uD83C[\uDDE6\uDDEA-\uDDED\uDDF0-\uDDF3\uDDF7-\uDDF9\uDDFC\uDDFE])|\uD83C\uDDFB(?:\uD83C[\uDDE6\uDDE8\uDDEA\uDDEC\uDDEE\uDDF3\uDDFA])|\uD83C\uDDF3(?:\uD83C[\uDDE6\uDDE8\uDDEA-\uDDEC\uDDEE\uDDF1\uDDF4\uDDF5\uDDF7\uDDFA\uDDFF])|\uD83C\uDDE8(?:\uD83C[\uDDE6\uDDE8\uDDE9\uDDEB-\uDDEE\uDDF0-\uDDF5\uDDF7\uDDFA-\uDDFF])|\uD83C\uDDF1(?:\uD83C[\uDDE6-\uDDE8\uDDEE\uDDF0\uDDF7-\uDDFB\uDDFE])|\uD83C\uDDFF(?:\uD83C[\uDDE6\uDDF2\uDDFC])|\uD83C\uDDFC(?:\uD83C[\uDDEB\uDDF8])|\uD83C\uDDFA(?:\uD83C[\uDDE6\uDDEC\uDDF2\uDDF3\uDDF8\uDDFE\uDDFF])|\uD83C\uDDEE(?:\uD83C[\uDDE8-\uDDEA\uDDF1-\uDDF4\uDDF6-\uDDF9])|\uD83C\uDDEF(?:\uD83C[\uDDEA\uDDF2\uDDF4\uDDF5])|(?:\uD83C[\uDFC3\uDFC4\uDFCA]|\uD83D[\uDC6E\uDC71\uDC73\uDC77\uDC81\uDC82\uDC86\uDC87\uDE45-\uDE47\uDE4B\uDE4D\uDE4E\uDEA3\uDEB4-\uDEB6]|\uD83E[\uDD26\uDD37-\uDD39\uDD3D\uDD3E\uDDB8\uDDB9\uDDCD-\uDDCF\uDDD6-\uDDDD])(?:\uD83C[\uDFFB-\uDFFF])|(?:\u26F9|\uD83C[\uDFCB\uDFCC]|\uD83D\uDD75)(?:\uD83C[\uDFFB-\uDFFF])|(?:[\u261D\u270A-\u270D]|\uD83C[\uDF85\uDFC2\uDFC7]|\uD83D[\uDC42\uDC43\uDC46-\uDC50\uDC66\uDC67\uDC6B-\uDC6D\uDC70\uDC72\uDC74-\uDC76\uDC78\uDC7C\uDC83\uDC85\uDCAA\uDD74\uDD7A\uDD90\uDD95\uDD96\uDE4C\uDE4F\uDEC0\uDECC]|\uD83E[\uDD0F\uDD18-\uDD1C\uDD1E\uDD1F\uDD30-\uDD36\uDDB5\uDDB6\uDDBB\uDDD2-\uDDD5])(?:\uD83C[\uDFFB-\uDFFF])|(?:[\u231A\u231B\u23E9-\u23EC\u23F0\u23F3\u25FD\u25FE\u2614\u2615\u2648-\u2653\u267F\u2693\u26A1\u26AA\u26AB\u26BD\u26BE\u26C4\u26C5\u26CE\u26D4\u26EA\u26F2\u26F3\u26F5\u26FA\u26FD\u2705\u270A\u270B\u2728\u274C\u274E\u2753-\u2755\u2757\u2795-\u2797\u27B0\u27BF\u2B1B\u2B1C\u2B50\u2B55]|\uD83C[\uDC04\uDCCF\uDD8E\uDD91-\uDD9A\uDDE6-\uDDFF\uDE01\uDE1A\uDE2F\uDE32-\uDE36\uDE38-\uDE3A\uDE50\uDE51\uDF00-\uDF20\uDF2D-\uDF35\uDF37-\uDF7C\uDF7E-\uDF93\uDFA0-\uDFCA\uDFCF-\uDFD3\uDFE0-\uDFF0\uDFF4\uDFF8-\uDFFF]|\uD83D[\uDC00-\uDC3E\uDC40\uDC42-\uDCFC\uDCFF-\uDD3D\uDD4B-\uDD4E\uDD50-\uDD67\uDD7A\uDD95\uDD96\uDDA4\uDDFB-\uDE4F\uDE80-\uDEC5\uDECC\uDED0-\uDED2\uDED5\uDEEB\uDEEC\uDEF4-\uDEFA\uDFE0-\uDFEB]|\uD83E[\uDD0D-\uDD3A\uDD3C-\uDD45\uDD47-\uDD71\uDD73-\uDD76\uDD7A-\uDDA2\uDDA5-\uDDAA\uDDAE-\uDDCA\uDDCD-\uDDFF\uDE70-\uDE73\uDE78-\uDE7A\uDE80-\uDE82\uDE90-\uDE95])|(?:[#\*0-9\xA9\xAE\u203C\u2049\u2122\u2139\u2194-\u2199\u21A9\u21AA\u231A\u231B\u2328\u23CF\u23E9-\u23F3\u23F8-\u23FA\u24C2\u25AA\u25AB\u25B6\u25C0\u25FB-\u25FE\u2600-\u2604\u260E\u2611\u2614\u2615\u2618\u261D\u2620\u2622\u2623\u2626\u262A\u262E\u262F\u2638-\u263A\u2640\u2642\u2648-\u2653\u265F\u2660\u2663\u2665\u2666\u2668\u267B\u267E\u267F\u2692-\u2697\u2699\u269B\u269C\u26A0\u26A1\u26AA\u26AB\u26B0\u26B1\u26BD\u26BE\u26C4\u26C5\u26C8\u26CE\u26CF\u26D1\u26D3\u26D4\u26E9\u26EA\u26F0-\u26F5\u26F7-\u26FA\u26FD\u2702\u2705\u2708-\u270D\u270F\u2712\u2714\u2716\u271D\u2721\u2728\u2733\u2734\u2744\u2747\u274C\u274E\u2753-\u2755\u2757\u2763\u2764\u2795-\u2797\u27A1\u27B0\u27BF\u2934\u2935\u2B05-\u2B07\u2B1B\u2B1C\u2B50\u2B55\u3030\u303D\u3297\u3299]|\uD83C[\uDC04\uDCCF\uDD70\uDD71\uDD7E\uDD7F\uDD8E\uDD91-\uDD9A\uDDE6-\uDDFF\uDE01\uDE02\uDE1A\uDE2F\uDE32-\uDE3A\uDE50\uDE51\uDF00-\uDF21\uDF24-\uDF93\uDF96\uDF97\uDF99-\uDF9B\uDF9E-\uDFF0\uDFF3-\uDFF5\uDFF7-\uDFFF]|\uD83D[\uDC00-\uDCFD\uDCFF-\uDD3D\uDD49-\uDD4E\uDD50-\uDD67\uDD6F\uDD70\uDD73-\uDD7A\uDD87\uDD8A-\uDD8D\uDD90\uDD95\uDD96\uDDA4\uDDA5\uDDA8\uDDB1\uDDB2\uDDBC\uDDC2-\uDDC4\uDDD1-\uDDD3\uDDDC-\uDDDE\uDDE1\uDDE3\uDDE8\uDDEF\uDDF3\uDDFA-\uDE4F\uDE80-\uDEC5\uDECB-\uDED2\uDED5\uDEE0-\uDEE5\uDEE9\uDEEB\uDEEC\uDEF0\uDEF3-\uDEFA\uDFE0-\uDFEB]|\uD83E[\uDD0D-\uDD3A\uDD3C-\uDD45\uDD47-\uDD71\uDD73-\uDD76\uDD7A-\uDDA2\uDDA5-\uDDAA\uDDAE-\uDDCA\uDDCD-\uDDFF\uDE70-\uDE73\uDE78-\uDE7A\uDE80-\uDE82\uDE90-\uDE95])\uFE0F|(?:[\u261D\u26F9\u270A-\u270D]|\uD83C[\uDF85\uDFC2-\uDFC4\uDFC7\uDFCA-\uDFCC]|\uD83D[\uDC42\uDC43\uDC46-\uDC50\uDC66-\uDC78\uDC7C\uDC81-\uDC83\uDC85-\uDC87\uDC8F\uDC91\uDCAA\uDD74\uDD75\uDD7A\uDD90\uDD95\uDD96\uDE45-\uDE47\uDE4B-\uDE4F\uDEA3\uDEB4-\uDEB6\uDEC0\uDECC]|\uD83E[\uDD0F\uDD18-\uDD1F\uDD26\uDD30-\uDD39\uDD3C-\uDD3E\uDDB5\uDDB6\uDDB8\uDDB9\uDDBB\uDDCD-\uDDCF\uDDD1-\uDDDD])/g; + }; + } +}); + +// node_modules/.pnpm/string-width@4.2.3/node_modules/string-width/index.js +var require_string_width = __commonJS({ + "node_modules/.pnpm/string-width@4.2.3/node_modules/string-width/index.js"(exports, module2) { + "use strict"; + var stripAnsi = require_strip_ansi(); + var isFullwidthCodePoint = require_is_fullwidth_code_point(); + var emojiRegex = require_emoji_regex(); + var stringWidth = (string) => { + if (typeof string !== "string" || string.length === 0) { + return 0; + } + string = stripAnsi(string); + if (string.length === 0) { + return 0; + } + string = string.replace(emojiRegex(), " "); + let width = 0; + for (let i = 0; i < string.length; i++) { + const code = string.codePointAt(i); + if (code <= 31 || code >= 127 && code <= 159) { + continue; + } + if (code >= 768 && code <= 879) { + continue; + } + if (code > 65535) { + i++; + } + width += isFullwidthCodePoint(code) ? 2 : 1; + } + return width; + }; + module2.exports = stringWidth; + module2.exports.default = stringWidth; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/format-value.js +var require_format_value = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/format-value.js"(exports, module2) { + module2.exports = function formatValue(v, options, type) { + if (options.autopadding !== true) { + return v; + } + function autopadding(value, length) { + return (options.autopaddingChar + value).slice(-length); + } + switch (type) { + case "percentage": + return autopadding(v, 3); + default: + return v; + } + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/format-bar.js +var require_format_bar = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/format-bar.js"(exports, module2) { + module2.exports = function formatBar(progress, options) { + const completeSize = Math.round(progress * options.barsize); + const incompleteSize = options.barsize - completeSize; + return options.barCompleteString.substr(0, completeSize) + options.barGlue + options.barIncompleteString.substr(0, incompleteSize); + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/format-time.js +var require_format_time = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/format-time.js"(exports, module2) { + module2.exports = function formatTime(t, options, roundToMultipleOf) { + function round(input) { + if (roundToMultipleOf) { + return roundToMultipleOf * Math.round(input / roundToMultipleOf); + } else { + return input; + } + } + function autopadding(v) { + return (options.autopaddingChar + v).slice(-2); + } + if (t > 3600) { + return autopadding(Math.floor(t / 3600)) + "h" + autopadding(round(t % 3600 / 60)) + "m"; + } else if (t > 60) { + return autopadding(Math.floor(t / 60)) + "m" + autopadding(round(t % 60)) + "s"; + } else if (t > 10) { + return autopadding(round(t)) + "s"; + } else { + return autopadding(t) + "s"; + } + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/formatter.js +var require_formatter = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/formatter.js"(exports, module2) { + var _stringWidth = require_string_width(); + var _defaultFormatValue = require_format_value(); + var _defaultFormatBar = require_format_bar(); + var _defaultFormatTime = require_format_time(); + module2.exports = function defaultFormatter(options, params, payload) { + let s = options.format; + const formatTime = options.formatTime || _defaultFormatTime; + const formatValue = options.formatValue || _defaultFormatValue; + const formatBar = options.formatBar || _defaultFormatBar; + const percentage = Math.floor(params.progress * 100) + ""; + const stopTime = params.stopTime || Date.now(); + const elapsedTime = Math.round((stopTime - params.startTime) / 1e3); + const context = Object.assign({}, payload, { + bar: formatBar(params.progress, options), + percentage: formatValue(percentage, options, "percentage"), + total: formatValue(params.total, options, "total"), + value: formatValue(params.value, options, "value"), + eta: formatValue(params.eta, options, "eta"), + eta_formatted: formatTime(params.eta, options, 5), + duration: formatValue(elapsedTime, options, "duration"), + duration_formatted: formatTime(elapsedTime, options, 1) + }); + s = s.replace(/\{(\w+)\}/g, function(match, key) { + if (typeof context[key] !== "undefined") { + return context[key]; + } + return match; + }); + const fullMargin = Math.max(0, params.maxWidth - _stringWidth(s) - 2); + const halfMargin = Math.floor(fullMargin / 2); + switch (options.align) { + case "right": + s = fullMargin > 0 ? " ".repeat(fullMargin) + s : s; + break; + case "center": + s = halfMargin > 0 ? " ".repeat(halfMargin) + s : s; + break; + case "left": + default: + break; + } + return s; + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/options.js +var require_options = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/options.js"(exports, module2) { + function mergeOption(v, defaultValue) { + if (typeof v === "undefined" || v === null) { + return defaultValue; + } else { + return v; + } + } + module2.exports = { + // set global options + parse: function parse(rawOptions, preset) { + const options = {}; + const opt = Object.assign({}, preset, rawOptions); + options.throttleTime = 1e3 / mergeOption(opt.fps, 10); + options.stream = mergeOption(opt.stream, process.stderr); + options.terminal = mergeOption(opt.terminal, null); + options.clearOnComplete = mergeOption(opt.clearOnComplete, false); + options.stopOnComplete = mergeOption(opt.stopOnComplete, false); + options.barsize = mergeOption(opt.barsize, 40); + options.align = mergeOption(opt.align, "left"); + options.hideCursor = mergeOption(opt.hideCursor, false); + options.linewrap = mergeOption(opt.linewrap, false); + options.barGlue = mergeOption(opt.barGlue, ""); + options.barCompleteChar = mergeOption(opt.barCompleteChar, "="); + options.barIncompleteChar = mergeOption(opt.barIncompleteChar, "-"); + options.format = mergeOption(opt.format, "progress [{bar}] {percentage}% | ETA: {eta}s | {value}/{total}"); + options.formatTime = mergeOption(opt.formatTime, null); + options.formatValue = mergeOption(opt.formatValue, null); + options.formatBar = mergeOption(opt.formatBar, null); + options.etaBufferLength = mergeOption(opt.etaBuffer, 10); + options.etaAsynchronousUpdate = mergeOption(opt.etaAsynchronousUpdate, false); + options.progressCalculationRelative = mergeOption(opt.progressCalculationRelative, false); + options.synchronousUpdate = mergeOption(opt.synchronousUpdate, true); + options.noTTYOutput = mergeOption(opt.noTTYOutput, false); + options.notTTYSchedule = mergeOption(opt.notTTYSchedule, 2e3); + options.emptyOnZero = mergeOption(opt.emptyOnZero, false); + options.forceRedraw = mergeOption(opt.forceRedraw, false); + options.autopadding = mergeOption(opt.autopadding, false); + options.gracefulExit = mergeOption(opt.gracefulExit, false); + return options; + }, + // derived options: instance specific, has to be created for every bar element + assignDerivedOptions: function assignDerivedOptions(options) { + options.barCompleteString = options.barCompleteChar.repeat(options.barsize + 1); + options.barIncompleteString = options.barIncompleteChar.repeat(options.barsize + 1); + options.autopaddingChar = options.autopadding ? mergeOption(options.autopaddingChar, " ") : ""; + return options; + } + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/generic-bar.js +var require_generic_bar = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/generic-bar.js"(exports, module2) { + var _ETA = require_eta(); + var _Terminal = require_terminal(); + var _formatter = require_formatter(); + var _options = require_options(); + var _EventEmitter = require("events"); + module2.exports = class GenericBar extends _EventEmitter { + constructor(options) { + super(); + this.options = _options.assignDerivedOptions(options); + this.terminal = this.options.terminal ? this.options.terminal : new _Terminal(this.options.stream); + this.value = 0; + this.startValue = 0; + this.total = 100; + this.lastDrawnString = null; + this.startTime = null; + this.stopTime = null; + this.lastRedraw = Date.now(); + this.eta = new _ETA(this.options.etaBufferLength, 0, 0); + this.payload = {}; + this.isActive = false; + this.formatter = typeof this.options.format === "function" ? this.options.format : _formatter; + } + // internal render function + render(forceRendering = false) { + const params = { + progress: this.getProgress(), + eta: this.eta.getTime(), + startTime: this.startTime, + stopTime: this.stopTime, + total: this.total, + value: this.value, + maxWidth: this.terminal.getWidth() + }; + if (this.options.etaAsynchronousUpdate) { + this.updateETA(); + } + const s = this.formatter(this.options, params, this.payload); + const forceRedraw = forceRendering || this.options.forceRedraw || this.options.noTTYOutput && !this.terminal.isTTY(); + if (forceRedraw || this.lastDrawnString != s) { + this.emit("redraw-pre"); + this.terminal.cursorTo(0, null); + this.terminal.write(s); + this.terminal.clearRight(); + this.lastDrawnString = s; + this.lastRedraw = Date.now(); + this.emit("redraw-post"); + } + } + // start the progress bar + start(total, startValue, payload) { + this.value = startValue || 0; + this.total = typeof total !== "undefined" && total >= 0 ? total : 100; + this.startValue = startValue || 0; + this.payload = payload || {}; + this.startTime = Date.now(); + this.stopTime = null; + this.lastDrawnString = ""; + this.eta = new _ETA(this.options.etaBufferLength, this.startTime, this.value); + this.isActive = true; + this.emit("start", total, startValue); + } + // stop the bar + stop() { + this.isActive = false; + this.stopTime = Date.now(); + this.emit("stop", this.total, this.value); + } + // update the bar value + // update(value, payload) + // update(payload) + update(arg0, arg1 = {}) { + if (typeof arg0 === "number") { + this.value = arg0; + this.eta.update(Date.now(), arg0, this.total); + } + const payloadData = (typeof arg0 === "object" ? arg0 : arg1) || {}; + this.emit("update", this.total, this.value); + for (const key in payloadData) { + this.payload[key] = payloadData[key]; + } + if (this.value >= this.getTotal() && this.options.stopOnComplete) { + this.stop(); + } + } + // calculate the actual progress value + getProgress() { + let progress = this.value / this.total; + if (this.options.progressCalculationRelative) { + progress = (this.value - this.startValue) / (this.total - this.startValue); + } + if (isNaN(progress)) { + progress = this.options && this.options.emptyOnZero ? 0 : 1; + } + progress = Math.min(Math.max(progress, 0), 1); + return progress; + } + // update the bar value + // increment(delta, payload) + // increment(payload) + increment(arg0 = 1, arg1 = {}) { + if (typeof arg0 === "object") { + this.update(this.value + 1, arg0); + } else { + this.update(this.value + arg0, arg1); + } + } + // get the total (limit) value + getTotal() { + return this.total; + } + // set the total (limit) value + setTotal(total) { + if (typeof total !== "undefined" && total >= 0) { + this.total = total; + } + } + // force eta calculation update (long running processes) + updateETA() { + this.eta.update(Date.now(), this.value, this.total); + } + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/single-bar.js +var require_single_bar = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/single-bar.js"(exports, module2) { + var _GenericBar = require_generic_bar(); + var _options = require_options(); + module2.exports = class SingleBar extends _GenericBar { + constructor(options, preset) { + super(_options.parse(options, preset)); + this.timer = null; + if (this.options.noTTYOutput && this.terminal.isTTY() === false) { + this.options.synchronousUpdate = false; + } + this.schedulingRate = this.terminal.isTTY() ? this.options.throttleTime : this.options.notTTYSchedule; + this.sigintCallback = null; + } + // internal render function + render() { + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + super.render(); + if (this.options.noTTYOutput && this.terminal.isTTY() === false) { + this.terminal.newline(); + } + this.timer = setTimeout(this.render.bind(this), this.schedulingRate); + } + update(current, payload) { + if (!this.timer) { + return; + } + super.update(current, payload); + if (this.options.synchronousUpdate && this.lastRedraw + this.options.throttleTime * 2 < Date.now()) { + this.render(); + } + } + // start the progress bar + start(total, startValue, payload) { + if (this.options.noTTYOutput === false && this.terminal.isTTY() === false) { + return; + } + if (this.sigintCallback === null && this.options.gracefulExit) { + this.sigintCallback = this.stop.bind(this); + process.once("SIGINT", this.sigintCallback); + process.once("SIGTERM", this.sigintCallback); + } + this.terminal.cursorSave(); + if (this.options.hideCursor === true) { + this.terminal.cursor(false); + } + if (this.options.linewrap === false) { + this.terminal.lineWrapping(false); + } + super.start(total, startValue, payload); + this.render(); + } + // stop the bar + stop() { + if (!this.timer) { + return; + } + if (this.sigintCallback) { + process.removeListener("SIGINT", this.sigintCallback); + process.removeListener("SIGTERM", this.sigintCallback); + this.sigintCallback = null; + } + this.render(); + super.stop(); + clearTimeout(this.timer); + this.timer = null; + if (this.options.hideCursor === true) { + this.terminal.cursor(true); + } + if (this.options.linewrap === false) { + this.terminal.lineWrapping(true); + } + this.terminal.cursorRestore(); + if (this.options.clearOnComplete) { + this.terminal.cursorTo(0, null); + this.terminal.clearLine(); + } else { + this.terminal.newline(); + } + } + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/multi-bar.js +var require_multi_bar = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/lib/multi-bar.js"(exports, module2) { + var _Terminal = require_terminal(); + var _BarElement = require_generic_bar(); + var _options = require_options(); + var _EventEmitter = require("events"); + module2.exports = class MultiBar extends _EventEmitter { + constructor(options, preset) { + super(); + this.bars = []; + this.options = _options.parse(options, preset); + this.options.synchronousUpdate = false; + this.terminal = this.options.terminal ? this.options.terminal : new _Terminal(this.options.stream); + this.timer = null; + this.isActive = false; + this.schedulingRate = this.terminal.isTTY() ? this.options.throttleTime : this.options.notTTYSchedule; + this.loggingBuffer = []; + this.sigintCallback = null; + } + // add a new bar to the stack + create(total, startValue, payload, barOptions = {}) { + const bar = new _BarElement(Object.assign( + {}, + // global options + this.options, + // terminal instance + { + terminal: this.terminal + }, + // overrides + barOptions + )); + this.bars.push(bar); + if (this.options.noTTYOutput === false && this.terminal.isTTY() === false) { + return bar; + } + if (this.sigintCallback === null && this.options.gracefulExit) { + this.sigintCallback = this.stop.bind(this); + process.once("SIGINT", this.sigintCallback); + process.once("SIGTERM", this.sigintCallback); + } + if (!this.isActive) { + if (this.options.hideCursor === true) { + this.terminal.cursor(false); + } + if (this.options.linewrap === false) { + this.terminal.lineWrapping(false); + } + this.timer = setTimeout(this.update.bind(this), this.schedulingRate); + } + this.isActive = true; + bar.start(total, startValue, payload); + this.emit("start"); + return bar; + } + // remove a bar from the stack + remove(bar) { + const index = this.bars.indexOf(bar); + if (index < 0) { + return false; + } + this.bars.splice(index, 1); + this.update(); + this.terminal.newline(); + this.terminal.clearBottom(); + return true; + } + // internal update routine + update() { + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + this.emit("update-pre"); + this.terminal.cursorRelativeReset(); + this.emit("redraw-pre"); + if (this.loggingBuffer.length > 0) { + this.terminal.clearLine(); + while (this.loggingBuffer.length > 0) { + this.terminal.write(this.loggingBuffer.shift(), true); + } + } + for (let i = 0; i < this.bars.length; i++) { + if (i > 0) { + this.terminal.newline(); + } + this.bars[i].render(); + } + this.emit("redraw-post"); + if (this.options.noTTYOutput && this.terminal.isTTY() === false) { + this.terminal.newline(); + this.terminal.newline(); + } + this.timer = setTimeout(this.update.bind(this), this.schedulingRate); + this.emit("update-post"); + if (this.options.stopOnComplete && !this.bars.find((bar) => bar.isActive)) { + this.stop(); + } + } + stop() { + clearTimeout(this.timer); + this.timer = null; + if (this.sigintCallback) { + process.removeListener("SIGINT", this.sigintCallback); + process.removeListener("SIGTERM", this.sigintCallback); + this.sigintCallback = null; + } + this.isActive = false; + if (this.options.hideCursor === true) { + this.terminal.cursor(true); + } + if (this.options.linewrap === false) { + this.terminal.lineWrapping(true); + } + this.terminal.cursorRelativeReset(); + this.emit("stop-pre-clear"); + if (this.options.clearOnComplete) { + this.terminal.clearBottom(); + } else { + for (let i = 0; i < this.bars.length; i++) { + if (i > 0) { + this.terminal.newline(); + } + this.bars[i].render(); + this.bars[i].stop(); + } + this.terminal.newline(); + } + this.emit("stop"); + } + log(s) { + this.loggingBuffer.push(s); + } + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/legacy.js +var require_legacy = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/legacy.js"(exports, module2) { + module2.exports = { + format: "progress [{bar}] {percentage}% | ETA: {eta}s | {value}/{total}", + barCompleteChar: "=", + barIncompleteChar: "-" + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/shades-classic.js +var require_shades_classic = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/shades-classic.js"(exports, module2) { + module2.exports = { + format: " {bar} {percentage}% | ETA: {eta}s | {value}/{total}", + barCompleteChar: "\u2588", + barIncompleteChar: "\u2591" + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/shades-grey.js +var require_shades_grey = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/shades-grey.js"(exports, module2) { + module2.exports = { + format: " \x1B[90m{bar}\x1B[0m {percentage}% | ETA: {eta}s | {value}/{total}", + barCompleteChar: "\u2588", + barIncompleteChar: "\u2591" + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/rect.js +var require_rect = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/rect.js"(exports, module2) { + module2.exports = { + format: " {bar}\u25A0 {percentage}% | ETA: {eta}s | {value}/{total}", + barCompleteChar: "\u25A0", + barIncompleteChar: " " + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/index.js +var require_presets = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/presets/index.js"(exports, module2) { + var _legacy = require_legacy(); + var _shades_classic = require_shades_classic(); + var _shades_grey = require_shades_grey(); + var _rect = require_rect(); + module2.exports = { + legacy: _legacy, + shades_classic: _shades_classic, + shades_grey: _shades_grey, + rect: _rect + }; + } +}); + +// node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/cli-progress.js +var require_cli_progress = __commonJS({ + "node_modules/.pnpm/cli-progress@3.12.0/node_modules/cli-progress/cli-progress.js"(exports, module2) { + var _SingleBar = require_single_bar(); + var _MultiBar = require_multi_bar(); + var _Presets = require_presets(); + var _Formatter = require_formatter(); + var _defaultFormatValue = require_format_value(); + var _defaultFormatBar = require_format_bar(); + var _defaultFormatTime = require_format_time(); + module2.exports = { + Bar: _SingleBar, + SingleBar: _SingleBar, + MultiBar: _MultiBar, + Presets: _Presets, + Format: { + Formatter: _Formatter, + BarFormat: _defaultFormatBar, + ValueFormat: _defaultFormatValue, + TimeFormat: _defaultFormatTime + } + }; + } +}); + +// cli.ts +var import_node_util = require("util"); + +// src/utils/typedEntries.ts +function typedEntries(obj) { + return Object.entries(obj); +} + +// src/lib/cache-management.ts +var import_node_os = require("os"); +var import_node_path = require("path"); +var import_promises = require("fs/promises"); + +// src/consts.ts +var HUB_URL = "https://huggingface.co"; + +// src/error.ts +async function createApiError(response, opts) { + const error = new HubApiError(response.url, response.status, response.headers.get("X-Request-Id") ?? opts?.requestId); + error.message = `Api error with status ${error.statusCode}${opts?.message ? `. ${opts.message}` : ""}`; + const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : void 0].filter(Boolean).join(". "); + if (response.headers.get("Content-Type")?.startsWith("application/json")) { + const json = await response.json(); + error.message = json.error || json.message || error.message; + if (json.error_description) { + error.message = error.message ? error.message + `: ${json.error_description}` : json.error_description; + } + error.data = json; + } else { + error.data = { message: await response.text() }; + } + error.message += `. ${trailer}`; + throw error; +} +var HubApiError = class extends Error { + statusCode; + url; + requestId; + data; + constructor(url, statusCode, requestId, message) { + super(message); + this.statusCode = statusCode; + this.requestId = requestId; + this.url = url; + } +}; +var InvalidApiResponseFormatError = class extends Error { +}; + +// src/utils/checkCredentials.ts +function checkAccessToken(accessToken) { + if (!accessToken.startsWith("hf_")) { + throw new TypeError("Your access token must start with 'hf_'"); + } +} +function checkCredentials(params) { + if (params.accessToken) { + checkAccessToken(params.accessToken); + return params.accessToken; + } + if (params.credentials?.accessToken) { + checkAccessToken(params.credentials.accessToken); + return params.credentials.accessToken; + } +} + +// src/utils/toRepoId.ts +function toRepoId(repo) { + if (typeof repo !== "string") { + return repo; + } + if (repo.startsWith("model/") || repo.startsWith("models/")) { + throw new TypeError( + "A repo designation for a model should not start with 'models/', directly specify the model namespace / name" + ); + } + if (repo.startsWith("space/")) { + throw new TypeError("Spaces should start with 'spaces/', plural, not 'space/'"); + } + if (repo.startsWith("dataset/")) { + throw new TypeError("Datasets should start with 'datasets/', plural, not 'dataset/'"); + } + if (repo.startsWith("bucket/")) { + throw new TypeError("Buckets should start with 'buckets/', plural, not 'bucket/'"); + } + if (repo.startsWith("kernel/")) { + throw new TypeError("Kernels should start with 'kernels/', plural, not 'kernel/'"); + } + const slashes = repo.split("/").length - 1; + if (repo.startsWith("spaces/")) { + if (slashes !== 2) { + throw new TypeError("Space Id must include namespace and name of the space"); + } + return { + type: "space", + name: repo.slice("spaces/".length) + }; + } + if (repo.startsWith("datasets/")) { + if (slashes > 2) { + throw new TypeError("Too many slashes in repo designation: " + repo); + } + return { + type: "dataset", + name: repo.slice("datasets/".length) + }; + } + if (repo.startsWith("buckets/")) { + if (slashes !== 2) { + throw new TypeError("Bucket Id must include namespace and name of the bucket"); + } + return { + type: "bucket", + name: repo.slice("buckets/".length) + }; + } + if (repo.startsWith("kernels/")) { + if (slashes !== 2) { + throw new TypeError("Kernel Id must include namespace and name of the kernel"); + } + return { + type: "kernel", + name: repo.slice("kernels/".length) + }; + } + if (slashes > 1) { + throw new TypeError("Too many slashes in repo designation: " + repo); + } + return { + type: "model", + name: repo + }; +} + +// src/utils/range.ts +function range(n, b) { + return b ? Array(b - n).fill(0).map((_, i) => n + i) : Array(n).fill(0).map((_, i) => i); +} + +// src/utils/chunk.ts +function chunk(arr, chunkSize) { + if (isNaN(chunkSize) || chunkSize < 1) { + throw new RangeError("Invalid chunk size: " + chunkSize); + } + if (!arr.length) { + return []; + } + if (arr.length <= chunkSize) { + return [arr]; + } + return range(Math.ceil(arr.length / chunkSize)).map((i) => { + return arr.slice(i * chunkSize, (i + 1) * chunkSize); + }); +} + +// src/utils/promisesQueue.ts +async function promisesQueue(factories, concurrency) { + const results = []; + const executing = /* @__PURE__ */ new Set(); + let index = 0; + for (const factory of factories) { + const closureIndex = index++; + const e = factory().then((r) => { + results[closureIndex] = r; + executing.delete(e); + }); + executing.add(e); + if (executing.size >= concurrency) { + await Promise.race(executing); + } + } + await Promise.all(executing); + return results; +} + +// src/utils/promisesQueueStreaming.ts +async function promisesQueueStreaming(factories, concurrency) { + const executing = []; + for await (const factory of factories) { + const e = factory().then(() => { + executing.splice(executing.indexOf(e), 1); + }); + executing.push(e); + if (executing.length >= concurrency) { + await Promise.race(executing); + } + } + await Promise.all(executing); +} + +// src/utils/eventToGenerator.ts +async function* eventToGenerator(cb) { + const promises = []; + function addPromise() { + let resolve3; + let reject; + const p = new Promise((res, rej) => { + resolve3 = res; + reject = rej; + }); + promises.push({ p, resolve: resolve3, reject }); + } + addPromise(); + const callbackRes = Promise.resolve().then( + () => cb( + (y) => { + addPromise(); + promises.at(-2)?.resolve({ done: false, value: y }); + }, + (r) => { + addPromise(); + promises.at(-2)?.resolve({ done: true, value: r }); + }, + (err) => promises.shift()?.reject(err) + ) + ).catch((err) => promises.shift()?.reject(err)); + while (1) { + const p = promises[0]; + if (!p) { + throw new Error("Logic error in eventGenerator, promises should never be empty"); + } + const result = await p.p; + promises.shift(); + if (result.done) { + await callbackRes; + return result.value; + } + yield result.value; + } + throw new Error("Unreachable"); +} + +// src/utils/hexFromBytes.ts +function hexFromBytes(arr) { + if (globalThis.Buffer) { + return globalThis.Buffer.from(arr).toString("hex"); + } else { + const bin = []; + arr.forEach((byte) => { + bin.push(byte.toString(16).padStart(2, "0")); + }); + return bin.join(""); + } +} + +// src/utils/isBackend.ts +var isBrowser = typeof window !== "undefined" && typeof window.document !== "undefined"; +var isWebWorker = typeof self === "object" && self.constructor && self.constructor.name === "DedicatedWorkerGlobalScope"; +var isBackend = !isBrowser && !isWebWorker; + +// src/utils/isFrontend.ts +var isFrontend = !isBackend; + +// src/utils/sha256.ts +async function getWebWorkerCode() { + const sha256Module = await Promise.resolve().then(() => (init_sha256_wrapper(), sha256_wrapper_exports)); + return URL.createObjectURL(new Blob([sha256Module.createSHA256WorkerCode()])); +} +var pendingWorkers = []; +var runningWorkers = /* @__PURE__ */ new Set(); +var resolve; +var waitPromise = new Promise((r) => { + resolve = r; +}); +async function getWorker(poolSize) { + { + const worker2 = pendingWorkers.pop(); + if (worker2) { + runningWorkers.add(worker2); + return worker2; + } + } + if (!poolSize) { + const worker2 = new Worker(await getWebWorkerCode()); + runningWorkers.add(worker2); + return worker2; + } + if (poolSize <= 0) { + throw new TypeError("Invalid webworker pool size: " + poolSize); + } + while (runningWorkers.size >= poolSize) { + await waitPromise; + } + const worker = new Worker(await getWebWorkerCode()); + runningWorkers.add(worker); + return worker; +} +async function freeWorker(worker, poolSize) { + if (!poolSize) { + return destroyWorker(worker); + } + runningWorkers.delete(worker); + pendingWorkers.push(worker); + const r = resolve; + waitPromise = new Promise((r2) => { + resolve = r2; + }); + r(); +} +function destroyWorker(worker) { + runningWorkers.delete(worker); + worker.terminate(); + const r = resolve; + waitPromise = new Promise((r2) => { + resolve = r2; + }); + r(); +} +async function* sha256(buffer, opts) { + yield 0; + const maxCryptoSize = typeof opts?.useWebWorker === "object" && opts?.useWebWorker.minSize !== void 0 ? opts.useWebWorker.minSize : 1e7; + if (buffer.size < maxCryptoSize && globalThis.crypto?.subtle) { + const res = hexFromBytes( + new Uint8Array( + await globalThis.crypto.subtle.digest("SHA-256", buffer instanceof Blob ? await buffer.arrayBuffer() : buffer) + ) + ); + yield 1; + return res; + } + if (isFrontend) { + if (opts?.useWebWorker) { + try { + const poolSize = typeof opts?.useWebWorker === "object" ? opts.useWebWorker.poolSize : void 0; + const worker = await getWorker(poolSize); + let messageHandler; + let errorHandler; + const cleanup = () => { + worker.removeEventListener("message", messageHandler); + worker.removeEventListener("error", errorHandler); + }; + return yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + messageHandler = (event) => { + if (event.data.sha256) { + cleanup(); + freeWorker(worker, poolSize); + returnCallback(event.data.sha256); + } else if (event.data.progress) { + yieldCallback(event.data.progress); + try { + opts.abortSignal?.throwIfAborted(); + } catch (err) { + cleanup(); + destroyWorker(worker); + rejectCallback(err); + } + } else { + cleanup(); + destroyWorker(worker); + rejectCallback(event); + } + }; + errorHandler = (event) => { + cleanup(); + destroyWorker(worker); + rejectCallback(event.error); + }; + if (opts?.abortSignal) { + try { + opts.abortSignal?.throwIfAborted(); + } catch (err) { + cleanup(); + destroyWorker(worker); + rejectCallback(opts.abortSignal.reason ?? new DOMException("Aborted", "AbortError")); + return; + } + const abortListener = () => { + cleanup(); + destroyWorker(worker); + rejectCallback(opts.abortSignal?.reason ?? new DOMException("Aborted", "AbortError")); + opts.abortSignal?.removeEventListener("abort", abortListener); + }; + opts.abortSignal.addEventListener("abort", abortListener); + } + worker.addEventListener("message", messageHandler); + worker.addEventListener("error", errorHandler); + worker.postMessage({ file: buffer }); + }); + } catch (err) { + console.warn("Failed to use web worker for sha256", err); + } + } + if (!wasmModule) { + wasmModule = await Promise.resolve().then(() => (init_sha256_wrapper(), sha256_wrapper_exports)); + } + const sha2562 = await wasmModule.createSHA256(); + sha2562.init(); + const reader = buffer.stream().getReader(); + const total = buffer.size; + let bytesDone = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + sha2562.update(value); + bytesDone += value.length; + yield bytesDone / total; + opts?.abortSignal?.throwIfAborted(); + } + return sha2562.digest("hex"); + } + if (!cryptoModule) { + cryptoModule = await Promise.resolve().then(() => (init_sha256_node(), sha256_node_exports)); + } + return yield* cryptoModule.sha256Node(buffer, { abortSignal: opts?.abortSignal }); +} +var cryptoModule; +var wasmModule; + +// src/utils/WebBlob.ts +var WebBlob = class extends Blob { + static async create(url, opts) { + const customFetch = opts?.fetch ?? fetch; + const probe = await customFetch(url, { + headers: { + Range: "bytes=0-0", + ...opts?.accessToken && { Authorization: `Bearer ${opts.accessToken}` } + } + }); + if (!probe.ok) { + throw await createApiError(probe); + } + const contentType = probe.headers.get("content-type") || ""; + if (probe.status === 206) { + const totalSize = Number(probe.headers.get("content-range")?.split("/").pop()); + await probe.body?.cancel(); + if (Number.isFinite(totalSize) && totalSize >= (opts?.cacheBelow ?? 1e6)) { + return new WebBlob(url, 0, totalSize, contentType, true, customFetch, opts?.accessToken); + } + const full = await customFetch(url, { + ...opts?.accessToken && { headers: { Authorization: `Bearer ${opts.accessToken}` } } + }); + if (!full.ok) { + throw await createApiError(full); + } + return full.blob(); + } + return probe.blob(); + } + url; + start; + end; + contentType; + full; + fetch; + accessToken; + constructor(url, start, end, contentType, full, customFetch, accessToken) { + super([]); + this.url = url; + this.start = start; + this.end = end; + this.contentType = contentType; + this.full = full; + this.fetch = customFetch; + this.accessToken = accessToken; + } + get size() { + return this.end - this.start; + } + get type() { + return this.contentType; + } + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on WebBlob.slice"); + } + const slice = new WebBlob( + this.url, + this.start + start, + Math.min(this.start + end, this.end), + this.contentType, + start === 0 && end === this.size ? this.full : false, + this.fetch, + this.accessToken + ); + return slice; + } + async arrayBuffer() { + const result = await this.fetchRange(); + return result.arrayBuffer(); + } + async text() { + const result = await this.fetchRange(); + return result.text(); + } + stream() { + const stream = new TransformStream(); + this.fetchRange().then((response) => response.body?.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message)); + return stream.readable; + } + fetchRange() { + const fetch2 = this.fetch; + if (this.full) { + return fetch2(this.url, { + ...this.accessToken && { + headers: { + Authorization: `Bearer ${this.accessToken}` + } + } + }).then((resp) => resp.ok ? resp : createApiError(resp)); + } + return fetch2(this.url, { + headers: { + Range: `bytes=${this.start}-${this.end - 1}`, + ...this.accessToken && { Authorization: `Bearer ${this.accessToken}` } + } + }).then((resp) => resp.ok ? resp : createApiError(resp)); + } +}; + +// src/utils/base64FromBytes.ts +function base64FromBytes(arr) { + if (globalThis.Buffer) { + return globalThis.Buffer.from(arr).toString("base64"); + } else { + const bin = []; + arr.forEach((byte) => { + bin.push(String.fromCharCode(byte)); + }); + return globalThis.btoa(bin.join("")); + } +} + +// src/utils/createBlobs.ts +async function createBlobs(url, destPath, opts) { + if (url.protocol === "http:" || url.protocol === "https:") { + const blob = await WebBlob.create(url, { fetch: opts?.fetch, accessToken: opts?.accessToken }); + return [{ path: destPath, blob }]; + } + if (isFrontend) { + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); + } + if (url.protocol === "file:") { + const { FileBlob: FileBlob2 } = await Promise.resolve().then(() => (init_FileBlob(), FileBlob_exports)); + const { subPaths: subPaths2 } = await Promise.resolve().then(() => (init_sub_paths(), sub_paths_exports)); + const paths = await subPaths2(url, opts?.maxFolderDepth); + if (paths.length === 1 && paths[0].relativePath === ".") { + const blob = await FileBlob2.create(url); + return [{ path: destPath, blob }]; + } + return Promise.all( + paths.map(async (path2) => ({ + path: `${destPath}/${path2.relativePath}`.replace(/\/[.]$/, "").replaceAll("//", "/").replace(/^[.]?\//, ""), + blob: await FileBlob2.create(new URL(path2.path)) + })) + ); + } + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); +} + +// src/vendor/lz4js/util.ts +function hashU32(a) { + a = a | 0; + a = a + 2127912214 + (a << 12) | 0; + a = a ^ -949894596 ^ a >>> 19; + a = a + 374761393 + (a << 5) | 0; + a = a + -744332180 ^ a << 9; + a = a + -42973499 + (a << 3) | 0; + return a ^ -1252372727 ^ a >>> 16 | 0; +} +function readU32(b, n) { + let x = 0; + x |= b[n++] << 0; + x |= b[n++] << 8; + x |= b[n++] << 16; + x |= b[n++] << 24; + return x; +} +function writeU32(b, n, x) { + b[n++] = x >> 0 & 255; + b[n++] = x >> 8 & 255; + b[n++] = x >> 16 & 255; + b[n++] = x >> 24 & 255; +} +function imul(a, b) { + const ah = a >>> 16; + const al = a & 65535; + const bh = b >>> 16; + const bl = b & 65535; + return al * bl + (ah * bl + al * bh << 16) | 0; +} + +// src/vendor/lz4js/xxh32.ts +var prime1 = 2654435761; +var prime2 = 2246822519; +var prime3 = 3266489917; +var prime4 = 668265263; +var prime5 = 374761393; +function rotl32(x, r) { + x = x | 0; + r = r | 0; + return x >>> (32 - r | 0) | x << r | 0; +} +function rotmul32(h, r, m) { + h = h | 0; + r = r | 0; + m = m | 0; + return imul(h >>> (32 - r | 0) | h << r, m) | 0; +} +function shiftxor32(h, s) { + h = h | 0; + s = s | 0; + return h >>> s ^ h | 0; +} +function xxhapply(h, src, m0, s, m1) { + return rotmul32(imul(src, m0) + h, s, m1); +} +function xxh1(h, src, index) { + return rotmul32(h + imul(src[index], prime5), 11, prime1); +} +function xxh4(h, src, index) { + return xxhapply(h, readU32(src, index), prime3, 17, prime4); +} +function xxh16(h, src, index) { + return [ + xxhapply(h[0], readU32(src, index + 0), prime2, 13, prime1), + xxhapply(h[1], readU32(src, index + 4), prime2, 13, prime1), + xxhapply(h[2], readU32(src, index + 8), prime2, 13, prime1), + xxhapply(h[3], readU32(src, index + 12), prime2, 13, prime1) + ]; +} +function xxh32(seed, src, index, len) { + let h; + const l = len; + if (len >= 16) { + h = [seed + prime1 + prime2, seed + prime2, seed, seed - prime1]; + while (len >= 16) { + h = xxh16(h, src, index); + index += 16; + len -= 16; + } + h = rotl32(h[0], 1) + rotl32(h[1], 7) + rotl32(h[2], 12) + rotl32(h[3], 18) + l; + } else { + h = seed + prime5 + len >>> 0; + } + while (len >= 4) { + h = xxh4(h, src, index); + index += 4; + len -= 4; + } + while (len > 0) { + h = xxh1(h, src, index); + index++; + len--; + } + h = shiftxor32(imul(shiftxor32(imul(shiftxor32(h, 15), prime2), 13), prime3), 16); + return h >>> 0; +} +var hash = xxh32; + +// src/vendor/lz4js/index.ts +var minMatch = 4; +var matchSearchLimit = 12; +var minTrailingLitterals = 5; +var skipTrigger = 6; +var hashSize = 1 << 16; +var mlBits = 4; +var mlMask = (1 << mlBits) - 1; +var runBits = 4; +var runMask = (1 << runBits) - 1; +var blockBuf = makeBuffer(5 << 20); +var hashTable = makeHashTable(); +var magicNum = 407708164; +var fdVersion = 64; +var bsDefault = 7; +var bsShift = 4; +var bsMap = { + 4: 65536, + 5: 262144, + 6: 1048576, + 7: 4194304 +}; +function makeHashTable() { + try { + return new Uint32Array(hashSize); + } catch (error) { + const hashTable2 = new Array(hashSize); + for (let i = 0; i < hashSize; i++) { + hashTable2[i] = 0; + } + return hashTable2; + } +} +function clearHashTable(table) { + for (let i = 0; i < hashSize; i++) { + table[i] = 0; + } +} +function makeBuffer(size) { + return new Uint8Array(size); +} +function sliceArray(array, start, end) { + return array.slice(start, end); +} +function compressBound(n) { + return n + n / 255 + 16 | 0; +} +function compressBlock(src, dst, sIndex, sLength, hashTable2) { + let mIndex, mAnchor, mLength, mOffset, mStep; + let literalCount, dIndex, sEnd, n; + dIndex = 0; + sEnd = sLength + sIndex; + mAnchor = sIndex; + let searchMatchCount = (1 << skipTrigger) + 3; + while (sIndex <= sEnd - matchSearchLimit) { + const seq = readU32(src, sIndex); + let hash2 = hashU32(seq) >>> 0; + hash2 = (hash2 >> 16 ^ hash2) >>> 0 & 65535; + mIndex = hashTable2[hash2] - 1; + hashTable2[hash2] = sIndex + 1; + if (mIndex < 0 || sIndex - mIndex >>> 16 > 0 || readU32(src, mIndex) !== seq) { + mStep = searchMatchCount++ >> skipTrigger; + sIndex += mStep; + continue; + } + searchMatchCount = (1 << skipTrigger) + 3; + literalCount = sIndex - mAnchor; + mOffset = sIndex - mIndex; + sIndex += minMatch; + mIndex += minMatch; + mLength = sIndex; + while (sIndex < sEnd - minTrailingLitterals && src[sIndex] === src[mIndex]) { + sIndex++; + mIndex++; + } + mLength = sIndex - mLength; + const token = mLength < mlMask ? mLength : mlMask; + if (literalCount >= runMask) { + dst[dIndex++] = (runMask << mlBits) + token; + for (n = literalCount - runMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } else { + dst[dIndex++] = (literalCount << mlBits) + token; + } + for (let i = 0; i < literalCount; i++) { + dst[dIndex++] = src[mAnchor + i]; + } + dst[dIndex++] = mOffset; + dst[dIndex++] = mOffset >> 8; + if (mLength >= mlMask) { + for (n = mLength - mlMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } + mAnchor = sIndex; + } + if (mAnchor === 0) { + return 0; + } + literalCount = sEnd - mAnchor; + if (literalCount >= runMask) { + dst[dIndex++] = runMask << mlBits; + for (n = literalCount - runMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } else { + dst[dIndex++] = literalCount << mlBits; + } + sIndex = mAnchor; + while (sIndex < sEnd) { + dst[dIndex++] = src[sIndex++]; + } + return dIndex; +} +function compressFrame(src, dst) { + let dIndex = 0; + writeU32(dst, dIndex, magicNum); + dIndex += 4; + dst[dIndex++] = fdVersion; + dst[dIndex++] = bsDefault << bsShift; + dst[dIndex] = hash(0, dst, 4, dIndex - 4) >> 8; + dIndex++; + const maxBlockSize = bsMap[bsDefault]; + let remaining = src.length; + let sIndex = 0; + clearHashTable(hashTable); + while (remaining > 0) { + let compSize = 0; + const blockSize = remaining > maxBlockSize ? maxBlockSize : remaining; + compSize = compressBlock(src, blockBuf, sIndex, blockSize, hashTable); + if (compSize > blockSize || compSize === 0) { + writeU32(dst, dIndex, 2147483648 | blockSize); + dIndex += 4; + for (let z = sIndex + blockSize; sIndex < z; ) { + dst[dIndex++] = src[sIndex++]; + } + remaining -= blockSize; + } else { + writeU32(dst, dIndex, compSize); + dIndex += 4; + for (let j = 0; j < compSize; ) { + dst[dIndex++] = blockBuf[j++]; + } + sIndex += blockSize; + remaining -= blockSize; + } + } + writeU32(dst, dIndex, 0); + dIndex += 4; + return dIndex; +} +function compress(src, maxSize) { + let dst, size; + if (maxSize === void 0) { + maxSize = compressBound(src.length); + } + dst = makeBuffer(maxSize); + size = compressFrame(src, dst); + if (size !== maxSize) { + dst = sliceArray(dst, 0, size); + } + return dst; +} + +// src/utils/XetBlob.ts +var XET_CHUNK_HEADER_BYTES = 8; +function bg4_split_bytes(bytes) { + const ret = new Uint8Array(bytes.byteLength); + const split = Math.floor(bytes.byteLength / 4); + const rem = bytes.byteLength % 4; + const g1_pos = split + (rem >= 1 ? 1 : 0); + const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); + const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); + for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + return ret; +} + +// src/utils/ChunkCache.ts +var CHUNK_CACHE_INITIAL_SIZE = 1e4; +var CHUNK_CACHE_GROW_FACTOR = 1.5; +var CHUNK_CACHE_MAX_SIZE = 1e6; +var ChunkCache = class { + index = 0; + // Index >= 0 means local xorb, < 0 means remote xorb + xorbIndices; + // Max 8K chunks per xorb, less than 64K uint16_t + chunkIndices; + map = /* @__PURE__ */ new Map(); + // hash -> chunkCacheIndex. Less overhead that way, empty object is 60+B and empty array is 40+B + hmacs = /* @__PURE__ */ new Set(); + // todo : remove old hmacs + maxSize; + constructor(maxSize = CHUNK_CACHE_MAX_SIZE) { + if (maxSize < 1) { + throw new Error("maxSize must be at least 1"); + } + this.maxSize = maxSize; + this.xorbIndices = new Int32Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); + this.chunkIndices = new Uint16Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); + } + addChunkToCache(hash2, xorbIndex, chunkIndex, hmac2) { + if (this.map.has(hash2)) { + return; + } + if (this.map.values().next().value === this.index) { + this.map.delete(this.map.keys().next().value); + } + this.map.set(hash2, this.index); + if (hmac2 !== null) { + this.hmacs.add(hmac2); + } + if (this.index >= this.xorbIndices.length) { + const oldXorbIndices = this.xorbIndices; + const oldChunkIndices = this.chunkIndices; + this.xorbIndices = new Int32Array(Math.min(this.xorbIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); + this.chunkIndices = new Uint16Array(Math.min(this.chunkIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); + this.xorbIndices.set(oldXorbIndices); + this.chunkIndices.set(oldChunkIndices); + } + this.xorbIndices[this.index] = xorbIndex; + this.chunkIndices[this.index] = chunkIndex; + this.index = (this.index + 1) % this.maxSize; + } + getChunk(hash2, hmacFunction) { + let index = this.map.get(hash2); + if (index === void 0 && hmacFunction !== null) { + for (const hmac2 of this.hmacs) { + index = this.map.get(hmacFunction(hash2, hmac2)); + if (index !== void 0) { + break; + } + } + } + if (index === void 0) { + return void 0; + } + return { + xorbIndex: this.xorbIndices[index], + chunkIndex: this.chunkIndices[index] + }; + } + updateChunkIndex(hash2, chunkIndex) { + const index = this.map.get(hash2); + if (index === void 0) { + throw new Error(`Chunk not found in cache: ${hash2}`); + } + this.chunkIndices[index] = chunkIndex; + } + removeChunkFromCache(hash2) { + this.map.delete(hash2); + } +}; + +// src/utils/xetWriteToken.ts +var JWT_SAFETY_PERIOD = 6e4; +var JWT_CACHE_SIZE = 1e3; +var jwtPromises = /* @__PURE__ */ new Map(); +var jwts = /* @__PURE__ */ new Map(); +async function xetWriteToken(params) { + if (params.xetParams.expiresAt && params.xetParams.casUrl && params.xetParams.accessToken && params.xetParams.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD)) { + return { accessToken: params.xetParams.accessToken, casUrl: params.xetParams.casUrl }; + } + const key = params.xetParams.refreshWriteTokenUrl; + const jwt = jwts.get(key); + if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD)) { + return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; + } + const existingPromise = jwtPromises.get(key); + if (existingPromise) { + return existingPromise; + } + const promise = (async () => { + const resp = await (params.fetch ?? fetch)(params.xetParams.refreshWriteTokenUrl, { + headers: { + ...params.accessToken ? { + Authorization: `Bearer ${params.accessToken}` + } : {}, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } + const json = await resp.json(); + const jwt2 = { + accessToken: json.accessToken, + expiresAt: new Date(json.exp * 1e3), + casUrl: json.casUrl + }; + jwtPromises.delete(key); + for (const [key2, value] of jwts.entries()) { + if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD)) { + jwts.delete(key2); + } else { + break; + } + } + if (jwts.size >= JWT_CACHE_SIZE) { + const keyToDelete = jwts.keys().next().value; + if (keyToDelete) { + jwts.delete(keyToDelete); + } + } + jwts.set(key, jwt2); + return { + accessToken: json.accessToken, + casUrl: json.casUrl + }; + })(); + jwtPromises.set(key, promise); + return promise; +} + +// src/utils/shardParser.ts +var HASH_LENGTH = 32; +var XORB_HASH_BOOKEND = "ff".repeat(HASH_LENGTH); +function readHashFromArray(array, offset) { + let hash2 = ""; + for (let i = 0; i < HASH_LENGTH; i += 8) { + hash2 += `${array[offset + i + 7].toString(16).padStart(2, "0")}${array[offset + i + 6].toString(16).padStart(2, "0")}${array[offset + i + 5].toString(16).padStart(2, "0")}${array[offset + i + 4].toString(16).padStart(2, "0")}${array[offset + i + 3].toString(16).padStart(2, "0")}${array[offset + i + 2].toString(16).padStart(2, "0")}${array[offset + i + 1].toString(16).padStart(2, "0")}${array[offset + i].toString(16).padStart(2, "0")}`; + } + return hash2; +} +async function parseShardData(shardBlob) { + const shard = new Uint8Array(await shardBlob.arrayBuffer()); + const shardView = new DataView(shard.buffer); + const magicTag = shard.slice(0, SHARD_MAGIC_TAG.length); + if (!magicTag.every((byte, i) => byte === SHARD_MAGIC_TAG[i])) { + throw new Error("Invalid shard magic tag"); + } + const version2 = shardView.getBigUint64(SHARD_MAGIC_TAG.length, true); + if (version2 !== SHARD_HEADER_VERSION) { + throw new Error(`Invalid shard version: ${version2}`); + } + const footerSize = Number(shardView.getBigUint64(SHARD_MAGIC_TAG.length + 8, true)); + const footerStart = shard.length - footerSize; + const footerVersion = shardView.getBigUint64(footerStart, true); + if (footerVersion !== SHARD_FOOTER_VERSION) { + throw new Error(`Invalid shard footer version: ${footerVersion}`); + } + const xorbInfoStart = Number(shardView.getBigUint64(footerStart + 16, true)); + const fileLookupStart = Number(shardView.getBigUint64(footerStart + 24, true)); + const hmacKey = readHashFromArray(shard, footerStart + 72); + const xorbs = []; + let offset = xorbInfoStart; + while (offset < fileLookupStart) { + const xorbHash2 = readHashFromArray(shard, offset); + offset += HASH_LENGTH; + if (xorbHash2 === XORB_HASH_BOOKEND) { + break; + } + offset += 4; + const chunkCount = shardView.getUint32(offset, true); + offset += 4; + offset += 4; + offset += 4; + const chunks = []; + for (let i = 0; i < chunkCount; i++) { + const chunkHash = readHashFromArray(shard, offset); + offset += HASH_LENGTH; + const startOffset = shardView.getUint32(offset, true); + offset += 4; + const length = shardView.getUint32(offset, true); + offset += 4; + offset += 8; + chunks.push({ + hash: chunkHash, + startOffset, + unpackedLength: length + }); + } + xorbs.push({ + hash: xorbHash2, + chunks + }); + } + return { + hmacKey, + xorbs + }; +} + +// src/utils/sum.ts +function sum(arr) { + return arr.reduce((a, b) => a + b, 0); +} + +// src/utils/SplicedBlob.ts +var SplicedBlob = class extends Blob { + originalBlob; + spliceOperations; + constructor(originalBlob, spliceOperations) { + super(); + this.originalBlob = originalBlob; + this.spliceOperations = spliceOperations; + } + static create(originalBlob, operations) { + for (const op of operations) { + if (op.start < 0 || op.end < 0) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + if (op.start > originalBlob.size || op.end > originalBlob.size) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + if (op.start > op.end) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + } + const sortedOps = [...operations].sort((a, b) => a.start - b.start); + for (let i = 0; i < sortedOps.length - 1; i++) { + if (sortedOps[i].end > sortedOps[i + 1].start) { + throw new Error("Overlapping splice operations are not supported"); + } + } + return new SplicedBlob(originalBlob, sortedOps); + } + /** + * Returns the size of the spliced blob. + * Size = original size - total replaced size + total insert size + */ + get size() { + let totalReplacedSize = 0; + let totalInsertSize = 0; + for (const op of this.spliceOperations) { + totalReplacedSize += op.end - op.start; + totalInsertSize += op.insert.size; + } + return this.originalBlob.size - totalReplacedSize + totalInsertSize; + } + /** + * Returns the MIME type of the original blob. + */ + get type() { + return this.originalBlob.type; + } + /** + * Returns a new instance of SplicedBlob that is a slice of the current one. + * + * The slice is inclusive of the start and exclusive of the end. + * The slice method does not support negative start/end. + * + * @param start beginning of the slice + * @param end end of the slice + */ + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + throw new TypeError("Unsupported negative start/end on SplicedBlob.slice"); + } + start = Math.min(start, this.size); + end = Math.min(end, this.size); + if (start >= end) { + return new Blob([]); + } + const segments = this.segments; + const segmentBoundaries = [0]; + let cumulativeSize = 0; + for (const segment of segments) { + cumulativeSize += segment.size; + segmentBoundaries.push(cumulativeSize); + } + const resultSegments = []; + for (let i = 0; i < segments.length; i++) { + const segmentStart = segmentBoundaries[i]; + const segmentEnd = segmentBoundaries[i + 1]; + if (segmentEnd <= start) { + continue; + } + if (segmentStart >= end) { + break; + } + const sliceStart = Math.max(0, start - segmentStart); + const sliceEnd = Math.min(segments[i].size, end - segmentStart); + if (sliceStart < sliceEnd) { + resultSegments.push(segments[i].slice(sliceStart, sliceEnd)); + } + } + return new Blob(resultSegments); + } + get firstSpliceIndex() { + return this.spliceOperations[0]?.start ?? Infinity; + } + /** + * Read the spliced blob content and returns it as an ArrayBuffer. + */ + async arrayBuffer() { + const segments = this.segments; + const buffers = await Promise.all(segments.map((segment) => segment.arrayBuffer())); + const totalSize = sum(buffers.map((buffer) => buffer.byteLength)); + const result = new Uint8Array(totalSize); + let offset = 0; + for (const buffer of buffers) { + result.set(new Uint8Array(buffer), offset); + offset += buffer.byteLength; + } + return result.buffer; + } + /** + * Read the spliced blob content and returns it as a string. + */ + async text() { + const buffer = await this.arrayBuffer(); + return new TextDecoder().decode(buffer); + } + /** + * Returns a stream around the spliced blob content. + */ + stream() { + const readable = new ReadableStream({ + start: async (controller) => { + try { + const segments = this.segments; + for (const segment of segments) { + const reader = segment.stream().getReader(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + controller.enqueue(value); + } + } finally { + reader.releaseLock(); + } + } + controller.close(); + } catch (error) { + controller.error(error); + } + } + }); + return readable; + } + /** + * Get all segments that make up the spliced blob. + * This includes original blob segments between splice operations and insert blobs. + */ + get segments() { + const segments = []; + let currentPosition = 0; + const sortedOps = [...this.spliceOperations].sort((a, b) => a.start - b.start); + for (const op of sortedOps) { + if (currentPosition < op.start) { + segments.push(this.originalBlob.slice(currentPosition, op.start)); + } + if (op.insert.size > 0) { + segments.push(op.insert); + } + currentPosition = op.end; + } + if (currentPosition < this.originalBlob.size) { + segments.push(this.originalBlob.slice(currentPosition)); + } + return segments; + } +}; + +// src/utils/createXorbs.ts +var import_xetchunk_wasm = require("@huggingface/xetchunk-wasm"); +var TARGET_CHUNK_SIZE = 64 * 1024; +var MAX_CHUNK_SIZE = 2 * TARGET_CHUNK_SIZE; +var XORB_SIZE = 64 * 1024 * 1024; +var MAX_XORB_CHUNKS = 8 * 1024; +var INTERVAL_BETWEEN_REMOTE_DEDUP = 4e6; +var PROCESSING_PROGRESS_RATIO = 0.1; +var UPLOADING_PROGRESS_RATIO = 1 - PROCESSING_PROGRESS_RATIO; +function computeXorbHashHex(chunks) { + const chunkObjs = chunks.map((c) => ({ hash: (0, import_xetchunk_wasm.hexToBytes)(c.hash), length: c.length })); + return (0, import_xetchunk_wasm.hashToHex)((0, import_xetchunk_wasm.xorbHash)(chunkObjs)); +} +function computeHmacHex(hash2, key) { + return (0, import_xetchunk_wasm.hashToHex)((0, import_xetchunk_wasm.hmac)((0, import_xetchunk_wasm.hexToBytes)(hash2), (0, import_xetchunk_wasm.hexToBytes)(key))); +} +function computeVerificationHashHex(hashes) { + return (0, import_xetchunk_wasm.hashToHex)((0, import_xetchunk_wasm.verificationHash)(hashes.map(import_xetchunk_wasm.hexToBytes))); +} +function computeFileHashHex(chunks) { + const chunkObjs = chunks.map((c) => ({ hash: (0, import_xetchunk_wasm.hexToBytes)(c.hash), length: c.length })); + return (0, import_xetchunk_wasm.hashToHex)((0, import_xetchunk_wasm.fileHash)(chunkObjs)); +} +function addDataToChunker(data, chunker) { + return (0, import_xetchunk_wasm.nextBlock)(chunker, data).map((c) => ({ hash: (0, import_xetchunk_wasm.hashToHex)(c.hash), length: c.length, dedup: false })); +} +function finalizeChunker(chunker) { + const last = (0, import_xetchunk_wasm.finalize)(chunker); + if (!last) { + return []; + } + return [{ hash: (0, import_xetchunk_wasm.hashToHex)(last.hash), length: last.length, dedup: false }]; +} +var CurrentXorbInfo = class { + id; + offset; + chunks; + fileProcessedBytes; + fileUploadedBytes; + fileSize; + data; + immutableData; + constructor() { + this.id = 0; + this.offset = 0; + this.chunks = []; + this.fileProcessedBytes = {}; + this.fileUploadedBytes = {}; + this.fileSize = {}; + this.data = new Uint8Array(XORB_SIZE); + this.immutableData = null; + } + event(computeXorbHash) { + const xorbChunksCleaned = this.chunks.map((chunk2) => ({ + hash: chunk2.hash, + length: chunk2.length + })); + return { + event: "xorb", + xorb: this.data.subarray(0, this.offset), + hash: computeXorbHash(xorbChunksCleaned), + chunks: xorbChunksCleaned, + id: this.id, + files: Object.entries(this.fileProcessedBytes).map(([path2, processedBytes]) => ({ + path: path2, + progress: processedBytes / this.fileSize[path2], + lastSentProgress: ((this.fileUploadedBytes[path2] ?? 0) + (processedBytes - (this.fileUploadedBytes[path2] ?? 0)) * PROCESSING_PROGRESS_RATIO) / this.fileSize[path2] + })) + }; + } +}; +async function* createXorbs(fileSources, params) { + const alreadyDoneFileSha256s = /* @__PURE__ */ new Set(); + let xorbId = 0; + const chunkCache = new ChunkCache(); + let xorb = new CurrentXorbInfo(); + const nextXorb = (currentFile) => { + const event = xorb.event(computeXorbHashHex); + xorbId++; + xorb = new CurrentXorbInfo(); + xorb.id = xorbId; + xorb.fileUploadedBytes = { + [currentFile.path]: currentFile.uploadedBytes + }; + xorb.fileSize[currentFile.path] = currentFile.size; + return event; + }; + const pendingFileEvents = []; + const remoteXorbHashes = [""]; + for await (const fileSource of fileSources) { + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: 0 + }); + if (fileSource.sha256 && alreadyDoneFileSha256s.has(fileSource.sha256)) { + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: 1 + }); + continue; + } + if (fileSource.sha256) { + alreadyDoneFileSha256s.add(fileSource.sha256); + } + const chunker = (0, import_xetchunk_wasm.createChunker)(TARGET_CHUNK_SIZE); + { + xorb.fileSize[fileSource.path] = fileSource.content.size; + if (fileSource.content instanceof SplicedBlob && fileSource.content.firstSpliceIndex < MAX_CHUNK_SIZE) { + await loadDedupInfoToCache( + fileSource.content.originalBlob.slice(0, MAX_CHUNK_SIZE), + remoteXorbHashes, + params, + chunkCache, + computeHmacHex, + { + maxChunks: 1, + isAtBeginning: true + } + ); + } + let bytesSinceRemoteDedup = Infinity; + let bytesSinceLastProgressEvent = 0; + let isFirstFileChunk = true; + const sourceChunks = []; + const reader = fileSource.content.stream().getReader(); + let processedBytes = 0; + let dedupedBytes = 0; + const fileChunks = []; + const chunkMetadata = []; + const addChunks = async function* (chunks) { + for (const chunk2 of chunks) { + if (isFirstFileChunk) { + chunk2.dedup = true; + isFirstFileChunk = false; + } + let chunkIndex = xorb.chunks.length; + let chunkXorbId = xorbId; + const chunkToCopy = removeChunkFromSourceData(sourceChunks, chunk2.length); + let cacheData = chunkCache.getChunk(chunk2.hash, computeHmacHex); + if (cacheData === void 0 && chunk2.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { + const token = await xetWriteToken(params); + bytesSinceRemoteDedup = 0; + const shardResp = await (params.fetch ?? fetch)(token.casUrl + "/v1/chunks/default/" + chunk2.hash, { + headers: { + Authorization: `Bearer ${token.accessToken}` + } + }); + if (shardResp.ok) { + const shard = await shardResp.blob(); + const shardData = await parseShardData(shard); + for (const xorb2 of shardData.xorbs) { + const remoteXorbId = -remoteXorbHashes.length; + remoteXorbHashes.push(xorb2.hash); + let i = 0; + for (const chunk3 of xorb2.chunks) { + chunkCache.addChunkToCache(chunk3.hash, remoteXorbId, i++, shardData.hmacKey); + } + } + cacheData = chunkCache.getChunk(chunk2.hash, computeHmacHex); + const oldDedupedBytes = dedupedBytes; + dedupedBytes = backtrackDedup(xorb, computeHmacHex, shardData, chunkCache, chunkMetadata, dedupedBytes); + if (dedupedBytes > oldDedupedBytes) { + xorb.fileUploadedBytes[fileSource.path] ??= 0; + xorb.fileUploadedBytes[fileSource.path] += dedupedBytes - oldDedupedBytes; + } + } + } + if (cacheData === void 0) { + if (!writeChunk(xorb, chunkToCopy, chunk2.hash)) { + yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); + chunkIndex = 0; + chunkXorbId = xorbId; + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } + pendingFileEvents.length = 0; + if (!writeChunk(xorb, chunkToCopy, chunk2.hash)) { + throw new Error("Failed to write chunk into xorb"); + } + } + chunkCache.addChunkToCache(chunk2.hash, xorbId, chunkIndex, null); + } else { + chunkXorbId = cacheData.xorbIndex; + chunkIndex = cacheData.chunkIndex; + dedupedBytes += chunk2.length; + xorb.fileUploadedBytes[fileSource.path] ??= 0; + xorb.fileUploadedBytes[fileSource.path] += chunk2.length; + } + bytesSinceRemoteDedup += chunk2.length; + bytesSinceLastProgressEvent += chunk2.length; + fileChunks.push({ hash: chunk2.hash, length: chunk2.length }); + chunkMetadata.push({ + xorbId: chunkXorbId, + chunkIndex, + length: chunk2.length + }); + xorb.fileProcessedBytes[fileSource.path] = processedBytes; + if (bytesSinceLastProgressEvent >= 1e6) { + bytesSinceLastProgressEvent = 0; + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: ((xorb.fileUploadedBytes[fileSource.path] ?? 0) + (xorb.fileProcessedBytes[fileSource.path] - (xorb.fileUploadedBytes[fileSource.path] ?? 0)) * PROCESSING_PROGRESS_RATIO) / fileSource.content.size + }); + } + if (xorb.chunks.length >= MAX_XORB_CHUNKS) { + yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } + pendingFileEvents.length = 0; + } + } + }; + while (true) { + const { done, value } = await reader.read(); + if (done) { + yield* addChunks(finalizeChunker(chunker)); + break; + } + processedBytes += value.length; + sourceChunks.push(value); + yield* addChunks(addDataToChunker(value, chunker)); + } + const fileRepresentation = buildFileRepresentation(chunkMetadata, fileChunks, computeVerificationHashHex); + xorb.immutableData = { + chunkIndex: xorb.chunks.length, + offset: xorb.offset + }; + const dedupRatio = fileSource.content.size > 0 ? dedupedBytes / fileSource.content.size : 0; + pendingFileEvents.push({ + event: "file", + path: fileSource.path, + hash: computeFileHashHex(fileChunks), + sha256: fileSource.sha256, + dedupRatio, + representation: fileRepresentation + }); + } + } + if (xorb.offset > 0) { + yield xorb.event(computeXorbHashHex); + } + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } +} +function backtrackDedup(xorb, computeHmac, shardData, chunkCache, chunkMetadata, dedupedBytes) { + const chunkIndexesToBacktrackFor = /* @__PURE__ */ new Map(); + for (let chunkToRecheckIndex = xorb.immutableData?.chunkIndex ?? 0; chunkToRecheckIndex < xorb.chunks.length; chunkToRecheckIndex++) { + const chunk2 = xorb.chunks[chunkToRecheckIndex]; + const hmacHash = computeHmac(chunk2.hash, shardData.hmacKey); + const cacheData = chunkCache.getChunk(hmacHash, null); + if (cacheData !== void 0) { + chunkIndexesToBacktrackFor.set(chunkToRecheckIndex, { + xorbId: cacheData.xorbIndex, + chunkIndex: cacheData.chunkIndex + }); + chunkCache.removeChunkFromCache(chunk2.hash); + } + } + for (const metadata of chunkMetadata) { + if (metadata.xorbId === xorb.id && chunkIndexesToBacktrackFor.has(metadata.chunkIndex)) { + const backtrackData = chunkIndexesToBacktrackFor.get(metadata.chunkIndex); + if (backtrackData !== void 0) { + metadata.xorbId = backtrackData.xorbId; + metadata.chunkIndex = backtrackData.chunkIndex; + dedupedBytes += metadata.length; + } + } + } + const xorbRangesToErase = []; + for (let i = 0; i < xorb.chunks.length; i++) { + const chunk2 = xorb.chunks[i]; + if (chunkIndexesToBacktrackFor.has(i)) { + xorbRangesToErase.push({ + start: chunk2.offset, + end: i < xorb.chunks.length - 1 ? xorb.chunks[i + 1].offset : xorb.offset + }); + } + } + const xorbRangesToKeep = []; + let currentStart = 0; + for (let i = 0; i < xorbRangesToErase.length; i++) { + const range2 = xorbRangesToErase[i]; + if (currentStart !== range2.start) { + xorbRangesToKeep.push({ start: currentStart, end: range2.start }); + } + currentStart = range2.end; + } + if (currentStart !== xorb.offset) { + xorbRangesToKeep.push({ start: currentStart, end: xorb.offset }); + } + let currentOffset = 0; + for (const range2 of xorbRangesToKeep) { + if (range2.start !== currentOffset) { + xorb.data.set(xorb.data.subarray(range2.start, range2.end), currentOffset); + } + currentOffset += range2.end - range2.start; + } + const newXorbChunks = []; + const oldIndexToNewIndex = /* @__PURE__ */ new Map(); + let erasedOffset = 0; + for (let i = 0; i < xorb.chunks.length; i++) { + const chunk2 = xorb.chunks[i]; + if (chunkIndexesToBacktrackFor.has(i)) { + if (i < xorb.chunks.length - 1) { + erasedOffset += xorb.chunks[i + 1].offset - chunk2.offset; + } + } else { + newXorbChunks.push({ + hash: chunk2.hash, + length: chunk2.length, + offset: chunk2.offset - erasedOffset + }); + if (erasedOffset > 0) { + oldIndexToNewIndex.set(i, newXorbChunks.length - 1); + } + } + } + xorb.chunks = newXorbChunks; + xorb.offset = currentOffset; + for (const chunk2 of chunkMetadata) { + if (chunk2.xorbId === xorb.id) { + const newIndex = oldIndexToNewIndex.get(chunk2.chunkIndex); + if (newIndex !== void 0) { + const cached = chunkCache.getChunk(xorb.chunks[newIndex].hash, null); + if (cached !== void 0 && cached.xorbIndex === chunk2.xorbId && cached.chunkIndex === chunk2.chunkIndex) { + chunkCache.updateChunkIndex(xorb.chunks[newIndex].hash, newIndex); + } + chunk2.chunkIndex = newIndex; + } + } + } + return dedupedBytes; +} +function removeChunkFromSourceData(sourceChunks, chunkLength) { + if (chunkLength === sourceChunks[0].length) { + const chunkToCopy = sourceChunks[0]; + sourceChunks.shift(); + return chunkToCopy; + } else if (chunkLength < sourceChunks[0].length) { + const chunkToCopy = sourceChunks[0].subarray(0, chunkLength); + sourceChunks[0] = sourceChunks[0].subarray(chunkLength); + return chunkToCopy; + } else { + const chunkToCopy = new Uint8Array(chunkLength); + let copyOffset = 0; + let index = 0; + let toSlice = -1; + while (copyOffset < chunkLength) { + const nToCopy = Math.min(sourceChunks[index].length, chunkLength - copyOffset); + chunkToCopy.set(sourceChunks[index].subarray(0, nToCopy), copyOffset); + copyOffset += nToCopy; + if (nToCopy === sourceChunks[index].length) { + index++; + } else { + toSlice = nToCopy; + } + } + sourceChunks.splice(0, index); + if (toSlice !== -1) { + sourceChunks[0] = sourceChunks[0].subarray(toSlice); + } + return chunkToCopy; + } +} +function writeChunk(xorb, chunk2, hash2) { + const regularCompressedChunk = compress(chunk2); + const bgCompressedChunk = compress(bg4_split_bytes(chunk2)); + const compressedChunk = bgCompressedChunk.length < regularCompressedChunk.length ? bgCompressedChunk : regularCompressedChunk; + const chunkToWrite = compressedChunk.length < chunk2.length ? compressedChunk : chunk2; + if (xorb.offset + XET_CHUNK_HEADER_BYTES + chunkToWrite.length > XORB_SIZE) { + return false; + } + xorb.data[xorb.offset] = 0; + xorb.data[xorb.offset + 1] = chunkToWrite.length & 255; + xorb.data[xorb.offset + 2] = chunkToWrite.length >> 8 & 255; + xorb.data[xorb.offset + 3] = chunkToWrite.length >> 16 & 255; + xorb.data[xorb.offset + 4] = chunkToWrite.length < chunk2.length ? bgCompressedChunk.length < regularCompressedChunk.length ? 2 /* ByteGroupingLZ4 */ : 1 /* LZ4 */ : 0 /* None */; + xorb.data[xorb.offset + 5] = chunk2.length & 255; + xorb.data[xorb.offset + 6] = chunk2.length >> 8 & 255; + xorb.data[xorb.offset + 7] = chunk2.length >> 16 & 255; + xorb.data.set(chunkToWrite, xorb.offset + XET_CHUNK_HEADER_BYTES); + xorb.chunks.push({ hash: hash2, length: chunk2.length, offset: xorb.offset }); + xorb.offset += XET_CHUNK_HEADER_BYTES + chunkToWrite.length; + return true; +} +var buildFileRepresentation = (metadata, chunks, computeVerificationHash) => { + if (metadata.length === 0) { + return []; + } + const representation = []; + let currentRange = { + xorbId: metadata[0].xorbId, + indexStart: metadata[0].chunkIndex, + indexEnd: metadata[0].chunkIndex + 1, + length: metadata[0].length, + chunkHashStart: 0 + }; + for (let i = 1; i < metadata.length; i++) { + const chunk2 = metadata[i]; + if (currentRange.xorbId === chunk2.xorbId && currentRange.indexEnd === chunk2.chunkIndex) { + currentRange.indexEnd = chunk2.chunkIndex + 1; + currentRange.length += chunk2.length; + } else { + const rangeHash2 = computeVerificationHash(chunks.slice(currentRange.chunkHashStart, i).map((x) => x.hash)); + representation.push({ + xorbId: currentRange.xorbId, + indexStart: currentRange.indexStart, + indexEnd: currentRange.indexEnd, + length: currentRange.length, + rangeHash: rangeHash2 + }); + currentRange = { + xorbId: chunk2.xorbId, + indexStart: chunk2.chunkIndex, + indexEnd: chunk2.chunkIndex + 1, + length: chunk2.length, + chunkHashStart: i + }; + } + } + const rangeHash = computeVerificationHash(chunks.slice(currentRange.chunkHashStart).map((x) => x.hash)); + representation.push({ + xorbId: currentRange.xorbId, + indexStart: currentRange.indexStart, + indexEnd: currentRange.indexEnd, + length: currentRange.length, + rangeHash + }); + return representation; +}; +async function loadDedupInfoToCache(content, remoteXorbHashes, params, chunkCache, computeHmacHex2, opts) { + const chunker = (0, import_xetchunk_wasm.createChunker)(TARGET_CHUNK_SIZE); + const cache = chunkCache; + let dedupedBytes = 0; + let chunksProcessed = 0; + let totalBytes = 0; + let bytesSinceRemoteDedup = Infinity; + const sourceChunks = []; + const reader = content.stream().getReader(); + const processChunks = async (chunks) => { + for (const chunk2 of chunks) { + chunksProcessed++; + if (opts?.isAtBeginning && chunksProcessed === 1) { + chunk2.dedup = true; + } + totalBytes += chunk2.length; + removeChunkFromSourceData(sourceChunks, chunk2.length); + let cacheData = cache.getChunk(chunk2.hash, computeHmacHex2); + if (cacheData !== void 0) { + dedupedBytes += chunk2.length; + bytesSinceRemoteDedup += chunk2.length; + continue; + } + if (chunk2.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { + const token = await xetWriteToken(params); + bytesSinceRemoteDedup = 0; + const shardResp = await (params.fetch ?? fetch)(token.casUrl + "/v1/chunks/default/" + chunk2.hash, { + headers: { + Authorization: `Bearer ${token.accessToken}` + } + }); + if (shardResp.ok) { + const shard = await shardResp.blob(); + const shardData = await parseShardData(shard); + for (const xorb of shardData.xorbs) { + const remoteXorbId = -remoteXorbHashes.length; + remoteXorbHashes.push(xorb.hash); + let i = 0; + for (const xorbChunk of xorb.chunks) { + cache.addChunkToCache(xorbChunk.hash, remoteXorbId, i++, shardData.hmacKey); + } + } + cacheData = cache.getChunk(chunk2.hash, computeHmacHex2); + } + } + if (cacheData !== void 0) { + dedupedBytes += chunk2.length; + } + bytesSinceRemoteDedup += chunk2.length; + } + }; + while (true) { + if (opts?.end !== void 0 && totalBytes >= opts.end) { + break; + } + if (opts?.maxChunks !== void 0 && chunksProcessed >= opts.maxChunks) { + break; + } + const { done, value } = await reader.read(); + if (done) { + await processChunks(finalizeChunker(chunker)); + break; + } + sourceChunks.push(value); + await processChunks(addDataToChunker(value, chunker)); + } +} + +// src/utils/uploadShards.ts +var SHARD_MAX_SIZE = 64 * 1024 * 1024; +var SHARD_HEADER_SIZE = 48; +var SHARD_FOOTER_SIZE = 200; +var HASH_LENGTH2 = 32; +var XORB_FOOTER_LENGTH = 48; +var FILE_FOOTER_LENGTH = 48; +var SHARD_HEADER_VERSION = 2n; +var SHARD_FOOTER_VERSION = 1n; +var MDB_FILE_FLAG_WITH_VERIFICATION = 2147483648; +var MDB_FILE_FLAG_WITH_METADATA_EXT = 1073741824; +var SHARD_MAGIC_TAG = new Uint8Array([ + "H".charCodeAt(0), + "F".charCodeAt(0), + "R".charCodeAt(0), + "e".charCodeAt(0), + "p".charCodeAt(0), + "o".charCodeAt(0), + "M".charCodeAt(0), + "e".charCodeAt(0), + "t".charCodeAt(0), + "a".charCodeAt(0), + "D".charCodeAt(0), + "a".charCodeAt(0), + "t".charCodeAt(0), + "a".charCodeAt(0), + 0, + 85, + 105, + 103, + 69, + 106, + 123, + 129, + 87, + 131, + 165, + 189, + 217, + 92, + 205, + 209, + 74, + 169 +]); +async function* uploadShards(source, params) { + const xorbHashes = []; + const seenFileXetHashes = /* @__PURE__ */ new Set(); + const fileInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.25); + const xorbInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.75); + const xorbView = new DataView(xorbInfoSection.buffer); + let xorbViewOffset = 0; + const fileInfoView = new DataView(fileInfoSection.buffer); + let fileViewOffset = 0; + let xorbTotalSize = 0n; + let fileTotalSize = 0n; + let xorbTotalUnpackedSize = 0n; + for await (const output of createXorbs(source, params)) { + switch (output.event) { + case "xorb": { + xorbHashes.push(output.hash); + const xorbEntrySize = HASH_LENGTH2 + 4 + 4 + 4 + 4; + const chunksSize = output.chunks.length * (HASH_LENGTH2 + 4 + 4 + 8); + const totalXorbSize = xorbEntrySize + chunksSize; + if (xorbViewOffset + totalXorbSize > xorbInfoSection.length) { + if (xorbViewOffset > 0 || fileViewOffset > 0) { + await uploadShard(createShard(), params); + } + } + writeHashToArray(output.hash, xorbInfoSection, xorbViewOffset); + xorbViewOffset += HASH_LENGTH2; + xorbView.setUint32(xorbViewOffset, 0, true); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, output.chunks.length, true); + xorbViewOffset += 4; + const xorbUnpackedSize = sum(output.chunks.map((x) => x.length)); + xorbView.setUint32(xorbViewOffset, xorbUnpackedSize, true); + xorbTotalUnpackedSize += BigInt(xorbUnpackedSize); + xorbTotalSize += BigInt(output.xorb.byteLength); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, output.xorb.byteLength, true); + xorbViewOffset += 4; + let chunkBytes = 0; + for (const chunk2 of output.chunks) { + writeHashToArray(chunk2.hash, xorbInfoSection, xorbViewOffset); + xorbViewOffset += HASH_LENGTH2; + xorbView.setUint32(xorbViewOffset, chunkBytes, true); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, chunk2.length, true); + xorbViewOffset += 4; + xorbView.setBigUint64(xorbViewOffset, 0n, true); + xorbViewOffset += 8; + chunkBytes += chunk2.length; + } + for (const file of output.files) { + yield { + event: "fileProgress", + path: file.path, + progress: file.lastSentProgress + }; + } + await uploadXorb(output, params); + for (const file of output.files) { + yield { event: "fileProgress", path: file.path, progress: file.progress }; + } + break; + } + case "file": { + yield { + event: "file", + path: output.path, + xetHash: output.hash, + sha256: output.sha256, + dedupRatio: output.dedupRatio + }; + if (seenFileXetHashes.has(output.hash)) { + break; + } + seenFileXetHashes.add(output.hash); + const fileHeaderSize = HASH_LENGTH2 + 4 + 4 + 8; + const representationSize = output.representation.length * (HASH_LENGTH2 + 4 + 4 + 4 + 4); + const verificationSize = output.representation.length * (HASH_LENGTH2 + 16); + const fileSha256 = output.sha256; + const hasMetadataExt = fileSha256 !== void 0; + const metadataSize = hasMetadataExt ? HASH_LENGTH2 + 16 : 0; + const totalFileSize = fileHeaderSize + representationSize + verificationSize + metadataSize; + if (fileViewOffset + totalFileSize > fileInfoSection.length) { + if (xorbViewOffset > 0 || fileViewOffset > 0) { + await uploadShard(createShard(), params); + } + } + writeHashToArray(output.hash, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + fileInfoView.setUint32( + fileViewOffset, + MDB_FILE_FLAG_WITH_VERIFICATION + (hasMetadataExt ? MDB_FILE_FLAG_WITH_METADATA_EXT : 0), + true + ); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, output.representation.length, true); + fileViewOffset += 4; + fileInfoView.setBigUint64(fileViewOffset, 0n, true); + fileViewOffset += 8; + for (const repItem of output.representation) { + writeHashToArray( + typeof repItem.xorbId === "number" ? xorbHashes[repItem.xorbId] : repItem.xorbId, + fileInfoSection, + fileViewOffset + ); + fileViewOffset += HASH_LENGTH2; + fileInfoView.setUint32(fileViewOffset, 0, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.length, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.indexStart, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.indexEnd, true); + fileViewOffset += 4; + } + for (const repItem of output.representation) { + writeHashToArray(repItem.rangeHash, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + for (let i = 0; i < 16; i++) { + fileInfoSection[fileViewOffset + i] = 0; + } + fileViewOffset += 16; + } + if (hasMetadataExt) { + writeHashToArray(fileSha256, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + for (let i = 0; i < 16; i++) { + fileInfoSection[fileViewOffset + i] = 0; + } + fileViewOffset += 16; + } + break; + } + } + } + function createShard() { + const shard = new Uint8Array( + SHARD_HEADER_SIZE + SHARD_FOOTER_SIZE + xorbViewOffset + XORB_FOOTER_LENGTH + fileViewOffset + FILE_FOOTER_LENGTH + ); + const shardView = new DataView(shard.buffer); + let shardOffset = 0; + shard.set(SHARD_MAGIC_TAG, shardOffset); + shardOffset += SHARD_MAGIC_TAG.length; + shardView.setBigUint64(shardOffset, SHARD_HEADER_VERSION, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(SHARD_FOOTER_SIZE), true); + shardOffset += 8; + shard.set(fileInfoSection.slice(0, fileViewOffset), shardOffset); + shardOffset += fileViewOffset; + for (let i = 0; i < 32; i++) { + shard[shardOffset + i] = 255; + } + shardOffset += 32; + for (let i = 0; i < 16; i++) { + shard[shardOffset + i] = 0; + } + shardOffset += 16; + const xorbInfoOffset = shardOffset; + shard.set(xorbInfoSection.slice(0, xorbViewOffset), shardOffset); + shardOffset += xorbViewOffset; + for (let i = 0; i < 32; i++) { + shard[shardOffset + i] = 255; + } + shardOffset += 32; + for (let i = 0; i < 16; i++) { + shard[shardOffset + i] = 0; + } + shardOffset += 16; + const footerOffset = shardOffset; + shardView.setBigUint64(shardOffset, SHARD_FOOTER_VERSION, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(SHARD_HEADER_SIZE), true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(xorbInfoOffset), true); + shardOffset += 8; + for (let i = 0; i < 48; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 48; + for (let i = 0; i < 32; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 32; + shardView.setBigUint64(shardOffset, BigInt(Math.floor(Date.now() / 1e3)), true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, 0n, true); + shardOffset += 8; + for (let i = 0; i < 48; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 48; + shardView.setBigUint64(shardOffset, xorbTotalSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, fileTotalSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, xorbTotalUnpackedSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(footerOffset), true); + xorbViewOffset = 0; + fileViewOffset = 0; + xorbTotalSize = 0n; + xorbTotalUnpackedSize = 0n; + fileTotalSize = 0n; + return shard; + } + if (xorbViewOffset || fileViewOffset) { + await uploadShard(createShard(), params); + } +} +function writeHashToArray(hash2, array, offset) { + for (let i = 0; i < hash2.length; i += 16) { + array[offset + i / 2] = parseInt(hash2.substring(i + 2 * 7, i + 2 * 8), 16); + array[offset + i / 2 + 1] = parseInt(hash2.substring(i + 2 * 6, i + 2 * 7), 16); + array[offset + i / 2 + 2] = parseInt(hash2.substring(i + 2 * 5, i + 2 * 6), 16); + array[offset + i / 2 + 3] = parseInt(hash2.substring(i + 2 * 4, i + 2 * 5), 16); + array[offset + i / 2 + 4] = parseInt(hash2.substring(i + 2 * 3, i + 2 * 4), 16); + array[offset + i / 2 + 5] = parseInt(hash2.substring(i + 2 * 2, i + 2 * 3), 16); + array[offset + i / 2 + 6] = parseInt(hash2.substring(i + 2 * 1, i + 2 * 2), 16); + array[offset + i / 2 + 7] = parseInt(hash2.substring(i + 2 * 0, i + 2 * 1), 16); + } +} +async function uploadXorb(xorb, params) { + const token = await xetWriteToken(params); + const resp = await (params.fetch ?? fetch)(`${token.casUrl}/v1/xorbs/default/${xorb.hash}`, { + method: "POST", + body: xorb.xorb, + headers: { + Authorization: `Bearer ${token.accessToken}`, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + }, + ...{ + progressHint: { + progressCallback: (progress) => { + for (const file of xorb.files) { + params.yieldCallback?.({ + event: "fileProgress", + path: file.path, + progress: file.lastSentProgress + (file.progress - file.lastSentProgress) * progress + }); + } + } + } + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } +} +async function uploadShard(shard, params) { + const token = await xetWriteToken(params); + const resp = await (params.fetch ?? fetch)(`${token.casUrl}/v1/shards`, { + method: "POST", + body: shard, + headers: { + Authorization: `Bearer ${token.accessToken}`, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } +} + +// src/utils/splitAsyncGenerator.ts +function splitAsyncGenerator(source, n) { + if (n <= 0) { + return []; + } + const sleep = (ms) => new Promise((resolve3) => setTimeout(resolve3, ms)); + let takenIndex = null; + const generators = []; + let remaining = n; + for (let i = 0; i < n; i++) { + generators.push({ + next: async () => { + while (takenIndex !== null) { + await sleep(1); + } + takenIndex = i; + return source.next().then((r) => { + takenIndex = null; + return r; + }); + }, + return: async () => { + remaining--; + if (remaining === 0) { + return source.return(void 0); + } + return { + done: true, + value: void 0 + }; + }, + throw: async (error) => { + return source.throw(error); + }, + [Symbol.asyncIterator]: () => generators[i] + }); + } + return generators; +} + +// src/lib/commit.ts +var CONCURRENT_SHAS = 5; +var CONCURRENT_LFS_UPLOADS = 5; +var MULTIPART_PARALLEL_UPLOAD = 5; +function isFileOperation(op) { + const ret = op.operation === "addOrUpdate"; + if (ret && !(op.content instanceof Blob)) { + throw new TypeError("Precondition failed: op.content should be a Blob"); + } + return ret; +} +async function* commitIter(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + if (repoId.type === "bucket") { + return yield* commitIterBucket(params); + } + if (params.operations.some((op) => op.operation === "copy")) { + throw new Error("'copy' operations are only supported when the destination repo is a bucket"); + } + yield { event: "phase", phase: "preuploading" }; + let useXet = params.useXet ?? true; + const lfsShas = /* @__PURE__ */ new Map(); + const abortController = new AbortController(); + const abortSignal = abortController.signal; + if (!abortSignal.throwIfAborted) { + abortSignal.throwIfAborted = () => { + if (abortSignal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + }; + } + if (params.abortSignal) { + params.abortSignal.addEventListener("abort", () => abortController.abort()); + } + try { + const allOperations = (await Promise.all( + params.operations.map(async (operation) => { + if (operation.operation === "edit") { + const splicedBlob = SplicedBlob.create( + operation.originalContent, + operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })) + ); + return { + operation: "addOrUpdate", + path: operation.path, + content: splicedBlob + }; + } + if (operation.operation !== "addOrUpdate") { + return operation; + } + if (!(operation.content instanceof URL)) { + return { ...operation, content: operation.content }; + } + const lazyBlobs = await createBlobs(operation.content, operation.path, { + fetch: params.fetch, + maxFolderDepth: params.maxFolderDepth + }); + abortSignal?.throwIfAborted(); + return lazyBlobs.map((blob) => ({ + ...operation, + content: blob.blob, + path: blob.path + })); + }) + )).flat(1); + const gitAttributes = allOperations.filter(isFileOperation).find((op) => op.path === ".gitattributes")?.content; + for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { + const payload = { + gitAttributes: gitAttributes && await gitAttributes.text(), + files: await Promise.all( + operations.map(async (operation) => ({ + path: operation.path, + size: operation.content.size, + sample: base64FromBytes(new Uint8Array(await operation.content.slice(0, 512).arrayBuffer())) + })) + ) + }; + abortSignal?.throwIfAborted(); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/preupload/${encodeURIComponent( + params.branch ?? "main" + )}` + (params.isPullRequest ? "?create_pr=1" : ""), + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/json" + }, + body: JSON.stringify(payload), + signal: abortSignal + } + ); + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + for (const file of json.files) { + if (file.uploadMode === "lfs") { + lfsShas.set(file.path, null); + } + } + } + yield { event: "phase", phase: "uploadingLargeFiles" }; + for (const operations of chunk( + allOperations.filter(isFileOperation).filter((op) => lfsShas.has(op.path)), + 100 + )) { + const shas = yield* eventToGenerator((yieldCallback, returnCallback, rejectCallack) => { + return promisesQueue( + operations.map((op) => async () => { + const iterator = sha256(op.content, { useWebWorker: params.useWebWorkers, abortSignal }); + let res2; + do { + res2 = await iterator.next(); + if (!res2.done) { + yieldCallback({ event: "fileProgress", path: op.path, progress: res2.value, state: "hashing" }); + } + } while (!res2.done); + const sha = res2.value; + lfsShas.set(op.path, res2.value); + return sha; + }), + CONCURRENT_SHAS + ).then(returnCallback, rejectCallack); + }); + abortSignal?.throwIfAborted(); + const payload = { + operation: "upload", + // multipart is a custom protocol for HF + transfers: ["basic", "multipart", ...useXet ? ["xet"] : []], + hash_algo: "sha_256", + ...!params.isPullRequest && { + ref: { + name: params.branch ?? "main" + } + }, + objects: operations.map((op, i) => ({ + oid: shas[i], + size: op.content.size + })) + }; + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/${repoId.type === "model" ? "" : repoId.type + "s/"}${repoId.name}.git/info/lfs/objects/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + Accept: "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json" + }, + body: JSON.stringify(payload), + signal: abortSignal + } + ); + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + const batchRequestId = res.headers.get("X-Request-Id") || void 0; + const shaToOperation = new Map(operations.map((op, i) => [shas[i], op])); + if (useXet && json.transfer !== "xet") { + useXet = false; + } + let xetParams = null; + if (useXet) { + for (const obj of json.objects) { + const op = shaToOperation.get(obj.oid); + if (!op) { + throw new InvalidApiResponseFormatError("Unrequested object ID in response"); + } + if (obj.error) { + const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${obj.error.message}${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; + throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); + } + if (!obj.actions?.upload) { + yield { + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }; + } else { + const headers = new Headers(obj.actions.upload.header); + xetParams = { + sessionId: headers.get("X-Xet-Session-Id") ?? void 0, + casUrl: headers.get("X-Xet-Cas-Url") ?? void 0, + accessToken: headers.get("X-Xet-Access-Token") ?? void 0, + expiresAt: headers.get("X-Xet-Token-Expiration") ? new Date(parseInt(headers.get("X-Xet-Token-Expiration") ?? "0") * 1e3) : void 0, + refreshWriteTokenUrl: obj.actions.upload.href + }; + } + } + const source = async function* () { + for (const obj of json.objects) { + const op = shaToOperation.get(obj.oid); + if (!op || !obj.actions?.upload) { + continue; + } + abortSignal?.throwIfAborted(); + yield { content: op.content, path: op.path, sha256: obj.oid }; + } + }(); + if (xetParams) { + const fixedXetParams = xetParams; + const sources = splitAsyncGenerator(source, 5); + yield* eventToGenerator( + (yieldCallback, returnCallback, rejectCallback) => Promise.all( + sources.map(async function(source2) { + for await (const event of uploadShards(source2, { + fetch: params.fetch, + accessToken, + hubUrl: params.hubUrl ?? HUB_URL, + repo: repoId, + xetParams: fixedXetParams, + // todo: maybe leave empty if PR? + rev: params.branch ?? "main", + isPullRequest: params.isPullRequest, + yieldCallback: (event2) => yieldCallback({ ...event2, state: "uploading" }) + })) { + if (event.event === "file") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: 1, + state: "uploading" + }); + } else if (event.event === "fileProgress") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: event.progress, + state: "uploading" + }); + } + } + }) + ).then(() => returnCallback(void 0), rejectCallback) + ); + } else { + } + } else { + yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + return promisesQueueStreaming( + json.objects.map((obj) => async () => { + const op = shaToOperation.get(obj.oid); + if (!op) { + throw new InvalidApiResponseFormatError("Unrequested object ID in response"); + } + abortSignal?.throwIfAborted(); + if (obj.error) { + const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${obj.error.message}${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; + throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); + } + if (!obj.actions?.upload) { + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + return; + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 0, + state: "uploading" + }); + const content = op.content; + const header = obj.actions.upload.header; + if (header?.chunk_size) { + const chunkSize = parseInt(header.chunk_size); + const completionUrl = obj.actions.upload.href; + const parts = Object.keys(header).filter((key) => /^[0-9]+$/.test(key)); + if (parts.length !== Math.ceil(content.size / chunkSize)) { + throw new Error("Invalid server response to upload large LFS file, wrong number of parts"); + } + const completeReq = { + oid: obj.oid, + parts: parts.map((part) => ({ + partNumber: +part, + etag: "" + })) + }; + const progressCallback = (progress) => yieldCallback({ event: "fileProgress", path: op.path, progress, state: "uploading" }); + await promisesQueueStreaming( + parts.map((part) => async () => { + abortSignal?.throwIfAborted(); + const index = parseInt(part) - 1; + const slice = content.slice(index * chunkSize, (index + 1) * chunkSize); + const res3 = await (params.fetch ?? fetch)(header[part], { + method: "PUT", + /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ + body: slice instanceof WebBlob && isFrontend ? await slice.arrayBuffer() : slice, + signal: abortSignal, + ...{ + progressHint: { + path: op.path, + part: index, + numParts: parts.length, + progressCallback + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + }); + if (!res3.ok) { + throw await createApiError(res3, { + requestId: batchRequestId, + message: `Error while uploading part ${part} of ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + const eTag = res3.headers.get("ETag"); + if (!eTag) { + throw new Error("Cannot get ETag of part during multipart upload"); + } + completeReq.parts[Number(part) - 1].etag = eTag; + }), + MULTIPART_PARALLEL_UPLOAD + ); + abortSignal?.throwIfAborted(); + const res2 = await (params.fetch ?? fetch)(completionUrl, { + method: "POST", + body: JSON.stringify(completeReq), + headers: { + Accept: "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json" + }, + signal: abortSignal + }); + if (!res2.ok) { + throw await createApiError(res2, { + requestId: batchRequestId, + message: `Error completing multipart upload of ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + } else { + const res2 = await (params.fetch ?? fetch)(obj.actions.upload.href, { + method: "PUT", + headers: { + ...batchRequestId ? { "X-Request-Id": batchRequestId } : void 0 + }, + /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ + body: content instanceof WebBlob && isFrontend ? await content.arrayBuffer() : content, + signal: abortSignal, + ...{ + progressHint: { + path: op.path, + progressCallback: (progress) => yieldCallback({ + event: "fileProgress", + path: op.path, + progress, + state: "uploading" + }) + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + }); + if (!res2.ok) { + throw await createApiError(res2, { + requestId: batchRequestId, + message: `Error while uploading ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + } + }), + CONCURRENT_LFS_UPLOADS + ).then(returnCallback, rejectCallback); + }); + } + } + abortSignal?.throwIfAborted(); + yield { event: "phase", phase: "committing" }; + return yield* eventToGenerator( + async (yieldCallback, returnCallback, rejectCallback) => (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commit/${encodeURIComponent( + params.branch ?? "main" + )}` + (params.isPullRequest ? "?create_pr=1" : ""), + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: [ + { + key: "header", + value: { + summary: params.title, + description: params.description, + parentCommit: params.parentCommit + } + }, + ...await Promise.all( + allOperations.map((operation) => { + if (isFileOperation(operation)) { + const sha = lfsShas.get(operation.path); + if (sha) { + return { + key: "lfsFile", + value: { + path: operation.path, + algo: "sha256", + size: operation.content.size, + oid: sha + } + }; + } + } + return convertOperationToNdJson(operation); + }) + ) + ].map((x) => JSON.stringify(x)).join("\n"), + signal: abortSignal, + ...{ + progressHint: { + progressCallback: (progress) => { + for (const op of allOperations) { + if (isFileOperation(op) && !lfsShas.has(op.path)) { + yieldCallback({ + event: "fileProgress", + path: op.path, + progress, + state: "uploading" + }); + } + } + } + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + } + ).then(async (res) => { + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + returnCallback({ + pullRequestUrl: json.pullRequestUrl, + commit: { + oid: json.commitOid, + url: json.commitUrl + }, + hookOutput: json.hookOutput + }); + }).catch(rejectCallback) + ); + } catch (err) { + abortController.abort(); + throw err; + } +} +async function* commitIterBucket(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + if (params.useXet === false) { + throw new Error("useXet must be true or undefined for buckets"); + } + const abortController = new AbortController(); + const abortSignal = abortController.signal; + if (!abortSignal.throwIfAborted) { + abortSignal.throwIfAborted = () => { + if (abortSignal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + }; + } + if (params.abortSignal) { + params.abortSignal.addEventListener("abort", () => abortController.abort()); + } + try { + const allOperations = (await Promise.all( + params.operations.map(async (operation) => { + if (operation.operation === "edit") { + const splicedBlob = SplicedBlob.create( + operation.originalContent, + operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })) + ); + return { + operation: "addOrUpdate", + path: operation.path, + content: splicedBlob + }; + } + if (operation.operation !== "addOrUpdate") { + return operation; + } + if (!(operation.content instanceof URL)) { + return { ...operation, content: operation.content }; + } + const lazyBlobs = await createBlobs(operation.content, operation.path, { + fetch: params.fetch, + maxFolderDepth: params.maxFolderDepth + }); + abortSignal?.throwIfAborted(); + return lazyBlobs.map((blob) => ({ + ...operation, + content: blob.blob, + path: blob.path + })); + }) + )).flat(1); + yield { event: "phase", phase: "uploadingLargeFiles" }; + for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { + const xetHashes = /* @__PURE__ */ new Map(); + abortSignal?.throwIfAborted(); + const source = async function* () { + for (const operation of operations) { + abortSignal?.throwIfAborted(); + yield { content: operation.content, path: operation.path }; + } + }(); + const xetParams = { + sessionId: crypto.randomUUID(), + refreshWriteTokenUrl: `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/xet-write-token` + }; + const sources = splitAsyncGenerator(source, 5); + yield* eventToGenerator( + (yieldCallback, returnCallback, rejectCallback) => Promise.all( + sources.map(async function(source2) { + for await (const event of uploadShards(source2, { + fetch: params.fetch, + accessToken, + hubUrl: params.hubUrl ?? HUB_URL, + repo: repoId, + xetParams, + rev: params.branch ?? "main", + yieldCallback: (event2) => yieldCallback({ ...event2, state: "uploading" }) + })) { + if (event.event === "file") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: 1, + state: "uploading" + }); + xetHashes.set(event.path, event.xetHash); + } else if (event.event === "fileProgress") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: event.progress, + state: "uploading" + }); + } + } + }) + ).then(() => returnCallback(void 0), rejectCallback) + ); + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: [...xetHashes.entries()].map( + ([path2, xetHash]) => JSON.stringify({ + type: "addFile", + path: path2, + xetHash + }) + ).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok && resp.status !== 422) { + throw await createApiError(resp); + } + const json = await resp.json(); + for (const failed of json.failed) { + yield { + event: "fileProgress", + path: failed.path, + progress: 0, + state: "error" + }; + } + } + abortSignal?.throwIfAborted(); + const copyOperations = allOperations.filter( + (operation) => operation.operation === "copy" + ); + for (const copyChunk of chunk(copyOperations, 100)) { + abortSignal?.throwIfAborted(); + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: copyChunk.map((op) => { + const sourceRepoId = toRepoId(op.sourceRepo); + return JSON.stringify({ + type: "copyFile", + path: op.path, + xetHash: op.sourceXetHash, + sourceRepoType: sourceRepoId.type, + sourceRepoId: sourceRepoId.name + }); + }).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok && resp.status !== 422) { + throw await createApiError(resp); + } + const json = await resp.json(); + for (const failed of json.failed) { + yield { + event: "fileProgress", + path: failed.path, + progress: 0, + state: "error" + }; + } + } + abortSignal?.throwIfAborted(); + const deletedOperations = allOperations.filter((operation) => operation.operation === "delete"); + if (deletedOperations.length > 0) { + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: deletedOperations.map( + (operation) => JSON.stringify({ + type: "deleteFile", + path: operation.path + }) + ).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok) { + throw await createApiError(resp); + } + const json = await resp.json(); + if (json.failed.length > 0) { + const failedPaths = json.failed.slice(0, 5).map((f) => f.path); + throw new Error( + `Failed to delete ${json.failed.length} file(s): ${failedPaths.join(", ")}${json.failed.length > 5 ? "..." : ""}, request ID: ${resp.headers.get("X-Request-Id")}` + ); + } + } + abortSignal?.throwIfAborted(); + } catch (err) { + abortController.abort(); + throw err; + } +} +async function convertOperationToNdJson(operation) { + switch (operation.operation) { + case "addOrUpdate": { + return { + key: "file", + value: { + content: base64FromBytes(new Uint8Array(await operation.content.arrayBuffer())), + path: operation.path, + encoding: "base64" + } + }; + } + case "delete": { + return { + key: "deletedFile", + value: { + path: operation.path + } + }; + } + case "edit": { + throw new Error( + "Edit operations should be converted to addOrUpdate operations before reaching convertOperationToNdJson" + ); + } + default: + throw new TypeError("Unknown operation: " + operation.operation); + } +} + +// src/utils/parseLinkHeader.ts +function parseLinkHeader(header) { + const regex = /<(https?:[/][/][^>]+)>;\s+rel="([^"]+)"/g; + return Object.fromEntries([...header.matchAll(regex)].map(([, url, rel]) => [rel, url])); +} + +// src/lib/create-repo.ts +async function createRepo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const [namespace, repoName] = repoId.name.split("/"); + const visibility = params.visibility ?? (params.private !== void 0 ? params.private ? "private" : "public" : void 0); + if (!namespace || !repoName) { + throw new TypeError( + `"${repoId.name}" is not a fully qualified repo name. It should be of the form "{namespace}/{repoName}".` + ); + } + const res = repoId.type === "bucket" ? await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/buckets/${namespace}/${repoName}`, { + method: "POST", + body: JSON.stringify({ + visibility, + resourceGroupId: params.resourceGroupId + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }) : await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/repos/create`, { + method: "POST", + body: JSON.stringify({ + name: repoName, + visibility, + organization: namespace, + resourceGroupId: params.resourceGroupId, + license: params.license, + ...repoId.type === "space" ? { + type: "space", + sdk: params.sdk ?? "static" + } : { + type: repoId.type + }, + files: params.files ? await Promise.all( + params.files.map(async (file) => ({ + encoding: "base64", + path: file.path, + content: base64FromBytes( + new Uint8Array(file.content instanceof Blob ? await file.content.arrayBuffer() : file.content) + ) + })) + ) : void 0 + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const output = await res.json(); + return { repoUrl: output.url, id: output.id }; +} + +// src/lib/create-branch.ts +async function createBranch(params) { + const repoId = toRepoId(params.repo); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + }, + body: JSON.stringify({ + startingPoint: params.revision, + ...params.empty && { emptyBranch: true }, + overwrite: params.overwrite + }) + } + ); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/delete-branch.ts +async function deleteBranch(params) { + const repoId = toRepoId(params.repo); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, + { + method: "DELETE", + headers: { + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + } + } + ); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/delete-repo.ts +async function deleteRepo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const [namespace, repoName] = repoId.name.split("/"); + const res = repoId.type === "bucket" ? await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/buckets/${namespace}/${repoName}`, { + method: "DELETE", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }) : await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/repos/delete`, { + method: "DELETE", + body: JSON.stringify({ + name: repoName, + organization: namespace, + type: repoId.type + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/download-file-to-cache-dir.ts +var import_node_path2 = require("path"); +var import_promises4 = require("fs/promises"); + +// src/utils/symlink.ts +var fs = __toESM(require("fs/promises")); +var path = __toESM(require("path")); +var os = __toESM(require("os")); + +// src/lib/download-file-to-cache-dir.ts +var import_node_stream3 = require("stream"); +var import_promises5 = require("stream/promises"); +var import_node_fs2 = require("fs"); +var REGEX_COMMIT_HASH = new RegExp("^[0-9a-f]{40}$"); + +// src/lib/jobs/get-job.ts +async function getJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/list-job-hardware.ts +async function listJobHardware(params) { + const accessToken = checkCredentials(params ?? {}); + const headers = {}; + if (accessToken) { + headers.Authorization = `Bearer ${accessToken}`; + } + const response = await (params?.fetch || fetch)(`${params?.hubUrl || HUB_URL}/api/jobs/hardware`, { + headers + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/list-jobs.ts +async function listJobs(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/run-job.ts +async function runJob(params) { + const accessToken = checkCredentials(params); + if (!params.dockerImage && !params.spaceId) { + throw new Error("Either dockerImage or spaceId must be provided"); + } + if (params.dockerImage && params.spaceId) { + throw new Error("Cannot provide both dockerImage and spaceId"); + } + const body = { + flavor: params.flavor, + environment: params.environment || {} + }; + if (params.dockerImage) { + body.dockerImage = params.dockerImage; + } + if (params.spaceId) { + body.spaceId = params.spaceId; + } + if (params.command) { + body.command = params.command; + } + if (params.arguments) { + body.arguments = params.arguments; + } + if (params.secrets) { + body.secrets = params.secrets; + } + if (params.arch) { + body.arch = params.arch; + } + if (params.timeoutSeconds !== void 0) { + body.timeoutSeconds = params.timeoutSeconds; + } + if (params.attempts !== void 0) { + body.attempts = params.attempts; + } + if (params.labels) { + body.labels = params.labels; + } + if (params.volumes?.length) { + body.volumes = params.volumes.map(({ source, ...rest }) => { + const repoId = toRepoId(source); + return { type: repoId.type, source: repoId.name, ...rest }; + }); + } + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }, + body: JSON.stringify(body) + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/stream-job-logs.ts +async function* streamJobLogs(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/logs`, + { + headers: { + Accept: "text/event-stream", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + if (!response.body) { + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const data = JSON.parse(line.slice(6)); + yield { message: data.data, timestamp: new Date(data.timestamp) }; + } catch { + yield { message: line.slice(6), timestamp: /* @__PURE__ */ new Date() }; + } + } + } + } + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const data = JSON.parse(line.slice(6)); + yield { message: data.data, timestamp: new Date(data.timestamp) }; + } catch { + yield { message: line.slice(6), timestamp: /* @__PURE__ */ new Date() }; + } + } + } + } + } finally { + reader.releaseLock(); + } +} + +// src/utils/normalizeInferenceProviderMapping.ts +function normalizeInferenceProviderMapping(hfModelId, inferenceProviderMapping) { + if (!inferenceProviderMapping) { + return []; + } + if (Array.isArray(inferenceProviderMapping)) { + return inferenceProviderMapping.map((entry) => ({ + ...entry, + hfModelId + })); + } + return Object.entries(inferenceProviderMapping).map(([provider, mapping]) => ({ + provider, + hfModelId, + providerId: mapping.providerId, + status: mapping.status, + task: mapping.task + })); +} + +// src/lib/list-models.ts +var MODEL_EXPAND_KEYS = [ + "pipeline_tag", + "private", + "gated", + "downloads", + "likes", + "lastModified" +]; +var MODEL_DERIVED_FIELD_TO_API_KEY = { + filePaths: "siblings" +}; +async function* listModels(params) { + const accessToken = params && checkCredentials(params); + let totalToFetch = params?.limit ?? Infinity; + const additionalExpandKeys = params?.additionalFields?.map( + (field) => MODEL_DERIVED_FIELD_TO_API_KEY[field] ?? field + ) ?? []; + const search = new URLSearchParams([ + ...Object.entries({ + limit: String(Math.min(totalToFetch, 500)), + ...params?.search?.owner ? { author: params.search.owner } : void 0, + ...params?.search?.task ? { pipeline_tag: params.search.task } : void 0, + ...params?.search?.query ? { search: params.search.query } : void 0, + ...params?.search?.inferenceProviders ? { inference_provider: params.search.inferenceProviders.join(",") } : void 0, + ...params?.search?.apps ? { apps: params.search.apps.join(",") } : void 0, + ...params?.sort ? { sort: params.sort } : void 0 + }), + ...params?.search?.tags?.map((tag) => ["filter", tag]) ?? [], + ...MODEL_EXPAND_KEYS.map((val) => ["expand", val]), + ...additionalExpandKeys.map((val) => ["expand", val]) + ]).toString(); + let url = `${params?.hubUrl || HUB_URL}/api/models?${search}`; + while (url) { + const res = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + const additional = {}; + if (params?.additionalFields) { + for (const field of params.additionalFields) { + if (field === "filePaths") { + additional.filePaths = (item.siblings ?? []).map((s) => s.rfilename); + } else if (field === "inferenceProviderMapping" && item.inferenceProviderMapping) { + additional.inferenceProviderMapping = normalizeInferenceProviderMapping( + item.id, + item.inferenceProviderMapping + ); + } else { + additional[field] = item[field]; + } + } + } + yield { + ...additional, + id: item._id, + name: item.id, + private: item.private, + task: item.pipeline_tag, + downloads: item.downloads, + gated: item.gated, + likes: item.likes, + updatedAt: new Date(item.lastModified) + }; + totalToFetch--; + if (totalToFetch <= 0) { + return; + } + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/utils/typedInclude.ts +function typedInclude(arr, v) { + return arr.includes(v); +} + +// src/lib/repo-exists.ts +async function repoExists(params) { + const repoId = toRepoId(params.repo); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}?expand[]=likes`, + { + method: "GET", + headers: { + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + } + } + ); + if (res.status === 404 || res.status === 401) { + return false; + } + if (!res.ok) { + throw await createApiError(res); + } + return true; +} + +// src/lib/snapshot-download.ts +var import_node_path3 = require("path"); +var import_promises6 = require("fs/promises"); + +// src/lib/upload-files-with-progress.ts +var multipartUploadTracking = /* @__PURE__ */ new WeakMap(); +async function* uploadFilesWithProgress(params) { + return yield* commitIter({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: params.files.map((file) => ({ + operation: "addOrUpdate", + path: file instanceof URL ? file.pathname.split("/").at(-1) ?? "file" : "path" in file ? file.path : file.name, + content: "content" in file ? file.content : file + })), + title: params.commitTitle ?? `Add ${params.files.length} files`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet, + fetch: async (input, init) => { + if (!init) { + return fetch(input); + } + if (!typedInclude(["PUT", "POST"], init.method) || !("progressHint" in init) || !init.progressHint || typeof XMLHttpRequest === "undefined" || typeof input !== "string" || !(init.body instanceof ArrayBuffer) && !(init.body instanceof Blob) && !(init.body instanceof File) && typeof init.body !== "string") { + return fetch(input, init); + } + const progressHint = init.progressHint; + const progressCallback = progressHint.progressCallback; + const xhr = new XMLHttpRequest(); + xhr.upload.addEventListener("progress", (event) => { + if (event.lengthComputable) { + if (progressHint.part !== void 0) { + let tracking = multipartUploadTracking.get(progressCallback); + if (!tracking) { + tracking = { numParts: progressHint.numParts, partsProgress: {} }; + multipartUploadTracking.set(progressCallback, tracking); + } + tracking.partsProgress[progressHint.part] = event.loaded / event.total; + let totalProgress = 0; + for (const partProgress of Object.values(tracking.partsProgress)) { + totalProgress += partProgress; + } + if (totalProgress === tracking.numParts) { + progressCallback(0.9999999999); + } else { + progressCallback(totalProgress / tracking.numParts); + } + } else { + if (event.loaded === event.total) { + progressCallback(0.9999999999); + } else { + progressCallback(event.loaded / event.total); + } + } + } + }); + xhr.open(init.method, input, true); + if (init.headers) { + const headers = new Headers(init.headers); + headers.forEach((value, key) => { + xhr.setRequestHeader(key, value); + }); + } + init.signal?.throwIfAborted(); + xhr.send(init.body); + return new Promise((resolve3, reject) => { + xhr.addEventListener("load", () => { + resolve3( + new Response(xhr.responseText, { + status: xhr.status, + statusText: xhr.statusText, + headers: Object.fromEntries( + xhr.getAllResponseHeaders().trim().split("\n").map((header) => [ + header.slice(0, header.indexOf(":")), + header.slice(header.indexOf(":") + 1).trim() + ]) + ) + }) + ); + }); + xhr.addEventListener("error", () => { + reject(new Error(xhr.statusText)); + }); + if (init.signal) { + init.signal.addEventListener("abort", () => { + xhr.abort(); + try { + init.signal?.throwIfAborted(); + } catch (err) { + reject(err); + } + }); + } + }); + } + }); +} + +// src/lib/who-am-i.ts +async function whoAmI(params) { + const accessToken = checkCredentials(params); + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/whoami-v2`, { + headers: { + Authorization: `Bearer ${accessToken}` + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const response = await res.json(); + if (typeof response.auth.accessToken?.createdAt === "string") { + response.auth.accessToken.createdAt = new Date(response.auth.accessToken.createdAt); + } + return response; +} + +// cli.ts +var import_node_url3 = require("url"); +var import_promises7 = require("fs/promises"); +var import_node_path4 = require("path"); + +// package.json +var version = "2.13.2"; + +// cli.ts +var UploadProgressManager = class { + multibar = null; + fileBars = /* @__PURE__ */ new Map(); + isQuiet; + cliProgressAvailable = false; + constructor(isQuiet = false) { + this.isQuiet = isQuiet; + } + async initialize() { + if (this.isQuiet) { + return; + } + try { + const cliProgress = await Promise.resolve().then(() => __toESM(require_cli_progress())); + this.cliProgressAvailable = true; + this.multibar = new cliProgress.MultiBar( + { + clearOnComplete: false, + hideCursor: true, + format: " {bar} | {filename} | {percentage}% | {state}", + barCompleteChar: "\u2588", + barIncompleteChar: "\u2591" + }, + cliProgress.Presets.shades_grey + ); + } catch (error) { + this.cliProgressAvailable = false; + } + } + handleEvent(event) { + if (this.isQuiet) { + return; + } + if (event.event === "phase") { + this.logPhase(event.phase); + } else if (event.event === "fileProgress") { + this.updateFileProgress(event.path, event.progress, event.state); + } + } + logPhase(phase) { + if (this.isQuiet) { + return; + } + const phaseMessages = { + preuploading: "\u{1F4CB} Preparing files for upload...", + uploadingLargeFiles: "\u2B06\uFE0F Uploading files...", + committing: "\u2728 Finalizing commit..." + }; + console.log(` +${phaseMessages[phase] || phase}`); + } + updateFileProgress(path2, progress, state) { + if (this.isQuiet) { + return; + } + if (this.cliProgressAvailable && this.multibar) { + let bar = this.fileBars.get(path2); + if (!bar) { + bar = this.multibar.create(100, 0, { + filename: this.truncateFilename(path2, 100), + state + }); + this.fileBars.set(path2, bar); + } + if (state === "error") { + bar.update(0, { state: "\u2717 error" }); + } else if (progress >= 1) { + bar.update(100, { state: state === "hashing" ? "\u2713 hashed" : "\u2713 uploaded" }); + } else { + const percentage = Math.round(progress * 100); + bar.update(percentage, { state }); + } + } else { + const percentage = Math.round(progress * 100); + const truncatedPath = this.truncateFilename(path2, 100); + if (state === "error") { + console.error(`\u2717 error: ${truncatedPath}`); + } else if (progress >= 1) { + const statusIcon = state === "hashing" ? "\u2713 hashed" : "\u2713 uploaded"; + console.log(`${statusIcon}: ${truncatedPath}`); + } else if (percentage % 25 === 0) { + console.log(`${state}: ${truncatedPath} (${percentage}%)`); + } + } + } + truncateFilename(filename, maxLength) { + if (filename.length <= maxLength) { + return filename; + } + return "..." + filename.slice(-(maxLength - 3)); + } + stop() { + if (!this.isQuiet && this.cliProgressAvailable && this.multibar) { + this.multibar.stop(); + } + } +}; +var commands = { + upload: { + description: "Upload a folder to a repo on the Hub", + args: [ + { + name: "repo-name", + description: "The name of the repo to upload to", + positional: true, + required: true + }, + { + name: "local-folder", + description: "The local folder to upload. Defaults to the current working directory", + positional: true, + default: () => process.cwd() + }, + { + name: "path-in-repo", + description: "The path in the repo to upload the folder to. Defaults to the root of the repo", + positional: true, + default: "." + }, + { + name: "quiet", + short: "q", + description: "Suppress all output", + boolean: true + }, + { + name: "repo-type", + enum: ["dataset", "model", "space", "bucket"], + description: "The type of repo to upload to. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name or buckets/username/repo-name" + }, + { + name: "revision", + description: "The revision to upload to. Defaults to the main branch", + default: "main" + }, + { + name: "commit-message", + description: "The commit message to use. Defaults to 'Upload files using @huggingface/hub'", + default: "Upload files using @huggingface/hub" + }, + { + name: "private", + description: "If creating a new repo, make it private", + boolean: true + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + }, + branch: { + description: "Manage repository branches", + subcommands: { + create: { + description: "Create a new branch in a repo, or update an existing one", + args: [ + { + name: "repo-name", + description: "The name of the repo to create the branch in", + positional: true, + required: true + }, + { + name: "branch", + description: "The name of the branch to create", + positional: true, + required: true + }, + { + name: "repo-type", + enum: ["dataset", "model", "space"], + description: "The type of the repo to create the branch into. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name" + }, + { + name: "revision", + description: "The revision to create the branch from. Defaults to the main branch, or existing branch if it exists." + }, + { + name: "empty", + boolean: true, + description: "Create an empty branch. This will erase all previous commits on the branch if it exists." + }, + { + name: "force", + short: "f", + boolean: true, + description: "Overwrite the branch if it already exists. Otherwise, throws an error if the branch already exists. No-ops if no revision is provided and the branch exists." + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + }, + delete: { + description: "Delete a branch in a repo", + args: [ + { + name: "repo-name", + description: "The name of the repo to delete the branch from", + positional: true, + required: true + }, + { + name: "branch", + description: "The name of the branch to delete", + positional: true, + required: true + }, + { + name: "repo-type", + enum: ["dataset", "model", "space"], + description: "The type of repo to delete the branch from. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name" + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + } + } + }, + repo: { + description: "Manage repositories on the Hub", + subcommands: { + delete: { + description: "Delete a repository from the Hub", + args: [ + { + name: "repo-name", + description: "The name of the repo to delete. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + positional: true, + required: true + }, + { + name: "repo-type", + enum: ["dataset", "model", "space"], + description: "The type of the repo to delete. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name" + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + } + } + }, + models: { + description: "Manage models on the Hub", + subcommands: { + list: { + description: "List models on the Hub (first page)", + args: [ + { + name: "search", + description: "Search query to filter models by name", + positional: true + }, + { + name: "sort", + enum: [ + "createdAt", + "downloads", + "likes", + "lastModified", + "likes30d", + "trendingScore", + "num_parameters" + // "mainSize", + // "id", + ], + description: "Sort models by a specific field" + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + } + } + }, + version: { + description: "Print the version of the CLI", + args: [] + }, + jobs: { + description: "Manage jobs on the Hub", + subcommands: { + run: { + description: "Run a new job", + args: [ + { + name: "docker-image-or-space", + description: "The Docker image to run (e.g., python:3.12) or Space ID (e.g., hf.co/spaces/username/space-name or username/space-name)", + positional: true, + required: true + }, + { + name: "command", + description: `The command to run (can be multiple arguments preceded by --, e.g., -- python -c 'import os; print(os.environ["FOO"])')`, + positional: true, + multiple: true + }, + { + name: "env", + short: "e", + multiple: true, + description: "Environment variable in the format KEY=VALUE (can be specified multiple times)" + }, + { + name: "secret", + short: "s", + multiple: true, + description: "Secret in the format KEY=VALUE (will be encrypted server-side, can be specified multiple times)" + }, + { + name: "label", + short: "l", + multiple: true, + description: "Label in the format KEY=VALUE or KEY alone (in this case VALUE defaults to empty string). Can be specified multiple times." + }, + { + name: "volume", + short: "v", + multiple: true, + description: "Volume to mount in the format SOURCE:MOUNTPATH[:OPTIONS]. SOURCE uses HuggingFace prefixes (datasets/user/repo, spaces/user/repo, buckets/user/bucket) or bare user/repo for models. OPTIONS are comma-separated: ro, revision=REV, path=SUBPATH. Can be specified multiple times." + }, + { + name: "flavor", + description: "Hardware flavor to use (defaults to cpu-basic)", + default: "cpu-basic" + }, + { + name: "attempts", + description: "Maximum number of attempts (defaults to 1)" + }, + { + name: "namespace", + description: "The namespace (username or organization name). Defaults to the current user's username." + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + }, + { + name: "detach", + short: "d", + description: "Don't stream logs after creating the job", + boolean: true + } + ] + }, + ps: { + description: "List jobs", + args: [ + { + name: "all", + short: "a", + description: "List all jobs (not just running ones)", + boolean: true + }, + { + name: "namespace", + description: "The namespace (username or organization name). Defaults to the current user's username." + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + }, + hardware: { + description: "List available hardware options for jobs", + args: [ + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + }, + logs: { + description: "Show logs for a job", + args: [ + { + name: "job-id", + description: "The job ID", + positional: true, + required: true + }, + { + name: "namespace", + description: "The namespace (username or organization name). Defaults to the current user's username." + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + } + } + } +}; +var mainCommandName = process.argv[2]; +var subCommandName; +var cliArgs; +if (mainCommandName && mainCommandName in commands && commands[mainCommandName] && "subcommands" in commands[mainCommandName]) { + subCommandName = process.argv[3]; + cliArgs = process.argv.slice(4); +} else { + cliArgs = process.argv.slice(3); +} +async function run() { + switch (mainCommandName) { + case void 0: + case "--help": + case "help": { + const helpArgs = mainCommandName === "help" ? process.argv.slice(3) : []; + if (helpArgs.length > 0) { + const cmdName = helpArgs[0]; + if (cmdName && commands[cmdName]) { + const cmdDef = commands[cmdName]; + if ("subcommands" in cmdDef) { + if (helpArgs.length > 1) { + const subCmdName = helpArgs[1]; + if (subCmdName in cmdDef.subcommands && cmdDef.subcommands[subCmdName]) { + console.log(detailedUsageForSubcommand(cmdName, subCmdName)); + break; + } else { + console.error(`Error: Unknown subcommand '${subCmdName}' for command '${cmdName}'.`); + console.log(listSubcommands(cmdName, cmdDef)); + process.exitCode = 1; + break; + } + } else { + console.log(listSubcommands(cmdName, cmdDef)); + break; + } + } else { + console.log(detailedUsageForCommand(cmdName)); + break; + } + } else { + console.error(`Error: Unknown command '${cmdName}' for help.`); + process.exitCode = 1; + } + } else { + console.log( + `Hugging Face CLI Tools (hfjs) + +Available commands: + +` + typedEntries(commands).map(([name, def]) => ` ${usage(name)}: ${def.description}`).join("\n") + ); + console.log("\nTo get help on a specific command, run `hfjs help ` or `hfjs --help`"); + console.log( + "For commands with subcommands (like 'branch'), run `hfjs help ` or `hfjs --help`" + ); + if (mainCommandName === void 0) { + process.exitCode = 1; + } + } + break; + } + case "upload": { + const cmdDef = commands.upload; + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + console.log(detailedUsageForCommand("upload")); + break; + } + const parsedArgs = advParseArgs(cliArgs, cmdDef.args, "upload"); + const { + repoName, + localFolder, + repoType, + revision, + token, + quiet, + commitMessage, + pathInRepo, + private: isPrivate + } = parsedArgs; + const repoId = repoType ? { type: repoType, name: repoName } : repoName; + if (!await repoExists({ repo: repoId, revision, accessToken: token, hubUrl: process.env.HF_ENDPOINT ?? HUB_URL })) { + if (!quiet) { + console.log(`Repo ${repoName} does not exist. Creating it...`); + } + await createRepo({ + repo: repoId, + accessToken: token, + private: !!isPrivate, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + } + const isFile = (await (0, import_promises7.stat)(localFolder)).isFile(); + const files = isFile ? [ + { + content: (0, import_node_url3.pathToFileURL)(localFolder), + path: (0, import_node_path4.join)(pathInRepo, `${(0, import_node_path4.basename)(localFolder)}`).replace(/^[.]?\//, "") + } + ] : [{ content: (0, import_node_url3.pathToFileURL)(localFolder), path: pathInRepo.replace(/^[.]?\//, "") }]; + const progressManager = new UploadProgressManager(!!quiet); + await progressManager.initialize(); + try { + for await (const event of uploadFilesWithProgress({ + repo: repoId, + files, + branch: revision, + accessToken: token, + commitTitle: commitMessage?.trim().split("\n")[0], + commitDescription: commitMessage?.trim().split("\n").slice(1).join("\n").trim(), + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + useXet: true + })) { + progressManager.handleEvent(event); + } + if (!quiet) { + console.log("\n\u2705 Upload completed successfully!"); + } + } catch (error) { + progressManager.stop(); + throw error; + } finally { + progressManager.stop(); + } + break; + } + case "branch": { + const branchCommandGroup = commands.branch; + const currentSubCommandName = subCommandName; + if (subCommandName === "--help" || subCommandName === "-h") { + console.log(listSubcommands("branch", branchCommandGroup)); + break; + } + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + if (currentSubCommandName && branchCommandGroup.subcommands[currentSubCommandName]) { + console.log(detailedUsageForSubcommand("branch", currentSubCommandName)); + } else { + console.log(listSubcommands("branch", branchCommandGroup)); + } + break; + } + if (!currentSubCommandName || !branchCommandGroup.subcommands[currentSubCommandName]) { + console.error(`Error: Missing or invalid subcommand for 'branch'.`); + console.log(listSubcommands("branch", branchCommandGroup)); + process.exitCode = 1; + break; + } + const subCmdDef = branchCommandGroup.subcommands[currentSubCommandName]; + switch (currentSubCommandName) { + case "create": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "branch create"); + const { repoName, branch, revision, empty, repoType, token, force } = parsedArgs; + await createBranch({ + repo: repoType ? { type: repoType, name: repoName } : repoName, + branch, + accessToken: token, + revision, + empty: empty ?? void 0, + overwrite: force ?? void 0, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + console.log(`Branch '${branch}' created successfully in repo '${repoName}'.`); + break; + } + case "delete": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "branch delete"); + const { repoName, branch, repoType, token } = parsedArgs; + await deleteBranch({ + repo: repoType ? { type: repoType, name: repoName } : repoName, + branch, + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + console.log(`Branch '${branch}' deleted successfully from repo '${repoName}'.`); + break; + } + default: + console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'branch'.`); + console.log(listSubcommands("branch", branchCommandGroup)); + process.exitCode = 1; + break; + } + break; + } + case "repo": { + const repoCommandGroup = commands.repo; + const currentSubCommandName = subCommandName; + if (subCommandName === "--help" || subCommandName === "-h") { + console.log(listSubcommands("repo", repoCommandGroup)); + break; + } + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + if (currentSubCommandName && repoCommandGroup.subcommands[currentSubCommandName]) { + console.log(detailedUsageForSubcommand("repo", currentSubCommandName)); + } else { + console.log(listSubcommands("repo", repoCommandGroup)); + } + break; + } + if (!currentSubCommandName || !repoCommandGroup.subcommands[currentSubCommandName]) { + console.error(`Error: Missing or invalid subcommand for 'repo'.`); + console.log(listSubcommands("repo", repoCommandGroup)); + process.exitCode = 1; + break; + } + const subCmdDef = repoCommandGroup.subcommands[currentSubCommandName]; + switch (currentSubCommandName) { + case "delete": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, `repo ${currentSubCommandName}`); + const { repoName, repoType, token } = parsedArgs; + const repoDesignation = repoType ? { type: repoType, name: repoName } : repoName; + await deleteRepo({ + repo: repoDesignation, + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + console.log(`Repository '${repoName}' deleted successfully.`); + break; + } + default: + console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'repo'.`); + console.log(listSubcommands("repo", repoCommandGroup)); + process.exitCode = 1; + break; + } + break; + } + case "models": { + const modelCommandGroup = commands.models; + const currentSubCommandName = subCommandName; + if (subCommandName === "--help" || subCommandName === "-h") { + console.log(listSubcommands("models", modelCommandGroup)); + break; + } + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + if (currentSubCommandName && modelCommandGroup.subcommands[currentSubCommandName]) { + console.log(detailedUsageForSubcommand("models", currentSubCommandName)); + } else { + console.log(listSubcommands("models", modelCommandGroup)); + } + break; + } + if (!currentSubCommandName || !modelCommandGroup.subcommands[currentSubCommandName]) { + console.error(`Error: Missing or invalid subcommand for 'models'.`); + console.log(listSubcommands("models", modelCommandGroup)); + process.exitCode = 1; + break; + } + const subCmdDef = modelCommandGroup.subcommands[currentSubCommandName]; + switch (currentSubCommandName) { + case "list": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "models list"); + const { search, sort, token } = parsedArgs; + const models = []; + for await (const model of listModels({ + search: search ? { query: search } : void 0, + sort, + limit: 20, + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + })) { + models.push(model); + } + if (models.length === 0) { + console.log("No models found."); + break; + } + console.log( + `${"MODEL".padEnd(45)} ${"TASK".padEnd(25)} ${"DOWNLOADS".padStart(10)} ${"LIKES".padStart(7)} ${"UPDATED"}` + ); + console.log("-".repeat(110)); + for (const model of models) { + const task = model.task || "N/A"; + const updatedAt = model.updatedAt.toLocaleDateString(); + console.log( + `${model.name.padEnd(45)} ${task.padEnd(25)} ${String(model.downloads).padStart(10)} ${String(model.likes).padStart(7)} ${updatedAt}` + ); + } + break; + } + default: + console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'models'.`); + console.log(listSubcommands("models", modelCommandGroup)); + process.exitCode = 1; + break; + } + break; + } + case "version": { + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + console.log(detailedUsageForCommand("version")); + break; + } + console.log(`hfjs version: ${version}`); + break; + } + case "jobs": { + const jobsCommandGroup = commands.jobs; + const currentSubCommandName = subCommandName; + if (subCommandName === "--help" || subCommandName === "-h") { + console.log(listSubcommands("jobs", jobsCommandGroup)); + break; + } + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + if (currentSubCommandName && jobsCommandGroup.subcommands[currentSubCommandName]) { + console.log(detailedUsageForSubcommand("jobs", currentSubCommandName)); + } else { + console.log(listSubcommands("jobs", jobsCommandGroup)); + } + break; + } + if (!currentSubCommandName || !jobsCommandGroup.subcommands[currentSubCommandName]) { + console.error(`Error: Missing or invalid subcommand for 'jobs'.`); + console.log(listSubcommands("jobs", jobsCommandGroup)); + process.exitCode = 1; + break; + } + const subCmdDef = jobsCommandGroup.subcommands[currentSubCommandName]; + switch (currentSubCommandName) { + case "run": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "jobs run"); + const { + dockerImageOrSpace: firstArg, + command: commandArray, + env, + secret, + label, + volume: volumeArgs, + flavor, + attempts: attemptsStr, + namespace, + token, + detach + } = parsedArgs; + const envVars = env; + const secretVars = secret; + const labelVars = label; + let attempts; + if (attemptsStr) { + const parsed = parseInt(attemptsStr, 10); + if (isNaN(parsed) || parsed < 1) { + throw new Error("Attempts must be a positive integer"); + } + attempts = parsed; + } + let dockerImage; + let spaceId; + const hfCoSpacesMatch = firstArg.match(/^hf\.co\/spaces\/(.+)$/); + if (hfCoSpacesMatch) { + spaceId = hfCoSpacesMatch[1]; + } else { + dockerImage = firstArg; + } + let finalNamespace = namespace; + if (!finalNamespace) { + if (!token) { + throw new Error( + "Cannot determine namespace without authentication. Please provide --namespace or --token." + ); + } + const userInfo = await whoAmI({ + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + if (userInfo.type !== "user") { + throw new Error("Cannot determine namespace. Please provide --namespace explicitly."); + } + finalNamespace = userInfo.name; + } + const environment = {}; + if (envVars) { + for (const envVar of envVars) { + const equalIndex = envVar.indexOf("="); + if (equalIndex === -1) { + throw new Error(`Invalid environment variable format: ${envVar}. Expected KEY=VALUE`); + } + const key = envVar.slice(0, equalIndex); + const value = envVar.slice(equalIndex + 1); + environment[key] = value; + } + } + const secrets = {}; + if (secretVars) { + for (const secretVar of secretVars) { + const equalIndex = secretVar.indexOf("="); + if (equalIndex === -1) { + throw new Error(`Invalid secret format: ${secretVar}. Expected KEY=VALUE`); + } + const key = secretVar.slice(0, equalIndex); + const value = secretVar.slice(equalIndex + 1); + secrets[key] = value; + } + } + const labels = {}; + if (labelVars) { + for (const labelVar of labelVars) { + const equalIndex = labelVar.indexOf("="); + const [key, value] = equalIndex > -1 ? [labelVar.slice(0, equalIndex), labelVar.slice(equalIndex + 1)] : [labelVar, ""]; + labels[key] = value; + } + } + const volumes = []; + if (volumeArgs) { + for (const volumeArg of volumeArgs) { + const colonIdx = volumeArg.indexOf(":"); + if (colonIdx === -1) { + throw new Error(`Invalid volume format: ${volumeArg}. Expected SOURCE:MOUNTPATH[:OPTIONS]`); + } + const source = volumeArg.slice(0, colonIdx); + const rest = volumeArg.slice(colonIdx + 1); + const secondColon = rest.indexOf(":"); + const mountPath = secondColon === -1 ? rest : rest.slice(0, secondColon); + const optionsStr = secondColon === -1 ? "" : rest.slice(secondColon + 1); + if (!mountPath.startsWith("/")) { + throw new Error(`Volume mountPath must start with "/": ${mountPath}`); + } + let readOnly; + let revision; + let subPath; + if (optionsStr) { + for (const opt of optionsStr.split(",")) { + if (opt === "ro") { + readOnly = true; + } else if (opt.startsWith("revision=")) { + revision = opt.slice("revision=".length); + } else if (opt.startsWith("path=")) { + subPath = opt.slice("path=".length); + } else { + throw new Error(`Unknown volume option: ${opt}`); + } + } + } + volumes.push({ + source, + mountPath, + ...revision ? { revision } : {}, + ...readOnly ? { readOnly } : {}, + ...subPath ? { path: subPath } : {} + }); + } + } + const jobParams = { + namespace: finalNamespace, + ...dockerImage ? { dockerImage } : {}, + ...spaceId ? { spaceId } : {}, + flavor, + command: commandArray.length > 0 ? commandArray : void 0, + environment, + secrets, + ...attempts !== void 0 ? { attempts } : {}, + ...Object.keys(labels).length > 0 ? { labels } : {}, + ...volumes.length > 0 ? { volumes } : {}, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + ...token ? { accessToken: token } : {} + }; + const job = await runJob(jobParams); + console.log(`Job created: ${job.id}`); + console.log(`Status: ${job.status.stage}`); + if (!detach) { + const logsParams = { + namespace: finalNamespace, + jobId: job.id, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + ...token ? { accessToken: token } : {} + }; + for await (const logChunk of streamJobLogs(logsParams)) { + console.log(logChunk.message); + } + } + break; + } + case "ps": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "jobs ps"); + const { all, namespace, token } = parsedArgs; + let finalNamespace = namespace; + if (!finalNamespace) { + if (!token) { + throw new Error( + "Cannot determine namespace without authentication. Please provide --namespace or --token." + ); + } + const userInfo = await whoAmI({ + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + if (userInfo.type !== "user") { + throw new Error("Cannot determine namespace. Please provide --namespace explicitly."); + } + finalNamespace = userInfo.name; + } + const jobs = await listJobs({ + namespace: finalNamespace, + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + const filteredJobs = all ? jobs : jobs.filter((job) => job.status.stage === "RUNNING"); + if (filteredJobs.length === 0) { + console.log(all ? "No jobs found." : "No running jobs found."); + break; + } + console.log(`${"ID".padEnd(40)} ${"STATUS".padEnd(12)} ${"CREATED".padEnd(20)} ${"DOCKER IMAGE"}`); + console.log("-".repeat(100)); + for (const job of filteredJobs) { + const createdAt = new Date(job.createdAt).toLocaleString(); + const dockerImage = job.dockerImage || job.spaceId || "N/A"; + const status = job.status.stage; + console.log(`${job.id.padEnd(40)} ${status.padEnd(12)} ${createdAt.padEnd(20)} ${dockerImage}`); + } + break; + } + case "hardware": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "jobs hardware"); + const { token } = parsedArgs; + const hardwareParams = { + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }; + if (token) { + hardwareParams.accessToken = token; + } + const hardware = await listJobHardware(hardwareParams); + console.log( + `${"NAME".padEnd(15)} ${"PRETTY NAME".padEnd(22)} ${"CPU".padEnd(8)} ${"RAM".padEnd(7)} ${"ACCELERATOR".padEnd(16)} ${"COST/MIN".padEnd(9)} ${"COST/HOUR"}` + ); + console.log("-".repeat(100)); + for (const hw of hardware) { + let accelerator = "N/A"; + if (hw.accelerator) { + accelerator = `${hw.accelerator.quantity}x ${hw.accelerator.model} (${hw.accelerator.vram})`; + } + const costPerMin = (hw.unitCostMicroUSD / 1e6).toFixed(4); + const costPerHour = (hw.unitCostMicroUSD / 1e6 * 60).toFixed(2); + console.log( + `${hw.name.padEnd(15)} ${hw.prettyName.padEnd(22)} ${hw.cpu.padEnd(8)} ${hw.ram.padEnd(7)} ${accelerator.padEnd(16)} $${costPerMin.padStart(8)} $${costPerHour.padStart(9)}` + ); + } + break; + } + case "logs": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "jobs logs"); + const { jobId, namespace, token } = parsedArgs; + let finalNamespace = namespace; + if (!finalNamespace) { + if (!token) { + throw new Error( + "Cannot determine namespace without authentication. Please provide --namespace or --token." + ); + } + const userInfo = await whoAmI({ + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + if (userInfo.type !== "user") { + throw new Error("Cannot determine namespace. Please provide --namespace explicitly."); + } + finalNamespace = userInfo.name; + } + const logsParams = { + namespace: finalNamespace, + jobId, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + ...token ? { accessToken: token } : {} + }; + const jobInfoParams = { + namespace: finalNamespace, + jobId, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + ...token ? { accessToken: token } : {} + }; + const jobInfo = await getJob(jobInfoParams); + if (jobInfo.status.stage === "ERROR" && jobInfo.status.message) { + console.error(` +\u274C Job failed: ${jobInfo.status.message} +`); + } + for await (const logChunk of streamJobLogs(logsParams)) { + console.log(logChunk.message); + } + break; + } + default: + console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'jobs'.`); + console.log(listSubcommands("jobs", jobsCommandGroup)); + process.exitCode = 1; + break; + } + break; + } + default: + console.error("Command not found: " + mainCommandName); + console.log( + ` +Available commands: + +` + typedEntries(commands).map(([name, def]) => ` ${usage(name)}: ${def.description}`).join("\n") + ); + console.log("\nTo get help on a specific command, run `hfjs help ` or `hfjs --help`"); + process.exitCode = 1; + break; + } +} +run().catch((err) => { + console.error("\x1B[31mError:\x1B[0m", err.message); + console.error(err); + process.exitCode = 1; +}); +function usage(commandName, subCommandName2) { + const commandEntry = commands[commandName]; + let cmdArgs; + let fullCommandName = commandName; + if ("subcommands" in commandEntry) { + if (subCommandName2 && subCommandName2 in commandEntry.subcommands) { + const subCmd = commandEntry.subcommands[subCommandName2]; + cmdArgs = subCmd.args; + fullCommandName = `${commandName} ${subCommandName2}`; + } else { + return `${commandName} `; + } + } else { + cmdArgs = commandEntry.args; + } + return `${fullCommandName} ${(cmdArgs || []).map((arg) => { + if (arg.positional) { + return arg.required ? `<${arg.name}>` : `[${arg.name}]`; + } + return `[--${arg.name}${arg.short ? `|-${arg.short}` : ""}${arg.enum ? ` {${arg.enum.join("|")}}` : arg.boolean ? "" : ` <${arg.name.toUpperCase().replace(/-/g, "_")}>`}]`; + }).join(" ")}`.trim(); +} +function _detailedUsage(args, usageLine, commandDescription) { + let ret = `usage: hfjs ${usageLine} +`; + if (commandDescription) { + ret += ` +${commandDescription} +`; + } + const positionals = args.filter((p) => p.positional); + const options = args.filter((p) => !p.positional); + if (positionals.length > 0) { + ret += ` +Positional arguments: +`; + for (const arg of positionals) { + ret += ` ${arg.name} ${arg.description}${arg.default ? ` (default: ${typeof arg.default === "function" ? arg.default() : arg.default})` : ""} +`; + } + } + if (options.length > 0) { + ret += ` +Options: +`; + for (const arg of options) { + const nameAndAlias = `--${arg.name}${arg.short ? `, -${arg.short}` : ""}`; + const valueHint = arg.enum ? `{${arg.enum.join("|")}}` : arg.boolean ? "" : `<${arg.name.toUpperCase().replace(/-/g, "_")}>`; + ret += ` ${nameAndAlias}${valueHint ? " " + valueHint : ""} ${arg.description}${arg.default !== void 0 ? ` (default: ${typeof arg.default === "function" ? arg.default() : arg.default})` : ""} +`; + } + } + ret += ` +`; + return ret; +} +function detailedUsageForCommand(commandName) { + const commandDef = commands[commandName]; + if ("subcommands" in commandDef) { + return listSubcommands(commandName, commandDef); + } + return _detailedUsage(commandDef.args, usage(commandName), commandDef.description); +} +function detailedUsageForSubcommand(commandName, subCommandName2) { + const commandGroup = commands[commandName]; + if (!("subcommands" in commandGroup)) { + throw new Error(`Command ${commandName} does not have subcommands`); + } + if (!(subCommandName2 in commandGroup.subcommands)) { + throw new Error(`Subcommand ${subCommandName2} not found for ${commandName}`); + } + const subCommandDef = commandGroup.subcommands[subCommandName2]; + return _detailedUsage(subCommandDef.args, usage(commandName, subCommandName2), subCommandDef.description); +} +function listSubcommands(commandName, commandGroup) { + let ret = `usage: hfjs ${commandName} [options] + +`; + ret += `${commandGroup.description} + +`; + ret += `Available subcommands for '${commandName}': +`; + ret += typedEntries(commandGroup.subcommands).map(([subName, subDef]) => ` ${subName} ${subDef.description}`).join("\n"); + if (commandName === "jobs") { + ret += ` + +Example: + hfjs jobs run -e FOO=foo -e BAR=bar python:3.12 -- python -c 'import os; print(os.environ["FOO"], os.environ["BAR"])'`; + } + ret += ` + +Run \`hfjs help ${commandName} \` for more information on a specific subcommand.`; + return ret; +} +function advParseArgs(args, argDefs, commandNameForError) { + const hasMultiplePositional = argDefs.some((arg) => arg.multiple && arg.positional); + const { tokens } = (0, import_node_util.parseArgs)({ + options: Object.fromEntries( + argDefs.filter((arg) => !arg.positional).map((arg) => { + const optionConfig = { + type: arg.boolean ? "boolean" : "string", + ...arg.short && { short: arg.short }, + ...arg.multiple && { multiple: true }, + ...arg.default !== void 0 && { + default: typeof arg.default === "function" ? arg.default() : arg.default + } + }; + return [arg.name, optionConfig]; + }) + ), + args, + allowPositionals: true, + strict: false, + // We do custom validation based on tokens and argDefs + tokens: true + }); + const expectedPositionals = argDefs.filter((arg) => arg.positional); + const providedPositionalTokens = tokens.filter((token) => token.kind === "positional"); + if (providedPositionalTokens.length < expectedPositionals.filter((arg) => arg.required).length) { + throw new Error( + `Command '${commandNameForError}': Missing required positional arguments. Usage: hfjs ${usage( + commandNameForError.split(" ")[0], + commandNameForError.split(" ")[1] + )}` + ); + } + if (providedPositionalTokens.length > expectedPositionals.length && !hasMultiplePositional) { + throw new Error( + `Command '${commandNameForError}': Too many positional arguments. Usage: hfjs ${usage( + commandNameForError.split(" ")[0], + commandNameForError.split(" ")[1] + )}` + ); + } + const result = {}; + for (const argDef of argDefs) { + if (argDef.default !== void 0) { + result[argDef.name] = typeof argDef.default === "function" ? argDef.default() : argDef.default; + } else if (argDef.boolean) { + result[argDef.name] = false; + } + } + expectedPositionals.forEach((argDef, i) => { + if (argDef.multiple) { + result[argDef.name] = providedPositionalTokens.slice(i).map((token) => token.value); + } else if (providedPositionalTokens[i]) { + result[argDef.name] = providedPositionalTokens[i].value; + } + }); + tokens.filter((token) => token.kind === "option").forEach((token) => { + const argDef = argDefs.find((def) => def.name === token.name || def.short === token.name); + if (!argDef) { + throw new Error(`Command '${commandNameForError}': Unknown option: ${token.rawName}`); + } + if (argDef.boolean) { + result[argDef.name] = true; + } else { + if (token.value === void 0) { + throw new Error(`Command '${commandNameForError}': Missing value for option: ${token.rawName}`); + } + if (argDef.enum && !argDef.enum.includes(token.value)) { + throw new Error( + `Command '${commandNameForError}': Invalid value '${token.value}' for option ${token.rawName}. Expected one of: ${argDef.enum.join(", ")}` + ); + } + if (argDef.multiple) { + const existing = result[argDef.name] || []; + existing.push(token.value); + result[argDef.name] = existing; + } else { + result[argDef.name] = token.value; + } + } + }); + for (const argDef of argDefs) { + if (argDef.required && result[argDef.name] === void 0) { + throw new Error(`Command '${commandNameForError}': Missing required argument: ${argDef.name}`); + } + } + return Object.fromEntries( + Object.entries(result).map(([name, val]) => [kebabToCamelCase(name), val]) + ); +} +function kebabToCamelCase(str) { + return str.replace(/-./g, (match) => match[1].toUpperCase()); +} diff --git a/node_modules/@huggingface/hub/dist/cli.mjs b/node_modules/@huggingface/hub/dist/cli.mjs new file mode 100644 index 0000000000000000000000000000000000000000..548e14b6a75e4d8a66a4d626872559dc5ef5ba79 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/cli.mjs @@ -0,0 +1,1232 @@ +#! /usr/bin/env node +import { + HUB_URL, + createBranch, + createRepo, + deleteBranch, + deleteRepo, + getJob, + listJobHardware, + listJobs, + listModels, + repoExists, + runJob, + streamJobLogs, + typedEntries, + uploadFilesWithProgress, + whoAmI +} from "./chunk-OPQ3EOKY.mjs"; +import "./chunk-FFYIGW52.mjs"; + +// cli.ts +import { parseArgs } from "util"; +import { pathToFileURL } from "url"; +import { stat } from "fs/promises"; +import { basename, join } from "path"; + +// package.json +var version = "2.13.2"; + +// cli.ts +var UploadProgressManager = class { + multibar = null; + fileBars = /* @__PURE__ */ new Map(); + isQuiet; + cliProgressAvailable = false; + constructor(isQuiet = false) { + this.isQuiet = isQuiet; + } + async initialize() { + if (this.isQuiet) { + return; + } + try { + const cliProgress = await import("./cli-progress-XP5T6RZP.mjs"); + this.cliProgressAvailable = true; + this.multibar = new cliProgress.MultiBar( + { + clearOnComplete: false, + hideCursor: true, + format: " {bar} | {filename} | {percentage}% | {state}", + barCompleteChar: "\u2588", + barIncompleteChar: "\u2591" + }, + cliProgress.Presets.shades_grey + ); + } catch (error) { + this.cliProgressAvailable = false; + } + } + handleEvent(event) { + if (this.isQuiet) { + return; + } + if (event.event === "phase") { + this.logPhase(event.phase); + } else if (event.event === "fileProgress") { + this.updateFileProgress(event.path, event.progress, event.state); + } + } + logPhase(phase) { + if (this.isQuiet) { + return; + } + const phaseMessages = { + preuploading: "\u{1F4CB} Preparing files for upload...", + uploadingLargeFiles: "\u2B06\uFE0F Uploading files...", + committing: "\u2728 Finalizing commit..." + }; + console.log(` +${phaseMessages[phase] || phase}`); + } + updateFileProgress(path, progress, state) { + if (this.isQuiet) { + return; + } + if (this.cliProgressAvailable && this.multibar) { + let bar = this.fileBars.get(path); + if (!bar) { + bar = this.multibar.create(100, 0, { + filename: this.truncateFilename(path, 100), + state + }); + this.fileBars.set(path, bar); + } + if (state === "error") { + bar.update(0, { state: "\u2717 error" }); + } else if (progress >= 1) { + bar.update(100, { state: state === "hashing" ? "\u2713 hashed" : "\u2713 uploaded" }); + } else { + const percentage = Math.round(progress * 100); + bar.update(percentage, { state }); + } + } else { + const percentage = Math.round(progress * 100); + const truncatedPath = this.truncateFilename(path, 100); + if (state === "error") { + console.error(`\u2717 error: ${truncatedPath}`); + } else if (progress >= 1) { + const statusIcon = state === "hashing" ? "\u2713 hashed" : "\u2713 uploaded"; + console.log(`${statusIcon}: ${truncatedPath}`); + } else if (percentage % 25 === 0) { + console.log(`${state}: ${truncatedPath} (${percentage}%)`); + } + } + } + truncateFilename(filename, maxLength) { + if (filename.length <= maxLength) { + return filename; + } + return "..." + filename.slice(-(maxLength - 3)); + } + stop() { + if (!this.isQuiet && this.cliProgressAvailable && this.multibar) { + this.multibar.stop(); + } + } +}; +var commands = { + upload: { + description: "Upload a folder to a repo on the Hub", + args: [ + { + name: "repo-name", + description: "The name of the repo to upload to", + positional: true, + required: true + }, + { + name: "local-folder", + description: "The local folder to upload. Defaults to the current working directory", + positional: true, + default: () => process.cwd() + }, + { + name: "path-in-repo", + description: "The path in the repo to upload the folder to. Defaults to the root of the repo", + positional: true, + default: "." + }, + { + name: "quiet", + short: "q", + description: "Suppress all output", + boolean: true + }, + { + name: "repo-type", + enum: ["dataset", "model", "space", "bucket"], + description: "The type of repo to upload to. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name or buckets/username/repo-name" + }, + { + name: "revision", + description: "The revision to upload to. Defaults to the main branch", + default: "main" + }, + { + name: "commit-message", + description: "The commit message to use. Defaults to 'Upload files using @huggingface/hub'", + default: "Upload files using @huggingface/hub" + }, + { + name: "private", + description: "If creating a new repo, make it private", + boolean: true + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + }, + branch: { + description: "Manage repository branches", + subcommands: { + create: { + description: "Create a new branch in a repo, or update an existing one", + args: [ + { + name: "repo-name", + description: "The name of the repo to create the branch in", + positional: true, + required: true + }, + { + name: "branch", + description: "The name of the branch to create", + positional: true, + required: true + }, + { + name: "repo-type", + enum: ["dataset", "model", "space"], + description: "The type of the repo to create the branch into. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name" + }, + { + name: "revision", + description: "The revision to create the branch from. Defaults to the main branch, or existing branch if it exists." + }, + { + name: "empty", + boolean: true, + description: "Create an empty branch. This will erase all previous commits on the branch if it exists." + }, + { + name: "force", + short: "f", + boolean: true, + description: "Overwrite the branch if it already exists. Otherwise, throws an error if the branch already exists. No-ops if no revision is provided and the branch exists." + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + }, + delete: { + description: "Delete a branch in a repo", + args: [ + { + name: "repo-name", + description: "The name of the repo to delete the branch from", + positional: true, + required: true + }, + { + name: "branch", + description: "The name of the branch to delete", + positional: true, + required: true + }, + { + name: "repo-type", + enum: ["dataset", "model", "space"], + description: "The type of repo to delete the branch from. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name" + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + } + } + }, + repo: { + description: "Manage repositories on the Hub", + subcommands: { + delete: { + description: "Delete a repository from the Hub", + args: [ + { + name: "repo-name", + description: "The name of the repo to delete. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + positional: true, + required: true + }, + { + name: "repo-type", + enum: ["dataset", "model", "space"], + description: "The type of the repo to delete. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name" + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + } + } + }, + models: { + description: "Manage models on the Hub", + subcommands: { + list: { + description: "List models on the Hub (first page)", + args: [ + { + name: "search", + description: "Search query to filter models by name", + positional: true + }, + { + name: "sort", + enum: [ + "createdAt", + "downloads", + "likes", + "lastModified", + "likes30d", + "trendingScore", + "num_parameters" + // "mainSize", + // "id", + ], + description: "Sort models by a specific field" + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + } + } + }, + version: { + description: "Print the version of the CLI", + args: [] + }, + jobs: { + description: "Manage jobs on the Hub", + subcommands: { + run: { + description: "Run a new job", + args: [ + { + name: "docker-image-or-space", + description: "The Docker image to run (e.g., python:3.12) or Space ID (e.g., hf.co/spaces/username/space-name or username/space-name)", + positional: true, + required: true + }, + { + name: "command", + description: `The command to run (can be multiple arguments preceded by --, e.g., -- python -c 'import os; print(os.environ["FOO"])')`, + positional: true, + multiple: true + }, + { + name: "env", + short: "e", + multiple: true, + description: "Environment variable in the format KEY=VALUE (can be specified multiple times)" + }, + { + name: "secret", + short: "s", + multiple: true, + description: "Secret in the format KEY=VALUE (will be encrypted server-side, can be specified multiple times)" + }, + { + name: "label", + short: "l", + multiple: true, + description: "Label in the format KEY=VALUE or KEY alone (in this case VALUE defaults to empty string). Can be specified multiple times." + }, + { + name: "volume", + short: "v", + multiple: true, + description: "Volume to mount in the format SOURCE:MOUNTPATH[:OPTIONS]. SOURCE uses HuggingFace prefixes (datasets/user/repo, spaces/user/repo, buckets/user/bucket) or bare user/repo for models. OPTIONS are comma-separated: ro, revision=REV, path=SUBPATH. Can be specified multiple times." + }, + { + name: "flavor", + description: "Hardware flavor to use (defaults to cpu-basic)", + default: "cpu-basic" + }, + { + name: "attempts", + description: "Maximum number of attempts (defaults to 1)" + }, + { + name: "namespace", + description: "The namespace (username or organization name). Defaults to the current user's username." + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + }, + { + name: "detach", + short: "d", + description: "Don't stream logs after creating the job", + boolean: true + } + ] + }, + ps: { + description: "List jobs", + args: [ + { + name: "all", + short: "a", + description: "List all jobs (not just running ones)", + boolean: true + }, + { + name: "namespace", + description: "The namespace (username or organization name). Defaults to the current user's username." + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + }, + hardware: { + description: "List available hardware options for jobs", + args: [ + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + }, + logs: { + description: "Show logs for a job", + args: [ + { + name: "job-id", + description: "The job ID", + positional: true, + required: true + }, + { + name: "namespace", + description: "The namespace (username or organization name). Defaults to the current user's username." + }, + { + name: "token", + description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN + } + ] + } + } + } +}; +var mainCommandName = process.argv[2]; +var subCommandName; +var cliArgs; +if (mainCommandName && mainCommandName in commands && commands[mainCommandName] && "subcommands" in commands[mainCommandName]) { + subCommandName = process.argv[3]; + cliArgs = process.argv.slice(4); +} else { + cliArgs = process.argv.slice(3); +} +async function run() { + switch (mainCommandName) { + case void 0: + case "--help": + case "help": { + const helpArgs = mainCommandName === "help" ? process.argv.slice(3) : []; + if (helpArgs.length > 0) { + const cmdName = helpArgs[0]; + if (cmdName && commands[cmdName]) { + const cmdDef = commands[cmdName]; + if ("subcommands" in cmdDef) { + if (helpArgs.length > 1) { + const subCmdName = helpArgs[1]; + if (subCmdName in cmdDef.subcommands && cmdDef.subcommands[subCmdName]) { + console.log(detailedUsageForSubcommand(cmdName, subCmdName)); + break; + } else { + console.error(`Error: Unknown subcommand '${subCmdName}' for command '${cmdName}'.`); + console.log(listSubcommands(cmdName, cmdDef)); + process.exitCode = 1; + break; + } + } else { + console.log(listSubcommands(cmdName, cmdDef)); + break; + } + } else { + console.log(detailedUsageForCommand(cmdName)); + break; + } + } else { + console.error(`Error: Unknown command '${cmdName}' for help.`); + process.exitCode = 1; + } + } else { + console.log( + `Hugging Face CLI Tools (hfjs) + +Available commands: + +` + typedEntries(commands).map(([name, def]) => ` ${usage(name)}: ${def.description}`).join("\n") + ); + console.log("\nTo get help on a specific command, run `hfjs help ` or `hfjs --help`"); + console.log( + "For commands with subcommands (like 'branch'), run `hfjs help ` or `hfjs --help`" + ); + if (mainCommandName === void 0) { + process.exitCode = 1; + } + } + break; + } + case "upload": { + const cmdDef = commands.upload; + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + console.log(detailedUsageForCommand("upload")); + break; + } + const parsedArgs = advParseArgs(cliArgs, cmdDef.args, "upload"); + const { + repoName, + localFolder, + repoType, + revision, + token, + quiet, + commitMessage, + pathInRepo, + private: isPrivate + } = parsedArgs; + const repoId = repoType ? { type: repoType, name: repoName } : repoName; + if (!await repoExists({ repo: repoId, revision, accessToken: token, hubUrl: process.env.HF_ENDPOINT ?? HUB_URL })) { + if (!quiet) { + console.log(`Repo ${repoName} does not exist. Creating it...`); + } + await createRepo({ + repo: repoId, + accessToken: token, + private: !!isPrivate, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + } + const isFile = (await stat(localFolder)).isFile(); + const files = isFile ? [ + { + content: pathToFileURL(localFolder), + path: join(pathInRepo, `${basename(localFolder)}`).replace(/^[.]?\//, "") + } + ] : [{ content: pathToFileURL(localFolder), path: pathInRepo.replace(/^[.]?\//, "") }]; + const progressManager = new UploadProgressManager(!!quiet); + await progressManager.initialize(); + try { + for await (const event of uploadFilesWithProgress({ + repo: repoId, + files, + branch: revision, + accessToken: token, + commitTitle: commitMessage?.trim().split("\n")[0], + commitDescription: commitMessage?.trim().split("\n").slice(1).join("\n").trim(), + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + useXet: true + })) { + progressManager.handleEvent(event); + } + if (!quiet) { + console.log("\n\u2705 Upload completed successfully!"); + } + } catch (error) { + progressManager.stop(); + throw error; + } finally { + progressManager.stop(); + } + break; + } + case "branch": { + const branchCommandGroup = commands.branch; + const currentSubCommandName = subCommandName; + if (subCommandName === "--help" || subCommandName === "-h") { + console.log(listSubcommands("branch", branchCommandGroup)); + break; + } + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + if (currentSubCommandName && branchCommandGroup.subcommands[currentSubCommandName]) { + console.log(detailedUsageForSubcommand("branch", currentSubCommandName)); + } else { + console.log(listSubcommands("branch", branchCommandGroup)); + } + break; + } + if (!currentSubCommandName || !branchCommandGroup.subcommands[currentSubCommandName]) { + console.error(`Error: Missing or invalid subcommand for 'branch'.`); + console.log(listSubcommands("branch", branchCommandGroup)); + process.exitCode = 1; + break; + } + const subCmdDef = branchCommandGroup.subcommands[currentSubCommandName]; + switch (currentSubCommandName) { + case "create": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "branch create"); + const { repoName, branch, revision, empty, repoType, token, force } = parsedArgs; + await createBranch({ + repo: repoType ? { type: repoType, name: repoName } : repoName, + branch, + accessToken: token, + revision, + empty: empty ?? void 0, + overwrite: force ?? void 0, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + console.log(`Branch '${branch}' created successfully in repo '${repoName}'.`); + break; + } + case "delete": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "branch delete"); + const { repoName, branch, repoType, token } = parsedArgs; + await deleteBranch({ + repo: repoType ? { type: repoType, name: repoName } : repoName, + branch, + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + console.log(`Branch '${branch}' deleted successfully from repo '${repoName}'.`); + break; + } + default: + console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'branch'.`); + console.log(listSubcommands("branch", branchCommandGroup)); + process.exitCode = 1; + break; + } + break; + } + case "repo": { + const repoCommandGroup = commands.repo; + const currentSubCommandName = subCommandName; + if (subCommandName === "--help" || subCommandName === "-h") { + console.log(listSubcommands("repo", repoCommandGroup)); + break; + } + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + if (currentSubCommandName && repoCommandGroup.subcommands[currentSubCommandName]) { + console.log(detailedUsageForSubcommand("repo", currentSubCommandName)); + } else { + console.log(listSubcommands("repo", repoCommandGroup)); + } + break; + } + if (!currentSubCommandName || !repoCommandGroup.subcommands[currentSubCommandName]) { + console.error(`Error: Missing or invalid subcommand for 'repo'.`); + console.log(listSubcommands("repo", repoCommandGroup)); + process.exitCode = 1; + break; + } + const subCmdDef = repoCommandGroup.subcommands[currentSubCommandName]; + switch (currentSubCommandName) { + case "delete": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, `repo ${currentSubCommandName}`); + const { repoName, repoType, token } = parsedArgs; + const repoDesignation = repoType ? { type: repoType, name: repoName } : repoName; + await deleteRepo({ + repo: repoDesignation, + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + console.log(`Repository '${repoName}' deleted successfully.`); + break; + } + default: + console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'repo'.`); + console.log(listSubcommands("repo", repoCommandGroup)); + process.exitCode = 1; + break; + } + break; + } + case "models": { + const modelCommandGroup = commands.models; + const currentSubCommandName = subCommandName; + if (subCommandName === "--help" || subCommandName === "-h") { + console.log(listSubcommands("models", modelCommandGroup)); + break; + } + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + if (currentSubCommandName && modelCommandGroup.subcommands[currentSubCommandName]) { + console.log(detailedUsageForSubcommand("models", currentSubCommandName)); + } else { + console.log(listSubcommands("models", modelCommandGroup)); + } + break; + } + if (!currentSubCommandName || !modelCommandGroup.subcommands[currentSubCommandName]) { + console.error(`Error: Missing or invalid subcommand for 'models'.`); + console.log(listSubcommands("models", modelCommandGroup)); + process.exitCode = 1; + break; + } + const subCmdDef = modelCommandGroup.subcommands[currentSubCommandName]; + switch (currentSubCommandName) { + case "list": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "models list"); + const { search, sort, token } = parsedArgs; + const models = []; + for await (const model of listModels({ + search: search ? { query: search } : void 0, + sort, + limit: 20, + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + })) { + models.push(model); + } + if (models.length === 0) { + console.log("No models found."); + break; + } + console.log( + `${"MODEL".padEnd(45)} ${"TASK".padEnd(25)} ${"DOWNLOADS".padStart(10)} ${"LIKES".padStart(7)} ${"UPDATED"}` + ); + console.log("-".repeat(110)); + for (const model of models) { + const task = model.task || "N/A"; + const updatedAt = model.updatedAt.toLocaleDateString(); + console.log( + `${model.name.padEnd(45)} ${task.padEnd(25)} ${String(model.downloads).padStart(10)} ${String(model.likes).padStart(7)} ${updatedAt}` + ); + } + break; + } + default: + console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'models'.`); + console.log(listSubcommands("models", modelCommandGroup)); + process.exitCode = 1; + break; + } + break; + } + case "version": { + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + console.log(detailedUsageForCommand("version")); + break; + } + console.log(`hfjs version: ${version}`); + break; + } + case "jobs": { + const jobsCommandGroup = commands.jobs; + const currentSubCommandName = subCommandName; + if (subCommandName === "--help" || subCommandName === "-h") { + console.log(listSubcommands("jobs", jobsCommandGroup)); + break; + } + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + if (currentSubCommandName && jobsCommandGroup.subcommands[currentSubCommandName]) { + console.log(detailedUsageForSubcommand("jobs", currentSubCommandName)); + } else { + console.log(listSubcommands("jobs", jobsCommandGroup)); + } + break; + } + if (!currentSubCommandName || !jobsCommandGroup.subcommands[currentSubCommandName]) { + console.error(`Error: Missing or invalid subcommand for 'jobs'.`); + console.log(listSubcommands("jobs", jobsCommandGroup)); + process.exitCode = 1; + break; + } + const subCmdDef = jobsCommandGroup.subcommands[currentSubCommandName]; + switch (currentSubCommandName) { + case "run": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "jobs run"); + const { + dockerImageOrSpace: firstArg, + command: commandArray, + env, + secret, + label, + volume: volumeArgs, + flavor, + attempts: attemptsStr, + namespace, + token, + detach + } = parsedArgs; + const envVars = env; + const secretVars = secret; + const labelVars = label; + let attempts; + if (attemptsStr) { + const parsed = parseInt(attemptsStr, 10); + if (isNaN(parsed) || parsed < 1) { + throw new Error("Attempts must be a positive integer"); + } + attempts = parsed; + } + let dockerImage; + let spaceId; + const hfCoSpacesMatch = firstArg.match(/^hf\.co\/spaces\/(.+)$/); + if (hfCoSpacesMatch) { + spaceId = hfCoSpacesMatch[1]; + } else { + dockerImage = firstArg; + } + let finalNamespace = namespace; + if (!finalNamespace) { + if (!token) { + throw new Error( + "Cannot determine namespace without authentication. Please provide --namespace or --token." + ); + } + const userInfo = await whoAmI({ + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + if (userInfo.type !== "user") { + throw new Error("Cannot determine namespace. Please provide --namespace explicitly."); + } + finalNamespace = userInfo.name; + } + const environment = {}; + if (envVars) { + for (const envVar of envVars) { + const equalIndex = envVar.indexOf("="); + if (equalIndex === -1) { + throw new Error(`Invalid environment variable format: ${envVar}. Expected KEY=VALUE`); + } + const key = envVar.slice(0, equalIndex); + const value = envVar.slice(equalIndex + 1); + environment[key] = value; + } + } + const secrets = {}; + if (secretVars) { + for (const secretVar of secretVars) { + const equalIndex = secretVar.indexOf("="); + if (equalIndex === -1) { + throw new Error(`Invalid secret format: ${secretVar}. Expected KEY=VALUE`); + } + const key = secretVar.slice(0, equalIndex); + const value = secretVar.slice(equalIndex + 1); + secrets[key] = value; + } + } + const labels = {}; + if (labelVars) { + for (const labelVar of labelVars) { + const equalIndex = labelVar.indexOf("="); + const [key, value] = equalIndex > -1 ? [labelVar.slice(0, equalIndex), labelVar.slice(equalIndex + 1)] : [labelVar, ""]; + labels[key] = value; + } + } + const volumes = []; + if (volumeArgs) { + for (const volumeArg of volumeArgs) { + const colonIdx = volumeArg.indexOf(":"); + if (colonIdx === -1) { + throw new Error(`Invalid volume format: ${volumeArg}. Expected SOURCE:MOUNTPATH[:OPTIONS]`); + } + const source = volumeArg.slice(0, colonIdx); + const rest = volumeArg.slice(colonIdx + 1); + const secondColon = rest.indexOf(":"); + const mountPath = secondColon === -1 ? rest : rest.slice(0, secondColon); + const optionsStr = secondColon === -1 ? "" : rest.slice(secondColon + 1); + if (!mountPath.startsWith("/")) { + throw new Error(`Volume mountPath must start with "/": ${mountPath}`); + } + let readOnly; + let revision; + let subPath; + if (optionsStr) { + for (const opt of optionsStr.split(",")) { + if (opt === "ro") { + readOnly = true; + } else if (opt.startsWith("revision=")) { + revision = opt.slice("revision=".length); + } else if (opt.startsWith("path=")) { + subPath = opt.slice("path=".length); + } else { + throw new Error(`Unknown volume option: ${opt}`); + } + } + } + volumes.push({ + source, + mountPath, + ...revision ? { revision } : {}, + ...readOnly ? { readOnly } : {}, + ...subPath ? { path: subPath } : {} + }); + } + } + const jobParams = { + namespace: finalNamespace, + ...dockerImage ? { dockerImage } : {}, + ...spaceId ? { spaceId } : {}, + flavor, + command: commandArray.length > 0 ? commandArray : void 0, + environment, + secrets, + ...attempts !== void 0 ? { attempts } : {}, + ...Object.keys(labels).length > 0 ? { labels } : {}, + ...volumes.length > 0 ? { volumes } : {}, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + ...token ? { accessToken: token } : {} + }; + const job = await runJob(jobParams); + console.log(`Job created: ${job.id}`); + console.log(`Status: ${job.status.stage}`); + if (!detach) { + const logsParams = { + namespace: finalNamespace, + jobId: job.id, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + ...token ? { accessToken: token } : {} + }; + for await (const logChunk of streamJobLogs(logsParams)) { + console.log(logChunk.message); + } + } + break; + } + case "ps": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "jobs ps"); + const { all, namespace, token } = parsedArgs; + let finalNamespace = namespace; + if (!finalNamespace) { + if (!token) { + throw new Error( + "Cannot determine namespace without authentication. Please provide --namespace or --token." + ); + } + const userInfo = await whoAmI({ + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + if (userInfo.type !== "user") { + throw new Error("Cannot determine namespace. Please provide --namespace explicitly."); + } + finalNamespace = userInfo.name; + } + const jobs = await listJobs({ + namespace: finalNamespace, + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + const filteredJobs = all ? jobs : jobs.filter((job) => job.status.stage === "RUNNING"); + if (filteredJobs.length === 0) { + console.log(all ? "No jobs found." : "No running jobs found."); + break; + } + console.log(`${"ID".padEnd(40)} ${"STATUS".padEnd(12)} ${"CREATED".padEnd(20)} ${"DOCKER IMAGE"}`); + console.log("-".repeat(100)); + for (const job of filteredJobs) { + const createdAt = new Date(job.createdAt).toLocaleString(); + const dockerImage = job.dockerImage || job.spaceId || "N/A"; + const status = job.status.stage; + console.log(`${job.id.padEnd(40)} ${status.padEnd(12)} ${createdAt.padEnd(20)} ${dockerImage}`); + } + break; + } + case "hardware": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "jobs hardware"); + const { token } = parsedArgs; + const hardwareParams = { + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }; + if (token) { + hardwareParams.accessToken = token; + } + const hardware = await listJobHardware(hardwareParams); + console.log( + `${"NAME".padEnd(15)} ${"PRETTY NAME".padEnd(22)} ${"CPU".padEnd(8)} ${"RAM".padEnd(7)} ${"ACCELERATOR".padEnd(16)} ${"COST/MIN".padEnd(9)} ${"COST/HOUR"}` + ); + console.log("-".repeat(100)); + for (const hw of hardware) { + let accelerator = "N/A"; + if (hw.accelerator) { + accelerator = `${hw.accelerator.quantity}x ${hw.accelerator.model} (${hw.accelerator.vram})`; + } + const costPerMin = (hw.unitCostMicroUSD / 1e6).toFixed(4); + const costPerHour = (hw.unitCostMicroUSD / 1e6 * 60).toFixed(2); + console.log( + `${hw.name.padEnd(15)} ${hw.prettyName.padEnd(22)} ${hw.cpu.padEnd(8)} ${hw.ram.padEnd(7)} ${accelerator.padEnd(16)} $${costPerMin.padStart(8)} $${costPerHour.padStart(9)}` + ); + } + break; + } + case "logs": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "jobs logs"); + const { jobId, namespace, token } = parsedArgs; + let finalNamespace = namespace; + if (!finalNamespace) { + if (!token) { + throw new Error( + "Cannot determine namespace without authentication. Please provide --namespace or --token." + ); + } + const userInfo = await whoAmI({ + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL + }); + if (userInfo.type !== "user") { + throw new Error("Cannot determine namespace. Please provide --namespace explicitly."); + } + finalNamespace = userInfo.name; + } + const logsParams = { + namespace: finalNamespace, + jobId, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + ...token ? { accessToken: token } : {} + }; + const jobInfoParams = { + namespace: finalNamespace, + jobId, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + ...token ? { accessToken: token } : {} + }; + const jobInfo = await getJob(jobInfoParams); + if (jobInfo.status.stage === "ERROR" && jobInfo.status.message) { + console.error(` +\u274C Job failed: ${jobInfo.status.message} +`); + } + for await (const logChunk of streamJobLogs(logsParams)) { + console.log(logChunk.message); + } + break; + } + default: + console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'jobs'.`); + console.log(listSubcommands("jobs", jobsCommandGroup)); + process.exitCode = 1; + break; + } + break; + } + default: + console.error("Command not found: " + mainCommandName); + console.log( + ` +Available commands: + +` + typedEntries(commands).map(([name, def]) => ` ${usage(name)}: ${def.description}`).join("\n") + ); + console.log("\nTo get help on a specific command, run `hfjs help ` or `hfjs --help`"); + process.exitCode = 1; + break; + } +} +run().catch((err) => { + console.error("\x1B[31mError:\x1B[0m", err.message); + console.error(err); + process.exitCode = 1; +}); +function usage(commandName, subCommandName2) { + const commandEntry = commands[commandName]; + let cmdArgs; + let fullCommandName = commandName; + if ("subcommands" in commandEntry) { + if (subCommandName2 && subCommandName2 in commandEntry.subcommands) { + const subCmd = commandEntry.subcommands[subCommandName2]; + cmdArgs = subCmd.args; + fullCommandName = `${commandName} ${subCommandName2}`; + } else { + return `${commandName} `; + } + } else { + cmdArgs = commandEntry.args; + } + return `${fullCommandName} ${(cmdArgs || []).map((arg) => { + if (arg.positional) { + return arg.required ? `<${arg.name}>` : `[${arg.name}]`; + } + return `[--${arg.name}${arg.short ? `|-${arg.short}` : ""}${arg.enum ? ` {${arg.enum.join("|")}}` : arg.boolean ? "" : ` <${arg.name.toUpperCase().replace(/-/g, "_")}>`}]`; + }).join(" ")}`.trim(); +} +function _detailedUsage(args, usageLine, commandDescription) { + let ret = `usage: hfjs ${usageLine} +`; + if (commandDescription) { + ret += ` +${commandDescription} +`; + } + const positionals = args.filter((p) => p.positional); + const options = args.filter((p) => !p.positional); + if (positionals.length > 0) { + ret += ` +Positional arguments: +`; + for (const arg of positionals) { + ret += ` ${arg.name} ${arg.description}${arg.default ? ` (default: ${typeof arg.default === "function" ? arg.default() : arg.default})` : ""} +`; + } + } + if (options.length > 0) { + ret += ` +Options: +`; + for (const arg of options) { + const nameAndAlias = `--${arg.name}${arg.short ? `, -${arg.short}` : ""}`; + const valueHint = arg.enum ? `{${arg.enum.join("|")}}` : arg.boolean ? "" : `<${arg.name.toUpperCase().replace(/-/g, "_")}>`; + ret += ` ${nameAndAlias}${valueHint ? " " + valueHint : ""} ${arg.description}${arg.default !== void 0 ? ` (default: ${typeof arg.default === "function" ? arg.default() : arg.default})` : ""} +`; + } + } + ret += ` +`; + return ret; +} +function detailedUsageForCommand(commandName) { + const commandDef = commands[commandName]; + if ("subcommands" in commandDef) { + return listSubcommands(commandName, commandDef); + } + return _detailedUsage(commandDef.args, usage(commandName), commandDef.description); +} +function detailedUsageForSubcommand(commandName, subCommandName2) { + const commandGroup = commands[commandName]; + if (!("subcommands" in commandGroup)) { + throw new Error(`Command ${commandName} does not have subcommands`); + } + if (!(subCommandName2 in commandGroup.subcommands)) { + throw new Error(`Subcommand ${subCommandName2} not found for ${commandName}`); + } + const subCommandDef = commandGroup.subcommands[subCommandName2]; + return _detailedUsage(subCommandDef.args, usage(commandName, subCommandName2), subCommandDef.description); +} +function listSubcommands(commandName, commandGroup) { + let ret = `usage: hfjs ${commandName} [options] + +`; + ret += `${commandGroup.description} + +`; + ret += `Available subcommands for '${commandName}': +`; + ret += typedEntries(commandGroup.subcommands).map(([subName, subDef]) => ` ${subName} ${subDef.description}`).join("\n"); + if (commandName === "jobs") { + ret += ` + +Example: + hfjs jobs run -e FOO=foo -e BAR=bar python:3.12 -- python -c 'import os; print(os.environ["FOO"], os.environ["BAR"])'`; + } + ret += ` + +Run \`hfjs help ${commandName} \` for more information on a specific subcommand.`; + return ret; +} +function advParseArgs(args, argDefs, commandNameForError) { + const hasMultiplePositional = argDefs.some((arg) => arg.multiple && arg.positional); + const { tokens } = parseArgs({ + options: Object.fromEntries( + argDefs.filter((arg) => !arg.positional).map((arg) => { + const optionConfig = { + type: arg.boolean ? "boolean" : "string", + ...arg.short && { short: arg.short }, + ...arg.multiple && { multiple: true }, + ...arg.default !== void 0 && { + default: typeof arg.default === "function" ? arg.default() : arg.default + } + }; + return [arg.name, optionConfig]; + }) + ), + args, + allowPositionals: true, + strict: false, + // We do custom validation based on tokens and argDefs + tokens: true + }); + const expectedPositionals = argDefs.filter((arg) => arg.positional); + const providedPositionalTokens = tokens.filter((token) => token.kind === "positional"); + if (providedPositionalTokens.length < expectedPositionals.filter((arg) => arg.required).length) { + throw new Error( + `Command '${commandNameForError}': Missing required positional arguments. Usage: hfjs ${usage( + commandNameForError.split(" ")[0], + commandNameForError.split(" ")[1] + )}` + ); + } + if (providedPositionalTokens.length > expectedPositionals.length && !hasMultiplePositional) { + throw new Error( + `Command '${commandNameForError}': Too many positional arguments. Usage: hfjs ${usage( + commandNameForError.split(" ")[0], + commandNameForError.split(" ")[1] + )}` + ); + } + const result = {}; + for (const argDef of argDefs) { + if (argDef.default !== void 0) { + result[argDef.name] = typeof argDef.default === "function" ? argDef.default() : argDef.default; + } else if (argDef.boolean) { + result[argDef.name] = false; + } + } + expectedPositionals.forEach((argDef, i) => { + if (argDef.multiple) { + result[argDef.name] = providedPositionalTokens.slice(i).map((token) => token.value); + } else if (providedPositionalTokens[i]) { + result[argDef.name] = providedPositionalTokens[i].value; + } + }); + tokens.filter((token) => token.kind === "option").forEach((token) => { + const argDef = argDefs.find((def) => def.name === token.name || def.short === token.name); + if (!argDef) { + throw new Error(`Command '${commandNameForError}': Unknown option: ${token.rawName}`); + } + if (argDef.boolean) { + result[argDef.name] = true; + } else { + if (token.value === void 0) { + throw new Error(`Command '${commandNameForError}': Missing value for option: ${token.rawName}`); + } + if (argDef.enum && !argDef.enum.includes(token.value)) { + throw new Error( + `Command '${commandNameForError}': Invalid value '${token.value}' for option ${token.rawName}. Expected one of: ${argDef.enum.join(", ")}` + ); + } + if (argDef.multiple) { + const existing = result[argDef.name] || []; + existing.push(token.value); + result[argDef.name] = existing; + } else { + result[argDef.name] = token.value; + } + } + }); + for (const argDef of argDefs) { + if (argDef.required && result[argDef.name] === void 0) { + throw new Error(`Command '${commandNameForError}': Missing required argument: ${argDef.name}`); + } + } + return Object.fromEntries( + Object.entries(result).map(([name, val]) => [kebabToCamelCase(name), val]) + ); +} +function kebabToCamelCase(str) { + return str.replace(/-./g, (match) => match[1].toUpperCase()); +} diff --git a/node_modules/@huggingface/hub/dist/index.d.ts b/node_modules/@huggingface/hub/dist/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..a92fbe1ee551ee13def8f72bd7f062db8c15a64e --- /dev/null +++ b/node_modules/@huggingface/hub/dist/index.d.ts @@ -0,0 +1,2 @@ +export * from "./src"; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/index.d.ts.map b/node_modules/@huggingface/hub/dist/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..01832f343733ac628ff0f5ae57d4d3efdc4773b9 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA,cAAc,OAAO,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/index.js b/node_modules/@huggingface/hub/dist/index.js new file mode 100644 index 0000000000000000000000000000000000000000..c6ec12c31e57b13c4c7aa9b7c2a765fef83a5147 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/index.js @@ -0,0 +1,7147 @@ +"use strict"; +var __create = Object.create; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __getProtoOf = Object.getPrototypeOf; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __esm = (fn, res) => function __init() { + return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res; +}; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( + // If the importer is in node compatibility mode or this is not an ESM + // file that has been converted to a CommonJS file using a Babel- + // compatible transform (i.e. "__esModule" has not been set), then set + // "default" to the CommonJS "module.exports" for node compatibility. + isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, + mod +)); +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); + +// src/vendor/hash-wasm/sha256.js +var import_meta, Module, sha256_default; +var init_sha256 = __esm({ + "src/vendor/hash-wasm/sha256.js"() { + "use strict"; + import_meta = {}; + Module = (() => { + var _unused = import_meta.url; + return function(moduleArg = {}) { + var Module2 = moduleArg; + var readyPromiseResolve, readyPromiseReject; + Module2["ready"] = new Promise((resolve3, reject) => { + readyPromiseResolve = resolve3; + readyPromiseReject = reject; + }); + var moduleOverrides = Object.assign({}, Module2); + var arguments_ = []; + var thisProgram = "./this.program"; + var quit_ = (status, toThrow) => { + throw toThrow; + }; + var ENVIRONMENT_IS_WEB = typeof window == "object"; + var ENVIRONMENT_IS_WORKER = typeof importScripts == "function"; + var ENVIRONMENT_IS_NODE = typeof process == "object" && typeof process.versions == "object" && typeof process.versions.node == "string"; + var ENVIRONMENT_IS_SHELL = !ENVIRONMENT_IS_WEB && !ENVIRONMENT_IS_NODE && !ENVIRONMENT_IS_WORKER; + var scriptDirectory = ""; + function locateFile(path2) { + if (Module2["locateFile"]) { + return Module2["locateFile"](path2, scriptDirectory); + } + return scriptDirectory + path2; + } + var read_, readAsync, readBinary; + if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { + if (ENVIRONMENT_IS_WORKER) { + scriptDirectory = self.location.href; + } else if (typeof document != "undefined" && document.currentScript) { + scriptDirectory = document.currentScript.src; + } + if (false) { + scriptDirectory = false; + } + if (scriptDirectory.startsWith("blob:")) { + scriptDirectory = ""; + } else { + scriptDirectory = scriptDirectory.substr(0, scriptDirectory.replace(/[?#].*/, "").lastIndexOf("/") + 1); + } + { + read_ = (url) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, false); + xhr.send(null); + return xhr.responseText; + }; + if (ENVIRONMENT_IS_WORKER) { + readBinary = (url) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, false); + xhr.responseType = "arraybuffer"; + xhr.send(null); + return new Uint8Array( + /** @type{!ArrayBuffer} */ + xhr.response + ); + }; + } + readAsync = (url, onload, onerror) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, true); + xhr.responseType = "arraybuffer"; + xhr.onload = () => { + if (xhr.status == 200 || xhr.status == 0 && xhr.response) { + onload(xhr.response); + return; + } + onerror(); + }; + xhr.onerror = onerror; + xhr.send(null); + }; + } + } else { + } + var out = Module2["print"] || console.log.bind(console); + var err = Module2["printErr"] || console.error.bind(console); + Object.assign(Module2, moduleOverrides); + moduleOverrides = null; + if (Module2["arguments"]) + arguments_ = Module2["arguments"]; + if (Module2["thisProgram"]) + thisProgram = Module2["thisProgram"]; + if (Module2["quit"]) + quit_ = Module2["quit"]; + var wasmBinary; + if (Module2["wasmBinary"]) + wasmBinary = Module2["wasmBinary"]; + if (typeof WebAssembly != "object") { + abort("no native wasm support detected"); + } + function intArrayFromBase64(s) { + var decoded = atob(s); + var bytes = new Uint8Array(decoded.length); + for (var i = 0; i < decoded.length; ++i) { + bytes[i] = decoded.charCodeAt(i); + } + return bytes; + } + function tryParseAsDataURI(filename) { + if (!isDataURI(filename)) { + return; + } + return intArrayFromBase64(filename.slice(dataURIPrefix.length)); + } + var wasmMemory; + var ABORT = false; + var EXITSTATUS; + function assert(condition, text) { + if (!condition) { + abort(text); + } + } + var HEAP, HEAP8, HEAPU8, HEAP16, HEAPU16, HEAP32, HEAPU32, HEAPF32, HEAPF64; + function updateMemoryViews() { + var b = wasmMemory.buffer; + Module2["HEAP8"] = HEAP8 = new Int8Array(b); + Module2["HEAP16"] = HEAP16 = new Int16Array(b); + Module2["HEAPU8"] = HEAPU8 = new Uint8Array(b); + Module2["HEAPU16"] = HEAPU16 = new Uint16Array(b); + Module2["HEAP32"] = HEAP32 = new Int32Array(b); + Module2["HEAPU32"] = HEAPU32 = new Uint32Array(b); + Module2["HEAPF32"] = HEAPF32 = new Float32Array(b); + Module2["HEAPF64"] = HEAPF64 = new Float64Array(b); + } + var __ATPRERUN__ = []; + var __ATINIT__ = []; + var __ATEXIT__ = []; + var __ATPOSTRUN__ = []; + var runtimeInitialized = false; + function preRun() { + if (Module2["preRun"]) { + if (typeof Module2["preRun"] == "function") + Module2["preRun"] = [Module2["preRun"]]; + while (Module2["preRun"].length) { + addOnPreRun(Module2["preRun"].shift()); + } + } + callRuntimeCallbacks(__ATPRERUN__); + } + function initRuntime() { + runtimeInitialized = true; + callRuntimeCallbacks(__ATINIT__); + } + function postRun() { + if (Module2["postRun"]) { + if (typeof Module2["postRun"] == "function") + Module2["postRun"] = [Module2["postRun"]]; + while (Module2["postRun"].length) { + addOnPostRun(Module2["postRun"].shift()); + } + } + callRuntimeCallbacks(__ATPOSTRUN__); + } + function addOnPreRun(cb) { + __ATPRERUN__.unshift(cb); + } + function addOnInit(cb) { + __ATINIT__.unshift(cb); + } + function addOnExit(cb) { + } + function addOnPostRun(cb) { + __ATPOSTRUN__.unshift(cb); + } + var runDependencies = 0; + var runDependencyWatcher = null; + var dependenciesFulfilled = null; + function getUniqueRunDependency(id) { + return id; + } + function addRunDependency(id) { + runDependencies++; + Module2["monitorRunDependencies"]?.(runDependencies); + } + function removeRunDependency(id) { + runDependencies--; + Module2["monitorRunDependencies"]?.(runDependencies); + if (runDependencies == 0) { + if (runDependencyWatcher !== null) { + clearInterval(runDependencyWatcher); + runDependencyWatcher = null; + } + if (dependenciesFulfilled) { + var callback = dependenciesFulfilled; + dependenciesFulfilled = null; + callback(); + } + } + } + function abort(what) { + Module2["onAbort"]?.(what); + what = "Aborted(" + what + ")"; + err(what); + ABORT = true; + EXITSTATUS = 1; + what += ". Build with -sASSERTIONS for more info."; + var e = new WebAssembly.RuntimeError(what); + readyPromiseReject(e); + throw e; + } + var dataURIPrefix = "data:application/octet-stream;base64,"; + var isDataURI = (filename) => filename.startsWith(dataURIPrefix); + var isFileURI = (filename) => filename.startsWith("file://"); + var wasmBinaryFile; + wasmBinaryFile = "data:application/octet-stream;base64,AGFzbQEAAAABHQZgAX8AYAABf2AAAGABfwF/YAJ/fwBgA39/fwF/Aw0MAgAEAgMBBQABAQADBAUBcAEBAQUGAQGAAoACBg4CfwFB8IuEBAt/AUEACweYAQoGbWVtb3J5AgARX193YXNtX2NhbGxfY3RvcnMAAAtIYXNoX1VwZGF0ZQABCkhhc2hfRmluYWwAAwlIYXNoX0luaXQABAxHZXRCdWZmZXJQdHIABRlfX2luZGlyZWN0X2Z1bmN0aW9uX3RhYmxlAQAJc3RhY2tTYXZlAAkMc3RhY2tSZXN0b3JlAAoKc3RhY2tBbGxvYwALCossDAIAC+4CAgV/AX5BACgCwAoiASABKQNAIgYgAK18NwNAAkACQAJAIAanQT9xIgINAEGACyEBIAAhAgwBC0HAACACayEDAkAgAEUNACADIAAgAyAASRshBCABIAJqIQVBACEBA0AgBSABIgFqQYALIAFqLQAAOgAAIAFBAWoiAiEBIAIgBEcNAAsLAkACQCAAIANJIgRFDQBBgAshASAAIQIMAQtBACgCwAoiAUHIAGogARACQYALIANqIQEgACADayECCyABIQEgAiECIAQNAQsgASEBAkACQCACIgJBwABPDQAgASEFIAIhAAwBCyACIQIgASEEA0BBACgCwApByABqIAQiBBACIAJBQGoiASECIARBwABqIgUhBCAFIQUgASEAIAFBP0sNAAsLIAUhBSAAIgBFDQBBACEBQQAhAgNAQQAoAsAKIAEiAWogBSABai0AADoAACACQQFqIgJB/wFxIgQhASACIQIgACAESw0ACwsLqCEBK38gACgCCCICIAAoAgQiAyAAKAIAIgRzcSADIARxcyAEQR53IARBE3dzIARBCndzaiAAKAIQIgVBGncgBUEVd3MgBUEHd3MgACgCHCIGaiAAKAIYIgcgACgCFCIIcyAFcSAHc2ogASgCACIJQRh0IAlBgP4DcUEIdHIgCUEIdkGA/gNxIAlBGHZyciIKakGY36iUBGoiC2oiCSAEcyADcSAJIARxcyAJQR53IAlBE3dzIAlBCndzaiAHIAEoAgQiDEEYdCAMQYD+A3FBCHRyIAxBCHZBgP4DcSAMQRh2cnIiDWogCyAAKAIMIg5qIg8gCCAFc3EgCHNqIA9BGncgD0EVd3MgD0EHd3NqQZGJ3YkHaiIQaiIMIAlzIARxIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAggASgCCCILQRh0IAtBgP4DcUEIdHIgC0EIdkGA/gNxIAtBGHZyciIRaiAQIAJqIhIgDyAFc3EgBXNqIBJBGncgEkEVd3MgEkEHd3NqQc/3g657aiITaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAUgASgCDCIQQRh0IBBBgP4DcUEIdHIgEEEIdkGA/gNxIBBBGHZyciIUaiATIANqIhMgEiAPc3EgD3NqIBNBGncgE0EVd3MgE0EHd3NqQaW3181+aiIVaiIQIAtzIAxxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIA8gASgCECIWQRh0IBZBgP4DcUEIdHIgFkEIdkGA/gNxIBZBGHZyciIXaiAVIARqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQduE28oDaiIYaiIPIBBzIAtxIA8gEHFzIA9BHncgD0ETd3MgD0EKd3NqIAEoAhQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGSASaiAYIAlqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQfGjxM8FaiIYaiIJIA9zIBBxIAkgD3FzIAlBHncgCUETd3MgCUEKd3NqIAEoAhgiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGiATaiAYIAxqIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQaSF/pF5aiIYaiIMIAlzIA9xIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAEoAhwiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGyAWaiAYIAtqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQdW98dh6aiIYaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAEoAiAiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHCASaiAYIBBqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQZjVnsB9aiIYaiIQIAtzIAxxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAiQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHSATaiAYIA9qIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQYG2jZQBaiIYaiIPIBBzIAtxIA8gEHFzIA9BHncgD0ETd3MgD0EKd3NqIAEoAigiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHiAWaiAYIAlqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQb6LxqECaiIYaiIJIA9zIBBxIAkgD3FzIAlBHncgCUETd3MgCUEKd3NqIAEoAiwiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHyASaiAYIAxqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQcP7sagFaiIYaiIMIAlzIA9xIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAEoAjAiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiICATaiAYIAtqIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQfS6+ZUHaiIYaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAEoAjQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiISAWaiAYIBBqIhAgEyASc3EgEnNqIBBBGncgEEEVd3MgEEEHd3NqQf7j+oZ4aiIYaiIWIAtzIAxxIBYgC3FzIBZBHncgFkETd3MgFkEKd3NqIAEoAjgiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiIiASaiAYIA9qIg8gECATc3EgE3NqIA9BGncgD0EVd3MgD0EHd3NqQaeN8N55aiIVaiISIBZzIAtxIBIgFnFzIBJBHncgEkETd3MgEkEKd3NqIAEoAjwiAUEYdCABQYD+A3FBCHRyIAFBCHZBgP4DcSABQRh2cnIiIyATaiAVIAlqIgEgDyAQc3EgEHNqIAFBGncgAUEVd3MgAUEHd3NqQfTi74x8aiIJaiEVIBIhGCAWISQgCyElIAkgDGohJiABIScgDyEoIBAhKSAjISMgIiEiICEhISAgISAgHyEfIB4hHiAdIR0gHCEcIBshGyAaIRogGSEZIBchFyAUIRQgESERIA0hECAKIQxBgAkhAUEQISoDQCAVIgkgGCIKcyAkIitxIAkgCnFzIAlBHncgCUETd3MgCUEKd3NqIBAiEEEZdyAQQQ53cyAQQQN2cyAMaiAdIh1qICIiFkEPdyAWQQ13cyAWQQp2c2oiDCApaiAmIhIgJyIPICgiE3NxIBNzaiASQRp3IBJBFXdzIBJBB3dzaiABIgEoAgBqIiRqIgsgCXMgCnEgCyAJcXMgC0EedyALQRN3cyALQQp3c2ogESIYQRl3IBhBDndzIBhBA3ZzIBBqIB4iHmogIyIVQQ93IBVBDXdzIBVBCnZzaiINIBNqIAEoAgRqICQgJWoiEyASIA9zcSAPc2ogE0EadyATQRV3cyATQQd3c2oiJWoiECALcyAJcSAQIAtxcyAQQR53IBBBE3dzIBBBCndzaiAUIiRBGXcgJEEOd3MgJEEDdnMgGGogHyIfaiAMQQ93IAxBDXdzIAxBCnZzaiIRIA9qIAEoAghqICUgK2oiGCATIBJzcSASc2ogGEEadyAYQRV3cyAYQQd3c2oiJWoiDyAQcyALcSAPIBBxcyAPQR53IA9BE3dzIA9BCndzaiAXIhdBGXcgF0EOd3MgF0EDdnMgJGogICIgaiANQQ93IA1BDXdzIA1BCnZzaiIUIBJqIAEoAgxqICUgCmoiCiAYIBNzcSATc2ogCkEadyAKQRV3cyAKQQd3c2oiJWoiEiAPcyAQcSASIA9xcyASQR53IBJBE3dzIBJBCndzaiATIBkiJEEZdyAkQQ53cyAkQQN2cyAXaiAhIiFqIBFBD3cgEUENd3MgEUEKdnNqIhdqIAEoAhBqICUgCWoiEyAKIBhzcSAYc2ogE0EadyATQRV3cyATQQd3c2oiJWoiCSAScyAPcSAJIBJxcyAJQR53IAlBE3dzIAlBCndzaiABKAIUIBoiGkEZdyAaQQ53cyAaQQN2cyAkaiAWaiAUQQ93IBRBDXdzIBRBCnZzaiIZaiAYaiAlIAtqIhggEyAKc3EgCnNqIBhBGncgGEEVd3MgGEEHd3NqIiVqIgsgCXMgEnEgCyAJcXMgC0EedyALQRN3cyALQQp3c2ogASgCGCAbIiRBGXcgJEEOd3MgJEEDdnMgGmogFWogF0EPdyAXQQ13cyAXQQp2c2oiGmogCmogJSAQaiIKIBggE3NxIBNzaiAKQRp3IApBFXdzIApBB3dzaiIlaiIQIAtzIAlxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAhwgHCIcQRl3IBxBDndzIBxBA3ZzICRqIAxqIBlBD3cgGUENd3MgGUEKdnNqIhtqIBNqICUgD2oiJCAKIBhzcSAYc2ogJEEadyAkQRV3cyAkQQd3c2oiE2oiDyAQcyALcSAPIBBxcyAPQR53IA9BE3dzIA9BCndzaiABKAIgIB1BGXcgHUEOd3MgHUEDdnMgHGogDWogGkEPdyAaQQ13cyAaQQp2c2oiHGogGGogEyASaiIYICQgCnNxIApzaiAYQRp3IBhBFXdzIBhBB3dzaiITaiISIA9zIBBxIBIgD3FzIBJBHncgEkETd3MgEkEKd3NqIAEoAiQgHkEZdyAeQQ53cyAeQQN2cyAdaiARaiAbQQ93IBtBDXdzIBtBCnZzaiIdaiAKaiATIAlqIgkgGCAkc3EgJHNqIAlBGncgCUEVd3MgCUEHd3NqIgpqIhMgEnMgD3EgEyAScXMgE0EedyATQRN3cyATQQp3c2ogASgCKCAfQRl3IB9BDndzIB9BA3ZzIB5qIBRqIBxBD3cgHEENd3MgHEEKdnNqIh5qICRqIAogC2oiCiAJIBhzcSAYc2ogCkEadyAKQRV3cyAKQQd3c2oiJGoiCyATcyAScSALIBNxcyALQR53IAtBE3dzIAtBCndzaiABKAIsICBBGXcgIEEOd3MgIEEDdnMgH2ogF2ogHUEPdyAdQQ13cyAdQQp2c2oiH2ogGGogJCAQaiIYIAogCXNxIAlzaiAYQRp3IBhBFXdzIBhBB3dzaiIkaiIQIAtzIBNxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAjAgIUEZdyAhQQ53cyAhQQN2cyAgaiAZaiAeQQ93IB5BDXdzIB5BCnZzaiIgaiAJaiAkIA9qIiQgGCAKc3EgCnNqICRBGncgJEEVd3MgJEEHd3NqIg9qIgkgEHMgC3EgCSAQcXMgCUEedyAJQRN3cyAJQQp3c2ogASgCNCAWQRl3IBZBDndzIBZBA3ZzICFqIBpqIB9BD3cgH0ENd3MgH0EKdnNqIiFqIApqIA8gEmoiDyAkIBhzcSAYc2ogD0EadyAPQRV3cyAPQQd3c2oiCmoiEiAJcyAQcSASIAlxcyASQR53IBJBE3dzIBJBCndzaiABKAI4IBVBGXcgFUEOd3MgFUEDdnMgFmogG2ogIEEPdyAgQQ13cyAgQQp2c2oiImogGGogCiATaiITIA8gJHNxICRzaiATQRp3IBNBFXdzIBNBB3dzaiIYaiIWIBJzIAlxIBYgEnFzIBZBHncgFkETd3MgFkEKd3NqIAEoAjwgDEEZdyAMQQ53cyAMQQN2cyAVaiAcaiAhQQ93ICFBDXdzICFBCnZzaiIKaiAkaiAYIAtqIgsgEyAPc3EgD3NqIAtBGncgC0EVd3MgC0EHd3NqIiZqIishFSAWIRggEiEkIAkhJSAmIBBqIiwhJiALIScgEyEoIA8hKSAKISMgIiEiICEhISAgISAgHyEfIB4hHiAdIR0gHCEcIBshGyAaIRogGSEZIBchFyAUIRQgESERIA0hECAMIQwgAUHAAGohASAqIgpBEGohKiAKQTBJDQALIAAgDyAGajYCHCAAIBMgB2o2AhggACALIAhqNgIUIAAgLCAFajYCECAAIAkgDmo2AgwgACASIAJqNgIIIAAgFiADajYCBCAAICsgBGo2AgAL1AMDBX8BfgF7QQAoAsAKIgAgACgCQCIBQQJ2QQ9xIgJBAnRqIgMgAygCAEF/IAFBA3QiAXRBf3NxQYABIAF0czYCAAJAAkAgAkEOTw0AIAJBAWohAAwBCwJAIAJBDkcNACAAQQA2AjwLIABByABqIAAQAkEAIQALAkAgACIAQQ1LDQBBACgCwAogAEECdCIAakEAQTggAGsQBhoLQQAoAsAKIgAgACkDQCIFpyICQRt0IAJBC3RBgID8B3FyIAJBBXZBgP4DcSACQQN0QRh2cnI2AjwgACAFQh2IpyICQRh0IAJBgP4DcUEIdHIgAkEIdkGA/gNxIAJBGHZycjYCOCAAQcgAaiAAEAJBACgCwApBPGohAUEAIQADQCABQQcgACIAa0ECdGoiAiAC/QACACAG/Q0MDQ4PCAkKCwQFBgcAAQIDIAb9DQMCAQAHBgUECwoJCA8ODQwgBv0NDA0ODwgJCgsEBQYHAAECA/0LAgAgAEEEaiICIQAgAkEIRw0ACwJAQQAoAsAKIgMoAmhFDQAgA0HIAGohBEEAIQBBACECA0BBgAsgACIAaiAEIABqLQAAOgAAIAJBAWoiAkH/AXEiASEAIAIhAiADKAJoIAFLDQALCwtxAQJ/QQAoAsAKIgFCADcDQCABQcgAaiECAkAgAEHgAUcNACABQRw2AmggAkEQakEA/QAEsAj9CwIAIAJBAP0ABKAI/QsCAEEADwsgAUEgNgJoIAJBEGpBAP0ABJAI/QsCACACQQD9AASACP0LAgBBAAsFAEGACwvyAgIDfwF+AkAgAkUNACAAIAE6AAAgACACaiIDQX9qIAE6AAAgAkEDSQ0AIAAgAToAAiAAIAE6AAEgA0F9aiABOgAAIANBfmogAToAACACQQdJDQAgACABOgADIANBfGogAToAACACQQlJDQAgAEEAIABrQQNxIgRqIgMgAUH/AXFBgYKECGwiATYCACADIAIgBGtBfHEiBGoiAkF8aiABNgIAIARBCUkNACADIAE2AgggAyABNgIEIAJBeGogATYCACACQXRqIAE2AgAgBEEZSQ0AIAMgATYCGCADIAE2AhQgAyABNgIQIAMgATYCDCACQXBqIAE2AgAgAkFsaiABNgIAIAJBaGogATYCACACQWRqIAE2AgAgBCADQQRxQRhyIgVrIgJBIEkNACABrUKBgICAEH4hBiADIAVqIQEDQCABIAY3AxggASAGNwMQIAEgBjcDCCABIAY3AwAgAUEgaiEBIAJBYGoiAkEfSw0ACwsgAAsGACAAJAELBAAjAQsEACMACwYAIAAkAAsSAQJ/IwAgAGtBcHEiASQAIAELC9ICAgBBgAgLwAJn5glqha5nu3Lzbjw69U+lf1IOUYxoBZur2YMfGc3gW9ieBcEH1Xw2F91wMDlZDvcxC8D/ERVYaKeP+WSkT/q+mC+KQpFEN3HP+8C1pdu16VvCVjnxEfFZpII/ktVeHKuYqgfYAVuDEr6FMSTDfQxVdF2+cv6x3oCnBtybdPGbwcFpm+SGR77vxp3BD8yhDCRvLOktqoR0StypsFzaiPl2UlE+mG3GMajIJwOwx39Zv/ML4MZHkafVUWPKBmcpKRSFCrcnOCEbLvxtLE0TDThTVHMKZbsKanYuycKBhSxykqHov6JLZhqocItLwqNRbMcZ6JLRJAaZ1oU1DvRwoGoQFsGkGQhsNx5Md0gntbywNLMMHDlKqthOT8qcW/NvLmjugo90b2OleBR4yIQIAseM+v++kOtsUKT3o/m+8nhxxgBBwAoLBIAFgAA="; + if (!isDataURI(wasmBinaryFile)) { + wasmBinaryFile = locateFile(wasmBinaryFile); + } + function getBinarySync(file) { + if (file == wasmBinaryFile && wasmBinary) { + return new Uint8Array(wasmBinary); + } + var binary = tryParseAsDataURI(file); + if (binary) { + return binary; + } + if (readBinary) { + return readBinary(file); + } + throw "both async and sync fetching of the wasm failed"; + } + function getBinaryPromise(binaryFile) { + return Promise.resolve().then(() => getBinarySync(binaryFile)); + } + function instantiateArrayBuffer(binaryFile, imports, receiver) { + return getBinaryPromise(binaryFile).then((binary) => { + return WebAssembly.instantiate(binary, imports); + }).then(receiver, (reason) => { + err(`failed to asynchronously prepare wasm: ${reason}`); + abort(reason); + }); + } + function instantiateAsync(binary, binaryFile, imports, callback) { + return instantiateArrayBuffer(binaryFile, imports, callback); + } + function createWasm() { + var info = { + "env": wasmImports, + "wasi_snapshot_preview1": wasmImports + }; + function receiveInstance(instance, module2) { + wasmExports = instance.exports; + wasmMemory = wasmExports["memory"]; + updateMemoryViews(); + addOnInit(wasmExports["__wasm_call_ctors"]); + removeRunDependency("wasm-instantiate"); + return wasmExports; + } + addRunDependency("wasm-instantiate"); + function receiveInstantiationResult(result) { + receiveInstance(result["instance"]); + } + if (Module2["instantiateWasm"]) { + try { + return Module2["instantiateWasm"](info, receiveInstance); + } catch (e) { + err(`Module.instantiateWasm callback failed with error: ${e}`); + readyPromiseReject(e); + } + } + instantiateAsync(wasmBinary, wasmBinaryFile, info, receiveInstantiationResult).catch(readyPromiseReject); + return {}; + } + var tempDouble; + var tempI64; + function ExitStatus(status) { + this.name = "ExitStatus"; + this.message = `Program terminated with exit(${status})`; + this.status = status; + } + var callRuntimeCallbacks = (callbacks) => { + while (callbacks.length > 0) { + callbacks.shift()(Module2); + } + }; + function getValue(ptr, type = "i8") { + if (type.endsWith("*")) + type = "*"; + switch (type) { + case "i1": + return HEAP8[ptr]; + case "i8": + return HEAP8[ptr]; + case "i16": + return HEAP16[ptr >> 1]; + case "i32": + return HEAP32[ptr >> 2]; + case "i64": + abort("to do getValue(i64) use WASM_BIGINT"); + case "float": + return HEAPF32[ptr >> 2]; + case "double": + return HEAPF64[ptr >> 3]; + case "*": + return HEAPU32[ptr >> 2]; + default: + abort(`invalid type for getValue: ${type}`); + } + } + var noExitRuntime = Module2["noExitRuntime"] || true; + function setValue(ptr, value, type = "i8") { + if (type.endsWith("*")) + type = "*"; + switch (type) { + case "i1": + HEAP8[ptr] = value; + break; + case "i8": + HEAP8[ptr] = value; + break; + case "i16": + HEAP16[ptr >> 1] = value; + break; + case "i32": + HEAP32[ptr >> 2] = value; + break; + case "i64": + abort("to do setValue(i64) use WASM_BIGINT"); + case "float": + HEAPF32[ptr >> 2] = value; + break; + case "double": + HEAPF64[ptr >> 3] = value; + break; + case "*": + HEAPU32[ptr >> 2] = value; + break; + default: + abort(`invalid type for setValue: ${type}`); + } + } + var wasmImports = {}; + var wasmExports = createWasm(); + var ___wasm_call_ctors = () => (___wasm_call_ctors = wasmExports["__wasm_call_ctors"])(); + var _Hash_Update = Module2["_Hash_Update"] = (a0) => (_Hash_Update = Module2["_Hash_Update"] = wasmExports["Hash_Update"])(a0); + var _Hash_Final = Module2["_Hash_Final"] = () => (_Hash_Final = Module2["_Hash_Final"] = wasmExports["Hash_Final"])(); + var _Hash_Init = Module2["_Hash_Init"] = (a0) => (_Hash_Init = Module2["_Hash_Init"] = wasmExports["Hash_Init"])(a0); + var _GetBufferPtr = Module2["_GetBufferPtr"] = () => (_GetBufferPtr = Module2["_GetBufferPtr"] = wasmExports["GetBufferPtr"])(); + var stackSave = () => (stackSave = wasmExports["stackSave"])(); + var stackRestore = (a0) => (stackRestore = wasmExports["stackRestore"])(a0); + var stackAlloc = (a0) => (stackAlloc = wasmExports["stackAlloc"])(a0); + var calledRun; + dependenciesFulfilled = function runCaller() { + if (!calledRun) + run(); + if (!calledRun) + dependenciesFulfilled = runCaller; + }; + function run() { + if (runDependencies > 0) { + return; + } + preRun(); + if (runDependencies > 0) { + return; + } + function doRun() { + if (calledRun) + return; + calledRun = true; + Module2["calledRun"] = true; + if (ABORT) + return; + initRuntime(); + readyPromiseResolve(Module2); + if (Module2["onRuntimeInitialized"]) + Module2["onRuntimeInitialized"](); + postRun(); + } + if (Module2["setStatus"]) { + Module2["setStatus"]("Running..."); + setTimeout(function() { + setTimeout(function() { + Module2["setStatus"](""); + }, 1); + doRun(); + }, 1); + } else { + doRun(); + } + } + if (Module2["preInit"]) { + if (typeof Module2["preInit"] == "function") + Module2["preInit"] = [Module2["preInit"]]; + while (Module2["preInit"].length > 0) { + Module2["preInit"].pop()(); + } + } + run(); + return moduleArg.ready; + }; + })(); + sha256_default = Module; + } +}); + +// src/vendor/hash-wasm/sha256-wrapper.ts +var sha256_wrapper_exports = {}; +__export(sha256_wrapper_exports, { + createSHA256: () => createSHA256, + createSHA256WorkerCode: () => createSHA256WorkerCode +}); +async function createSHA256(isInsideWorker = false) { + const BUFFER_MAX_SIZE = 8 * 1024 * 1024; + const wasm = isInsideWorker ? ( + // @ts-expect-error WasmModule will be populated inside self object + await self["SHA256WasmModule"]() + ) : await sha256_default(); + const heap = wasm.HEAPU8.subarray(wasm._GetBufferPtr()); + return { + init() { + wasm._Hash_Init(256); + }, + update(data) { + let byteUsed = 0; + while (byteUsed < data.byteLength) { + const bytesLeft = data.byteLength - byteUsed; + const length = Math.min(bytesLeft, BUFFER_MAX_SIZE); + heap.set(data.subarray(byteUsed, byteUsed + length)); + wasm._Hash_Update(length); + byteUsed += length; + } + }, + digest(method) { + if (method !== "hex") { + throw new Error("Only digest hex is supported"); + } + wasm._Hash_Final(); + const result = Array.from(heap.slice(0, 32)); + return result.map((b) => b.toString(16).padStart(2, "0")).join(""); + } + }; +} +function createSHA256WorkerCode() { + return ` + self.addEventListener('message', async (event) => { + const { file } = event.data; + const sha256 = await self.createSHA256(true); + sha256.init(); + const reader = file.stream().getReader(); + const total = file.size; + let bytesDone = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + sha256.update(value); + bytesDone += value.length; + postMessage({ progress: bytesDone / total }); + } + postMessage({ sha256: sha256.digest('hex') }); + }); + self.SHA256WasmModule = ${sha256_default.toString()}; + self.createSHA256 = ${createSHA256.toString()}; + `; +} +var init_sha256_wrapper = __esm({ + "src/vendor/hash-wasm/sha256-wrapper.ts"() { + "use strict"; + init_sha256(); + } +}); + +// src/utils/sha256-node.ts +var sha256_node_exports = {}; +__export(sha256_node_exports, { + sha256Node: () => sha256Node +}); +async function* sha256Node(buffer, opts) { + const sha256Stream = (0, import_node_crypto.createHash)("sha256"); + const size = buffer instanceof Blob ? buffer.size : buffer.byteLength; + let done = 0; + const readable = buffer instanceof Blob ? import_node_stream.Readable.fromWeb(buffer.stream()) : import_node_stream.Readable.from(Buffer.from(buffer)); + for await (const buffer2 of readable) { + sha256Stream.update(buffer2); + done += buffer2.length; + yield done / size; + opts?.abortSignal?.throwIfAborted(); + } + return sha256Stream.digest("hex"); +} +var import_node_stream, import_node_crypto; +var init_sha256_node = __esm({ + "src/utils/sha256-node.ts"() { + "use strict"; + import_node_stream = require("stream"); + import_node_crypto = require("crypto"); + } +}); + +// src/utils/FileBlob.ts +var FileBlob_exports = {}; +__export(FileBlob_exports, { + FileBlob: () => FileBlob +}); +var import_node_fs, import_promises2, import_node_stream2, import_node_url, FileBlob; +var init_FileBlob = __esm({ + "src/utils/FileBlob.ts"() { + "use strict"; + import_node_fs = require("fs"); + import_promises2 = require("fs/promises"); + import_node_stream2 = require("stream"); + import_node_url = require("url"); + FileBlob = class extends Blob { + /** + * Creates a new FileBlob on the provided file. + * + * @param path Path to the file to be lazy readed + */ + static async create(path2) { + path2 = path2 instanceof URL ? (0, import_node_url.fileURLToPath)(path2) : path2; + const { size } = await (0, import_promises2.stat)(path2); + const fileBlob = new FileBlob(path2, 0, size); + return fileBlob; + } + path; + start; + end; + constructor(path2, start, end) { + super(); + this.path = path2; + this.start = start; + this.end = end; + } + /** + * Returns the size of the blob. + */ + get size() { + return this.end - this.start; + } + /** + * Returns a new instance of FileBlob that is a slice of the current one. + * + * The slice is inclusive of the start and exclusive of the end. + * + * The slice method does not supports negative start/end. + * + * @param start beginning of the slice + * @param end end of the slice + */ + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on FileBlob.slice"); + } + const slice = new FileBlob(this.path, this.start + start, Math.min(this.start + end, this.end)); + return slice; + } + /** + * Read the part of the file delimited by the FileBlob and returns it as an ArrayBuffer. + */ + async arrayBuffer() { + const slice = await this.execute((file) => file.read(Buffer.alloc(this.size), 0, this.size, this.start)); + return slice.buffer; + } + /** + * Read the part of the file delimited by the FileBlob and returns it as a string. + */ + async text() { + const buffer = await this.arrayBuffer(); + return buffer.toString("utf8"); + } + /** + * Returns a stream around the part of the file delimited by the FileBlob. + */ + stream() { + if (this.start === this.end) { + return new Blob([]).stream(); + } + return import_node_stream2.Readable.toWeb((0, import_node_fs.createReadStream)(this.path, { start: this.start, end: this.end - 1 })); + } + /** + * We are opening and closing the file for each action to prevent file descriptor leaks. + * + * It is an intended choice of developer experience over performances. + */ + async execute(action) { + const file = await (0, import_promises2.open)(this.path, "r"); + try { + return await action(file); + } finally { + await file.close(); + } + } + }; + } +}); + +// src/utils/sub-paths.ts +var sub_paths_exports = {}; +__export(sub_paths_exports, { + subPaths: () => subPaths +}); +async function subPaths(path2, maxDepth = 10) { + const state = await (0, import_promises3.stat)(path2); + if (!state.isDirectory()) { + return [{ path: path2, relativePath: "." }]; + } + const files = await (0, import_promises3.readdir)(path2, { withFileTypes: true }); + const ret = []; + for (const file of files) { + const filePath = (0, import_node_url2.pathToFileURL)((0, import_node_url2.fileURLToPath)(path2) + "/" + file.name); + if (file.isDirectory()) { + ret.push( + ...(await subPaths(filePath, maxDepth - 1)).map((subPath) => ({ + ...subPath, + relativePath: `${file.name}/${subPath.relativePath}` + })) + ); + } else { + ret.push({ path: filePath, relativePath: file.name }); + } + } + return ret; +} +var import_promises3, import_node_url2; +var init_sub_paths = __esm({ + "src/utils/sub-paths.ts"() { + "use strict"; + import_promises3 = require("fs/promises"); + import_node_url2 = require("url"); + } +}); + +// index.ts +var hub_exports = {}; +__export(hub_exports, { + DATASET_EXPANDABLE_KEYS: () => DATASET_EXPANDABLE_KEYS, + DATASET_EXPAND_KEYS: () => DATASET_EXPAND_KEYS, + DEFAULT_REVISION: () => DEFAULT_REVISION, + HUB_URL: () => HUB_URL, + HubApiError: () => HubApiError, + InvalidApiResponseFormatError: () => InvalidApiResponseFormatError, + MODEL_DERIVED_FIELD_TO_API_KEY: () => MODEL_DERIVED_FIELD_TO_API_KEY, + MODEL_EXPANDABLE_KEYS: () => MODEL_EXPANDABLE_KEYS, + MODEL_EXPAND_KEYS: () => MODEL_EXPAND_KEYS, + REGEX_COMMIT_HASH: () => REGEX_COMMIT_HASH, + REPO_ID_SEPARATOR: () => REPO_ID_SEPARATOR, + RE_SAFETENSORS_FILE: () => RE_SAFETENSORS_FILE, + RE_SAFETENSORS_INDEX_FILE: () => RE_SAFETENSORS_INDEX_FILE, + RE_SAFETENSORS_SHARD_FILE: () => RE_SAFETENSORS_SHARD_FILE, + SAFETENSORS_FILE: () => SAFETENSORS_FILE, + SAFETENSORS_INDEX_FILE: () => SAFETENSORS_INDEX_FILE, + SPACE_EXPANDABLE_KEYS: () => SPACE_EXPANDABLE_KEYS, + SPACE_EXPAND_KEYS: () => SPACE_EXPAND_KEYS, + __internal_XetBlob: () => XetBlob, + __internal_sha256: () => sha256, + cancelJob: () => cancelJob, + checkRepoAccess: () => checkRepoAccess, + commit: () => commit, + commitIter: () => commitIter, + commitIterBucket: () => commitIterBucket, + copyFile: () => copyFile, + copyFileIter: () => copyFileIter, + copyFiles: () => copyFiles, + copyFilesIter: () => copyFilesIter, + copyFolder: () => copyFolder, + copyFolderIter: () => copyFolderIter, + countCommits: () => countCommits, + createBranch: () => createBranch, + createCollection: () => createCollection, + createRepo: () => createRepo, + createScheduledJob: () => createScheduledJob, + datasetInfo: () => datasetInfo, + deleteBranch: () => deleteBranch, + deleteCollection: () => deleteCollection, + deleteFile: () => deleteFile, + deleteFiles: () => deleteFiles, + deleteRepo: () => deleteRepo, + deleteScheduledJob: () => deleteScheduledJob, + downloadFile: () => downloadFile, + downloadFileToCacheDir: () => downloadFileToCacheDir, + duplicateJob: () => duplicateJob, + fileDownloadInfo: () => fileDownloadInfo, + fileExists: () => fileExists, + getBlobStat: () => getBlobStat, + getHFHubCachePath: () => getHFHubCachePath, + getJob: () => getJob, + getRepoFolderName: () => getRepoFolderName, + getScheduledJob: () => getScheduledJob, + globMatch: () => globMatch, + isQuantizedTensor: () => isQuantizedTensor, + listCollections: () => listCollections, + listCommits: () => listCommits, + listDatasets: () => listDatasets, + listFiles: () => listFiles, + listJobHardware: () => listJobHardware, + listJobs: () => listJobs, + listModels: () => listModels, + listScheduledJobs: () => listScheduledJobs, + listSpaces: () => listSpaces, + matchesCompressedTensorsTarget: () => matchesCompressedTensorsTarget, + modelInfo: () => modelInfo, + oauthHandleRedirect: () => oauthHandleRedirect, + oauthHandleRedirectIfPresent: () => oauthHandleRedirectIfPresent, + oauthLoginUrl: () => oauthLoginUrl, + parseRepoType: () => parseRepoType, + parseSafetensorsMetadata: () => parseSafetensorsMetadata, + parseSafetensorsShardFilename: () => parseSafetensorsShardFilename, + pathsInfo: () => pathsInfo, + relativeUnderFolder: () => relativeUnderFolder, + repoExists: () => repoExists, + resumeScheduledJob: () => resumeScheduledJob, + runJob: () => runJob, + runScheduledJob: () => runScheduledJob, + scanCacheDir: () => scanCacheDir, + scanCachedRepo: () => scanCachedRepo, + scanRefsDir: () => scanRefsDir, + scanSnapshotDir: () => scanSnapshotDir, + snapshotDownload: () => snapshotDownload, + spaceInfo: () => spaceInfo, + streamJobEvents: () => streamJobEvents, + streamJobLogs: () => streamJobLogs, + streamJobMetrics: () => streamJobMetrics, + suspendScheduledJob: () => suspendScheduledJob, + uploadFile: () => uploadFile, + uploadFiles: () => uploadFiles, + uploadFilesWithProgress: () => uploadFilesWithProgress, + whoAmI: () => whoAmI +}); +module.exports = __toCommonJS(hub_exports); + +// src/lib/cache-management.ts +var import_node_os = require("os"); +var import_node_path = require("path"); +var import_promises = require("fs/promises"); +function getDefaultHome() { + return (0, import_node_path.join)((0, import_node_os.homedir)(), ".cache"); +} +function getDefaultCachePath() { + return (0, import_node_path.join)(process.env["HF_HOME"] ?? (0, import_node_path.join)(process.env["XDG_CACHE_HOME"] ?? getDefaultHome(), "huggingface"), "hub"); +} +function getHuggingFaceHubCache() { + return process.env["HUGGINGFACE_HUB_CACHE"] ?? getDefaultCachePath(); +} +function getHFHubCachePath() { + return process.env["HF_HUB_CACHE"] ?? getHuggingFaceHubCache(); +} +var FILES_TO_IGNORE = [".DS_Store"]; +var REPO_ID_SEPARATOR = "--"; +function getRepoFolderName({ name, type }) { + const parts = [`${type}s`, ...name.split("/")]; + return parts.join(REPO_ID_SEPARATOR); +} +async function scanCacheDir(cacheDir = void 0) { + if (!cacheDir) { + cacheDir = getHFHubCachePath(); + } + const s = await (0, import_promises.stat)(cacheDir); + if (!s.isDirectory()) { + throw new Error( + `Scan cache expects a directory but found a file: ${cacheDir}. Please use \`cacheDir\` argument or set \`HF_HUB_CACHE\` environment variable.` + ); + } + const repos = []; + const warnings = []; + const directories = await (0, import_promises.readdir)(cacheDir); + for (const repo of directories) { + if (repo === ".locks") { + continue; + } + const absolute = (0, import_node_path.join)(cacheDir, repo); + const s2 = await (0, import_promises.stat)(absolute); + if (!s2.isDirectory()) { + continue; + } + try { + const cached = await scanCachedRepo(absolute); + repos.push(cached); + } catch (err) { + warnings.push(err); + } + } + return { + repos, + size: [...repos.values()].reduce((sum2, repo) => sum2 + repo.size, 0), + warnings + }; +} +async function scanCachedRepo(repoPath) { + const name = (0, import_node_path.basename)(repoPath); + if (!name.includes(REPO_ID_SEPARATOR)) { + throw new Error(`Repo path is not a valid HuggingFace cache directory: ${name}`); + } + const [type, ...remaining] = name.split(REPO_ID_SEPARATOR); + const repoType = parseRepoType(type); + const repoId = remaining.join("/"); + const snapshotsPath = (0, import_node_path.join)(repoPath, "snapshots"); + const refsPath = (0, import_node_path.join)(repoPath, "refs"); + const snapshotStat = await (0, import_promises.stat)(snapshotsPath); + if (!snapshotStat.isDirectory()) { + throw new Error(`Snapshots dir doesn't exist in cached repo ${snapshotsPath}`); + } + const refsByHash = /* @__PURE__ */ new Map(); + const refsStat = await (0, import_promises.stat)(refsPath); + if (refsStat.isDirectory()) { + await scanRefsDir(refsPath, refsByHash); + } + const cachedRevisions = []; + const blobStats = /* @__PURE__ */ new Map(); + const snapshotDirs = await (0, import_promises.readdir)(snapshotsPath); + for (const dir of snapshotDirs) { + if (FILES_TO_IGNORE.includes(dir)) { + continue; + } + const revisionPath = (0, import_node_path.join)(snapshotsPath, dir); + const revisionStat = await (0, import_promises.stat)(revisionPath); + if (!revisionStat.isDirectory()) { + throw new Error(`Snapshots folder corrupted. Found a file: ${revisionPath}`); + } + const cachedFiles = []; + await scanSnapshotDir(revisionPath, cachedFiles, blobStats); + const revisionLastModified = cachedFiles.length > 0 ? Math.max(...[...cachedFiles].map((file) => file.blob.lastModifiedAt.getTime())) : revisionStat.mtimeMs; + cachedRevisions.push({ + commitOid: dir, + files: cachedFiles, + refs: refsByHash.get(dir) || [], + size: [...cachedFiles].reduce((sum2, file) => sum2 + file.blob.size, 0), + path: revisionPath, + lastModifiedAt: new Date(revisionLastModified) + }); + refsByHash.delete(dir); + } + if (refsByHash.size > 0) { + throw new Error( + `Reference(s) refer to missing commit hashes: ${JSON.stringify(Object.fromEntries(refsByHash))} (${repoPath})` + ); + } + const repoStats = await (0, import_promises.stat)(repoPath); + const repoLastAccessed = blobStats.size > 0 ? Math.max(...[...blobStats.values()].map((stat5) => stat5.atimeMs)) : repoStats.atimeMs; + const repoLastModified = blobStats.size > 0 ? Math.max(...[...blobStats.values()].map((stat5) => stat5.mtimeMs)) : repoStats.mtimeMs; + return { + id: { + name: repoId, + type: repoType + }, + path: repoPath, + filesCount: blobStats.size, + revisions: cachedRevisions, + size: [...blobStats.values()].reduce((sum2, stat5) => sum2 + stat5.size, 0), + lastAccessedAt: new Date(repoLastAccessed), + lastModifiedAt: new Date(repoLastModified) + }; +} +async function scanRefsDir(refsPath, refsByHash) { + const refFiles = await (0, import_promises.readdir)(refsPath, { withFileTypes: true }); + for (const refFile of refFiles) { + const refFilePath = (0, import_node_path.join)(refsPath, refFile.name); + if (refFile.isDirectory()) { + continue; + } + const commitHash = await (0, import_promises.readFile)(refFilePath, "utf-8"); + const refName = refFile.name; + if (!refsByHash.has(commitHash)) { + refsByHash.set(commitHash, []); + } + refsByHash.get(commitHash)?.push(refName); + } +} +async function scanSnapshotDir(revisionPath, cachedFiles, blobStats) { + const files = await (0, import_promises.readdir)(revisionPath, { withFileTypes: true }); + for (const file of files) { + if (file.isDirectory()) { + continue; + } + const filePath = (0, import_node_path.join)(revisionPath, file.name); + const blobPath = await (0, import_promises.realpath)(filePath); + const blobStat = await getBlobStat(blobPath, blobStats); + cachedFiles.push({ + path: filePath, + blob: { + path: blobPath, + size: blobStat.size, + lastAccessedAt: new Date(blobStat.atimeMs), + lastModifiedAt: new Date(blobStat.mtimeMs) + } + }); + } +} +async function getBlobStat(blobPath, blobStats) { + const blob = blobStats.get(blobPath); + if (!blob) { + const statResult = await (0, import_promises.lstat)(blobPath); + blobStats.set(blobPath, statResult); + return statResult; + } + return blob; +} +function parseRepoType(type) { + switch (type) { + case "models": + return "model"; + case "datasets": + return "dataset"; + case "spaces": + return "space"; + case "buckets": + return "bucket"; + case "kernels": + return "kernel"; + default: + throw new TypeError(`Invalid repo type: ${type}`); + } +} + +// src/consts.ts +var HUB_URL = "https://huggingface.co"; + +// src/error.ts +async function createApiError(response, opts) { + const error = new HubApiError(response.url, response.status, response.headers.get("X-Request-Id") ?? opts?.requestId); + error.message = `Api error with status ${error.statusCode}${opts?.message ? `. ${opts.message}` : ""}`; + const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : void 0].filter(Boolean).join(". "); + if (response.headers.get("Content-Type")?.startsWith("application/json")) { + const json = await response.json(); + error.message = json.error || json.message || error.message; + if (json.error_description) { + error.message = error.message ? error.message + `: ${json.error_description}` : json.error_description; + } + error.data = json; + } else { + error.data = { message: await response.text() }; + } + error.message += `. ${trailer}`; + throw error; +} +var HubApiError = class extends Error { + statusCode; + url; + requestId; + data; + constructor(url, statusCode, requestId, message) { + super(message); + this.statusCode = statusCode; + this.requestId = requestId; + this.url = url; + } +}; +var InvalidApiResponseFormatError = class extends Error { +}; + +// src/utils/checkCredentials.ts +function checkAccessToken(accessToken) { + if (!accessToken.startsWith("hf_")) { + throw new TypeError("Your access token must start with 'hf_'"); + } +} +function checkCredentials(params) { + if (params.accessToken) { + checkAccessToken(params.accessToken); + return params.accessToken; + } + if (params.credentials?.accessToken) { + checkAccessToken(params.credentials.accessToken); + return params.credentials.accessToken; + } +} + +// src/utils/toRepoId.ts +function toRepoId(repo) { + if (typeof repo !== "string") { + return repo; + } + if (repo.startsWith("model/") || repo.startsWith("models/")) { + throw new TypeError( + "A repo designation for a model should not start with 'models/', directly specify the model namespace / name" + ); + } + if (repo.startsWith("space/")) { + throw new TypeError("Spaces should start with 'spaces/', plural, not 'space/'"); + } + if (repo.startsWith("dataset/")) { + throw new TypeError("Datasets should start with 'datasets/', plural, not 'dataset/'"); + } + if (repo.startsWith("bucket/")) { + throw new TypeError("Buckets should start with 'buckets/', plural, not 'bucket/'"); + } + if (repo.startsWith("kernel/")) { + throw new TypeError("Kernels should start with 'kernels/', plural, not 'kernel/'"); + } + const slashes = repo.split("/").length - 1; + if (repo.startsWith("spaces/")) { + if (slashes !== 2) { + throw new TypeError("Space Id must include namespace and name of the space"); + } + return { + type: "space", + name: repo.slice("spaces/".length) + }; + } + if (repo.startsWith("datasets/")) { + if (slashes > 2) { + throw new TypeError("Too many slashes in repo designation: " + repo); + } + return { + type: "dataset", + name: repo.slice("datasets/".length) + }; + } + if (repo.startsWith("buckets/")) { + if (slashes !== 2) { + throw new TypeError("Bucket Id must include namespace and name of the bucket"); + } + return { + type: "bucket", + name: repo.slice("buckets/".length) + }; + } + if (repo.startsWith("kernels/")) { + if (slashes !== 2) { + throw new TypeError("Kernel Id must include namespace and name of the kernel"); + } + return { + type: "kernel", + name: repo.slice("kernels/".length) + }; + } + if (slashes > 1) { + throw new TypeError("Too many slashes in repo designation: " + repo); + } + return { + type: "model", + name: repo + }; +} + +// src/lib/check-repo-access.ts +async function checkRepoAccess(params) { + const accessToken = params && checkCredentials(params); + const repoId = toRepoId(params.repo); + const response = await (params.fetch || fetch)(`${params?.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}`, { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + }); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/utils/range.ts +function range(n, b) { + return b ? Array(b - n).fill(0).map((_, i) => n + i) : Array(n).fill(0).map((_, i) => i); +} + +// src/utils/chunk.ts +function chunk(arr, chunkSize) { + if (isNaN(chunkSize) || chunkSize < 1) { + throw new RangeError("Invalid chunk size: " + chunkSize); + } + if (!arr.length) { + return []; + } + if (arr.length <= chunkSize) { + return [arr]; + } + return range(Math.ceil(arr.length / chunkSize)).map((i) => { + return arr.slice(i * chunkSize, (i + 1) * chunkSize); + }); +} + +// src/utils/promisesQueue.ts +async function promisesQueue(factories, concurrency) { + const results = []; + const executing = /* @__PURE__ */ new Set(); + let index = 0; + for (const factory of factories) { + const closureIndex = index++; + const e = factory().then((r) => { + results[closureIndex] = r; + executing.delete(e); + }); + executing.add(e); + if (executing.size >= concurrency) { + await Promise.race(executing); + } + } + await Promise.all(executing); + return results; +} + +// src/utils/promisesQueueStreaming.ts +async function promisesQueueStreaming(factories, concurrency) { + const executing = []; + for await (const factory of factories) { + const e = factory().then(() => { + executing.splice(executing.indexOf(e), 1); + }); + executing.push(e); + if (executing.length >= concurrency) { + await Promise.race(executing); + } + } + await Promise.all(executing); +} + +// src/utils/eventToGenerator.ts +async function* eventToGenerator(cb) { + const promises = []; + function addPromise() { + let resolve3; + let reject; + const p = new Promise((res, rej) => { + resolve3 = res; + reject = rej; + }); + promises.push({ p, resolve: resolve3, reject }); + } + addPromise(); + const callbackRes = Promise.resolve().then( + () => cb( + (y) => { + addPromise(); + promises.at(-2)?.resolve({ done: false, value: y }); + }, + (r) => { + addPromise(); + promises.at(-2)?.resolve({ done: true, value: r }); + }, + (err) => promises.shift()?.reject(err) + ) + ).catch((err) => promises.shift()?.reject(err)); + while (1) { + const p = promises[0]; + if (!p) { + throw new Error("Logic error in eventGenerator, promises should never be empty"); + } + const result = await p.p; + promises.shift(); + if (result.done) { + await callbackRes; + return result.value; + } + yield result.value; + } + throw new Error("Unreachable"); +} + +// src/utils/hexFromBytes.ts +function hexFromBytes(arr) { + if (globalThis.Buffer) { + return globalThis.Buffer.from(arr).toString("hex"); + } else { + const bin = []; + arr.forEach((byte) => { + bin.push(byte.toString(16).padStart(2, "0")); + }); + return bin.join(""); + } +} + +// src/utils/isBackend.ts +var isBrowser = typeof window !== "undefined" && typeof window.document !== "undefined"; +var isWebWorker = typeof self === "object" && self.constructor && self.constructor.name === "DedicatedWorkerGlobalScope"; +var isBackend = !isBrowser && !isWebWorker; + +// src/utils/isFrontend.ts +var isFrontend = !isBackend; + +// src/utils/sha256.ts +async function getWebWorkerCode() { + const sha256Module = await Promise.resolve().then(() => (init_sha256_wrapper(), sha256_wrapper_exports)); + return URL.createObjectURL(new Blob([sha256Module.createSHA256WorkerCode()])); +} +var pendingWorkers = []; +var runningWorkers = /* @__PURE__ */ new Set(); +var resolve; +var waitPromise = new Promise((r) => { + resolve = r; +}); +async function getWorker(poolSize) { + { + const worker2 = pendingWorkers.pop(); + if (worker2) { + runningWorkers.add(worker2); + return worker2; + } + } + if (!poolSize) { + const worker2 = new Worker(await getWebWorkerCode()); + runningWorkers.add(worker2); + return worker2; + } + if (poolSize <= 0) { + throw new TypeError("Invalid webworker pool size: " + poolSize); + } + while (runningWorkers.size >= poolSize) { + await waitPromise; + } + const worker = new Worker(await getWebWorkerCode()); + runningWorkers.add(worker); + return worker; +} +async function freeWorker(worker, poolSize) { + if (!poolSize) { + return destroyWorker(worker); + } + runningWorkers.delete(worker); + pendingWorkers.push(worker); + const r = resolve; + waitPromise = new Promise((r2) => { + resolve = r2; + }); + r(); +} +function destroyWorker(worker) { + runningWorkers.delete(worker); + worker.terminate(); + const r = resolve; + waitPromise = new Promise((r2) => { + resolve = r2; + }); + r(); +} +async function* sha256(buffer, opts) { + yield 0; + const maxCryptoSize = typeof opts?.useWebWorker === "object" && opts?.useWebWorker.minSize !== void 0 ? opts.useWebWorker.minSize : 1e7; + if (buffer.size < maxCryptoSize && globalThis.crypto?.subtle) { + const res = hexFromBytes( + new Uint8Array( + await globalThis.crypto.subtle.digest("SHA-256", buffer instanceof Blob ? await buffer.arrayBuffer() : buffer) + ) + ); + yield 1; + return res; + } + if (isFrontend) { + if (opts?.useWebWorker) { + try { + const poolSize = typeof opts?.useWebWorker === "object" ? opts.useWebWorker.poolSize : void 0; + const worker = await getWorker(poolSize); + let messageHandler; + let errorHandler; + const cleanup = () => { + worker.removeEventListener("message", messageHandler); + worker.removeEventListener("error", errorHandler); + }; + return yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + messageHandler = (event) => { + if (event.data.sha256) { + cleanup(); + freeWorker(worker, poolSize); + returnCallback(event.data.sha256); + } else if (event.data.progress) { + yieldCallback(event.data.progress); + try { + opts.abortSignal?.throwIfAborted(); + } catch (err) { + cleanup(); + destroyWorker(worker); + rejectCallback(err); + } + } else { + cleanup(); + destroyWorker(worker); + rejectCallback(event); + } + }; + errorHandler = (event) => { + cleanup(); + destroyWorker(worker); + rejectCallback(event.error); + }; + if (opts?.abortSignal) { + try { + opts.abortSignal?.throwIfAborted(); + } catch (err) { + cleanup(); + destroyWorker(worker); + rejectCallback(opts.abortSignal.reason ?? new DOMException("Aborted", "AbortError")); + return; + } + const abortListener = () => { + cleanup(); + destroyWorker(worker); + rejectCallback(opts.abortSignal?.reason ?? new DOMException("Aborted", "AbortError")); + opts.abortSignal?.removeEventListener("abort", abortListener); + }; + opts.abortSignal.addEventListener("abort", abortListener); + } + worker.addEventListener("message", messageHandler); + worker.addEventListener("error", errorHandler); + worker.postMessage({ file: buffer }); + }); + } catch (err) { + console.warn("Failed to use web worker for sha256", err); + } + } + if (!wasmModule) { + wasmModule = await Promise.resolve().then(() => (init_sha256_wrapper(), sha256_wrapper_exports)); + } + const sha2562 = await wasmModule.createSHA256(); + sha2562.init(); + const reader = buffer.stream().getReader(); + const total = buffer.size; + let bytesDone = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + sha2562.update(value); + bytesDone += value.length; + yield bytesDone / total; + opts?.abortSignal?.throwIfAborted(); + } + return sha2562.digest("hex"); + } + if (!cryptoModule) { + cryptoModule = await Promise.resolve().then(() => (init_sha256_node(), sha256_node_exports)); + } + return yield* cryptoModule.sha256Node(buffer, { abortSignal: opts?.abortSignal }); +} +var cryptoModule; +var wasmModule; + +// src/utils/WebBlob.ts +var WebBlob = class extends Blob { + static async create(url, opts) { + const customFetch = opts?.fetch ?? fetch; + const probe = await customFetch(url, { + headers: { + Range: "bytes=0-0", + ...opts?.accessToken && { Authorization: `Bearer ${opts.accessToken}` } + } + }); + if (!probe.ok) { + throw await createApiError(probe); + } + const contentType = probe.headers.get("content-type") || ""; + if (probe.status === 206) { + const totalSize = Number(probe.headers.get("content-range")?.split("/").pop()); + await probe.body?.cancel(); + if (Number.isFinite(totalSize) && totalSize >= (opts?.cacheBelow ?? 1e6)) { + return new WebBlob(url, 0, totalSize, contentType, true, customFetch, opts?.accessToken); + } + const full = await customFetch(url, { + ...opts?.accessToken && { headers: { Authorization: `Bearer ${opts.accessToken}` } } + }); + if (!full.ok) { + throw await createApiError(full); + } + return full.blob(); + } + return probe.blob(); + } + url; + start; + end; + contentType; + full; + fetch; + accessToken; + constructor(url, start, end, contentType, full, customFetch, accessToken) { + super([]); + this.url = url; + this.start = start; + this.end = end; + this.contentType = contentType; + this.full = full; + this.fetch = customFetch; + this.accessToken = accessToken; + } + get size() { + return this.end - this.start; + } + get type() { + return this.contentType; + } + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on WebBlob.slice"); + } + const slice = new WebBlob( + this.url, + this.start + start, + Math.min(this.start + end, this.end), + this.contentType, + start === 0 && end === this.size ? this.full : false, + this.fetch, + this.accessToken + ); + return slice; + } + async arrayBuffer() { + const result = await this.fetchRange(); + return result.arrayBuffer(); + } + async text() { + const result = await this.fetchRange(); + return result.text(); + } + stream() { + const stream = new TransformStream(); + this.fetchRange().then((response) => response.body?.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message)); + return stream.readable; + } + fetchRange() { + const fetch2 = this.fetch; + if (this.full) { + return fetch2(this.url, { + ...this.accessToken && { + headers: { + Authorization: `Bearer ${this.accessToken}` + } + } + }).then((resp) => resp.ok ? resp : createApiError(resp)); + } + return fetch2(this.url, { + headers: { + Range: `bytes=${this.start}-${this.end - 1}`, + ...this.accessToken && { Authorization: `Bearer ${this.accessToken}` } + } + }).then((resp) => resp.ok ? resp : createApiError(resp)); + } +}; + +// src/utils/base64FromBytes.ts +function base64FromBytes(arr) { + if (globalThis.Buffer) { + return globalThis.Buffer.from(arr).toString("base64"); + } else { + const bin = []; + arr.forEach((byte) => { + bin.push(String.fromCharCode(byte)); + }); + return globalThis.btoa(bin.join("")); + } +} + +// src/utils/createBlobs.ts +async function createBlobs(url, destPath, opts) { + if (url.protocol === "http:" || url.protocol === "https:") { + const blob = await WebBlob.create(url, { fetch: opts?.fetch, accessToken: opts?.accessToken }); + return [{ path: destPath, blob }]; + } + if (isFrontend) { + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); + } + if (url.protocol === "file:") { + const { FileBlob: FileBlob2 } = await Promise.resolve().then(() => (init_FileBlob(), FileBlob_exports)); + const { subPaths: subPaths2 } = await Promise.resolve().then(() => (init_sub_paths(), sub_paths_exports)); + const paths = await subPaths2(url, opts?.maxFolderDepth); + if (paths.length === 1 && paths[0].relativePath === ".") { + const blob = await FileBlob2.create(url); + return [{ path: destPath, blob }]; + } + return Promise.all( + paths.map(async (path2) => ({ + path: `${destPath}/${path2.relativePath}`.replace(/\/[.]$/, "").replaceAll("//", "/").replace(/^[.]?\//, ""), + blob: await FileBlob2.create(new URL(path2.path)) + })) + ); + } + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); +} + +// src/utils/combineUint8Arrays.ts +function combineUint8Arrays(a, b) { + const aLength = a.length; + const combinedBytes = new Uint8Array(aLength + b.length); + combinedBytes.set(a); + combinedBytes.set(b, aLength); + return combinedBytes; +} + +// src/vendor/lz4js/util.ts +function hashU32(a) { + a = a | 0; + a = a + 2127912214 + (a << 12) | 0; + a = a ^ -949894596 ^ a >>> 19; + a = a + 374761393 + (a << 5) | 0; + a = a + -744332180 ^ a << 9; + a = a + -42973499 + (a << 3) | 0; + return a ^ -1252372727 ^ a >>> 16 | 0; +} +function readU64(b, n) { + let x = 0; + x |= b[n++] << 0; + x |= b[n++] << 8; + x |= b[n++] << 16; + x |= b[n++] << 24; + x |= b[n++] << 32; + x |= b[n++] << 40; + x |= b[n++] << 48; + x |= b[n++] << 56; + return x; +} +function readU32(b, n) { + let x = 0; + x |= b[n++] << 0; + x |= b[n++] << 8; + x |= b[n++] << 16; + x |= b[n++] << 24; + return x; +} +function writeU32(b, n, x) { + b[n++] = x >> 0 & 255; + b[n++] = x >> 8 & 255; + b[n++] = x >> 16 & 255; + b[n++] = x >> 24 & 255; +} +function imul(a, b) { + const ah = a >>> 16; + const al = a & 65535; + const bh = b >>> 16; + const bl = b & 65535; + return al * bl + (ah * bl + al * bh << 16) | 0; +} + +// src/vendor/lz4js/xxh32.ts +var prime1 = 2654435761; +var prime2 = 2246822519; +var prime3 = 3266489917; +var prime4 = 668265263; +var prime5 = 374761393; +function rotl32(x, r) { + x = x | 0; + r = r | 0; + return x >>> (32 - r | 0) | x << r | 0; +} +function rotmul32(h, r, m) { + h = h | 0; + r = r | 0; + m = m | 0; + return imul(h >>> (32 - r | 0) | h << r, m) | 0; +} +function shiftxor32(h, s) { + h = h | 0; + s = s | 0; + return h >>> s ^ h | 0; +} +function xxhapply(h, src, m0, s, m1) { + return rotmul32(imul(src, m0) + h, s, m1); +} +function xxh1(h, src, index) { + return rotmul32(h + imul(src[index], prime5), 11, prime1); +} +function xxh4(h, src, index) { + return xxhapply(h, readU32(src, index), prime3, 17, prime4); +} +function xxh16(h, src, index) { + return [ + xxhapply(h[0], readU32(src, index + 0), prime2, 13, prime1), + xxhapply(h[1], readU32(src, index + 4), prime2, 13, prime1), + xxhapply(h[2], readU32(src, index + 8), prime2, 13, prime1), + xxhapply(h[3], readU32(src, index + 12), prime2, 13, prime1) + ]; +} +function xxh32(seed, src, index, len) { + let h; + const l = len; + if (len >= 16) { + h = [seed + prime1 + prime2, seed + prime2, seed, seed - prime1]; + while (len >= 16) { + h = xxh16(h, src, index); + index += 16; + len -= 16; + } + h = rotl32(h[0], 1) + rotl32(h[1], 7) + rotl32(h[2], 12) + rotl32(h[3], 18) + l; + } else { + h = seed + prime5 + len >>> 0; + } + while (len >= 4) { + h = xxh4(h, src, index); + index += 4; + len -= 4; + } + while (len > 0) { + h = xxh1(h, src, index); + index++; + len--; + } + h = shiftxor32(imul(shiftxor32(imul(shiftxor32(h, 15), prime2), 13), prime3), 16); + return h >>> 0; +} +var hash = xxh32; + +// src/vendor/lz4js/index.ts +var minMatch = 4; +var matchSearchLimit = 12; +var minTrailingLitterals = 5; +var skipTrigger = 6; +var hashSize = 1 << 16; +var mlBits = 4; +var mlMask = (1 << mlBits) - 1; +var runBits = 4; +var runMask = (1 << runBits) - 1; +var blockBuf = makeBuffer(5 << 20); +var hashTable = makeHashTable(); +var magicNum = 407708164; +var fdContentChksum = 4; +var fdContentSize = 8; +var fdBlockChksum = 16; +var fdVersion = 64; +var fdVersionMask = 192; +var bsUncompressed = 2147483648; +var bsDefault = 7; +var bsShift = 4; +var bsMask = 7; +var bsMap = { + 4: 65536, + 5: 262144, + 6: 1048576, + 7: 4194304 +}; +function makeHashTable() { + try { + return new Uint32Array(hashSize); + } catch (error) { + const hashTable2 = new Array(hashSize); + for (let i = 0; i < hashSize; i++) { + hashTable2[i] = 0; + } + return hashTable2; + } +} +function clearHashTable(table) { + for (let i = 0; i < hashSize; i++) { + table[i] = 0; + } +} +function makeBuffer(size) { + return new Uint8Array(size); +} +function sliceArray(array, start, end) { + return array.slice(start, end); +} +function compressBound(n) { + return n + n / 255 + 16 | 0; +} +function decompressBound(src) { + let sIndex = 0; + if (readU32(src, sIndex) !== magicNum) { + throw new Error("invalid magic number"); + } + sIndex += 4; + const descriptor = src[sIndex++]; + if ((descriptor & fdVersionMask) !== fdVersion) { + throw new Error("incompatible descriptor version " + (descriptor & fdVersionMask)); + } + const useBlockSum = (descriptor & fdBlockChksum) !== 0; + const useContentSize = (descriptor & fdContentSize) !== 0; + const bsIdx = src[sIndex++] >> bsShift & bsMask; + if (bsMap[bsIdx] === void 0) { + throw new Error("invalid block size " + bsIdx); + } + const maxBlockSize = bsMap[bsIdx]; + if (useContentSize) { + return readU64(src, sIndex); + } + sIndex++; + let maxSize = 0; + while (true) { + let blockSize = readU32(src, sIndex); + sIndex += 4; + if (blockSize & bsUncompressed) { + blockSize &= ~bsUncompressed; + maxSize += blockSize; + } else if (blockSize > 0) { + maxSize += maxBlockSize; + } + if (blockSize === 0) { + return maxSize; + } + if (useBlockSum) { + sIndex += 4; + } + sIndex += blockSize; + } +} +function decompressBlock(src, dst, sIndex, sLength, dIndex) { + let mLength, mOffset, sEnd, n, i; + const hasCopyWithin = dst.copyWithin !== void 0 && dst.fill !== void 0; + sEnd = sIndex + sLength; + while (sIndex < sEnd) { + const token = src[sIndex++]; + let literalCount = token >> 4; + if (literalCount > 0) { + if (literalCount === 15) { + while (true) { + literalCount += src[sIndex]; + if (src[sIndex++] !== 255) { + break; + } + } + } + for (n = sIndex + literalCount; sIndex < n; ) { + dst[dIndex++] = src[sIndex++]; + } + } + if (sIndex >= sEnd) { + break; + } + mLength = token & 15; + mOffset = src[sIndex++] | src[sIndex++] << 8; + if (mLength === 15) { + while (true) { + mLength += src[sIndex]; + if (src[sIndex++] !== 255) { + break; + } + } + } + mLength += minMatch; + if (hasCopyWithin && mOffset === 1) { + dst.fill(dst[dIndex - 1] | 0, dIndex, dIndex + mLength); + dIndex += mLength; + } else if (hasCopyWithin && mOffset > mLength && mLength > 31) { + dst.copyWithin(dIndex, dIndex - mOffset, dIndex - mOffset + mLength); + dIndex += mLength; + } else { + for (i = dIndex - mOffset, n = i + mLength; i < n; ) { + dst[dIndex++] = dst[i++] | 0; + } + } + } + return dIndex; +} +function compressBlock(src, dst, sIndex, sLength, hashTable2) { + let mIndex, mAnchor, mLength, mOffset, mStep; + let literalCount, dIndex, sEnd, n; + dIndex = 0; + sEnd = sLength + sIndex; + mAnchor = sIndex; + let searchMatchCount = (1 << skipTrigger) + 3; + while (sIndex <= sEnd - matchSearchLimit) { + const seq = readU32(src, sIndex); + let hash2 = hashU32(seq) >>> 0; + hash2 = (hash2 >> 16 ^ hash2) >>> 0 & 65535; + mIndex = hashTable2[hash2] - 1; + hashTable2[hash2] = sIndex + 1; + if (mIndex < 0 || sIndex - mIndex >>> 16 > 0 || readU32(src, mIndex) !== seq) { + mStep = searchMatchCount++ >> skipTrigger; + sIndex += mStep; + continue; + } + searchMatchCount = (1 << skipTrigger) + 3; + literalCount = sIndex - mAnchor; + mOffset = sIndex - mIndex; + sIndex += minMatch; + mIndex += minMatch; + mLength = sIndex; + while (sIndex < sEnd - minTrailingLitterals && src[sIndex] === src[mIndex]) { + sIndex++; + mIndex++; + } + mLength = sIndex - mLength; + const token = mLength < mlMask ? mLength : mlMask; + if (literalCount >= runMask) { + dst[dIndex++] = (runMask << mlBits) + token; + for (n = literalCount - runMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } else { + dst[dIndex++] = (literalCount << mlBits) + token; + } + for (let i = 0; i < literalCount; i++) { + dst[dIndex++] = src[mAnchor + i]; + } + dst[dIndex++] = mOffset; + dst[dIndex++] = mOffset >> 8; + if (mLength >= mlMask) { + for (n = mLength - mlMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } + mAnchor = sIndex; + } + if (mAnchor === 0) { + return 0; + } + literalCount = sEnd - mAnchor; + if (literalCount >= runMask) { + dst[dIndex++] = runMask << mlBits; + for (n = literalCount - runMask; n >= 255; n -= 255) { + dst[dIndex++] = 255; + } + dst[dIndex++] = n; + } else { + dst[dIndex++] = literalCount << mlBits; + } + sIndex = mAnchor; + while (sIndex < sEnd) { + dst[dIndex++] = src[sIndex++]; + } + return dIndex; +} +function decompressFrame(src, dst) { + let useBlockSum, useContentSum, useContentSize, descriptor; + let sIndex = 0; + let dIndex = 0; + if (readU32(src, sIndex) !== magicNum) { + throw new Error("invalid magic number"); + } + sIndex += 4; + descriptor = src[sIndex++]; + if ((descriptor & fdVersionMask) !== fdVersion) { + throw new Error("incompatible descriptor version"); + } + useBlockSum = (descriptor & fdBlockChksum) !== 0; + useContentSum = (descriptor & fdContentChksum) !== 0; + useContentSize = (descriptor & fdContentSize) !== 0; + const bsIdx = src[sIndex++] >> bsShift & bsMask; + if (bsMap[bsIdx] === void 0) { + throw new Error("invalid block size"); + } + if (useContentSize) { + sIndex += 8; + } + sIndex++; + while (true) { + var compSize; + compSize = readU32(src, sIndex); + sIndex += 4; + if (compSize === 0) { + break; + } + if (useBlockSum) { + sIndex += 4; + } + if ((compSize & bsUncompressed) !== 0) { + compSize &= ~bsUncompressed; + for (let j = 0; j < compSize; j++) { + dst[dIndex++] = src[sIndex++]; + } + } else { + dIndex = decompressBlock(src, dst, sIndex, compSize, dIndex); + sIndex += compSize; + } + } + if (useContentSum) { + sIndex += 4; + } + return dIndex; +} +function compressFrame(src, dst) { + let dIndex = 0; + writeU32(dst, dIndex, magicNum); + dIndex += 4; + dst[dIndex++] = fdVersion; + dst[dIndex++] = bsDefault << bsShift; + dst[dIndex] = hash(0, dst, 4, dIndex - 4) >> 8; + dIndex++; + const maxBlockSize = bsMap[bsDefault]; + let remaining = src.length; + let sIndex = 0; + clearHashTable(hashTable); + while (remaining > 0) { + let compSize = 0; + const blockSize = remaining > maxBlockSize ? maxBlockSize : remaining; + compSize = compressBlock(src, blockBuf, sIndex, blockSize, hashTable); + if (compSize > blockSize || compSize === 0) { + writeU32(dst, dIndex, 2147483648 | blockSize); + dIndex += 4; + for (let z = sIndex + blockSize; sIndex < z; ) { + dst[dIndex++] = src[sIndex++]; + } + remaining -= blockSize; + } else { + writeU32(dst, dIndex, compSize); + dIndex += 4; + for (let j = 0; j < compSize; ) { + dst[dIndex++] = blockBuf[j++]; + } + sIndex += blockSize; + remaining -= blockSize; + } + } + writeU32(dst, dIndex, 0); + dIndex += 4; + return dIndex; +} +function decompress(src, maxSize) { + let dst, size; + if (maxSize === void 0) { + maxSize = decompressBound(src); + } + dst = makeBuffer(maxSize); + size = decompressFrame(src, dst); + if (size !== maxSize) { + dst = sliceArray(dst, 0, size); + } + return dst; +} +function compress(src, maxSize) { + let dst, size; + if (maxSize === void 0) { + maxSize = compressBound(src.length); + } + dst = makeBuffer(maxSize); + size = compressFrame(src, dst); + if (size !== maxSize) { + dst = sliceArray(dst, 0, size); + } + return dst; +} + +// src/utils/RangeList.ts +var RangeList = class { + ranges = []; + /** + * Add a range to the list. If it overlaps with existing ranges, + * it will split them and increment reference counts accordingly. + */ + add(start, end) { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + const overlappingRanges = []; + for (let i = 0; i < this.ranges.length; i++) { + const range2 = this.ranges[i]; + if (start < range2.end && end > range2.start) { + overlappingRanges.push({ index: i, range: range2 }); + } + if (range2.data !== null) { + throw new Error("Overlapping range already has data"); + } + } + if (overlappingRanges.length === 0) { + this.ranges.push({ start, end, refCount: 1, data: null }); + this.ranges.sort((a, b) => a.start - b.start); + return; + } + const newRanges = []; + let currentPos = start; + for (let i = 0; i < overlappingRanges.length; i++) { + const { range: range2 } = overlappingRanges[i]; + if (currentPos < range2.start) { + newRanges.push({ + start: currentPos, + end: range2.start, + refCount: 1, + data: null + }); + } else if (range2.start < currentPos) { + newRanges.push({ + start: range2.start, + end: currentPos, + refCount: range2.refCount, + data: null + }); + } + newRanges.push({ + start: Math.max(currentPos, range2.start), + end: Math.min(end, range2.end), + refCount: range2.refCount + 1, + data: null + }); + if (range2.end > end) { + newRanges.push({ + start: end, + end: range2.end, + refCount: range2.refCount, + data: null + }); + } + currentPos = Math.max(currentPos, range2.end); + } + if (currentPos < end) { + newRanges.push({ + start: currentPos, + end, + refCount: 1, + data: null + }); + } + const firstIndex = overlappingRanges[0].index; + const lastIndex = overlappingRanges[overlappingRanges.length - 1].index; + this.ranges.splice(firstIndex, lastIndex - firstIndex + 1, ...newRanges); + this.ranges.sort((a, b) => a.start - b.start); + } + /** + * Remove a range from the list. The range must start and end at existing boundaries. + */ + remove(start, end) { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + const affectedRanges = []; + for (let i = 0; i < this.ranges.length; i++) { + const range2 = this.ranges[i]; + if (start < range2.end && end > range2.start) { + affectedRanges.push({ index: i, range: range2 }); + } + } + if (affectedRanges.length === 0) { + throw new Error("No ranges found to remove"); + } + if (start !== affectedRanges[0].range.start || end !== affectedRanges[affectedRanges.length - 1].range.end) { + throw new Error("Range boundaries must match existing boundaries"); + } + for (let i = 0; i < affectedRanges.length; i++) { + const { range: range2 } = affectedRanges[i]; + range2.refCount--; + } + this.ranges = this.ranges.filter((range2) => range2.refCount > 0); + } + /** + * Get all ranges within the specified boundaries. + */ + getRanges(start, end) { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + return this.ranges.filter((range2) => start < range2.end && end > range2.start); + } + /** + * Get all ranges in the list + */ + getAllRanges() { + return [...this.ranges]; + } +}; + +// src/utils/XetBlob.ts +var JWT_SAFETY_PERIOD = 6e4; +var JWT_CACHE_SIZE = 1e3; +var compressionSchemeLabels = { + [0 /* None */]: "None", + [1 /* LZ4 */]: "LZ4", + [2 /* ByteGroupingLZ4 */]: "ByteGroupingLZ4" +}; +var XET_CHUNK_HEADER_BYTES = 8; +var XetBlob = class extends Blob { + fetch; + accessToken; + refreshUrl; + reconstructionUrl; + hash; + start = 0; + end = 0; + internalLogging = false; + reconstructionInfo; + listener; + constructor(params) { + super([]); + this.fetch = params.fetch ?? fetch.bind(globalThis); + this.accessToken = checkCredentials(params); + this.refreshUrl = params.refreshUrl; + this.end = params.size; + this.reconstructionUrl = params.reconstructionUrl; + this.hash = params.hash; + this.listener = params.listener; + this.internalLogging = params.internalLogging ?? false; + if (params.readToken) { + const key = cacheKey({ refreshUrl: this.refreshUrl, initialAccessToken: this.accessToken }); + jwts.set(key, { + accessToken: params.readToken.accessToken, + expiresAt: new Date(params.readToken.exp * 1e3), + casUrl: params.readToken.casUrl + }); + } + } + get size() { + return this.end - this.start; + } + #clone() { + const blob = new XetBlob({ + fetch: this.fetch, + hash: this.hash, + refreshUrl: this.refreshUrl, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + reconstructionUrl: this.reconstructionUrl, + size: this.size + }); + blob.accessToken = this.accessToken; + blob.start = this.start; + blob.end = this.end; + blob.reconstructionInfo = this.reconstructionInfo; + blob.listener = this.listener; + blob.internalLogging = this.internalLogging; + return blob; + } + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on XetBlob.slice"); + } + const slice = this.#clone(); + slice.start = this.start + start; + slice.end = Math.min(this.start + end, this.end); + if (slice.start !== this.start || slice.end !== this.end) { + slice.reconstructionInfo = void 0; + } + return slice; + } + #reconstructionInfoPromise; + #loadReconstructionInfo() { + if (this.#reconstructionInfoPromise) { + return this.#reconstructionInfoPromise; + } + this.#reconstructionInfoPromise = (async () => { + const connParams = await getAccessToken(this.accessToken, this.fetch, this.refreshUrl); + const resp = await this.fetch(this.reconstructionUrl ?? `${connParams.casUrl}/v1/reconstructions/${this.hash}`, { + headers: { + Authorization: `Bearer ${connParams.accessToken}`, + Range: `bytes=${this.start}-${this.end - 1}` + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } + this.reconstructionInfo = await resp.json(); + return this.reconstructionInfo; + })().finally(() => this.#reconstructionInfoPromise = void 0); + return this.#reconstructionInfoPromise; + } + async #fetch() { + if (this.size === 0) { + return new ReadableStream({ + start(controller) { + controller.close(); + } + }); + } + if (!this.reconstructionInfo) { + await this.#loadReconstructionInfo(); + } + const rangeLists = /* @__PURE__ */ new Map(); + if (!this.reconstructionInfo) { + throw new Error("Failed to load reconstruction info"); + } + for (const term of this.reconstructionInfo.terms) { + let rangeList = rangeLists.get(term.hash); + if (!rangeList) { + rangeList = new RangeList(); + rangeLists.set(term.hash, rangeList); + } + rangeList.add(term.range.start, term.range.end); + } + const listener = this.listener; + const log = this.internalLogging ? (...args) => console.log(...args) : () => { + }; + async function* readData(reconstructionInfo, customFetch, maxBytes, reloadReconstructionInfo) { + let totalBytesRead = 0; + let readBytesToSkip = reconstructionInfo.offset_into_first_range; + for (const term of reconstructionInfo.terms) { + if (totalBytesRead >= maxBytes) { + break; + } + const rangeList = rangeLists.get(term.hash); + if (!rangeList) { + throw new Error(`Failed to find range list for term ${term.hash}`); + } + { + const termRanges = rangeList.getRanges(term.range.start, term.range.end); + if (termRanges.every((range2) => range2.data)) { + log("all data available for term", term.hash, readBytesToSkip); + rangeLoop: + for (const range2 of termRanges) { + for (let chunk2 of range2.data) { + if (readBytesToSkip) { + const skipped = Math.min(readBytesToSkip, chunk2.byteLength); + chunk2 = chunk2.slice(skipped); + readBytesToSkip -= skipped; + if (!chunk2.byteLength) { + continue; + } + } + if (chunk2.byteLength > maxBytes - totalBytesRead) { + chunk2 = chunk2.slice(0, maxBytes - totalBytesRead); + } + totalBytesRead += chunk2.byteLength; + yield range2.refCount > 1 ? chunk2.slice() : chunk2; + listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } }); + if (totalBytesRead >= maxBytes) { + break rangeLoop; + } + } + } + rangeList.remove(term.range.start, term.range.end); + continue; + } + } + let fetchInfo = reconstructionInfo.fetch_info[term.hash].find( + (info) => info.range.start <= term.range.start && info.range.end >= term.range.end + ); + if (!fetchInfo) { + throw new Error( + `Failed to find fetch info for term ${term.hash} and range ${term.range.start}-${term.range.end}` + ); + } + log("term", term); + log("fetchinfo", fetchInfo); + log("readBytesToSkip", readBytesToSkip); + let resp = await customFetch(fetchInfo.url, { + headers: { + Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}` + } + }); + if (resp.status === 403) { + reconstructionInfo = await reloadReconstructionInfo(); + fetchInfo = reconstructionInfo.fetch_info[term.hash]?.find( + (info) => info.range.start <= term.range.start && info.range.end >= term.range.end + ); + if (!fetchInfo) { + throw new Error( + `Failed to find fetch info for term ${term.hash} and range ${term.range.start}-${term.range.end} after refresh` + ); + } + resp = await customFetch(fetchInfo.url, { + headers: { + Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}` + } + }); + } + if (!resp.ok) { + throw await createApiError(resp); + } + log( + "expected content length", + resp.headers.get("content-length"), + "range", + fetchInfo.url_range, + resp.headers.get("content-range") + ); + const reader = resp.body?.getReader(); + if (!reader) { + throw new Error("Failed to get reader from response body"); + } + let done = false; + let chunkIndex = fetchInfo.range.start; + const ranges = rangeList.getRanges(fetchInfo.range.start, fetchInfo.range.end); + let leftoverBytes = void 0; + let totalFetchBytes = 0; + fetchData: + while (!done && totalBytesRead < maxBytes) { + const result = await reader.read(); + listener?.({ event: "read" }); + done = result.done; + log("read", result.value?.byteLength, "bytes", "total read", totalBytesRead, "toSkip", readBytesToSkip); + if (!result.value) { + log("no data in result, cancelled", result); + continue; + } + totalFetchBytes += result.value.byteLength; + if (leftoverBytes) { + result.value = combineUint8Arrays(leftoverBytes, result.value); + leftoverBytes = void 0; + } + while (totalBytesRead < maxBytes && result.value?.byteLength) { + if (result.value.byteLength < 8) { + leftoverBytes = result.value; + continue fetchData; + } + const header = new DataView(result.value.buffer, result.value.byteOffset, XET_CHUNK_HEADER_BYTES); + const chunkHeader = { + version: header.getUint8(0), + compressed_length: header.getUint8(1) | header.getUint8(2) << 8 | header.getUint8(3) << 16, + compression_scheme: header.getUint8(4), + uncompressed_length: header.getUint8(5) | header.getUint8(6) << 8 | header.getUint8(7) << 16 + }; + log("chunk header", chunkHeader, "to skip", readBytesToSkip); + if (chunkHeader.version !== 0) { + throw new Error(`Unsupported chunk version ${chunkHeader.version}`); + } + if (chunkHeader.compression_scheme !== 0 /* None */ && chunkHeader.compression_scheme !== 1 /* LZ4 */ && chunkHeader.compression_scheme !== 2 /* ByteGroupingLZ4 */) { + throw new Error( + `Unsupported compression scheme ${compressionSchemeLabels[chunkHeader.compression_scheme] ?? chunkHeader.compression_scheme}` + ); + } + if (result.value.byteLength < chunkHeader.compressed_length + XET_CHUNK_HEADER_BYTES) { + leftoverBytes = result.value; + continue fetchData; + } + result.value = result.value.slice(XET_CHUNK_HEADER_BYTES); + let uncompressed = chunkHeader.compression_scheme === 1 /* LZ4 */ ? decompress(result.value.slice(0, chunkHeader.compressed_length), chunkHeader.uncompressed_length) : chunkHeader.compression_scheme === 2 /* ByteGroupingLZ4 */ ? bg4_regroup_bytes( + decompress( + result.value.slice(0, chunkHeader.compressed_length), + chunkHeader.uncompressed_length + ) + ) : result.value.slice(0, chunkHeader.compressed_length); + const range2 = ranges.find((range3) => chunkIndex >= range3.start && chunkIndex < range3.end); + const shouldYield = chunkIndex >= term.range.start && chunkIndex < term.range.end; + const minRefCountToStore = shouldYield ? 2 : 1; + let stored = false; + if (range2 && range2.refCount >= minRefCountToStore) { + range2.data ??= []; + range2.data.push(uncompressed); + stored = true; + } + if (shouldYield) { + if (readBytesToSkip) { + const skipped = Math.min(readBytesToSkip, uncompressed.byteLength); + uncompressed = uncompressed.slice(readBytesToSkip); + readBytesToSkip -= skipped; + } + if (uncompressed.byteLength > maxBytes - totalBytesRead) { + uncompressed = uncompressed.slice(0, maxBytes - totalBytesRead); + } + if (uncompressed.byteLength) { + log( + "yield", + uncompressed.byteLength, + "bytes", + result.value.byteLength, + "total read", + totalBytesRead, + stored + ); + totalBytesRead += uncompressed.byteLength; + yield stored ? uncompressed.slice() : uncompressed; + listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } }); + } + } + chunkIndex++; + result.value = result.value.slice(chunkHeader.compressed_length); + } + } + if (done && totalBytesRead < maxBytes && totalFetchBytes < fetchInfo.url_range.end - fetchInfo.url_range.start + 1) { + log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes); + log("failed to fetch all data for term", term.hash); + throw new Error( + `Failed to fetch all data for term ${term.hash}, fetched ${totalFetchBytes} bytes out of ${fetchInfo.url_range.end - fetchInfo.url_range.start + 1}` + ); + } + log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes); + log("cancel reader"); + await reader.cancel(); + } + } + const iterator = readData( + this.reconstructionInfo, + this.fetch, + this.end - this.start, + this.#loadReconstructionInfo.bind(this) + ); + return new ReadableStream( + { + // todo: when Safari supports it, type controller as ReadableByteStreamController + async pull(controller) { + const result = await iterator.next(); + if (result.value) { + controller.enqueue(result.value); + } + if (result.done) { + controller.close(); + } + }, + type: "bytes" + // todo: when Safari supports it, add autoAllocateChunkSize param + }, + // todo : use ByteLengthQueuingStrategy when there's good support for it, currently in Node.js it fails due to size being a function + { + highWaterMark: 1e3 + // 1_000 chunks for ~1MB of RAM + } + ); + } + async arrayBuffer() { + const result = await this.#fetch(); + return new Response(result).arrayBuffer(); + } + async text() { + const result = await this.#fetch(); + return new Response(result).text(); + } + async response() { + const result = await this.#fetch(); + return new Response(result); + } + stream() { + const stream = new TransformStream(); + this.#fetch().then((response) => response.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message)); + return stream.readable; + } +}; +var jwtPromises = /* @__PURE__ */ new Map(); +var jwts = /* @__PURE__ */ new Map(); +function cacheKey(params) { + return JSON.stringify([params.refreshUrl, params.initialAccessToken]); +} +function bg4_regroup_bytes(bytes) { + const split = Math.floor(bytes.byteLength / 4); + const rem = bytes.byteLength % 4; + const g1_pos = split + (rem >= 1 ? 1 : 0); + const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); + const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); + const ret = new Uint8Array(bytes.byteLength); + for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + return ret; +} +function bg4_split_bytes(bytes) { + const ret = new Uint8Array(bytes.byteLength); + const split = Math.floor(bytes.byteLength / 4); + const rem = bytes.byteLength % 4; + const g1_pos = split + (rem >= 1 ? 1 : 0); + const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); + const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); + for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + return ret; +} +async function getAccessToken(initialAccessToken, customFetch, refreshUrl) { + const key = cacheKey({ refreshUrl, initialAccessToken }); + const jwt = jwts.get(key); + if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD)) { + return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; + } + const existingPromise = jwtPromises.get(key); + if (existingPromise) { + return existingPromise; + } + const promise = (async () => { + const resp = await customFetch(refreshUrl, { + headers: { + ...initialAccessToken ? { + Authorization: `Bearer ${initialAccessToken}` + } : {} + } + }); + if (!resp.ok) { + throw new Error(`Failed to get JWT token: ${resp.status} ${await resp.text()}`); + } + const json = await resp.json(); + const jwt2 = { + accessToken: json.accessToken, + expiresAt: new Date(json.exp * 1e3), + casUrl: json.casUrl + }; + jwtPromises.delete(key); + for (const [key2, value] of jwts.entries()) { + if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD)) { + jwts.delete(key2); + } else { + break; + } + } + if (jwts.size >= JWT_CACHE_SIZE) { + const keyToDelete = jwts.keys().next().value; + if (keyToDelete) { + jwts.delete(keyToDelete); + } + } + jwts.set(key, jwt2); + return { + accessToken: json.accessToken, + casUrl: json.casUrl + }; + })(); + jwtPromises.set(key, promise); + return promise; +} + +// src/utils/ChunkCache.ts +var CHUNK_CACHE_INITIAL_SIZE = 1e4; +var CHUNK_CACHE_GROW_FACTOR = 1.5; +var CHUNK_CACHE_MAX_SIZE = 1e6; +var ChunkCache = class { + index = 0; + // Index >= 0 means local xorb, < 0 means remote xorb + xorbIndices; + // Max 8K chunks per xorb, less than 64K uint16_t + chunkIndices; + map = /* @__PURE__ */ new Map(); + // hash -> chunkCacheIndex. Less overhead that way, empty object is 60+B and empty array is 40+B + hmacs = /* @__PURE__ */ new Set(); + // todo : remove old hmacs + maxSize; + constructor(maxSize = CHUNK_CACHE_MAX_SIZE) { + if (maxSize < 1) { + throw new Error("maxSize must be at least 1"); + } + this.maxSize = maxSize; + this.xorbIndices = new Int32Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); + this.chunkIndices = new Uint16Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); + } + addChunkToCache(hash2, xorbIndex, chunkIndex, hmac2) { + if (this.map.has(hash2)) { + return; + } + if (this.map.values().next().value === this.index) { + this.map.delete(this.map.keys().next().value); + } + this.map.set(hash2, this.index); + if (hmac2 !== null) { + this.hmacs.add(hmac2); + } + if (this.index >= this.xorbIndices.length) { + const oldXorbIndices = this.xorbIndices; + const oldChunkIndices = this.chunkIndices; + this.xorbIndices = new Int32Array(Math.min(this.xorbIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); + this.chunkIndices = new Uint16Array(Math.min(this.chunkIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); + this.xorbIndices.set(oldXorbIndices); + this.chunkIndices.set(oldChunkIndices); + } + this.xorbIndices[this.index] = xorbIndex; + this.chunkIndices[this.index] = chunkIndex; + this.index = (this.index + 1) % this.maxSize; + } + getChunk(hash2, hmacFunction) { + let index = this.map.get(hash2); + if (index === void 0 && hmacFunction !== null) { + for (const hmac2 of this.hmacs) { + index = this.map.get(hmacFunction(hash2, hmac2)); + if (index !== void 0) { + break; + } + } + } + if (index === void 0) { + return void 0; + } + return { + xorbIndex: this.xorbIndices[index], + chunkIndex: this.chunkIndices[index] + }; + } + updateChunkIndex(hash2, chunkIndex) { + const index = this.map.get(hash2); + if (index === void 0) { + throw new Error(`Chunk not found in cache: ${hash2}`); + } + this.chunkIndices[index] = chunkIndex; + } + removeChunkFromCache(hash2) { + this.map.delete(hash2); + } +}; + +// src/utils/xetWriteToken.ts +var JWT_SAFETY_PERIOD2 = 6e4; +var JWT_CACHE_SIZE2 = 1e3; +var jwtPromises2 = /* @__PURE__ */ new Map(); +var jwts2 = /* @__PURE__ */ new Map(); +async function xetWriteToken(params) { + if (params.xetParams.expiresAt && params.xetParams.casUrl && params.xetParams.accessToken && params.xetParams.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD2)) { + return { accessToken: params.xetParams.accessToken, casUrl: params.xetParams.casUrl }; + } + const key = params.xetParams.refreshWriteTokenUrl; + const jwt = jwts2.get(key); + if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD2)) { + return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; + } + const existingPromise = jwtPromises2.get(key); + if (existingPromise) { + return existingPromise; + } + const promise = (async () => { + const resp = await (params.fetch ?? fetch)(params.xetParams.refreshWriteTokenUrl, { + headers: { + ...params.accessToken ? { + Authorization: `Bearer ${params.accessToken}` + } : {}, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } + const json = await resp.json(); + const jwt2 = { + accessToken: json.accessToken, + expiresAt: new Date(json.exp * 1e3), + casUrl: json.casUrl + }; + jwtPromises2.delete(key); + for (const [key2, value] of jwts2.entries()) { + if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD2)) { + jwts2.delete(key2); + } else { + break; + } + } + if (jwts2.size >= JWT_CACHE_SIZE2) { + const keyToDelete = jwts2.keys().next().value; + if (keyToDelete) { + jwts2.delete(keyToDelete); + } + } + jwts2.set(key, jwt2); + return { + accessToken: json.accessToken, + casUrl: json.casUrl + }; + })(); + jwtPromises2.set(key, promise); + return promise; +} + +// src/utils/shardParser.ts +var HASH_LENGTH = 32; +var XORB_HASH_BOOKEND = "ff".repeat(HASH_LENGTH); +function readHashFromArray(array, offset) { + let hash2 = ""; + for (let i = 0; i < HASH_LENGTH; i += 8) { + hash2 += `${array[offset + i + 7].toString(16).padStart(2, "0")}${array[offset + i + 6].toString(16).padStart(2, "0")}${array[offset + i + 5].toString(16).padStart(2, "0")}${array[offset + i + 4].toString(16).padStart(2, "0")}${array[offset + i + 3].toString(16).padStart(2, "0")}${array[offset + i + 2].toString(16).padStart(2, "0")}${array[offset + i + 1].toString(16).padStart(2, "0")}${array[offset + i].toString(16).padStart(2, "0")}`; + } + return hash2; +} +async function parseShardData(shardBlob) { + const shard = new Uint8Array(await shardBlob.arrayBuffer()); + const shardView = new DataView(shard.buffer); + const magicTag = shard.slice(0, SHARD_MAGIC_TAG.length); + if (!magicTag.every((byte, i) => byte === SHARD_MAGIC_TAG[i])) { + throw new Error("Invalid shard magic tag"); + } + const version = shardView.getBigUint64(SHARD_MAGIC_TAG.length, true); + if (version !== SHARD_HEADER_VERSION) { + throw new Error(`Invalid shard version: ${version}`); + } + const footerSize = Number(shardView.getBigUint64(SHARD_MAGIC_TAG.length + 8, true)); + const footerStart = shard.length - footerSize; + const footerVersion = shardView.getBigUint64(footerStart, true); + if (footerVersion !== SHARD_FOOTER_VERSION) { + throw new Error(`Invalid shard footer version: ${footerVersion}`); + } + const xorbInfoStart = Number(shardView.getBigUint64(footerStart + 16, true)); + const fileLookupStart = Number(shardView.getBigUint64(footerStart + 24, true)); + const hmacKey = readHashFromArray(shard, footerStart + 72); + const xorbs = []; + let offset = xorbInfoStart; + while (offset < fileLookupStart) { + const xorbHash2 = readHashFromArray(shard, offset); + offset += HASH_LENGTH; + if (xorbHash2 === XORB_HASH_BOOKEND) { + break; + } + offset += 4; + const chunkCount = shardView.getUint32(offset, true); + offset += 4; + offset += 4; + offset += 4; + const chunks = []; + for (let i = 0; i < chunkCount; i++) { + const chunkHash = readHashFromArray(shard, offset); + offset += HASH_LENGTH; + const startOffset = shardView.getUint32(offset, true); + offset += 4; + const length = shardView.getUint32(offset, true); + offset += 4; + offset += 8; + chunks.push({ + hash: chunkHash, + startOffset, + unpackedLength: length + }); + } + xorbs.push({ + hash: xorbHash2, + chunks + }); + } + return { + hmacKey, + xorbs + }; +} + +// src/utils/sum.ts +function sum(arr) { + return arr.reduce((a, b) => a + b, 0); +} + +// src/utils/SplicedBlob.ts +var SplicedBlob = class extends Blob { + originalBlob; + spliceOperations; + constructor(originalBlob, spliceOperations) { + super(); + this.originalBlob = originalBlob; + this.spliceOperations = spliceOperations; + } + static create(originalBlob, operations) { + for (const op of operations) { + if (op.start < 0 || op.end < 0) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + if (op.start > originalBlob.size || op.end > originalBlob.size) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + if (op.start > op.end) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + } + const sortedOps = [...operations].sort((a, b) => a.start - b.start); + for (let i = 0; i < sortedOps.length - 1; i++) { + if (sortedOps[i].end > sortedOps[i + 1].start) { + throw new Error("Overlapping splice operations are not supported"); + } + } + return new SplicedBlob(originalBlob, sortedOps); + } + /** + * Returns the size of the spliced blob. + * Size = original size - total replaced size + total insert size + */ + get size() { + let totalReplacedSize = 0; + let totalInsertSize = 0; + for (const op of this.spliceOperations) { + totalReplacedSize += op.end - op.start; + totalInsertSize += op.insert.size; + } + return this.originalBlob.size - totalReplacedSize + totalInsertSize; + } + /** + * Returns the MIME type of the original blob. + */ + get type() { + return this.originalBlob.type; + } + /** + * Returns a new instance of SplicedBlob that is a slice of the current one. + * + * The slice is inclusive of the start and exclusive of the end. + * The slice method does not support negative start/end. + * + * @param start beginning of the slice + * @param end end of the slice + */ + slice(start = 0, end = this.size) { + if (start < 0 || end < 0) { + throw new TypeError("Unsupported negative start/end on SplicedBlob.slice"); + } + start = Math.min(start, this.size); + end = Math.min(end, this.size); + if (start >= end) { + return new Blob([]); + } + const segments = this.segments; + const segmentBoundaries = [0]; + let cumulativeSize = 0; + for (const segment of segments) { + cumulativeSize += segment.size; + segmentBoundaries.push(cumulativeSize); + } + const resultSegments = []; + for (let i = 0; i < segments.length; i++) { + const segmentStart = segmentBoundaries[i]; + const segmentEnd = segmentBoundaries[i + 1]; + if (segmentEnd <= start) { + continue; + } + if (segmentStart >= end) { + break; + } + const sliceStart = Math.max(0, start - segmentStart); + const sliceEnd = Math.min(segments[i].size, end - segmentStart); + if (sliceStart < sliceEnd) { + resultSegments.push(segments[i].slice(sliceStart, sliceEnd)); + } + } + return new Blob(resultSegments); + } + get firstSpliceIndex() { + return this.spliceOperations[0]?.start ?? Infinity; + } + /** + * Read the spliced blob content and returns it as an ArrayBuffer. + */ + async arrayBuffer() { + const segments = this.segments; + const buffers = await Promise.all(segments.map((segment) => segment.arrayBuffer())); + const totalSize = sum(buffers.map((buffer) => buffer.byteLength)); + const result = new Uint8Array(totalSize); + let offset = 0; + for (const buffer of buffers) { + result.set(new Uint8Array(buffer), offset); + offset += buffer.byteLength; + } + return result.buffer; + } + /** + * Read the spliced blob content and returns it as a string. + */ + async text() { + const buffer = await this.arrayBuffer(); + return new TextDecoder().decode(buffer); + } + /** + * Returns a stream around the spliced blob content. + */ + stream() { + const readable = new ReadableStream({ + start: async (controller) => { + try { + const segments = this.segments; + for (const segment of segments) { + const reader = segment.stream().getReader(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + controller.enqueue(value); + } + } finally { + reader.releaseLock(); + } + } + controller.close(); + } catch (error) { + controller.error(error); + } + } + }); + return readable; + } + /** + * Get all segments that make up the spliced blob. + * This includes original blob segments between splice operations and insert blobs. + */ + get segments() { + const segments = []; + let currentPosition = 0; + const sortedOps = [...this.spliceOperations].sort((a, b) => a.start - b.start); + for (const op of sortedOps) { + if (currentPosition < op.start) { + segments.push(this.originalBlob.slice(currentPosition, op.start)); + } + if (op.insert.size > 0) { + segments.push(op.insert); + } + currentPosition = op.end; + } + if (currentPosition < this.originalBlob.size) { + segments.push(this.originalBlob.slice(currentPosition)); + } + return segments; + } +}; + +// src/utils/createXorbs.ts +var import_xetchunk_wasm = require("@huggingface/xetchunk-wasm"); +var TARGET_CHUNK_SIZE = 64 * 1024; +var MAX_CHUNK_SIZE = 2 * TARGET_CHUNK_SIZE; +var XORB_SIZE = 64 * 1024 * 1024; +var MAX_XORB_CHUNKS = 8 * 1024; +var INTERVAL_BETWEEN_REMOTE_DEDUP = 4e6; +var PROCESSING_PROGRESS_RATIO = 0.1; +var UPLOADING_PROGRESS_RATIO = 1 - PROCESSING_PROGRESS_RATIO; +function computeXorbHashHex(chunks) { + const chunkObjs = chunks.map((c) => ({ hash: (0, import_xetchunk_wasm.hexToBytes)(c.hash), length: c.length })); + return (0, import_xetchunk_wasm.hashToHex)((0, import_xetchunk_wasm.xorbHash)(chunkObjs)); +} +function computeHmacHex(hash2, key) { + return (0, import_xetchunk_wasm.hashToHex)((0, import_xetchunk_wasm.hmac)((0, import_xetchunk_wasm.hexToBytes)(hash2), (0, import_xetchunk_wasm.hexToBytes)(key))); +} +function computeVerificationHashHex(hashes) { + return (0, import_xetchunk_wasm.hashToHex)((0, import_xetchunk_wasm.verificationHash)(hashes.map(import_xetchunk_wasm.hexToBytes))); +} +function computeFileHashHex(chunks) { + const chunkObjs = chunks.map((c) => ({ hash: (0, import_xetchunk_wasm.hexToBytes)(c.hash), length: c.length })); + return (0, import_xetchunk_wasm.hashToHex)((0, import_xetchunk_wasm.fileHash)(chunkObjs)); +} +function addDataToChunker(data, chunker) { + return (0, import_xetchunk_wasm.nextBlock)(chunker, data).map((c) => ({ hash: (0, import_xetchunk_wasm.hashToHex)(c.hash), length: c.length, dedup: false })); +} +function finalizeChunker(chunker) { + const last = (0, import_xetchunk_wasm.finalize)(chunker); + if (!last) { + return []; + } + return [{ hash: (0, import_xetchunk_wasm.hashToHex)(last.hash), length: last.length, dedup: false }]; +} +var CurrentXorbInfo = class { + id; + offset; + chunks; + fileProcessedBytes; + fileUploadedBytes; + fileSize; + data; + immutableData; + constructor() { + this.id = 0; + this.offset = 0; + this.chunks = []; + this.fileProcessedBytes = {}; + this.fileUploadedBytes = {}; + this.fileSize = {}; + this.data = new Uint8Array(XORB_SIZE); + this.immutableData = null; + } + event(computeXorbHash) { + const xorbChunksCleaned = this.chunks.map((chunk2) => ({ + hash: chunk2.hash, + length: chunk2.length + })); + return { + event: "xorb", + xorb: this.data.subarray(0, this.offset), + hash: computeXorbHash(xorbChunksCleaned), + chunks: xorbChunksCleaned, + id: this.id, + files: Object.entries(this.fileProcessedBytes).map(([path2, processedBytes]) => ({ + path: path2, + progress: processedBytes / this.fileSize[path2], + lastSentProgress: ((this.fileUploadedBytes[path2] ?? 0) + (processedBytes - (this.fileUploadedBytes[path2] ?? 0)) * PROCESSING_PROGRESS_RATIO) / this.fileSize[path2] + })) + }; + } +}; +async function* createXorbs(fileSources, params) { + const alreadyDoneFileSha256s = /* @__PURE__ */ new Set(); + let xorbId = 0; + const chunkCache = new ChunkCache(); + let xorb = new CurrentXorbInfo(); + const nextXorb = (currentFile) => { + const event = xorb.event(computeXorbHashHex); + xorbId++; + xorb = new CurrentXorbInfo(); + xorb.id = xorbId; + xorb.fileUploadedBytes = { + [currentFile.path]: currentFile.uploadedBytes + }; + xorb.fileSize[currentFile.path] = currentFile.size; + return event; + }; + const pendingFileEvents = []; + const remoteXorbHashes = [""]; + for await (const fileSource of fileSources) { + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: 0 + }); + if (fileSource.sha256 && alreadyDoneFileSha256s.has(fileSource.sha256)) { + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: 1 + }); + continue; + } + if (fileSource.sha256) { + alreadyDoneFileSha256s.add(fileSource.sha256); + } + const chunker = (0, import_xetchunk_wasm.createChunker)(TARGET_CHUNK_SIZE); + { + xorb.fileSize[fileSource.path] = fileSource.content.size; + if (fileSource.content instanceof SplicedBlob && fileSource.content.firstSpliceIndex < MAX_CHUNK_SIZE) { + await loadDedupInfoToCache( + fileSource.content.originalBlob.slice(0, MAX_CHUNK_SIZE), + remoteXorbHashes, + params, + chunkCache, + computeHmacHex, + { + maxChunks: 1, + isAtBeginning: true + } + ); + } + let bytesSinceRemoteDedup = Infinity; + let bytesSinceLastProgressEvent = 0; + let isFirstFileChunk = true; + const sourceChunks = []; + const reader = fileSource.content.stream().getReader(); + let processedBytes = 0; + let dedupedBytes = 0; + const fileChunks = []; + const chunkMetadata = []; + const addChunks = async function* (chunks) { + for (const chunk2 of chunks) { + if (isFirstFileChunk) { + chunk2.dedup = true; + isFirstFileChunk = false; + } + let chunkIndex = xorb.chunks.length; + let chunkXorbId = xorbId; + const chunkToCopy = removeChunkFromSourceData(sourceChunks, chunk2.length); + let cacheData = chunkCache.getChunk(chunk2.hash, computeHmacHex); + if (cacheData === void 0 && chunk2.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { + const token = await xetWriteToken(params); + bytesSinceRemoteDedup = 0; + const shardResp = await (params.fetch ?? fetch)(token.casUrl + "/v1/chunks/default/" + chunk2.hash, { + headers: { + Authorization: `Bearer ${token.accessToken}` + } + }); + if (shardResp.ok) { + const shard = await shardResp.blob(); + const shardData = await parseShardData(shard); + for (const xorb2 of shardData.xorbs) { + const remoteXorbId = -remoteXorbHashes.length; + remoteXorbHashes.push(xorb2.hash); + let i = 0; + for (const chunk3 of xorb2.chunks) { + chunkCache.addChunkToCache(chunk3.hash, remoteXorbId, i++, shardData.hmacKey); + } + } + cacheData = chunkCache.getChunk(chunk2.hash, computeHmacHex); + const oldDedupedBytes = dedupedBytes; + dedupedBytes = backtrackDedup(xorb, computeHmacHex, shardData, chunkCache, chunkMetadata, dedupedBytes); + if (dedupedBytes > oldDedupedBytes) { + xorb.fileUploadedBytes[fileSource.path] ??= 0; + xorb.fileUploadedBytes[fileSource.path] += dedupedBytes - oldDedupedBytes; + } + } + } + if (cacheData === void 0) { + if (!writeChunk(xorb, chunkToCopy, chunk2.hash)) { + yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); + chunkIndex = 0; + chunkXorbId = xorbId; + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } + pendingFileEvents.length = 0; + if (!writeChunk(xorb, chunkToCopy, chunk2.hash)) { + throw new Error("Failed to write chunk into xorb"); + } + } + chunkCache.addChunkToCache(chunk2.hash, xorbId, chunkIndex, null); + } else { + chunkXorbId = cacheData.xorbIndex; + chunkIndex = cacheData.chunkIndex; + dedupedBytes += chunk2.length; + xorb.fileUploadedBytes[fileSource.path] ??= 0; + xorb.fileUploadedBytes[fileSource.path] += chunk2.length; + } + bytesSinceRemoteDedup += chunk2.length; + bytesSinceLastProgressEvent += chunk2.length; + fileChunks.push({ hash: chunk2.hash, length: chunk2.length }); + chunkMetadata.push({ + xorbId: chunkXorbId, + chunkIndex, + length: chunk2.length + }); + xorb.fileProcessedBytes[fileSource.path] = processedBytes; + if (bytesSinceLastProgressEvent >= 1e6) { + bytesSinceLastProgressEvent = 0; + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: ((xorb.fileUploadedBytes[fileSource.path] ?? 0) + (xorb.fileProcessedBytes[fileSource.path] - (xorb.fileUploadedBytes[fileSource.path] ?? 0)) * PROCESSING_PROGRESS_RATIO) / fileSource.content.size + }); + } + if (xorb.chunks.length >= MAX_XORB_CHUNKS) { + yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } + pendingFileEvents.length = 0; + } + } + }; + while (true) { + const { done, value } = await reader.read(); + if (done) { + yield* addChunks(finalizeChunker(chunker)); + break; + } + processedBytes += value.length; + sourceChunks.push(value); + yield* addChunks(addDataToChunker(value, chunker)); + } + const fileRepresentation = buildFileRepresentation(chunkMetadata, fileChunks, computeVerificationHashHex); + xorb.immutableData = { + chunkIndex: xorb.chunks.length, + offset: xorb.offset + }; + const dedupRatio = fileSource.content.size > 0 ? dedupedBytes / fileSource.content.size : 0; + pendingFileEvents.push({ + event: "file", + path: fileSource.path, + hash: computeFileHashHex(fileChunks), + sha256: fileSource.sha256, + dedupRatio, + representation: fileRepresentation + }); + } + } + if (xorb.offset > 0) { + yield xorb.event(computeXorbHashHex); + } + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] + })); + yield event; + } +} +function backtrackDedup(xorb, computeHmac, shardData, chunkCache, chunkMetadata, dedupedBytes) { + const chunkIndexesToBacktrackFor = /* @__PURE__ */ new Map(); + for (let chunkToRecheckIndex = xorb.immutableData?.chunkIndex ?? 0; chunkToRecheckIndex < xorb.chunks.length; chunkToRecheckIndex++) { + const chunk2 = xorb.chunks[chunkToRecheckIndex]; + const hmacHash = computeHmac(chunk2.hash, shardData.hmacKey); + const cacheData = chunkCache.getChunk(hmacHash, null); + if (cacheData !== void 0) { + chunkIndexesToBacktrackFor.set(chunkToRecheckIndex, { + xorbId: cacheData.xorbIndex, + chunkIndex: cacheData.chunkIndex + }); + chunkCache.removeChunkFromCache(chunk2.hash); + } + } + for (const metadata of chunkMetadata) { + if (metadata.xorbId === xorb.id && chunkIndexesToBacktrackFor.has(metadata.chunkIndex)) { + const backtrackData = chunkIndexesToBacktrackFor.get(metadata.chunkIndex); + if (backtrackData !== void 0) { + metadata.xorbId = backtrackData.xorbId; + metadata.chunkIndex = backtrackData.chunkIndex; + dedupedBytes += metadata.length; + } + } + } + const xorbRangesToErase = []; + for (let i = 0; i < xorb.chunks.length; i++) { + const chunk2 = xorb.chunks[i]; + if (chunkIndexesToBacktrackFor.has(i)) { + xorbRangesToErase.push({ + start: chunk2.offset, + end: i < xorb.chunks.length - 1 ? xorb.chunks[i + 1].offset : xorb.offset + }); + } + } + const xorbRangesToKeep = []; + let currentStart = 0; + for (let i = 0; i < xorbRangesToErase.length; i++) { + const range2 = xorbRangesToErase[i]; + if (currentStart !== range2.start) { + xorbRangesToKeep.push({ start: currentStart, end: range2.start }); + } + currentStart = range2.end; + } + if (currentStart !== xorb.offset) { + xorbRangesToKeep.push({ start: currentStart, end: xorb.offset }); + } + let currentOffset = 0; + for (const range2 of xorbRangesToKeep) { + if (range2.start !== currentOffset) { + xorb.data.set(xorb.data.subarray(range2.start, range2.end), currentOffset); + } + currentOffset += range2.end - range2.start; + } + const newXorbChunks = []; + const oldIndexToNewIndex = /* @__PURE__ */ new Map(); + let erasedOffset = 0; + for (let i = 0; i < xorb.chunks.length; i++) { + const chunk2 = xorb.chunks[i]; + if (chunkIndexesToBacktrackFor.has(i)) { + if (i < xorb.chunks.length - 1) { + erasedOffset += xorb.chunks[i + 1].offset - chunk2.offset; + } + } else { + newXorbChunks.push({ + hash: chunk2.hash, + length: chunk2.length, + offset: chunk2.offset - erasedOffset + }); + if (erasedOffset > 0) { + oldIndexToNewIndex.set(i, newXorbChunks.length - 1); + } + } + } + xorb.chunks = newXorbChunks; + xorb.offset = currentOffset; + for (const chunk2 of chunkMetadata) { + if (chunk2.xorbId === xorb.id) { + const newIndex = oldIndexToNewIndex.get(chunk2.chunkIndex); + if (newIndex !== void 0) { + const cached = chunkCache.getChunk(xorb.chunks[newIndex].hash, null); + if (cached !== void 0 && cached.xorbIndex === chunk2.xorbId && cached.chunkIndex === chunk2.chunkIndex) { + chunkCache.updateChunkIndex(xorb.chunks[newIndex].hash, newIndex); + } + chunk2.chunkIndex = newIndex; + } + } + } + return dedupedBytes; +} +function removeChunkFromSourceData(sourceChunks, chunkLength) { + if (chunkLength === sourceChunks[0].length) { + const chunkToCopy = sourceChunks[0]; + sourceChunks.shift(); + return chunkToCopy; + } else if (chunkLength < sourceChunks[0].length) { + const chunkToCopy = sourceChunks[0].subarray(0, chunkLength); + sourceChunks[0] = sourceChunks[0].subarray(chunkLength); + return chunkToCopy; + } else { + const chunkToCopy = new Uint8Array(chunkLength); + let copyOffset = 0; + let index = 0; + let toSlice = -1; + while (copyOffset < chunkLength) { + const nToCopy = Math.min(sourceChunks[index].length, chunkLength - copyOffset); + chunkToCopy.set(sourceChunks[index].subarray(0, nToCopy), copyOffset); + copyOffset += nToCopy; + if (nToCopy === sourceChunks[index].length) { + index++; + } else { + toSlice = nToCopy; + } + } + sourceChunks.splice(0, index); + if (toSlice !== -1) { + sourceChunks[0] = sourceChunks[0].subarray(toSlice); + } + return chunkToCopy; + } +} +function writeChunk(xorb, chunk2, hash2) { + const regularCompressedChunk = compress(chunk2); + const bgCompressedChunk = compress(bg4_split_bytes(chunk2)); + const compressedChunk = bgCompressedChunk.length < regularCompressedChunk.length ? bgCompressedChunk : regularCompressedChunk; + const chunkToWrite = compressedChunk.length < chunk2.length ? compressedChunk : chunk2; + if (xorb.offset + XET_CHUNK_HEADER_BYTES + chunkToWrite.length > XORB_SIZE) { + return false; + } + xorb.data[xorb.offset] = 0; + xorb.data[xorb.offset + 1] = chunkToWrite.length & 255; + xorb.data[xorb.offset + 2] = chunkToWrite.length >> 8 & 255; + xorb.data[xorb.offset + 3] = chunkToWrite.length >> 16 & 255; + xorb.data[xorb.offset + 4] = chunkToWrite.length < chunk2.length ? bgCompressedChunk.length < regularCompressedChunk.length ? 2 /* ByteGroupingLZ4 */ : 1 /* LZ4 */ : 0 /* None */; + xorb.data[xorb.offset + 5] = chunk2.length & 255; + xorb.data[xorb.offset + 6] = chunk2.length >> 8 & 255; + xorb.data[xorb.offset + 7] = chunk2.length >> 16 & 255; + xorb.data.set(chunkToWrite, xorb.offset + XET_CHUNK_HEADER_BYTES); + xorb.chunks.push({ hash: hash2, length: chunk2.length, offset: xorb.offset }); + xorb.offset += XET_CHUNK_HEADER_BYTES + chunkToWrite.length; + return true; +} +var buildFileRepresentation = (metadata, chunks, computeVerificationHash) => { + if (metadata.length === 0) { + return []; + } + const representation = []; + let currentRange = { + xorbId: metadata[0].xorbId, + indexStart: metadata[0].chunkIndex, + indexEnd: metadata[0].chunkIndex + 1, + length: metadata[0].length, + chunkHashStart: 0 + }; + for (let i = 1; i < metadata.length; i++) { + const chunk2 = metadata[i]; + if (currentRange.xorbId === chunk2.xorbId && currentRange.indexEnd === chunk2.chunkIndex) { + currentRange.indexEnd = chunk2.chunkIndex + 1; + currentRange.length += chunk2.length; + } else { + const rangeHash2 = computeVerificationHash(chunks.slice(currentRange.chunkHashStart, i).map((x) => x.hash)); + representation.push({ + xorbId: currentRange.xorbId, + indexStart: currentRange.indexStart, + indexEnd: currentRange.indexEnd, + length: currentRange.length, + rangeHash: rangeHash2 + }); + currentRange = { + xorbId: chunk2.xorbId, + indexStart: chunk2.chunkIndex, + indexEnd: chunk2.chunkIndex + 1, + length: chunk2.length, + chunkHashStart: i + }; + } + } + const rangeHash = computeVerificationHash(chunks.slice(currentRange.chunkHashStart).map((x) => x.hash)); + representation.push({ + xorbId: currentRange.xorbId, + indexStart: currentRange.indexStart, + indexEnd: currentRange.indexEnd, + length: currentRange.length, + rangeHash + }); + return representation; +}; +async function loadDedupInfoToCache(content, remoteXorbHashes, params, chunkCache, computeHmacHex2, opts) { + const chunker = (0, import_xetchunk_wasm.createChunker)(TARGET_CHUNK_SIZE); + const cache = chunkCache; + let dedupedBytes = 0; + let chunksProcessed = 0; + let totalBytes = 0; + let bytesSinceRemoteDedup = Infinity; + const sourceChunks = []; + const reader = content.stream().getReader(); + const processChunks = async (chunks) => { + for (const chunk2 of chunks) { + chunksProcessed++; + if (opts?.isAtBeginning && chunksProcessed === 1) { + chunk2.dedup = true; + } + totalBytes += chunk2.length; + removeChunkFromSourceData(sourceChunks, chunk2.length); + let cacheData = cache.getChunk(chunk2.hash, computeHmacHex2); + if (cacheData !== void 0) { + dedupedBytes += chunk2.length; + bytesSinceRemoteDedup += chunk2.length; + continue; + } + if (chunk2.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { + const token = await xetWriteToken(params); + bytesSinceRemoteDedup = 0; + const shardResp = await (params.fetch ?? fetch)(token.casUrl + "/v1/chunks/default/" + chunk2.hash, { + headers: { + Authorization: `Bearer ${token.accessToken}` + } + }); + if (shardResp.ok) { + const shard = await shardResp.blob(); + const shardData = await parseShardData(shard); + for (const xorb of shardData.xorbs) { + const remoteXorbId = -remoteXorbHashes.length; + remoteXorbHashes.push(xorb.hash); + let i = 0; + for (const xorbChunk of xorb.chunks) { + cache.addChunkToCache(xorbChunk.hash, remoteXorbId, i++, shardData.hmacKey); + } + } + cacheData = cache.getChunk(chunk2.hash, computeHmacHex2); + } + } + if (cacheData !== void 0) { + dedupedBytes += chunk2.length; + } + bytesSinceRemoteDedup += chunk2.length; + } + }; + while (true) { + if (opts?.end !== void 0 && totalBytes >= opts.end) { + break; + } + if (opts?.maxChunks !== void 0 && chunksProcessed >= opts.maxChunks) { + break; + } + const { done, value } = await reader.read(); + if (done) { + await processChunks(finalizeChunker(chunker)); + break; + } + sourceChunks.push(value); + await processChunks(addDataToChunker(value, chunker)); + } +} + +// src/utils/uploadShards.ts +var SHARD_MAX_SIZE = 64 * 1024 * 1024; +var SHARD_HEADER_SIZE = 48; +var SHARD_FOOTER_SIZE = 200; +var HASH_LENGTH2 = 32; +var XORB_FOOTER_LENGTH = 48; +var FILE_FOOTER_LENGTH = 48; +var SHARD_HEADER_VERSION = 2n; +var SHARD_FOOTER_VERSION = 1n; +var MDB_FILE_FLAG_WITH_VERIFICATION = 2147483648; +var MDB_FILE_FLAG_WITH_METADATA_EXT = 1073741824; +var SHARD_MAGIC_TAG = new Uint8Array([ + "H".charCodeAt(0), + "F".charCodeAt(0), + "R".charCodeAt(0), + "e".charCodeAt(0), + "p".charCodeAt(0), + "o".charCodeAt(0), + "M".charCodeAt(0), + "e".charCodeAt(0), + "t".charCodeAt(0), + "a".charCodeAt(0), + "D".charCodeAt(0), + "a".charCodeAt(0), + "t".charCodeAt(0), + "a".charCodeAt(0), + 0, + 85, + 105, + 103, + 69, + 106, + 123, + 129, + 87, + 131, + 165, + 189, + 217, + 92, + 205, + 209, + 74, + 169 +]); +async function* uploadShards(source, params) { + const xorbHashes = []; + const seenFileXetHashes = /* @__PURE__ */ new Set(); + const fileInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.25); + const xorbInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.75); + const xorbView = new DataView(xorbInfoSection.buffer); + let xorbViewOffset = 0; + const fileInfoView = new DataView(fileInfoSection.buffer); + let fileViewOffset = 0; + let xorbTotalSize = 0n; + let fileTotalSize = 0n; + let xorbTotalUnpackedSize = 0n; + for await (const output of createXorbs(source, params)) { + switch (output.event) { + case "xorb": { + xorbHashes.push(output.hash); + const xorbEntrySize = HASH_LENGTH2 + 4 + 4 + 4 + 4; + const chunksSize = output.chunks.length * (HASH_LENGTH2 + 4 + 4 + 8); + const totalXorbSize = xorbEntrySize + chunksSize; + if (xorbViewOffset + totalXorbSize > xorbInfoSection.length) { + if (xorbViewOffset > 0 || fileViewOffset > 0) { + await uploadShard(createShard(), params); + } + } + writeHashToArray(output.hash, xorbInfoSection, xorbViewOffset); + xorbViewOffset += HASH_LENGTH2; + xorbView.setUint32(xorbViewOffset, 0, true); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, output.chunks.length, true); + xorbViewOffset += 4; + const xorbUnpackedSize = sum(output.chunks.map((x) => x.length)); + xorbView.setUint32(xorbViewOffset, xorbUnpackedSize, true); + xorbTotalUnpackedSize += BigInt(xorbUnpackedSize); + xorbTotalSize += BigInt(output.xorb.byteLength); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, output.xorb.byteLength, true); + xorbViewOffset += 4; + let chunkBytes = 0; + for (const chunk2 of output.chunks) { + writeHashToArray(chunk2.hash, xorbInfoSection, xorbViewOffset); + xorbViewOffset += HASH_LENGTH2; + xorbView.setUint32(xorbViewOffset, chunkBytes, true); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, chunk2.length, true); + xorbViewOffset += 4; + xorbView.setBigUint64(xorbViewOffset, 0n, true); + xorbViewOffset += 8; + chunkBytes += chunk2.length; + } + for (const file of output.files) { + yield { + event: "fileProgress", + path: file.path, + progress: file.lastSentProgress + }; + } + await uploadXorb(output, params); + for (const file of output.files) { + yield { event: "fileProgress", path: file.path, progress: file.progress }; + } + break; + } + case "file": { + yield { + event: "file", + path: output.path, + xetHash: output.hash, + sha256: output.sha256, + dedupRatio: output.dedupRatio + }; + if (seenFileXetHashes.has(output.hash)) { + break; + } + seenFileXetHashes.add(output.hash); + const fileHeaderSize = HASH_LENGTH2 + 4 + 4 + 8; + const representationSize = output.representation.length * (HASH_LENGTH2 + 4 + 4 + 4 + 4); + const verificationSize = output.representation.length * (HASH_LENGTH2 + 16); + const fileSha256 = output.sha256; + const hasMetadataExt = fileSha256 !== void 0; + const metadataSize = hasMetadataExt ? HASH_LENGTH2 + 16 : 0; + const totalFileSize = fileHeaderSize + representationSize + verificationSize + metadataSize; + if (fileViewOffset + totalFileSize > fileInfoSection.length) { + if (xorbViewOffset > 0 || fileViewOffset > 0) { + await uploadShard(createShard(), params); + } + } + writeHashToArray(output.hash, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + fileInfoView.setUint32( + fileViewOffset, + MDB_FILE_FLAG_WITH_VERIFICATION + (hasMetadataExt ? MDB_FILE_FLAG_WITH_METADATA_EXT : 0), + true + ); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, output.representation.length, true); + fileViewOffset += 4; + fileInfoView.setBigUint64(fileViewOffset, 0n, true); + fileViewOffset += 8; + for (const repItem of output.representation) { + writeHashToArray( + typeof repItem.xorbId === "number" ? xorbHashes[repItem.xorbId] : repItem.xorbId, + fileInfoSection, + fileViewOffset + ); + fileViewOffset += HASH_LENGTH2; + fileInfoView.setUint32(fileViewOffset, 0, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.length, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.indexStart, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.indexEnd, true); + fileViewOffset += 4; + } + for (const repItem of output.representation) { + writeHashToArray(repItem.rangeHash, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + for (let i = 0; i < 16; i++) { + fileInfoSection[fileViewOffset + i] = 0; + } + fileViewOffset += 16; + } + if (hasMetadataExt) { + writeHashToArray(fileSha256, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH2; + for (let i = 0; i < 16; i++) { + fileInfoSection[fileViewOffset + i] = 0; + } + fileViewOffset += 16; + } + break; + } + } + } + function createShard() { + const shard = new Uint8Array( + SHARD_HEADER_SIZE + SHARD_FOOTER_SIZE + xorbViewOffset + XORB_FOOTER_LENGTH + fileViewOffset + FILE_FOOTER_LENGTH + ); + const shardView = new DataView(shard.buffer); + let shardOffset = 0; + shard.set(SHARD_MAGIC_TAG, shardOffset); + shardOffset += SHARD_MAGIC_TAG.length; + shardView.setBigUint64(shardOffset, SHARD_HEADER_VERSION, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(SHARD_FOOTER_SIZE), true); + shardOffset += 8; + shard.set(fileInfoSection.slice(0, fileViewOffset), shardOffset); + shardOffset += fileViewOffset; + for (let i = 0; i < 32; i++) { + shard[shardOffset + i] = 255; + } + shardOffset += 32; + for (let i = 0; i < 16; i++) { + shard[shardOffset + i] = 0; + } + shardOffset += 16; + const xorbInfoOffset = shardOffset; + shard.set(xorbInfoSection.slice(0, xorbViewOffset), shardOffset); + shardOffset += xorbViewOffset; + for (let i = 0; i < 32; i++) { + shard[shardOffset + i] = 255; + } + shardOffset += 32; + for (let i = 0; i < 16; i++) { + shard[shardOffset + i] = 0; + } + shardOffset += 16; + const footerOffset = shardOffset; + shardView.setBigUint64(shardOffset, SHARD_FOOTER_VERSION, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(SHARD_HEADER_SIZE), true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(xorbInfoOffset), true); + shardOffset += 8; + for (let i = 0; i < 48; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 48; + for (let i = 0; i < 32; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 32; + shardView.setBigUint64(shardOffset, BigInt(Math.floor(Date.now() / 1e3)), true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, 0n, true); + shardOffset += 8; + for (let i = 0; i < 48; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 48; + shardView.setBigUint64(shardOffset, xorbTotalSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, fileTotalSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, xorbTotalUnpackedSize, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(footerOffset), true); + xorbViewOffset = 0; + fileViewOffset = 0; + xorbTotalSize = 0n; + xorbTotalUnpackedSize = 0n; + fileTotalSize = 0n; + return shard; + } + if (xorbViewOffset || fileViewOffset) { + await uploadShard(createShard(), params); + } +} +function writeHashToArray(hash2, array, offset) { + for (let i = 0; i < hash2.length; i += 16) { + array[offset + i / 2] = parseInt(hash2.substring(i + 2 * 7, i + 2 * 8), 16); + array[offset + i / 2 + 1] = parseInt(hash2.substring(i + 2 * 6, i + 2 * 7), 16); + array[offset + i / 2 + 2] = parseInt(hash2.substring(i + 2 * 5, i + 2 * 6), 16); + array[offset + i / 2 + 3] = parseInt(hash2.substring(i + 2 * 4, i + 2 * 5), 16); + array[offset + i / 2 + 4] = parseInt(hash2.substring(i + 2 * 3, i + 2 * 4), 16); + array[offset + i / 2 + 5] = parseInt(hash2.substring(i + 2 * 2, i + 2 * 3), 16); + array[offset + i / 2 + 6] = parseInt(hash2.substring(i + 2 * 1, i + 2 * 2), 16); + array[offset + i / 2 + 7] = parseInt(hash2.substring(i + 2 * 0, i + 2 * 1), 16); + } +} +async function uploadXorb(xorb, params) { + const token = await xetWriteToken(params); + const resp = await (params.fetch ?? fetch)(`${token.casUrl}/v1/xorbs/default/${xorb.hash}`, { + method: "POST", + body: xorb.xorb, + headers: { + Authorization: `Bearer ${token.accessToken}`, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + }, + ...{ + progressHint: { + progressCallback: (progress) => { + for (const file of xorb.files) { + params.yieldCallback?.({ + event: "fileProgress", + path: file.path, + progress: file.lastSentProgress + (file.progress - file.lastSentProgress) * progress + }); + } + } + } + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } +} +async function uploadShard(shard, params) { + const token = await xetWriteToken(params); + const resp = await (params.fetch ?? fetch)(`${token.casUrl}/v1/shards`, { + method: "POST", + body: shard, + headers: { + Authorization: `Bearer ${token.accessToken}`, + ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} + } + }); + if (!resp.ok) { + throw await createApiError(resp); + } +} + +// src/utils/splitAsyncGenerator.ts +function splitAsyncGenerator(source, n) { + if (n <= 0) { + return []; + } + const sleep = (ms) => new Promise((resolve3) => setTimeout(resolve3, ms)); + let takenIndex = null; + const generators = []; + let remaining = n; + for (let i = 0; i < n; i++) { + generators.push({ + next: async () => { + while (takenIndex !== null) { + await sleep(1); + } + takenIndex = i; + return source.next().then((r) => { + takenIndex = null; + return r; + }); + }, + return: async () => { + remaining--; + if (remaining === 0) { + return source.return(void 0); + } + return { + done: true, + value: void 0 + }; + }, + throw: async (error) => { + return source.throw(error); + }, + [Symbol.asyncIterator]: () => generators[i] + }); + } + return generators; +} + +// src/lib/commit.ts +var CONCURRENT_SHAS = 5; +var CONCURRENT_LFS_UPLOADS = 5; +var MULTIPART_PARALLEL_UPLOAD = 5; +function isFileOperation(op) { + const ret = op.operation === "addOrUpdate"; + if (ret && !(op.content instanceof Blob)) { + throw new TypeError("Precondition failed: op.content should be a Blob"); + } + return ret; +} +async function* commitIter(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + if (repoId.type === "bucket") { + return yield* commitIterBucket(params); + } + if (params.operations.some((op) => op.operation === "copy")) { + throw new Error("'copy' operations are only supported when the destination repo is a bucket"); + } + yield { event: "phase", phase: "preuploading" }; + let useXet = params.useXet ?? true; + const lfsShas = /* @__PURE__ */ new Map(); + const abortController = new AbortController(); + const abortSignal = abortController.signal; + if (!abortSignal.throwIfAborted) { + abortSignal.throwIfAborted = () => { + if (abortSignal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + }; + } + if (params.abortSignal) { + params.abortSignal.addEventListener("abort", () => abortController.abort()); + } + try { + const allOperations = (await Promise.all( + params.operations.map(async (operation) => { + if (operation.operation === "edit") { + const splicedBlob = SplicedBlob.create( + operation.originalContent, + operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })) + ); + return { + operation: "addOrUpdate", + path: operation.path, + content: splicedBlob + }; + } + if (operation.operation !== "addOrUpdate") { + return operation; + } + if (!(operation.content instanceof URL)) { + return { ...operation, content: operation.content }; + } + const lazyBlobs = await createBlobs(operation.content, operation.path, { + fetch: params.fetch, + maxFolderDepth: params.maxFolderDepth + }); + abortSignal?.throwIfAborted(); + return lazyBlobs.map((blob) => ({ + ...operation, + content: blob.blob, + path: blob.path + })); + }) + )).flat(1); + const gitAttributes = allOperations.filter(isFileOperation).find((op) => op.path === ".gitattributes")?.content; + for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { + const payload = { + gitAttributes: gitAttributes && await gitAttributes.text(), + files: await Promise.all( + operations.map(async (operation) => ({ + path: operation.path, + size: operation.content.size, + sample: base64FromBytes(new Uint8Array(await operation.content.slice(0, 512).arrayBuffer())) + })) + ) + }; + abortSignal?.throwIfAborted(); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/preupload/${encodeURIComponent( + params.branch ?? "main" + )}` + (params.isPullRequest ? "?create_pr=1" : ""), + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/json" + }, + body: JSON.stringify(payload), + signal: abortSignal + } + ); + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + for (const file of json.files) { + if (file.uploadMode === "lfs") { + lfsShas.set(file.path, null); + } + } + } + yield { event: "phase", phase: "uploadingLargeFiles" }; + for (const operations of chunk( + allOperations.filter(isFileOperation).filter((op) => lfsShas.has(op.path)), + 100 + )) { + const shas = yield* eventToGenerator((yieldCallback, returnCallback, rejectCallack) => { + return promisesQueue( + operations.map((op) => async () => { + const iterator = sha256(op.content, { useWebWorker: params.useWebWorkers, abortSignal }); + let res2; + do { + res2 = await iterator.next(); + if (!res2.done) { + yieldCallback({ event: "fileProgress", path: op.path, progress: res2.value, state: "hashing" }); + } + } while (!res2.done); + const sha = res2.value; + lfsShas.set(op.path, res2.value); + return sha; + }), + CONCURRENT_SHAS + ).then(returnCallback, rejectCallack); + }); + abortSignal?.throwIfAborted(); + const payload = { + operation: "upload", + // multipart is a custom protocol for HF + transfers: ["basic", "multipart", ...useXet ? ["xet"] : []], + hash_algo: "sha_256", + ...!params.isPullRequest && { + ref: { + name: params.branch ?? "main" + } + }, + objects: operations.map((op, i) => ({ + oid: shas[i], + size: op.content.size + })) + }; + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/${repoId.type === "model" ? "" : repoId.type + "s/"}${repoId.name}.git/info/lfs/objects/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + Accept: "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json" + }, + body: JSON.stringify(payload), + signal: abortSignal + } + ); + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + const batchRequestId = res.headers.get("X-Request-Id") || void 0; + const shaToOperation = new Map(operations.map((op, i) => [shas[i], op])); + if (useXet && json.transfer !== "xet") { + useXet = false; + } + let xetParams = null; + if (useXet) { + for (const obj of json.objects) { + const op = shaToOperation.get(obj.oid); + if (!op) { + throw new InvalidApiResponseFormatError("Unrequested object ID in response"); + } + if (obj.error) { + const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${obj.error.message}${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; + throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); + } + if (!obj.actions?.upload) { + yield { + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }; + } else { + const headers = new Headers(obj.actions.upload.header); + xetParams = { + sessionId: headers.get("X-Xet-Session-Id") ?? void 0, + casUrl: headers.get("X-Xet-Cas-Url") ?? void 0, + accessToken: headers.get("X-Xet-Access-Token") ?? void 0, + expiresAt: headers.get("X-Xet-Token-Expiration") ? new Date(parseInt(headers.get("X-Xet-Token-Expiration") ?? "0") * 1e3) : void 0, + refreshWriteTokenUrl: obj.actions.upload.href + }; + } + } + const source = async function* () { + for (const obj of json.objects) { + const op = shaToOperation.get(obj.oid); + if (!op || !obj.actions?.upload) { + continue; + } + abortSignal?.throwIfAborted(); + yield { content: op.content, path: op.path, sha256: obj.oid }; + } + }(); + if (xetParams) { + const fixedXetParams = xetParams; + const sources = splitAsyncGenerator(source, 5); + yield* eventToGenerator( + (yieldCallback, returnCallback, rejectCallback) => Promise.all( + sources.map(async function(source2) { + for await (const event of uploadShards(source2, { + fetch: params.fetch, + accessToken, + hubUrl: params.hubUrl ?? HUB_URL, + repo: repoId, + xetParams: fixedXetParams, + // todo: maybe leave empty if PR? + rev: params.branch ?? "main", + isPullRequest: params.isPullRequest, + yieldCallback: (event2) => yieldCallback({ ...event2, state: "uploading" }) + })) { + if (event.event === "file") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: 1, + state: "uploading" + }); + } else if (event.event === "fileProgress") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: event.progress, + state: "uploading" + }); + } + } + }) + ).then(() => returnCallback(void 0), rejectCallback) + ); + } else { + } + } else { + yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + return promisesQueueStreaming( + json.objects.map((obj) => async () => { + const op = shaToOperation.get(obj.oid); + if (!op) { + throw new InvalidApiResponseFormatError("Unrequested object ID in response"); + } + abortSignal?.throwIfAborted(); + if (obj.error) { + const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${obj.error.message}${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; + throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); + } + if (!obj.actions?.upload) { + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + return; + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 0, + state: "uploading" + }); + const content = op.content; + const header = obj.actions.upload.header; + if (header?.chunk_size) { + const chunkSize = parseInt(header.chunk_size); + const completionUrl = obj.actions.upload.href; + const parts = Object.keys(header).filter((key) => /^[0-9]+$/.test(key)); + if (parts.length !== Math.ceil(content.size / chunkSize)) { + throw new Error("Invalid server response to upload large LFS file, wrong number of parts"); + } + const completeReq = { + oid: obj.oid, + parts: parts.map((part) => ({ + partNumber: +part, + etag: "" + })) + }; + const progressCallback = (progress) => yieldCallback({ event: "fileProgress", path: op.path, progress, state: "uploading" }); + await promisesQueueStreaming( + parts.map((part) => async () => { + abortSignal?.throwIfAborted(); + const index = parseInt(part) - 1; + const slice = content.slice(index * chunkSize, (index + 1) * chunkSize); + const res3 = await (params.fetch ?? fetch)(header[part], { + method: "PUT", + /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ + body: slice instanceof WebBlob && isFrontend ? await slice.arrayBuffer() : slice, + signal: abortSignal, + ...{ + progressHint: { + path: op.path, + part: index, + numParts: parts.length, + progressCallback + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + }); + if (!res3.ok) { + throw await createApiError(res3, { + requestId: batchRequestId, + message: `Error while uploading part ${part} of ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + const eTag = res3.headers.get("ETag"); + if (!eTag) { + throw new Error("Cannot get ETag of part during multipart upload"); + } + completeReq.parts[Number(part) - 1].etag = eTag; + }), + MULTIPART_PARALLEL_UPLOAD + ); + abortSignal?.throwIfAborted(); + const res2 = await (params.fetch ?? fetch)(completionUrl, { + method: "POST", + body: JSON.stringify(completeReq), + headers: { + Accept: "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json" + }, + signal: abortSignal + }); + if (!res2.ok) { + throw await createApiError(res2, { + requestId: batchRequestId, + message: `Error completing multipart upload of ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + } else { + const res2 = await (params.fetch ?? fetch)(obj.actions.upload.href, { + method: "PUT", + headers: { + ...batchRequestId ? { "X-Request-Id": batchRequestId } : void 0 + }, + /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ + body: content instanceof WebBlob && isFrontend ? await content.arrayBuffer() : content, + signal: abortSignal, + ...{ + progressHint: { + path: op.path, + progressCallback: (progress) => yieldCallback({ + event: "fileProgress", + path: op.path, + progress, + state: "uploading" + }) + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + }); + if (!res2.ok) { + throw await createApiError(res2, { + requestId: batchRequestId, + message: `Error while uploading ${operations[shas.indexOf(obj.oid)].path} to LFS storage` + }); + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading" + }); + } + }), + CONCURRENT_LFS_UPLOADS + ).then(returnCallback, rejectCallback); + }); + } + } + abortSignal?.throwIfAborted(); + yield { event: "phase", phase: "committing" }; + return yield* eventToGenerator( + async (yieldCallback, returnCallback, rejectCallback) => (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commit/${encodeURIComponent( + params.branch ?? "main" + )}` + (params.isPullRequest ? "?create_pr=1" : ""), + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: [ + { + key: "header", + value: { + summary: params.title, + description: params.description, + parentCommit: params.parentCommit + } + }, + ...await Promise.all( + allOperations.map((operation) => { + if (isFileOperation(operation)) { + const sha = lfsShas.get(operation.path); + if (sha) { + return { + key: "lfsFile", + value: { + path: operation.path, + algo: "sha256", + size: operation.content.size, + oid: sha + } + }; + } + } + return convertOperationToNdJson(operation); + }) + ) + ].map((x) => JSON.stringify(x)).join("\n"), + signal: abortSignal, + ...{ + progressHint: { + progressCallback: (progress) => { + for (const op of allOperations) { + if (isFileOperation(op) && !lfsShas.has(op.path)) { + yieldCallback({ + event: "fileProgress", + path: op.path, + progress, + state: "uploading" + }); + } + } + } + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } + } + ).then(async (res) => { + if (!res.ok) { + throw await createApiError(res); + } + const json = await res.json(); + returnCallback({ + pullRequestUrl: json.pullRequestUrl, + commit: { + oid: json.commitOid, + url: json.commitUrl + }, + hookOutput: json.hookOutput + }); + }).catch(rejectCallback) + ); + } catch (err) { + abortController.abort(); + throw err; + } +} +async function* commitIterBucket(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + if (params.useXet === false) { + throw new Error("useXet must be true or undefined for buckets"); + } + const abortController = new AbortController(); + const abortSignal = abortController.signal; + if (!abortSignal.throwIfAborted) { + abortSignal.throwIfAborted = () => { + if (abortSignal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + }; + } + if (params.abortSignal) { + params.abortSignal.addEventListener("abort", () => abortController.abort()); + } + try { + const allOperations = (await Promise.all( + params.operations.map(async (operation) => { + if (operation.operation === "edit") { + const splicedBlob = SplicedBlob.create( + operation.originalContent, + operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })) + ); + return { + operation: "addOrUpdate", + path: operation.path, + content: splicedBlob + }; + } + if (operation.operation !== "addOrUpdate") { + return operation; + } + if (!(operation.content instanceof URL)) { + return { ...operation, content: operation.content }; + } + const lazyBlobs = await createBlobs(operation.content, operation.path, { + fetch: params.fetch, + maxFolderDepth: params.maxFolderDepth + }); + abortSignal?.throwIfAborted(); + return lazyBlobs.map((blob) => ({ + ...operation, + content: blob.blob, + path: blob.path + })); + }) + )).flat(1); + yield { event: "phase", phase: "uploadingLargeFiles" }; + for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { + const xetHashes = /* @__PURE__ */ new Map(); + abortSignal?.throwIfAborted(); + const source = async function* () { + for (const operation of operations) { + abortSignal?.throwIfAborted(); + yield { content: operation.content, path: operation.path }; + } + }(); + const xetParams = { + sessionId: crypto.randomUUID(), + refreshWriteTokenUrl: `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/xet-write-token` + }; + const sources = splitAsyncGenerator(source, 5); + yield* eventToGenerator( + (yieldCallback, returnCallback, rejectCallback) => Promise.all( + sources.map(async function(source2) { + for await (const event of uploadShards(source2, { + fetch: params.fetch, + accessToken, + hubUrl: params.hubUrl ?? HUB_URL, + repo: repoId, + xetParams, + rev: params.branch ?? "main", + yieldCallback: (event2) => yieldCallback({ ...event2, state: "uploading" }) + })) { + if (event.event === "file") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: 1, + state: "uploading" + }); + xetHashes.set(event.path, event.xetHash); + } else if (event.event === "fileProgress") { + yieldCallback({ + event: "fileProgress", + path: event.path, + progress: event.progress, + state: "uploading" + }); + } + } + }) + ).then(() => returnCallback(void 0), rejectCallback) + ); + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: [...xetHashes.entries()].map( + ([path2, xetHash]) => JSON.stringify({ + type: "addFile", + path: path2, + xetHash + }) + ).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok && resp.status !== 422) { + throw await createApiError(resp); + } + const json = await resp.json(); + for (const failed of json.failed) { + yield { + event: "fileProgress", + path: failed.path, + progress: 0, + state: "error" + }; + } + } + abortSignal?.throwIfAborted(); + const copyOperations = allOperations.filter( + (operation) => operation.operation === "copy" + ); + for (const copyChunk of chunk(copyOperations, 100)) { + abortSignal?.throwIfAborted(); + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: copyChunk.map((op) => { + const sourceRepoId = toRepoId(op.sourceRepo); + return JSON.stringify({ + type: "copyFile", + path: op.path, + xetHash: op.sourceXetHash, + sourceRepoType: sourceRepoId.type, + sourceRepoId: sourceRepoId.name + }); + }).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok && resp.status !== 422) { + throw await createApiError(resp); + } + const json = await resp.json(); + for (const failed of json.failed) { + yield { + event: "fileProgress", + path: failed.path, + progress: 0, + state: "error" + }; + } + } + abortSignal?.throwIfAborted(); + const deletedOperations = allOperations.filter((operation) => operation.operation === "delete"); + if (deletedOperations.length > 0) { + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...accessToken && { Authorization: `Bearer ${accessToken}` }, + "Content-Type": "application/x-ndjson" + }, + body: deletedOperations.map( + (operation) => JSON.stringify({ + type: "deleteFile", + path: operation.path + }) + ).join("\n"), + signal: abortSignal + } + ); + if (!resp.ok) { + throw await createApiError(resp); + } + const json = await resp.json(); + if (json.failed.length > 0) { + const failedPaths = json.failed.slice(0, 5).map((f) => f.path); + throw new Error( + `Failed to delete ${json.failed.length} file(s): ${failedPaths.join(", ")}${json.failed.length > 5 ? "..." : ""}, request ID: ${resp.headers.get("X-Request-Id")}` + ); + } + } + abortSignal?.throwIfAborted(); + } catch (err) { + abortController.abort(); + throw err; + } +} +async function commit(params) { + const iterator = commitIter(params); + const failedPaths = []; + let failedCount = 0; + let res = await iterator.next(); + while (!res.done) { + if (res.value.event === "fileProgress" && res.value.state === "error") { + failedCount++; + if (failedPaths.length < 5) { + failedPaths.push(res.value.path); + } + } + res = await iterator.next(); + } + if (failedCount > 0) { + throw new Error( + `Failed to upload ${failedCount} file(s): ${failedPaths.join(", ")}${failedCount > 5 ? "..." : ""}` + ); + } + return res.value; +} +async function convertOperationToNdJson(operation) { + switch (operation.operation) { + case "addOrUpdate": { + return { + key: "file", + value: { + content: base64FromBytes(new Uint8Array(await operation.content.arrayBuffer())), + path: operation.path, + encoding: "base64" + } + }; + } + case "delete": { + return { + key: "deletedFile", + value: { + path: operation.path + } + }; + } + case "edit": { + throw new Error( + "Edit operations should be converted to addOrUpdate operations before reaching convertOperationToNdJson" + ); + } + default: + throw new TypeError("Unknown operation: " + operation.operation); + } +} + +// src/utils/formatBytes.ts +function formatBytes(bytes) { + if (!Number.isFinite(bytes) || bytes < 0) { + return `${bytes} B`; + } + const units = ["B", "kB", "MB", "GB", "TB", "PB"]; + let value = bytes; + let i = 0; + while (value >= 1e3 && i < units.length - 1) { + value /= 1e3; + i++; + } + const formatted = i === 0 ? value.toString() : value.toFixed(value >= 100 ? 0 : value >= 10 ? 1 : 2); + return `${formatted} ${units[i]}`; +} + +// src/utils/parseLinkHeader.ts +function parseLinkHeader(header) { + const regex = /<(https?:[/][/][^>]+)>;\s+rel="([^"]+)"/g; + return Object.fromEntries([...header.matchAll(regex)].map(([, url, rel]) => [rel, url])); +} + +// src/lib/file-download-info.ts +async function fileDownloadInfo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const hubUrl = params.hubUrl ?? HUB_URL; + const revision = repoId.type === "bucket" ? void 0 : params.revision ?? "main"; + const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${params.raw ? "raw" : "resolve"}${revision ? `/${encodeURIComponent(revision)}` : ""}/${params.path}` + (params.noContentDisposition ? "?noContentDisposition=1" : ""); + const resp = await (params.fetch ?? fetch)(url, { + method: "GET", + headers: { + ...accessToken && { + Authorization: `Bearer ${accessToken}` + }, + Range: "bytes=0-0", + Accept: "application/vnd.xet-fileinfo+json, */*" + } + }); + if (resp.status === 404 && resp.headers.get("X-Error-Code") === "EntryNotFound") { + return null; + } + if (!resp.ok) { + throw await createApiError(resp); + } + let size; + let xetInfo; + if (resp.headers.get("Content-Type")?.includes("application/vnd.xet-fileinfo+json")) { + size = parseInt(resp.headers.get("X-Linked-Size") ?? "invalid"); + if (isNaN(size)) { + throw new InvalidApiResponseFormatError("Invalid file size received in X-Linked-Size header"); + } + const hash2 = resp.headers.get("X-Xet-Hash"); + const links = parseLinkHeader(resp.headers.get("Link") ?? ""); + const reconstructionUrl = (() => { + try { + return new URL(links["xet-reconstruction-info"]); + } catch { + return null; + } + })(); + const refreshUrl = (() => { + try { + return new URL(links["xet-auth"]); + } catch { + return null; + } + })(); + if (!hash2) { + throw new InvalidApiResponseFormatError("No hash received in X-Xet-Hash header"); + } + if (!reconstructionUrl || !refreshUrl) { + throw new InvalidApiResponseFormatError("No xet-reconstruction-info or xet-auth link header"); + } + xetInfo = { + hash: hash2, + refreshUrl, + reconstructionUrl + }; + } + if (size === void 0 || isNaN(size)) { + const contentRangeHeader = resp.headers.get("content-range"); + if (!contentRangeHeader) { + throw new InvalidApiResponseFormatError("Expected size information"); + } + const [, parsedSize] = contentRangeHeader.split("/"); + size = parseInt(parsedSize); + if (isNaN(size)) { + throw new InvalidApiResponseFormatError("Invalid file size received"); + } + } + const etag = resp.headers.get("X-Linked-ETag") ?? resp.headers.get("ETag") ?? void 0; + if (!etag) { + throw new InvalidApiResponseFormatError("Expected ETag"); + } + return { + etag, + size, + xet: xetInfo, + // Cannot use resp.url in case it's a S3 url and the user adds an Authorization header to it. + url: resp.url && (new URL(resp.url).origin === new URL(hubUrl).origin || resp.headers.get("X-Cache")?.endsWith(" cloudfront")) ? resp.url : url + }; +} + +// src/lib/download-file.ts +async function downloadFile(params) { + const accessToken = checkCredentials(params); + const info = params.downloadInfo ?? await fileDownloadInfo({ + accessToken, + repo: params.repo, + path: params.path, + revision: params.revision, + hubUrl: params.hubUrl, + fetch: params.fetch, + raw: params.raw + }); + if (!info) { + return null; + } + if (info.xet && params.xet !== false) { + return new XetBlob({ + refreshUrl: info.xet.refreshUrl.href, + reconstructionUrl: info.xet.reconstructionUrl.href, + fetch: params.fetch, + accessToken, + size: info.size, + readToken: typeof params.xet === "object" ? params.xet.readToken : void 0 + }); + } + return new WebBlob(new URL(info.url), 0, info.size, "", true, params.fetch ?? fetch, accessToken); +} + +// src/lib/list-files.ts +async function* listFiles(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const revision = repoId.type === "bucket" ? void 0 : params.revision || "main"; + let url = `${params.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}/tree${revision ? `/${revision}` : ""}${params.path ? "/" + params.path : ""}?recursive=${!!params.recursive}&expand=${!!params.expand}`; + while (url) { + const res = await (params.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + yield item; + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/paths-info.ts +async function pathsInfo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const hubUrl = params.hubUrl ?? HUB_URL; + const revision = repoId.type === "bucket" ? void 0 : params.revision ?? "main"; + const url = `${hubUrl}/api/${repoId.type}s/${repoId.name}/paths-info${revision ? `/${encodeURIComponent(revision)}` : ""}`; + const resp = await (params.fetch ?? fetch)(url, { + method: "POST", + headers: { + ...accessToken && { + Authorization: `Bearer ${accessToken}` + }, + Accept: "application/json", + "Content-Type": "application/json" + }, + body: JSON.stringify({ + paths: params.paths, + expand: params.expand + }) + }); + if (!resp.ok) { + throw await createApiError(resp); + } + const json = await resp.json(); + if (!Array.isArray(json)) { + throw new Error("malformed response: expected array"); + } + return json.map((item) => ({ + path: item.path, + lfs: item.lfs, + type: item.type, + oid: item.oid, + size: item.size, + xetHash: item.xetHash, + uploadedAt: item.uploadedAt, + securityFileStatus: item.securityFileStatus, + lastCommit: item.lastCommit ? { + date: new Date(item.lastCommit.date), + title: item.lastCommit.title, + id: item.lastCommit.id + } : void 0 + })); +} + +// src/lib/copy-files.ts +var DOWNLOAD_CONCURRENCY = 5; +var PATHS_INFO_BATCH_SIZE = 100; +var MAX_REPORTED_LFS_PATHS = 5; +function copyFile(params) { + return copyFiles({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + destination: params.destination.repo, + files: [ + { + source: params.source, + destinationPath: params.destination.path + } + ], + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); +} +function copyFileIter(params) { + return copyFilesIter({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + destination: params.destination.repo, + files: [ + { + source: params.source, + destinationPath: params.destination.path + } + ], + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); +} +async function copyFiles(params) { + const iterator = copyFilesIter(params); + while (true) { + const res = await iterator.next(); + if (res.done) { + return void 0; + } + } +} +async function* copyFilesIter(params) { + if (params.files.length === 0) { + return void 0; + } + const operations = yield* resolveCopyOperationsIter(params, params.files); + await commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.destination, + operations, + title: "", + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); + return void 0; +} +async function copyFolder(params) { + const iterator = copyFolderIter(params); + while (true) { + const res = await iterator.next(); + if (res.done) { + return void 0; + } + } +} +async function* copyFolderIter(params) { + const accessToken = checkCredentials(params); + const sourceRepoId = toRepoId(params.source.repo); + const sourcePath = (params.source.path ?? "").replace(/\/+$/, ""); + const destinationPrefix = (params.destination.path ?? "").replace(/\/+$/, ""); + const sourceRevision = sourceRepoId.type === "bucket" ? void 0 : params.source.revision ?? "main"; + const operations = []; + const pendingDownloads = []; + const lfsOffenders = []; + for await (const item of listFiles({ + repo: sourceRepoId, + path: sourcePath || void 0, + recursive: true, + revision: sourceRevision, + accessToken, + hubUrl: params.hubUrl, + fetch: params.fetch + })) { + if (item.type !== "file") { + continue; + } + const relPath = relativeUnderFolder(item.path, sourcePath); + const destPath = destinationPrefix ? `${destinationPrefix}/${relPath}` : relPath; + switch (classifySourceFile(item)) { + case "copy": + operations.push({ + operation: "copy", + path: destPath, + sourceXetHash: item.xetHash, + sourceRepo: sourceRepoId + }); + continue; + case "lfs": + lfsOffenders.push({ path: item.path, size: item.lfs?.size ?? item.size }); + continue; + case "download": + pendingDownloads.push({ + index: operations.length, + repoId: sourceRepoId, + revision: sourceRevision, + sourcePath: item.path + }); + operations.push({ + operation: "addOrUpdate", + path: destPath, + content: new Blob([]) + }); + continue; + } + } + if (lfsOffenders.length > 0) { + throwUnmigratedLfsError(sourceRepoId, lfsOffenders); + } + if (operations.length === 0) { + return void 0; + } + yield* downloadAndFillBlobsIter({ + pendingDownloads, + operations, + accessToken, + hubUrl: params.hubUrl, + fetch: params.fetch + }); + await commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.destination.repo, + operations, + title: "", + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal + }); + return void 0; +} +async function* resolveCopyOperationsIter(shared, files) { + const accessToken = checkCredentials(shared); + const groups = /* @__PURE__ */ new Map(); + for (let i = 0; i < files.length; i++) { + const file = files[i]; + const repoId = toRepoId(file.source.repo); + const revision = repoId.type === "bucket" ? void 0 : file.source.revision ?? "main"; + const key = `${repoId.type}\0${repoId.name}\0${revision ?? ""}`; + let group = groups.get(key); + if (!group) { + group = { repoId, revision, entries: [] }; + groups.set(key, group); + } + group.entries.push({ index: i, file }); + } + const operations = new Array(files.length); + const pendingDownloads = []; + for (const group of groups.values()) { + const paths = group.entries.map((e) => e.file.source.path); + const infos = []; + for (let offset = 0; offset < paths.length; offset += PATHS_INFO_BATCH_SIZE) { + const slice = paths.slice(offset, offset + PATHS_INFO_BATCH_SIZE); + const res = await pathsInfo({ + repo: group.repoId, + paths: slice, + revision: group.revision, + accessToken, + hubUrl: shared.hubUrl, + fetch: shared.fetch + }); + infos.push(...res); + } + const infoByPath = new Map(infos.map((i) => [i.path, i])); + const lfsOffenders = []; + for (const { index, file } of group.entries) { + const info = infoByPath.get(file.source.path); + if (!info) { + throw new Error(`Source file not found: '${file.source.path}' in ${group.repoId.type}s/${group.repoId.name}`); + } + if (info.type !== "file") { + throw new Error( + `Source path '${file.source.path}' in ${group.repoId.type}s/${group.repoId.name} is a folder; use copyFolder() instead.` + ); + } + switch (classifySourceFile(info)) { + case "copy": + operations[index] = { + operation: "copy", + path: file.destinationPath, + sourceXetHash: info.xetHash, + sourceRepo: group.repoId + }; + continue; + case "lfs": + lfsOffenders.push({ path: file.source.path, size: info.lfs?.size ?? info.size }); + continue; + case "download": + pendingDownloads.push({ + index, + repoId: group.repoId, + revision: group.revision, + sourcePath: file.source.path + }); + operations[index] = { + operation: "addOrUpdate", + path: file.destinationPath, + content: new Blob([]) + }; + continue; + } + } + if (lfsOffenders.length > 0) { + throwUnmigratedLfsError(group.repoId, lfsOffenders); + } + } + yield* downloadAndFillBlobsIter({ + pendingDownloads, + operations, + accessToken, + hubUrl: shared.hubUrl, + fetch: shared.fetch + }); + return operations; +} +function downloadAndFillBlobsIter(args) { + const total = args.pendingDownloads.length; + return eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + if (total === 0) { + returnCallback(); + return; + } + let downloaded = 0; + promisesQueue( + args.pendingDownloads.map(({ index, repoId, revision, sourcePath }) => async () => { + const blob = await downloadFile({ + repo: repoId, + path: sourcePath, + revision, + accessToken: args.accessToken, + hubUrl: args.hubUrl, + fetch: args.fetch + }); + if (!blob) { + throw new Error(`Failed to download '${sourcePath}' from ${repoId.type}s/${repoId.name}`); + } + const op = args.operations[index]; + if (op.operation !== "addOrUpdate") { + throw new Error("Internal: expected addOrUpdate placeholder operation"); + } + op.content = blob; + downloaded++; + yieldCallback({ event: "fileDownloaded", path: sourcePath, downloaded, total }); + }), + DOWNLOAD_CONCURRENCY + ).then( + () => returnCallback(), + (err) => rejectCallback(err) + ); + }); +} +function relativeUnderFolder(filePath, folderPath) { + if (!folderPath) { + return filePath; + } + if (filePath === folderPath) { + return filePath.split("/").pop() ?? filePath; + } + if (filePath.startsWith(folderPath + "/")) { + return filePath.slice(folderPath.length + 1); + } + throw new Error(`Path '${filePath}' is not inside folder '${folderPath}'`); +} +function classifySourceFile(file) { + if (file.xetHash) { + return "copy"; + } + if (file.lfs) { + return "lfs"; + } + return "download"; +} +function throwUnmigratedLfsError(repoId, entries) { + const head = entries.slice(0, MAX_REPORTED_LFS_PATHS).map((e) => `'${e.path}' (${formatBytes(e.size)})`).join(", "); + const more = entries.length > MAX_REPORTED_LFS_PATHS ? ` (and ${entries.length - MAX_REPORTED_LFS_PATHS} more)` : ""; + throw new Error( + `Cannot copy ${entries.length} LFS file(s) from ${repoId.type}s/${repoId.name} that have not been migrated to xet: ${head}${more}. Migrate these files to xet before copying.` + ); +} + +// src/lib/count-commits.ts +async function countCommits(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const url = `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commits/${params.revision ?? "main"}?limit=1`; + const res = await (params.fetch ?? fetch)(url, { + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }); + if (!res.ok) { + throw await createApiError(res); + } + return parseInt(res.headers.get("x-total-count") ?? "0", 10); +} + +// src/lib/create-repo.ts +async function createRepo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const [namespace, repoName] = repoId.name.split("/"); + const visibility = params.visibility ?? (params.private !== void 0 ? params.private ? "private" : "public" : void 0); + if (!namespace || !repoName) { + throw new TypeError( + `"${repoId.name}" is not a fully qualified repo name. It should be of the form "{namespace}/{repoName}".` + ); + } + const res = repoId.type === "bucket" ? await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/buckets/${namespace}/${repoName}`, { + method: "POST", + body: JSON.stringify({ + visibility, + resourceGroupId: params.resourceGroupId + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }) : await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/repos/create`, { + method: "POST", + body: JSON.stringify({ + name: repoName, + visibility, + organization: namespace, + resourceGroupId: params.resourceGroupId, + license: params.license, + ...repoId.type === "space" ? { + type: "space", + sdk: params.sdk ?? "static" + } : { + type: repoId.type + }, + files: params.files ? await Promise.all( + params.files.map(async (file) => ({ + encoding: "base64", + path: file.path, + content: base64FromBytes( + new Uint8Array(file.content instanceof Blob ? await file.content.arrayBuffer() : file.content) + ) + })) + ) : void 0 + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const output = await res.json(); + return { repoUrl: output.url, id: output.id }; +} + +// src/lib/create-branch.ts +async function createBranch(params) { + const repoId = toRepoId(params.repo); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + }, + body: JSON.stringify({ + startingPoint: params.revision, + ...params.empty && { emptyBranch: true }, + overwrite: params.overwrite + }) + } + ); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/create-collection.ts +async function createCollection(params) { + const accessToken = checkCredentials(params); + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/collections`, { + method: "POST", + body: JSON.stringify(params.collection), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const output = await res.json(); + return { slug: output.slug }; +} + +// src/utils/pick.ts +function pick(o, props) { + return Object.assign( + {}, + ...props.map((prop) => { + if (o[prop] !== void 0) { + return { [prop]: o[prop] }; + } + }) + ); +} + +// src/lib/list-datasets.ts +var DATASET_EXPAND_KEYS = [ + "private", + "downloads", + "gated", + "likes", + "lastModified" +]; +var DATASET_EXPANDABLE_KEYS = [ + "author", + "cardData", + "citation", + "createdAt", + "disabled", + "description", + "downloads", + "downloadsAllTime", + "gated", + "gitalyUid", + "lastModified", + "likes", + "paperswithcode_id", + "private", + // "siblings", + "sha", + "tags" +]; +async function* listDatasets(params) { + const accessToken = params && checkCredentials(params); + let totalToFetch = params?.limit ?? Infinity; + const search = new URLSearchParams([ + ...Object.entries({ + limit: String(Math.min(totalToFetch, 500)), + ...params?.search?.owner ? { author: params.search.owner } : void 0, + ...params?.search?.query ? { search: params.search.query } : void 0, + ...params?.sort ? { sort: params.sort } : void 0 + }), + ...params?.search?.tags?.map((tag) => ["filter", tag]) ?? [], + ...DATASET_EXPAND_KEYS.map((val) => ["expand", val]), + ...params?.additionalFields?.map((val) => ["expand", val]) ?? [] + ]).toString(); + let url = `${params?.hubUrl || HUB_URL}/api/datasets` + (search ? "?" + search : ""); + while (url) { + const res = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + yield { + ...params?.additionalFields && pick(item, params.additionalFields), + id: item._id, + name: item.id, + private: item.private, + downloads: item.downloads, + likes: item.likes, + gated: item.gated, + updatedAt: new Date(item.lastModified) + }; + totalToFetch--; + if (totalToFetch <= 0) { + return; + } + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/dataset-info.ts +async function datasetInfo(params) { + const accessToken = params && checkCredentials(params); + const search = new URLSearchParams([ + ...DATASET_EXPAND_KEYS.map((val) => ["expand", val]), + ...params?.additionalFields?.map((val) => ["expand", val]) ?? [] + ]).toString(); + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/datasets/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD" + )}?${search.toString()}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + const data = await response.json(); + return { + ...params?.additionalFields && pick(data, params.additionalFields), + id: data._id, + name: data.id, + private: data.private, + downloads: data.downloads, + likes: data.likes, + gated: data.gated, + updatedAt: new Date(data.lastModified) + }; +} + +// src/lib/delete-branch.ts +async function deleteBranch(params) { + const repoId = toRepoId(params.repo); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, + { + method: "DELETE", + headers: { + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + } + } + ); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/delete-file.ts +function deleteFile(params) { + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: [ + { + operation: "delete", + path: params.path + } + ], + title: params.commitTitle ?? `Delete ${params.path}`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch + }); +} + +// src/lib/delete-files.ts +function deleteFiles(params) { + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: params.paths.map((path2) => ({ + operation: "delete", + path: path2 + })), + title: params.commitTitle ?? `Deletes ${params.paths.length} files`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch + }); +} + +// src/lib/delete-repo.ts +async function deleteRepo(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const [namespace, repoName] = repoId.name.split("/"); + const res = repoId.type === "bucket" ? await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/buckets/${namespace}/${repoName}`, { + method: "DELETE", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }) : await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/repos/delete`, { + method: "DELETE", + body: JSON.stringify({ + name: repoName, + organization: namespace, + type: repoId.type + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/delete-collection.ts +async function deleteCollection(params) { + if (!params.slug) { + throw new TypeError("slug is required"); + } + const accessToken = checkCredentials(params); + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/collections/${params.slug}`, { + method: "DELETE", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json" + } + }); + if (!res.ok) { + throw await createApiError(res); + } +} + +// src/lib/download-file-to-cache-dir.ts +var import_node_path2 = require("path"); +var import_promises4 = require("fs/promises"); + +// src/utils/symlink.ts +var fs = __toESM(require("fs/promises")); +var path = __toESM(require("path")); +var os = __toESM(require("os")); +function expandUser(path2) { + if (path2.startsWith("~")) { + return path2.replace("~", os.homedir()); + } + return path2; +} +async function createSymlink(params) { + const abs_src = path.resolve(expandUser(params.sourcePath)); + const abs_dst = path.resolve(expandUser(params.finalPath)); + try { + await fs.rm(abs_dst); + } catch { + } + try { + await fs.symlink(path.relative(path.dirname(abs_dst), abs_src), abs_dst); + } catch { + console.info(`Symlink not supported. Copying file from ${abs_src} to ${abs_dst}`); + await fs.copyFile(abs_src, abs_dst); + } +} + +// src/lib/download-file-to-cache-dir.ts +var import_node_stream3 = require("stream"); +var import_promises5 = require("stream/promises"); +var import_node_fs2 = require("fs"); +var REGEX_COMMIT_HASH = new RegExp("^[0-9a-f]{40}$"); +function getFilePointer(storageFolder, revision, relativeFilename) { + const snapshotPath = (0, import_node_path2.join)(storageFolder, "snapshots"); + return (0, import_node_path2.join)(snapshotPath, revision, relativeFilename); +} +async function exists(path2, followSymlinks) { + try { + if (followSymlinks) { + await (0, import_promises4.stat)(path2); + } else { + await (0, import_promises4.lstat)(path2); + } + return true; + } catch (err) { + return false; + } +} +async function downloadFileToCacheDir(params) { + const repoId = toRepoId(params.repo); + if (repoId.type === "bucket") { + throw new Error("downloadFileToCacheDir is not supported for bucket repos."); + } + const revision = params.revision ?? "main"; + const cacheDir = params.cacheDir ?? getHFHubCachePath(); + const storageFolder = (0, import_node_path2.join)(cacheDir, getRepoFolderName(repoId)); + let commitHash; + if (revision && REGEX_COMMIT_HASH.test(revision)) { + commitHash = revision; + const pointerPath2 = getFilePointer(storageFolder, revision, params.path); + if (await exists(pointerPath2, true)) { + return pointerPath2; + } + } + const pathsInformation = await pathsInfo({ + ...params, + paths: [params.path], + revision, + expand: true + }); + if (!pathsInformation || pathsInformation.length !== 1) { + throw new Error(`cannot get path info for ${params.path}`); + } + const info = pathsInformation[0]; + let etag; + if (info.lfs) { + etag = info.lfs.oid; + } else if (info.xetHash) { + etag = info.xetHash; + } else if (info.oid) { + etag = info.oid; + } else { + throw new Error(`cannot determine etag for ${params.path}`); + } + const snapshotId = commitHash ?? info.lastCommit?.id ?? etag; + const pointerPath = getFilePointer(storageFolder, snapshotId, params.path); + const blobPath = (0, import_node_path2.join)(storageFolder, "blobs", etag); + if (await exists(pointerPath, true)) { + return pointerPath; + } + await (0, import_promises4.mkdir)((0, import_node_path2.dirname)(blobPath), { recursive: true }); + await (0, import_promises4.mkdir)((0, import_node_path2.dirname)(pointerPath), { recursive: true }); + if (await exists(blobPath)) { + await createSymlink({ sourcePath: blobPath, finalPath: pointerPath }); + return pointerPath; + } + const incomplete = `${blobPath}.incomplete`; + console.debug(`Downloading ${params.path} to ${incomplete}`); + const blob = await downloadFile({ + ...params, + revision + }); + if (!blob) { + throw new Error(`invalid response for file ${params.path}`); + } + await (0, import_promises5.pipeline)(import_node_stream3.Readable.fromWeb(blob.stream()), (0, import_node_fs2.createWriteStream)(incomplete)); + await (0, import_promises4.rename)(incomplete, blobPath); + await createSymlink({ sourcePath: blobPath, finalPath: pointerPath }); + return pointerPath; +} + +// src/lib/file-exists.ts +async function fileExists(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const hubUrl = params.hubUrl ?? HUB_URL; + const revision = repoId.type === "bucket" ? void 0 : params.revision ?? "main"; + const endpoint = repoId.type === "bucket" ? "resolve" : "raw"; + const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${endpoint}${revision ? `/${encodeURIComponent(revision)}` : ""}/${params.path}`; + const resp = await (params.fetch ?? fetch)(url, { + method: "HEAD", + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }); + if (resp.status === 404) { + return false; + } + if (!resp.ok) { + throw await createApiError(resp); + } + return true; +} + +// src/lib/jobs/cancel-job.ts +async function cancelJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/cancel`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/create-scheduled-job.ts +async function createScheduledJob(params) { + const accessToken = checkCredentials(params); + const { namespace, hubUrl, fetch: customFetch, ...rest } = params; + if (!rest.jobSpec.dockerImage && !rest.jobSpec.spaceId) { + throw new Error("Either dockerImage or spaceId must be provided in jobSpec"); + } + if (rest.jobSpec.dockerImage && rest.jobSpec.spaceId) { + throw new Error("Cannot provide both dockerImage and spaceId in jobSpec"); + } + const body = { + jobSpec: { + flavor: rest.jobSpec.flavor + }, + schedule: rest.schedule, + suspend: rest.suspend ?? false, + concurrency: rest.concurrency ?? false + }; + if (rest.jobSpec.dockerImage) { + body.jobSpec.dockerImage = rest.jobSpec.dockerImage; + } + if (rest.jobSpec.spaceId) { + body.jobSpec.spaceId = rest.jobSpec.spaceId; + } + if (rest.jobSpec.command) { + body.jobSpec.command = rest.jobSpec.command; + } + body.jobSpec.environment = rest.jobSpec.environment || {}; + if (rest.jobSpec.secrets) { + body.jobSpec.secrets = rest.jobSpec.secrets; + } + if (rest.jobSpec.arch) { + body.jobSpec.arch = rest.jobSpec.arch; + } + if (rest.jobSpec.timeoutSeconds !== void 0) { + body.jobSpec.timeoutSeconds = rest.jobSpec.timeoutSeconds; + } + if (rest.jobSpec.attempts !== void 0) { + body.jobSpec.attempts = rest.jobSpec.attempts; + } + if (rest.jobSpec.labels) { + body.jobSpec.labels = rest.jobSpec.labels; + } + if (rest.jobSpec.volumes?.length) { + body.jobSpec.volumes = rest.jobSpec.volumes.map(({ source, ...rest2 }) => { + const repoId = toRepoId(source); + return { type: repoId.type, source: repoId.name, ...rest2 }; + }); + } + const response = await (customFetch || fetch)(`${hubUrl || HUB_URL}/api/scheduled-jobs/${namespace}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }, + body: JSON.stringify(body) + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/delete-scheduled-job.ts +async function deleteScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}`, + { + method: "DELETE", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/lib/jobs/duplicate-job.ts +async function duplicateJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/duplicate`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/get-job.ts +async function getJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/get-scheduled-job.ts +async function getScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/list-job-hardware.ts +async function listJobHardware(params) { + const accessToken = checkCredentials(params ?? {}); + const headers = {}; + if (accessToken) { + headers.Authorization = `Bearer ${accessToken}`; + } + const response = await (params?.fetch || fetch)(`${params?.hubUrl || HUB_URL}/api/jobs/hardware`, { + headers + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/list-jobs.ts +async function listJobs(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/list-scheduled-jobs.ts +async function listScheduledJobs(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}`, { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/resume-scheduled-job.ts +async function resumeScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/resume`, + { + method: "POST", + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/lib/jobs/run-job.ts +async function runJob(params) { + const accessToken = checkCredentials(params); + if (!params.dockerImage && !params.spaceId) { + throw new Error("Either dockerImage or spaceId must be provided"); + } + if (params.dockerImage && params.spaceId) { + throw new Error("Cannot provide both dockerImage and spaceId"); + } + const body = { + flavor: params.flavor, + environment: params.environment || {} + }; + if (params.dockerImage) { + body.dockerImage = params.dockerImage; + } + if (params.spaceId) { + body.spaceId = params.spaceId; + } + if (params.command) { + body.command = params.command; + } + if (params.arguments) { + body.arguments = params.arguments; + } + if (params.secrets) { + body.secrets = params.secrets; + } + if (params.arch) { + body.arch = params.arch; + } + if (params.timeoutSeconds !== void 0) { + body.timeoutSeconds = params.timeoutSeconds; + } + if (params.attempts !== void 0) { + body.attempts = params.attempts; + } + if (params.labels) { + body.labels = params.labels; + } + if (params.volumes?.length) { + body.volumes = params.volumes.map(({ source, ...rest }) => { + const repoId = toRepoId(source); + return { type: repoId.type, source: repoId.name, ...rest }; + }); + } + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }, + body: JSON.stringify(body) + }); + if (!response.ok) { + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/run-scheduled-job.ts +async function runScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/run`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + if (response.status === 409) { + return null; + } + throw await createApiError(response); + } + return await response.json(); +} + +// src/lib/jobs/stream-job-events.ts +async function* streamJobEvents(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/events`, + { + headers: { + Accept: "text/event-stream", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + if (!response.body) { + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + } finally { + reader.releaseLock(); + } +} + +// src/lib/jobs/stream-job-logs.ts +async function* streamJobLogs(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/logs`, + { + headers: { + Accept: "text/event-stream", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + if (!response.body) { + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const data = JSON.parse(line.slice(6)); + yield { message: data.data, timestamp: new Date(data.timestamp) }; + } catch { + yield { message: line.slice(6), timestamp: /* @__PURE__ */ new Date() }; + } + } + } + } + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const data = JSON.parse(line.slice(6)); + yield { message: data.data, timestamp: new Date(data.timestamp) }; + } catch { + yield { message: line.slice(6), timestamp: /* @__PURE__ */ new Date() }; + } + } + } + } + } finally { + reader.releaseLock(); + } +} + +// src/lib/jobs/stream-job-metrics.ts +async function* streamJobMetrics(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/metrics`, + { + headers: { + Accept: "text/event-stream", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + if (!response.body) { + return; + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + } finally { + reader.releaseLock(); + } +} + +// src/lib/jobs/suspend-scheduled-job.ts +async function suspendScheduledJob(params) { + const accessToken = checkCredentials(params); + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/suspend`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } +} + +// src/lib/list-commits.ts +async function* listCommits(params) { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + let url = `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commits/${params.revision ?? "main"}?limit=${params.batchSize ?? 100}`; + while (url) { + const res = await (params.fetch ?? fetch)(url, { + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + }); + if (!res.ok) { + throw await createApiError(res); + } + const resJson = await res.json(); + for (const commit2 of resJson) { + yield { + oid: commit2.id, + title: commit2.title, + message: commit2.message, + authors: commit2.authors.map((author) => ({ + username: author.user, + avatarUrl: author.avatar + })), + date: new Date(commit2.date) + }; + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/utils/normalizeInferenceProviderMapping.ts +function normalizeInferenceProviderMapping(hfModelId, inferenceProviderMapping) { + if (!inferenceProviderMapping) { + return []; + } + if (Array.isArray(inferenceProviderMapping)) { + return inferenceProviderMapping.map((entry) => ({ + ...entry, + hfModelId + })); + } + return Object.entries(inferenceProviderMapping).map(([provider, mapping]) => ({ + provider, + hfModelId, + providerId: mapping.providerId, + status: mapping.status, + task: mapping.task + })); +} + +// src/lib/list-models.ts +var MODEL_EXPAND_KEYS = [ + "pipeline_tag", + "private", + "gated", + "downloads", + "likes", + "lastModified" +]; +var MODEL_EXPANDABLE_KEYS = [ + "author", + "cardData", + "config", + "createdAt", + "disabled", + "downloads", + "downloadsAllTime", + "gated", + "gitalyUid", + "inferenceProviderMapping", + "lastModified", + "library_name", + "likes", + "model-index", + "pipeline_tag", + "private", + "safetensors", + "sha", + "spaces", + "tags", + "transformersInfo" +]; +var MODEL_DERIVED_FIELD_TO_API_KEY = { + filePaths: "siblings" +}; +async function* listModels(params) { + const accessToken = params && checkCredentials(params); + let totalToFetch = params?.limit ?? Infinity; + const additionalExpandKeys = params?.additionalFields?.map( + (field) => MODEL_DERIVED_FIELD_TO_API_KEY[field] ?? field + ) ?? []; + const search = new URLSearchParams([ + ...Object.entries({ + limit: String(Math.min(totalToFetch, 500)), + ...params?.search?.owner ? { author: params.search.owner } : void 0, + ...params?.search?.task ? { pipeline_tag: params.search.task } : void 0, + ...params?.search?.query ? { search: params.search.query } : void 0, + ...params?.search?.inferenceProviders ? { inference_provider: params.search.inferenceProviders.join(",") } : void 0, + ...params?.search?.apps ? { apps: params.search.apps.join(",") } : void 0, + ...params?.sort ? { sort: params.sort } : void 0 + }), + ...params?.search?.tags?.map((tag) => ["filter", tag]) ?? [], + ...MODEL_EXPAND_KEYS.map((val) => ["expand", val]), + ...additionalExpandKeys.map((val) => ["expand", val]) + ]).toString(); + let url = `${params?.hubUrl || HUB_URL}/api/models?${search}`; + while (url) { + const res = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + const additional = {}; + if (params?.additionalFields) { + for (const field of params.additionalFields) { + if (field === "filePaths") { + additional.filePaths = (item.siblings ?? []).map((s) => s.rfilename); + } else if (field === "inferenceProviderMapping" && item.inferenceProviderMapping) { + additional.inferenceProviderMapping = normalizeInferenceProviderMapping( + item.id, + item.inferenceProviderMapping + ); + } else { + additional[field] = item[field]; + } + } + } + yield { + ...additional, + id: item._id, + name: item.id, + private: item.private, + task: item.pipeline_tag, + downloads: item.downloads, + gated: item.gated, + likes: item.likes, + updatedAt: new Date(item.lastModified) + }; + totalToFetch--; + if (totalToFetch <= 0) { + return; + } + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/list-spaces.ts +var SPACE_EXPAND_KEYS = [ + "sdk", + "likes", + "private", + "lastModified" +]; +var SPACE_EXPANDABLE_KEYS = [ + "author", + "cardData", + "datasets", + "disabled", + "gitalyUid", + "lastModified", + "createdAt", + "likes", + "private", + "runtime", + "sdk", + // "siblings", + "sha", + "subdomain", + "tags", + "models" +]; +async function* listSpaces(params) { + const accessToken = params && checkCredentials(params); + const search = new URLSearchParams([ + ...Object.entries({ + limit: "500", + ...params?.search?.owner ? { author: params.search.owner } : void 0, + ...params?.search?.query ? { search: params.search.query } : void 0, + ...params?.sort ? { sort: params.sort } : void 0 + }), + ...params?.search?.tags?.map((tag) => ["filter", tag]) ?? [], + ...[...SPACE_EXPAND_KEYS, ...params?.additionalFields ?? []].map( + (val) => ["expand", val] + ) + ]).toString(); + let url = `${params?.hubUrl || HUB_URL}/api/spaces?${search}`; + while (url) { + const res = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const items = await res.json(); + for (const item of items) { + yield { + ...params?.additionalFields && pick(item, params.additionalFields), + id: item._id, + name: item.id, + sdk: item.sdk, + likes: item.likes, + private: item.private, + updatedAt: new Date(item.lastModified) + }; + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/list-collections.ts +async function* listCollections(params) { + const accessToken = params && checkCredentials(params); + const searchParams = new URLSearchParams(); + let totalToFetch = params?.limit ?? Infinity; + searchParams.append("limit", String(Math.min(totalToFetch, 100))); + if (params?.sort) { + searchParams.append("sort", params.sort); + } + if (params?.search?.owner) { + for (const owner of params.search.owner) { + searchParams.append("owner", owner); + } + } + if (params?.search?.item) { + for (const item of params.search.item) { + searchParams.append("item", item); + } + } + if (params?.search?.q) { + searchParams.append("q", params.search.q); + } + let url = `${params?.hubUrl || HUB_URL}/api/collections?${searchParams}`; + while (url) { + const res = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const collections = await res.json(); + for (const collection of collections) { + yield collection; + totalToFetch--; + if (totalToFetch <= 0) { + return; + } + } + const linkHeader = res.headers.get("Link"); + url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; + } +} + +// src/lib/model-info.ts +async function modelInfo(params) { + const accessToken = params && checkCredentials(params); + const additionalExpandKeys = params?.additionalFields?.map( + (field) => MODEL_DERIVED_FIELD_TO_API_KEY[field] ?? field + ) ?? []; + const search = new URLSearchParams([ + ...MODEL_EXPAND_KEYS.map((val) => ["expand", val]), + ...additionalExpandKeys.map((val) => ["expand", val]) + ]).toString(); + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/models/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD" + )}?${search.toString()}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + const data = await response.json(); + const additional = {}; + if (params?.additionalFields) { + for (const field of params.additionalFields) { + if (field === "filePaths") { + additional.filePaths = (data.siblings ?? []).map((s) => s.rfilename); + } else if (field === "inferenceProviderMapping" && data.inferenceProviderMapping) { + additional.inferenceProviderMapping = normalizeInferenceProviderMapping(data.id, data.inferenceProviderMapping); + } else { + additional[field] = data[field]; + } + } + } + return { + ...additional, + id: data._id, + name: data.id, + private: data.private, + task: data.pipeline_tag, + downloads: data.downloads, + gated: data.gated, + likes: data.likes, + updatedAt: new Date(data.lastModified) + }; +} + +// src/lib/oauth-handle-redirect.ts +async function oauthHandleRedirect(opts) { + if (typeof window === "undefined" && !opts?.redirectedUrl) { + throw new Error("oauthHandleRedirect is only available in the browser, unless you provide redirectedUrl"); + } + if (typeof localStorage === "undefined" && (!opts?.nonce || !opts?.codeVerifier)) { + throw new Error( + "oauthHandleRedirect requires localStorage to be available, unless you provide nonce and codeVerifier" + ); + } + const redirectedUrl = opts?.redirectedUrl ?? window.location.href; + const searchParams = (() => { + try { + return new URL(redirectedUrl).searchParams; + } catch (err) { + throw new Error("Failed to parse redirected URL: " + redirectedUrl); + } + })(); + const [error, errorDescription] = [searchParams.get("error"), searchParams.get("error_description")]; + if (error) { + throw new Error(`${error}: ${errorDescription}`); + } + const code = searchParams.get("code"); + const nonce = opts?.nonce ?? localStorage.getItem("huggingface.co:oauth:nonce"); + if (!code) { + throw new Error("Missing oauth code from query parameters in redirected URL: " + redirectedUrl); + } + if (!nonce) { + throw new Error("Missing oauth nonce from localStorage"); + } + const codeVerifier = opts?.codeVerifier ?? localStorage.getItem("huggingface.co:oauth:code_verifier"); + if (!codeVerifier) { + throw new Error("Missing oauth code_verifier from localStorage"); + } + const state = searchParams.get("state"); + if (!state) { + throw new Error("Missing oauth state from query parameters in redirected URL"); + } + let parsedState; + try { + parsedState = JSON.parse(state); + } catch { + throw new Error("Invalid oauth state in redirected URL, unable to parse JSON: " + state); + } + if (parsedState.nonce !== nonce) { + throw new Error("Invalid oauth state in redirected URL"); + } + const hubUrl = opts?.hubUrl || HUB_URL; + const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; + const openidConfigRes = await fetch(openidConfigUrl, { + headers: { + Accept: "application/json" + } + }); + if (!openidConfigRes.ok) { + throw await createApiError(openidConfigRes); + } + const openidConfig = await openidConfigRes.json(); + const tokenRes = await fetch(openidConfig.token_endpoint, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded" + }, + body: new URLSearchParams({ + grant_type: "authorization_code", + code, + redirect_uri: parsedState.redirectUri, + code_verifier: codeVerifier + }).toString() + }); + if (!opts?.codeVerifier) { + localStorage.removeItem("huggingface.co:oauth:code_verifier"); + } + if (!opts?.nonce) { + localStorage.removeItem("huggingface.co:oauth:nonce"); + } + if (!tokenRes.ok) { + throw await createApiError(tokenRes); + } + const token = await tokenRes.json(); + const accessTokenExpiresAt = new Date(Date.now() + token.expires_in * 1e3); + const userInfoRes = await fetch(openidConfig.userinfo_endpoint, { + headers: { + Authorization: `Bearer ${token.access_token}` + } + }); + if (!userInfoRes.ok) { + throw await createApiError(userInfoRes); + } + const userInfo = await userInfoRes.json(); + return { + accessToken: token.access_token, + accessTokenExpiresAt, + userInfo, + state: parsedState.state, + scope: token.scope + }; +} +async function oauthHandleRedirectIfPresent(opts) { + if (typeof window === "undefined" && !opts?.redirectedUrl) { + throw new Error("oauthHandleRedirect is only available in the browser, unless you provide redirectedUrl"); + } + if (typeof localStorage === "undefined" && (!opts?.nonce || !opts?.codeVerifier)) { + throw new Error( + "oauthHandleRedirect requires localStorage to be available, unless you provide nonce and codeVerifier" + ); + } + const searchParams = new URLSearchParams(opts?.redirectedUrl ?? window.location.search); + if (searchParams.has("error")) { + return oauthHandleRedirect(opts); + } + if (searchParams.has("code")) { + if (!localStorage.getItem("huggingface.co:oauth:nonce")) { + console.warn( + "Missing oauth nonce from localStorage. This can happen when the user refreshes the page after logging in, without changing the URL." + ); + return false; + } + return oauthHandleRedirect(opts); + } + return false; +} + +// src/lib/oauth-login-url.ts +async function oauthLoginUrl(opts) { + if (typeof window === "undefined" && (!opts?.redirectUrl || !opts?.clientId)) { + throw new Error("oauthLogin is only available in the browser, unless you provide clientId and redirectUrl"); + } + if (typeof localStorage === "undefined" && !opts?.localStorage) { + throw new Error( + "oauthLogin requires localStorage to be available in the context, unless you provide a localStorage empty object as argument" + ); + } + const hubUrl = opts?.hubUrl || HUB_URL; + const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; + const openidConfigRes = await fetch(openidConfigUrl, { + headers: { + Accept: "application/json" + } + }); + if (!openidConfigRes.ok) { + throw await createApiError(openidConfigRes); + } + const opendidConfig = await openidConfigRes.json(); + const newNonce = globalThis.crypto.randomUUID(); + const newCodeVerifier = globalThis.crypto.randomUUID() + globalThis.crypto.randomUUID(); + if (opts?.localStorage) { + if (opts.localStorage.codeVerifier !== void 0 && opts.localStorage.codeVerifier !== null) { + throw new Error( + "localStorage.codeVerifier must be initially set to null or undefined, and will be filled by oauthLoginUrl" + ); + } + if (opts.localStorage.nonce !== void 0 && opts.localStorage.nonce !== null) { + throw new Error( + "localStorage.nonce must be initially set to null or undefined, and will be filled by oauthLoginUrl" + ); + } + opts.localStorage.codeVerifier = newCodeVerifier; + opts.localStorage.nonce = newNonce; + } else { + localStorage.setItem("huggingface.co:oauth:nonce", newNonce); + localStorage.setItem("huggingface.co:oauth:code_verifier", newCodeVerifier); + } + const redirectUri = opts?.redirectUrl || (typeof window !== "undefined" ? window.location.href : void 0); + if (!redirectUri) { + throw new Error("Missing redirectUrl"); + } + const state = JSON.stringify({ + nonce: newNonce, + redirectUri, + state: opts?.state + }); + const variables = ( + // @ts-expect-error window.huggingface is defined inside static Spaces. + typeof window !== "undefined" ? window.huggingface?.variables ?? null : null + ); + const clientId = opts?.clientId || variables?.OAUTH_CLIENT_ID; + if (!clientId) { + if (variables) { + throw new Error("Missing clientId, please add hf_oauth: true to the README.md's metadata in your static Space"); + } + throw new Error("Missing clientId"); + } + const challenge = base64FromBytes( + new Uint8Array(await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(newCodeVerifier))) + ).replace(/[+]/g, "-").replace(/[/]/g, "_").replace(/=/g, ""); + return `${opendidConfig.authorization_endpoint}?${new URLSearchParams({ + client_id: clientId, + scope: opts?.scopes || variables?.OAUTH_SCOPES || "openid profile", + response_type: "code", + redirect_uri: redirectUri, + state, + code_challenge: challenge, + code_challenge_method: "S256" + }).toString()}`; +} + +// src/utils/typedInclude.ts +function typedInclude(arr, v) { + return arr.includes(v); +} + +// src/utils/omit.ts +function omit(o, props) { + const propsArr = Array.isArray(props) ? props : [props]; + const letsKeep = Object.keys(o).filter((prop) => !typedInclude(propsArr, prop)); + return pick(o, letsKeep); +} + +// src/utils/typedEntries.ts +function typedEntries(obj) { + return Object.entries(obj); +} + +// src/lib/parse-safetensors-metadata.ts +var SAFETENSORS_FILE = "model.safetensors"; +var SAFETENSORS_INDEX_FILE = "model.safetensors.index.json"; +var RE_SAFETENSORS_FILE = /\.safetensors$/; +var RE_SAFETENSORS_INDEX_FILE = /\.safetensors\.index\.json$/; +var RE_SAFETENSORS_SHARD_FILE = /^(?(?.*?)[_-])(?\d{5,6})-of-(?\d{5,6})\.safetensors$/; +function parseSafetensorsShardFilename(filename) { + const match = RE_SAFETENSORS_SHARD_FILE.exec(filename); + if (match && match.groups) { + return { + prefix: match.groups["prefix"], + basePrefix: match.groups["basePrefix"], + shard: match.groups["shard"], + total: match.groups["total"] + }; + } + return null; +} +var PARALLEL_DOWNLOADS = 20; +var MAX_HEADER_LENGTH = 25e6; +var MAX_CONFIG_LENGTH = 1e7; +var MAX_SHARD_COUNT = 1e4; +var GPTQ_QWEIGHT_SUFFIX = "qweight"; +var GPTQ_AWQ_AUXILIARY_SUFFIXES = ["qzeros", "g_idx", "scales"]; +var SafetensorParseError = class extends Error { +}; +async function fetchModelConfig(params) { + try { + const configBlob = await downloadFile({ + ...params, + path: "config.json" + }); + if (!configBlob) { + return null; + } + const config = JSON.parse(await configBlob.slice(0, MAX_CONFIG_LENGTH).text()); + return config; + } catch (error) { + return null; + } +} +async function parseSingleFile(path2, params) { + const blob = await downloadFile({ ...params, path: path2 }); + if (!blob) { + throw new SafetensorParseError(`Failed to parse file ${path2}: failed to fetch safetensors header length.`); + } + const bufLengthOfHeaderLE = await blob.slice(0, 8).arrayBuffer(); + const lengthOfHeader = new DataView(bufLengthOfHeaderLE).getBigUint64(0, true); + if (lengthOfHeader <= 0) { + throw new SafetensorParseError(`Failed to parse file ${path2}: safetensors header is malformed.`); + } + if (lengthOfHeader > MAX_HEADER_LENGTH) { + throw new SafetensorParseError( + `Failed to parse file ${path2}: safetensor header is too big. Maximum supported size is ${MAX_HEADER_LENGTH} bytes.` + ); + } + try { + const header = JSON.parse(await blob.slice(8, 8 + Number(lengthOfHeader)).text()); + return header; + } catch (err) { + throw new SafetensorParseError(`Failed to parse file ${path2}: safetensors header is not valid JSON.`); + } +} +async function parseShardedIndex(path2, params) { + const indexBlob = await downloadFile({ + ...params, + path: path2 + }); + if (!indexBlob) { + throw new SafetensorParseError(`Failed to parse file ${path2}: failed to fetch safetensors index.`); + } + try { + const index = JSON.parse(await indexBlob.slice(0, MAX_HEADER_LENGTH).text()); + return index; + } catch (error) { + throw new SafetensorParseError(`Failed to parse file ${path2}: not a valid JSON.`); + } +} +async function fetchAllHeaders(path2, index, params) { + const pathPrefix = path2.slice(0, path2.lastIndexOf("/") + 1); + const filenames = [...new Set(Object.values(index.weight_map))]; + if (filenames.length > MAX_SHARD_COUNT) { + throw new SafetensorParseError( + `Too many shard files (${filenames.length}). Maximum supported is ${MAX_SHARD_COUNT}.` + ); + } + for (const filename of filenames) { + if (filename.includes("..") || filename.startsWith("/") || filename.includes("://")) { + throw new SafetensorParseError(`Unsafe shard filename in weight_map: "${filename}"`); + } + } + const shardedMap = Object.fromEntries( + await promisesQueue( + filenames.map( + (filename) => async () => [filename, await parseSingleFile(pathPrefix + filename, params)] + ), + PARALLEL_DOWNLOADS + ) + ); + return shardedMap; +} +function parseTotalParameters(value) { + if (!value) { + return void 0; + } + if (typeof value === "number") { + return value; + } + return parseInt(value); +} +async function parseSafetensorsMetadata(params) { + const repoId = toRepoId(params.repo); + if (repoId.type !== "model") { + throw new TypeError("Only model repos should contain safetensors files."); + } + const modelConfig = params.computeParametersCount ? await fetchModelConfig(params) : null; + const quantConfig = modelConfig?.quantization_config ?? modelConfig?.text_config?.quantization_config; + if (params.path && RE_SAFETENSORS_FILE.test(params.path) || await fileExists({ ...params, path: SAFETENSORS_FILE })) { + const header = await parseSingleFile(params.path ?? SAFETENSORS_FILE, params); + const paramStats = params.computeParametersCount ? { + parameterCount: computeNumOfParamsByDtypeSingleFile(header, quantConfig), + /// shortcut: get param count directly from metadata + parameterTotal: parseTotalParameters(header.__metadata__?.total_parameters) + } : void 0; + return { + sharded: false, + header, + ...paramStats, + filepaths: [params.path ?? SAFETENSORS_FILE] + }; + } else if (params.path && RE_SAFETENSORS_INDEX_FILE.test(params.path) || await fileExists({ ...params, path: SAFETENSORS_INDEX_FILE })) { + const path2 = params.path ?? SAFETENSORS_INDEX_FILE; + const index = await parseShardedIndex(path2, params); + const shardedMap = await fetchAllHeaders(path2, index, params); + const pathPrefix = path2.slice(0, path2.lastIndexOf("/") + 1); + const paramStats = params.computeParametersCount ? { + parameterCount: computeNumOfParamsByDtypeSharded(shardedMap, quantConfig), + /// shortcut: get param count directly from metadata + parameterTotal: parseTotalParameters(index.metadata?.total_parameters) + } : void 0; + return { + sharded: true, + index, + headers: shardedMap, + ...paramStats, + filepaths: [path2, ...Object.keys(shardedMap).map((filename) => pathPrefix + filename)] + }; + } else { + throw new Error("model id does not seem to contain safetensors weights"); + } +} +function globMatch(pattern, str) { + const parts = pattern.split("*"); + if (parts.length === 1) { + return pattern === str; + } + if (!str.startsWith(parts[0])) { + return false; + } + let pos = parts[0].length; + const lastPart = parts[parts.length - 1]; + if (!str.endsWith(lastPart)) { + return false; + } + const end = str.length - lastPart.length; + for (let i = 1; i < parts.length - 1; i++) { + const idx = str.indexOf(parts[i], pos); + if (idx === -1 || idx + parts[i].length > end) { + return false; + } + pos = idx + parts[i].length; + } + return pos <= end; +} +function isQuantizedTensor(tensorName, quantConfig) { + if (!quantConfig) { + return false; + } + const patterns = quantConfig.modules_to_not_convert; + if (!patterns?.length) { + return true; + } + return !patterns.some( + (pattern) => pattern.includes("*") ? globMatch(pattern, tensorName) : tensorName.includes(pattern) + ); +} +function matchesCompressedTensorsTarget(target, moduleName) { + if (!target.startsWith("re:")) { + return target === moduleName; + } + let pattern = target.slice(3); + if (pattern.startsWith("^")) { + pattern = pattern.slice(1); + } + if (pattern.endsWith("$")) { + pattern = pattern.slice(0, -1); + } else { + pattern += ".*"; + } + const glob = pattern.replaceAll(".*", "*").replaceAll("\\.", "."); + if (/[\\+?()[\]{}|^$]/.test(glob)) { + return false; + } + return globMatch(glob, moduleName); +} +function getQuantizationMultiplier(tensorName, dtype, quantConfig) { + if (!quantConfig || !isQuantizedTensor(tensorName, quantConfig)) { + return 1; + } + const quantMethod = quantConfig.quant_method?.toLowerCase(); + switch (quantMethod) { + case "mxfp4": + if (dtype === "U8" && tensorName.includes("_blocks")) { + return 2; + } + return 1; + case "gptq": + case "awq": + if (getTensorSuffix(tensorName) === GPTQ_QWEIGHT_SUFFIX) { + const bits = quantConfig.bits && quantConfig.bits > 0 ? quantConfig.bits : 4; + return Math.max(1, Math.floor(32 / bits)); + } + if (quantConfig.bits === 4 && dtype === "U8") { + return 2; + } + if (quantConfig.bits === 2 && dtype === "U8") { + return 4; + } + return 1; + case "compressed-tensors": + if (dtype === "I32") { + const groups = Object.values(quantConfig.config_groups ?? {}); + const suffixIndex = tensorName.lastIndexOf(".weight"); + const moduleName = suffixIndex === -1 ? tensorName : tensorName.slice(0, suffixIndex); + const group = groups.find( + (g) => g.targets?.some((target) => matchesCompressedTensorsTarget(target, moduleName)) + ); + if (group) { + if ((group.format ?? quantConfig.format) !== "pack-quantized") { + return 1; + } + const numBits = group.weights?.num_bits ?? 4; + return Math.max(1, Math.floor(32 / numBits)); + } + if (quantConfig.format === "pack-quantized") { + const numBits = groups.find((g) => g.weights?.num_bits)?.weights?.num_bits ?? 4; + return Math.max(1, Math.floor(32 / numBits)); + } + } + return 1; + case "bitsandbytes": + if (quantConfig.load_in_4bit && dtype === "U8") { + return 2; + } + return 1; + default: + if (dtype === "U8" && (quantConfig.load_in_4bit || quantConfig.bits === 4)) { + return 2; + } + return 1; + } +} +function computeNumOfParamsByDtypeSingleFile(header, quantConfig) { + const counter = {}; + const tensors = omit(header, "__metadata__"); + for (const [tensorName, v] of typedEntries(tensors)) { + if (shouldSkipTensor(tensorName, quantConfig)) { + continue; + } + if (v.shape.length === 0) { + continue; + } + const elements = v.shape.reduce((a, b) => a * b); + if (!Number.isFinite(elements)) { + continue; + } + const multiplier = quantConfig ? getQuantizationMultiplier(tensorName, v.dtype, quantConfig) : 1; + if (multiplier === 0) { + continue; + } + counter[v.dtype] = (counter[v.dtype] ?? 0) + elements * multiplier; + } + return counter; +} +function computeNumOfParamsByDtypeSharded(shardedMap, quantConfig) { + const counter = {}; + for (const header of Object.values(shardedMap)) { + for (const [k, v] of typedEntries(computeNumOfParamsByDtypeSingleFile(header, quantConfig))) { + counter[k] = (counter[k] ?? 0) + (v ?? 0); + } + } + return counter; +} +function getTensorSuffix(tensorName) { + const lastDotIndex = tensorName.lastIndexOf("."); + return lastDotIndex === -1 ? tensorName : tensorName.slice(lastDotIndex + 1); +} +function shouldSkipTensor(tensorName, quantConfig) { + if (!quantConfig) { + return false; + } + const quantMethod = quantConfig.quant_method?.toLowerCase(); + if (quantMethod !== "gptq" && quantMethod !== "awq") { + return false; + } + if (!isQuantizedTensor(tensorName, quantConfig)) { + return false; + } + const suffix = getTensorSuffix(tensorName); + return suffix !== GPTQ_QWEIGHT_SUFFIX && GPTQ_AWQ_AUXILIARY_SUFFIXES.includes(suffix); +} + +// src/lib/repo-exists.ts +async function repoExists(params) { + const repoId = toRepoId(params.repo); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}?expand[]=likes`, + { + method: "GET", + headers: { + ...params.accessToken && { + Authorization: `Bearer ${params.accessToken}` + } + } + } + ); + if (res.status === 404 || res.status === 401) { + return false; + } + if (!res.ok) { + throw await createApiError(res); + } + return true; +} + +// src/lib/space-info.ts +async function spaceInfo(params) { + const accessToken = params && checkCredentials(params); + const search = new URLSearchParams([ + ...SPACE_EXPAND_KEYS.map((val) => ["expand", val]), + ...params?.additionalFields?.map((val) => ["expand", val]) ?? [] + ]).toString(); + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/spaces/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD" + )}?${search.toString()}`, + { + headers: { + ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} + } + } + ); + if (!response.ok) { + throw await createApiError(response); + } + const data = await response.json(); + return { + ...params?.additionalFields && pick(data, params.additionalFields), + id: data._id, + name: data.id, + sdk: data.sdk, + likes: data.likes, + private: data.private, + updatedAt: new Date(data.lastModified) + }; +} + +// src/lib/snapshot-download.ts +var import_node_path3 = require("path"); +var import_promises6 = require("fs/promises"); +var DEFAULT_REVISION = "main"; +async function snapshotDownload(params) { + let cacheDir; + if (params.cacheDir) { + cacheDir = params.cacheDir; + } else { + cacheDir = getHFHubCachePath(); + } + let revision; + if (params.revision) { + revision = params.revision; + } else { + revision = DEFAULT_REVISION; + } + const repoId = toRepoId(params.repo); + const storageFolder = (0, import_node_path3.join)(cacheDir, getRepoFolderName(repoId)); + let repoInfo; + switch (repoId.type) { + case "space": + repoInfo = await spaceInfo({ + ...params, + name: repoId.name, + additionalFields: ["sha"], + revision + }); + break; + case "dataset": + repoInfo = await datasetInfo({ + ...params, + name: repoId.name, + additionalFields: ["sha"], + revision + }); + break; + case "model": + repoInfo = await modelInfo({ + ...params, + name: repoId.name, + additionalFields: ["sha"], + revision + }); + break; + default: + throw new Error( + `Unsupported repository type: ${repoId.type}. snapshotDownload is not supported for bucket repos.` + ); + } + const commitHash = repoInfo.sha; + if (revision !== commitHash) { + const refPath = (0, import_node_path3.join)(storageFolder, "refs", revision); + await (0, import_promises6.mkdir)((0, import_node_path3.dirname)(refPath), { recursive: true }); + await (0, import_promises6.writeFile)(refPath, commitHash); + } + const snapshotFolder = (0, import_node_path3.join)(storageFolder, "snapshots", commitHash); + const cursor = listFiles({ + ...params, + repo: params.repo, + recursive: true, + revision: commitHash + }); + for await (const entry of cursor) { + switch (entry.type) { + case "file": + await downloadFileToCacheDir({ + ...params, + path: entry.path, + revision: commitHash, + cacheDir + }); + break; + case "directory": + await (0, import_promises6.mkdir)((0, import_node_path3.join)(snapshotFolder, entry.path), { recursive: true }); + break; + default: + throw new Error(`unknown entry type: ${entry.type}`); + } + } + return snapshotFolder; +} + +// src/lib/upload-file.ts +function uploadFile(params) { + const path2 = params.file instanceof URL ? params.file.pathname.split("/").at(-1) ?? "file" : "path" in params.file ? params.file.path : params.file.name; + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: [ + { + operation: "addOrUpdate", + path: path2, + content: "content" in params.file ? params.file.content : params.file + } + ], + title: params.commitTitle ?? `Add ${path2}`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet + }); +} + +// src/lib/upload-files.ts +function uploadFiles(params) { + return commit({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: params.files.map((file) => ({ + operation: "addOrUpdate", + path: file instanceof URL ? file.pathname.split("/").at(-1) ?? "file" : "path" in file ? file.path : file.name, + content: "content" in file ? file.content : file + })), + title: params.commitTitle ?? `Add ${params.files.length} files`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet + }); +} + +// src/lib/upload-files-with-progress.ts +var multipartUploadTracking = /* @__PURE__ */ new WeakMap(); +async function* uploadFilesWithProgress(params) { + return yield* commitIter({ + ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, + repo: params.repo, + operations: params.files.map((file) => ({ + operation: "addOrUpdate", + path: file instanceof URL ? file.pathname.split("/").at(-1) ?? "file" : "path" in file ? file.path : file.name, + content: "content" in file ? file.content : file + })), + title: params.commitTitle ?? `Add ${params.files.length} files`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet, + fetch: async (input, init) => { + if (!init) { + return fetch(input); + } + if (!typedInclude(["PUT", "POST"], init.method) || !("progressHint" in init) || !init.progressHint || typeof XMLHttpRequest === "undefined" || typeof input !== "string" || !(init.body instanceof ArrayBuffer) && !(init.body instanceof Blob) && !(init.body instanceof File) && typeof init.body !== "string") { + return fetch(input, init); + } + const progressHint = init.progressHint; + const progressCallback = progressHint.progressCallback; + const xhr = new XMLHttpRequest(); + xhr.upload.addEventListener("progress", (event) => { + if (event.lengthComputable) { + if (progressHint.part !== void 0) { + let tracking = multipartUploadTracking.get(progressCallback); + if (!tracking) { + tracking = { numParts: progressHint.numParts, partsProgress: {} }; + multipartUploadTracking.set(progressCallback, tracking); + } + tracking.partsProgress[progressHint.part] = event.loaded / event.total; + let totalProgress = 0; + for (const partProgress of Object.values(tracking.partsProgress)) { + totalProgress += partProgress; + } + if (totalProgress === tracking.numParts) { + progressCallback(0.9999999999); + } else { + progressCallback(totalProgress / tracking.numParts); + } + } else { + if (event.loaded === event.total) { + progressCallback(0.9999999999); + } else { + progressCallback(event.loaded / event.total); + } + } + } + }); + xhr.open(init.method, input, true); + if (init.headers) { + const headers = new Headers(init.headers); + headers.forEach((value, key) => { + xhr.setRequestHeader(key, value); + }); + } + init.signal?.throwIfAborted(); + xhr.send(init.body); + return new Promise((resolve3, reject) => { + xhr.addEventListener("load", () => { + resolve3( + new Response(xhr.responseText, { + status: xhr.status, + statusText: xhr.statusText, + headers: Object.fromEntries( + xhr.getAllResponseHeaders().trim().split("\n").map((header) => [ + header.slice(0, header.indexOf(":")), + header.slice(header.indexOf(":") + 1).trim() + ]) + ) + }) + ); + }); + xhr.addEventListener("error", () => { + reject(new Error(xhr.statusText)); + }); + if (init.signal) { + init.signal.addEventListener("abort", () => { + xhr.abort(); + try { + init.signal?.throwIfAborted(); + } catch (err) { + reject(err); + } + }); + } + }); + } + }); +} + +// src/lib/who-am-i.ts +async function whoAmI(params) { + const accessToken = checkCredentials(params); + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/whoami-v2`, { + headers: { + Authorization: `Bearer ${accessToken}` + } + }); + if (!res.ok) { + throw await createApiError(res); + } + const response = await res.json(); + if (typeof response.auth.accessToken?.createdAt === "string") { + response.auth.accessToken.createdAt = new Date(response.auth.accessToken.createdAt); + } + return response; +} +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + DATASET_EXPANDABLE_KEYS, + DATASET_EXPAND_KEYS, + DEFAULT_REVISION, + HUB_URL, + HubApiError, + InvalidApiResponseFormatError, + MODEL_DERIVED_FIELD_TO_API_KEY, + MODEL_EXPANDABLE_KEYS, + MODEL_EXPAND_KEYS, + REGEX_COMMIT_HASH, + REPO_ID_SEPARATOR, + RE_SAFETENSORS_FILE, + RE_SAFETENSORS_INDEX_FILE, + RE_SAFETENSORS_SHARD_FILE, + SAFETENSORS_FILE, + SAFETENSORS_INDEX_FILE, + SPACE_EXPANDABLE_KEYS, + SPACE_EXPAND_KEYS, + __internal_XetBlob, + __internal_sha256, + cancelJob, + checkRepoAccess, + commit, + commitIter, + commitIterBucket, + copyFile, + copyFileIter, + copyFiles, + copyFilesIter, + copyFolder, + copyFolderIter, + countCommits, + createBranch, + createCollection, + createRepo, + createScheduledJob, + datasetInfo, + deleteBranch, + deleteCollection, + deleteFile, + deleteFiles, + deleteRepo, + deleteScheduledJob, + downloadFile, + downloadFileToCacheDir, + duplicateJob, + fileDownloadInfo, + fileExists, + getBlobStat, + getHFHubCachePath, + getJob, + getRepoFolderName, + getScheduledJob, + globMatch, + isQuantizedTensor, + listCollections, + listCommits, + listDatasets, + listFiles, + listJobHardware, + listJobs, + listModels, + listScheduledJobs, + listSpaces, + matchesCompressedTensorsTarget, + modelInfo, + oauthHandleRedirect, + oauthHandleRedirectIfPresent, + oauthLoginUrl, + parseRepoType, + parseSafetensorsMetadata, + parseSafetensorsShardFilename, + pathsInfo, + relativeUnderFolder, + repoExists, + resumeScheduledJob, + runJob, + runScheduledJob, + scanCacheDir, + scanCachedRepo, + scanRefsDir, + scanSnapshotDir, + snapshotDownload, + spaceInfo, + streamJobEvents, + streamJobLogs, + streamJobMetrics, + suspendScheduledJob, + uploadFile, + uploadFiles, + uploadFilesWithProgress, + whoAmI +}); diff --git a/node_modules/@huggingface/hub/dist/index.mjs b/node_modules/@huggingface/hub/dist/index.mjs new file mode 100644 index 0000000000000000000000000000000000000000..4f7428f3c9c7c129c3540d898f06d9bd270ddfd8 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/index.mjs @@ -0,0 +1,189 @@ +import { + DATASET_EXPANDABLE_KEYS, + DATASET_EXPAND_KEYS, + DEFAULT_REVISION, + HUB_URL, + HubApiError, + InvalidApiResponseFormatError, + MODEL_DERIVED_FIELD_TO_API_KEY, + MODEL_EXPANDABLE_KEYS, + MODEL_EXPAND_KEYS, + REGEX_COMMIT_HASH, + REPO_ID_SEPARATOR, + RE_SAFETENSORS_FILE, + RE_SAFETENSORS_INDEX_FILE, + RE_SAFETENSORS_SHARD_FILE, + SAFETENSORS_FILE, + SAFETENSORS_INDEX_FILE, + SPACE_EXPANDABLE_KEYS, + SPACE_EXPAND_KEYS, + XetBlob, + cancelJob, + checkRepoAccess, + commit, + commitIter, + commitIterBucket, + copyFile, + copyFileIter, + copyFiles, + copyFilesIter, + copyFolder, + copyFolderIter, + countCommits, + createBranch, + createCollection, + createRepo, + createScheduledJob, + datasetInfo, + deleteBranch, + deleteCollection, + deleteFile, + deleteFiles, + deleteRepo, + deleteScheduledJob, + downloadFile, + downloadFileToCacheDir, + duplicateJob, + fileDownloadInfo, + fileExists, + getBlobStat, + getHFHubCachePath, + getJob, + getRepoFolderName, + getScheduledJob, + globMatch, + isQuantizedTensor, + listCollections, + listCommits, + listDatasets, + listFiles, + listJobHardware, + listJobs, + listModels, + listScheduledJobs, + listSpaces, + matchesCompressedTensorsTarget, + modelInfo, + oauthHandleRedirect, + oauthHandleRedirectIfPresent, + oauthLoginUrl, + parseRepoType, + parseSafetensorsMetadata, + parseSafetensorsShardFilename, + pathsInfo, + relativeUnderFolder, + repoExists, + resumeScheduledJob, + runJob, + runScheduledJob, + scanCacheDir, + scanCachedRepo, + scanRefsDir, + scanSnapshotDir, + sha256, + snapshotDownload, + spaceInfo, + streamJobEvents, + streamJobLogs, + streamJobMetrics, + suspendScheduledJob, + uploadFile, + uploadFiles, + uploadFilesWithProgress, + whoAmI +} from "./chunk-OPQ3EOKY.mjs"; +import "./chunk-FFYIGW52.mjs"; +export { + DATASET_EXPANDABLE_KEYS, + DATASET_EXPAND_KEYS, + DEFAULT_REVISION, + HUB_URL, + HubApiError, + InvalidApiResponseFormatError, + MODEL_DERIVED_FIELD_TO_API_KEY, + MODEL_EXPANDABLE_KEYS, + MODEL_EXPAND_KEYS, + REGEX_COMMIT_HASH, + REPO_ID_SEPARATOR, + RE_SAFETENSORS_FILE, + RE_SAFETENSORS_INDEX_FILE, + RE_SAFETENSORS_SHARD_FILE, + SAFETENSORS_FILE, + SAFETENSORS_INDEX_FILE, + SPACE_EXPANDABLE_KEYS, + SPACE_EXPAND_KEYS, + XetBlob as __internal_XetBlob, + sha256 as __internal_sha256, + cancelJob, + checkRepoAccess, + commit, + commitIter, + commitIterBucket, + copyFile, + copyFileIter, + copyFiles, + copyFilesIter, + copyFolder, + copyFolderIter, + countCommits, + createBranch, + createCollection, + createRepo, + createScheduledJob, + datasetInfo, + deleteBranch, + deleteCollection, + deleteFile, + deleteFiles, + deleteRepo, + deleteScheduledJob, + downloadFile, + downloadFileToCacheDir, + duplicateJob, + fileDownloadInfo, + fileExists, + getBlobStat, + getHFHubCachePath, + getJob, + getRepoFolderName, + getScheduledJob, + globMatch, + isQuantizedTensor, + listCollections, + listCommits, + listDatasets, + listFiles, + listJobHardware, + listJobs, + listModels, + listScheduledJobs, + listSpaces, + matchesCompressedTensorsTarget, + modelInfo, + oauthHandleRedirect, + oauthHandleRedirectIfPresent, + oauthLoginUrl, + parseRepoType, + parseSafetensorsMetadata, + parseSafetensorsShardFilename, + pathsInfo, + relativeUnderFolder, + repoExists, + resumeScheduledJob, + runJob, + runScheduledJob, + scanCacheDir, + scanCachedRepo, + scanRefsDir, + scanSnapshotDir, + snapshotDownload, + spaceInfo, + streamJobEvents, + streamJobLogs, + streamJobMetrics, + suspendScheduledJob, + uploadFile, + uploadFiles, + uploadFilesWithProgress, + whoAmI +}; diff --git a/node_modules/@huggingface/hub/dist/sha256-node-ZPWO3OWR.mjs b/node_modules/@huggingface/hub/dist/sha256-node-ZPWO3OWR.mjs new file mode 100644 index 0000000000000000000000000000000000000000..a6e23fac7dd00390723c690334f53397ff7fe2f7 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/sha256-node-ZPWO3OWR.mjs @@ -0,0 +1,21 @@ +import "./chunk-FFYIGW52.mjs"; + +// src/utils/sha256-node.ts +import { Readable } from "stream"; +import { createHash } from "crypto"; +async function* sha256Node(buffer, opts) { + const sha256Stream = createHash("sha256"); + const size = buffer instanceof Blob ? buffer.size : buffer.byteLength; + let done = 0; + const readable = buffer instanceof Blob ? Readable.fromWeb(buffer.stream()) : Readable.from(Buffer.from(buffer)); + for await (const buffer2 of readable) { + sha256Stream.update(buffer2); + done += buffer2.length; + yield done / size; + opts?.abortSignal?.throwIfAborted(); + } + return sha256Stream.digest("hex"); +} +export { + sha256Node +}; diff --git a/node_modules/@huggingface/hub/dist/sha256-wrapper-ITDNMKRK.mjs b/node_modules/@huggingface/hub/dist/sha256-wrapper-ITDNMKRK.mjs new file mode 100644 index 0000000000000000000000000000000000000000..9ff354726365047f05d48ed04a08577d48a955ce --- /dev/null +++ b/node_modules/@huggingface/hub/dist/sha256-wrapper-ITDNMKRK.mjs @@ -0,0 +1,460 @@ +import "./chunk-FFYIGW52.mjs"; + +// src/vendor/hash-wasm/sha256.js +var Module = (() => { + var _unused = import.meta.url; + return function(moduleArg = {}) { + var Module2 = moduleArg; + var readyPromiseResolve, readyPromiseReject; + Module2["ready"] = new Promise((resolve, reject) => { + readyPromiseResolve = resolve; + readyPromiseReject = reject; + }); + var moduleOverrides = Object.assign({}, Module2); + var arguments_ = []; + var thisProgram = "./this.program"; + var quit_ = (status, toThrow) => { + throw toThrow; + }; + var ENVIRONMENT_IS_WEB = typeof window == "object"; + var ENVIRONMENT_IS_WORKER = typeof importScripts == "function"; + var ENVIRONMENT_IS_NODE = typeof process == "object" && typeof process.versions == "object" && typeof process.versions.node == "string"; + var ENVIRONMENT_IS_SHELL = !ENVIRONMENT_IS_WEB && !ENVIRONMENT_IS_NODE && !ENVIRONMENT_IS_WORKER; + var scriptDirectory = ""; + function locateFile(path) { + if (Module2["locateFile"]) { + return Module2["locateFile"](path, scriptDirectory); + } + return scriptDirectory + path; + } + var read_, readAsync, readBinary; + if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { + if (ENVIRONMENT_IS_WORKER) { + scriptDirectory = self.location.href; + } else if (typeof document != "undefined" && document.currentScript) { + scriptDirectory = document.currentScript.src; + } + if (false) { + scriptDirectory = false; + } + if (scriptDirectory.startsWith("blob:")) { + scriptDirectory = ""; + } else { + scriptDirectory = scriptDirectory.substr(0, scriptDirectory.replace(/[?#].*/, "").lastIndexOf("/") + 1); + } + { + read_ = (url) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, false); + xhr.send(null); + return xhr.responseText; + }; + if (ENVIRONMENT_IS_WORKER) { + readBinary = (url) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, false); + xhr.responseType = "arraybuffer"; + xhr.send(null); + return new Uint8Array( + /** @type{!ArrayBuffer} */ + xhr.response + ); + }; + } + readAsync = (url, onload, onerror) => { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, true); + xhr.responseType = "arraybuffer"; + xhr.onload = () => { + if (xhr.status == 200 || xhr.status == 0 && xhr.response) { + onload(xhr.response); + return; + } + onerror(); + }; + xhr.onerror = onerror; + xhr.send(null); + }; + } + } else { + } + var out = Module2["print"] || console.log.bind(console); + var err = Module2["printErr"] || console.error.bind(console); + Object.assign(Module2, moduleOverrides); + moduleOverrides = null; + if (Module2["arguments"]) + arguments_ = Module2["arguments"]; + if (Module2["thisProgram"]) + thisProgram = Module2["thisProgram"]; + if (Module2["quit"]) + quit_ = Module2["quit"]; + var wasmBinary; + if (Module2["wasmBinary"]) + wasmBinary = Module2["wasmBinary"]; + if (typeof WebAssembly != "object") { + abort("no native wasm support detected"); + } + function intArrayFromBase64(s) { + var decoded = atob(s); + var bytes = new Uint8Array(decoded.length); + for (var i = 0; i < decoded.length; ++i) { + bytes[i] = decoded.charCodeAt(i); + } + return bytes; + } + function tryParseAsDataURI(filename) { + if (!isDataURI(filename)) { + return; + } + return intArrayFromBase64(filename.slice(dataURIPrefix.length)); + } + var wasmMemory; + var ABORT = false; + var EXITSTATUS; + function assert(condition, text) { + if (!condition) { + abort(text); + } + } + var HEAP, HEAP8, HEAPU8, HEAP16, HEAPU16, HEAP32, HEAPU32, HEAPF32, HEAPF64; + function updateMemoryViews() { + var b = wasmMemory.buffer; + Module2["HEAP8"] = HEAP8 = new Int8Array(b); + Module2["HEAP16"] = HEAP16 = new Int16Array(b); + Module2["HEAPU8"] = HEAPU8 = new Uint8Array(b); + Module2["HEAPU16"] = HEAPU16 = new Uint16Array(b); + Module2["HEAP32"] = HEAP32 = new Int32Array(b); + Module2["HEAPU32"] = HEAPU32 = new Uint32Array(b); + Module2["HEAPF32"] = HEAPF32 = new Float32Array(b); + Module2["HEAPF64"] = HEAPF64 = new Float64Array(b); + } + var __ATPRERUN__ = []; + var __ATINIT__ = []; + var __ATEXIT__ = []; + var __ATPOSTRUN__ = []; + var runtimeInitialized = false; + function preRun() { + if (Module2["preRun"]) { + if (typeof Module2["preRun"] == "function") + Module2["preRun"] = [Module2["preRun"]]; + while (Module2["preRun"].length) { + addOnPreRun(Module2["preRun"].shift()); + } + } + callRuntimeCallbacks(__ATPRERUN__); + } + function initRuntime() { + runtimeInitialized = true; + callRuntimeCallbacks(__ATINIT__); + } + function postRun() { + if (Module2["postRun"]) { + if (typeof Module2["postRun"] == "function") + Module2["postRun"] = [Module2["postRun"]]; + while (Module2["postRun"].length) { + addOnPostRun(Module2["postRun"].shift()); + } + } + callRuntimeCallbacks(__ATPOSTRUN__); + } + function addOnPreRun(cb) { + __ATPRERUN__.unshift(cb); + } + function addOnInit(cb) { + __ATINIT__.unshift(cb); + } + function addOnExit(cb) { + } + function addOnPostRun(cb) { + __ATPOSTRUN__.unshift(cb); + } + var runDependencies = 0; + var runDependencyWatcher = null; + var dependenciesFulfilled = null; + function getUniqueRunDependency(id) { + return id; + } + function addRunDependency(id) { + runDependencies++; + Module2["monitorRunDependencies"]?.(runDependencies); + } + function removeRunDependency(id) { + runDependencies--; + Module2["monitorRunDependencies"]?.(runDependencies); + if (runDependencies == 0) { + if (runDependencyWatcher !== null) { + clearInterval(runDependencyWatcher); + runDependencyWatcher = null; + } + if (dependenciesFulfilled) { + var callback = dependenciesFulfilled; + dependenciesFulfilled = null; + callback(); + } + } + } + function abort(what) { + Module2["onAbort"]?.(what); + what = "Aborted(" + what + ")"; + err(what); + ABORT = true; + EXITSTATUS = 1; + what += ". Build with -sASSERTIONS for more info."; + var e = new WebAssembly.RuntimeError(what); + readyPromiseReject(e); + throw e; + } + var dataURIPrefix = "data:application/octet-stream;base64,"; + var isDataURI = (filename) => filename.startsWith(dataURIPrefix); + var isFileURI = (filename) => filename.startsWith("file://"); + var wasmBinaryFile; + wasmBinaryFile = "data:application/octet-stream;base64,AGFzbQEAAAABHQZgAX8AYAABf2AAAGABfwF/YAJ/fwBgA39/fwF/Aw0MAgAEAgMBBQABAQADBAUBcAEBAQUGAQGAAoACBg4CfwFB8IuEBAt/AUEACweYAQoGbWVtb3J5AgARX193YXNtX2NhbGxfY3RvcnMAAAtIYXNoX1VwZGF0ZQABCkhhc2hfRmluYWwAAwlIYXNoX0luaXQABAxHZXRCdWZmZXJQdHIABRlfX2luZGlyZWN0X2Z1bmN0aW9uX3RhYmxlAQAJc3RhY2tTYXZlAAkMc3RhY2tSZXN0b3JlAAoKc3RhY2tBbGxvYwALCossDAIAC+4CAgV/AX5BACgCwAoiASABKQNAIgYgAK18NwNAAkACQAJAIAanQT9xIgINAEGACyEBIAAhAgwBC0HAACACayEDAkAgAEUNACADIAAgAyAASRshBCABIAJqIQVBACEBA0AgBSABIgFqQYALIAFqLQAAOgAAIAFBAWoiAiEBIAIgBEcNAAsLAkACQCAAIANJIgRFDQBBgAshASAAIQIMAQtBACgCwAoiAUHIAGogARACQYALIANqIQEgACADayECCyABIQEgAiECIAQNAQsgASEBAkACQCACIgJBwABPDQAgASEFIAIhAAwBCyACIQIgASEEA0BBACgCwApByABqIAQiBBACIAJBQGoiASECIARBwABqIgUhBCAFIQUgASEAIAFBP0sNAAsLIAUhBSAAIgBFDQBBACEBQQAhAgNAQQAoAsAKIAEiAWogBSABai0AADoAACACQQFqIgJB/wFxIgQhASACIQIgACAESw0ACwsLqCEBK38gACgCCCICIAAoAgQiAyAAKAIAIgRzcSADIARxcyAEQR53IARBE3dzIARBCndzaiAAKAIQIgVBGncgBUEVd3MgBUEHd3MgACgCHCIGaiAAKAIYIgcgACgCFCIIcyAFcSAHc2ogASgCACIJQRh0IAlBgP4DcUEIdHIgCUEIdkGA/gNxIAlBGHZyciIKakGY36iUBGoiC2oiCSAEcyADcSAJIARxcyAJQR53IAlBE3dzIAlBCndzaiAHIAEoAgQiDEEYdCAMQYD+A3FBCHRyIAxBCHZBgP4DcSAMQRh2cnIiDWogCyAAKAIMIg5qIg8gCCAFc3EgCHNqIA9BGncgD0EVd3MgD0EHd3NqQZGJ3YkHaiIQaiIMIAlzIARxIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAggASgCCCILQRh0IAtBgP4DcUEIdHIgC0EIdkGA/gNxIAtBGHZyciIRaiAQIAJqIhIgDyAFc3EgBXNqIBJBGncgEkEVd3MgEkEHd3NqQc/3g657aiITaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAUgASgCDCIQQRh0IBBBgP4DcUEIdHIgEEEIdkGA/gNxIBBBGHZyciIUaiATIANqIhMgEiAPc3EgD3NqIBNBGncgE0EVd3MgE0EHd3NqQaW3181+aiIVaiIQIAtzIAxxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIA8gASgCECIWQRh0IBZBgP4DcUEIdHIgFkEIdkGA/gNxIBZBGHZyciIXaiAVIARqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQduE28oDaiIYaiIPIBBzIAtxIA8gEHFzIA9BHncgD0ETd3MgD0EKd3NqIAEoAhQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGSASaiAYIAlqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQfGjxM8FaiIYaiIJIA9zIBBxIAkgD3FzIAlBHncgCUETd3MgCUEKd3NqIAEoAhgiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGiATaiAYIAxqIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQaSF/pF5aiIYaiIMIAlzIA9xIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAEoAhwiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGyAWaiAYIAtqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQdW98dh6aiIYaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAEoAiAiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHCASaiAYIBBqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQZjVnsB9aiIYaiIQIAtzIAxxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAiQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHSATaiAYIA9qIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQYG2jZQBaiIYaiIPIBBzIAtxIA8gEHFzIA9BHncgD0ETd3MgD0EKd3NqIAEoAigiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHiAWaiAYIAlqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQb6LxqECaiIYaiIJIA9zIBBxIAkgD3FzIAlBHncgCUETd3MgCUEKd3NqIAEoAiwiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHyASaiAYIAxqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQcP7sagFaiIYaiIMIAlzIA9xIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAEoAjAiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiICATaiAYIAtqIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQfS6+ZUHaiIYaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAEoAjQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiISAWaiAYIBBqIhAgEyASc3EgEnNqIBBBGncgEEEVd3MgEEEHd3NqQf7j+oZ4aiIYaiIWIAtzIAxxIBYgC3FzIBZBHncgFkETd3MgFkEKd3NqIAEoAjgiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiIiASaiAYIA9qIg8gECATc3EgE3NqIA9BGncgD0EVd3MgD0EHd3NqQaeN8N55aiIVaiISIBZzIAtxIBIgFnFzIBJBHncgEkETd3MgEkEKd3NqIAEoAjwiAUEYdCABQYD+A3FBCHRyIAFBCHZBgP4DcSABQRh2cnIiIyATaiAVIAlqIgEgDyAQc3EgEHNqIAFBGncgAUEVd3MgAUEHd3NqQfTi74x8aiIJaiEVIBIhGCAWISQgCyElIAkgDGohJiABIScgDyEoIBAhKSAjISMgIiEiICEhISAgISAgHyEfIB4hHiAdIR0gHCEcIBshGyAaIRogGSEZIBchFyAUIRQgESERIA0hECAKIQxBgAkhAUEQISoDQCAVIgkgGCIKcyAkIitxIAkgCnFzIAlBHncgCUETd3MgCUEKd3NqIBAiEEEZdyAQQQ53cyAQQQN2cyAMaiAdIh1qICIiFkEPdyAWQQ13cyAWQQp2c2oiDCApaiAmIhIgJyIPICgiE3NxIBNzaiASQRp3IBJBFXdzIBJBB3dzaiABIgEoAgBqIiRqIgsgCXMgCnEgCyAJcXMgC0EedyALQRN3cyALQQp3c2ogESIYQRl3IBhBDndzIBhBA3ZzIBBqIB4iHmogIyIVQQ93IBVBDXdzIBVBCnZzaiINIBNqIAEoAgRqICQgJWoiEyASIA9zcSAPc2ogE0EadyATQRV3cyATQQd3c2oiJWoiECALcyAJcSAQIAtxcyAQQR53IBBBE3dzIBBBCndzaiAUIiRBGXcgJEEOd3MgJEEDdnMgGGogHyIfaiAMQQ93IAxBDXdzIAxBCnZzaiIRIA9qIAEoAghqICUgK2oiGCATIBJzcSASc2ogGEEadyAYQRV3cyAYQQd3c2oiJWoiDyAQcyALcSAPIBBxcyAPQR53IA9BE3dzIA9BCndzaiAXIhdBGXcgF0EOd3MgF0EDdnMgJGogICIgaiANQQ93IA1BDXdzIA1BCnZzaiIUIBJqIAEoAgxqICUgCmoiCiAYIBNzcSATc2ogCkEadyAKQRV3cyAKQQd3c2oiJWoiEiAPcyAQcSASIA9xcyASQR53IBJBE3dzIBJBCndzaiATIBkiJEEZdyAkQQ53cyAkQQN2cyAXaiAhIiFqIBFBD3cgEUENd3MgEUEKdnNqIhdqIAEoAhBqICUgCWoiEyAKIBhzcSAYc2ogE0EadyATQRV3cyATQQd3c2oiJWoiCSAScyAPcSAJIBJxcyAJQR53IAlBE3dzIAlBCndzaiABKAIUIBoiGkEZdyAaQQ53cyAaQQN2cyAkaiAWaiAUQQ93IBRBDXdzIBRBCnZzaiIZaiAYaiAlIAtqIhggEyAKc3EgCnNqIBhBGncgGEEVd3MgGEEHd3NqIiVqIgsgCXMgEnEgCyAJcXMgC0EedyALQRN3cyALQQp3c2ogASgCGCAbIiRBGXcgJEEOd3MgJEEDdnMgGmogFWogF0EPdyAXQQ13cyAXQQp2c2oiGmogCmogJSAQaiIKIBggE3NxIBNzaiAKQRp3IApBFXdzIApBB3dzaiIlaiIQIAtzIAlxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAhwgHCIcQRl3IBxBDndzIBxBA3ZzICRqIAxqIBlBD3cgGUENd3MgGUEKdnNqIhtqIBNqICUgD2oiJCAKIBhzcSAYc2ogJEEadyAkQRV3cyAkQQd3c2oiE2oiDyAQcyALcSAPIBBxcyAPQR53IA9BE3dzIA9BCndzaiABKAIgIB1BGXcgHUEOd3MgHUEDdnMgHGogDWogGkEPdyAaQQ13cyAaQQp2c2oiHGogGGogEyASaiIYICQgCnNxIApzaiAYQRp3IBhBFXdzIBhBB3dzaiITaiISIA9zIBBxIBIgD3FzIBJBHncgEkETd3MgEkEKd3NqIAEoAiQgHkEZdyAeQQ53cyAeQQN2cyAdaiARaiAbQQ93IBtBDXdzIBtBCnZzaiIdaiAKaiATIAlqIgkgGCAkc3EgJHNqIAlBGncgCUEVd3MgCUEHd3NqIgpqIhMgEnMgD3EgEyAScXMgE0EedyATQRN3cyATQQp3c2ogASgCKCAfQRl3IB9BDndzIB9BA3ZzIB5qIBRqIBxBD3cgHEENd3MgHEEKdnNqIh5qICRqIAogC2oiCiAJIBhzcSAYc2ogCkEadyAKQRV3cyAKQQd3c2oiJGoiCyATcyAScSALIBNxcyALQR53IAtBE3dzIAtBCndzaiABKAIsICBBGXcgIEEOd3MgIEEDdnMgH2ogF2ogHUEPdyAdQQ13cyAdQQp2c2oiH2ogGGogJCAQaiIYIAogCXNxIAlzaiAYQRp3IBhBFXdzIBhBB3dzaiIkaiIQIAtzIBNxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAjAgIUEZdyAhQQ53cyAhQQN2cyAgaiAZaiAeQQ93IB5BDXdzIB5BCnZzaiIgaiAJaiAkIA9qIiQgGCAKc3EgCnNqICRBGncgJEEVd3MgJEEHd3NqIg9qIgkgEHMgC3EgCSAQcXMgCUEedyAJQRN3cyAJQQp3c2ogASgCNCAWQRl3IBZBDndzIBZBA3ZzICFqIBpqIB9BD3cgH0ENd3MgH0EKdnNqIiFqIApqIA8gEmoiDyAkIBhzcSAYc2ogD0EadyAPQRV3cyAPQQd3c2oiCmoiEiAJcyAQcSASIAlxcyASQR53IBJBE3dzIBJBCndzaiABKAI4IBVBGXcgFUEOd3MgFUEDdnMgFmogG2ogIEEPdyAgQQ13cyAgQQp2c2oiImogGGogCiATaiITIA8gJHNxICRzaiATQRp3IBNBFXdzIBNBB3dzaiIYaiIWIBJzIAlxIBYgEnFzIBZBHncgFkETd3MgFkEKd3NqIAEoAjwgDEEZdyAMQQ53cyAMQQN2cyAVaiAcaiAhQQ93ICFBDXdzICFBCnZzaiIKaiAkaiAYIAtqIgsgEyAPc3EgD3NqIAtBGncgC0EVd3MgC0EHd3NqIiZqIishFSAWIRggEiEkIAkhJSAmIBBqIiwhJiALIScgEyEoIA8hKSAKISMgIiEiICEhISAgISAgHyEfIB4hHiAdIR0gHCEcIBshGyAaIRogGSEZIBchFyAUIRQgESERIA0hECAMIQwgAUHAAGohASAqIgpBEGohKiAKQTBJDQALIAAgDyAGajYCHCAAIBMgB2o2AhggACALIAhqNgIUIAAgLCAFajYCECAAIAkgDmo2AgwgACASIAJqNgIIIAAgFiADajYCBCAAICsgBGo2AgAL1AMDBX8BfgF7QQAoAsAKIgAgACgCQCIBQQJ2QQ9xIgJBAnRqIgMgAygCAEF/IAFBA3QiAXRBf3NxQYABIAF0czYCAAJAAkAgAkEOTw0AIAJBAWohAAwBCwJAIAJBDkcNACAAQQA2AjwLIABByABqIAAQAkEAIQALAkAgACIAQQ1LDQBBACgCwAogAEECdCIAakEAQTggAGsQBhoLQQAoAsAKIgAgACkDQCIFpyICQRt0IAJBC3RBgID8B3FyIAJBBXZBgP4DcSACQQN0QRh2cnI2AjwgACAFQh2IpyICQRh0IAJBgP4DcUEIdHIgAkEIdkGA/gNxIAJBGHZycjYCOCAAQcgAaiAAEAJBACgCwApBPGohAUEAIQADQCABQQcgACIAa0ECdGoiAiAC/QACACAG/Q0MDQ4PCAkKCwQFBgcAAQIDIAb9DQMCAQAHBgUECwoJCA8ODQwgBv0NDA0ODwgJCgsEBQYHAAECA/0LAgAgAEEEaiICIQAgAkEIRw0ACwJAQQAoAsAKIgMoAmhFDQAgA0HIAGohBEEAIQBBACECA0BBgAsgACIAaiAEIABqLQAAOgAAIAJBAWoiAkH/AXEiASEAIAIhAiADKAJoIAFLDQALCwtxAQJ/QQAoAsAKIgFCADcDQCABQcgAaiECAkAgAEHgAUcNACABQRw2AmggAkEQakEA/QAEsAj9CwIAIAJBAP0ABKAI/QsCAEEADwsgAUEgNgJoIAJBEGpBAP0ABJAI/QsCACACQQD9AASACP0LAgBBAAsFAEGACwvyAgIDfwF+AkAgAkUNACAAIAE6AAAgACACaiIDQX9qIAE6AAAgAkEDSQ0AIAAgAToAAiAAIAE6AAEgA0F9aiABOgAAIANBfmogAToAACACQQdJDQAgACABOgADIANBfGogAToAACACQQlJDQAgAEEAIABrQQNxIgRqIgMgAUH/AXFBgYKECGwiATYCACADIAIgBGtBfHEiBGoiAkF8aiABNgIAIARBCUkNACADIAE2AgggAyABNgIEIAJBeGogATYCACACQXRqIAE2AgAgBEEZSQ0AIAMgATYCGCADIAE2AhQgAyABNgIQIAMgATYCDCACQXBqIAE2AgAgAkFsaiABNgIAIAJBaGogATYCACACQWRqIAE2AgAgBCADQQRxQRhyIgVrIgJBIEkNACABrUKBgICAEH4hBiADIAVqIQEDQCABIAY3AxggASAGNwMQIAEgBjcDCCABIAY3AwAgAUEgaiEBIAJBYGoiAkEfSw0ACwsgAAsGACAAJAELBAAjAQsEACMACwYAIAAkAAsSAQJ/IwAgAGtBcHEiASQAIAELC9ICAgBBgAgLwAJn5glqha5nu3Lzbjw69U+lf1IOUYxoBZur2YMfGc3gW9ieBcEH1Xw2F91wMDlZDvcxC8D/ERVYaKeP+WSkT/q+mC+KQpFEN3HP+8C1pdu16VvCVjnxEfFZpII/ktVeHKuYqgfYAVuDEr6FMSTDfQxVdF2+cv6x3oCnBtybdPGbwcFpm+SGR77vxp3BD8yhDCRvLOktqoR0StypsFzaiPl2UlE+mG3GMajIJwOwx39Zv/ML4MZHkafVUWPKBmcpKRSFCrcnOCEbLvxtLE0TDThTVHMKZbsKanYuycKBhSxykqHov6JLZhqocItLwqNRbMcZ6JLRJAaZ1oU1DvRwoGoQFsGkGQhsNx5Md0gntbywNLMMHDlKqthOT8qcW/NvLmjugo90b2OleBR4yIQIAseM+v++kOtsUKT3o/m+8nhxxgBBwAoLBIAFgAA="; + if (!isDataURI(wasmBinaryFile)) { + wasmBinaryFile = locateFile(wasmBinaryFile); + } + function getBinarySync(file) { + if (file == wasmBinaryFile && wasmBinary) { + return new Uint8Array(wasmBinary); + } + var binary = tryParseAsDataURI(file); + if (binary) { + return binary; + } + if (readBinary) { + return readBinary(file); + } + throw "both async and sync fetching of the wasm failed"; + } + function getBinaryPromise(binaryFile) { + return Promise.resolve().then(() => getBinarySync(binaryFile)); + } + function instantiateArrayBuffer(binaryFile, imports, receiver) { + return getBinaryPromise(binaryFile).then((binary) => { + return WebAssembly.instantiate(binary, imports); + }).then(receiver, (reason) => { + err(`failed to asynchronously prepare wasm: ${reason}`); + abort(reason); + }); + } + function instantiateAsync(binary, binaryFile, imports, callback) { + return instantiateArrayBuffer(binaryFile, imports, callback); + } + function createWasm() { + var info = { + "env": wasmImports, + "wasi_snapshot_preview1": wasmImports + }; + function receiveInstance(instance, module) { + wasmExports = instance.exports; + wasmMemory = wasmExports["memory"]; + updateMemoryViews(); + addOnInit(wasmExports["__wasm_call_ctors"]); + removeRunDependency("wasm-instantiate"); + return wasmExports; + } + addRunDependency("wasm-instantiate"); + function receiveInstantiationResult(result) { + receiveInstance(result["instance"]); + } + if (Module2["instantiateWasm"]) { + try { + return Module2["instantiateWasm"](info, receiveInstance); + } catch (e) { + err(`Module.instantiateWasm callback failed with error: ${e}`); + readyPromiseReject(e); + } + } + instantiateAsync(wasmBinary, wasmBinaryFile, info, receiveInstantiationResult).catch(readyPromiseReject); + return {}; + } + var tempDouble; + var tempI64; + function ExitStatus(status) { + this.name = "ExitStatus"; + this.message = `Program terminated with exit(${status})`; + this.status = status; + } + var callRuntimeCallbacks = (callbacks) => { + while (callbacks.length > 0) { + callbacks.shift()(Module2); + } + }; + function getValue(ptr, type = "i8") { + if (type.endsWith("*")) + type = "*"; + switch (type) { + case "i1": + return HEAP8[ptr]; + case "i8": + return HEAP8[ptr]; + case "i16": + return HEAP16[ptr >> 1]; + case "i32": + return HEAP32[ptr >> 2]; + case "i64": + abort("to do getValue(i64) use WASM_BIGINT"); + case "float": + return HEAPF32[ptr >> 2]; + case "double": + return HEAPF64[ptr >> 3]; + case "*": + return HEAPU32[ptr >> 2]; + default: + abort(`invalid type for getValue: ${type}`); + } + } + var noExitRuntime = Module2["noExitRuntime"] || true; + function setValue(ptr, value, type = "i8") { + if (type.endsWith("*")) + type = "*"; + switch (type) { + case "i1": + HEAP8[ptr] = value; + break; + case "i8": + HEAP8[ptr] = value; + break; + case "i16": + HEAP16[ptr >> 1] = value; + break; + case "i32": + HEAP32[ptr >> 2] = value; + break; + case "i64": + abort("to do setValue(i64) use WASM_BIGINT"); + case "float": + HEAPF32[ptr >> 2] = value; + break; + case "double": + HEAPF64[ptr >> 3] = value; + break; + case "*": + HEAPU32[ptr >> 2] = value; + break; + default: + abort(`invalid type for setValue: ${type}`); + } + } + var wasmImports = {}; + var wasmExports = createWasm(); + var ___wasm_call_ctors = () => (___wasm_call_ctors = wasmExports["__wasm_call_ctors"])(); + var _Hash_Update = Module2["_Hash_Update"] = (a0) => (_Hash_Update = Module2["_Hash_Update"] = wasmExports["Hash_Update"])(a0); + var _Hash_Final = Module2["_Hash_Final"] = () => (_Hash_Final = Module2["_Hash_Final"] = wasmExports["Hash_Final"])(); + var _Hash_Init = Module2["_Hash_Init"] = (a0) => (_Hash_Init = Module2["_Hash_Init"] = wasmExports["Hash_Init"])(a0); + var _GetBufferPtr = Module2["_GetBufferPtr"] = () => (_GetBufferPtr = Module2["_GetBufferPtr"] = wasmExports["GetBufferPtr"])(); + var stackSave = () => (stackSave = wasmExports["stackSave"])(); + var stackRestore = (a0) => (stackRestore = wasmExports["stackRestore"])(a0); + var stackAlloc = (a0) => (stackAlloc = wasmExports["stackAlloc"])(a0); + var calledRun; + dependenciesFulfilled = function runCaller() { + if (!calledRun) + run(); + if (!calledRun) + dependenciesFulfilled = runCaller; + }; + function run() { + if (runDependencies > 0) { + return; + } + preRun(); + if (runDependencies > 0) { + return; + } + function doRun() { + if (calledRun) + return; + calledRun = true; + Module2["calledRun"] = true; + if (ABORT) + return; + initRuntime(); + readyPromiseResolve(Module2); + if (Module2["onRuntimeInitialized"]) + Module2["onRuntimeInitialized"](); + postRun(); + } + if (Module2["setStatus"]) { + Module2["setStatus"]("Running..."); + setTimeout(function() { + setTimeout(function() { + Module2["setStatus"](""); + }, 1); + doRun(); + }, 1); + } else { + doRun(); + } + } + if (Module2["preInit"]) { + if (typeof Module2["preInit"] == "function") + Module2["preInit"] = [Module2["preInit"]]; + while (Module2["preInit"].length > 0) { + Module2["preInit"].pop()(); + } + } + run(); + return moduleArg.ready; + }; +})(); +var sha256_default = Module; + +// src/vendor/hash-wasm/sha256-wrapper.ts +async function createSHA256(isInsideWorker = false) { + const BUFFER_MAX_SIZE = 8 * 1024 * 1024; + const wasm = isInsideWorker ? ( + // @ts-expect-error WasmModule will be populated inside self object + await self["SHA256WasmModule"]() + ) : await sha256_default(); + const heap = wasm.HEAPU8.subarray(wasm._GetBufferPtr()); + return { + init() { + wasm._Hash_Init(256); + }, + update(data) { + let byteUsed = 0; + while (byteUsed < data.byteLength) { + const bytesLeft = data.byteLength - byteUsed; + const length = Math.min(bytesLeft, BUFFER_MAX_SIZE); + heap.set(data.subarray(byteUsed, byteUsed + length)); + wasm._Hash_Update(length); + byteUsed += length; + } + }, + digest(method) { + if (method !== "hex") { + throw new Error("Only digest hex is supported"); + } + wasm._Hash_Final(); + const result = Array.from(heap.slice(0, 32)); + return result.map((b) => b.toString(16).padStart(2, "0")).join(""); + } + }; +} +function createSHA256WorkerCode() { + return ` + self.addEventListener('message', async (event) => { + const { file } = event.data; + const sha256 = await self.createSHA256(true); + sha256.init(); + const reader = file.stream().getReader(); + const total = file.size; + let bytesDone = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + sha256.update(value); + bytesDone += value.length; + postMessage({ progress: bytesDone / total }); + } + postMessage({ sha256: sha256.digest('hex') }); + }); + self.SHA256WasmModule = ${sha256_default.toString()}; + self.createSHA256 = ${createSHA256.toString()}; + `; +} +export { + createSHA256, + createSHA256WorkerCode +}; diff --git a/node_modules/@huggingface/hub/dist/src/consts.d.ts b/node_modules/@huggingface/hub/dist/src/consts.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..eb6738efab2fb480c708b5d2a86ae84e3d8f6722 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/consts.d.ts @@ -0,0 +1,2 @@ +export declare const HUB_URL = "https://huggingface.co"; +//# sourceMappingURL=consts.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/consts.d.ts.map b/node_modules/@huggingface/hub/dist/src/consts.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..8533b83498b905f30ebd0a0f2ba4abaf239158f5 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/consts.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"consts.d.ts","sourceRoot":"","sources":["../../src/consts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,2BAA2B,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/error.d.ts b/node_modules/@huggingface/hub/dist/src/error.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ca693abdd9d17e254b26d665401e275c831c8089 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/error.d.ts @@ -0,0 +1,18 @@ +import type { JsonObject } from "./vendor/type-fest/basic"; +export declare function createApiError(response: Response, opts?: { + requestId?: string; + message?: string; +}): Promise; +/** + * Error thrown when an API call to the Hugging Face Hub fails. + */ +export declare class HubApiError extends Error { + statusCode: number; + url: string; + requestId?: string; + data?: JsonObject; + constructor(url: string, statusCode: number, requestId?: string, message?: string); +} +export declare class InvalidApiResponseFormatError extends Error { +} +//# sourceMappingURL=error.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/error.d.ts.map b/node_modules/@huggingface/hub/dist/src/error.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..ec6c608476abf048706f07371abefe7f0d5f028c --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/error.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"error.d.ts","sourceRoot":"","sources":["../../src/error.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAE3D,wBAAsB,cAAc,CACnC,QAAQ,EAAE,QAAQ,EAClB,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7C,OAAO,CAAC,KAAK,CAAC,CAuBhB;AAED;;GAEG;AACH,qBAAa,WAAY,SAAQ,KAAK;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,UAAU,CAAC;gBAEN,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM;CAOjF;AAED,qBAAa,6BAA8B,SAAQ,KAAK;CAAG"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/index.d.ts b/node_modules/@huggingface/hub/dist/src/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..def842a815dbb31f1ef051f9cae5693975466fc2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/index.d.ts @@ -0,0 +1,11 @@ +export * from "./lib"; +export type { AccessToken, AccessTokenRole, AuthType, Credentials, PipelineType, RepoDesignation, RepoFullName, RepoId, RepoType, SpaceHardwareFlavor, SpaceResourceConfig, SpaceResourceRequirement, SpaceRuntime, SpaceSdk, SpaceStage, } from "./types/public"; +export { HubApiError, InvalidApiResponseFormatError } from "./error"; +export { HUB_URL } from "./consts"; +/** + * Only exported for E2Es convenience + */ +export { sha256 as __internal_sha256 } from "./utils/sha256"; +export { XetBlob as __internal_XetBlob } from "./utils/XetBlob"; +export type { XetReadToken } from "./utils/XetBlob"; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/index.d.ts.map b/node_modules/@huggingface/hub/dist/src/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..9c534cdcd688d677eb3c49bdfdcaebac62038dfb --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,OAAO,CAAC;AAEtB,YAAY,EACX,WAAW,EACX,eAAe,EACf,QAAQ,EACR,WAAW,EACX,YAAY,EACZ,eAAe,EACf,YAAY,EACZ,MAAM,EACN,QAAQ,EACR,mBAAmB,EACnB,mBAAmB,EACnB,wBAAwB,EACxB,YAAY,EACZ,QAAQ,EACR,UAAU,GACV,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,6BAA6B,EAAE,MAAM,SAAS,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AACnC;;GAEG;AACH,OAAO,EAAE,MAAM,IAAI,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,OAAO,IAAI,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAChE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/add-collection-item.d.ts b/node_modules/@huggingface/hub/dist/src/lib/add-collection-item.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..941576a00135d9537655b260fc3259bf85a45e17 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/add-collection-item.d.ts @@ -0,0 +1,24 @@ +import type { CredentialsParams } from "../types/public"; +export declare function addCollectionItem(params: { + /** + * The slug of the collection to add the item to. + */ + slug: string; + /** + * The item to add to the collection. + */ + item: { + type: "paper" | "collection" | "space" | "model" | "dataset"; + id: string; + }; + /** + * A note to attach to the item in the collection. The maximum size for a note is 500 characters. + */ + note?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise; +//# sourceMappingURL=add-collection-item.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/add-collection-item.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/add-collection-item.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..057031dde233d9f6bed5d00d2bdc213a78710a5a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/add-collection-item.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"add-collection-item.d.ts","sourceRoot":"","sources":["../../../src/lib/add-collection-item.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAGzD,wBAAsB,iBAAiB,CACtC,MAAM,EAAE;IACP;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE;QACL,IAAI,EAAE,OAAO,GAAG,YAAY,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;QAC7D,EAAE,EAAE,MAAM,CAAC;KACX,CAAC;IACF;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,IAAI,CAAC,CAsBf"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/add-collection-item.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/add-collection-item.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b087f52d09a11b782a3ea8446e48b90e2495d60c --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/add-collection-item.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=add-collection-item.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/add-collection-item.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/add-collection-item.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d151aa218acb12542c66125dca851fb1b0e7e711 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/add-collection-item.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"add-collection-item.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/add-collection-item.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/cache-management.d.ts b/node_modules/@huggingface/hub/dist/src/lib/cache-management.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..8112147debe5857120441ed3e4967c3358632d7e --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/cache-management.d.ts @@ -0,0 +1,46 @@ +import type { Stats } from "node:fs"; +import type { RepoType, RepoId } from "../types/public"; +export declare function getHFHubCachePath(): string; +export declare const REPO_ID_SEPARATOR: string; +export declare function getRepoFolderName({ name, type }: RepoId): string; +export interface CachedFileInfo { + path: string; + /** + * Underlying file - which `path` is symlinked to + */ + blob: { + size: number; + path: string; + lastModifiedAt: Date; + lastAccessedAt: Date; + }; +} +export interface CachedRevisionInfo { + commitOid: string; + path: string; + size: number; + files: CachedFileInfo[]; + refs: string[]; + lastModifiedAt: Date; +} +export interface CachedRepoInfo { + id: RepoId; + path: string; + size: number; + filesCount: number; + revisions: CachedRevisionInfo[]; + lastAccessedAt: Date; + lastModifiedAt: Date; +} +export interface HFCacheInfo { + size: number; + repos: CachedRepoInfo[]; + warnings: Error[]; +} +export declare function scanCacheDir(cacheDir?: string | undefined): Promise; +export declare function scanCachedRepo(repoPath: string): Promise; +export declare function scanRefsDir(refsPath: string, refsByHash: Map): Promise; +export declare function scanSnapshotDir(revisionPath: string, cachedFiles: CachedFileInfo[], blobStats: Map): Promise; +export declare function getBlobStat(blobPath: string, blobStats: Map): Promise; +export declare function parseRepoType(type: string): RepoType; +//# sourceMappingURL=cache-management.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/cache-management.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/cache-management.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..958e8b0c129cd7644a5750063aa30b2e2f7d9d3c --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/cache-management.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"cache-management.d.ts","sourceRoot":"","sources":["../../../src/lib/cache-management.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAcxD,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C;AAID,eAAO,MAAM,iBAAiB,EAAE,MAAa,CAAC;AAE9C,wBAAgB,iBAAiB,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,MAAM,GAAG,MAAM,CAGhE;AAED,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,cAAc,EAAE,IAAI,CAAC;QACrB,cAAc,EAAE,IAAI,CAAC;KACrB,CAAC;CACF;AAED,MAAM,WAAW,kBAAkB;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,cAAc,EAAE,IAAI,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAEhC,cAAc,EAAE,IAAI,CAAC;IACrB,cAAc,EAAE,IAAI,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,QAAQ,EAAE,KAAK,EAAE,CAAC;CAClB;AAED,wBAAsB,YAAY,CAAC,QAAQ,GAAE,MAAM,GAAG,SAAqB,GAAG,OAAO,CAAC,WAAW,CAAC,CA4CjG;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CA0F9E;AAED,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAepG;AAED,wBAAsB,eAAe,CACpC,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,cAAc,EAAE,EAC7B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,GAC3B,OAAO,CAAC,IAAI,CAAC,CAqBf;AAED,wBAAsB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAQjG;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,CAepD"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/cache-management.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/cache-management.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..0373b1959aacd93e9de1690903ef3493a839051c --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/cache-management.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=cache-management.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/cache-management.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/cache-management.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..697ad504d66506002f0899cb5e27c7a815e24526 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/cache-management.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"cache-management.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/cache-management.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/check-repo-access.d.ts b/node_modules/@huggingface/hub/dist/src/lib/check-repo-access.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2d15375d1d1c197a7b3e2e51d95837bcae3c90fe --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/check-repo-access.d.ts @@ -0,0 +1,12 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +/** + * Check if we have read access to a repository. + * + * Throw a {@link HubApiError} error if we don't have access. HubApiError.statusCode will be 401, 403 or 404. + */ +export declare function checkRepoAccess(params: { + repo: RepoDesignation; + hubUrl?: string; + fetch?: typeof fetch; +} & Partial): Promise; +//# sourceMappingURL=check-repo-access.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/check-repo-access.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/check-repo-access.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..3a470c4c75088592242ebf5067f6267badd2f1ef --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/check-repo-access.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"check-repo-access.d.ts","sourceRoot":"","sources":["../../../src/lib/check-repo-access.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAI1E;;;;GAIG;AACH,wBAAsB,eAAe,CACpC,MAAM,EAAE;IACP,IAAI,EAAE,eAAe,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,IAAI,CAAC,CAaf"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/check-repo-access.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/check-repo-access.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..0e9067a51472b2defbda225ccee9af890e47b336 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/check-repo-access.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=check-repo-access.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/check-repo-access.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/check-repo-access.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..b92cd17f8306ca5f9559c749dc44ce07655998f3 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/check-repo-access.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"check-repo-access.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/check-repo-access.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/collection-info.d.ts b/node_modules/@huggingface/hub/dist/src/lib/collection-info.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..3ba10e00637508eb6d0c8fa85c4e567387eb8726 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/collection-info.d.ts @@ -0,0 +1,17 @@ +import type { ApiCollectionInfo } from "../types/api/api-collection"; +import type { CredentialsParams } from "../types/public"; +export declare function collectionInfo(params: { + /** + * The slug of the collection. + */ + slug: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise; +//# sourceMappingURL=collection-info.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/collection-info.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/collection-info.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..ae80cdad1918833b58432da0fb9186a02577c3f3 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/collection-info.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"collection-info.d.ts","sourceRoot":"","sources":["../../../src/lib/collection-info.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAGzD,wBAAsB,cAAc,CACnC,MAAM,EAAE;IACP;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,iBAAiB,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC,CAerE"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/collection-info.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/collection-info.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..62c27db438f160af07f39c5cc319c320608e09d9 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/collection-info.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=collection-info.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/collection-info.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/collection-info.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..fc80b51de15a77b4e3c259784b0f47c945f9fb06 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/collection-info.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"collection-info.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/collection-info.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/commit.d.ts b/node_modules/@huggingface/hub/dist/src/lib/commit.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..76759fa8532d025198e4da0eac2a1ef7e1167451 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/commit.d.ts @@ -0,0 +1,131 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +export interface CommitDeletedEntry { + operation: "delete"; + path: string; +} +export type ContentSource = Blob | URL; +export interface CommitFile { + operation: "addOrUpdate"; + path: string; + content: ContentSource; +} +/** + * Opitmized when only the beginning or the end of the file is replaced + * + * todo: handle other cases + */ +export interface CommitEditFile { + operation: "edit"; + path: string; + /** Later, will be ContentSource. For now simpler to just handle blobs */ + originalContent: Blob; + edits: Array<{ + /** + * Later, will be ContentSource. For now simpler to just handle blobs + * + * originalContent from [start, end) will be replaced by this + */ + content: Blob; + /** + * The start position of the edit in the original content + */ + start: number; + /** + * The end position of the edit in the original content + * + * originalContent from [start, end) will be replaced by the edit + */ + end: number; + }>; +} +/** + * Server-side copy of a file from a source repo/bucket to the destination repo. + * + * Only supported when the destination repo is a bucket. The source file must be xet-backed, + * so the caller is responsible for resolving the source path to its {@link sourceXetHash} + * (typically via {@link pathsInfo} or {@link listFiles}). + * + * For higher-level helpers that perform the resolution and handle non-xet source files, + * see {@link copyFile}, {@link copyFiles} and {@link copyFolder}. + */ +export interface CommitCopyFile { + operation: "copy"; + path: string; + sourceXetHash: string; + sourceRepo: RepoDesignation; +} +export type CommitOperation = CommitDeletedEntry | CommitFile | CommitEditFile | CommitCopyFile; +export type CommitParams = { + title: string; + description?: string; + repo: RepoDesignation; + operations: CommitOperation[]; + /** @default "main" */ + branch?: string; + /** + * Parent commit. Optional + * + * - When opening a PR: will use parentCommit as the parent commit + * - When committing on a branch: Will make sure that there were no intermediate commits + */ + parentCommit?: string; + isPullRequest?: boolean; + hubUrl?: string; + /** + * Whether to use web workers to compute SHA256 hashes. + * + * @default false + */ + useWebWorkers?: boolean | { + minSize?: number; + poolSize?: number; + }; + /** + * Maximum depth of folders to upload. Files deeper than this will be ignored + * + * @default 5 + */ + maxFolderDepth?: number; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + abortSignal?: AbortSignal; + /** + * @default true + * + * Use xet protocol: https://huggingface.co/blog/xet-on-the-hub to upload, rather than a basic S3 PUT + */ + useXet?: boolean; +} & Partial; +export interface CommitOutput { + pullRequestUrl?: string; + commit: { + oid: string; + url: string; + }; + hookOutput: string; +} +export type CommitProgressEvent = { + event: "phase"; + phase: "preuploading" | "uploadingLargeFiles" | "committing"; +} | { + event: "fileProgress"; + path: string; + progress: number; + state: "hashing" | "uploading" | "error"; +}; +/** + * Internal function for now, used by commit. + * + * Can be exposed later to offer fine-tuned progress info + * + * CommitOutput is not present for bucket commits + */ +export declare function commitIter(params: CommitParams): AsyncGenerator; +export declare function commitIterBucket(params: CommitParams): AsyncGenerator; +/** + * @returns undefined for bucket uploads, CommitOutput otherwise + */ +export declare function commit(params: CommitParams): Promise; +//# sourceMappingURL=commit.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/commit.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/commit.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..be06a753b9f16367348f75ef12d26f9b1252d1b7 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/commit.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"commit.d.ts","sourceRoot":"","sources":["../../../src/lib/commit.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAqB1E,MAAM,WAAW,kBAAkB;IAClC,SAAS,EAAE,QAAQ,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACb;AAED,MAAM,MAAM,aAAa,GAAG,IAAI,GAAG,GAAG,CAAC;AAEvC,MAAM,WAAW,UAAU;IAC1B,SAAS,EAAE,aAAa,CAAC;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,aAAa,CAAC;CAEvB;AAED;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,yEAAyE;IACzE,eAAe,EAAE,IAAI,CAAC;IACtB,KAAK,EAAE,KAAK,CAAC;QACZ;;;;WAIG;QACH,OAAO,EAAE,IAAI,CAAC;QACd;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QACd;;;;WAIG;QACH,GAAG,EAAE,MAAM,CAAC;KACZ,CAAC,CAAC;CACH;AAYD;;;;;;;;;GASG;AACH,MAAM,WAAW,cAAc;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,eAAe,CAAC;CAC5B;AAED,MAAM,MAAM,eAAe,GACxB,kBAAkB,GAClB,UAAU,GACV,cAAc,GACd,cAAc,CAA0B;AAG3C,MAAM,MAAM,YAAY,GAAG;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,eAAe,CAAC;IACtB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,sBAAsB;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,GAAG;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAClE;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;;;;OAIG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAEjB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAE/B,MAAM,WAAW,YAAY;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE;QACP,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;KACZ,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;CACnB;AAYD,MAAM,MAAM,mBAAmB,GAC5B;IACA,KAAK,EAAE,OAAO,CAAC;IACf,KAAK,EAAE,cAAc,GAAG,qBAAqB,GAAG,YAAY,CAAC;CAC5D,GACD;IACA,KAAK,EAAE,cAAc,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,SAAS,GAAG,WAAW,GAAG,OAAO,CAAC;CACxC,CAAC;AAEL;;;;;;GAMG;AACH,wBAAuB,UAAU,CAAC,MAAM,EAAE,YAAY,GAAG,cAAc,CAAC,mBAAmB,EAAE,YAAY,GAAG,SAAS,CAAC,CAyiBrH;AAED,wBAAuB,gBAAgB,CAAC,MAAM,EAAE,YAAY,GAAG,cAAc,CAAC,mBAAmB,CAAC,CA0PjG;AAED;;GAEG;AACH,wBAAsB,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CAoBpF"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/commit.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/commit.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fb66fc5a53dcf9f2ec7d34ffb5aed8ae2bbe7d7c --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/commit.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=commit.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/commit.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/commit.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f5b06e5691c53eb723cec2cbe573b5a48bba6989 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/commit.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"commit.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/commit.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/copy-files.d.ts b/node_modules/@huggingface/hub/dist/src/lib/copy-files.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..97a28e4d057eb1cc038ee21473bf79e64947edcd --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/copy-files.d.ts @@ -0,0 +1,219 @@ +import type { BucketDesignation, CredentialsParams, RepoDesignation } from "../types/public"; +import type { CommitParams } from "./commit"; +/** + * Progress events yielded by {@link copyFileIter} / {@link copyFilesIter} / {@link copyFolderIter}. + * + * Currently only `fileDownloaded` is emitted: one event per source file that had to be downloaded + * (small git-stored files that can't be copied server-side). Xet-backed files are copied + * server-side and do not produce events. + */ +export interface CopyProgressEvent { + event: "fileDownloaded"; + /** Source path of the file that was just downloaded. */ + path: string; + /** Number of files downloaded so far (including this one). */ + downloaded: number; + /** Total number of files that will be downloaded. */ + total: number; +} +/** + * Source location of a file in {@link copyFile} / {@link copyFiles} / {@link copyFolder}. + */ +export interface CopySource { + repo: RepoDesignation; + /** + * Path of the file (or folder, for {@link copyFolder}) inside the source repo. + * Leave empty in {@link copyFolder} to copy the whole repo. + */ + path: string; + /** + * Git revision to read the source from. Ignored for bucket sources. + * + * @default "main" + */ + revision?: string; +} +/** + * Destination location for {@link copyFile} / {@link copyFolder}. + * + * The destination repo must be a bucket — server-side copy is currently only supported + * towards buckets. + */ +export interface CopyDestination { + repo: BucketDesignation; + /** + * Exact destination path within the destination bucket. For {@link copyFolder}, + * acts as a prefix; leave empty to copy under the bucket root. + */ + path: string; +} +/** + * One file to copy in a {@link copyFiles} call. + */ +export interface CopyFilesEntry { + source: CopySource; + /** + * Exact path within the destination bucket. The bucket itself is shared with the + * other entries via the top-level {@link copyFiles} `destination` parameter. + */ + destinationPath: string; +} +type SharedParams = { + hubUrl?: CommitParams["hubUrl"]; + fetch?: CommitParams["fetch"]; + abortSignal?: CommitParams["abortSignal"]; +} & Partial; +/** + * Copy a single file from a source repo/bucket to the destination bucket. + * + * The copy is server-side (no data transfer) when the source file is xet-backed. + * For small non-xet repo files (e.g. `config.json`) the file is downloaded and + * re-uploaded to the destination bucket in the same commit. + * + * LFS pointer files that have not been migrated to xet are rejected up front + * (they would otherwise require downloading the full LFS blob). + * + * @example + * ```ts + * await copyFile({ + * source: { + * repo: { type: "model", name: "username/my-model" }, + * path: "model.safetensors", + * }, + * destination: { + * repo: { type: "bucket", name: "username/my-bucket" }, + * path: "models/my-model/model.safetensors", + * }, + * accessToken: "hf_...", + * }); + * ``` + */ +export declare function copyFile(params: { + source: CopySource; + destination: CopyDestination; +} & SharedParams): Promise; +/** + * Async-iterator variant of {@link copyFile} that yields {@link CopyProgressEvent}s while + * downloading non-xet source files (xet-backed files are copied server-side and do not + * emit events). See {@link copyFile} for the semantics. + * + * @example + * ```ts + * for await (const event of copyFileIter({ source, destination, accessToken })) { + * console.log(`downloaded ${event.path} (${event.downloaded}/${event.total})`); + * } + * ``` + */ +export declare function copyFileIter(params: { + source: CopySource; + destination: CopyDestination; +} & SharedParams): AsyncGenerator; +/** + * Copy multiple files (potentially from different source repos/buckets) to the destination + * bucket in a single commit. + * + * For xet-backed source files, the copy is performed server-side with no data transfer. + * For non-xet source files (typically small git-stored repo files), the file is + * downloaded and re-uploaded as part of the same commit. + * + * LFS pointer files that have not been migrated to xet are rejected up front. + * + * @example + * ```ts + * await copyFiles({ + * destination: { type: "bucket", name: "username/my-bucket" }, + * files: [ + * { + * source: { + * repo: { type: "bucket", name: "username/other-bucket" }, + * path: "data.bin", + * }, + * destinationPath: "data.bin", + * }, + * { + * source: { + * repo: { type: "model", name: "username/my-model" }, + * path: "model.safetensors", + * }, + * destinationPath: "models/my-model/model.safetensors", + * }, + * ], + * accessToken: "hf_...", + * }); + * ``` + */ +export declare function copyFiles(params: { + destination: BucketDesignation; + files: CopyFilesEntry[]; +} & SharedParams): Promise; +/** + * Async-iterator variant of {@link copyFiles} that yields {@link CopyProgressEvent}s while + * downloading non-xet source files (xet-backed files are copied server-side and do not + * emit events). See {@link copyFiles} for the semantics. + */ +export declare function copyFilesIter(params: { + destination: BucketDesignation; + files: CopyFilesEntry[]; +} & SharedParams): AsyncGenerator; +/** + * Copy a folder (recursively) from a source repo/bucket to the destination bucket + * in a single commit. + * + * Per-file paths are resolved relative to {@link CopySource.path}; the source folder + * itself is not preserved in the destination unless {@link CopyDestination.path} + * keeps it. + * + * @example + * ```ts + * // Copy an entire dataset under "datasets/my-dataset/" in the bucket + * await copyFolder({ + * source: { repo: { type: "dataset", name: "username/my-dataset" } }, + * destination: { + * repo: { type: "bucket", name: "username/my-bucket" }, + * path: "datasets/my-dataset/", + * }, + * accessToken: "hf_...", + * }); + * + * // Copy a subfolder + * await copyFolder({ + * source: { + * repo: { type: "bucket", name: "username/src-bucket" }, + * path: "models/", + * }, + * destination: { + * repo: { type: "bucket", name: "username/dst-bucket" }, + * path: "backup/", + * }, + * accessToken: "hf_...", + * }); + * ``` + */ +export declare function copyFolder(params: { + source: Omit & { + path?: string; + }; + destination: Omit & { + path?: string; + }; +} & SharedParams): Promise; +/** + * Async-iterator variant of {@link copyFolder} that yields {@link CopyProgressEvent}s while + * downloading non-xet source files (xet-backed files are copied server-side and do not + * emit events). See {@link copyFolder} for the semantics. + */ +export declare function copyFolderIter(params: { + source: Omit & { + path?: string; + }; + destination: Omit & { + path?: string; + }; +} & SharedParams): AsyncGenerator; +/** + * Compute the path of `filePath` relative to `folderPath`. Used to map source paths + * under a folder being copied to destination paths under the new prefix. + */ +export declare function relativeUnderFolder(filePath: string, folderPath: string): string; +export {}; +//# sourceMappingURL=copy-files.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/copy-files.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/copy-files.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..2301bd3cef0102d6802957d925eb9615586729f2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/copy-files.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"copy-files.d.ts","sourceRoot":"","sources":["../../../src/lib/copy-files.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,eAAe,EAAU,MAAM,iBAAiB,CAAC;AAMrG,OAAO,KAAK,EAAmB,YAAY,EAAE,MAAM,UAAU,CAAC;AAQ9D;;;;;;GAMG;AACH,MAAM,WAAW,iBAAiB;IACjC,KAAK,EAAE,gBAAgB,CAAC;IACxB,wDAAwD;IACxD,IAAI,EAAE,MAAM,CAAC;IACb,8DAA8D;IAC9D,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;CACd;AAKD;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,IAAI,EAAE,eAAe,CAAC;IACtB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,iBAAiB,CAAC;IACxB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;CACb;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,MAAM,EAAE,UAAU,CAAC;IACnB;;;OAGG;IACH,eAAe,EAAE,MAAM,CAAC;CACxB;AAED,KAAK,YAAY,GAAG;IACnB,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChC,KAAK,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAC9B,WAAW,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;CAC1C,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAE/B;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,QAAQ,CACvB,MAAM,EAAE;IACP,MAAM,EAAE,UAAU,CAAC;IACnB,WAAW,EAAE,eAAe,CAAC;CAC7B,GAAG,YAAY,GACd,OAAO,CAAC,SAAS,CAAC,CAcpB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,YAAY,CAC3B,MAAM,EAAE;IACP,MAAM,EAAE,UAAU,CAAC;IACnB,WAAW,EAAE,eAAe,CAAC;CAC7B,GAAG,YAAY,GACd,cAAc,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAc9C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAsB,SAAS,CAC9B,MAAM,EAAE;IACP,WAAW,EAAE,iBAAiB,CAAC;IAC/B,KAAK,EAAE,cAAc,EAAE,CAAC;CACxB,GAAG,YAAY,GACd,OAAO,CAAC,SAAS,CAAC,CAQpB;AAED;;;;GAIG;AACH,wBAAuB,aAAa,CACnC,MAAM,EAAE;IACP,WAAW,EAAE,iBAAiB,CAAC;IAC/B,KAAK,EAAE,cAAc,EAAE,CAAC;CACxB,GAAG,YAAY,GACd,cAAc,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAiB9C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAsB,UAAU,CAC/B,MAAM,EAAE;IACP,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,GAAG;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,WAAW,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,GAAG;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/D,GAAG,YAAY,GACd,OAAO,CAAC,SAAS,CAAC,CAQpB;AAED;;;;GAIG;AACH,wBAAuB,cAAc,CACpC,MAAM,EAAE;IACP,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,GAAG;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,WAAW,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,GAAG;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/D,GAAG,YAAY,GACd,cAAc,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAkF9C;AA0KD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAWhF"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/copy-files.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/copy-files.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b4e9d8cb285edefca108d5c0eaafd5d59b64f008 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/copy-files.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=copy-files.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/copy-files.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/copy-files.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f70a084a833fc26f27fed61806c9d4eeddeecbaf --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/copy-files.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"copy-files.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/copy-files.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/count-commits.d.ts b/node_modules/@huggingface/hub/dist/src/lib/count-commits.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..e3418916f35b778acfb2d8d436bcc669a9f80f23 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/count-commits.d.ts @@ -0,0 +1,11 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +export declare function countCommits(params: { + repo: RepoDesignation; + /** + * Revision to list commits from. Defaults to the default branch. + */ + revision?: string; + hubUrl?: string; + fetch?: typeof fetch; +} & Partial): Promise; +//# sourceMappingURL=count-commits.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/count-commits.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/count-commits.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..159002e1f44df1d8e551f2fb06bcd2152215b1bb --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/count-commits.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"count-commits.d.ts","sourceRoot":"","sources":["../../../src/lib/count-commits.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAI1E,wBAAsB,YAAY,CACjC,MAAM,EAAE;IACP,IAAI,EAAE,eAAe,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,MAAM,CAAC,CAkBjB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/count-commits.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/count-commits.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..0a2672397b44dafd9085dd324362e025f69b8156 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/count-commits.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=count-commits.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/count-commits.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/count-commits.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..3c5c244c038c12851531bac47c3f05a9a4a630ba --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/count-commits.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"count-commits.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/count-commits.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/create-branch.d.ts b/node_modules/@huggingface/hub/dist/src/lib/create-branch.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..6b30a187a6690d6348f218555f7b74949d1cd127 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/create-branch.d.ts @@ -0,0 +1,28 @@ +import type { AccessToken, RepoDesignation } from "../types/public"; +export declare function createBranch(params: { + repo: RepoDesignation; + /** + * Revision to create the branch from. Defaults to the default branch. + * + * Use empty: true to create an empty branch. + */ + revision?: string; + hubUrl?: string; + accessToken?: AccessToken; + fetch?: typeof fetch; + /** + * The name of the branch to create + */ + branch: string; + /** + * Use this to create an empty branch, with no commits. + */ + empty?: boolean; + /** + * Use this to overwrite the branch if it already exists. + * + * If you only specify `overwrite` and no `revision`/`empty`, and the branch already exists, it will be a no-op. + */ + overwrite?: boolean; +}): Promise; +//# sourceMappingURL=create-branch.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/create-branch.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/create-branch.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d58e407814418eb6663d5375d5d7750280683a03 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/create-branch.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"create-branch.d.ts","sourceRoot":"","sources":["../../../src/lib/create-branch.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGpE,wBAAsB,YAAY,CAAC,MAAM,EAAE;IAC1C,IAAI,EAAE,eAAe,CAAC;IACtB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACpB,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBhB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/create-branch.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/create-branch.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b99f6b76f298288324555588ff1dff86c067f334 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/create-branch.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=create-branch.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/create-branch.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/create-branch.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e8f81a4ce52abfa6142e4627f76c66c941a0c37b --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/create-branch.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"create-branch.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/create-branch.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/create-collection.d.ts b/node_modules/@huggingface/hub/dist/src/lib/create-collection.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..0ece694bb2b107f060b84bb2920ce004b6e09cbc --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/create-collection.d.ts @@ -0,0 +1,13 @@ +import type { ApiCreateCollectionPayload } from "../types/api/api-create-collection"; +import type { CredentialsParams } from "../types/public"; +export declare function createCollection(params: { + collection: ApiCreateCollectionPayload; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise<{ + slug: string; +}>; +//# sourceMappingURL=create-collection.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/create-collection.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/create-collection.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..3a50e67dded85691dbd1aa8c1d434d7b37ce51c2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/create-collection.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"create-collection.d.ts","sourceRoot":"","sources":["../../../src/lib/create-collection.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAC;AACrF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAGzD,wBAAsB,gBAAgB,CACrC,MAAM,EAAE;IACP,UAAU,EAAE,0BAA0B,CAAC;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAmB3B"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/create-collection.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/create-collection.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..6ab3fcced4e0caed608efad803ea292e12460ae2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/create-collection.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=create-collection.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/create-collection.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/create-collection.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..3c5a4027bdeeba2aa856a7cf0c8039c76959d44c --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/create-collection.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"create-collection.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/create-collection.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/create-repo.d.ts b/node_modules/@huggingface/hub/dist/src/lib/create-repo.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..1ea2c723be6dac37d903256d01a78303a9368c4d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/create-repo.d.ts @@ -0,0 +1,35 @@ +import type { CredentialsParams, RepoDesignation, SpaceSdk } from "../types/public"; +export declare function createRepo(params: { + repo: RepoDesignation; + /** + * If unset, will follow the organization's default setting. (typically public, except for some Enterprise organizations) + */ + visibility?: "public" | "private" | "protected"; + /** + * @deprecated Use {@link visibility} instead. + */ + private?: boolean; + resourceGroupId?: string; + /** + * Does not work for buckets + */ + license?: string; + /** + * Only a few lightweight files are supported at repo creation - and not for buckets + */ + files?: Array<{ + content: ArrayBuffer | Blob; + path: string; + }>; + /** @required for when {@link repo.type} === "space" */ + sdk?: SpaceSdk; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): Promise<{ + repoUrl: string; + id: string; +}>; +//# sourceMappingURL=create-repo.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/create-repo.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/create-repo.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c4d09e0d078e7563bca981981d36b62d359d98a3 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/create-repo.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"create-repo.d.ts","sourceRoot":"","sources":["../../../src/lib/create-repo.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAKpF,wBAAsB,UAAU,CAC/B,MAAM,EAAE;IACP,IAAI,EAAE,eAAe,CAAC;IACtB;;OAEG;IACH,UAAU,CAAC,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,CAAC;IAChD;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,KAAK,CAAC,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC7D,uDAAuD;IACvD,GAAG,CAAC,EAAE,QAAQ,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAiE1C"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/create-repo.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/create-repo.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..270ba5d1fa0472bccb38f73d4382cf58355ec243 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/create-repo.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=create-repo.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/create-repo.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/create-repo.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a7c90fdf162621b4df82e2c275bfa6c03a65192a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/create-repo.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"create-repo.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/create-repo.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/dataset-info.d.ts b/node_modules/@huggingface/hub/dist/src/lib/dataset-info.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..491a5db3066f6d2937156c6dc19b9aa129bbfc64 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/dataset-info.d.ts @@ -0,0 +1,17 @@ +import type { ApiDatasetInfo } from "../types/api/api-dataset"; +import type { CredentialsParams } from "../types/public"; +import { type DATASET_EXPANDABLE_KEYS, DATASET_EXPAND_KEYS, type DatasetEntry } from "./list-datasets"; +export declare function datasetInfo = never>(params: { + name: string; + hubUrl?: string; + additionalFields?: T[]; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + */ + revision?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise>; +//# sourceMappingURL=dataset-info.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/dataset-info.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/dataset-info.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..fcc0f07cc9086f0f64d2690cf2a1c3d9ef737d78 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/dataset-info.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"dataset-info.d.ts","sourceRoot":"","sources":["../../../src/lib/dataset-info.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAGzD,OAAO,EAAE,KAAK,uBAAuB,EAAE,mBAAmB,EAAE,KAAK,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAEvG,wBAAsB,WAAW,CAChC,KAAK,CAAC,CAAC,SAAS,OAAO,CAAC,CAAC,OAAO,uBAAuB,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,EAE/G,MAAM,EAAE;IACP,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,CAAC,EAAE,CAAC;IACvB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAmCjD"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/dataset-info.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/dataset-info.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..77b2e5264179c0963a1a5390b306ae4b438b029e --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/dataset-info.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=dataset-info.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/dataset-info.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/dataset-info.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..4d4065e192dfa1b446e01f2118d65c66d363138f --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/dataset-info.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"dataset-info.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/dataset-info.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-branch.d.ts b/node_modules/@huggingface/hub/dist/src/lib/delete-branch.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..05c3ea4b8c22549c9c3fec1263f104e79d79781c --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-branch.d.ts @@ -0,0 +1,12 @@ +import type { AccessToken, RepoDesignation } from "../types/public"; +export declare function deleteBranch(params: { + repo: RepoDesignation; + /** + * The name of the branch to delete + */ + branch: string; + hubUrl?: string; + accessToken?: AccessToken; + fetch?: typeof fetch; +}): Promise; +//# sourceMappingURL=delete-branch.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-branch.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/delete-branch.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..0f892bf0613efd665aa95a675841133dcaf59cc2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-branch.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"delete-branch.d.ts","sourceRoot":"","sources":["../../../src/lib/delete-branch.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGpE,wBAAsB,YAAY,CAAC,MAAM,EAAE;IAC1C,IAAI,EAAE,eAAe,CAAC;IACtB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiBhB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-branch.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/delete-branch.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..cde6a23ddb883aea225dbd8aaa31f41de3b6e651 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-branch.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=delete-branch.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-branch.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/delete-branch.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..95303a80b82dcb2584a8fbd0e2f8dfe985f73ee8 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-branch.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"delete-branch.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/delete-branch.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-collection-item.d.ts b/node_modules/@huggingface/hub/dist/src/lib/delete-collection-item.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..c484ea53adabcb94f0cedfe4f688f8597a1824d5 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-collection-item.d.ts @@ -0,0 +1,18 @@ +import type { CredentialsParams } from "../types/public"; +export declare function deleteCollectionItem(params: { + /** + * The slug of the collection to delete the item from. + */ + slug: string; + /** + * The item object id which is different from the repo_id/paper_id provided when adding the item to the collection. + * This should be the _id property of the item. + */ + itemId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise; +//# sourceMappingURL=delete-collection-item.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-collection-item.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/delete-collection-item.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6b7d1d81d22c7d92f4e4f17bf727b5689c462a08 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-collection-item.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"delete-collection-item.d.ts","sourceRoot":"","sources":["../../../src/lib/delete-collection-item.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAGzD,wBAAsB,oBAAoB,CACzC,MAAM,EAAE;IACP;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,IAAI,CAAC,CAiBf"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-collection.d.ts b/node_modules/@huggingface/hub/dist/src/lib/delete-collection.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..5185848c35d0849e07a6779a02c6d308453b9707 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-collection.d.ts @@ -0,0 +1,13 @@ +import type { CredentialsParams } from "../types/public"; +export declare function deleteCollection(params: { + /** + * The slug of the collection to delete. + */ + slug: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise; +//# sourceMappingURL=delete-collection.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-collection.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/delete-collection.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c4eab52ce13111c2d10aa561d90e3d50df59189d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-collection.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"delete-collection.d.ts","sourceRoot":"","sources":["../../../src/lib/delete-collection.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAGzD,wBAAsB,gBAAgB,CACrC,MAAM,EAAE;IACP;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,IAAI,CAAC,CAkBf"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-file.d.ts b/node_modules/@huggingface/hub/dist/src/lib/delete-file.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..f392e5c9736169deb1e339cc32ace3865bab8212 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-file.d.ts @@ -0,0 +1,14 @@ +import type { CredentialsParams } from "../types/public"; +import type { CommitOutput, CommitParams } from "./commit"; +export declare function deleteFile(params: { + repo: CommitParams["repo"]; + path: string; + commitTitle?: CommitParams["title"]; + commitDescription?: CommitParams["description"]; + hubUrl?: CommitParams["hubUrl"]; + fetch?: CommitParams["fetch"]; + branch?: CommitParams["branch"]; + isPullRequest?: CommitParams["isPullRequest"]; + parentCommit?: CommitParams["parentCommit"]; +} & CredentialsParams): Promise; +//# sourceMappingURL=delete-file.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-file.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/delete-file.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6725759323e93a3d747349a34827915249d7d9e7 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-file.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"delete-file.d.ts","sourceRoot":"","sources":["../../../src/lib/delete-file.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAG3D,wBAAgB,UAAU,CACzB,MAAM,EAAE;IACP,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IACpC,iBAAiB,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAChD,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChC,KAAK,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAC9B,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChC,aAAa,CAAC,EAAE,YAAY,CAAC,eAAe,CAAC,CAAC;IAC9C,YAAY,CAAC,EAAE,YAAY,CAAC,cAAc,CAAC,CAAC;CAC5C,GAAG,iBAAiB,GACnB,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CAkBnC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-file.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/delete-file.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..de56d5e1937527ceaf00929900d0e63eae538847 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-file.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=delete-file.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-file.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/delete-file.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..76730de16109dc557c82460b6fa2ffffc4dd0333 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-file.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"delete-file.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/delete-file.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-files.d.ts b/node_modules/@huggingface/hub/dist/src/lib/delete-files.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d2e1b65049ecc4e75a6154dcb3b63bed72c215bc --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-files.d.ts @@ -0,0 +1,14 @@ +import type { CredentialsParams } from "../types/public"; +import type { CommitOutput, CommitParams } from "./commit"; +export declare function deleteFiles(params: { + repo: CommitParams["repo"]; + paths: string[]; + commitTitle?: CommitParams["title"]; + commitDescription?: CommitParams["description"]; + hubUrl?: CommitParams["hubUrl"]; + branch?: CommitParams["branch"]; + isPullRequest?: CommitParams["isPullRequest"]; + parentCommit?: CommitParams["parentCommit"]; + fetch?: CommitParams["fetch"]; +} & CredentialsParams): Promise; +//# sourceMappingURL=delete-files.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-files.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/delete-files.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a5fedf1663814906320c1cb024082893aa8e435f --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-files.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"delete-files.d.ts","sourceRoot":"","sources":["../../../src/lib/delete-files.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAG3D,wBAAgB,WAAW,CAC1B,MAAM,EAAE;IACP,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;IAC3B,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IACpC,iBAAiB,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAChD,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChC,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChC,aAAa,CAAC,EAAE,YAAY,CAAC,eAAe,CAAC,CAAC;IAC9C,YAAY,CAAC,EAAE,YAAY,CAAC,cAAc,CAAC,CAAC;IAC5C,KAAK,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;CAC9B,GAAG,iBAAiB,GACnB,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CAgBnC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-files.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/delete-files.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..37e127b0a03951796d2632f2dabc42ad830f6bfc --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-files.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=delete-files.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-files.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/delete-files.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..ddfacd00b9a937d6137521d7e6eacce2bd70b8dc --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-files.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"delete-files.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/delete-files.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-repo.d.ts b/node_modules/@huggingface/hub/dist/src/lib/delete-repo.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..8065e73476d6c31b9ee9826967a83fb2fe96dc4e --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-repo.d.ts @@ -0,0 +1,10 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +export declare function deleteRepo(params: { + repo: RepoDesignation; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): Promise; +//# sourceMappingURL=delete-repo.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/delete-repo.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/delete-repo.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c9188a8b02e215341be8918469ff7bacfd243dcc --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/delete-repo.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"delete-repo.d.ts","sourceRoot":"","sources":["../../../src/lib/delete-repo.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAI1E,wBAAsB,UAAU,CAC/B,MAAM,EAAE;IACP,IAAI,EAAE,eAAe,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,OAAO,CAAC,IAAI,CAAC,CA8Bf"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/download-file-to-cache-dir.d.ts b/node_modules/@huggingface/hub/dist/src/lib/download-file-to-cache-dir.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..0d296592fd8bd4a60b45c8223f1b3ee8a939eba7 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/download-file-to-cache-dir.d.ts @@ -0,0 +1,30 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +export declare const REGEX_COMMIT_HASH: RegExp; +/** + * Download a given file if it's not already present in the local cache. + * @param params + * @return the symlink to the blob object + */ +export declare function downloadFileToCacheDir(params: { + repo: RepoDesignation; + path: string; + /** + * If true, will download the raw git file. + * + * For example, when calling on a file stored with Git LFS, the pointer file will be downloaded instead. + */ + raw?: boolean; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + * + * @default "main" + */ + revision?: string; + hubUrl?: string; + cacheDir?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise; +//# sourceMappingURL=download-file-to-cache-dir.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/download-file-to-cache-dir.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/download-file-to-cache-dir.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c1739818e982759f46b1cfbd4573ff92425d7d83 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/download-file-to-cache-dir.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"download-file-to-cache-dir.d.ts","sourceRoot":"","sources":["../../../src/lib/download-file-to-cache-dir.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAS1E,eAAO,MAAM,iBAAiB,EAAE,MAAqC,CAAC;AAyBtE;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC3C,MAAM,EAAE;IACP,IAAI,EAAE,eAAe,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,GAAG,CAAC,EAAE,OAAO,CAAC;IACd;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,MAAM,CAAC,CAgFjB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/download-file-to-cache-dir.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/download-file-to-cache-dir.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..877fde63e081bf510b9659e3f2c17210d9370779 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/download-file-to-cache-dir.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=download-file-to-cache-dir.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/download-file-to-cache-dir.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/download-file-to-cache-dir.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a4a149600740909d75e4350a6210ad004cf9f249 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/download-file-to-cache-dir.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"download-file-to-cache-dir.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/download-file-to-cache-dir.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/download-file.d.ts b/node_modules/@huggingface/hub/dist/src/lib/download-file.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..be551a0d89f55287283d679ecd20a3f012942abe --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/download-file.d.ts @@ -0,0 +1,43 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import type { XetReadToken } from "../utils/XetBlob"; +import type { FileDownloadInfoOutput } from "./file-download-info"; +/** + * @returns null when the file doesn't exist + */ +export declare function downloadFile(params: { + repo: RepoDesignation; + path: string; + /** + * If true, will download the raw git file. + * + * For example, when calling on a file stored with Git LFS, the pointer file will be downloaded instead. + */ + raw?: boolean; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + * + * @default "main" + */ + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + /** + * Whether to use the xet protocol to download the file (if applicable). + * + * When an object with `readToken` is provided along with `downloadInfo`, + * the xet download can skip the token refresh roundtrip. + * + * @default true + */ + xet?: boolean | { + readToken: XetReadToken; + }; + /** + * Can save an http request if provided + */ + downloadInfo?: FileDownloadInfoOutput; +} & Partial): Promise; +//# sourceMappingURL=download-file.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/download-file.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/download-file.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5c4c3048d3422c8494f3f1dbb7ea75934e536f22 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/download-file.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"download-file.d.ts","sourceRoot":"","sources":["../../../src/lib/download-file.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAI1E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAGnE;;GAEG;AACH,wBAAsB,YAAY,CACjC,MAAM,EAAE;IACP,IAAI,EAAE,eAAe,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,GAAG,CAAC,EAAE,OAAO,CAAC;IACd;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB;;;;;;;OAOG;IACH,GAAG,CAAC,EAAE,OAAO,GAAG;QAAE,SAAS,EAAE,YAAY,CAAA;KAAE,CAAC;IAC5C;;OAEG;IACH,YAAY,CAAC,EAAE,sBAAsB,CAAC;CACtC,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,CA+BtB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/download-file.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/download-file.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4a3b1c59ec7e6cf069d6bcdcd0a780c972be1cda --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/download-file.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=download-file.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/download-file.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/download-file.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6b111e57f0768fd6d4c07656fd8f0bfe296f1f98 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/download-file.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"download-file.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/download-file.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/file-download-info.d.ts b/node_modules/@huggingface/hub/dist/src/lib/file-download-info.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..8fc52c1a1d3ef04dd4dbf57e274791acb2f401fd --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/file-download-info.d.ts @@ -0,0 +1,39 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +export interface XetFileInfo { + hash: string; + refreshUrl: URL; + /** + * Can be directly used instead of the hash. + */ + reconstructionUrl: URL; +} +export interface FileDownloadInfoOutput { + size: number; + etag: string; + xet?: XetFileInfo; + url: string; +} +/** + * @returns null when the file doesn't exist + */ +export declare function fileDownloadInfo(params: { + repo: RepoDesignation; + path: string; + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + /** + * To get the raw pointer file behind a LFS file + */ + raw?: boolean; + /** + * To avoid the content-disposition header in the `downloadLink` for LFS files + * + * So that on browsers you can use the URL in an iframe for example + */ + noContentDisposition?: boolean; +} & Partial): Promise; +//# sourceMappingURL=file-download-info.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/file-download-info.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/file-download-info.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7ad12c1ca2204c80bcf5b7f2b670aa4844622612 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/file-download-info.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"file-download-info.d.ts","sourceRoot":"","sources":["../../../src/lib/file-download-info.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAK1E,MAAM,WAAW,WAAW;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,GAAG,CAAC;IAChB;;OAEG;IACH,iBAAiB,EAAE,GAAG,CAAC;CACvB;AAED,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,WAAW,CAAC;IAElB,GAAG,EAAE,MAAM,CAAC;CACZ;AACD;;GAEG;AACH,wBAAsB,gBAAgB,CACrC,MAAM,EAAE;IACP,IAAI,EAAE,eAAe,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB;;OAEG;IACH,GAAG,CAAC,EAAE,OAAO,CAAC;IACd;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAC/B,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAwGxC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/file-download-info.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/file-download-info.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2fca4384f7e94c8c0cfb38fb2ceddbb949459aa5 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/file-download-info.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=file-download-info.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/file-download-info.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/file-download-info.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..bca08215540bb7a16469114e160a3e99bc9f2365 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/file-download-info.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"file-download-info.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/file-download-info.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/file-exists.d.ts b/node_modules/@huggingface/hub/dist/src/lib/file-exists.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..1e546f37098ba44544c5c6dae0c95c2fd571fd5d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/file-exists.d.ts @@ -0,0 +1,12 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +export declare function fileExists(params: { + repo: RepoDesignation; + path: string; + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise; +//# sourceMappingURL=file-exists.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/file-exists.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/file-exists.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..34a4a2e7ea1dd960e3fea506356026bc499f61f5 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/file-exists.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"file-exists.d.ts","sourceRoot":"","sources":["../../../src/lib/file-exists.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAI1E,wBAAsB,UAAU,CAC/B,MAAM,EAAE;IACP,IAAI,EAAE,eAAe,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,OAAO,CAAC,CAyBlB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/file-exists.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/file-exists.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..c153d9c7630e5ca4510a7664a551fe0fa77614a8 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/file-exists.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=file-exists.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/file-exists.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/file-exists.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6560306e2c341b5262a9a193d86cc8b917f2f4c5 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/file-exists.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"file-exists.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/file-exists.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/index.d.ts b/node_modules/@huggingface/hub/dist/src/lib/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..c07f59df1a7960e4a811d3decf3a8ee2c50b5753 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/index.d.ts @@ -0,0 +1,38 @@ +export * from "./cache-management"; +export * from "./check-repo-access"; +export * from "./commit"; +export * from "./copy-files"; +export * from "./count-commits"; +export * from "./create-repo"; +export * from "./create-branch"; +export * from "./create-collection"; +export * from "./dataset-info"; +export * from "./delete-branch"; +export * from "./delete-file"; +export * from "./delete-files"; +export * from "./delete-repo"; +export * from "./delete-collection"; +export * from "./download-file"; +export * from "./download-file-to-cache-dir"; +export * from "./file-download-info"; +export * from "./file-exists"; +export * from "./jobs"; +export * from "./list-commits"; +export * from "./list-datasets"; +export * from "./list-files"; +export * from "./list-models"; +export * from "./list-spaces"; +export * from "./list-collections"; +export * from "./model-info"; +export * from "./oauth-handle-redirect"; +export * from "./oauth-login-url"; +export * from "./parse-safetensors-metadata"; +export * from "./paths-info"; +export * from "./repo-exists"; +export * from "./snapshot-download"; +export * from "./space-info"; +export * from "./upload-file"; +export * from "./upload-files"; +export * from "./upload-files-with-progress"; +export * from "./who-am-i"; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/index.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..73f52a423cc42c3b955ac2f1d64fb7be60a286c0 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC;AAC7B,cAAc,iBAAiB,CAAC;AAChC,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,eAAe,CAAC;AAC9B,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,sBAAsB,CAAC;AACrC,cAAc,eAAe,CAAC;AAC9B,cAAc,QAAQ,CAAC;AACvB,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC;AAClC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC;AAC9B,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,8BAA8B,CAAC;AAC7C,cAAc,YAAY,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/cancel-job.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/cancel-job.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..64ed7fac07f6314e967304b1592d02a9900f7767 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/cancel-job.d.ts @@ -0,0 +1,21 @@ +import type { CredentialsParams } from "../../types/public"; +import type { ApiJob } from "../../types/api/api-jobs"; +/** + * Cancel a job. + */ +export declare function cancelJob(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): Promise; +//# sourceMappingURL=cancel-job.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/cancel-job.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/cancel-job.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..042c4d7a907865b590f47d168242e86c74dab5bc --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/cancel-job.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"cancel-job.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/cancel-job.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE5D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAEvD;;GAEG;AACH,wBAAsB,SAAS,CAC9B,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,OAAO,CAAC,MAAM,CAAC,CAmBjB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/create-scheduled-job.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/create-scheduled-job.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..9d6ccb2212d167dd4e6147d385d31d8a29c8732d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/create-scheduled-job.d.ts @@ -0,0 +1,17 @@ +import type { CredentialsParams } from "../../types/public"; +import type { ApiScheduledJob, CreateScheduledJobOptions } from "../../types/api/api-jobs"; +/** + * Create a scheduled job. + */ +export declare function createScheduledJob(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CreateScheduledJobOptions & CredentialsParams): Promise; +//# sourceMappingURL=create-scheduled-job.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/create-scheduled-job.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/create-scheduled-job.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..36f172afbe76f3037231a0e02cd7393eb82d747c --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/create-scheduled-job.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"create-scheduled-job.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/create-scheduled-job.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAG5D,OAAO,KAAK,EAAE,eAAe,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AAE3F;;GAEG;AACH,wBAAsB,kBAAkB,CACvC,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,yBAAyB,GAC5B,iBAAiB,GAChB,OAAO,CAAC,eAAe,CAAC,CAoE1B"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/delete-scheduled-job.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/delete-scheduled-job.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..5509893ed3adb64006ed1b92735dcdcd7c142687 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/delete-scheduled-job.d.ts @@ -0,0 +1,20 @@ +import type { CredentialsParams } from "../../types/public"; +/** + * Delete a scheduled job. + */ +export declare function deleteScheduledJob(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The scheduled job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): Promise; +//# sourceMappingURL=delete-scheduled-job.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/delete-scheduled-job.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/delete-scheduled-job.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6fcff09479eb30134f6cf967305c3a705468bb46 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/delete-scheduled-job.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"delete-scheduled-job.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/delete-scheduled-job.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAG5D;;GAEG;AACH,wBAAsB,kBAAkB,CACvC,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,OAAO,CAAC,IAAI,CAAC,CAiBf"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/duplicate-job.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/duplicate-job.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..931cc533c4d1989e7d0c73440ccc0929ca19107a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/duplicate-job.d.ts @@ -0,0 +1,21 @@ +import type { CredentialsParams } from "../../types/public"; +import type { ApiJob } from "../../types/api/api-jobs"; +/** + * Duplicate a job (re-run with the same spec). + */ +export declare function duplicateJob(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The job ID to duplicate + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): Promise; +//# sourceMappingURL=duplicate-job.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/duplicate-job.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/duplicate-job.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..01b0a15c2eed2bdcf2d8292687c9443bd44587ec --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/duplicate-job.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"duplicate-job.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/duplicate-job.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE5D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAEvD;;GAEG;AACH,wBAAsB,YAAY,CACjC,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,OAAO,CAAC,MAAM,CAAC,CAmBjB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/get-job.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/get-job.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4a578f0faa6d365c2d9f2ff4a5287496945a8751 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/get-job.d.ts @@ -0,0 +1,21 @@ +import type { CredentialsParams } from "../../types/public"; +import type { ApiJob } from "../../types/api/api-jobs"; +/** + * Get a specific job by ID. + */ +export declare function getJob(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): Promise; +//# sourceMappingURL=get-job.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/get-job.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/get-job.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..0c5d39cb76fc6165180ad50e6b6131c3a1413dac --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/get-job.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"get-job.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/get-job.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE5D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAEvD;;GAEG;AACH,wBAAsB,MAAM,CAC3B,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,OAAO,CAAC,MAAM,CAAC,CAiBjB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/get-scheduled-job.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/get-scheduled-job.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ad727f023622f2a5fa8f142aece75492c0f44435 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/get-scheduled-job.d.ts @@ -0,0 +1,21 @@ +import type { CredentialsParams } from "../../types/public"; +import type { ApiScheduledJob } from "../../types/api/api-jobs"; +/** + * Get a specific scheduled job by ID. + */ +export declare function getScheduledJob(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The scheduled job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): Promise; +//# sourceMappingURL=get-scheduled-job.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/get-scheduled-job.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/get-scheduled-job.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..818cf5e1a6d97d7d9f77c929079400392c7a48b4 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/get-scheduled-job.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"get-scheduled-job.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/get-scheduled-job.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE5D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAEhE;;GAEG;AACH,wBAAsB,eAAe,CACpC,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,OAAO,CAAC,eAAe,CAAC,CAiB1B"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/index.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..6d691cfce8791153c8c7d5062103cb130c26e460 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/index.d.ts @@ -0,0 +1,17 @@ +export * from "./cancel-job"; +export * from "./create-scheduled-job"; +export * from "./delete-scheduled-job"; +export * from "./duplicate-job"; +export * from "./get-job"; +export * from "./get-scheduled-job"; +export * from "./list-job-hardware"; +export * from "./list-jobs"; +export * from "./list-scheduled-jobs"; +export * from "./resume-scheduled-job"; +export * from "./run-job"; +export * from "./run-scheduled-job"; +export * from "./stream-job-events"; +export * from "./stream-job-logs"; +export * from "./stream-job-metrics"; +export * from "./suspend-scheduled-job"; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/index.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..71acf13901c3eff1057490a57660cdc07467180e --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,iBAAiB,CAAC;AAChC,cAAc,WAAW,CAAC;AAC1B,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,cAAc,aAAa,CAAC;AAC5B,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,WAAW,CAAC;AAC1B,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,yBAAyB,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/list-job-hardware.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/list-job-hardware.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4b92d8233f0b3c07c1e72c179bb67d5f4a12bbe2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/list-job-hardware.d.ts @@ -0,0 +1,14 @@ +import type { CredentialsParams } from "../../types/public"; +import type { ApiJobHardware } from "../../types/api/api-jobs"; +/** + * Get the list of available hardware for jobs. + * This endpoint is public and does not require authentication, but authentication is optional. + */ +export declare function listJobHardware(params?: { + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise; +//# sourceMappingURL=list-job-hardware.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/list-job-hardware.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/list-job-hardware.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..67466a3e6f109ede95c942318236ee52c3b97202 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/list-job-hardware.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-job-hardware.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/list-job-hardware.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE5D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAE/D;;;GAGG;AACH,wBAAsB,eAAe,CACpC,MAAM,CAAC,EAAE;IACR,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,cAAc,EAAE,CAAC,CAkB3B"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/list-jobs.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/list-jobs.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..566252ea02c15eaece5baa781d6834094aec2ffa --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/list-jobs.d.ts @@ -0,0 +1,17 @@ +import type { CredentialsParams } from "../../types/public"; +import type { ApiJob } from "../../types/api/api-jobs"; +/** + * List jobs for a namespace (user or organization). + */ +export declare function listJobs(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): Promise; +//# sourceMappingURL=list-jobs.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/list-jobs.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/list-jobs.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d797154c2d0283b3f3e9bbb13838e81eaec36519 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/list-jobs.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-jobs.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/list-jobs.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE5D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAEvD;;GAEG;AACH,wBAAsB,QAAQ,CAC7B,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,OAAO,CAAC,MAAM,EAAE,CAAC,CAcnB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/list-scheduled-jobs.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/list-scheduled-jobs.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..10721343d81fee600c979d3b1a304406d28aa128 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/list-scheduled-jobs.d.ts @@ -0,0 +1,17 @@ +import type { CredentialsParams } from "../../types/public"; +import type { ApiScheduledJob } from "../../types/api/api-jobs"; +/** + * List scheduled jobs for a namespace. + */ +export declare function listScheduledJobs(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): Promise; +//# sourceMappingURL=list-scheduled-jobs.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/list-scheduled-jobs.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/list-scheduled-jobs.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..18eb7e7b865f0cf78ebcd19129c704da241a7a57 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/list-scheduled-jobs.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-scheduled-jobs.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/list-scheduled-jobs.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE5D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAEhE;;GAEG;AACH,wBAAsB,iBAAiB,CACtC,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,OAAO,CAAC,eAAe,EAAE,CAAC,CAc5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/resume-scheduled-job.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/resume-scheduled-job.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ed90c4a2d6f77835f7b774dc42f24954b50147b8 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/resume-scheduled-job.d.ts @@ -0,0 +1,20 @@ +import type { CredentialsParams } from "../../types/public"; +/** + * Resume a scheduled job. + */ +export declare function resumeScheduledJob(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The scheduled job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): Promise; +//# sourceMappingURL=resume-scheduled-job.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/resume-scheduled-job.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/resume-scheduled-job.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f4399de8175267ef432a6b2d58338ee207df29aa --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/resume-scheduled-job.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"resume-scheduled-job.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/resume-scheduled-job.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAG5D;;GAEG;AACH,wBAAsB,kBAAkB,CACvC,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,OAAO,CAAC,IAAI,CAAC,CAgBf"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/run-job.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/run-job.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..9530a7ffc049265712208acb27243b9738b495c4 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/run-job.d.ts @@ -0,0 +1,18 @@ +import type { CredentialsParams } from "../../types/public"; +import type { ApiJob, CreateJobOptions } from "../../types/api/api-jobs"; +export type { JobVolume } from "../../types/api/api-jobs"; +/** + * Run a new job. + */ +export declare function runJob(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CreateJobOptions & CredentialsParams): Promise; +//# sourceMappingURL=run-job.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/run-job.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/run-job.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a1f027d7641133cd6e7c7669fef97c7218b9ee7e --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/run-job.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"run-job.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/run-job.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAG5D,OAAO,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACzE,YAAY,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAE1D;;GAEG;AACH,wBAAsB,MAAM,CAC3B,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,gBAAgB,GACnB,iBAAiB,GAChB,OAAO,CAAC,MAAM,CAAC,CAgEjB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/run-scheduled-job.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/run-scheduled-job.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..cf92a7a09d1f144a51c8d1263f080a2f71864e4f --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/run-scheduled-job.d.ts @@ -0,0 +1,22 @@ +import type { CredentialsParams } from "../../types/public"; +import type { ApiJob } from "../../types/api/api-jobs"; +/** + * Trigger a scheduled job to run immediately. + * Returns the job that was triggered, or null if another instance is already running. + */ +export declare function runScheduledJob(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The scheduled job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): Promise; +//# sourceMappingURL=run-scheduled-job.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/run-scheduled-job.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/run-scheduled-job.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f3d23dd6a751b2d8e846c492764308edb5f5c945 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/run-scheduled-job.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"run-scheduled-job.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/run-scheduled-job.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE5D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAEvD;;;GAGG;AACH,wBAAsB,eAAe,CACpC,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAuBxB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-events.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-events.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ec45c1c86652c8a2374bb92efdc3420e0b0ae4ce --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-events.d.ts @@ -0,0 +1,21 @@ +import type { CredentialsParams } from "../../types/public"; +/** + * Stream job events using Server-Sent Events (SSE). + * Returns an async iterable of event chunks. + */ +export declare function streamJobEvents(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): AsyncGenerator; +//# sourceMappingURL=stream-job-events.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-events.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-events.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..34428eac8cf31a80866ea6cfe6ca83be8c976792 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-events.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"stream-job-events.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/stream-job-events.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAG5D;;;GAGG;AACH,wBAAuB,eAAe,CACrC,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,cAAc,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAuDvC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-logs.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-logs.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..debe63c185db000d71c8d9fa8136216224893143 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-logs.d.ts @@ -0,0 +1,24 @@ +import type { CredentialsParams } from "../../types/public"; +/** + * Stream job logs using Server-Sent Events (SSE). + * Returns an async iterable of log chunks. + */ +export declare function streamJobLogs(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): AsyncGenerator<{ + message: string; + timestamp: Date; +}, void, unknown>; +//# sourceMappingURL=stream-job-logs.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-logs.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-logs.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..68671d851d27bd0e008cda3926689cd90af13199 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-logs.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"stream-job-logs.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/stream-job-logs.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAG5D;;;GAGG;AACH,wBAAuB,aAAa,CACnC,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,cAAc,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,EAAE,IAAI,EAAE,OAAO,CAAC,CAiErE"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-metrics.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-metrics.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..92b10fdfdfa7d3a9b85f3a68783fa759c46d0068 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-metrics.d.ts @@ -0,0 +1,21 @@ +import type { CredentialsParams } from "../../types/public"; +/** + * Stream job metrics using Server-Sent Events (SSE). + * Returns an async iterable of metric chunks. + */ +export declare function streamJobMetrics(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): AsyncGenerator; +//# sourceMappingURL=stream-job-metrics.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-metrics.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-metrics.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7d94b855eef463b2865fa60606d70efc927699b6 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/stream-job-metrics.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"stream-job-metrics.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/stream-job-metrics.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAG5D;;;GAGG;AACH,wBAAuB,gBAAgB,CACtC,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,cAAc,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAuDvC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/suspend-scheduled-job.d.ts b/node_modules/@huggingface/hub/dist/src/lib/jobs/suspend-scheduled-job.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..97b1cc4c15da1bbd0c169678b390cee4ee0a3179 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/suspend-scheduled-job.d.ts @@ -0,0 +1,20 @@ +import type { CredentialsParams } from "../../types/public"; +/** + * Suspend (pause) a scheduled job. + */ +export declare function suspendScheduledJob(params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The scheduled job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): Promise; +//# sourceMappingURL=suspend-scheduled-job.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/jobs/suspend-scheduled-job.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/jobs/suspend-scheduled-job.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..73354a1af2e1bdafaa1614c39a52804fe4ea753d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/jobs/suspend-scheduled-job.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"suspend-scheduled-job.d.ts","sourceRoot":"","sources":["../../../../src/lib/jobs/suspend-scheduled-job.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAG5D;;GAEG;AACH,wBAAsB,mBAAmB,CACxC,MAAM,EAAE;IACP;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,OAAO,CAAC,IAAI,CAAC,CAiBf"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-collections.d.ts b/node_modules/@huggingface/hub/dist/src/lib/list-collections.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..89d4c269adb1207f6e01f7057fbaa4773537216b --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-collections.d.ts @@ -0,0 +1,34 @@ +import type { CredentialsParams } from "../types/public"; +import type { ApiCollectionInfo } from "../types/api/api-collection"; +export declare function listCollections(params?: { + search?: { + /** + * Filter collections created by specific owners (users or organizations). + */ + owner?: string[]; + /** + * Filter collections containing specific items. + * Value must be the item_type and item_id concatenated. + * Example: "models/teknium/OpenHermes-2.5-Mistral-7B", "datasets/rajpurkar/squad" or "papers/2311.12983". + */ + item?: string[]; + /** + * Filter based on substrings for titles & descriptions. + */ + q?: string; + }; + /** + * Sort the returned collections. Supported values are "lastModified", "trending" (default) and "upvotes". + */ + sort?: "lastModified" | "trending" | "upvotes"; + /** + * Set to limit the number of collections returned. + */ + limit?: number; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): AsyncGenerator; +//# sourceMappingURL=list-collections.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-collections.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/list-collections.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..67ec7d14cdc09cc09399fa25d6816cfae068d60c --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-collections.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-collections.d.ts","sourceRoot":"","sources":["../../../src/lib/list-collections.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAGzD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAMrE,wBAAuB,eAAe,CACrC,MAAM,CAAC,EAAE;IACR,MAAM,CAAC,EAAE;QACR;;WAEG;QACH,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;QACjB;;;;WAIG;QACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB;;WAEG;QACH,CAAC,CAAC,EAAE,MAAM,CAAC;KACX,CAAC;IACF;;OAEG;IACH,IAAI,CAAC,EAAE,cAAc,GAAG,UAAU,GAAG,SAAS,CAAC;IAC/C;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,cAAc,CAAC,iBAAiB,CAAC,CA0DnC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-collections.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/list-collections.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..345b6da35996bd241fa7c68ae0abf7f5ccb20455 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-collections.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=list-collections.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-collections.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/list-collections.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..18b2dda0371e61dcaa2952fb48eff8728516ce8b --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-collections.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-collections.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/list-collections.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-commits.d.ts b/node_modules/@huggingface/hub/dist/src/lib/list-commits.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..9a4986670dcf07c9c9805573f42245244853bcfb --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-commits.d.ts @@ -0,0 +1,28 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +export interface CommitData { + oid: string; + title: string; + message: string; + authors: Array<{ + username: string; + avatarUrl: string; + }>; + date: Date; +} +export declare function listCommits(params: { + repo: RepoDesignation; + /** + * Revision to list commits from. Defaults to the default branch. + */ + revision?: string; + hubUrl?: string; + /** + * Number of commits to fetch from the hub each http call. Defaults to 100. Can be set to 1000. + */ + batchSize?: number; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): AsyncGenerator; +//# sourceMappingURL=list-commits.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-commits.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/list-commits.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f0ed8143ac9ba451d3e337498b73791de1396cfe --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-commits.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-commits.d.ts","sourceRoot":"","sources":["../../../src/lib/list-commits.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAK1E,MAAM,WAAW,UAAU;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACxD,IAAI,EAAE,IAAI,CAAC;CACX;AAED,wBAAuB,WAAW,CACjC,MAAM,EAAE;IACP,IAAI,EAAE,eAAe,CAAC;IACtB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,cAAc,CAAC,UAAU,CAAC,CAoC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-commits.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/list-commits.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..145da942cfd67dbfcaf233a0d9c72c03ca48f7f7 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-commits.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=list-commits.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-commits.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/list-commits.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a2800c70818e14a5f25f8682ec4ae9e34022c273 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-commits.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-commits.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/list-commits.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-datasets.d.ts b/node_modules/@huggingface/hub/dist/src/lib/list-datasets.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..cc5a43dfa3ecba52e2a16a2e0daebcd9effa1cb1 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-datasets.d.ts @@ -0,0 +1,38 @@ +import type { ApiDatasetInfo } from "../types/api/api-dataset"; +import type { CredentialsParams } from "../types/public"; +export declare const DATASET_EXPAND_KEYS: readonly ["private", "downloads", "gated", "likes", "lastModified"]; +export declare const DATASET_EXPANDABLE_KEYS: readonly ["author", "cardData", "citation", "createdAt", "disabled", "description", "downloads", "downloadsAllTime", "gated", "gitalyUid", "lastModified", "likes", "paperswithcode_id", "private", "sha", "tags"]; +export interface DatasetEntry { + id: string; + name: string; + private: boolean; + downloads: number; + gated: false | "auto" | "manual"; + likes: number; + updatedAt: Date; +} +export declare function listDatasets = never>(params?: { + search?: { + /** + * Will search in the dataset name for matches + */ + query?: string; + owner?: string; + tags?: string[]; + }; + hubUrl?: string; + additionalFields?: T[]; + /** + * Set to limit the number of datasets returned. + */ + limit?: number; + /** + * Sort datasets by a specific field. + */ + sort?: "createdAt" | "downloads" | "likes" | "lastModified" | "likes30d" | "trendingScore" | "datasetsServerInfo.numRows" | "mainSize" | "id"; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): AsyncGenerator>; +//# sourceMappingURL=list-datasets.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-datasets.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/list-datasets.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..89fd6dcda3b296bd6c0aae23112a90a43eb38437 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-datasets.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-datasets.d.ts","sourceRoot":"","sources":["../../../src/lib/list-datasets.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC/D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAKzD,eAAO,MAAM,mBAAmB,qEAMsB,CAAC;AAEvD,eAAO,MAAM,uBAAuB,oNAkBkB,CAAC;AAEvD,MAAM,WAAW,YAAY;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;CAChB;AAED,wBAAuB,YAAY,CAClC,KAAK,CAAC,CAAC,SAAS,OAAO,CAAC,CAAC,OAAO,uBAAuB,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,EAE/G,MAAM,CAAC,EAAE;IACR,MAAM,CAAC,EAAE;QACR;;WAEG;QACH,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;KAChB,CAAC;IACF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,CAAC,EAAE,CAAC;IACvB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,IAAI,CAAC,EACF,WAAW,GACX,WAAW,GACX,OAAO,GACP,cAAc,GACd,UAAU,GACV,eAAe,GACf,4BAA4B,GAC5B,UAAU,GACV,IAAI,CAAC;IACR;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,cAAc,CAAC,YAAY,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAoDxD"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-datasets.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/list-datasets.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..519940c354cd662287e9f5367f13af6ec5b2ce1a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-datasets.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=list-datasets.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-datasets.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/list-datasets.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..fed961e2907d53002f954edbc08ecbfc9b23dff4 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-datasets.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-datasets.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/list-datasets.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-files.d.ts b/node_modules/@huggingface/hub/dist/src/lib/list-files.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..26df64962d3669ea36e11f528978678d0a2edd8d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-files.d.ts @@ -0,0 +1,67 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +export interface ListFileEntry { + type: "file" | "directory" | "unknown"; + size: number; + path: string; + /** + * Not available for bucket repos. + */ + oid?: string; + lfs?: { + oid: string; + size: number; + /** Size of the raw pointer file, 100~200 bytes */ + pointerSize: number; + }; + /** + * Xet-backed hash, a new protocol replacing LFS for big files. + */ + xetHash?: string; + /** + * Only fetched if `expand` is set to `true` in the `listFiles` call. + * + * Not available for bucket repos, use {@link uploadedAt} instead. + */ + lastCommit?: { + date: string; + id: string; + title: string; + }; + /** + * Only fetched if `expand` is set to `true` in the `listFiles` call. + * + * Only available for bucket repos. + */ + uploadedAt?: string; + /** + * Only fetched if `expand` is set to `true` in the `listFiles` call. + */ + securityFileStatus?: unknown; +} +/** + * List files in a folder. To list ALL files in the directory, call it + * with {@link params.recursive} set to `true`. + */ +export declare function listFiles(params: { + repo: RepoDesignation; + /** + * Do we want to list files in subdirectories? + */ + recursive?: boolean; + /** + * Eg 'data' for listing all files in the 'data' folder. Leave it empty to list all + * files in the repo. + */ + path?: string; + /** + * Fetch `lastCommit` and `securityFileStatus` for each file. + */ + expand?: boolean; + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): AsyncGenerator; +//# sourceMappingURL=list-files.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-files.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/list-files.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d8bc981cf92d35c35aa35b83705fdbf3b48e137b --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-files.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-files.d.ts","sourceRoot":"","sources":["../../../src/lib/list-files.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAK1E,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAAC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE;QACL,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,kDAAkD;QAClD,WAAW,EAAE,MAAM,CAAC;KACpB,CAAC;IACF;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,UAAU,CAAC,EAAE;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,EAAE,EAAE,MAAM,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;KACd,CAAC;IACF;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;;GAGG;AACH,wBAAuB,SAAS,CAC/B,MAAM,EAAE;IACP,IAAI,EAAE,eAAe,CAAC;IACtB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,cAAc,CAAC,aAAa,CAAC,CA8B/B"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-files.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/list-files.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ad583c6ddbb16d54cbfbf4bdd81967667d92ec9d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-files.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=list-files.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-files.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/list-files.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..b1bccb466b1eb437c23214480376d87e4664d5f0 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-files.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-files.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/list-files.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-models.d.ts b/node_modules/@huggingface/hub/dist/src/lib/list-models.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..363ef06df455ea13ed648865cd562901bef63093 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-models.d.ts @@ -0,0 +1,54 @@ +import type { ApiModelInfo } from "../types/api/api-model"; +import type { CredentialsParams, PipelineType } from "../types/public"; +export declare const MODEL_EXPAND_KEYS: readonly ["pipeline_tag", "private", "gated", "downloads", "likes", "lastModified"]; +export declare const MODEL_EXPANDABLE_KEYS: readonly ["author", "cardData", "config", "createdAt", "disabled", "downloads", "downloadsAllTime", "gated", "gitalyUid", "inferenceProviderMapping", "lastModified", "library_name", "likes", "model-index", "pipeline_tag", "private", "safetensors", "sha", "spaces", "tags", "transformersInfo"]; +export interface ModelDerivedFields { + filePaths: string[]; +} +export declare const MODEL_DERIVED_FIELD_TO_API_KEY: Record; +export type ModelAdditionalField = Exclude<(typeof MODEL_EXPANDABLE_KEYS)[number], (typeof MODEL_EXPAND_KEYS)[number]> | keyof ModelDerivedFields; +export type ResolveModelAdditionalFields = Pick & Pick; +export interface ModelEntry { + id: string; + name: string; + private: boolean; + gated: false | "auto" | "manual"; + task?: PipelineType; + likes: number; + downloads: number; + updatedAt: Date; +} +export declare function listModels(params?: { + search?: { + /** + * Will search in the model name for matches + */ + query?: string; + owner?: string; + task?: PipelineType; + tags?: string[]; + /** + * Will search for models that have one of the inference providers in the list. + */ + inferenceProviders?: string[]; + /** + * Will search for models that support at least one of those local apps (eg "lmstudio", "mlx-lm", ...) + */ + apps?: string[]; + }; + hubUrl?: string; + additionalFields?: T[]; + /** + * Set to limit the number of models returned. + */ + limit?: number; + /** + * Sort models by a specific field. + */ + sort?: "createdAt" | "downloads" | "likes" | "lastModified" | "likes30d" | "trendingScore" | "num_parameters" | "mainSize" | "id"; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): AsyncGenerator>; +//# sourceMappingURL=list-models.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-models.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/list-models.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..9e69e5d9e2ebf83f629013fe9a985198c4960e35 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-models.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-models.d.ts","sourceRoot":"","sources":["../../../src/lib/list-models.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAKvE,eAAO,MAAM,iBAAiB,qFAOsB,CAAC;AAErD,eAAO,MAAM,qBAAqB,sSAsBkB,CAAC;AAErD,MAAM,WAAW,kBAAkB;IAClC,SAAS,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,eAAO,MAAM,8BAA8B,EAAE,MAAM,CAAC,MAAM,kBAAkB,EAAE,MAAM,YAAY,CAE/F,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAC7B,OAAO,CAAC,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC,GACnF,MAAM,kBAAkB,CAAC;AAE5B,MAAM,MAAM,4BAA4B,CAAC,CAAC,SAAS,oBAAoB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC,GAAG,MAAM,YAAY,CAAC,GACpH,IAAI,CAAC,kBAAkB,EAAE,CAAC,GAAG,MAAM,kBAAkB,CAAC,CAAC;AAExD,MAAM,WAAW,UAAU;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAC;IACjC,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,IAAI,CAAC;CAChB;AAED,wBAAuB,UAAU,CAAC,KAAK,CAAC,CAAC,SAAS,oBAAoB,GAAG,KAAK,EAC7E,MAAM,CAAC,EAAE;IACR,MAAM,CAAC,EAAE;QACR;;WAEG;QACH,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB;;WAEG;QACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;QAC9B;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;KAChB,CAAC;IACF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,CAAC,EAAE,CAAC;IACvB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,IAAI,CAAC,EACF,WAAW,GACX,WAAW,GACX,OAAO,GACP,cAAc,GACd,UAAU,GACV,eAAe,GACf,gBAAgB,GAChB,UAAU,GACV,IAAI,CAAC;IACR;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,cAAc,CAAC,UAAU,GAAG,4BAA4B,CAAC,CAAC,CAAC,CAAC,CA+E9D"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-models.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/list-models.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..5485cfb68c947b882c2fabb975f2559e4d311c6a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-models.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=list-models.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-models.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/list-models.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..9c79d0f1f444fa47f349016a9d542c05a530624d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-models.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-models.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/list-models.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-spaces.d.ts b/node_modules/@huggingface/hub/dist/src/lib/list-spaces.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..9dc53fd06ef6a4b9a92c13b82801ec4d15e2567d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-spaces.d.ts @@ -0,0 +1,36 @@ +import type { ApiSpaceInfo } from "../types/api/api-space"; +import type { CredentialsParams, SpaceSdk } from "../types/public"; +export declare const SPACE_EXPAND_KEYS: readonly ["sdk", "likes", "private", "lastModified"]; +export declare const SPACE_EXPANDABLE_KEYS: readonly ["author", "cardData", "datasets", "disabled", "gitalyUid", "lastModified", "createdAt", "likes", "private", "runtime", "sdk", "sha", "subdomain", "tags", "models"]; +export interface SpaceEntry { + id: string; + name: string; + sdk?: SpaceSdk; + likes: number; + private: boolean; + updatedAt: Date; +} +export declare function listSpaces = never>(params?: { + search?: { + /** + * Will search in the space name for matches + */ + query?: string; + owner?: string; + tags?: string[]; + }; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + /** + * Additional fields to fetch from huggingface.co. + */ + additionalFields?: T[]; + /** + * Sort spaces by a specific field. + */ + sort?: "createdAt" | "downloads" | "likes" | "lastModified" | "likes30d" | "trendingScore" | "mainSize" | "id"; +} & Partial): AsyncGenerator>; +//# sourceMappingURL=list-spaces.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-spaces.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/list-spaces.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..4b612437ff76e6b8d7bb1bf77a5d47dd0326ba56 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-spaces.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-spaces.d.ts","sourceRoot":"","sources":["../../../src/lib/list-spaces.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAKnE,eAAO,MAAM,iBAAiB,sDAKsB,CAAC;AACrD,eAAO,MAAM,qBAAqB,+KAiBkB,CAAC;AAErD,MAAM,WAAW,UAAU;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,IAAI,CAAC;CAEhB;AAED,wBAAuB,UAAU,CAChC,KAAK,CAAC,CAAC,SAAS,OAAO,CAAC,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,EAE3G,MAAM,CAAC,EAAE;IACR,MAAM,CAAC,EAAE;QACR;;WAEG;QACH,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;KAChB,CAAC;IACF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB;;OAEG;IACH,gBAAgB,CAAC,EAAE,CAAC,EAAE,CAAC;IACvB;;OAEG;IACH,IAAI,CAAC,EAAE,WAAW,GAAG,WAAW,GAAG,OAAO,GAAG,cAAc,GAAG,UAAU,GAAG,eAAe,GAAG,UAAU,GAAG,IAAI,CAAC;CAC/G,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,cAAc,CAAC,UAAU,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CA8CpD"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-spaces.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/list-spaces.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d6849257e8c610df7d7c6f0f141ef435528a2976 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-spaces.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=list-spaces.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/list-spaces.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/list-spaces.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7c994733a15bf93f15abd61fd15ec1c67df14442 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/list-spaces.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"list-spaces.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/list-spaces.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/model-info.d.ts b/node_modules/@huggingface/hub/dist/src/lib/model-info.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..225698f945925b2c07435ce2ed8f0c7038caf507 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/model-info.d.ts @@ -0,0 +1,16 @@ +import type { CredentialsParams } from "../types/public"; +import { type ModelAdditionalField, type ResolveModelAdditionalFields, type ModelEntry } from "./list-models"; +export declare function modelInfo(params: { + name: string; + hubUrl?: string; + additionalFields?: T[]; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + */ + revision?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise>; +//# sourceMappingURL=model-info.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/model-info.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/model-info.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..b744ceee75f5b0e584bc24cd0182d1b4534e3d12 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/model-info.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"model-info.d.ts","sourceRoot":"","sources":["../../../src/lib/model-info.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAGzD,OAAO,EAGN,KAAK,oBAAoB,EAEzB,KAAK,4BAA4B,EACjC,KAAK,UAAU,EACf,MAAM,eAAe,CAAC;AAEvB,wBAAsB,SAAS,CAAC,KAAK,CAAC,CAAC,SAAS,oBAAoB,GAAG,KAAK,EAC3E,MAAM,EAAE;IACP,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,CAAC,EAAE,CAAC;IACvB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,UAAU,GAAG,4BAA4B,CAAC,CAAC,CAAC,CAAC,CAsDvD"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/model-info.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/model-info.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2d95a6a1246ab6ffdbcc80d34080189d6db30f15 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/model-info.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=model-info.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/model-info.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/model-info.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d718c2cb9ddb44cb9cef95d4757b0e19ff9c4bfd --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/model-info.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"model-info.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/model-info.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/oauth-handle-redirect.d.ts b/node_modules/@huggingface/hub/dist/src/lib/oauth-handle-redirect.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d22d6cfce447e26ac8feb2cb5376465d87d19a0e --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/oauth-handle-redirect.d.ts @@ -0,0 +1,172 @@ +export interface UserInfo { + /** + * OpenID Connect field. Unique identifier for the user, even in case of rename. + */ + sub: string; + /** + * OpenID Connect field. The user's full name. + */ + name: string; + /** + * OpenID Connect field. The user's username. + */ + preferred_username: string; + /** + * OpenID Connect field, available if scope "email" was granted. + */ + email_verified?: boolean; + /** + * OpenID Connect field, available if scope "email" was granted. + */ + email?: string; + /** + * OpenID Connect field. The user's profile picture URL. + */ + picture: string; + /** + * OpenID Connect field. The user's profile URL. + */ + profile: string; + /** + * OpenID Connect field. The user's website URL. + */ + website?: string; + /** + * Hugging Face field. Whether the user is a pro user. + */ + isPro: boolean; + /** + * Hugging Face field. Whether the user has a payment method set up. Needs "read-billing" scope. + */ + canPay?: boolean; + /** + * Hugging Face field. The user's orgs + */ + orgs?: Array<{ + /** + * OpenID Connect field. Unique identifier for the org. + */ + sub: string; + /** + * OpenID Connect field. The org's full name. + */ + name: string; + /** + * OpenID Connect field. The org's username. + */ + preferred_username: string; + /** + * OpenID Connect field. The org's profile picture URL. + */ + picture: string; + /** + * Hugging Face field. The org's plan (e.g., "enterprise", "team"). + */ + plan?: string; + /** + * Hugging Face field. Whether the org has a payment method set up. Needs "read-billing" scope, and the user needs to approve access to the org in the OAuth page. + */ + canPay?: boolean; + /** + * Hugging Face field. The user's role in the org. The user needs to approve access to the org in the OAuth page. + */ + roleInOrg?: string; + /** + * @deprecated Use securityRestrictions instead with "sso" + * HuggingFace field. When the user granted the oauth app access to the org, but didn't complete SSO. + * + * Should never happen directly after the oauth flow. + */ + pendingSSO?: boolean; + /** + * @deprecated Use securityRestrictions instead with "mfa" + * + * HuggingFace field. When the user granted the oauth app access to the org, but didn't complete MFA. + * + * Should never happen directly after the oauth flow. + */ + missingMFA?: boolean; + /** + * HuggingFace field. When the user granted the oauth app access to the org, but didn't complete following security restrictions. + * + * Should never happen directly after the oauth flow. + */ + securityRestrictions?: ("mfa" | "sso" | "ip" | "token-policy")[]; + }>; +} +export interface OAuthResult { + accessToken: string; + accessTokenExpiresAt: Date; + userInfo: UserInfo; + /** + * State passed to the OAuth provider in the original request to the OAuth provider. + */ + state?: string; + /** + * Granted scope + */ + scope: string; +} +/** + * To call after the OAuth provider redirects back to the app. + * + * There is also a helper function {@link oauthHandleRedirectIfPresent}, which will call `oauthHandleRedirect` if the URL contains an oauth code + * in the query parameters and return `false` otherwise. + */ +export declare function oauthHandleRedirect(opts?: { + /** + * The URL of the hub. Defaults to {@link HUB_URL}. + */ + hubUrl?: string; + /** + * The URL to analyze. + * + * @default window.location.href + */ + redirectedUrl?: string; + /** + * nonce generated by oauthLoginUrl + * + * @default localStorage.getItem("huggingface.co:oauth:nonce") + */ + nonce?: string; + /** + * codeVerifier generated by oauthLoginUrl + * + * @default localStorage.getItem("huggingface.co:oauth:code_verifier") + */ + codeVerifier?: string; +}): Promise; +/** + * To call after the OAuth provider redirects back to the app. + * + * It returns false if the URL does not contain an oauth code in the query parameters, otherwise + * it calls {@link oauthHandleRedirect}. + * + * Depending on your app, you may want to call {@link oauthHandleRedirect} directly instead. + */ +export declare function oauthHandleRedirectIfPresent(opts?: { + /** + * The URL of the hub. Defaults to {@link HUB_URL}. + */ + hubUrl?: string; + /** + * The URL to analyze. + * + * @default window.location.href + */ + redirectedUrl?: string; + /** + * nonce generated by oauthLoginUrl + * + * @default localStorage.getItem("huggingface.co:oauth:nonce") + */ + nonce?: string; + /** + * codeVerifier generated by oauthLoginUrl + * + * @default localStorage.getItem("huggingface.co:oauth:code_verifier") + */ + codeVerifier?: string; +}): Promise; +//# sourceMappingURL=oauth-handle-redirect.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/oauth-handle-redirect.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/oauth-handle-redirect.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..11bc9aa34815e34894d3037aac67bc01031d73d2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/oauth-handle-redirect.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"oauth-handle-redirect.d.ts","sourceRoot":"","sources":["../../../src/lib/oauth-handle-redirect.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,QAAQ;IACxB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;OAEG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;OAEG;IACH,IAAI,CAAC,EAAE,KAAK,CAAC;QACZ;;WAEG;QACH,GAAG,EAAE,MAAM,CAAC;QACZ;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QACb;;WAEG;QACH,kBAAkB,EAAE,MAAM,CAAC;QAC3B;;WAEG;QACH,OAAO,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QACd;;WAEG;QACH,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB;;WAEG;QACH,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB;;;;;WAKG;QACH,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB;;;;;;WAMG;QACH,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB;;;;WAIG;QACH,oBAAoB,CAAC,EAAE,CAAC,KAAK,GAAG,KAAK,GAAG,IAAI,GAAG,cAAc,CAAC,EAAE,CAAC;KACjE,CAAC,CAAC;CACH;AAED,MAAM,WAAW,WAAW;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,oBAAoB,EAAE,IAAI,CAAC;IAC3B,QAAQ,EAAE,QAAQ,CAAC;IACnB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;CACd;AAED;;;;;GAKG;AACH,wBAAsB,mBAAmB,CAAC,IAAI,CAAC,EAAE;IAChD;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,WAAW,CAAC,CAqIvB;AAMD;;;;;;;GAOG;AACH,wBAAsB,4BAA4B,CAAC,IAAI,CAAC,EAAE;IACzD;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,WAAW,GAAG,KAAK,CAAC,CA2B/B"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/oauth-handle-redirect.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/oauth-handle-redirect.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ea62996fe81b500ec61c6b5552c9491a81f92902 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/oauth-handle-redirect.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=oauth-handle-redirect.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/oauth-handle-redirect.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/oauth-handle-redirect.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..cad458c55b25add3ebf75bdf3967074621d93945 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/oauth-handle-redirect.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"oauth-handle-redirect.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/oauth-handle-redirect.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/oauth-login-url.d.ts b/node_modules/@huggingface/hub/dist/src/lib/oauth-login-url.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..50e5f403b9f633055964009cbe952487ace2879d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/oauth-login-url.d.ts @@ -0,0 +1,74 @@ +/** + * Use "Sign in with Hub" to authenticate a user, and get oauth user info / access token. + * + * Returns an url to redirect to. After the user is redirected back to your app, call `oauthHandleRedirect` to get the oauth user info / access token. + * + * When called from inside a static Space with OAuth enabled, it will load the config from the space, otherwise you need to at least specify + * the client ID of your OAuth App. + * + * @example + * ```ts + * import { oauthLoginUrl, oauthHandleRedirectIfPresent } from "@huggingface/hub"; + * + * const oauthResult = await oauthHandleRedirectIfPresent(); + * + * if (!oauthResult) { + * // If the user is not logged in, redirect to the login page + * window.location.href = await oauthLoginUrl(); + * } + * + * // You can use oauthResult.accessToken, oauthResult.accessTokenExpiresAt and oauthResult.userInfo + * console.log(oauthResult); + * ``` + * + * (Theoretically, this function could be used to authenticate a user for any OAuth provider supporting PKCE and OpenID Connect by changing `hubUrl`, + * but it is currently only tested with the Hugging Face Hub.) + */ +export declare function oauthLoginUrl(opts?: { + /** + * OAuth client ID. + * + * For static Spaces, you can omit this and it will be loaded from the Space config, as long as `hf_oauth: true` is present in the README.md's metadata. + * For other Spaces, it is available to the backend in the OAUTH_CLIENT_ID environment variable, as long as `hf_oauth: true` is present in the README.md's metadata. + * + * You can also create a Developer Application at https://huggingface.co/settings/connected-applications and use its client ID. + */ + clientId?: string; + hubUrl?: string; + /** + * OAuth scope, a list of space-separated scopes. + * + * For static Spaces, you can omit this and it will be loaded from the Space config, as long as `hf_oauth: true` is present in the README.md's metadata. + * For other Spaces, it is available to the backend in the OAUTH_SCOPES environment variable, as long as `hf_oauth: true` is present in the README.md's metadata. + * + * Defaults to "openid profile". + * + * You can also create a Developer Application at https://huggingface.co/settings/connected-applications and use its scopes. + * + * See https://huggingface.co/docs/hub/oauth for a list of available scopes. + */ + scopes?: string; + /** + * Redirect URI, defaults to the current URL. + * + * For Spaces, any URL within the Space is allowed. + * + * For Developer Applications, you can add any URL you want to the list of allowed redirect URIs at https://huggingface.co/settings/connected-applications. + */ + redirectUrl?: string; + /** + * State to pass to the OAuth provider, which will be returned in the call to `oauthLogin` after the redirect. + */ + state?: string; + /** + * If provided, will be filled with the code verifier and nonce used for the OAuth flow, + * instead of using localStorage. + * + * When calling {@link `oauthHandleRedirectIfPresent`} or {@link `oauthHandleRedirect`} you will need to provide the same values. + */ + localStorage?: { + codeVerifier?: string; + nonce?: string; + }; +}): Promise; +//# sourceMappingURL=oauth-login-url.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/oauth-login-url.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/oauth-login-url.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..bb92a164d0d2e2610aba3a0a2fbf4492581a4d75 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/oauth-login-url.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"oauth-login-url.d.ts","sourceRoot":"","sources":["../../../src/lib/oauth-login-url.ts"],"names":[],"mappings":"AAIA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,aAAa,CAAC,IAAI,CAAC,EAAE;IAC1C;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,YAAY,CAAC,EAAE;QACd,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,KAAK,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACF,GAAG,OAAO,CAAC,MAAM,CAAC,CAyFlB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/parse-safetensors-metadata.d.ts b/node_modules/@huggingface/hub/dist/src/lib/parse-safetensors-metadata.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..22c915d39e515b6b22639d10699d9b94a8687a55 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/parse-safetensors-metadata.d.ts @@ -0,0 +1,140 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import type { SetRequired } from "../vendor/type-fest/set-required"; +export declare const SAFETENSORS_FILE = "model.safetensors"; +export declare const SAFETENSORS_INDEX_FILE = "model.safetensors.index.json"; +export declare const RE_SAFETENSORS_FILE: RegExp; +export declare const RE_SAFETENSORS_INDEX_FILE: RegExp; +export declare const RE_SAFETENSORS_SHARD_FILE: RegExp; +export interface SafetensorsShardFileInfo { + prefix: string; + basePrefix: string; + shard: string; + total: string; +} +export declare function parseSafetensorsShardFilename(filename: string): SafetensorsShardFileInfo | null; +type FileName = string; +export type TensorName = string; +export type Dtype = "F64" | "F32" | "C64" | "F16" | "F8_E4M3" | "F8_E4M3FNUZ" | "F8_E5M2" | "F8_E5M2FNUZ" | "F8_E8M0" | "E8M0" | "F6_E3M2" | "F6_E2M3" | "F4" | "FP4" | "BF16" | "I64" | "U64" | "I32" | "U32" | "I16" | "I8" | "U16" | "U8" | "UE8" | "BOOL"; +export interface TensorInfo { + dtype: Dtype; + shape: number[]; + data_offsets: [number, number]; +} +export type SafetensorsFileHeader = Record & { + __metadata__?: { + total_parameters?: string | number; + } & Record; +}; +export interface SafetensorsIndexJson { + dtype?: string; + metadata?: { + total_parameters?: string | number; + } & Record; + weight_map: Record; +} +export type SafetensorsShardedHeaders = Record; +export type SafetensorsParseFromRepo = { + sharded: false; + header: SafetensorsFileHeader; + parameterCount?: Partial>; + parameterTotal?: number; + filepaths: string[]; +} | { + sharded: true; + index: SafetensorsIndexJson; + headers: SafetensorsShardedHeaders; + parameterCount?: Partial>; + parameterTotal?: number; + filepaths: string[]; +}; +/** + * Analyze model.safetensors.index.json or model.safetensors from a model hosted + * on Hugging Face using smart range requests to extract its metadata. + */ +export declare function parseSafetensorsMetadata(params: { + /** Only models are supported */ + repo: RepoDesignation; + /** + * Relative file path to safetensors file inside `repo`. Defaults to `SAFETENSORS_FILE` or `SAFETENSORS_INDEX_FILE` (whichever one exists). + */ + path?: string; + /** + * Will include SafetensorsParseFromRepo["parameterCount"], an object containing the number of parameters for each DType + * + * @default false + */ + computeParametersCount: true; + hubUrl?: string; + revision?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise>; +export declare function parseSafetensorsMetadata(params: { + /** Only models are supported */ + repo: RepoDesignation; + path?: string; + /** + * Will include SafetensorsParseFromRepo["parameterCount"], an object containing the number of parameters for each DType + * + * @default false + */ + computeParametersCount?: boolean; + hubUrl?: string; + revision?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise; +export interface QuantizationConfig { + quant_method?: string; + modules_to_not_convert?: string[]; + bits?: number; + load_in_4bit?: boolean; + load_in_8bit?: boolean; + format?: string; + config_groups?: Record; +} +export interface ModelConfig { + quantization_config?: QuantizationConfig; + text_config?: { + quantization_config?: QuantizationConfig; + }; +} +/** + * @internal + * Glob match without RegExp: splits pattern on `*` and checks that each literal + * segment appears in order within `str`. Avoids RegExp entirely (no ReDoS risk, + * no SyntaxError from attacker-controlled patterns in config.json). + */ +export declare function globMatch(pattern: string, str: string): boolean; +/** + * Determines if a tensor is quantized based on quantization config and tensor name. + * + * Python's transformers uses plain substring matching for `modules_to_not_convert`, + * so bare names like `"lm_head"` must match `"model.lm_head.weight"`. When the + * pattern contains a `*` we fall back to proper glob matching for flexibility. + */ +export declare function isQuantizedTensor(tensorName: string, quantConfig?: QuantizationConfig): boolean; +/** + * @internal + * Matches a module name against a compressed-tensors target. + * + * Targets are either exact module names, class names (e.g. `"Linear"`, which we + * cannot resolve from tensor names and therefore ignore), or Python regexes + * prefixed with `re:`. To avoid evaluating attacker-controlled RegExp from + * config.json (ReDoS, SyntaxError — see globMatch), we translate the common + * regex subset (`.*` wildcard, `^`/`$` anchors, `\.` escapes) to globMatch and + * treat targets using any other regex syntax as non-matching. + */ +export declare function matchesCompressedTensorsTarget(target: string, moduleName: string): boolean; +export {}; +//# sourceMappingURL=parse-safetensors-metadata.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/parse-safetensors-metadata.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/parse-safetensors-metadata.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..cb78684f8fce7c2c404d46d1b784319ff1e112b3 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/parse-safetensors-metadata.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"parse-safetensors-metadata.d.ts","sourceRoot":"","sources":["../../../src/lib/parse-safetensors-metadata.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAO1E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kCAAkC,CAAC;AAEpE,eAAO,MAAM,gBAAgB,sBAAsB,CAAC;AACpD,eAAO,MAAM,sBAAsB,iCAAiC,CAAC;AAGrE,eAAO,MAAM,mBAAmB,QAAmB,CAAC;AACpD,eAAO,MAAM,yBAAyB,QAAgC,CAAC;AACvE,eAAO,MAAM,yBAAyB,QACmD,CAAC;AAC1F,MAAM,WAAW,wBAAwB;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACd;AACD,wBAAgB,6BAA6B,CAAC,QAAQ,EAAE,MAAM,GAAG,wBAAwB,GAAG,IAAI,CAW/F;AAWD,KAAK,QAAQ,GAAG,MAAM,CAAC;AAEvB,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC;AAChC,MAAM,MAAM,KAAK,GACd,KAAK,GACL,KAAK,GACL,KAAK,GACL,KAAK,GACL,SAAS,GACT,aAAa,GACb,SAAS,GACT,aAAa,GACb,SAAS,GACT,MAAM,GACN,SAAS,GACT,SAAS,GACT,IAAI,GACJ,KAAK,GACL,MAAM,GACN,KAAK,GACL,KAAK,GACL,KAAK,GACL,KAAK,GACL,KAAK,GACL,IAAI,GACJ,KAAK,GACL,IAAI,GACJ,KAAK,GACL,MAAM,CAAC;AAEV,MAAM,WAAW,UAAU;IAC1B,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC/B;AAED,MAAM,MAAM,qBAAqB,GAAG,MAAM,CAAC,UAAU,EAAE,UAAU,CAAC,GAAG;IACpE,YAAY,CAAC,EAAE;QAAE,gBAAgB,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC/E,CAAC;AAEF,MAAM,WAAW,oBAAoB;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,QAAQ,CAAC,EAAE;QAAE,gBAAgB,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE3E,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;CACzC;AAED,MAAM,MAAM,yBAAyB,GAAG,MAAM,CAAC,QAAQ,EAAE,qBAAqB,CAAC,CAAC;AAEhF,MAAM,MAAM,wBAAwB,GACjC;IACA,OAAO,EAAE,KAAK,CAAC;IACf,MAAM,EAAE,qBAAqB,CAAC;IAC9B,cAAc,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;IAChD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,EAAE,CAAC;CACnB,GACD;IACA,OAAO,EAAE,IAAI,CAAC;IACd,KAAK,EAAE,oBAAoB,CAAC;IAC5B,OAAO,EAAE,yBAAyB,CAAC;IACnC,cAAc,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;IAChD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,EAAE,CAAC;CACnB,CAAC;AAmJL;;;GAGG;AACH,wBAAsB,wBAAwB,CAC7C,MAAM,EAAE;IACP,gCAAgC;IAChC,IAAI,EAAE,eAAe,CAAC;IACtB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,sBAAsB,EAAE,IAAI,CAAC;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,WAAW,CAAC,wBAAwB,EAAE,gBAAgB,CAAC,CAAC,CAAC;AACpE,wBAAsB,wBAAwB,CAC7C,MAAM,EAAE;IACP,gCAAgC;IAChC,IAAI,EAAE,eAAe,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,wBAAwB,CAAC,CAAC;AAsErC,MAAM,WAAW,kBAAkB;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sBAAsB,CAAC,EAAE,MAAM,EAAE,CAAC;IAClC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,CAAC,EAAE;YAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,WAAW;IAC3B,mBAAmB,CAAC,EAAE,kBAAkB,CAAC;IACzC,WAAW,CAAC,EAAE;QAAE,mBAAmB,CAAC,EAAE,kBAAkB,CAAA;KAAE,CAAC;CAC3D;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CA2B/D;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAW/F;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,8BAA8B,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAoB1F"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/parse-safetensors-metadata.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/parse-safetensors-metadata.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..c781076e3abe952949abdabe873f645b2b014406 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/parse-safetensors-metadata.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=parse-safetensors-metadata.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/parse-safetensors-metadata.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/parse-safetensors-metadata.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..48e6b84cbf71fab528aa031cc04330eaf886b1c5 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/parse-safetensors-metadata.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"parse-safetensors-metadata.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/parse-safetensors-metadata.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/paths-info.d.ts b/node_modules/@huggingface/hub/dist/src/lib/paths-info.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..a6d8de6cc452d94d184b44b09bb7a61254749a2b --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/paths-info.d.ts @@ -0,0 +1,66 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +export interface LfsPathInfo { + oid: string; + size: number; + pointerSize: number; +} +export interface CommitInfo { + id: string; + title: string; + date: Date; +} +export interface SecurityFileStatus { + status: string; +} +export interface PathInfo { + path: string; + type: string; + /** + * Not available for bucket repos. + */ + oid?: string; + size: number; + /** + * Only defined when path is LFS pointer. Not available for bucket repos. + */ + lfs?: LfsPathInfo; + /** + * Xet-backed hash. Always present for bucket file entries. + */ + xetHash?: string; + /** + * Not available for bucket repos, use {@link uploadedAt} instead. + */ + lastCommit?: CommitInfo; + /** + * Only available for bucket repos. + */ + uploadedAt?: string; + securityFileStatus?: SecurityFileStatus; +} +export declare function pathsInfo(params: { + repo: RepoDesignation; + paths: string[]; + expand: true; + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise<(PathInfo & { + lastCommit: CommitInfo; + securityFileStatus: SecurityFileStatus; +})[]>; +export declare function pathsInfo(params: { + repo: RepoDesignation; + paths: string[]; + expand?: boolean; + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise; +//# sourceMappingURL=paths-info.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/paths-info.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/paths-info.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7f8ad0a260f3b3c5449682555d96eb39438ad1a6 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/paths-info.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"paths-info.d.ts","sourceRoot":"","sources":["../../../src/lib/paths-info.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAM1E,MAAM,WAAW,WAAW;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,UAAU;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,IAAI,CAAC;CACX;AAED,MAAM,WAAW,kBAAkB;IAClC,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,QAAQ;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,GAAG,CAAC,EAAE,WAAW,CAAC;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACxC;AAGD,wBAAgB,SAAS,CACxB,MAAM,EAAE;IACP,IAAI,EAAE,eAAe,CAAC;IACtB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,IAAI,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,CAAC,QAAQ,GAAG;IAAE,UAAU,EAAE,UAAU,CAAC;IAAC,kBAAkB,EAAE,kBAAkB,CAAA;CAAE,CAAC,EAAE,CAAC,CAAC;AAC9F,wBAAgB,SAAS,CACxB,MAAM,EAAE;IACP,IAAI,EAAE,eAAe,CAAC;IACtB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/paths-info.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/paths-info.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..9c94e1d347a5971f3b51de846929d205fe90d091 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/paths-info.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=paths-info.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/paths-info.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/paths-info.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..0f444672581f4abaaa495a8f6735cf129bb8b391 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/paths-info.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"paths-info.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/paths-info.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/repo-exists.d.ts b/node_modules/@huggingface/hub/dist/src/lib/repo-exists.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..6f48cd5ae8a49f5e539ec2abf2077b7f58009c65 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/repo-exists.d.ts @@ -0,0 +1,15 @@ +import type { RepoDesignation } from "../types/public"; +export declare function repoExists(params: { + repo: RepoDesignation; + hubUrl?: string; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + */ + revision?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + accessToken?: string; +}): Promise; +//# sourceMappingURL=repo-exists.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/repo-exists.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/repo-exists.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..66eb6fa7ed688e3cfbb975679a72caac9e8d09c5 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/repo-exists.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"repo-exists.d.ts","sourceRoot":"","sources":["../../../src/lib/repo-exists.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGvD,wBAAsB,UAAU,CAAC,MAAM,EAAE;IACxC,IAAI,EAAE,eAAe,CAAC;IAEtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,OAAO,CAAC,OAAO,CAAC,CAwBnB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/repo-exists.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/repo-exists.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d863da4915c8018fe13da8229bff74d3e9fd5f12 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/repo-exists.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=repo-exists.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/repo-exists.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/repo-exists.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d53c3fa35ee1d41d26047a45e5f51f9017cc5657 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/repo-exists.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"repo-exists.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/repo-exists.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/snapshot-download.d.ts b/node_modules/@huggingface/hub/dist/src/lib/snapshot-download.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..a1b9e44dbb157cad06b5e5a6055bc596b45c8cb7 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/snapshot-download.d.ts @@ -0,0 +1,23 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +export declare const DEFAULT_REVISION = "main"; +/** + * Downloads an entire repository at a given revision in the cache directory {@link getHFHubCachePath}. + * You can list all cached repositories using {@link scanCachedRepo} + * @remarks It uses internally {@link downloadFileToCacheDir}. + */ +export declare function snapshotDownload(params: { + repo: RepoDesignation; + cacheDir?: string; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + * + * @default "main" + */ + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise; +//# sourceMappingURL=snapshot-download.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/snapshot-download.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/snapshot-download.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..0334234db9d09f42ca288f118b00bf9915c91d2a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/snapshot-download.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"snapshot-download.d.ts","sourceRoot":"","sources":["../../../src/lib/snapshot-download.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAW1E,eAAO,MAAM,gBAAgB,SAAS,CAAC;AAEvC;;;;GAIG;AACH,wBAAsB,gBAAgB,CACrC,MAAM,EAAE;IACP,IAAI,EAAE,eAAe,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,MAAM,CAAC,CAsFjB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/snapshot-download.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/snapshot-download.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b30ca61e456c5cce98cd14f7924e8e71f3b05030 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/snapshot-download.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=snapshot-download.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/snapshot-download.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/snapshot-download.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5cc460f188f3ca2d235aa04a3844e1fff083c294 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/snapshot-download.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"snapshot-download.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/snapshot-download.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/space-info.d.ts b/node_modules/@huggingface/hub/dist/src/lib/space-info.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..bb5da6bcb12f5457bf62b1e3305af411e7e1e4ab --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/space-info.d.ts @@ -0,0 +1,18 @@ +import type { ApiSpaceInfo } from "../types/api/api-space"; +import type { CredentialsParams } from "../types/public"; +import type { SPACE_EXPANDABLE_KEYS, SpaceEntry } from "./list-spaces"; +import { SPACE_EXPAND_KEYS } from "./list-spaces"; +export declare function spaceInfo = never>(params: { + name: string; + hubUrl?: string; + additionalFields?: T[]; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + */ + revision?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & Partial): Promise>; +//# sourceMappingURL=space-info.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/space-info.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/space-info.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f02a162dd57a96d3c6bccae584765e4fb02ba3ec --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/space-info.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"space-info.d.ts","sourceRoot":"","sources":["../../../src/lib/space-info.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAGzD,OAAO,KAAK,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAElD,wBAAsB,SAAS,CAC9B,KAAK,CAAC,CAAC,SAAS,OAAO,CAAC,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,EAE3G,MAAM,EAAE;IACP,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,CAAC,EAAE,CAAC;IACvB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAkC7C"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/space-info.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/space-info.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..492a7dc07496c3ea833f538a54326f4c3a2cceee --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/space-info.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=space-info.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/space-info.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/space-info.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..0f443b9fe75d49d12128305a8a6e824f8e4aef14 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/space-info.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"space-info.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/space-info.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/upload-file.d.ts b/node_modules/@huggingface/hub/dist/src/lib/upload-file.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..de98cd492b6b57773886e5b6f6b6328464e27886 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/upload-file.d.ts @@ -0,0 +1,20 @@ +import type { CredentialsParams } from "../types/public"; +import type { CommitOutput, CommitParams, ContentSource } from "./commit"; +export declare function uploadFile(params: { + repo: CommitParams["repo"]; + file: URL | File | { + path: string; + content: ContentSource; + }; + commitTitle?: CommitParams["title"]; + commitDescription?: CommitParams["description"]; + hubUrl?: CommitParams["hubUrl"]; + branch?: CommitParams["branch"]; + isPullRequest?: CommitParams["isPullRequest"]; + parentCommit?: CommitParams["parentCommit"]; + fetch?: CommitParams["fetch"]; + useWebWorkers?: CommitParams["useWebWorkers"]; + abortSignal?: CommitParams["abortSignal"]; + useXet?: CommitParams["useXet"]; +} & Partial): Promise; +//# sourceMappingURL=upload-file.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/upload-file.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/upload-file.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..31080ea7108c079bfa7dfbdd70e2568688701dc5 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/upload-file.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"upload-file.d.ts","sourceRoot":"","sources":["../../../src/lib/upload-file.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG1E,wBAAgB,UAAU,CACzB,MAAM,EAAE;IACP,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;IAC3B,IAAI,EAAE,GAAG,GAAG,IAAI,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,aAAa,CAAA;KAAE,CAAC;IAC5D,WAAW,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IACpC,iBAAiB,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAChD,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChC,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChC,aAAa,CAAC,EAAE,YAAY,CAAC,eAAe,CAAC,CAAC;IAC9C,YAAY,CAAC,EAAE,YAAY,CAAC,cAAc,CAAC,CAAC;IAC5C,KAAK,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAC9B,aAAa,CAAC,EAAE,YAAY,CAAC,eAAe,CAAC,CAAC;IAC9C,WAAW,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAC1C,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;CAChC,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CA6BnC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/upload-file.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/upload-file.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..abd67bbcd82b8e7e1b3a4ac47cdd555278ebb0aa --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/upload-file.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=upload-file.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/upload-file.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/upload-file.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5d7a1fc716da9f90f3dcaa868bf3e96271fb504f --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/upload-file.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"upload-file.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/upload-file.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/upload-files-with-progress.d.ts b/node_modules/@huggingface/hub/dist/src/lib/upload-files-with-progress.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ed8671f076e40d92e6448d97cd272a3adc8cfaa1 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/upload-files-with-progress.d.ts @@ -0,0 +1,29 @@ +import type { CredentialsParams } from "../types/public"; +import type { CommitOutput, CommitParams, CommitProgressEvent, ContentSource } from "./commit"; +/** + * Uploads with progress + * + * Needs XMLHttpRequest to be available for progress events for uploads on models, datasets and spaces. + * Set useWebWorkers to true in order to have progress events for hashing for models, datasets and spaces. + */ +export declare function uploadFilesWithProgress(params: { + repo: CommitParams["repo"]; + files: Array; + commitTitle?: CommitParams["title"]; + commitDescription?: CommitParams["description"]; + hubUrl?: CommitParams["hubUrl"]; + branch?: CommitParams["branch"]; + isPullRequest?: CommitParams["isPullRequest"]; + parentCommit?: CommitParams["parentCommit"]; + abortSignal?: CommitParams["abortSignal"]; + maxFolderDepth?: CommitParams["maxFolderDepth"]; + useXet?: CommitParams["useXet"]; + /** + * Set this to true in order to have progress events for hashing + */ + useWebWorkers?: CommitParams["useWebWorkers"]; +} & Partial): AsyncGenerator; +//# sourceMappingURL=upload-files-with-progress.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/upload-files-with-progress.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/upload-files-with-progress.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..70b3df2c678cd0d2b6360a855e7011de3f71c3e0 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/upload-files-with-progress.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"upload-files-with-progress.d.ts","sourceRoot":"","sources":["../../../src/lib/upload-files-with-progress.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEzD,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAW/F;;;;;GAKG;AACH,wBAAuB,uBAAuB,CAC7C,MAAM,EAAE;IACP,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;IAC3B,KAAK,EAAE,KAAK,CAAC,GAAG,GAAG,IAAI,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IACpE,WAAW,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IACpC,iBAAiB,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAChD,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChC,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChC,aAAa,CAAC,EAAE,YAAY,CAAC,eAAe,CAAC,CAAC;IAC9C,YAAY,CAAC,EAAE,YAAY,CAAC,cAAc,CAAC,CAAC;IAC5C,WAAW,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAC1C,cAAc,CAAC,EAAE,YAAY,CAAC,gBAAgB,CAAC,CAAC;IAChD,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChC;;OAEG;IACH,aAAa,CAAC,EAAE,YAAY,CAAC,eAAe,CAAC,CAAC;CAC9C,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,cAAc,CAAC,mBAAmB,EAAE,YAAY,GAAG,SAAS,CAAC,CAyH/D"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/upload-files-with-progress.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/upload-files-with-progress.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..be30c98f9d1b7bb6b35bb18c5f8b1c64257a8ff9 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/upload-files-with-progress.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=upload-files-with-progress.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/upload-files-with-progress.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/upload-files-with-progress.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..554c4450c0ff875214647536415eab792418d3e3 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/upload-files-with-progress.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"upload-files-with-progress.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/upload-files-with-progress.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/upload-files.d.ts b/node_modules/@huggingface/hub/dist/src/lib/upload-files.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..024b081f10d936eaeb68ba49bfa2de3bcf36f669 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/upload-files.d.ts @@ -0,0 +1,21 @@ +import type { CredentialsParams } from "../types/public"; +import type { CommitOutput, CommitParams, ContentSource } from "./commit"; +export declare function uploadFiles(params: { + repo: CommitParams["repo"]; + files: Array; + commitTitle?: CommitParams["title"]; + commitDescription?: CommitParams["description"]; + hubUrl?: CommitParams["hubUrl"]; + branch?: CommitParams["branch"]; + isPullRequest?: CommitParams["isPullRequest"]; + parentCommit?: CommitParams["parentCommit"]; + fetch?: CommitParams["fetch"]; + useWebWorkers?: CommitParams["useWebWorkers"]; + maxFolderDepth?: CommitParams["maxFolderDepth"]; + abortSignal?: CommitParams["abortSignal"]; + useXet?: CommitParams["useXet"]; +} & Partial): Promise; +//# sourceMappingURL=upload-files.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/upload-files.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/upload-files.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..869dd0e3a51c7740392cc302144317d73bf3a491 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/upload-files.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"upload-files.d.ts","sourceRoot":"","sources":["../../../src/lib/upload-files.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG1E,wBAAgB,WAAW,CAC1B,MAAM,EAAE;IACP,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;IAC3B,KAAK,EAAE,KAAK,CAAC,GAAG,GAAG,IAAI,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IACpE,WAAW,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IACpC,iBAAiB,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAChD,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChC,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChC,aAAa,CAAC,EAAE,YAAY,CAAC,eAAe,CAAC,CAAC;IAC9C,YAAY,CAAC,EAAE,YAAY,CAAC,cAAc,CAAC,CAAC;IAC5C,KAAK,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;IAC9B,aAAa,CAAC,EAAE,YAAY,CAAC,eAAe,CAAC,CAAC;IAC9C,cAAc,CAAC,EAAE,YAAY,CAAC,gBAAgB,CAAC,CAAC;IAChD,WAAW,CAAC,EAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAC1C,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;CAChC,GAAG,OAAO,CAAC,iBAAiB,CAAC,GAC5B,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CAoBnC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/upload-files.fs.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/upload-files.fs.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d5185a7b3f266c6ff006a34d244759a5f9ea4ca4 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/upload-files.fs.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=upload-files.fs.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/upload-files.fs.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/upload-files.fs.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..45e27e07142683746cfed07213e97b9757f2659a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/upload-files.fs.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"upload-files.fs.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/upload-files.fs.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/upload-files.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/upload-files.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..c74223ec254a9815f5eacd3151a0a622c4450c79 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/upload-files.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=upload-files.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/upload-files.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/upload-files.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..3ea1dc8b4eddb0dd3fbc868f989bf1c83e19c82b --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/upload-files.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"upload-files.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/upload-files.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/who-am-i.d.ts b/node_modules/@huggingface/hub/dist/src/lib/who-am-i.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..1baef0f46f541ffcecf5afc98529c86dd3975807 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/who-am-i.d.ts @@ -0,0 +1,62 @@ +import type { AccessTokenRole, AuthType, CredentialsParams } from "../types/public"; +export interface WhoAmIUser { + /** Unique ID persistent across renames */ + id: string; + type: "user"; + email: string; + emailVerified: boolean; + isPro: boolean; + orgs: WhoAmIOrg[]; + name: string; + fullname: string; + canPay: boolean; + avatarUrl: string; + /** + * Unix timestamp in seconds + */ + periodEnd: number | null; + billingMode: "postpaid" | "prepaid"; +} +export interface WhoAmIOrg { + /** Unique ID persistent across renames */ + id: string; + type: "org"; + name: string; + fullname: string; + email: string | null; + canPay: boolean; + avatarUrl: string; + /** + * Unix timestamp in seconds + */ + periodEnd: number | null; +} +export interface WhoAmIApp { + id: string; + type: "app"; + name: string; + scope?: { + entities: string[]; + role: "admin" | "write" | "contributor" | "read"; + }; +} +export type WhoAmI = WhoAmIApp | WhoAmIOrg | WhoAmIUser; +export interface AuthInfo { + type: AuthType; + accessToken?: { + displayName: string; + role: AccessTokenRole; + createdAt: Date; + }; + expiresAt?: Date; +} +export declare function whoAmI(params: { + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} & CredentialsParams): Promise; +//# sourceMappingURL=who-am-i.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/who-am-i.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/who-am-i.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e0e3427e31bf0dd8ef3211d94bc3841a5311342f --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/who-am-i.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"who-am-i.d.ts","sourceRoot":"","sources":["../../../src/lib/who-am-i.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAGpF,MAAM,WAAW,UAAU;IAC1B,0CAA0C;IAC1C,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,OAAO,CAAC;IACvB,KAAK,EAAE,OAAO,CAAC;IACf,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,UAAU,GAAG,SAAS,CAAC;CACpC;AAED,MAAM,WAAW,SAAS;IACzB,0CAA0C;IAC1C,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,KAAK,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,SAAS;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,KAAK,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE;QACP,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,IAAI,EAAE,OAAO,GAAG,OAAO,GAAG,aAAa,GAAG,MAAM,CAAC;KACjD,CAAC;CACF;AAED,MAAM,MAAM,MAAM,GAAG,SAAS,GAAG,SAAS,GAAG,UAAU,CAAC;AACxD,MAAM,WAAW,QAAQ;IACxB,IAAI,EAAE,QAAQ,CAAC;IACf,WAAW,CAAC,EAAE;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,eAAe,CAAC;QACtB,SAAS,EAAE,IAAI,CAAC;KAChB,CAAC;IACF,SAAS,CAAC,EAAE,IAAI,CAAC;CACjB;AAED,wBAAsB,MAAM,CAC3B,MAAM,EAAE;IACP,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACrB,GAAG,iBAAiB,GACnB,OAAO,CAAC,MAAM,GAAG;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,CAAC,CAsBtC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/who-am-i.spec.d.ts b/node_modules/@huggingface/hub/dist/src/lib/who-am-i.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fb98d015629183178ada0b1c088ab41e89b388d6 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/who-am-i.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=who-am-i.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/lib/who-am-i.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/lib/who-am-i.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..82fce42e56637a318aea43a2b7ea4b12c4784c48 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/lib/who-am-i.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"who-am-i.spec.d.ts","sourceRoot":"","sources":["../../../src/lib/who-am-i.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/test/consts.d.ts b/node_modules/@huggingface/hub/dist/src/test/consts.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b3ebcdd1ecc1eb288c1688ff9cb04d69ee0cd67a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/test/consts.d.ts @@ -0,0 +1,5 @@ +export declare const TEST_HUB_URL = "https://hub-ci.huggingface.co"; +export declare const TEST_USER = "hub.js"; +export declare const TEST_ACCESS_TOKEN = "hf_hub.js"; +export declare const TEST_COOKIE = "huggingface-hub.js-cookie"; +//# sourceMappingURL=consts.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/test/consts.d.ts.map b/node_modules/@huggingface/hub/dist/src/test/consts.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..700fd7d3b5fb00ae7fa159a578278ea99c54f822 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/test/consts.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"consts.d.ts","sourceRoot":"","sources":["../../../src/test/consts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,YAAY,kCAAkC,CAAC;AAC5D,eAAO,MAAM,SAAS,WAAW,CAAC;AAClC,eAAO,MAAM,iBAAiB,cAAc,CAAC;AAC7C,eAAO,MAAM,WAAW,8BAA8B,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-author.d.ts b/node_modules/@huggingface/hub/dist/src/types/api/api-author.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..605d7d100e840b87a090b3b145065e26c97fbb78 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-author.d.ts @@ -0,0 +1,25 @@ +export type ApiAuthor = { + avatarUrl: string; + fullname: string; + name: string; + isHf: boolean; + isHfAdmin: boolean; + isMod: boolean; + followerCount?: number; + type: "org"; + plan?: string; + isUserFollowing?: boolean; +} | { + avatarUrl: string; + fullname: string; + name: string; + isHf: boolean; + isHfAdmin: boolean; + isMod: boolean; + followerCount?: number; + type: "user"; + isPro: boolean; + _id: string; + isUserFollowing?: boolean; +}; +//# sourceMappingURL=api-author.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-author.d.ts.map b/node_modules/@huggingface/hub/dist/src/types/api/api-author.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..dd66328a7228d6799a213c37ac7c693034611aec --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-author.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"api-author.d.ts","sourceRoot":"","sources":["../../../../src/types/api/api-author.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,SAAS,GAClB;IACA,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,IAAI,EAAE,KAAK,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,OAAO,CAAC;CACzB,GACD;IACA,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,OAAO,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,OAAO,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,OAAO,CAAC;CACzB,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-collection.d.ts b/node_modules/@huggingface/hub/dist/src/types/api/api-collection.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..c663b6350ab7aa483b7200cceb1e9f11db8dc0af --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-collection.d.ts @@ -0,0 +1,170 @@ +import type { ApiAuthor } from "./api-author"; +export interface ApiCollectionInfo { + slug: string; + title: string; + description?: string; + lastUpdated: string; + gating: true | (false | { + mode: "auto"; + } | { + mode: "manual"; + notifications: { + mode: "bulk" | "real-time"; + email?: string; + }; + }); + owner: ApiAuthor; + /** + * Note that it's limited to 4 items when the listing endpoint is used. + */ + items: ApiCollectionItem[]; + theme: "orange" | "blue" | "green" | "purple" | "pink" | "indigo"; + private: boolean; + upvotes: number; + isUpvotedByUser: boolean; +} +interface ApiCollectionItemBase { + _id: string; + position: number; + note?: { + html: string; + text: string; + }; + gallery?: string[]; +} +interface ApiCollectionItemModel extends ApiCollectionItemBase { + type: "model"; + author: string; + downloads: number; + id: string; + availableInferenceProviders: { + provider: "black-forest-labs" | "cerebras" | "cohere" | "fal-ai" | "featherless-ai" | "fireworks-ai" | "groq" | "hf-inference" | "hyperbolic" | "nebius" | "novita" | "nscale" | "openai" | "ovhcloud" | "replicate" | "sambanova" | "together"; + providerStatus: "live" | "staging" | "error"; + modelStatus: "live" | "staging" | "error"; + providerId: string; + task: "text-classification" | "token-classification" | "table-question-answering" | "question-answering" | "zero-shot-classification" | "translation" | "summarization" | "feature-extraction" | "text-generation" | "text2text-generation" | "fill-mask" | "sentence-similarity" | "text-to-speech" | "text-to-audio" | "automatic-speech-recognition" | "audio-to-audio" | "audio-classification" | "audio-text-to-text" | "voice-activity-detection" | "depth-estimation" | "image-classification" | "object-detection" | "image-segmentation" | "text-to-image" | "image-to-text" | "image-to-image" | "image-to-video" | "unconditional-image-generation" | "video-classification" | "reinforcement-learning" | "robotics" | "tabular-classification" | "tabular-regression" | "tabular-to-text" | "table-to-text" | "multiple-choice" | "text-ranking" | "text-retrieval" | "time-series-forecasting" | "text-to-video" | "image-text-to-text" | "visual-question-answering" | "document-question-answering" | "zero-shot-image-classification" | "graph-ml" | "mask-generation" | "zero-shot-object-detection" | "text-to-3d" | "image-to-3d" | "image-feature-extraction" | "video-text-to-text" | "keypoint-detection" | "visual-document-retrieval" | "any-to-any" | "video-to-video" | "other" | "conversational"; + adapterType?: "lora"; + adapterWeightsPath?: string; + }[]; + isLikedByUser: boolean; + lastModified: string; + likes: number; + pipeline_tag?: string; + private: boolean; + repoType: "model"; + gated: false | ("auto" | "manual"); + resourceGroup?: { + id: string; + name: string; + numUsers: number; + }; + numParameters?: number; + authorData?: ApiAuthor; + widgetOutputUrls?: string[]; +} +interface ApiCollectionItemDataset extends ApiCollectionItemBase { + type: "dataset"; + author: string; + id: string; + isLikedByUser: boolean; + likes: number; + datasetsServerInfo?: { + viewer: "preview" | "viewer-partial" | "viewer"; + numRows: number | null; + libraries: ("mlcroissant" | "webdataset" | "datasets" | "pandas" | "dask" | "distilabel" | "fiftyone" | "argilla" | "polars" | "duckdb")[]; + formats: ("json" | "csv" | "parquet" | "imagefolder" | "audiofolder" | "webdataset" | "text" | "arrow")[]; + modalities: ("3d" | "audio" | "document" | "geospatial" | "image" | "tabular" | "text" | "timeseries" | "video")[]; + }; + private: boolean; + repoType: "dataset"; + downloads: number; + gated: false | ("auto" | "manual"); + lastModified: string; + resourceGroup?: { + id: string; + name: string; + numUsers: number; + }; +} +interface ApiCollectionItemSpace extends ApiCollectionItemBase { + type: "space"; + author: string; + colorFrom: string; + colorTo: string; + createdAt: string; + emoji: string; + id: string; + isLikedByUser: boolean; + lastModified: string; + likes: number; + pinned: boolean; + private: boolean; + featured: boolean; + repoType: "space"; + title: string; + sdk?: "gradio" | "docker" | "static" | "streamlit"; + runtime: { + stage: "NO_APP_FILE" | "CONFIG_ERROR" | "BUILDING" | "BUILD_ERROR" | "APP_STARTING" | "RUNNING" | "RUNNING_BUILDING" | "RUNNING_APP_STARTING" | "RUNTIME_ERROR" | "DELETING" | "STOPPED" | "PAUSED" | "SLEEPING"; + hardware: { + current: ("cpu-basic" | "cpu-upgrade" | "cpu-performance" | "cpu-xl" | "zero-a10g" | "t4-small" | "t4-medium" | "l4x1" | "l4x4" | "l40sx1" | "l40sx4" | "l40sx8" | "a10g-small" | "a10g-large" | "a10g-largex2" | "a10g-largex4" | "a100-large" | "h100" | "h100x8") | null; + requested: ("cpu-basic" | "cpu-upgrade" | "cpu-performance" | "cpu-xl" | "zero-a10g" | "t4-small" | "t4-medium" | "l4x1" | "l4x4" | "l40sx1" | "l40sx4" | "l40sx8" | "a10g-small" | "a10g-large" | "a10g-largex2" | "a10g-largex4" | "a100-large" | "h100" | "h100x8") | null; + }; + storage: ("small" | "medium" | "large") | null; + errorMessage?: string; + gcTimeout?: number | null; + replicas: { + current?: number | null; + requested: number | "auto"; + }; + devMode?: boolean; + domains?: { + domain: string; + isCustom?: boolean | null; + stage: "READY" | "PENDING"; + }[]; + sha?: string; + }; + originSpace?: { + author: ApiAuthor; + name: string; + }; + ai_short_description?: string; + ai_category?: string; + trendingScore?: number; + resourceGroup?: { + id: string; + name: string; + numUsers: number; + }; + tags: string[]; + authorData?: ApiAuthor; + shortDescription?: string; + semanticRelevancyScore?: number; + visibility?: "public" | "private" | "protected"; +} +interface ApiCollectionItemPaper extends ApiCollectionItemBase { + type: "paper"; + id: string; + title: string; + upvotes: number; + publishedAt: string; + thumbnailUrl?: string; + isUpvotedByUser?: boolean; +} +interface ApiCollectionItemCollection extends ApiCollectionItemBase { + type: "collection"; + slug: string; + lastUpdated: string; + description?: string; + owner: ApiAuthor; + title: string; + theme: "orange" | "blue" | "green" | "purple" | "pink" | "indigo"; + upvotes: number; + isUpvotedByUser: boolean; + id: string; + numberItems: number; + shareUrl: string; +} +type ApiCollectionItem = ApiCollectionItemModel | ApiCollectionItemDataset | ApiCollectionItemSpace | ApiCollectionItemPaper | ApiCollectionItemCollection; +export {}; +//# sourceMappingURL=api-collection.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-collection.d.ts.map b/node_modules/@huggingface/hub/dist/src/types/api/api-collection.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f2b689352fcdf2f1c31d175d4eb4ab38b5c8b2f4 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-collection.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"api-collection.d.ts","sourceRoot":"","sources":["../../../../src/types/api/api-collection.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9C,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EACH,IAAI,GACJ,CACE,KAAK,GACL;QACA,IAAI,EAAE,MAAM,CAAC;KACZ,GACD;QACA,IAAI,EAAE,QAAQ,CAAC;QACf,aAAa,EAAE;YACd,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;YAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;SACf,CAAC;KACD,CACF,CAAC;IACL,KAAK,EAAE,SAAS,CAAC;IACjB;;OAEG;IACH,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,KAAK,EAAE,QAAQ,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,QAAQ,CAAC;IAClE,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,OAAO,CAAC;CACzB;AAED,UAAU,qBAAqB;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;KACb,CAAC;IACF,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,UAAU,sBAAuB,SAAQ,qBAAqB;IAC7D,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,2BAA2B,EAAE;QAC5B,QAAQ,EACL,mBAAmB,GACnB,UAAU,GACV,QAAQ,GACR,QAAQ,GACR,gBAAgB,GAChB,cAAc,GACd,MAAM,GACN,cAAc,GACd,YAAY,GACZ,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,UAAU,GACV,WAAW,GACX,WAAW,GACX,UAAU,CAAC;QACd,cAAc,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;QAC7C,WAAW,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;QAC1C,UAAU,EAAE,MAAM,CAAC;QACnB,IAAI,EACD,qBAAqB,GACrB,sBAAsB,GACtB,0BAA0B,GAC1B,oBAAoB,GACpB,0BAA0B,GAC1B,aAAa,GACb,eAAe,GACf,oBAAoB,GACpB,iBAAiB,GACjB,sBAAsB,GACtB,WAAW,GACX,qBAAqB,GACrB,gBAAgB,GAChB,eAAe,GACf,8BAA8B,GAC9B,gBAAgB,GAChB,sBAAsB,GACtB,oBAAoB,GACpB,0BAA0B,GAC1B,kBAAkB,GAClB,sBAAsB,GACtB,kBAAkB,GAClB,oBAAoB,GACpB,eAAe,GACf,eAAe,GACf,gBAAgB,GAChB,gBAAgB,GAChB,gCAAgC,GAChC,sBAAsB,GACtB,wBAAwB,GACxB,UAAU,GACV,wBAAwB,GACxB,oBAAoB,GACpB,iBAAiB,GACjB,eAAe,GACf,iBAAiB,GACjB,cAAc,GACd,gBAAgB,GAChB,yBAAyB,GACzB,eAAe,GACf,oBAAoB,GACpB,2BAA2B,GAC3B,6BAA6B,GAC7B,gCAAgC,GAChC,UAAU,GACV,iBAAiB,GACjB,4BAA4B,GAC5B,YAAY,GACZ,aAAa,GACb,0BAA0B,GAC1B,oBAAoB,GACpB,oBAAoB,GACpB,2BAA2B,GAC3B,YAAY,GACZ,gBAAgB,GAChB,OAAO,GACP,gBAAgB,CAAC;QACpB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC5B,EAAE,CAAC;IACJ,aAAa,EAAE,OAAO,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,OAAO,CAAC;IAClB,KAAK,EAAE,KAAK,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC;IACnC,aAAa,CAAC,EAAE;QACf,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,UAAU,wBAAyB,SAAQ,qBAAqB;IAC/D,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;IACX,aAAa,EAAE,OAAO,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE;QACpB,MAAM,EAAE,SAAS,GAAG,gBAAgB,GAAG,QAAQ,CAAC;QAChD,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QACvB,SAAS,EAAE,CACR,aAAa,GACb,YAAY,GACZ,UAAU,GACV,QAAQ,GACR,MAAM,GACN,YAAY,GACZ,UAAU,GACV,SAAS,GACT,QAAQ,GACR,QAAQ,CACV,EAAE,CAAC;QACJ,OAAO,EAAE,CAAC,MAAM,GAAG,KAAK,GAAG,SAAS,GAAG,aAAa,GAAG,aAAa,GAAG,YAAY,GAAG,MAAM,GAAG,OAAO,CAAC,EAAE,CAAC;QAC1G,UAAU,EAAE,CAAC,IAAI,GAAG,OAAO,GAAG,UAAU,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,GAAG,MAAM,GAAG,YAAY,GAAG,OAAO,CAAC,EAAE,CAAC;KACnH,CAAC;IACF,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,SAAS,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,KAAK,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC;IACnC,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE;QACf,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KACjB,CAAC;CACF;AAED,UAAU,sBAAuB,SAAQ,qBAAqB;IAC7D,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,aAAa,EAAE,OAAO,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,OAAO,CAAC;IAClB,QAAQ,EAAE,OAAO,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;IACnD,OAAO,EAAE;QACR,KAAK,EACF,aAAa,GACb,cAAc,GACd,UAAU,GACV,aAAa,GACb,cAAc,GACd,SAAS,GACT,kBAAkB,GAClB,sBAAsB,GACtB,eAAe,GACf,UAAU,GACV,SAAS,GACT,QAAQ,GACR,UAAU,CAAC;QACd,QAAQ,EAAE;YACT,OAAO,EACJ,CACE,WAAW,GACX,aAAa,GACb,iBAAiB,GACjB,QAAQ,GACR,WAAW,GACX,UAAU,GACV,WAAW,GACX,MAAM,GACN,MAAM,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,YAAY,GACZ,YAAY,GACZ,cAAc,GACd,cAAc,GACd,YAAY,GACZ,MAAM,GACN,QAAQ,CACT,GACD,IAAI,CAAC;YACR,SAAS,EACN,CACE,WAAW,GACX,aAAa,GACb,iBAAiB,GACjB,QAAQ,GACR,WAAW,GACX,UAAU,GACV,WAAW,GACX,MAAM,GACN,MAAM,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,YAAY,GACZ,YAAY,GACZ,cAAc,GACd,cAAc,GACd,YAAY,GACZ,MAAM,GACN,QAAQ,CACT,GACD,IAAI,CAAC;SACR,CAAC;QACF,OAAO,EAAE,CAAC,OAAO,GAAG,QAAQ,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC;QAC/C,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,QAAQ,EAAE;YACT,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;YACxB,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;SAC3B,CAAC;QACF,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,OAAO,CAAC,EAAE;YACT,MAAM,EAAE,MAAM,CAAC;YACf,QAAQ,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;YAC1B,KAAK,EAAE,OAAO,GAAG,SAAS,CAAC;SAC3B,EAAE,CAAC;QACJ,GAAG,CAAC,EAAE,MAAM,CAAC;KACb,CAAC;IACF,WAAW,CAAC,EAAE;QACb,MAAM,EAAE,SAAS,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;KACb,CAAC;IACF,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE;QACf,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,UAAU,CAAC,EAAE,SAAS,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,UAAU,CAAC,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,CAAC;CAChD;AAED,UAAU,sBAAuB,SAAQ,qBAAqB;IAC7D,IAAI,EAAE,OAAO,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,UAAU,2BAA4B,SAAQ,qBAAqB;IAClE,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,SAAS,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,QAAQ,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,GAAG,QAAQ,CAAC;IAClE,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,OAAO,CAAC;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,KAAK,iBAAiB,GACnB,sBAAsB,GACtB,wBAAwB,GACxB,sBAAsB,GACtB,sBAAsB,GACtB,2BAA2B,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-commit.d.ts b/node_modules/@huggingface/hub/dist/src/types/api/api-commit.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..e6f5ac29cd61ddf27324d8ad7dca6ccbef741887 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-commit.d.ts @@ -0,0 +1,198 @@ +export interface ApiLfsBatchRequest { + operation: "download" | "upload"; + transfers?: Array; + /** + * Optional object describing the server ref that the objects belong to. Note: Added in v2.4. + * + * We use this object for QOL and to fail early for users when they're trying to push to the wrong reference. + * But it does nothing for security. + */ + ref?: { + name: string; + } | null; + objects: { + oid: string; + /** + * Integer byte size of the LFS object. Must be at least zero. + */ + size: number; + }[]; + /** + * The hash algorithm used to name Git LFS objects. Optional; defaults to sha256 if not specified. + * */ + hash_algo?: string; +} +export interface ApiLfsBatchResponse { + transfer?: ApiLfsResponseTransfer; + objects: ApiLfsResponseObject[]; +} +export type ApiLfsResponseTransfer = "basic" | "multipart" | "xet"; +export interface ApiLfsCompleteMultipartRequest { + oid: string; + parts: { + etag: string; + partNumber: number; + }[]; +} +export interface ApiLfsResponseObject { + /** + * Optional boolean specifying whether the request + * for this specific object is authenticated. + * If omitted or false, Git LFS will attempt to find credentials for this URL. + */ + authenticated?: boolean; + oid: string; + /** + * Integer byte size of the LFS object. Must be at least zero. + */ + size: number; + /** + * Applicable actions depend on which `operation` is specified in the request. + * How these properties are interpreted depends on which transfer adapter + * the client will be using. + */ + actions?: { + /** + * Download operations MUST specify a download action, + * or an object error if the object cannot be downloaded for some reason + */ + download?: ApiLfsAction; + /** + * Upload operations can specify an upload and a verify action. + * The upload action describes how to upload the object. + */ + upload?: ApiLfsAction; + /** + * The LFS client will hit this URL after a successful upload. + * Servers can use this for extra verification, if needed. + */ + verify?: ApiLfsAction; + }; + /** + * If there are problems accessing individual objects, servers should continue + * to return a 200 status code, and provide per-object errors + */ + error?: { + code: number; + message: string; + }; +} +export interface ApiLfsAction { + href: string; + /** + * Optional hash of String HTTP header key/value pairs to apply to the request + */ + header?: { + [key: string]: string; + } & { + chunk_size?: string; + }; + /** + * Whole number of seconds after local client time when transfer will expire. + * Preferred over `expires_at` if both are provided. + * Maximum of 2147483647, minimum of -2147483647. + */ + expires_in?: number; + /** + * String uppercase RFC 3339-formatted timestamp with second precision + * for when the given action expires (usually due to a temporary token). + */ + expires_at?: string; +} +export interface ApiPreuploadRequest { + /** + * Optional, otherwise takes the existing content of `.gitattributes` for the revision. + * + * Provide this parameter if you plan to modify `.gitattributes` yourself at the same + * time as uploading LFS files. + * + * Note that this is not needed if you solely rely on automatic LFS detection from HF: the commit endpoint + * will automatically edit the `.gitattributes` file to track the files passed to its `lfsFiles` param. + */ + gitAttributes?: string; + files: Array<{ + /** + * Path of the LFS file + */ + path: string; + /** + * Full size of the LFS file + */ + size: number; + /** + * Base64-encoded sample of the first 512 bytes of the file + */ + sample: string; + }>; +} +export interface ApiBucketBatchResponse { + /** True if all files were successfully added */ + success: boolean; + /** Total number of operations attempted */ + processed: number; + /** Number of successful operations */ + succeeded: number; + /** List of failed operations */ + failed: Array<{ + path: string; + error: string; + }>; +} +export interface ApiPreuploadResponse { + files: Array<{ + path: string; + uploadMode: "lfs" | "regular"; + }>; +} +export interface ApiCommitHeader { + summary: string; + description?: string; + /** + * Parent commit. Optional + * + * - When opening a PR: will use parentCommit as the parent commit + * - When committing on a branch: Will make sure that there were no intermediate commits + */ + parentCommit?: string; +} +export interface ApiCommitDeletedEntry { + path: string; +} +export interface ApiCommitLfsFile { + path: string; + oldPath?: string; + /** Required if {@link oldPath} is not set */ + algo?: "sha256"; + /** Required if {@link oldPath} is not set */ + oid?: string; + size?: number; +} +export interface ApiCommitFile { + /** Required if {@link oldPath} is not set */ + content?: string; + path: string; + oldPath?: string; + encoding?: "utf-8" | "base64"; +} +export type ApiCommitOperation = { + key: "file"; + value: ApiCommitFile; +} | { + key: "lfsFile"; + value: ApiCommitLfsFile; +} | { + key: "deletedFile"; + value: ApiCommitDeletedEntry; +}; +export interface ApiCommitData { + id: string; + title: string; + message: string; + authors: Array<{ + user: string; + avatar: string; + }>; + date: string; + formatted?: string; +} +//# sourceMappingURL=api-commit.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-commit.d.ts.map b/node_modules/@huggingface/hub/dist/src/types/api/api-commit.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..739574dc874764752afcc729e0196b6ac026e9d2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-commit.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"api-commit.d.ts","sourceRoot":"","sources":["../../../../src/types/api/api-commit.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IAElC,SAAS,EAAE,UAAU,GAAG,QAAQ,CAAC;IACjC,SAAS,CAAC,EAAE,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC1C;;;;;OAKG;IACH,GAAG,CAAC,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;KACb,GAAG,IAAI,CAAC;IACT,OAAO,EAAE;QACR,GAAG,EAAE,MAAM,CAAC;QACZ;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;KACb,EAAE,CAAC;IACJ;;SAEK;IACL,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IACnC,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,OAAO,EAAE,oBAAoB,EAAE,CAAC;CAChC;AAED,MAAM,MAAM,sBAAsB,GAAG,OAAO,GAAG,WAAW,GAAG,KAAK,CAAC;AAEnE,MAAM,WAAW,8BAA8B;IAC9C,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC9C;AAED,MAAM,WAAW,oBAAoB;IACpC;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,OAAO,CAAC,EAAE;QACT;;;WAGG;QACH,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB;;;WAGG;QACH,MAAM,CAAC,EAAE,YAAY,CAAC;QACtB;;;WAGG;QACH,MAAM,CAAC,EAAE,YAAY,CAAC;KACtB,CAAC;IACF;;;OAGG;IACH,KAAK,CAAC,EAAE;QACP,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KAChB,CAAC;CACF;AAED,MAAM,WAAW,YAAY;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,MAAM,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,GAAG;QAAE,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7D;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,mBAAmB;IACnC;;;;;;;;OAQG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,KAAK,CAAC;QACZ;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QACb;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QACb;;WAEG;QACH,MAAM,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;CACH;AACD,MAAM,WAAW,sBAAsB;IACtC,gDAAgD;IAChD,OAAO,EAAE,OAAO,CAAC;IACjB,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;KACd,CAAC,CAAC;CACH;AAED,MAAM,WAAW,oBAAoB;IACpC,KAAK,EAAE,KAAK,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,UAAU,EAAE,KAAK,GAAG,SAAS,CAAC;KAC9B,CAAC,CAAC;CACH;AAED,MAAM,WAAW,eAAe;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,qBAAqB;IACrC,IAAI,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6CAA6C;IAC7C,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,6CAA6C;IAC7C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,aAAa;IAC7B,6CAA6C;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC;CAC9B;AAED,MAAM,MAAM,kBAAkB,GAC3B;IACA,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,aAAa,CAAC;CACpB,GACD;IACA,GAAG,EAAE,SAAS,CAAC;IACf,KAAK,EAAE,gBAAgB,CAAC;CACvB,GACD;IACA,GAAG,EAAE,aAAa,CAAC;IACnB,KAAK,EAAE,qBAAqB,CAAC;CAC5B,CAAC;AAEL,MAAM,WAAW,aAAa;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-create-collection.d.ts b/node_modules/@huggingface/hub/dist/src/types/api/api-create-collection.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4dc723003a5d39c2ac52bfd4ca099d880a67567c --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-create-collection.d.ts @@ -0,0 +1,20 @@ +export interface ApiCreateCollectionPayload { + /** + * Title of the collection to create. + */ + title: string; + /** + * Namespace of the collection to create (username or org). + */ + namespace: string; + /** + * Description of the collection to create. + */ + description?: string; + /** + * Whether the collection should be private or not. Defaults to False (i.e. public collection). + * @default false + */ + private?: boolean; +} +//# sourceMappingURL=api-create-collection.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-create-collection.d.ts.map b/node_modules/@huggingface/hub/dist/src/types/api/api-create-collection.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7bb2b04e3420ab3d356202c5a9ce3fbe2c2cafba --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-create-collection.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"api-create-collection.d.ts","sourceRoot":"","sources":["../../../../src/types/api/api-create-collection.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,0BAA0B;IAC1C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;CAClB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-create-repo.d.ts b/node_modules/@huggingface/hub/dist/src/types/api/api-create-repo.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..eaabad12ad64f06ba889eb9849a252942d0ec576 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-create-repo.d.ts @@ -0,0 +1,22 @@ +import type { SetRequired } from "../../vendor/type-fest/set-required"; +import type { RepoType, SpaceHardwareFlavor, SpaceSdk } from "../public"; +import type { ApiCommitFile } from "./api-commit"; +export type ApiCreateRepoPayload = { + name: string; + canonical?: boolean; + license?: string; + resourceGroupId?: string; + template?: string; + organization?: string; + visibility?: "public" | "private" | "protected"; + lfsmultipartthresh?: number; + files?: SetRequired[]; +} & ({ + type: Exclude; +} | { + type: "space"; + hardware?: SpaceHardwareFlavor; + sdk: SpaceSdk; + sdkVersion?: string; +}); +//# sourceMappingURL=api-create-repo.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-create-repo.d.ts.map b/node_modules/@huggingface/hub/dist/src/types/api/api-create-repo.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..72b9b712ccab1af0e09eb82a2915bc6e074bb282 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-create-repo.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"api-create-repo.d.ts","sourceRoot":"","sources":["../../../../src/types/api/api-create-repo.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qCAAqC,CAAC;AACvE,OAAO,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACzE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAElD,MAAM,MAAM,oBAAoB,GAAG;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,CAAC;IAChD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC,aAAa,EAAE,SAAS,CAAC,EAAE,CAAC;CAChD,GAAG,CACD;IACA,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;CAChC,GACD;IACA,IAAI,EAAE,OAAO,CAAC;IACd,QAAQ,CAAC,EAAE,mBAAmB,CAAC;IAC/B,GAAG,EAAE,QAAQ,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;CACnB,CACH,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-dataset.d.ts b/node_modules/@huggingface/hub/dist/src/types/api/api-dataset.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..65257d1d3b97dab7e9b65eba2786b875e295e204 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-dataset.d.ts @@ -0,0 +1,95 @@ +import type { License } from "../public"; +export interface ApiDatasetInfo { + _id: string; + id: string; + arxivIds?: string[]; + author?: string; + cardExists?: true; + cardError?: unknown; + cardData?: ApiDatasetMetadata; + contributors?: Array<{ + user: string; + _id: string; + }>; + disabled: boolean; + discussionsDisabled: boolean; + gated: false | "auto" | "manual"; + gitalyUid: string; + lastAuthor: { + email: string; + user?: string; + }; + lastModified: string; + likes: number; + likesRecent: number; + private: boolean; + updatedAt: string; + createdAt: string; + tags: string[]; + paperswithcode_id?: string; + sha: string; + files?: string[]; + citation?: string; + description?: string; + downloads: number; + downloadsAllTime: number; + previewable?: boolean; + doi?: { + id: string; + commit: string; + }; +} +export interface ApiDatasetMetadata { + licenses?: undefined; + license?: License | License[]; + license_name?: string; + license_link?: "LICENSE" | "LICENSE.md" | string; + license_details?: string; + languages?: undefined; + language?: string | string[]; + language_bcp47?: string[]; + language_details?: string; + tags?: string[]; + task_categories?: string[]; + task_ids?: string[]; + config_names?: string[]; + configs?: { + config_name: string; + data_files?: string | string[] | { + split: string; + path: string | string[]; + }[]; + data_dir?: string; + }[]; + benchmark?: string; + paperswithcode_id?: string | null; + pretty_name?: string; + viewer?: boolean; + viewer_display_urls?: boolean; + thumbnail?: string | null; + description?: string | null; + annotations_creators?: string[]; + language_creators?: string[]; + multilinguality?: string[]; + size_categories?: string[]; + source_datasets?: string[]; + extra_gated_prompt?: string; + extra_gated_fields?: { + /** + * "text" | "checkbox" | "date_picker" | "country" | "ip_location" | { type: "text" | "checkbox" | "date_picker" | "country" | "ip_location" } | { type: "select", options: Array } Property + */ + [x: string]: "text" | "checkbox" | "date_picker" | "country" | "ip_location" | { + type: "text" | "checkbox" | "date_picker" | "country" | "ip_location"; + } | { + type: "select"; + options: Array; + }; + }; + extra_gated_heading?: string; + extra_gated_description?: string; + extra_gated_button_content?: string; +} +//# sourceMappingURL=api-dataset.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-dataset.d.ts.map b/node_modules/@huggingface/hub/dist/src/types/api/api-dataset.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a6edd3577ab63ce94adea5ef3ad0e97b0788dda4 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-dataset.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"api-dataset.d.ts","sourceRoot":"","sources":["../../../../src/types/api/api-dataset.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEzC,MAAM,WAAW,cAAc;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,IAAI,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B,YAAY,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACpD,QAAQ,EAAE,OAAO,CAAC;IAClB,mBAAmB,EAAE,OAAO,CAAC;IAC7B,KAAK,EAAE,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,GAAG,CAAC,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CACrC;AAED,MAAM,WAAW,kBAAkB;IAClC,QAAQ,CAAC,EAAE,SAAS,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,GAAG,OAAO,EAAE,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,SAAS,GAAG,YAAY,GAAG,MAAM,CAAC;IACjD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,CAAC,EAAE;QACT,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EACR,MAAM,GACN,MAAM,EAAE,GACR;YACA,KAAK,EAAE,MAAM,CAAC;YACd,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;SACvB,EAAE,CAAC;QACP,QAAQ,CAAC,EAAE,MAAM,CAAC;KAClB,EAAE,CAAC;IACJ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kBAAkB,CAAC,EAAE;QACpB;;WAEG;QACH,CAAC,CAAC,EAAE,MAAM,GACP,MAAM,GACN,UAAU,GACV,aAAa,GACb,SAAS,GACT,aAAa,GACb;YAAE,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,aAAa,GAAG,SAAS,GAAG,aAAa,CAAA;SAAE,GACzE;YAAE,IAAI,EAAE,QAAQ,CAAC;YAAC,OAAO,EAAE,KAAK,CAAC,MAAM,GAAG;gBAAE,KAAK,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAA;aAAE,CAAC,CAAA;SAAE,CAAC;KACjF,CAAC;IACF,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,0BAA0B,CAAC,EAAE,MAAM,CAAC;CACpC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-index-tree.d.ts b/node_modules/@huggingface/hub/dist/src/types/api/api-index-tree.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..1e85d4fd1c9c660e998e851443a7732d9c5d902b --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-index-tree.d.ts @@ -0,0 +1,48 @@ +export interface ApiIndexTreeEntry { + type: "file" | "directory" | "unknown"; + size: number; + path: string; + oid: string; + lfs?: { + oid: string; + size: number; + /** Size of the raw pointer file, 100~200 bytes */ + pointerSize: number; + }; + /** + * Xet content hash. Set for bucket file entries (always) and for repo LFS entries + * that have been migrated to xet. + */ + xetHash?: string; + lastCommit?: { + date: string; + id: string; + title: string; + }; + security?: ApiFileScanResult; +} +export interface ApiFileScanResult { + /** namespaced by repo type (models/, datasets/, spaces/) */ + repositoryId: string; + blobId: string; + name: string; + safe: boolean; + avScan?: ApiAVScan; + pickleImportScan?: ApiPickleImportScan; +} +interface ApiAVScan { + virusFound: boolean; + virusNames?: string[]; +} +type ApiSafetyLevel = "innocuous" | "suspicious" | "dangerous"; +interface ApiPickleImport { + module: string; + name: string; + safety: ApiSafetyLevel; +} +interface ApiPickleImportScan { + highestSafetyLevel: ApiSafetyLevel; + imports: ApiPickleImport[]; +} +export {}; +//# sourceMappingURL=api-index-tree.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-index-tree.d.ts.map b/node_modules/@huggingface/hub/dist/src/types/api/api-index-tree.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d1b04aa188ace68e6f35a11608f1e66030122d3a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-index-tree.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"api-index-tree.d.ts","sourceRoot":"","sources":["../../../../src/types/api/api-index-tree.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAAC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE;QACL,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,kDAAkD;QAClD,WAAW,EAAE,MAAM,CAAC;KACpB,CAAC;IACF;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,EAAE,EAAE,MAAM,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;KACd,CAAC;IACF,QAAQ,CAAC,EAAE,iBAAiB,CAAC;CAC7B;AAED,MAAM,WAAW,iBAAiB;IACjC,4DAA4D;IAC5D,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,gBAAgB,CAAC,EAAE,mBAAmB,CAAC;CACvC;AAED,UAAU,SAAS;IAClB,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,KAAK,cAAc,GAAG,WAAW,GAAG,YAAY,GAAG,WAAW,CAAC;AAE/D,UAAU,eAAe;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,cAAc,CAAC;CACvB;AAED,UAAU,mBAAmB;IAC5B,kBAAkB,EAAE,cAAc,CAAC;IACnC,OAAO,EAAE,eAAe,EAAE,CAAC;CAC3B"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-jobs.d.ts b/node_modules/@huggingface/hub/dist/src/types/api/api-jobs.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..bc16dee1a6eb25f512eb0581aa1d33f913c0027a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-jobs.d.ts @@ -0,0 +1,165 @@ +import type { RepoDesignation, RepoType, SpaceHardwareFlavor } from "../public"; +export interface ApiJobVolume { + type: RepoType; + source: string; + mountPath: string; + revision?: string; + readOnly?: boolean; + path?: string; +} +export interface JobVolume { + /** Source repo, e.g. "datasets/user/my-dataset", "user/my-model", or { type: "dataset", name: "user/my-dataset" } */ + source: RepoDesignation; + /** Mount path inside the container, e.g. "/data" */ + mountPath: string; + /** Git revision (only for repos, defaults to "main") */ + revision?: string; + /** Read-only mount (forced true for repos, defaults to false for buckets) */ + readOnly?: boolean; + /** Subfolder prefix inside the bucket/repo to mount, e.g. "path/to/dir" */ + path?: string; +} +export interface ApiJobHardware { + name: string; + prettyName: string; + cpu: string; + ram: string; + accelerator: { + type: "gpu" | "neuron"; + model: string; + quantity: string; + vram: string; + manufacturer: "Nvidia" | "AWS"; + } | null; + unitCostMicroUSD: number; + unitCostUSD: number; + unitLabel: string; +} +export type JobStatusStage = "DELETING" | "RUNNING" | "PAUSED" | "STOPPED" | "UPDATING" | "ERROR"; +export interface ApiJobStatus { + stage: JobStatusStage; + message?: string | null; + failureCount: number; +} +export interface ApiJobUser { + id: string; + name: string; + type?: "user" | "org"; + avatarUrl?: string; +} +export interface ApiJob { + type: "job"; + id: string; + status: ApiJobStatus; + createdAt: string; + updatedAt?: string; + startedAt?: string | null; + finishedAt?: string | null; + createdBy?: { + id: string; + name: string; + }; + dockerImage?: string | null; + spaceId?: string | null; + command?: string[] | null; + arguments?: string[] | null; + environment?: Record | null; + flavor: SpaceHardwareFlavor; + arch?: "amd64" | "arm64" | null; + timeoutSeconds?: number | null; + attempts?: number; + owner?: ApiJobUser; + initiator?: ApiJobUser; + secrets?: string[]; + labels?: Record | null; + volumes?: ApiJobVolume[] | null; +} +export interface ApiScheduledJob { + id: string; + schedule: string; + suspend: boolean; + concurrency: boolean; + createdAt: string; + updatedAt: string; + jobSpec: { + dockerImage?: string | null; + spaceId?: string | null; + command?: string[] | null; + environment?: Record | null; + flavor: SpaceHardwareFlavor; + arch?: "amd64" | "arm64" | null; + timeoutSeconds?: number | null; + attempts?: number; + labels?: Record | null; + volumes?: ApiJobVolume[] | null; + }; +} +export interface CreateJobOptions { + /** + * The Docker image to run (e.g., "python:3.12" or "pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel") + */ + dockerImage?: string; + /** + * The Space ID to run (e.g., "username/space-name") + */ + spaceId?: string; + /** + * The command to run (array of strings) + */ + command?: string[]; + /** + * Additional arguments to pass to the command + */ + arguments?: string[]; + /** + * Environment variables to set + */ + environment?: Record; + /** + * Secrets to pass (will be encrypted server-side) + */ + secrets?: Record; + /** + * Hardware flavor to use + */ + flavor: SpaceHardwareFlavor; + /** + * Architecture (defaults to "amd64") + */ + arch?: "amd64" | "arm64"; + /** + * Timeout in seconds + */ + timeoutSeconds?: number | null; + /** + * Maximum number of attempts (defaults to 1) + */ + attempts?: number; + /** + * Labels to attach to the job (key-value pairs) + */ + labels?: Record; + /** + * HuggingFace Buckets or Repos to mount as volumes in the job container + */ + volumes?: JobVolume[]; +} +export interface CreateScheduledJobOptions { + /** + * The job specification + */ + jobSpec: Omit; + /** + * CRON schedule expression (e.g., "0 9 * * 1" for 9 AM every Monday) or shortcuts like "@hourly", "@daily" + */ + schedule: string; + /** + * Whether the scheduled job is suspended (paused) + */ + suspend?: boolean; + /** + * Whether multiple instances of this job can run concurrently + */ + concurrency?: boolean; +} +//# sourceMappingURL=api-jobs.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-jobs.d.ts.map b/node_modules/@huggingface/hub/dist/src/types/api/api-jobs.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..3136fa8a29d899f7649d802bdfa7415ed996fdbb --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-jobs.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"api-jobs.d.ts","sourceRoot":"","sources":["../../../../src/types/api/api-jobs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AAEhF,MAAM,WAAW,YAAY;IAC5B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,SAAS;IACzB,qHAAqH;IACrH,MAAM,EAAE,eAAe,CAAC;IACxB,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6EAA6E;IAC7E,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,2EAA2E;IAC3E,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE;QACZ,IAAI,EAAE,KAAK,GAAG,QAAQ,CAAC;QACvB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,YAAY,EAAE,QAAQ,GAAG,KAAK,CAAC;KAC/B,GAAG,IAAI,CAAC;IACT,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,SAAS,GAAG,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,CAAC;AAElG,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,cAAc,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,MAAM;IACtB,IAAI,EAAE,KAAK,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,YAAY,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,SAAS,CAAC,EAAE;QACX,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;KACb,CAAC;IACF,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IAC5C,MAAM,EAAE,mBAAmB,CAAC;IAC5B,IAAI,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,IAAI,CAAC;IAChC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,SAAS,CAAC,EAAE,UAAU,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;CAChC;AAED,MAAM,WAAW,eAAe;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,OAAO,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QACR,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAC1B,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;QAC5C,MAAM,EAAE,mBAAmB,CAAC;QAC5B,IAAI,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,IAAI,CAAC;QAChC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;QACvC,OAAO,CAAC,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;KAChC,CAAC;CACF;AAED,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;;OAEG;IACH,MAAM,EAAE,mBAAmB,CAAC;IAC5B;;OAEG;IACH,IAAI,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC;IACzB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC;;OAEG;IACH,OAAO,CAAC,EAAE,SAAS,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,yBAAyB;IACzC;;OAEG;IACH,OAAO,EAAE,IAAI,CAAC,gBAAgB,EAAE,WAAW,CAAC,CAAC;IAC7C;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACtB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-model.d.ts b/node_modules/@huggingface/hub/dist/src/types/api/api-model.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..3e655423dec735f0c1b0be8a4eef60c453b14b76 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-model.d.ts @@ -0,0 +1,279 @@ +import type { ModelLibraryKey, TransformersInfo, WidgetType } from "@huggingface/tasks"; +import type { License, PipelineType } from "../public"; +export interface ApiModelInfo { + _id: string; + id: string; + arxivIds: string[]; + author?: string; + cardData?: ApiModelMetadata; + cardError: unknown; + cardExists?: true; + config: unknown; + contributors: Array<{ + user: string; + _id: string; + }>; + disabled: boolean; + discussionsDisabled: boolean; + doi?: { + id: string; + commit: string; + }; + downloads: number; + downloadsAllTime: number; + files: string[]; + gitalyUid: string; + inferenceProviderMapping?: ApiModelInferenceProviderMappingEntry[]; + lastAuthor: { + email: string; + user?: string; + }; + lastModified: string; + library_name?: ModelLibraryKey; + likes: number; + likesRecent: number; + private: boolean; + gated: false | "auto" | "manual"; + sha: string; + spaces: string[]; + updatedAt: string; + createdAt: string; + pipeline_tag: PipelineType; + tags: string[]; + "model-index": unknown; + safetensors?: { + parameters: Record; + total: number; + }; + siblings: Array<{ + rfilename: string; + }>; + transformersInfo?: TransformersInfo; +} +export interface ApiModelIndex { + name: string; + results: { + task: { + /** + * Example: automatic-speech-recognition +Use task id from https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/tasksData.ts + */ + type: string; + /** + * Example: Speech Recognition + */ + name?: string; + }; + /** + * This will switch to required at some point. +in any case, we need them to link to PWC + */ + dataset?: { + /** + * Example: common_voice. Use dataset id from https://hf.co/datasets + */ + type: string; + /** + * A pretty name for the dataset. Example: Common Voice zh-CN +Also encode config params into the name if relevant. + */ + name: string; + /** + * Optional. The name of the dataset configuration used in `load_dataset()` + */ + config?: string; + /** + * Optional. Example: test + */ + split?: string; + /** + * Optional. Example: 5503434ddd753f426f4b38109466949a1217c2bb + */ + revision?: string; + args?: string | { + /** + * String Property + */ + [x: string]: string; + }; + }; + metrics: { + /** + * Example: wer. Use metric id from https://hf.co/metrics + */ + type: string; + /** + * Required. Example: 20.0 or "20.0 ± 1.2" + */ + value: unknown; + /** + * Example: Test WER + */ + name?: string; + /** + * Optional. The name of the metric configuration used in `load_metric()`. + */ + config?: string; + args?: string | { + /** + * String Property + */ + [x: string]: string; + }; + /** + * [Automatically computed, do not set] Dynamically overridden by huggingface in API calls to indicate if it was verified by Hugging Face. + */ + verified?: boolean; + /** + * Generated by Hugging Face to prove the results are valid. + */ + verifyToken?: string; + }[]; + /** + * The source for this evaluation result. + */ + source?: { + /** + * Example: Open LLM Leaderboard + */ + name?: string; + /** + * Example: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard + */ + url: string; + }; + }[]; +} +export interface ApiWidgetExampleFromModelcard { + example_title?: string; + group?: string; + text?: string; + src?: string; + table?: { + /** + * (string | number)[] Property + */ + [x: string]: (string | number)[]; + }; + structured_data?: { + /** + * (string | number)[] Property + */ + [x: string]: (string | number)[]; + }; + candidate_labels?: string; + messages?: { + role: "system" | "user" | "assistant"; + content: string; + }[]; + multi_class?: boolean; + source_sentence?: string; + sentences?: string[]; + parameters?: { + aggregation_strategy?: string; + top_k?: number; + top_p?: number; + temperature?: number; + max_new_tokens?: number; + do_sample?: boolean; + negative_prompt?: string; + guidance_scale?: number; + num_inference_steps?: number; + }; + output?: { + label: string; + score: number; + }[] | { + answer: string; + score: number; + } | { + text: string; + } | { + url: string; + }; +} +export interface ApiModelMetadata { + datasets?: string | string[]; + license?: License | License[]; + license_name?: string; + license_link?: "LICENSE" | "LICENSE.md" | string; + license_details?: string; + inference?: boolean | { + parameters?: { + aggregation_strategy?: string; + top_k?: number; + top_p?: number; + temperature?: number; + max_new_tokens?: number; + do_sample?: boolean; + negative_prompt?: string; + guidance_scale?: number; + num_inference_steps?: number; + }; + }; + language?: string | string[]; + language_bcp47?: string[]; + language_details?: string; + tags?: string[]; + pipeline_tag?: string; + co2_eq_emissions?: number | { + /** + * Emissions in grams of CO2 + */ + emissions: number; + /** + * source of the information, either directly from AutoTrain, code carbon or from a scientific article documenting the model + */ + source?: string; + /** + * pre-training or fine-tuning + */ + training_type?: string; + /** + * as granular as possible, for instance Quebec, Canada or Brooklyn, NY, USA + */ + geographical_location?: string; + /** + * how much compute and what kind, e.g. 8 v100 GPUs + */ + hardware_used?: string; + }; + library_name?: string; + thumbnail?: string | null; + description?: string | null; + mask_token?: string; + widget?: ApiWidgetExampleFromModelcard[]; + "model-index"?: ApiModelIndex[]; + finetuned_from?: string; + base_model?: string | string[]; + instance_prompt?: string | null; + extra_gated_prompt?: string; + extra_gated_fields?: { + /** + * "text" | "checkbox" | "date_picker" | "country" | "ip_location" | { type: "text" | "checkbox" | "date_picker" | "country" | "ip_location" } | { type: "select", options: Array } Property + */ + [x: string]: "text" | "checkbox" | "date_picker" | "country" | "ip_location" | { + type: "text" | "checkbox" | "date_picker" | "country" | "ip_location"; + } | { + type: "select"; + options: Array; + }; + }; + extra_gated_heading?: string; + extra_gated_description?: string; + extra_gated_button_content?: string; +} +export interface ApiModelInferenceProviderMappingEntry { + provider: string; + hfModelId: string; + providerId: string; + status: "live" | "staging"; + task: WidgetType; + adapter?: string; + adapterWeightsPath?: string; + type?: "single-file" | "tag-filter"; +} +//# sourceMappingURL=api-model.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-model.d.ts.map b/node_modules/@huggingface/hub/dist/src/types/api/api-model.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..2771c3d9625fae6e805ddd5d16ab91893e4136f6 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-model.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"api-model.d.ts","sourceRoot":"","sources":["../../../../src/types/api/api-model.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACxF,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEvD,MAAM,WAAW,YAAY;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,IAAI,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACnD,QAAQ,EAAE,OAAO,CAAC;IAClB,mBAAmB,EAAE,OAAO,CAAC;IAC7B,GAAG,CAAC,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,wBAAwB,CAAC,EAAE,qCAAqC,EAAE,CAAC;IACnE,UAAU,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,eAAe,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAC;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,YAAY,CAAC;IAC3B,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE;QACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,KAAK,EAAE,MAAM,CAAC;KACd,CAAC;IACF,QAAQ,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACpC;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE;QACR,IAAI,EAAE;YACL;;;eAGG;YACH,IAAI,EAAE,MAAM,CAAC;YACb;;eAEG;YACH,IAAI,CAAC,EAAE,MAAM,CAAC;SACd,CAAC;QACF;;;WAGG;QACH,OAAO,CAAC,EAAE;YACT;;eAEG;YACH,IAAI,EAAE,MAAM,CAAC;YACb;;;eAGG;YACH,IAAI,EAAE,MAAM,CAAC;YACb;;eAEG;YACH,MAAM,CAAC,EAAE,MAAM,CAAC;YAChB;;eAEG;YACH,KAAK,CAAC,EAAE,MAAM,CAAC;YACf;;eAEG;YACH,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,IAAI,CAAC,EACF,MAAM,GACN;gBACA;;mBAEG;gBACH,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;aACnB,CAAC;SACL,CAAC;QACF,OAAO,EAAE;YACR;;eAEG;YACH,IAAI,EAAE,MAAM,CAAC;YACb;;eAEG;YACH,KAAK,EAAE,OAAO,CAAC;YACf;;eAEG;YACH,IAAI,CAAC,EAAE,MAAM,CAAC;YACd;;eAEG;YACH,MAAM,CAAC,EAAE,MAAM,CAAC;YAChB,IAAI,CAAC,EACF,MAAM,GACN;gBACA;;mBAEG;gBACH,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;aACnB,CAAC;YACL;;eAEG;YACH,QAAQ,CAAC,EAAE,OAAO,CAAC;YACnB;;eAEG;YACH,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,EAAE,CAAC;QACJ;;WAEG;QACH,MAAM,CAAC,EAAE;YACR;;eAEG;YACH,IAAI,CAAC,EAAE,MAAM,CAAC;YACd;;eAEG;YACH,GAAG,EAAE,MAAM,CAAC;SACZ,CAAC;KACF,EAAE,CAAC;CACJ;AAED,MAAM,WAAW,6BAA6B;IAC7C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE;QACP;;WAEG;QACH,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;KACjC,CAAC;IACF,eAAe,CAAC,EAAE;QACjB;;WAEG;QACH,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;KACjC,CAAC;IACF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,EAAE;QACV,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;QACtC,OAAO,EAAE,MAAM,CAAC;KAChB,EAAE,CAAC;IACJ,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE;QACZ,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,mBAAmB,CAAC,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF,MAAM,CAAC,EACJ;QACA,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACb,EAAE,GACH;QACA,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACb,GACD;QACA,IAAI,EAAE,MAAM,CAAC;KACZ,GACD;QACA,GAAG,EAAE,MAAM,CAAC;KACX,CAAC;CACL;AAED,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC7B,OAAO,CAAC,EAAE,OAAO,GAAG,OAAO,EAAE,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,SAAS,GAAG,YAAY,GAAG,MAAM,CAAC;IACjD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EACP,OAAO,GACP;QACA,UAAU,CAAC,EAAE;YACZ,oBAAoB,CAAC,EAAE,MAAM,CAAC;YAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,KAAK,CAAC,EAAE,MAAM,CAAC;YACf,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,cAAc,CAAC,EAAE,MAAM,CAAC;YACxB,SAAS,CAAC,EAAE,OAAO,CAAC;YACpB,eAAe,CAAC,EAAE,MAAM,CAAC;YACzB,cAAc,CAAC,EAAE,MAAM,CAAC;YACxB,mBAAmB,CAAC,EAAE,MAAM,CAAC;SAC7B,CAAC;KACD,CAAC;IACL,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EACd,MAAM,GACN;QACA;;WAEG;QACH,SAAS,EAAE,MAAM,CAAC;QAClB;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;WAEG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB;;WAEG;QACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B;;WAEG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;IACL,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,6BAA6B,EAAE,CAAC;IACzC,aAAa,CAAC,EAAE,aAAa,EAAE,CAAC;IAChC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kBAAkB,CAAC,EAAE;QACpB;;WAEG;QACH,CAAC,CAAC,EAAE,MAAM,GACP,MAAM,GACN,UAAU,GACV,aAAa,GACb,SAAS,GACT,aAAa,GACb;YAAE,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,aAAa,GAAG,SAAS,GAAG,aAAa,CAAA;SAAE,GACzE;YAAE,IAAI,EAAE,QAAQ,CAAC;YAAC,OAAO,EAAE,KAAK,CAAC,MAAM,GAAG;gBAAE,KAAK,EAAE,MAAM,CAAC;gBAAC,KAAK,EAAE,MAAM,CAAA;aAAE,CAAC,CAAA;SAAE,CAAC;KACjF,CAAC;IACF,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,0BAA0B,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,MAAM,WAAW,qCAAqC;IACrD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,IAAI,EAAE,UAAU,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,IAAI,CAAC,EAAE,aAAa,GAAG,YAAY,CAAC;CACpC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-space.d.ts b/node_modules/@huggingface/hub/dist/src/types/api/api-space.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..9adcefe9ab29392235bc5bb66d5b8c5cefb1a181 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-space.d.ts @@ -0,0 +1,92 @@ +import type { License, SpaceRuntime, SpaceSdk } from "../public"; +type Color = "red" | "yellow" | "green" | "blue" | "indigo" | "purple" | "pink" | "gray"; +export interface ApiSpaceInfo { + _id: string; + id: string; + arxivIds?: string[]; + author: string; + cardExists?: true; + cardError?: unknown; + cardData?: unknown; + contributors?: Array<{ + user: string; + _id: string; + }>; + disabled: boolean; + discussionsDisabled: boolean; + duplicationDisabled: boolean; + gated: false | "auto" | "manual"; + gitalyUid: string; + lastAuthor: { + email: string; + user?: string; + }; + lastModified: string; + likes: number; + likesRecent: number; + private: boolean; + updatedAt: string; + createdAt: string; + tags: string[]; + sha: string; + subdomain: string; + title: string; + emoji: string; + colorFrom: Color; + colorTo: Color; + pinned: boolean; + siblings: Array<{ + rfilename: string; + }>; + sdk?: SpaceSdk; + runtime?: SpaceRuntime; + models?: string[]; + datasets?: string[]; + originSpace?: { + _id: string; + authorId: string; + }; +} +export interface ApiSpaceMetadata { + license?: License | License[]; + tags?: string[]; + title?: string; + colorFrom?: "red" | "yellow" | "green" | "blue" | "indigo" | "purple" | "pink" | "gray"; + colorTo?: "red" | "yellow" | "green" | "blue" | "indigo" | "purple" | "pink" | "gray"; + emoji?: string; + sdk?: "streamlit" | "gradio" | "docker" | "static"; + sdk_version?: string | string; + python_version?: string | string; + fullWidth?: boolean; + header?: "mini" | "default"; + app_file?: string; + app_port?: number; + base_path?: string; + models?: string[]; + datasets?: string[]; + pinned?: boolean; + metaTitle?: string; + description?: string; + thumbnail?: string; + /** + * If enabled, will associate an oauth app to the Space, adding variables and secrets to the Space's environment + */ + hf_oauth?: boolean; + /** + * The expiration of access tokens for your oauth app in minutes. max 30 days (43,200 minutes). Defaults to 8 hours (480 minutes) + */ + hf_oauth_expiration_minutes?: number; + /** + * OAuth scopes to request. By default you have access to the user's profile, you can request access to their repos or inference-api. + */ + hf_oauth_scopes?: ("email" | "read-repos" | "write-repos" | "manage-repos" | "inference-api")[]; + suggested_hardware?: "cpu-basic" | "zero-a10g" | "cpu-upgrade" | "cpu-xl" | "t4-small" | "t4-medium" | "a10g-small" | "a10g-large" | "a10g-largex2" | "a10g-largex4" | "a100-large"; + suggested_storage?: "small" | "medium" | "large"; + custom_headers?: { + "cross-origin-embedder-policy"?: "unsafe-none" | "require-corp" | "credentialless"; + "cross-origin-opener-policy"?: "same-origin" | "same-origin-allow-popups" | "unsafe-none"; + "cross-origin-resource-policy"?: "same-site" | "same-origin" | "cross-origin"; + }; +} +export {}; +//# sourceMappingURL=api-space.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-space.d.ts.map b/node_modules/@huggingface/hub/dist/src/types/api/api-space.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d8951d8f85a48df3be5177f029ee2929f460e8b0 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-space.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"api-space.d.ts","sourceRoot":"","sources":["../../../../src/types/api/api-space.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAEjE,KAAK,KAAK,GAAG,KAAK,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAC;AAEzF,MAAM,WAAW,YAAY;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,IAAI,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,YAAY,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACpD,QAAQ,EAAE,OAAO,CAAC;IAClB,mBAAmB,EAAE,OAAO,CAAC;IAC7B,mBAAmB,EAAE,OAAO,CAAC;IAC7B,KAAK,EAAE,KAAK,GAAG,MAAM,GAAG,QAAQ,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,KAAK,CAAC;IACjB,OAAO,EAAE,KAAK,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvC,GAAG,CAAC,EAAE,QAAQ,CAAC;IACf,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,WAAW,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;CAChD;AAED,MAAM,WAAW,gBAAgB;IAChC,OAAO,CAAC,EAAE,OAAO,GAAG,OAAO,EAAE,CAAC;IAC9B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAC;IACxF,OAAO,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAC;IACtF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACnD,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC9B,cAAc,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACjC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;OAEG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC;;OAEG;IACH,eAAe,CAAC,EAAE,CAAC,OAAO,GAAG,YAAY,GAAG,aAAa,GAAG,cAAc,GAAG,eAAe,CAAC,EAAE,CAAC;IAChG,kBAAkB,CAAC,EAChB,WAAW,GACX,WAAW,GACX,aAAa,GACb,QAAQ,GACR,UAAU,GACV,WAAW,GACX,YAAY,GACZ,YAAY,GACZ,cAAc,GACd,cAAc,GACd,YAAY,CAAC;IAChB,iBAAiB,CAAC,EAAE,OAAO,GAAG,QAAQ,GAAG,OAAO,CAAC;IACjD,cAAc,CAAC,EAAE;QAChB,8BAA8B,CAAC,EAAE,aAAa,GAAG,cAAc,GAAG,gBAAgB,CAAC;QACnF,4BAA4B,CAAC,EAAE,aAAa,GAAG,0BAA0B,GAAG,aAAa,CAAC;QAC1F,8BAA8B,CAAC,EAAE,WAAW,GAAG,aAAa,GAAG,cAAc,CAAC;KAC9E,CAAC;CACF"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-who-am-i.d.ts b/node_modules/@huggingface/hub/dist/src/types/api/api-who-am-i.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..62d92b26d78590a7da5a1683fc90b3b35876d79a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-who-am-i.d.ts @@ -0,0 +1,47 @@ +import type { AccessTokenRole, AuthType } from "../public"; +interface ApiWhoAmIBase { + /** Unique ID persistent across renames */ + id: string; + type: "user" | "org" | "app"; + name: string; +} +interface ApiWhoAmIEntityBase extends ApiWhoAmIBase { + fullname: string; + email: string | null; + canPay: boolean; + avatarUrl: string; + /** + * Unix timestamp in seconds + */ + periodEnd: number | null; +} +interface ApiWhoAmIOrg extends ApiWhoAmIEntityBase { + type: "org"; +} +interface ApiWhoAmIUser extends ApiWhoAmIEntityBase { + type: "user"; + email: string; + emailVerified: boolean; + isPro: boolean; + orgs: ApiWhoAmIOrg[]; + billingMode: "postpaid" | "prepaid"; +} +interface ApiWhoAmIApp extends ApiWhoAmIBase { + type: "app"; + name: string; + scope?: { + entities: string[]; + role: AccessTokenRole; + }; +} +export type ApiWhoAmIReponse = ApiWhoAmIUser | ApiWhoAmIOrg | ApiWhoAmIApp; +export interface ApiWhoAmIAuthInfo { + type: AuthType; + accessToken?: { + displayName: string; + expiration?: string; + role: AccessTokenRole; + }; +} +export {}; +//# sourceMappingURL=api-who-am-i.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/api/api-who-am-i.d.ts.map b/node_modules/@huggingface/hub/dist/src/types/api/api-who-am-i.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..eb31de33ff8a6dabad5bc68653812e99c83c494f --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/api/api-who-am-i.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"api-who-am-i.d.ts","sourceRoot":"","sources":["../../../../src/types/api/api-who-am-i.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE3D,UAAU,aAAa;IACtB,0CAA0C;IAC1C,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,CAAC;IAC7B,IAAI,EAAE,MAAM,CAAC;CACb;AAED,UAAU,mBAAoB,SAAQ,aAAa;IAClD,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,UAAU,YAAa,SAAQ,mBAAmB;IACjD,IAAI,EAAE,KAAK,CAAC;CACZ;AAED,UAAU,aAAc,SAAQ,mBAAmB;IAClD,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,OAAO,CAAC;IACvB,KAAK,EAAE,OAAO,CAAC;IACf,IAAI,EAAE,YAAY,EAAE,CAAC;IACrB,WAAW,EAAE,UAAU,GAAG,SAAS,CAAC;CACpC;AAED,UAAU,YAAa,SAAQ,aAAa;IAC3C,IAAI,EAAE,KAAK,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE;QACP,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,IAAI,EAAE,eAAe,CAAC;KACtB,CAAC;CACF;AAED,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG,YAAY,GAAG,YAAY,CAAC;AAE3E,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,QAAQ,CAAC;IACf,WAAW,CAAC,EAAE;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,eAAe,CAAC;KACtB,CAAC;CACF"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/public.d.ts b/node_modules/@huggingface/hub/dist/src/types/public.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4b3824c3408ab74649b6a772be2fa298c72d1eb6 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/public.d.ts @@ -0,0 +1,77 @@ +import type { PipelineType } from "@huggingface/tasks"; +export type RepoType = "space" | "dataset" | "model" | "bucket" | "kernel"; +export interface RepoId { + name: string; + type: RepoType; +} +export type RepoFullName = string | `spaces/${string}` | `datasets/${string}` | `buckets/${string}` | `kernels/${string}`; +export type RepoDesignation = RepoId | RepoFullName; +/** + * A {@link RepoDesignation} narrowed to bucket repos. + * + * Used by APIs that only operate on buckets (e.g. {@link copyFile}, {@link copyFiles}, + * {@link copyFolder}). + */ +export type BucketDesignation = { + type: "bucket"; + name: string; +} | `buckets/${string}`; +/** Actually `hf_${string}`, but for convenience, using the string type */ +export type AccessToken = string; +/** + * @deprecated Use `AccessToken` instead. Pass { accessToken: "hf_..." } instead of { credentials: { accessToken: "hf_..." } } + */ +export interface Credentials { + accessToken: AccessToken; +} +export type CredentialsParams = { + accessToken?: undefined; + /** + * @deprecated Use `accessToken` instead + */ + credentials: Credentials; +} | { + accessToken: AccessToken; + /** + * @deprecated Use `accessToken` instead + */ + credentials?: undefined; +}; +export type SpaceHardwareFlavor = "cpu-basic" | "cpu-upgrade" | "cpu-performance" | "cpu-xl" | "sprx8" | "zero-a10g" | "inf2x6" | "t4-small" | "t4-medium" | "l4x1" | "l4x4" | "l40sx1" | "l40sx4" | "l40sx8" | "a10g-small" | "a10g-large" | "a10g-largex2" | "a10g-largex4" | "a100-large" | "a100x4" | "a100x8"; +export type SpaceSdk = "streamlit" | "gradio" | "docker" | "static"; +export type SpaceStage = "NO_APP_FILE" | "CONFIG_ERROR" | "BUILDING" | "BUILD_ERROR" | "RUNNING" | "RUNNING_BUILDING" | "RUNTIME_ERROR" | "DELETING" | "PAUSED" | "SLEEPING"; +export type AccessTokenRole = "admin" | "write" | "contributor" | "read"; +export type AuthType = "access_token" | "app_token" | "app_token_as_user"; +export type { PipelineType }; +export interface SpaceRuntime { + stage: SpaceStage; + sdk?: SpaceSdk; + sdkVersion?: string; + errorMessage?: string; + hardware?: { + current: SpaceHardwareFlavor | null; + currentPrettyName?: string; + requested: SpaceHardwareFlavor | null; + requestedPrettyName?: string; + }; + /** when calling /spaces, those props are only fetched if ?full=true */ + resources?: SpaceResourceConfig; + /** in seconds */ + gcTimeout?: number | null; +} +export interface SpaceResourceRequirement { + cpu?: string; + memory?: string; + gpu?: string; + gpuModel?: string; + ephemeral?: string; +} +export interface SpaceResourceConfig { + requests: SpaceResourceRequirement; + limits: SpaceResourceRequirement; + replicas?: number; + throttled?: boolean; + is_custom?: boolean; +} +export type License = "apache-2.0" | "mit" | "openrail" | "bigscience-openrail-m" | "creativeml-openrail-m" | "bigscience-bloom-rail-1.0" | "bigcode-openrail-m" | "afl-3.0" | "artistic-2.0" | "bsl-1.0" | "bsd" | "bsd-2-clause" | "bsd-3-clause" | "bsd-3-clause-clear" | "c-uda" | "cc" | "cc0-1.0" | "cc-by-2.0" | "cc-by-2.5" | "cc-by-3.0" | "cc-by-4.0" | "cc-by-sa-3.0" | "cc-by-sa-4.0" | "cc-by-nc-2.0" | "cc-by-nc-3.0" | "cc-by-nc-4.0" | "cc-by-nd-4.0" | "cc-by-nc-nd-3.0" | "cc-by-nc-nd-4.0" | "cc-by-nc-sa-2.0" | "cc-by-nc-sa-3.0" | "cc-by-nc-sa-4.0" | "cdla-sharing-1.0" | "cdla-permissive-1.0" | "cdla-permissive-2.0" | "wtfpl" | "ecl-2.0" | "epl-1.0" | "epl-2.0" | "etalab-2.0" | "eupl-1.1" | "agpl-3.0" | "gfdl" | "gpl" | "gpl-2.0" | "gpl-3.0" | "lgpl" | "lgpl-2.1" | "lgpl-3.0" | "isc" | "lppl-1.3c" | "ms-pl" | "mpl-2.0" | "odc-by" | "odbl" | "openrail++" | "osl-3.0" | "postgresql" | "ofl-1.1" | "ncsa" | "unlicense" | "zlib" | "pddl" | "lgpl-lr" | "deepfloyd-if-license" | "llama2" | "llama3" | "llama3.1" | "llama3.2" | "llama3.3" | "gemma" | "apple-ascl" | "apple-amlr" | "unknown" | "other"; +//# sourceMappingURL=public.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/types/public.d.ts.map b/node_modules/@huggingface/hub/dist/src/types/public.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..644d3440d6cc814020484d93b9c02cdff5745976 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/types/public.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"public.d.ts","sourceRoot":"","sources":["../../../src/types/public.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAE3E,MAAM,WAAW,MAAM;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,CAAC;CACf;AAED,MAAM,MAAM,YAAY,GACrB,MAAM,GACN,UAAU,MAAM,EAAE,GAClB,YAAY,MAAM,EAAE,GACpB,WAAW,MAAM,EAAE,GACnB,WAAW,MAAM,EAAE,CAAC;AAEvB,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,YAAY,CAAC;AAEpD;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,MAAM,EAAE,CAAC;AAEvF,0EAA0E;AAC1E,MAAM,MAAM,WAAW,GAAG,MAAM,CAAC;AAEjC;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B,WAAW,EAAE,WAAW,CAAC;CACzB;AAED,MAAM,MAAM,iBAAiB,GAC1B;IACA,WAAW,CAAC,EAAE,SAAS,CAAC;IACxB;;OAEG;IACH,WAAW,EAAE,WAAW,CAAC;CACxB,GACD;IACA,WAAW,EAAE,WAAW,CAAC;IACzB;;OAEG;IACH,WAAW,CAAC,EAAE,SAAS,CAAC;CACvB,CAAC;AAEL,MAAM,MAAM,mBAAmB,GAC5B,WAAW,GACX,aAAa,GACb,iBAAiB,GACjB,QAAQ,GACR,OAAO,GACP,WAAW,GACX,QAAQ,GACR,UAAU,GACV,WAAW,GACX,MAAM,GACN,MAAM,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,YAAY,GACZ,YAAY,GACZ,cAAc,GACd,cAAc,GACd,YAAY,GACZ,QAAQ,GACR,QAAQ,CAAC;AAEZ,MAAM,MAAM,QAAQ,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEpE,MAAM,MAAM,UAAU,GACnB,aAAa,GACb,cAAc,GACd,UAAU,GACV,aAAa,GACb,SAAS,GACT,kBAAkB,GAClB,eAAe,GACf,UAAU,GACV,QAAQ,GACR,UAAU,CAAC;AAEd,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,OAAO,GAAG,aAAa,GAAG,MAAM,CAAC;AAEzE,MAAM,MAAM,QAAQ,GAAG,cAAc,GAAG,WAAW,GAAG,mBAAmB,CAAC;AAE1E,YAAY,EAAE,YAAY,EAAE,CAAC;AAE7B,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,UAAU,CAAC;IAClB,GAAG,CAAC,EAAE,QAAQ,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE;QACV,OAAO,EAAE,mBAAmB,GAAG,IAAI,CAAC;QACpC,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,SAAS,EAAE,mBAAmB,GAAG,IAAI,CAAC;QACtC,mBAAmB,CAAC,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF,uEAAuE;IACvE,SAAS,CAAC,EAAE,mBAAmB,CAAC;IAChC,iBAAiB;IACjB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,MAAM,WAAW,wBAAwB;IACxC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IACnC,QAAQ,EAAE,wBAAwB,CAAC;IACnC,MAAM,EAAE,wBAAwB,CAAC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,MAAM,OAAO,GAChB,YAAY,GACZ,KAAK,GACL,UAAU,GACV,uBAAuB,GACvB,uBAAuB,GACvB,2BAA2B,GAC3B,oBAAoB,GACpB,SAAS,GACT,cAAc,GACd,SAAS,GACT,KAAK,GACL,cAAc,GACd,cAAc,GACd,oBAAoB,GACpB,OAAO,GACP,IAAI,GACJ,SAAS,GACT,WAAW,GACX,WAAW,GACX,WAAW,GACX,WAAW,GACX,cAAc,GACd,cAAc,GACd,cAAc,GACd,cAAc,GACd,cAAc,GACd,cAAc,GACd,iBAAiB,GACjB,iBAAiB,GACjB,iBAAiB,GACjB,iBAAiB,GACjB,iBAAiB,GACjB,kBAAkB,GAClB,qBAAqB,GACrB,qBAAqB,GACrB,OAAO,GACP,SAAS,GACT,SAAS,GACT,SAAS,GACT,YAAY,GACZ,UAAU,GACV,UAAU,GACV,MAAM,GACN,KAAK,GACL,SAAS,GACT,SAAS,GACT,MAAM,GACN,UAAU,GACV,UAAU,GACV,KAAK,GACL,WAAW,GACX,OAAO,GACP,SAAS,GACT,QAAQ,GACR,MAAM,GACN,YAAY,GACZ,SAAS,GACT,YAAY,GACZ,SAAS,GACT,MAAM,GACN,WAAW,GACX,MAAM,GACN,MAAM,GACN,SAAS,GACT,sBAAsB,GACtB,QAAQ,GACR,QAAQ,GACR,UAAU,GACV,UAAU,GACV,UAAU,GACV,OAAO,GACP,YAAY,GACZ,YAAY,GACZ,SAAS,GACT,OAAO,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/ChunkCache.d.ts b/node_modules/@huggingface/hub/dist/src/utils/ChunkCache.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..adac54f9e1194f85f3b793c3140435a677679f35 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/ChunkCache.d.ts @@ -0,0 +1,21 @@ +export declare class ChunkCache { + index: number; + xorbIndices: Int32Array; + chunkIndices: Uint16Array; + map: Map; + hmacs: Set; + maxSize: number; + constructor(maxSize?: number); + addChunkToCache(hash: string, xorbIndex: number, chunkIndex: number, hmac: string | null): void; + getChunk(hash: string, + /** + * Set to null if you only want to check against locally created chunks, or the hash is already a hmac + */ + hmacFunction: ((hash: string, key: string) => string) | null): { + xorbIndex: number; + chunkIndex: number; + } | undefined; + updateChunkIndex(hash: string, chunkIndex: number): void; + removeChunkFromCache(hash: string): void; +} +//# sourceMappingURL=ChunkCache.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/ChunkCache.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/ChunkCache.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..37fdd6e22605a95b32df26a2123bc3ea1fabbff2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/ChunkCache.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"ChunkCache.d.ts","sourceRoot":"","sources":["../../../src/utils/ChunkCache.ts"],"names":[],"mappings":"AAIA,qBAAa,UAAU;IACtB,KAAK,SAAK;IAEV,WAAW,EAAE,UAAU,CAAC;IAExB,YAAY,EAAE,WAAW,CAAC;IAC1B,GAAG,sBAA6B;IAChC,KAAK,cAAqB;IAC1B,OAAO,EAAE,MAAM,CAAC;gBAEJ,OAAO,GAAE,MAA6B;IASlD,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI;IAkC/F,QAAQ,CACP,IAAI,EAAE,MAAM;IACZ;;OAEG;IACH,YAAY,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC,GAAG,IAAI,GAE1D;QACA,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;KAClB,GACD,SAAS;IAmBZ,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAQxD,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;CAGxC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/ChunkCache.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/ChunkCache.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4c53ed26eba0f9c94b5dc571a9535f6c5a69e4e7 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/ChunkCache.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=ChunkCache.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/ChunkCache.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/ChunkCache.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..99b179f37390cc20f054e7b66fba3dce58bc6ff9 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/ChunkCache.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"ChunkCache.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/ChunkCache.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/FileBlob.d.ts b/node_modules/@huggingface/hub/dist/src/utils/FileBlob.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..33567f7e559d2ce6b3f19e39c7094ca4e3f7ba9d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/FileBlob.d.ts @@ -0,0 +1,61 @@ +/** + * @internal + * + * A FileBlob is a replacement for the Blob class that allows to lazy read files + * in order to preserve memory. + * + * It is a drop-in replacement for the Blob class, so you can use it as a Blob. + * + * The main difference is the instantiation, which is done asynchronously using the `FileBlob.create` method. + * + * @example + * const fileBlob = await FileBlob.create("path/to/package.json"); + * + * await fetch("https://aschen.tech", { method: "POST", body: fileBlob }); + */ +export declare class FileBlob extends Blob { + /** + * Creates a new FileBlob on the provided file. + * + * @param path Path to the file to be lazy readed + */ + static create(path: string | URL): Promise; + private path; + private start; + private end; + private constructor(); + /** + * Returns the size of the blob. + */ + get size(): number; + /** + * Returns a new instance of FileBlob that is a slice of the current one. + * + * The slice is inclusive of the start and exclusive of the end. + * + * The slice method does not supports negative start/end. + * + * @param start beginning of the slice + * @param end end of the slice + */ + slice(start?: number, end?: number): FileBlob; + /** + * Read the part of the file delimited by the FileBlob and returns it as an ArrayBuffer. + */ + arrayBuffer(): Promise; + /** + * Read the part of the file delimited by the FileBlob and returns it as a string. + */ + text(): Promise; + /** + * Returns a stream around the part of the file delimited by the FileBlob. + */ + stream(): ReturnType; + /** + * We are opening and closing the file for each action to prevent file descriptor leaks. + * + * It is an intended choice of developer experience over performances. + */ + private execute; +} +//# sourceMappingURL=FileBlob.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/FileBlob.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/FileBlob.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..2a7d1312e2d3bbc84475efc6836b66934f97e43f --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/FileBlob.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"FileBlob.d.ts","sourceRoot":"","sources":["../../../src/utils/FileBlob.ts"],"names":[],"mappings":"AAMA;;;;;;;;;;;;;;GAcG;AACH,qBAAa,QAAS,SAAQ,IAAI;IACjC;;;;OAIG;WACU,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC;IAU1D,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,GAAG,CAAS;IAEpB,OAAO;IAQP;;OAEG;IACH,IAAa,IAAI,IAAI,MAAM,CAE1B;IAED;;;;;;;;;OASG;IACM,KAAK,CAAC,KAAK,SAAI,EAAE,GAAG,SAAY,GAAG,QAAQ;IAUpD;;OAEG;IACY,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC;IAMlD;;OAEG;IACY,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;IAMtC;;OAEG;IACM,MAAM,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAU7C;;;;OAIG;YACW,OAAO;CASrB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/FileBlob.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/FileBlob.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..3dc4b9bdaa2e5d1f61764724aeb956a9f1decb9e --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/FileBlob.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=FileBlob.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/FileBlob.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/FileBlob.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..fe05adadf8a8c2290c395959ddeb770a8bf20a1b --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/FileBlob.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"FileBlob.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/FileBlob.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/RangeList.d.ts b/node_modules/@huggingface/hub/dist/src/utils/RangeList.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..6643303da47626cc99319eabc91bd9fc35d1bff6 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/RangeList.d.ts @@ -0,0 +1,52 @@ +/** + * Code generated with this prompt by Cursor: + * + * I want to build a class to manage ranges + * + * I can add ranges to it with a start& an end (both integer, end > start). It should store those ranges efficiently. + * + * When several ranges overlap, eg [1, 100] and [30, 50], I want the class to split the range into non-overlapping ranges, and add a "ref counter" to the ranges. For example, [1, 30], [30, 50] * 2, [50, 100] + * + * I also want to be able to remove ranges, it will decrease the ref counter or remove the range altogether. I can only remove ranges at existing boundaries. For example, with the [1, 30], [30, 50] * 2, [50, 100] configuration + * + * - removing [1, 100] => the only range remaning is [30, 50] + * - removing [2, 50] => error, because "2' is not a boundary + * - removing [30, 50] => [1, 30], [30, 50], [50, 100] (do not "merge" the ranges back together) + * + * I want to be able to associate data to each range. And I want to be able to get the ranges inside boundaries. For example , with [1, 30], [30, 50] * 2, [50, 100] configuration + * + * - getting [30, 100] => I receive [30, 50] * 2, [50, 100], and I can get / modify the data associated to each range by accessing their data prop. Note the "*2" is just the ref counter, there is onlly one range object for the interval returned + * - getting [2, 50] => I get [30, 50] * 2 + * + * ---- + * + * Could optimize with binary search, but the ranges we want to handle are not that many. + */ +interface Range { + start: number; + end: number; + refCount: number; + data: T | null; +} +export declare class RangeList { + private ranges; + /** + * Add a range to the list. If it overlaps with existing ranges, + * it will split them and increment reference counts accordingly. + */ + add(start: number, end: number): void; + /** + * Remove a range from the list. The range must start and end at existing boundaries. + */ + remove(start: number, end: number): void; + /** + * Get all ranges within the specified boundaries. + */ + getRanges(start: number, end: number): Range[]; + /** + * Get all ranges in the list + */ + getAllRanges(): Range[]; +} +export {}; +//# sourceMappingURL=RangeList.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/RangeList.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/RangeList.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..1bb233291c606acfada9677da4ad6518c23425ba --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/RangeList.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"RangeList.d.ts","sourceRoot":"","sources":["../../../src/utils/RangeList.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,UAAU,KAAK,CAAC,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC;CACf;AAED,qBAAa,SAAS,CAAC,CAAC;IACvB,OAAO,CAAC,MAAM,CAAkB;IAEhC;;;OAGG;IACH,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAsFrC;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAkCxC;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE;IAQjD;;OAEG;IACH,YAAY,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE;CAG1B"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/RangeList.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/RangeList.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..a859f578163d9a221f250055706ca0c5bbeb6c12 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/RangeList.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=RangeList.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/RangeList.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/RangeList.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c0d70f8a61ca0678f80575c4651ad140101ba520 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/RangeList.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"RangeList.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/RangeList.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/SplicedBlob.d.ts b/node_modules/@huggingface/hub/dist/src/utils/SplicedBlob.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..8580c55a2e5707af0ab536261f4181df857ed8fd --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/SplicedBlob.d.ts @@ -0,0 +1,68 @@ +/** + * Represents a single splice operation + */ +interface SpliceOperation { + insert: Blob; + start: number; + end: number; +} +/** + * @internal + * + * A SplicedBlob is a Blob that represents the result of splicing one or more insert blobs + * into an original blob at specified positions, replacing content between start and end. + * + * It is a drop-in replacement for the Blob class, so you can use it as a Blob. + * The splicing is done virtually without copying data until accessed. + * + * @example + * const originalBlob = new Blob(["Hello, World!"]); + * const insertBlob = new Blob(["Beautiful "]); + * const splicedBlob = SplicedBlob.create(originalBlob, insertBlob, 7, 7); + * // Result represents: "Hello, Beautiful World!" + */ +export declare class SplicedBlob extends Blob { + originalBlob: Blob; + spliceOperations: SpliceOperation[]; + private constructor(); + static create(originalBlob: Blob, operations: SpliceOperation[]): SplicedBlob; + /** + * Returns the size of the spliced blob. + * Size = original size - total replaced size + total insert size + */ + get size(): number; + /** + * Returns the MIME type of the original blob. + */ + get type(): string; + /** + * Returns a new instance of SplicedBlob that is a slice of the current one. + * + * The slice is inclusive of the start and exclusive of the end. + * The slice method does not support negative start/end. + * + * @param start beginning of the slice + * @param end end of the slice + */ + slice(start?: number, end?: number): Blob; + get firstSpliceIndex(): number; + /** + * Read the spliced blob content and returns it as an ArrayBuffer. + */ + arrayBuffer(): Promise; + /** + * Read the spliced blob content and returns it as a string. + */ + text(): Promise; + /** + * Returns a stream around the spliced blob content. + */ + stream(): ReturnType; + /** + * Get all segments that make up the spliced blob. + * This includes original blob segments between splice operations and insert blobs. + */ + private get segments(); +} +export {}; +//# sourceMappingURL=SplicedBlob.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/SplicedBlob.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/SplicedBlob.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..26219e324401b837f58542d3c53a70713ee73131 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/SplicedBlob.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"SplicedBlob.d.ts","sourceRoot":"","sources":["../../../src/utils/SplicedBlob.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,UAAU,eAAe;IACxB,MAAM,EAAE,IAAI,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACZ;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,WAAY,SAAQ,IAAI;IAC7B,YAAY,EAAE,IAAI,CAAC;IACnB,gBAAgB,EAAE,eAAe,EAAE,CAAC;IAE3C,OAAO;IAOP,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,eAAe,EAAE,GAAG,WAAW;IAyB7E;;;OAGG;IACH,IAAa,IAAI,IAAI,MAAM,CAU1B;IAED;;OAEG;IACH,IAAa,IAAI,IAAI,MAAM,CAE1B;IAED;;;;;;;;OAQG;IACM,KAAK,CAAC,KAAK,SAAI,EAAE,GAAG,SAAY,GAAG,IAAI;IAmDhD,IAAI,gBAAgB,IAAI,MAAM,CAE7B;IAED;;OAEG;IACY,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC;IAiBlD;;OAEG;IACY,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;IAKtC;;OAEG;IACM,MAAM,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAgC7C;;;OAGG;IACH,OAAO,KAAK,QAAQ,GA4BnB;CACD"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/SplicedBlob.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/SplicedBlob.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..51358346368fe397cabb0c4f822edd282172d9e2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/SplicedBlob.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=SplicedBlob.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/SplicedBlob.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/SplicedBlob.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d856cb7a3e3040e0f2ed956043a406740dc5be0d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/SplicedBlob.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"SplicedBlob.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/SplicedBlob.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/WebBlob.d.ts b/node_modules/@huggingface/hub/dist/src/utils/WebBlob.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..abcb924cc8a4c80f97051db67eb5a31f91d4e9d9 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/WebBlob.d.ts @@ -0,0 +1,37 @@ +/** + * WebBlob is a Blob implementation for web resources that supports range requests. + */ +interface WebBlobCreateOptions { + /** + * @default 1_000_000 + * + * Objects below that size will immediately be fetched and put in RAM, rather + * than streamed ad-hoc + */ + cacheBelow?: number; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + accessToken: string | undefined; +} +export declare class WebBlob extends Blob { + static create(url: URL, opts?: WebBlobCreateOptions): Promise; + private url; + private start; + private end; + private contentType; + private full; + private fetch; + private accessToken; + constructor(url: URL, start: number, end: number, contentType: string, full: boolean, customFetch: typeof fetch, accessToken: string | undefined); + get size(): number; + get type(): string; + slice(start?: number, end?: number): WebBlob; + arrayBuffer(): Promise; + text(): Promise; + stream(): ReturnType; + private fetchRange; +} +export {}; +//# sourceMappingURL=WebBlob.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/WebBlob.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/WebBlob.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..363b437d39710af6d5ca5cca744716753df1bd17 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/WebBlob.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"WebBlob.d.ts","sourceRoot":"","sources":["../../../src/utils/WebBlob.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,UAAU,oBAAoB;IAC7B;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CAChC;AAED,qBAAa,OAAQ,SAAQ,IAAI;WACnB,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqDzE,OAAO,CAAC,GAAG,CAAM;IACjB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,IAAI,CAAU;IACtB,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,WAAW,CAAqB;gBAGvC,GAAG,EAAE,GAAG,EACR,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,OAAO,EACb,WAAW,EAAE,OAAO,KAAK,EACzB,WAAW,EAAE,MAAM,GAAG,SAAS;IAahC,IAAa,IAAI,IAAI,MAAM,CAE1B;IAED,IAAa,IAAI,IAAI,MAAM,CAE1B;IAEQ,KAAK,CAAC,KAAK,SAAI,EAAE,GAAG,SAAY,GAAG,OAAO;IAkBpC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC;IAMnC,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;IAM7B,MAAM,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAU7C,OAAO,CAAC,UAAU;CAkBlB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/WebBlob.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/WebBlob.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..aca418dff45130924e8f7cc7c44436d28336cbfb --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/WebBlob.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=WebBlob.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/WebBlob.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/WebBlob.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..91978e677249d646b7b1152ae4291b3aa8216e18 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/WebBlob.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"WebBlob.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/WebBlob.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/XetBlob.d.ts b/node_modules/@huggingface/hub/dist/src/utils/XetBlob.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..92a5392c6c9bff22b6c28863ed320a850d4f8244 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/XetBlob.d.ts @@ -0,0 +1,107 @@ +import type { CredentialsParams } from "../types/public"; +export interface XetReadToken { + accessToken: string; + casUrl: string; + exp: number; +} +type XetBlobCreateOptions = { + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + refreshUrl: string; + size: number; + listener?: (arg: { + event: "read"; + } | { + event: "progress"; + progress: { + read: number; + total: number; + }; + }) => void; + internalLogging?: boolean; + /** + * Pre-fetched read token to avoid the refresh URL roundtrip. + */ + readToken?: XetReadToken; +} & ({ + hash: string; + reconstructionUrl?: string; +} | { + hash?: string; + reconstructionUrl: string; +}) & Partial; +export interface ReconstructionInfo { + /** + * List of CAS blocks + */ + terms: Array<{ + /** Hash of the CAS block */ + hash: string; + /** Total uncompressed length of data of the chunks from range.start to range.end - 1 */ + unpacked_length: number; + /** Chunks. Eg start: 10, end: 100 = chunks 10-99 */ + range: { + start: number; + end: number; + }; + }>; + /** + * Dictionnary of CAS block hash => list of ranges in the block + url to fetch it + */ + fetch_info: Record>; + /** + * When doing a range request, the offset into the term's uncompressed data. Can be multiple chunks' worth of data. + */ + offset_into_first_range: number; +} +export declare enum XetChunkCompressionScheme { + None = 0, + LZ4 = 1, + ByteGroupingLZ4 = 2 +} +export declare const XET_CHUNK_HEADER_BYTES = 8; +/** + * XetBlob is a blob implementation that fetches data directly from the Xet storage + */ +export declare class XetBlob extends Blob { + #private; + fetch: typeof fetch; + accessToken?: string; + refreshUrl: string; + reconstructionUrl?: string; + hash?: string; + start: number; + end: number; + internalLogging: boolean; + reconstructionInfo: ReconstructionInfo | undefined; + listener: XetBlobCreateOptions["listener"]; + constructor(params: XetBlobCreateOptions); + get size(): number; + slice(start?: number, end?: number): XetBlob; + arrayBuffer(): Promise; + text(): Promise; + response(): Promise; + stream(): ReturnType; +} +export declare function bg4_regroup_bytes(bytes: Uint8Array): Uint8Array; +export declare function bg4_split_bytes(bytes: Uint8Array): Uint8Array; +export {}; +//# sourceMappingURL=XetBlob.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/XetBlob.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/XetBlob.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a1126a7f6c8fdb32effe60bb6e22ce628778708c --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/XetBlob.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"XetBlob.d.ts","sourceRoot":"","sources":["../../../src/utils/XetBlob.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AASzD,MAAM,WAAW,YAAY;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;CACZ;AAED,KAAK,oBAAoB,GAAG;IAC3B;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IAErB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,KAAK,EAAE,UAAU,CAAC;QAAC,QAAQ,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,KAAK,IAAI,CAAC;IAC/G,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;OAEG;IACH,SAAS,CAAC,EAAE,YAAY,CAAC;CACzB,GAAG,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,MAAM,CAAA;CAAE,CAAC,GAChG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAE5B,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,KAAK,EAAE,KAAK,CAAC;QACZ,4BAA4B;QAC5B,IAAI,EAAE,MAAM,CAAC;QACb,wFAAwF;QACxF,eAAe,EAAE,MAAM,CAAC;QACxB,oDAAoD;QACpD,KAAK,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,MAAM,CAAA;SAAE,CAAC;KACtC,CAAC,CAAC;IAEH;;OAEG;IACH,UAAU,EAAE,MAAM,CACjB,MAAM,EACN,KAAK,CAAC;QACL,GAAG,EAAE,MAAM,CAAC;QACZ,kBAAkB;QAClB,KAAK,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,MAAM,CAAA;SAAE,CAAC;QACtC;;;;WAIG;QACH,SAAS,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,MAAM,CAAA;SAAE,CAAC;KAC1C,CAAC,CACF,CAAC;IACF;;OAEG;IACH,uBAAuB,EAAE,MAAM,CAAC;CAChC;AAED,oBAAY,yBAAyB;IACpC,IAAI,IAAI;IACR,GAAG,IAAI;IACP,eAAe,IAAI;CACnB;AAeD,eAAO,MAAM,sBAAsB,IAAI,CAAC;AAExC;;GAEG;AACH,qBAAa,OAAQ,SAAQ,IAAI;;IAChC,KAAK,EAAE,OAAO,KAAK,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,SAAK;IACV,GAAG,SAAK;IACR,eAAe,UAAS;IACxB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,CAAC;IACnD,QAAQ,EAAE,oBAAoB,CAAC,UAAU,CAAC,CAAC;gBAE/B,MAAM,EAAE,oBAAoB;IAsBxC,IAAa,IAAI,IAAI,MAAM,CAE1B;IAsBQ,KAAK,CAAC,KAAK,SAAI,EAAE,GAAG,SAAY,GAAG,OAAO;IAgXpC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC;IAMnC,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;IAMhC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC;IAM1B,MAAM,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;CAS7C;AAoBD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,UAAU,GAAG,UAAU,CA2D/D;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,UAAU,GAAG,UAAU,CAgC7D"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/XetBlob.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/XetBlob.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..6b9e66998f8e4532f6b32273faee81dce58ec3c6 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/XetBlob.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=XetBlob.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/XetBlob.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/XetBlob.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..3ec609ddf408832fefea7342b1f6369794f9f90b --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/XetBlob.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"XetBlob.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/XetBlob.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/base64FromBytes.d.ts b/node_modules/@huggingface/hub/dist/src/utils/base64FromBytes.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4cbee918664971dbe986ef21375120c44924979a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/base64FromBytes.d.ts @@ -0,0 +1,2 @@ +export declare function base64FromBytes(arr: Uint8Array): string; +//# sourceMappingURL=base64FromBytes.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/base64FromBytes.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/base64FromBytes.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6f58a178c9da0fcdd6e1fd27282e2b1bc612489e --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/base64FromBytes.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"base64FromBytes.d.ts","sourceRoot":"","sources":["../../../src/utils/base64FromBytes.ts"],"names":[],"mappings":"AAAA,wBAAgB,eAAe,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAUvD"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/checkCredentials.d.ts b/node_modules/@huggingface/hub/dist/src/utils/checkCredentials.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4eed32a6feb4eb241402ce09531509c11e735259 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/checkCredentials.d.ts @@ -0,0 +1,4 @@ +import type { CredentialsParams } from "../types/public"; +export declare function checkAccessToken(accessToken: string): void; +export declare function checkCredentials(params: Partial): string | undefined; +//# sourceMappingURL=checkCredentials.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/checkCredentials.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/checkCredentials.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..3c683f78b508a6c78207a8e38e72688cfb92fba0 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/checkCredentials.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"checkCredentials.d.ts","sourceRoot":"","sources":["../../../src/utils/checkCredentials.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEzD,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAI1D;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,MAAM,GAAG,SAAS,CASvF"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/chunk.d.ts b/node_modules/@huggingface/hub/dist/src/utils/chunk.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..5ef12cdda9226612e4cf1c1492b0c662e997143e --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/chunk.d.ts @@ -0,0 +1,7 @@ +/** + * Chunk array into arrays of length at most `chunkSize` + * + * @param chunkSize must be greater than or equal to 1 + */ +export declare function chunk(arr: T, chunkSize: number): T[]; +//# sourceMappingURL=chunk.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/chunk.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/chunk.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..80708bab7af0c2e2462199377b7cfa54d050ae00 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/chunk.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../../src/utils/chunk.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,wBAAgB,KAAK,CAAC,CAAC,SAAS,OAAO,EAAE,GAAG,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,EAAE,CAiBlF"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/combineUint8Arrays.d.ts b/node_modules/@huggingface/hub/dist/src/utils/combineUint8Arrays.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..306fc41c5ed6fb4168dfdbf8024c577478de3a80 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/combineUint8Arrays.d.ts @@ -0,0 +1,2 @@ +export declare function combineUint8Arrays(a: Uint8Array, b: Uint8Array): Uint8Array; +//# sourceMappingURL=combineUint8Arrays.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/combineUint8Arrays.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/combineUint8Arrays.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..577accbb7d85426405fa205f315d9c32a80a001a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/combineUint8Arrays.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"combineUint8Arrays.d.ts","sourceRoot":"","sources":["../../../src/utils/combineUint8Arrays.ts"],"names":[],"mappings":"AAAA,wBAAgB,kBAAkB,CACjC,CAAC,EAAE,UAAU,CAAC,eAAe,CAAC,EAC9B,CAAC,EAAE,UAAU,CAAC,eAAe,CAAC,GAC5B,UAAU,CAAC,WAAW,CAAC,CAMzB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/combineUint8Arrays.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/combineUint8Arrays.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..94c4c0680fee2056c16262b6a28f7b3593d2c253 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/combineUint8Arrays.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=combineUint8Arrays.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/combineUint8Arrays.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/combineUint8Arrays.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7143c2d6c3a7aec245278fc8ec6fc41cca7f65d2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/combineUint8Arrays.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"combineUint8Arrays.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/combineUint8Arrays.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/createBlob.d.ts b/node_modules/@huggingface/hub/dist/src/utils/createBlob.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..24d0d70fce75368728806fb00ca7341ac19f0b56 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/createBlob.d.ts @@ -0,0 +1,15 @@ +/** + * This function allow to retrieve either a FileBlob or a WebBlob from a URL. + * + * From the backend: + * - support local files + * - support http resources with absolute URLs + * + * From the frontend: + * - support http resources with absolute or relative URLs + */ +export declare function createBlob(url: URL, opts?: { + fetch?: typeof fetch; + accessToken?: string; +}): Promise; +//# sourceMappingURL=createBlob.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/createBlob.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/createBlob.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c8f042d169b38c108cbae5e7fb1387ad2c069933 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/createBlob.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"createBlob.d.ts","sourceRoot":"","sources":["../../../src/utils/createBlob.ts"],"names":[],"mappings":"AAGA;;;;;;;;;GASG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB/G"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/createBlobs.d.ts b/node_modules/@huggingface/hub/dist/src/utils/createBlobs.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2e43bdfd6a18871abce8069b848d7c38cf7769e0 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/createBlobs.d.ts @@ -0,0 +1,20 @@ +/** + * This function allow to retrieve either a FileBlob or a WebBlob from a URL. + * + * From the backend: + * - support local files + * - support local folders + * - support http resources with absolute URLs + * + * From the frontend: + * - support http resources with absolute or relative URLs + */ +export declare function createBlobs(url: URL, destPath: string, opts?: { + fetch?: typeof fetch; + maxFolderDepth?: number; + accessToken?: string; +}): Promise>; +//# sourceMappingURL=createBlobs.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/createBlobs.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/createBlobs.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5e21137842d11d48799812977745f79925650e9f --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/createBlobs.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"createBlobs.d.ts","sourceRoot":"","sources":["../../../src/utils/createBlobs.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAChC,GAAG,EAAE,GAAG,EACR,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAC5E,OAAO,CAAC,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,IAAI,CAAA;CAAE,CAAC,CAAC,CAgC9C"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/createXorb.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/createXorb.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..207d9fa0ed1d7db1728a19c7feeef1101b1a97a0 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/createXorb.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=createXorb.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/createXorb.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/createXorb.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..72bdab5719c1245a868b7fc1c3f68a52fc4b04d6 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/createXorb.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"createXorb.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/createXorb.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/createXorbs.d.ts b/node_modules/@huggingface/hub/dist/src/utils/createXorbs.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..bef532b0d827821ec150d48cc577353f92ba9db2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/createXorbs.d.ts @@ -0,0 +1,73 @@ +import { ChunkCache } from "./ChunkCache"; +import { type XetWriteTokenParams } from "./xetWriteToken"; +import type { ShardData } from "./shardParser"; +interface XorbEvent { + event: "xorb"; + xorb: Uint8Array; + hash: string; + id: number; + chunks: Array<{ + hash: string; + length: number; + }>; + files: Array<{ + path: string; + progress: number; + lastSentProgress: number; + }>; +} +export declare class CurrentXorbInfo { + id: number; + offset: number; + chunks: Array<{ + hash: string; + length: number; + offset: number; + }>; + fileProcessedBytes: Record; + fileUploadedBytes: Record; + fileSize: Record; + data: Uint8Array; + immutableData: { + chunkIndex: number; + offset: number; + } | null; + constructor(); + event(computeXorbHash: (chunks: { + hash: string; + length: number; + }[]) => string): XorbEvent; +} +export declare function createXorbs(fileSources: AsyncGenerator<{ + content: Blob; + path: string; + sha256?: string; +}>, params: XetWriteTokenParams & { + yieldCallback?: (event: { + event: "fileProgress"; + path: string; + progress: number; + }) => void; +}): AsyncGenerator; +}, void, undefined>; +export declare function backtrackDedup(xorb: CurrentXorbInfo, computeHmac: (hash: string, key: string) => string, shardData: ShardData, chunkCache: ChunkCache, chunkMetadata: { + xorbId: number | string; + chunkIndex: number; + length: number; +}[], dedupedBytes: number): number; +export {}; +//# sourceMappingURL=createXorbs.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/createXorbs.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/createXorbs.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7dc00b78946573908f860922edd86925ea01afd2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/createXorbs.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"createXorbs.d.ts","sourceRoot":"","sources":["../../../src/utils/createXorbs.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAiB,KAAK,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAiE/C,UAAU,SAAS;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChD,KAAK,EAAE,KAAK,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;QACjB,gBAAgB,EAAE,MAAM,CAAC;KACzB,CAAC,CAAC;CACH;AAED,qBAAa,eAAe;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAEhE,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,IAAI,EAAE,UAAU,CAAC;IACjB,aAAa,EAAE;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;KACf,GAAG,IAAI,CAAC;;IAaT,KAAK,CAAC,eAAe,EAAE,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,KAAK,MAAM,GAAG,SAAS;CAsBzF;AAED,wBAAuB,WAAW,CACjC,WAAW,EAAE,cAAc,CAAC;IAAE,OAAO,EAAE,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,EAC7E,MAAM,EAAE,mBAAmB,GAAG;IAC7B,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,cAAc,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CAC3F,GACC,cAAc,CACd,SAAS,GACT;IACA,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4DAA4D;IAC5D,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,KAAK,CAAC;QACrB,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,sBAAsB;QACtB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;CACF,EACH,IAAI,EACJ,SAAS,CACT,CA6PA;AAED,wBAAgB,cAAc,CAC7B,IAAI,EAAE,eAAe,EACrB,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,MAAM,EAClD,SAAS,EAAE,SAAS,EACpB,UAAU,EAAE,UAAU,EACtB,aAAa,EAAE;IAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EAAE,EAChF,YAAY,EAAE,MAAM,GAClB,MAAM,CAmGR"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/eventToGenerator.d.ts b/node_modules/@huggingface/hub/dist/src/utils/eventToGenerator.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4023ceea374623c495a4b391ff20eb85eb98a847 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/eventToGenerator.d.ts @@ -0,0 +1,2 @@ +export declare function eventToGenerator(cb: (yieldCallback: (y: YieldType) => void, returnCallback: (r: ReturnType) => void, rejectCallack: (reason: unknown) => void) => unknown): AsyncGenerator; +//# sourceMappingURL=eventToGenerator.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/eventToGenerator.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/eventToGenerator.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c145f34e3a8f0ffe9d8e47880d3a2c81860bc222 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/eventToGenerator.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"eventToGenerator.d.ts","sourceRoot":"","sources":["../../../src/utils/eventToGenerator.ts"],"names":[],"mappings":"AAAA,wBAAuB,gBAAgB,CAAC,SAAS,EAAE,UAAU,EAC5D,EAAE,EAAE,CACH,aAAa,EAAE,CAAC,CAAC,EAAE,SAAS,KAAK,IAAI,EACrC,cAAc,EAAE,CAAC,CAAC,EAAE,UAAU,KAAK,IAAI,EACvC,aAAa,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,IAAI,KACpC,OAAO,GACV,cAAc,CAAC,SAAS,EAAE,UAAU,CAAC,CAyDvC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/eventToGenerator.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/eventToGenerator.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..48665837d283aabc4512d4dda06f00a1a148a77e --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/eventToGenerator.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=eventToGenerator.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/eventToGenerator.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/eventToGenerator.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..0bb4d9a5f7782ee078caf6c44b6c73fd0a92a7ba --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/eventToGenerator.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"eventToGenerator.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/eventToGenerator.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/formatBytes.d.ts b/node_modules/@huggingface/hub/dist/src/utils/formatBytes.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d8c2e4c39be522cd832078381768667424c5b279 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/formatBytes.d.ts @@ -0,0 +1,7 @@ +/** + * Format a byte count using SI units (multiples of 1000, e.g. `1.2 GB`). + * + * Negative or non-finite inputs are returned as `" B"` without unit conversion. + */ +export declare function formatBytes(bytes: number): string; +//# sourceMappingURL=formatBytes.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/formatBytes.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/formatBytes.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c727b2bb656cf3651b6497ea252ca0fca082e0ea --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/formatBytes.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"formatBytes.d.ts","sourceRoot":"","sources":["../../../src/utils/formatBytes.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAajD"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/formatBytes.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/formatBytes.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..9773eb49b7aac1d0391a19d09273418836da9771 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/formatBytes.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=formatBytes.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/formatBytes.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/formatBytes.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..b6de3bf103d9442c0d3a4900791aecc708bde62a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/formatBytes.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"formatBytes.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/formatBytes.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/hexFromBytes.d.ts b/node_modules/@huggingface/hub/dist/src/utils/hexFromBytes.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..c46b6770008f612d0c2bb7bf4c058f14d1b88d11 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/hexFromBytes.d.ts @@ -0,0 +1,2 @@ +export declare function hexFromBytes(arr: Uint8Array): string; +//# sourceMappingURL=hexFromBytes.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/hexFromBytes.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/hexFromBytes.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7648ff81154157295bec5f561ba4c77cdc769916 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/hexFromBytes.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"hexFromBytes.d.ts","sourceRoot":"","sources":["../../../src/utils/hexFromBytes.ts"],"names":[],"mappings":"AAAA,wBAAgB,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,CAUpD"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/insecureRandomString.d.ts b/node_modules/@huggingface/hub/dist/src/utils/insecureRandomString.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..44f3e0e242ecdd766bfb500cbeceda0b855d3f40 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/insecureRandomString.d.ts @@ -0,0 +1,2 @@ +export declare function insecureRandomString(): string; +//# sourceMappingURL=insecureRandomString.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/insecureRandomString.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/insecureRandomString.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..45728c06f25c99fcc0205f065452f69a2b0e5833 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/insecureRandomString.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"insecureRandomString.d.ts","sourceRoot":"","sources":["../../../src/utils/insecureRandomString.ts"],"names":[],"mappings":"AAAA,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/isBackend.d.ts b/node_modules/@huggingface/hub/dist/src/utils/isBackend.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..069d8727ffbc8bb2c70bae99fe1413db4bf0e7a2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/isBackend.d.ts @@ -0,0 +1,2 @@ +export declare const isBackend: boolean; +//# sourceMappingURL=isBackend.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/isBackend.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/isBackend.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..56291c33b8f5490fd2cc3d194f43bd791f599280 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/isBackend.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"isBackend.d.ts","sourceRoot":"","sources":["../../../src/utils/isBackend.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,SAAS,SAA6B,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/isFrontend.d.ts b/node_modules/@huggingface/hub/dist/src/utils/isFrontend.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..f48051727350420cea4a870a0df61fd629c8f47a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/isFrontend.d.ts @@ -0,0 +1,2 @@ +export declare const isFrontend: boolean; +//# sourceMappingURL=isFrontend.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/isFrontend.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/isFrontend.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..eb97bede136119de84b957123d53836f5de4ea2f --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/isFrontend.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"isFrontend.d.ts","sourceRoot":"","sources":["../../../src/utils/isFrontend.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,UAAU,SAAa,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/mergeAsyncGenerators.d.ts b/node_modules/@huggingface/hub/dist/src/utils/mergeAsyncGenerators.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4699ddbde6aca8408916607eeee718facdbb31d0 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/mergeAsyncGenerators.d.ts @@ -0,0 +1,5 @@ +/** + * Merge outputs of multiple async generators. + */ +export declare function mergeAsyncGenerators(generators: AsyncGenerator[]): AsyncGenerator; +//# sourceMappingURL=mergeAsyncGenerators.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/mergeAsyncGenerators.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/mergeAsyncGenerators.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..4a387c0b5c473da5d40d7b3c8f77436332a64061 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/mergeAsyncGenerators.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"mergeAsyncGenerators.d.ts","sourceRoot":"","sources":["../../../src/utils/mergeAsyncGenerators.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,wBAAuB,oBAAoB,CAAC,CAAC,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC,EAAE,GAAG,cAAc,CAAC,CAAC,CAAC,CA8BjG"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/mergeAsyncGenerators.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/mergeAsyncGenerators.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..9eabdfde96de2c9954f01e54588899a69576423c --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/mergeAsyncGenerators.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=mergeAsyncGenerators.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/mergeAsyncGenerators.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/mergeAsyncGenerators.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..1a80e950ca139f98c499df308eff46a656c40da2 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/mergeAsyncGenerators.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"mergeAsyncGenerators.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/mergeAsyncGenerators.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/normalizeInferenceProviderMapping.d.ts b/node_modules/@huggingface/hub/dist/src/utils/normalizeInferenceProviderMapping.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..c74be59900e109bcc3b1c0eed77eb9b435dd077a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/normalizeInferenceProviderMapping.d.ts @@ -0,0 +1,14 @@ +import type { WidgetType } from "@huggingface/tasks"; +import type { ApiModelInferenceProviderMappingEntry } from "../types/api/api-model"; +/** + * Normalize inferenceProviderMapping to always return an array format. + * + * Little hack to simplify Inference Providers logic and make it backward and forward compatible. + * Right now, API returns a dict on model-info and a list on list-models. Let's harmonize to list. + */ +export declare function normalizeInferenceProviderMapping(hfModelId: string, inferenceProviderMapping?: ApiModelInferenceProviderMappingEntry[] | Record): ApiModelInferenceProviderMappingEntry[]; +//# sourceMappingURL=normalizeInferenceProviderMapping.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/normalizeInferenceProviderMapping.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/normalizeInferenceProviderMapping.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..0185b247025c3e6ed8b8563af1a939846cc5d891 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/normalizeInferenceProviderMapping.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"normalizeInferenceProviderMapping.d.ts","sourceRoot":"","sources":["../../../src/utils/normalizeInferenceProviderMapping.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,wBAAwB,CAAC;AAEpF;;;;;GAKG;AACH,wBAAgB,iCAAiC,CAChD,SAAS,EAAE,MAAM,EACjB,wBAAwB,CAAC,EACtB,qCAAqC,EAAE,GACvC,MAAM,CAAC,MAAM,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAE,CAAC,GACrF,qCAAqC,EAAE,CAqBzC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/omit.d.ts b/node_modules/@huggingface/hub/dist/src/utils/omit.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..131e4d9f9406b8ebc3190b712141be127d7b2445 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/omit.d.ts @@ -0,0 +1,5 @@ +/** + * Return copy of object, omitting blacklisted array of props + */ +export declare function omit, K extends keyof T>(o: T, props: K[] | K): Pick>; +//# sourceMappingURL=omit.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/omit.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/omit.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f921f39722761d48dd1135985e7bd6ca20a232c3 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/omit.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"omit.d.ts","sourceRoot":"","sources":["../../../src/utils/omit.ts"],"names":[],"mappings":"AAGA;;GAEG;AACH,wBAAgB,IAAI,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,EACxE,CAAC,EAAE,CAAC,EACJ,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,GACZ,IAAI,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAI9B"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/parseLinkHeader.d.ts b/node_modules/@huggingface/hub/dist/src/utils/parseLinkHeader.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..63c5d9becba662bd7ba37e9beb205a61ee3d995f --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/parseLinkHeader.d.ts @@ -0,0 +1,5 @@ +/** + * Parse Link HTTP header, eg `; rel="next"` + */ +export declare function parseLinkHeader(header: string): Record; +//# sourceMappingURL=parseLinkHeader.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/parseLinkHeader.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/parseLinkHeader.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..32793a7ff82c66134a054e75eca37a38b9bdc694 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/parseLinkHeader.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"parseLinkHeader.d.ts","sourceRoot":"","sources":["../../../src/utils/parseLinkHeader.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAItE"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/pick.d.ts b/node_modules/@huggingface/hub/dist/src/utils/pick.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..263f5eea8d53d5aec52cae6143fc9d1f27fbba1d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/pick.d.ts @@ -0,0 +1,5 @@ +/** + * Return copy of object, only keeping whitelisted properties. + */ +export declare function pick(o: T, props: K[] | ReadonlyArray): Pick; +//# sourceMappingURL=pick.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/pick.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/pick.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..0ea81b4da845f65ccce0ceb28d60fff5e33ba7bf --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/pick.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"pick.d.ts","sourceRoot":"","sources":["../../../src/utils/pick.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,wBAAgB,IAAI,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAS1F"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/promisesQueue.d.ts b/node_modules/@huggingface/hub/dist/src/utils/promisesQueue.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..9f6a9d15a5989ed0f03f81737167ecaae75102bc --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/promisesQueue.d.ts @@ -0,0 +1,7 @@ +/** + * Execute queue of promises. + * + * Inspired by github.com/rxaviers/async-pool + */ +export declare function promisesQueue(factories: (() => Promise)[], concurrency: number): Promise; +//# sourceMappingURL=promisesQueue.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/promisesQueue.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/promisesQueue.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..69d485dfe767ec32ba609cf24babafa2a7205280 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/promisesQueue.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"promisesQueue.d.ts","sourceRoot":"","sources":["../../../src/utils/promisesQueue.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAiBzG"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/promisesQueue.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/promisesQueue.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..34dcd72d62f718a8a7343dcdbcf05cf8073a8941 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/promisesQueue.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=promisesQueue.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/promisesQueue.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/promisesQueue.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..cfe1a6213806b3ed46a7601a8c39b14c6693aa9e --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/promisesQueue.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"promisesQueue.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/promisesQueue.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/promisesQueueStreaming.d.ts b/node_modules/@huggingface/hub/dist/src/utils/promisesQueueStreaming.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..f2a7e3d6b659b85c33452a7bc55f7a4bd835bb5a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/promisesQueueStreaming.d.ts @@ -0,0 +1,11 @@ +/** + * Execute queue of promises in a streaming fashion. + * + * Optimized for streaming: + * - Expects an iterable as input + * - Does not return a list of all results + * + * Inspired by github.com/rxaviers/async-pool + */ +export declare function promisesQueueStreaming(factories: AsyncIterable<() => Promise> | Iterable<() => Promise>, concurrency: number): Promise; +//# sourceMappingURL=promisesQueueStreaming.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/promisesQueueStreaming.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/promisesQueueStreaming.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..122e476a2ec906ed1fd2c815d1d2ab2ed95e383e --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/promisesQueueStreaming.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"promisesQueueStreaming.d.ts","sourceRoot":"","sources":["../../../src/utils/promisesQueueStreaming.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAAC,CAAC,EAC7C,SAAS,EAAE,aAAa,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,EACvE,WAAW,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC,CAYf"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/range.d.ts b/node_modules/@huggingface/hub/dist/src/utils/range.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..cb4f113f7e4dedaa552f82ef826c650bee0932ca --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/range.d.ts @@ -0,0 +1,6 @@ +/** + * One param: create list of integers from 0 (inclusive) to n (exclusive) + * Two params: create list of integers from a (inclusive) to b (exclusive) + */ +export declare function range(n: number, b?: number): number[]; +//# sourceMappingURL=range.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/range.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/range.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..ae3ed0568677ff5deb9da878250e3d8961513ccb --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/range.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"range.d.ts","sourceRoot":"","sources":["../../../src/utils/range.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAgB,KAAK,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAQrD"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/sha256-node.d.ts b/node_modules/@huggingface/hub/dist/src/utils/sha256-node.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..64c666ae2771e2f16450eb97d7553ce8aaff5d02 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/sha256-node.d.ts @@ -0,0 +1,4 @@ +export declare function sha256Node(buffer: ArrayBuffer | Blob, opts?: { + abortSignal?: AbortSignal; +}): AsyncGenerator; +//# sourceMappingURL=sha256-node.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/sha256-node.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/sha256-node.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..208ca3e88b6ce69fe378aadc4e401db1b468ddc6 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/sha256-node.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"sha256-node.d.ts","sourceRoot":"","sources":["../../../src/utils/sha256-node.ts"],"names":[],"mappings":"AAIA,wBAAuB,UAAU,CAChC,MAAM,EAAE,WAAW,GAAG,IAAI,EAC1B,IAAI,CAAC,EAAE;IACN,WAAW,CAAC,EAAE,WAAW,CAAC;CAC1B,GACC,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAgBhC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/sha256.d.ts b/node_modules/@huggingface/hub/dist/src/utils/sha256.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ff8e7ac961d5db66f81781189ecb0df563ce341f --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/sha256.d.ts @@ -0,0 +1,12 @@ +/** + * @returns hex-encoded sha + * @yields progress (0-1) + */ +export declare function sha256(buffer: Blob, opts?: { + useWebWorker?: boolean | { + minSize?: number; + poolSize?: number; + }; + abortSignal?: AbortSignal; +}): AsyncGenerator; +//# sourceMappingURL=sha256.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/sha256.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/sha256.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c9fab8e218016e2c3c35312b7f56a03bb7f30ca7 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/sha256.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"sha256.d.ts","sourceRoot":"","sources":["../../../src/utils/sha256.ts"],"names":[],"mappings":"AAmEA;;;GAGG;AACH,wBAAuB,MAAM,CAC5B,MAAM,EAAE,IAAI,EACZ,IAAI,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,OAAO,GAAG;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAAC,WAAW,CAAC,EAAE,WAAW,CAAA;CAAE,GACpG,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CA8HhC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/sha256.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/sha256.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..8f8113753430e5bac2b71d036748c666904ce676 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/sha256.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=sha256.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/sha256.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/sha256.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5b7d9a4c4c2d5167f96b864fe6fd3b3e30205223 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/sha256.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"sha256.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/sha256.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/shardParser.d.ts b/node_modules/@huggingface/hub/dist/src/utils/shardParser.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..a25ad37be5bfe0afa6efd66a461234a9f58ad9f1 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/shardParser.d.ts @@ -0,0 +1,13 @@ +export interface ShardData { + hmacKey: string; + xorbs: Array<{ + hash: string; + chunks: Array<{ + hash: string; + startOffset: number; + unpackedLength: number; + }>; + }>; +} +export declare function parseShardData(shardBlob: Blob): Promise; +//# sourceMappingURL=shardParser.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/shardParser.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/shardParser.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..472cf769dad7fb8e92ff9a940b475f3dcaad0a8e --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/shardParser.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"shardParser.d.ts","sourceRoot":"","sources":["../../../src/utils/shardParser.ts"],"names":[],"mappings":"AAsBA,MAAM,WAAW,SAAS;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,KAAK,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,KAAK,CAAC;YACb,IAAI,EAAE,MAAM,CAAC;YACb,WAAW,EAAE,MAAM,CAAC;YACpB,cAAc,EAAE,MAAM,CAAC;SACvB,CAAC,CAAC;KACH,CAAC,CAAC;CACH;AAED,wBAAsB,cAAc,CAAC,SAAS,EAAE,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAkHxE"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/shardParser.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/shardParser.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..cef9dc426a38e6e434abe3478ad726a0eea696ea --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/shardParser.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=shardParser.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/shardParser.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/shardParser.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..0ecbe9928efae0f83114d2faff85cb8ca4a2daeb --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/shardParser.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"shardParser.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/shardParser.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/splitAsyncGenerator.d.ts b/node_modules/@huggingface/hub/dist/src/utils/splitAsyncGenerator.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fbec9588e8b793a345be6d2338bdb540e3c73474 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/splitAsyncGenerator.d.ts @@ -0,0 +1,5 @@ +/** + * Split an async generator into multiple async generators, all drawing from the same source. + */ +export declare function splitAsyncGenerator(source: AsyncGenerator, n: number): Array>; +//# sourceMappingURL=splitAsyncGenerator.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/splitAsyncGenerator.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/splitAsyncGenerator.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5bd217c94a64b2dfeccc9de5d40a2f400cc48a69 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/splitAsyncGenerator.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"splitAsyncGenerator.d.ts","sourceRoot":"","sources":["../../../src/utils/splitAsyncGenerator.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAuCrG"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/sub-paths.d.ts b/node_modules/@huggingface/hub/dist/src/utils/sub-paths.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..67a5958cb6a4f67d3bdc8ae942e42bd7580a54ec --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/sub-paths.d.ts @@ -0,0 +1,8 @@ +/** + * Recursively retrieves all sub-paths of a given directory up to a specified depth. + */ +export declare function subPaths(path: URL, maxDepth?: number): Promise>; +//# sourceMappingURL=sub-paths.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/sub-paths.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/sub-paths.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6bd4d8d0c30e7c6ccb76659278e1ec9504a2463d --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/sub-paths.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"sub-paths.d.ts","sourceRoot":"","sources":["../../../src/utils/sub-paths.ts"],"names":[],"mappings":"AAGA;;GAEG;AACH,wBAAsB,QAAQ,CAC7B,IAAI,EAAE,GAAG,EACT,QAAQ,SAAK,GACX,OAAO,CACT,KAAK,CAAC;IACL,IAAI,EAAE,GAAG,CAAC;IACV,YAAY,EAAE,MAAM,CAAC;CACrB,CAAC,CACF,CAuBA"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/sub-paths.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/sub-paths.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..56526ea3edaf4e9c68b422b7e05f7ddc05b5496b --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/sub-paths.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=sub-paths.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/sub-paths.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/sub-paths.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..96337009bb1ed3b6d01c12f0a18b6e11bff10b4f --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/sub-paths.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"sub-paths.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/sub-paths.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/sum.d.ts b/node_modules/@huggingface/hub/dist/src/utils/sum.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2938657fe7ad469a00bb6c9746352594b4e9d2fe --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/sum.d.ts @@ -0,0 +1,5 @@ +/** + * Sum of elements in array + */ +export declare function sum(arr: number[]): number; +//# sourceMappingURL=sum.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/sum.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/sum.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7752efe0d45ab8028279894b920d0a7a3149d1bf --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/sum.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"sum.d.ts","sourceRoot":"","sources":["../../../src/utils/sum.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,wBAAgB,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,CAEzC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/symlink.d.ts b/node_modules/@huggingface/hub/dist/src/utils/symlink.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ead0f37bf2ff381400dd6ba2ee13db69fa8a2817 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/symlink.d.ts @@ -0,0 +1,38 @@ +/** + * Heavily inspired by https://github.com/huggingface/huggingface_hub/blob/fcfd14361bd03f23f82efced1aa65a7cbfa4b922/src/huggingface_hub/file_download.py#L517 + */ +/** + * Create a symbolic link named dst pointing to src. + * + * By default, it will try to create a symlink using a relative path. Relative paths have 2 advantages: + * - If the cache_folder is moved (example: back-up on a shared drive), relative paths within the cache folder will + * not break. + * - Relative paths seems to be better handled on Windows. Issue was reported 3 times in less than a week when + * changing from relative to absolute paths. See https://github.com/huggingface/huggingface_hub/issues/1398, + * https://github.com/huggingface/diffusers/issues/2729 and https://github.com/huggingface/transformers/pull/22228. + * NOTE: The issue with absolute paths doesn't happen on admin mode. + * When creating a symlink from the cache to a local folder, it is possible that a relative path cannot be created. + * This happens when paths are not on the same volume. In that case, we use absolute paths. + * + * The result layout looks something like + * └── [ 128] snapshots + * ├── [ 128] 2439f60ef33a0d46d85da5001d52aeda5b00ce9f + * │ ├── [ 52] README.md -> ../../../blobs/d7edf6bd2a681fb0175f7735299831ee1b22b812 + * │ └── [ 76] pytorch_model.bin -> ../../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd + * + * If symlinks cannot be created on this platform (most likely to be Windows), the workaround is to avoid symlinks by + * having the actual file in `dst`. If it is a new file (`new_blob=True`), we move it to `dst`. If it is not a new file + * (`new_blob=False`), we don't know if the blob file is already referenced elsewhere. To avoid breaking existing + * cache, the file is duplicated on the disk. + */ +export declare function createSymlink(params: { + /** + * The path to the symlink. + */ + finalPath: string; + /** + * The path the symlink should point to. + */ + sourcePath: string; +}): Promise; +//# sourceMappingURL=symlink.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/symlink.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/symlink.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..198f26130e63ea4d89a084f208672d1a1a16884b --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/symlink.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"symlink.d.ts","sourceRoot":"","sources":["../../../src/utils/symlink.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,aAAa,CAAC,MAAM,EAAE;IAC3C;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC,IAAI,CAAC,CAgBhB"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/symlink.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/symlink.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..84af2aadfaf2cdecc028476e26cfdaf9372f47a6 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/symlink.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=symlink.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/symlink.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/symlink.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..75ef4b18596ce4e5f1bdaa9b2550ea8140f481a1 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/symlink.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"symlink.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/symlink.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/toRepoId.d.ts b/node_modules/@huggingface/hub/dist/src/utils/toRepoId.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..f633788be346a4e3954687933a336dc3a6f58cec --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/toRepoId.d.ts @@ -0,0 +1,3 @@ +import type { RepoDesignation, RepoId } from "../types/public"; +export declare function toRepoId(repo: RepoDesignation): RepoId; +//# sourceMappingURL=toRepoId.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/toRepoId.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/toRepoId.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..489ceb6043d317596011b30529a40a37d0b6239a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/toRepoId.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"toRepoId.d.ts","sourceRoot":"","sources":["../../../src/utils/toRepoId.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAE/D,wBAAgB,QAAQ,CAAC,IAAI,EAAE,eAAe,GAAG,MAAM,CAiFtD"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/typedEntries.d.ts b/node_modules/@huggingface/hub/dist/src/utils/typedEntries.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..a6f0116ecba6e1afd76c415e5c2bd0826754776c --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/typedEntries.d.ts @@ -0,0 +1,5 @@ +import type { Entries } from "../vendor/type-fest/entries"; +export declare function typedEntries>(obj: T): Entries; +//# sourceMappingURL=typedEntries.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/typedEntries.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/typedEntries.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d84b2b6c5d05414c37d268af991b7762841afea8 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/typedEntries.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"typedEntries.d.ts","sourceRoot":"","sources":["../../../src/utils/typedEntries.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAE3D,wBAAgB,YAAY,CAAC,CAAC,SAAS;IAAE,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;CAAE,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAE9G"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/typedInclude.d.ts b/node_modules/@huggingface/hub/dist/src/utils/typedInclude.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..18fcc761228d7347ac431d408cd967cdb1f60c47 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/typedInclude.d.ts @@ -0,0 +1,2 @@ +export declare function typedInclude(arr: readonly T[], v: V): v is T; +//# sourceMappingURL=typedInclude.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/typedInclude.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/typedInclude.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..4512a06ceb2dca2c2c3207dc352faef3e3099512 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/typedInclude.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"typedInclude.d.ts","sourceRoot":"","sources":["../../../src/utils/typedInclude.ts"],"names":[],"mappings":"AAAA,wBAAgB,YAAY,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAE5E"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/uploadShards.d.ts b/node_modules/@huggingface/hub/dist/src/utils/uploadShards.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4d2ab81aecc77c89030728efa3caa1869bdb9e36 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/uploadShards.d.ts @@ -0,0 +1,45 @@ +import type { RepoId } from "../types/public"; +export declare const SHARD_HEADER_VERSION = 2n; +export declare const SHARD_FOOTER_VERSION = 1n; +export declare const SHARD_MAGIC_TAG: Uint8Array; +export interface XetTokenParams { + sessionId?: string; + casUrl?: string; + accessToken?: string; + expiresAt?: Date; + refreshWriteTokenUrl: string; +} +interface UploadShardsParams { + accessToken: string | undefined; + hubUrl: string; + xetParams: XetTokenParams; + fetch?: typeof fetch; + repo: RepoId; + rev: string; + isPullRequest?: boolean; + yieldCallback?: (event: { + event: "fileProgress"; + path: string; + progress: number; + }) => void; +} +/** + * Outputs the file sha256 after their xorbs/shards have been uploaded. + */ +export declare function uploadShards(source: AsyncGenerator<{ + content: Blob; + path: string; + sha256?: string; +}>, params: UploadShardsParams): AsyncGenerator<{ + event: "file"; + path: string; + xetHash: string; + sha256: string | undefined; + dedupRatio: number; +} | { + event: "fileProgress"; + path: string; + progress: number; +}>; +export {}; +//# sourceMappingURL=uploadShards.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/uploadShards.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/uploadShards.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5dd1a2f3c671197a6c85e30670e16278d99ed232 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/uploadShards.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"uploadShards.d.ts","sourceRoot":"","sources":["../../../src/utils/uploadShards.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAW9C,eAAO,MAAM,oBAAoB,KAAK,CAAC;AACvC,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAKvC,eAAO,MAAM,eAAe,yBAiC1B,CAAC;AAEH,MAAM,WAAW,cAAc;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,IAAI,CAAC;IACjB,oBAAoB,EAAE,MAAM,CAAC;CAC7B;AAED,UAAU,kBAAkB;IAC3B,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,cAAc,CAAC;IAC1B,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,cAAc,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CAC3F;AAED;;GAEG;AACH,wBAAuB,YAAY,CAClC,MAAM,EAAE,cAAc,CAAC;IAAE,OAAO,EAAE,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,EACxE,MAAM,EAAE,kBAAkB,GACxB,cAAc,CACd;IACA,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;CAClB,GACD;IAAE,KAAK,EAAE,cAAc,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAC3D,CA+RA"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/uploadShards.spec.d.ts b/node_modules/@huggingface/hub/dist/src/utils/uploadShards.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..08f7c496f9766ed114a3ec68b6ce3108e9ac233b --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/uploadShards.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=uploadShards.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/uploadShards.spec.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/uploadShards.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..650a78f154f31f0a7a681f0ca248b025467a04e0 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/uploadShards.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"uploadShards.spec.d.ts","sourceRoot":"","sources":["../../../src/utils/uploadShards.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/xetWriteToken.d.ts b/node_modules/@huggingface/hub/dist/src/utils/xetWriteToken.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..0d91b92d45f85d8e96a5281862bd53049fafa928 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/xetWriteToken.d.ts @@ -0,0 +1,11 @@ +import type { XetTokenParams } from "./uploadShards"; +export interface XetWriteTokenParams { + accessToken: string | undefined; + fetch?: typeof fetch; + xetParams: XetTokenParams; +} +export declare function xetWriteToken(params: XetWriteTokenParams): Promise<{ + accessToken: string; + casUrl: string; +}>; +//# sourceMappingURL=xetWriteToken.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/utils/xetWriteToken.d.ts.map b/node_modules/@huggingface/hub/dist/src/utils/xetWriteToken.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c65476dc0e46629814ec3b88fb2bad4d24196a04 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/utils/xetWriteToken.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"xetWriteToken.d.ts","sourceRoot":"","sources":["../../../src/utils/xetWriteToken.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAErD,MAAM,WAAW,mBAAmB;IACnC,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,SAAS,EAAE,cAAc,CAAC;CAC1B;AAkBD,wBAAsB,aAAa,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAwEjH"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/hash-wasm/sha256-wrapper.d.ts b/node_modules/@huggingface/hub/dist/src/vendor/hash-wasm/sha256-wrapper.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..5abd70c5ba17fbeaa88ceff3ed5d65d737e425cd --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/hash-wasm/sha256-wrapper.d.ts @@ -0,0 +1,7 @@ +export declare function createSHA256(isInsideWorker?: boolean): Promise<{ + init(): void; + update(data: Uint8Array): void; + digest(method: "hex"): string; +}>; +export declare function createSHA256WorkerCode(): string; +//# sourceMappingURL=sha256-wrapper.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/hash-wasm/sha256-wrapper.d.ts.map b/node_modules/@huggingface/hub/dist/src/vendor/hash-wasm/sha256-wrapper.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..8f093a79f4f5eae87c5881fe0ae4387f9cfa81db --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/hash-wasm/sha256-wrapper.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"sha256-wrapper.d.ts","sourceRoot":"","sources":["../../../../src/vendor/hash-wasm/sha256-wrapper.ts"],"names":[],"mappings":"AAEA,wBAAsB,YAAY,CAAC,cAAc,UAAQ,GAAG,OAAO,CAAC;IACnE,IAAI,IAAI,IAAI,CAAC;IACb,MAAM,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI,CAAC;IAC/B,MAAM,CAAC,MAAM,EAAE,KAAK,GAAG,MAAM,CAAC;CAC9B,CAAC,CA8BD;AAED,wBAAgB,sBAAsB,IAAI,MAAM,CAuB/C"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/lz4js/index.d.ts b/node_modules/@huggingface/hub/dist/src/vendor/lz4js/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d7234eb842a5738c47a26b1ebe0d9a92b6565e38 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/lz4js/index.d.ts @@ -0,0 +1,9 @@ +export declare function compressBound(n: number): number; +export declare function decompressBound(src: Uint8Array): number; +export declare function decompressBlock(src: Uint8Array, dst: Uint8Array, sIndex: number, sLength: number, dIndex: number): number; +export declare function compressBlock(src: Uint8Array, dst: Uint8Array, sIndex: number, sLength: number, hashTable: Uint32Array | number[]): number; +export declare function decompressFrame(src: Uint8Array, dst: Uint8Array): number; +export declare function compressFrame(src: Uint8Array, dst: Uint8Array): number; +export declare function decompress(src: Uint8Array, maxSize: number): Uint8Array; +export declare function compress(src: Uint8Array, maxSize?: number): Uint8Array; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/lz4js/index.d.ts.map b/node_modules/@huggingface/hub/dist/src/vendor/lz4js/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..b62f63cbbd71b0db8af0dd62c144d4dadce44b90 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/lz4js/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/vendor/lz4js/index.ts"],"names":[],"mappings":"AA8FA,wBAAgB,aAAa,CAAC,CAAC,EAAE,MAAM,UAEtC;AAGD,wBAAgB,eAAe,CAAC,GAAG,EAAE,UAAU,UA8D9C;AAGD,wBAAgB,eAAe,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,UAuEhH;AAGD,wBAAgB,aAAa,CAC5B,GAAG,EAAE,UAAU,EACf,GAAG,EAAE,UAAU,EACf,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,WAAW,GAAG,MAAM,EAAE,UAgHjC;AAGD,wBAAgB,eAAe,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,UA6E/D;AAGD,wBAAgB,aAAa,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,UA2D7D;AAKD,wBAAgB,UAAU,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,2BAc1D;AAKD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,MAAM,2BAezD"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/lz4js/util.d.ts b/node_modules/@huggingface/hub/dist/src/vendor/lz4js/util.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..6cb832d0cc53fbd05a1c374d3ab487b55cf0075f --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/lz4js/util.d.ts @@ -0,0 +1,6 @@ +export declare function hashU32(a: number): number; +export declare function readU64(b: Uint8Array, n: number): number; +export declare function readU32(b: Uint8Array, n: number): number; +export declare function writeU32(b: Uint8Array, n: number, x: number): void; +export declare function imul(a: number, b: number): number; +//# sourceMappingURL=util.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/lz4js/util.d.ts.map b/node_modules/@huggingface/hub/dist/src/vendor/lz4js/util.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..4881104c4e19850b7c8a3d4275b12f69b17006c5 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/lz4js/util.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../../../src/vendor/lz4js/util.ts"],"names":[],"mappings":"AAEA,wBAAgB,OAAO,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAQzC;AAGD,wBAAgB,OAAO,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAWxD;AAGD,wBAAgB,OAAO,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAOxD;AAGD,wBAAgB,QAAQ,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,IAAI,CAKlE;AAID,wBAAgB,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAOjD"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/lz4js/xxh32.d.ts b/node_modules/@huggingface/hub/dist/src/vendor/lz4js/xxh32.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..a99c38224ea73de8171588fc034d3cdaf4505cd8 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/lz4js/xxh32.d.ts @@ -0,0 +1,4 @@ +declare function xxh32(seed: number, src: Uint8Array, index: number, len: number): number; +export declare const hash: typeof xxh32; +export {}; +//# sourceMappingURL=xxh32.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/lz4js/xxh32.d.ts.map b/node_modules/@huggingface/hub/dist/src/vendor/lz4js/xxh32.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c8f48fd66fd4709107d7ffb9f09b38150a8eea3a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/lz4js/xxh32.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"xxh32.d.ts","sourceRoot":"","sources":["../../../../src/vendor/lz4js/xxh32.ts"],"names":[],"mappings":"AA0DA,iBAAS,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAmChF;AAED,eAAO,MAAM,IAAI,cAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/type-fest/basic.d.ts b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/basic.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..563bf416b94e4e2537bf9d16cca53b16e75c2ffc --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/basic.d.ts @@ -0,0 +1,33 @@ +/** +Matches a JSON object. + +This type can be useful to enforce some input to be JSON-compatible or as a super-type to be extended from. Don't use this as a direct return type as the user would have to double-cast it: `jsonObject as unknown as CustomResponse`. Instead, you could extend your CustomResponse type from it to ensure your type only uses JSON-compatible types: `interface CustomResponse extends JsonObject { … }`. + +@category JSON +*/ +export type JsonObject = { + [Key in string]: JsonValue; +} & { + [Key in string]?: JsonValue | undefined; +}; +/** +Matches a JSON array. + +@category JSON +*/ +export type JsonArray = JsonValue[] | readonly JsonValue[]; +/** +Matches any valid JSON primitive value. + +@category JSON +*/ +export type JsonPrimitive = string | number | boolean | null; +/** +Matches any valid JSON value. + +@see `Jsonify` if you need to transform a type to one that is assignable to `JsonValue`. + +@category JSON +*/ +export type JsonValue = JsonPrimitive | JsonObject | JsonArray; +//# sourceMappingURL=basic.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/type-fest/basic.d.ts.map b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/basic.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a331b50d25c11e92c8a89e9ee1bce19c2c95ed7b --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/basic.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"basic.d.ts","sourceRoot":"","sources":["../../../../src/vendor/type-fest/basic.ts"],"names":[],"mappings":"AAAA;;;;;;EAME;AACF,MAAM,MAAM,UAAU,GAAG;KAAG,GAAG,IAAI,MAAM,GAAG,SAAS;CAAE,GAAG;KAAG,GAAG,IAAI,MAAM,CAAC,CAAC,EAAE,SAAS,GAAG,SAAS;CAAE,CAAC;AAEtG;;;;EAIE;AACF,MAAM,MAAM,SAAS,GAAG,SAAS,EAAE,GAAG,SAAS,SAAS,EAAE,CAAC;AAE3D;;;;EAIE;AACF,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;AAE7D;;;;;;EAME;AACF,MAAM,MAAM,SAAS,GAAG,aAAa,GAAG,UAAU,GAAG,SAAS,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/type-fest/entries.d.ts b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/entries.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..400567db2d5865d041eab92aac280623c2203f48 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/entries.d.ts @@ -0,0 +1,57 @@ +import type { ArrayEntry, MapEntry, ObjectEntry, SetEntry } from "./entry"; +type ArrayEntries = Array>; +type MapEntries = Array>; +type ObjectEntries = Array>; +type SetEntries> = Array>; +/** +Many collections have an `entries` method which returns an array of a given object's own enumerable string-keyed property [key, value] pairs. The `Entries` type will return the type of that collection's entries. + +For example the {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/entries|`Object`}, {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map/entries|`Map`}, {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/entries|`Array`}, and {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set/entries|`Set`} collections all have this method. Note that `WeakMap` and `WeakSet` do not have this method since their entries are not enumerable. + +@see `Entry` if you want to just access the type of a single entry. + +@example +``` +import type {Entries} from 'type-fest'; + +interface Example { + someKey: number; +} + +const manipulatesEntries = (examples: Entries) => examples.map(example => [ + // Does some arbitrary processing on the key (with type information available) + example[0].toUpperCase(), + + // Does some arbitrary processing on the value (with type information available) + example[1].toFixed() +]); + +const example: Example = {someKey: 1}; +const entries = Object.entries(example) as Entries; +const output = manipulatesEntries(entries); + +// Objects +const objectExample = {a: 1}; +const objectEntries: Entries = [['a', 1]]; + +// Arrays +const arrayExample = ['a', 1]; +const arrayEntries: Entries = [[0, 'a'], [1, 1]]; + +// Maps +const mapExample = new Map([['a', 1]]); +const mapEntries: Entries = [['a', 1]]; + +// Sets +const setExample = new Set(['a', 1]); +const setEntries: Entries = [['a', 'a'], [1, 1]]; +``` + +@category Object +@category Map +@category Set +@category Array +*/ +export type Entries = BaseType extends Map ? MapEntries : BaseType extends Set ? SetEntries : BaseType extends readonly unknown[] ? ArrayEntries : BaseType extends object ? ObjectEntries : never; +export {}; +//# sourceMappingURL=entries.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/type-fest/entries.d.ts.map b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/entries.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a225268a8eed76c85b076d1c289b470f51fe1352 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/entries.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"entries.d.ts","sourceRoot":"","sources":["../../../../src/vendor/type-fest/entries.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAE3E,KAAK,YAAY,CAAC,QAAQ,SAAS,SAAS,OAAO,EAAE,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;AACrF,KAAK,UAAU,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;AACtD,KAAK,aAAa,CAAC,QAAQ,IAAI,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC5D,KAAK,UAAU,CAAC,QAAQ,SAAS,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE3E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAgDE;AACF,MAAM,MAAM,OAAO,CAAC,QAAQ,IAC3B,QAAQ,SAAS,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,GACnC,UAAU,CAAC,QAAQ,CAAC,GACpB,QAAQ,SAAS,GAAG,CAAC,OAAO,CAAC,GAC5B,UAAU,CAAC,QAAQ,CAAC,GACpB,QAAQ,SAAS,SAAS,OAAO,EAAE,GAClC,YAAY,CAAC,QAAQ,CAAC,GACtB,QAAQ,SAAS,MAAM,GACtB,aAAa,CAAC,QAAQ,CAAC,GACvB,KAAK,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/type-fest/entry.d.ts b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/entry.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..8b3a5f48de6771db03dd5df3223a24267ec3f2bc --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/entry.d.ts @@ -0,0 +1,60 @@ +type MapKey = BaseType extends Map ? KeyType : never; +type MapValue = BaseType extends Map ? ValueType : never; +export type ArrayEntry = [number, BaseType[number]]; +export type MapEntry = [MapKey, MapValue]; +export type ObjectEntry = [keyof BaseType, BaseType[keyof BaseType]]; +export type SetEntry = BaseType extends Set ? [ItemType, ItemType] : never; +/** +Many collections have an `entries` method which returns an array of a given object's own enumerable string-keyed property [key, value] pairs. The `Entry` type will return the type of that collection's entry. + +For example the {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/entries|`Object`}, {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map/entries|`Map`}, {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/entries|`Array`}, and {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set/entries|`Set`} collections all have this method. Note that `WeakMap` and `WeakSet` do not have this method since their entries are not enumerable. + +@see `Entries` if you want to just access the type of the array of entries (which is the return of the `.entries()` method). + +@example +``` +import type {Entry} from 'type-fest'; + +interface Example { + someKey: number; +} + +const manipulatesEntry = (example: Entry) => [ + // Does some arbitrary processing on the key (with type information available) + example[0].toUpperCase(), + + // Does some arbitrary processing on the value (with type information available) + example[1].toFixed(), +]; + +const example: Example = {someKey: 1}; +const entry = Object.entries(example)[0] as Entry; +const output = manipulatesEntry(entry); + +// Objects +const objectExample = {a: 1}; +const objectEntry: Entry = ['a', 1]; + +// Arrays +const arrayExample = ['a', 1]; +const arrayEntryString: Entry = [0, 'a']; +const arrayEntryNumber: Entry = [1, 1]; + +// Maps +const mapExample = new Map([['a', 1]]); +const mapEntry: Entry = ['a', 1]; + +// Sets +const setExample = new Set(['a', 1]); +const setEntryString: Entry = ['a', 'a']; +const setEntryNumber: Entry = [1, 1]; +``` + +@category Object +@category Map +@category Array +@category Set +*/ +export type Entry = BaseType extends Map ? MapEntry : BaseType extends Set ? SetEntry : BaseType extends readonly unknown[] ? ArrayEntry : BaseType extends object ? ObjectEntry : never; +export {}; +//# sourceMappingURL=entry.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/type-fest/entry.d.ts.map b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/entry.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f9c31621eaf9ef1ded5fc674b32f40737f78b6ae --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/entry.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"entry.d.ts","sourceRoot":"","sources":["../../../../src/vendor/type-fest/entry.ts"],"names":[],"mappings":"AAAA,KAAK,MAAM,CAAC,QAAQ,IAAI,QAAQ,SAAS,GAAG,CAAC,MAAM,OAAO,EAAE,OAAO,CAAC,GAAG,OAAO,GAAG,KAAK,CAAC;AACvF,KAAK,QAAQ,CAAC,QAAQ,IAAI,QAAQ,SAAS,GAAG,CAAC,OAAO,EAAE,MAAM,SAAS,CAAC,GAAG,SAAS,GAAG,KAAK,CAAC;AAE7F,MAAM,MAAM,UAAU,CAAC,QAAQ,SAAS,SAAS,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;AACzF,MAAM,MAAM,QAAQ,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;AACxE,MAAM,MAAM,WAAW,CAAC,QAAQ,IAAI,CAAC,MAAM,QAAQ,EAAE,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC,CAAC;AAC/E,MAAM,MAAM,QAAQ,CAAC,QAAQ,IAAI,QAAQ,SAAS,GAAG,CAAC,MAAM,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC;AAErG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAkDE;AACF,MAAM,MAAM,KAAK,CAAC,QAAQ,IACzB,QAAQ,SAAS,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,GACnC,QAAQ,CAAC,QAAQ,CAAC,GAClB,QAAQ,SAAS,GAAG,CAAC,OAAO,CAAC,GAC5B,QAAQ,CAAC,QAAQ,CAAC,GAClB,QAAQ,SAAS,SAAS,OAAO,EAAE,GAClC,UAAU,CAAC,QAAQ,CAAC,GACpB,QAAQ,SAAS,MAAM,GACtB,WAAW,CAAC,QAAQ,CAAC,GACrB,KAAK,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/type-fest/except.d.ts b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/except.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..6086c8f26a8e3766ec2ff7c8a3649b2a9365fba7 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/except.d.ts @@ -0,0 +1,67 @@ +import type { IsEqual } from "./is-equal"; +/** +Filter out keys from an object. + +Returns `never` if `Exclude` is strictly equal to `Key`. +Returns `never` if `Key` extends `Exclude`. +Returns `Key` otherwise. + +@example +``` +type Filtered = Filter<'foo', 'foo'>; +//=> never +``` + +@example +``` +type Filtered = Filter<'bar', string>; +//=> never +``` + +@example +``` +type Filtered = Filter<'bar', 'foo'>; +//=> 'bar' +``` + +@see {Except} +*/ +type Filter = IsEqual extends true ? never : KeyType extends ExcludeType ? never : KeyType; +/** +Create a type from an object type without certain keys. + +We recommend setting the `requireExactProps` option to `true`. + +This type is a stricter version of [`Omit`](https://www.typescriptlang.org/docs/handbook/release-notes/typescript-3-5.html#the-omit-helper-type). The `Omit` type does not restrict the omitted keys to be keys present on the given type, while `Except` does. The benefits of a stricter type are avoiding typos and allowing the compiler to pick up on rename refactors automatically. + +This type was proposed to the TypeScript team, which declined it, saying they prefer that libraries implement stricter versions of the built-in types ([microsoft/TypeScript#30825](https://github.com/microsoft/TypeScript/issues/30825#issuecomment-523668235)). + +@example +``` +import type {Except} from 'type-fest'; + +type Foo = { + a: number; + b: string; +}; + +type FooWithoutA = Except; +//=> {b: string} + +const fooWithoutA: FooWithoutA = {a: 1, b: '2'}; +//=> errors: 'a' does not exist in type '{ b: string; }' + +type FooWithoutB = Except; +//=> {a: number} & Partial> + +const fooWithoutB: FooWithoutB = {a: 1, b: '2'}; +//=> errors at 'b': Type 'string' is not assignable to type 'undefined'. +``` + +@category Object +*/ +export type Except = { + [KeyType in keyof ObjectType as Filter]: ObjectType[KeyType]; +}; +export {}; +//# sourceMappingURL=except.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/type-fest/except.d.ts.map b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/except.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f8626d74797a3a748c421a8125bb0477587591b3 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/except.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"except.d.ts","sourceRoot":"","sources":["../../../../src/vendor/type-fest/except.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C;;;;;;;;;;;;;;;;;;;;;;;;;;EA0BE;AACF,KAAK,MAAM,CAAC,OAAO,EAAE,WAAW,IAC/B,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,SAAS,IAAI,GAAG,KAAK,GAAG,OAAO,SAAS,WAAW,GAAG,KAAK,GAAG,OAAO,CAAC;AAEpG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAgCE;AACF,MAAM,MAAM,MAAM,CAAC,UAAU,EAAE,QAAQ,SAAS,MAAM,UAAU,IAAI;KAClE,OAAO,IAAI,MAAM,UAAU,IAAI,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC;CAC/E,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/type-fest/is-equal.d.ts b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/is-equal.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..379e2bcde32c2f236ef6e29a1f9eef18b1e029d4 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/is-equal.d.ts @@ -0,0 +1,28 @@ +/** +Returns a boolean for whether the two given types are equal. + +@link https://github.com/microsoft/TypeScript/issues/27024#issuecomment-421529650 +@link https://stackoverflow.com/questions/68961864/how-does-the-equals-work-in-typescript/68963796#68963796 + +Use-cases: +- If you want to make a conditional branch based on the result of a comparison of two types. + +@example +``` +import type {IsEqual} from 'type-fest'; + +// This type returns a boolean for whether the given array includes the given item. +// `IsEqual` is used to compare the given array at position 0 and the given item and then return true if they are equal. +type Includes = + Value extends readonly [Value[0], ...infer rest] + ? IsEqual extends true + ? true + : Includes + : false; +``` + +@category Type Guard +@category Utilities +*/ +export type IsEqual = (() => G extends A ? 1 : 2) extends () => G extends B ? 1 : 2 ? true : false; +//# sourceMappingURL=is-equal.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/type-fest/is-equal.d.ts.map b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/is-equal.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..4459d190f6c8e6fd72b602a418672b8cbd263e4a --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/is-equal.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"is-equal.d.ts","sourceRoot":"","sources":["../../../../src/vendor/type-fest/is-equal.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;EAyBE;AACF,MAAM,MAAM,OAAO,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,KAAK,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/type-fest/set-required.d.ts b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/set-required.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..a5db5b2e7bab99032aa7de0e1eac925a5e5a2f80 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/set-required.d.ts @@ -0,0 +1,29 @@ +import type { Except } from "./except"; +import type { Simplify } from "./simplify"; +/** +Create a type that makes the given keys required. The remaining keys are kept as is. The sister of the `SetOptional` type. + +Use-case: You want to define a single model where the only thing that changes is whether or not some of the keys are required. + +@example +``` +import type {SetRequired} from 'type-fest'; + +type Foo = { + a?: number; + b: string; + c?: boolean; +} + +type SomeRequired = SetRequired; +// type SomeRequired = { +// a?: number; +// b: string; // Was already required and still is. +// c: boolean; // Is now required. +// } +``` + +@category Object +*/ +export type SetRequired = Simplify & Required>>; +//# sourceMappingURL=set-required.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/type-fest/set-required.d.ts.map b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/set-required.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..69f1d38f92b485da1d1b2f166e65668fbfac06a3 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/set-required.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"set-required.d.ts","sourceRoot":"","sources":["../../../../src/vendor/type-fest/set-required.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;EAwBE;AACF,MAAM,MAAM,WAAW,CAAC,QAAQ,EAAE,IAAI,SAAS,MAAM,QAAQ,IAAI,QAAQ,CAExE,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,GAErB,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAC/B,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/type-fest/simplify.d.ts b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/simplify.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..1fd7581d344fef70c7c423e3ec805b7b604181ff --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/simplify.d.ts @@ -0,0 +1,61 @@ +/** +Useful to flatten the type output to improve type hints shown in editors. And also to transform an interface into a type to aide with assignability. + +@example +``` +import type {Simplify} from 'type-fest'; + +type PositionProps = { + top: number; + left: number; +}; + +type SizeProps = { + width: number; + height: number; +}; + +// In your editor, hovering over `Props` will show a flattened object with all the properties. +type Props = Simplify; +``` + +Sometimes it is desired to pass a value as a function argument that has a different type. At first inspection it may seem assignable, and then you discover it is not because the `value`'s type definition was defined as an interface. In the following example, `fn` requires an argument of type `Record`. If the value is defined as a literal, then it is assignable. And if the `value` is defined as type using the `Simplify` utility the value is assignable. But if the `value` is defined as an interface, it is not assignable because the interface is not sealed and elsewhere a non-string property could be added to the interface. + +If the type definition must be an interface (perhaps it was defined in a third-party npm package), then the `value` can be defined as `const value: Simplify = ...`. Then `value` will be assignable to the `fn` argument. Or the `value` can be cast as `Simplify` if you can't re-declare the `value`. + +@example +``` +import type {Simplify} from 'type-fest'; + +interface SomeInterface { + foo: number; + bar?: string; + baz: number | undefined; +} + +type SomeType = { + foo: number; + bar?: string; + baz: number | undefined; +}; + +const literal = {foo: 123, bar: 'hello', baz: 456}; +const someType: SomeType = literal; +const someInterface: SomeInterface = literal; + +function fn(object: Record): void {} + +fn(literal); // Good: literal object type is sealed +fn(someType); // Good: type is sealed +fn(someInterface); // Error: Index signature for type 'string' is missing in type 'someInterface'. Because `interface` can be re-opened +fn(someInterface as Simplify); // Good: transform an `interface` into a `type` +``` + +@link https://github.com/microsoft/TypeScript/issues/15300 + +@category Object +*/ +export type Simplify = { + [KeyType in keyof T]: T[KeyType]; +} & {}; +//# sourceMappingURL=simplify.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/src/vendor/type-fest/simplify.d.ts.map b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/simplify.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..fdcf4ee12d0336ecd5d484fd90d1dd63c893f3c5 --- /dev/null +++ b/node_modules/@huggingface/hub/dist/src/vendor/type-fest/simplify.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"simplify.d.ts","sourceRoot":"","sources":["../../../../src/vendor/type-fest/simplify.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwDE;AAEF,MAAM,MAAM,QAAQ,CAAC,CAAC,IAAI;KAAG,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;CAAE,GAAG,EAAE,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/dist/sub-paths-HFKHI55E.mjs b/node_modules/@huggingface/hub/dist/sub-paths-HFKHI55E.mjs new file mode 100644 index 0000000000000000000000000000000000000000..42518bd92be1ec1fe73d472c1d529db6789206bb --- /dev/null +++ b/node_modules/@huggingface/hub/dist/sub-paths-HFKHI55E.mjs @@ -0,0 +1,30 @@ +import "./chunk-FFYIGW52.mjs"; + +// src/utils/sub-paths.ts +import { readdir, stat } from "fs/promises"; +import { fileURLToPath, pathToFileURL } from "url"; +async function subPaths(path, maxDepth = 10) { + const state = await stat(path); + if (!state.isDirectory()) { + return [{ path, relativePath: "." }]; + } + const files = await readdir(path, { withFileTypes: true }); + const ret = []; + for (const file of files) { + const filePath = pathToFileURL(fileURLToPath(path) + "/" + file.name); + if (file.isDirectory()) { + ret.push( + ...(await subPaths(filePath, maxDepth - 1)).map((subPath) => ({ + ...subPath, + relativePath: `${file.name}/${subPath.relativePath}` + })) + ); + } else { + ret.push({ path: filePath, relativePath: file.name }); + } + } + return ret; +} +export { + subPaths +}; diff --git a/node_modules/@huggingface/hub/index.ts b/node_modules/@huggingface/hub/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..3bd16e178a03831ae8cbf9578aa6df4b8be5926d --- /dev/null +++ b/node_modules/@huggingface/hub/index.ts @@ -0,0 +1 @@ +export * from "./src"; diff --git a/node_modules/@huggingface/hub/package.json b/node_modules/@huggingface/hub/package.json new file mode 100644 index 0000000000000000000000000000000000000000..cefb390a3b5c174c03a7ea406d42280a0c3d2d70 --- /dev/null +++ b/node_modules/@huggingface/hub/package.json @@ -0,0 +1,75 @@ +{ + "name": "@huggingface/hub", + "version": "2.13.2", + "description": "Utilities to interact with the Hugging Face hub", + "keywords": [ + "api", + "client", + "face", + "hub", + "hugging", + "huggingface" + ], + "license": "MIT", + "author": "Hugging Face", + "repository": "https://github.com/huggingface/huggingface.js.git", + "bin": { + "hfjs": "./dist/cli.js" + }, + "source": "index.ts", + "files": [ + "dist", + "index.ts", + "src", + "tsconfig.json" + ], + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "browser": { + "./src/utils/sha256-node.ts": false, + "./src/utils/sub-paths.ts": false, + "./src/utils/FileBlob.ts": false, + "./src/lib/cache-management.ts": false, + "./src/lib/download-file-to-cache-dir.ts": false, + "./src/lib/snapshot-download.ts": false, + "./dist/index.js": "./dist/browser/index.js", + "./dist/index.mjs": "./dist/browser/index.mjs" + }, + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "require": "./dist/index.js", + "import": "./dist/index.mjs" + } + }, + "publishConfig": { + "access": "public" + }, + "dependencies": { + "@huggingface/tasks": "^0.21.13", + "@huggingface/xetchunk-wasm": "^0.1.0" + }, + "devDependencies": { + "@types/cli-progress": "^3.11.6" + }, + "optionalDependencies": { + "cli-progress": "^3.12.0" + }, + "engines": { + "node": ">=18" + }, + "scripts": { + "lint": "eslint --quiet --fix --ext .cjs,.ts .", + "lint:check": "eslint --ext .cjs,.ts .", + "format": "oxfmt .", + "format:check": "oxfmt --check .", + "build": "tsup && tsc --emitDeclarationOnly --declaration", + "test": "vitest run", + "test:browser": "vitest run --browser.name=chrome --browser.headless --config vitest-browser.config.mts", + "check": "tsc", + "build:xet-wasm": "./scripts/build-xet-wasm.sh -t bundler --clean", + "bench": "tsx scripts/bench.ts", + "debug-xet": "tsx scripts/debug-xet.ts" + } +} \ No newline at end of file diff --git a/node_modules/@huggingface/hub/src/consts.ts b/node_modules/@huggingface/hub/src/consts.ts new file mode 100644 index 0000000000000000000000000000000000000000..5d34e9caecf3f843e34c24f341de91f6c715f17e --- /dev/null +++ b/node_modules/@huggingface/hub/src/consts.ts @@ -0,0 +1 @@ +export const HUB_URL = "https://huggingface.co"; diff --git a/node_modules/@huggingface/hub/src/error.ts b/node_modules/@huggingface/hub/src/error.ts new file mode 100644 index 0000000000000000000000000000000000000000..9c3c675d8755f0f4f14d4786db9a809074df3b6e --- /dev/null +++ b/node_modules/@huggingface/hub/src/error.ts @@ -0,0 +1,49 @@ +import type { JsonObject } from "./vendor/type-fest/basic"; + +export async function createApiError( + response: Response, + opts?: { requestId?: string; message?: string }, +): Promise { + const error = new HubApiError(response.url, response.status, response.headers.get("X-Request-Id") ?? opts?.requestId); + + error.message = `Api error with status ${error.statusCode}${opts?.message ? `. ${opts.message}` : ""}`; + + const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : undefined] + .filter(Boolean) + .join(". "); + + if (response.headers.get("Content-Type")?.startsWith("application/json")) { + const json = await response.json(); + error.message = json.error || json.message || error.message; + if (json.error_description) { + error.message = error.message ? error.message + `: ${json.error_description}` : json.error_description; + } + error.data = json; + } else { + error.data = { message: await response.text() }; + } + + error.message += `. ${trailer}`; + + throw error; +} + +/** + * Error thrown when an API call to the Hugging Face Hub fails. + */ +export class HubApiError extends Error { + statusCode: number; + url: string; + requestId?: string; + data?: JsonObject; + + constructor(url: string, statusCode: number, requestId?: string, message?: string) { + super(message); + + this.statusCode = statusCode; + this.requestId = requestId; + this.url = url; + } +} + +export class InvalidApiResponseFormatError extends Error {} diff --git a/node_modules/@huggingface/hub/src/index.ts b/node_modules/@huggingface/hub/src/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..6e26804c37d48c5a086673ac3e5d1f377680291b --- /dev/null +++ b/node_modules/@huggingface/hub/src/index.ts @@ -0,0 +1,27 @@ +export * from "./lib"; +// Typescript 5 will add 'export type *' +export type { + AccessToken, + AccessTokenRole, + AuthType, + Credentials, + PipelineType, + RepoDesignation, + RepoFullName, + RepoId, + RepoType, + SpaceHardwareFlavor, + SpaceResourceConfig, + SpaceResourceRequirement, + SpaceRuntime, + SpaceSdk, + SpaceStage, +} from "./types/public"; +export { HubApiError, InvalidApiResponseFormatError } from "./error"; +export { HUB_URL } from "./consts"; +/** + * Only exported for E2Es convenience + */ +export { sha256 as __internal_sha256 } from "./utils/sha256"; +export { XetBlob as __internal_XetBlob } from "./utils/XetBlob"; +export type { XetReadToken } from "./utils/XetBlob"; diff --git a/node_modules/@huggingface/hub/src/lib/add-collection-item.spec.ts b/node_modules/@huggingface/hub/src/lib/add-collection-item.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..8261e2e9124dddc1e05afbc69e02af1976ca8604 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/add-collection-item.spec.ts @@ -0,0 +1,84 @@ +import { it, describe, expect } from "vitest"; + +import { TEST_HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../test/consts"; +import { addCollectionItem } from "./add-collection-item"; +import { collectionInfo } from "./collection-info"; +import { deleteCollectionItem } from "./delete-collection-item"; +import { createCollection } from "./create-collection"; +import { deleteCollection } from "./delete-collection"; + +describe("addCollectionItem", () => { + it("should add a item to a collection", async () => { + let slug: string = ""; + + const randomString = crypto.randomUUID(); + const title = `Test Collection ${randomString}`; + + try { + const result = await createCollection({ + collection: { + title, + namespace: TEST_USER, + description: "This is a test collection", + private: false, + }, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + slug = result.slug; + + expect(result.slug.startsWith(`${TEST_USER}/test-collection-${randomString}`)).toBe(true); + + await addCollectionItem({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + slug, + item: { + type: "collection", + // temporary, later the slug should work on its own + id: result.slug.slice(-24), + }, + note: "This is a test item", + }); + + const items = await collectionInfo({ + slug, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + expect(items.items.length).toBe(1); + expect(items.items[0].type).toBe("collection"); + // temporary, later the slug should work on its own right? + expect(items.items[0].id).toBe(result.slug.slice(-24)); + expect(items.items[0].note).toEqual({ + html: "This is a test item", + text: "This is a test item", + }); + + await deleteCollectionItem({ + slug, + itemId: items.items[0]._id, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + const items2 = await collectionInfo({ + slug, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + expect(items2.items.length).toBe(0); + } finally { + if (slug) { + await deleteCollection({ + slug, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + } + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/add-collection-item.ts b/node_modules/@huggingface/hub/src/lib/add-collection-item.ts new file mode 100644 index 0000000000000000000000000000000000000000..27e926bca96c31da836ede834400b7fbe6568fb6 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/add-collection-item.ts @@ -0,0 +1,51 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { CredentialsParams } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; + +export async function addCollectionItem( + params: { + /** + * The slug of the collection to add the item to. + */ + slug: string; + /** + * The item to add to the collection. + */ + item: { + type: "paper" | "collection" | "space" | "model" | "dataset"; + id: string; + }; + /** + * A note to attach to the item in the collection. The maximum size for a note is 500 characters. + */ + note?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise { + if (!params.slug) { + throw new TypeError("slug is required"); + } + + const accessToken = checkCredentials(params); + + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/collections/${params.slug}/items`, { + method: "POST", + body: JSON.stringify({ + item: params.item, + note: params.note, + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!res.ok) { + throw await createApiError(res); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/cache-management.spec.ts b/node_modules/@huggingface/hub/src/lib/cache-management.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..a4a6544114b4851c24155b265445da0983ef9241 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/cache-management.spec.ts @@ -0,0 +1,139 @@ +import { describe, test, expect, vi, beforeEach } from "vitest"; +import { + scanCacheDir, + scanCachedRepo, + scanSnapshotDir, + parseRepoType, + getBlobStat, + type CachedFileInfo, +} from "./cache-management"; +import { stat, readdir, realpath, lstat } from "node:fs/promises"; +import type { Stats } from "node:fs"; +import { join } from "node:path"; + +// Mocks +vi.mock("node:fs/promises"); + +beforeEach(() => { + vi.resetAllMocks(); + vi.restoreAllMocks(); +}); + +describe("scanCacheDir", () => { + test("should throw an error if cacheDir is not a directory", async () => { + vi.mocked(stat).mockResolvedValueOnce({ + isDirectory: () => false, + } as Stats); + + await expect(scanCacheDir("/fake/dir")).rejects.toThrow("Scan cache expects a directory"); + }); + + test("empty directory should return an empty set of repository and no warnings", async () => { + vi.mocked(stat).mockResolvedValueOnce({ + isDirectory: () => true, + } as Stats); + + // mock empty cache folder + vi.mocked(readdir).mockResolvedValue([]); + + const result = await scanCacheDir("/fake/dir"); + + // cacheDir must have been read + expect(readdir).toHaveBeenCalledWith("/fake/dir"); + + expect(result.warnings.length).toBe(0); + expect(result.repos).toHaveLength(0); + expect(result.size).toBe(0); + }); +}); + +describe("scanCachedRepo", () => { + test("should throw an error for invalid repo path", async () => { + await expect(() => { + return scanCachedRepo("/fake/repo_path"); + }).rejects.toThrow("Repo path is not a valid HuggingFace cache directory"); + }); + + test("should throw an error if the snapshot folder does not exist", async () => { + vi.mocked(readdir).mockResolvedValue([]); + vi.mocked(stat).mockResolvedValue({ + isDirectory: () => false, + } as Stats); + + await expect(() => { + return scanCachedRepo("/fake/cacheDir/models--hello-world--name"); + }).rejects.toThrow("Snapshots dir doesn't exist in cached repo"); + }); + + test("should properly parse the repository name", async () => { + const repoPath = "/fake/cacheDir/models--hello-world--name"; + vi.mocked(readdir).mockResolvedValue([]); + vi.mocked(stat).mockResolvedValue({ + isDirectory: () => true, + } as Stats); + + const result = await scanCachedRepo(repoPath); + expect(readdir).toHaveBeenCalledWith(join(repoPath, "refs"), { + withFileTypes: true, + }); + + expect(result.id.name).toBe("hello-world/name"); + expect(result.id.type).toBe("model"); + }); +}); + +describe("scanSnapshotDir", () => { + test("should scan a valid snapshot directory", async () => { + const cachedFiles: CachedFileInfo[] = []; + const blobStats = new Map(); + vi.mocked(readdir).mockResolvedValueOnce([ + { name: "file1", isDirectory: () => false } as unknown as Awaited>[0], + ]); + + vi.mocked(realpath).mockResolvedValueOnce("/fake/realpath"); + vi.mocked(lstat).mockResolvedValueOnce({ size: 1024, atimeMs: Date.now(), mtimeMs: Date.now() } as Stats); + + await scanSnapshotDir("/fake/revision", cachedFiles, blobStats); + + expect(cachedFiles).toHaveLength(1); + expect(blobStats.size).toBe(1); + }); +}); + +describe("getBlobStat", () => { + test("should retrieve blob stat if already cached", async () => { + const blobStats = new Map([["/fake/blob", { size: 1024 } as Stats]]); + const result = await getBlobStat("/fake/blob", blobStats); + + expect(lstat).not.toHaveBeenCalled(); + expect(result.size).toBe(1024); + }); + + test("should fetch and cache blob stat if not cached", async () => { + const blobStats = new Map(); + vi.mocked(lstat).mockResolvedValueOnce({ size: 2048 } as Stats); + + const result = await getBlobStat("/fake/blob", blobStats); + + expect(result.size).toBe(2048); + expect(blobStats.size).toBe(1); + }); +}); + +describe("parseRepoType", () => { + test("should parse models repo type", () => { + expect(parseRepoType("models")).toBe("model"); + }); + + test("should parse dataset repo type", () => { + expect(parseRepoType("datasets")).toBe("dataset"); + }); + + test("should parse space repo type", () => { + expect(parseRepoType("spaces")).toBe("space"); + }); + + test("should throw an error for invalid repo type", () => { + expect(() => parseRepoType("invalid")).toThrowError("Invalid repo type: invalid"); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/cache-management.ts b/node_modules/@huggingface/hub/src/lib/cache-management.ts new file mode 100644 index 0000000000000000000000000000000000000000..423643960faa7baed561545a4e0b644a70919660 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/cache-management.ts @@ -0,0 +1,279 @@ +import { homedir } from "node:os"; +import { join, basename } from "node:path"; +import { stat, readdir, readFile, realpath, lstat } from "node:fs/promises"; +import type { Stats } from "node:fs"; +import type { RepoType, RepoId } from "../types/public"; + +function getDefaultHome(): string { + return join(homedir(), ".cache"); +} + +function getDefaultCachePath(): string { + return join(process.env["HF_HOME"] ?? join(process.env["XDG_CACHE_HOME"] ?? getDefaultHome(), "huggingface"), "hub"); +} + +function getHuggingFaceHubCache(): string { + return process.env["HUGGINGFACE_HUB_CACHE"] ?? getDefaultCachePath(); +} + +export function getHFHubCachePath(): string { + return process.env["HF_HUB_CACHE"] ?? getHuggingFaceHubCache(); +} + +const FILES_TO_IGNORE: string[] = [".DS_Store"]; + +export const REPO_ID_SEPARATOR: string = "--"; + +export function getRepoFolderName({ name, type }: RepoId): string { + const parts = [`${type}s`, ...name.split("/")]; + return parts.join(REPO_ID_SEPARATOR); +} + +export interface CachedFileInfo { + path: string; + /** + * Underlying file - which `path` is symlinked to + */ + blob: { + size: number; + path: string; + lastModifiedAt: Date; + lastAccessedAt: Date; + }; +} + +export interface CachedRevisionInfo { + commitOid: string; + path: string; + size: number; + files: CachedFileInfo[]; + refs: string[]; + + lastModifiedAt: Date; +} + +export interface CachedRepoInfo { + id: RepoId; + path: string; + size: number; + filesCount: number; + revisions: CachedRevisionInfo[]; + + lastAccessedAt: Date; + lastModifiedAt: Date; +} + +export interface HFCacheInfo { + size: number; + repos: CachedRepoInfo[]; + warnings: Error[]; +} + +export async function scanCacheDir(cacheDir: string | undefined = undefined): Promise { + if (!cacheDir) { + cacheDir = getHFHubCachePath(); + } + + const s = await stat(cacheDir); + if (!s.isDirectory()) { + throw new Error( + `Scan cache expects a directory but found a file: ${cacheDir}. Please use \`cacheDir\` argument or set \`HF_HUB_CACHE\` environment variable.`, + ); + } + + const repos: CachedRepoInfo[] = []; + const warnings: Error[] = []; + + const directories = await readdir(cacheDir); + for (const repo of directories) { + // skip .locks folder + if (repo === ".locks") { + continue; + } + + // get the absolute path of the repo + const absolute = join(cacheDir, repo); + + // ignore non-directory element + const s = await stat(absolute); + if (!s.isDirectory()) { + continue; + } + + try { + const cached = await scanCachedRepo(absolute); + repos.push(cached); + } catch (err: unknown) { + warnings.push(err as Error); + } + } + + return { + repos: repos, + size: [...repos.values()].reduce((sum, repo) => sum + repo.size, 0), + warnings: warnings, + }; +} + +export async function scanCachedRepo(repoPath: string): Promise { + // get the directory name + const name = basename(repoPath); + if (!name.includes(REPO_ID_SEPARATOR)) { + throw new Error(`Repo path is not a valid HuggingFace cache directory: ${name}`); + } + + // parse the repoId from directory name + const [type, ...remaining] = name.split(REPO_ID_SEPARATOR); + const repoType = parseRepoType(type); + const repoId = remaining.join("/"); + + const snapshotsPath = join(repoPath, "snapshots"); + const refsPath = join(repoPath, "refs"); + + const snapshotStat = await stat(snapshotsPath); + if (!snapshotStat.isDirectory()) { + throw new Error(`Snapshots dir doesn't exist in cached repo ${snapshotsPath}`); + } + + // Check if the refs directory exists and scan it + const refsByHash: Map = new Map(); + const refsStat = await stat(refsPath); + if (refsStat.isDirectory()) { + await scanRefsDir(refsPath, refsByHash); + } + + // Scan snapshots directory and collect cached revision information + const cachedRevisions: CachedRevisionInfo[] = []; + const blobStats: Map = new Map(); // Store blob stats + + const snapshotDirs = await readdir(snapshotsPath); + for (const dir of snapshotDirs) { + if (FILES_TO_IGNORE.includes(dir)) { + continue; + } // Ignore unwanted files + + const revisionPath = join(snapshotsPath, dir); + const revisionStat = await stat(revisionPath); + if (!revisionStat.isDirectory()) { + throw new Error(`Snapshots folder corrupted. Found a file: ${revisionPath}`); + } + + const cachedFiles: CachedFileInfo[] = []; + await scanSnapshotDir(revisionPath, cachedFiles, blobStats); + + const revisionLastModified = + cachedFiles.length > 0 + ? Math.max(...[...cachedFiles].map((file) => file.blob.lastModifiedAt.getTime())) + : revisionStat.mtimeMs; + + cachedRevisions.push({ + commitOid: dir, + files: cachedFiles, + refs: refsByHash.get(dir) || [], + size: [...cachedFiles].reduce((sum, file) => sum + file.blob.size, 0), + path: revisionPath, + lastModifiedAt: new Date(revisionLastModified), + }); + + refsByHash.delete(dir); + } + + // Verify that all refs refer to a valid revision + if (refsByHash.size > 0) { + throw new Error( + `Reference(s) refer to missing commit hashes: ${JSON.stringify(Object.fromEntries(refsByHash))} (${repoPath})`, + ); + } + + const repoStats = await stat(repoPath); + const repoLastAccessed = + blobStats.size > 0 ? Math.max(...[...blobStats.values()].map((stat) => stat.atimeMs)) : repoStats.atimeMs; + + const repoLastModified = + blobStats.size > 0 ? Math.max(...[...blobStats.values()].map((stat) => stat.mtimeMs)) : repoStats.mtimeMs; + + // Return the constructed CachedRepoInfo object + return { + id: { + name: repoId, + type: repoType, + }, + path: repoPath, + filesCount: blobStats.size, + revisions: cachedRevisions, + size: [...blobStats.values()].reduce((sum, stat) => sum + stat.size, 0), + lastAccessedAt: new Date(repoLastAccessed), + lastModifiedAt: new Date(repoLastModified), + }; +} + +export async function scanRefsDir(refsPath: string, refsByHash: Map): Promise { + const refFiles = await readdir(refsPath, { withFileTypes: true }); + for (const refFile of refFiles) { + const refFilePath = join(refsPath, refFile.name); + if (refFile.isDirectory()) { + continue; // Skip directories + } + + const commitHash = await readFile(refFilePath, "utf-8"); + const refName = refFile.name; + if (!refsByHash.has(commitHash)) { + refsByHash.set(commitHash, []); + } + refsByHash.get(commitHash)?.push(refName); + } +} + +export async function scanSnapshotDir( + revisionPath: string, + cachedFiles: CachedFileInfo[], + blobStats: Map, +): Promise { + const files = await readdir(revisionPath, { withFileTypes: true }); + for (const file of files) { + if (file.isDirectory()) { + continue; // Skip directories + } + + const filePath = join(revisionPath, file.name); + const blobPath = await realpath(filePath); + const blobStat = await getBlobStat(blobPath, blobStats); + + cachedFiles.push({ + path: filePath, + blob: { + path: blobPath, + size: blobStat.size, + lastAccessedAt: new Date(blobStat.atimeMs), + lastModifiedAt: new Date(blobStat.mtimeMs), + }, + }); + } +} + +export async function getBlobStat(blobPath: string, blobStats: Map): Promise { + const blob = blobStats.get(blobPath); + if (!blob) { + const statResult = await lstat(blobPath); + blobStats.set(blobPath, statResult); + return statResult; + } + return blob; +} + +export function parseRepoType(type: string): RepoType { + switch (type) { + case "models": + return "model"; + case "datasets": + return "dataset"; + case "spaces": + return "space"; + case "buckets": + return "bucket"; + case "kernels": + return "kernel"; + default: + throw new TypeError(`Invalid repo type: ${type}`); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/check-repo-access.spec.ts b/node_modules/@huggingface/hub/src/lib/check-repo-access.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..12ad5cd92a2cf3549b1e62f4125b875336b55d6a --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/check-repo-access.spec.ts @@ -0,0 +1,34 @@ +import { assert, describe, expect, it } from "vitest"; +import { checkRepoAccess } from "./check-repo-access"; +import { HubApiError } from "../error"; +import { TEST_ACCESS_TOKEN, TEST_HUB_URL } from "../test/consts"; + +describe("checkRepoAccess", () => { + it("should throw 401 when accessing unexisting repo unauthenticated", async () => { + try { + await checkRepoAccess({ repo: { name: "i--d/dont", type: "model" } }); + assert(false, "should have thrown"); + } catch (err) { + expect(err).toBeInstanceOf(HubApiError); + expect((err as HubApiError).statusCode).toBe(401); + } + }); + + it("should throw 404 when accessing unexisting repo authenticated", async () => { + try { + await checkRepoAccess({ + repo: { name: "i--d/dont", type: "model" }, + hubUrl: TEST_HUB_URL, + accessToken: TEST_ACCESS_TOKEN, + }); + assert(false, "should have thrown"); + } catch (err) { + expect(err).toBeInstanceOf(HubApiError); + expect((err as HubApiError).statusCode).toBe(404); + } + }); + + it("should not throw when accessing public repo", async () => { + await checkRepoAccess({ repo: { name: "openai-community/gpt2", type: "model" } }); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/check-repo-access.ts b/node_modules/@huggingface/hub/src/lib/check-repo-access.ts new file mode 100644 index 0000000000000000000000000000000000000000..4c997291b4c78bb77859d7c8ba56c49291eb4f5a --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/check-repo-access.ts @@ -0,0 +1,32 @@ +import { HUB_URL } from "../consts"; +// eslint-disable-next-line @typescript-eslint/no-unused-vars +import { createApiError, type HubApiError } from "../error"; +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { toRepoId } from "../utils/toRepoId"; + +/** + * Check if we have read access to a repository. + * + * Throw a {@link HubApiError} error if we don't have access. HubApiError.statusCode will be 401, 403 or 404. + */ +export async function checkRepoAccess( + params: { + repo: RepoDesignation; + hubUrl?: string; + fetch?: typeof fetch; + } & Partial, +): Promise { + const accessToken = params && checkCredentials(params); + const repoId = toRepoId(params.repo); + + const response = await (params.fetch || fetch)(`${params?.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}`, { + headers: { + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }); + + if (!response.ok) { + throw await createApiError(response); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/collection-info.spec.ts b/node_modules/@huggingface/hub/src/lib/collection-info.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..0ed4aeaf915a01c1c5d00718d66fbb1b6a2ad717 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/collection-info.spec.ts @@ -0,0 +1,175 @@ +import { describe, expect, it } from "vitest"; +import { collectionInfo } from "./collection-info"; + +describe("collectionInfo", () => { + it("should return the collection info", async () => { + const collection = await collectionInfo({ + slug: "huggingfacejs/test-collection-690df2897fa1945492b8cf42", + }); + + // Check all properties of the collection except items + expect(collection).toEqual({ + slug: "huggingfacejs/test-collection-690df2897fa1945492b8cf42", + title: "Test Collection", + description: "Only used in E2E tests", + gating: false, + lastUpdated: expect.any(String), + owner: { + _id: "6414d83b385a75d7790d5a58", + avatarUrl: expect.any(String), + fullname: "Huggingface.js", + name: "huggingfacejs", + type: "org", + followerCount: expect.any(Number), + isHf: false, + isHfAdmin: false, + isMod: false, + isUserFollowing: expect.any(Boolean), + }, + items: [ + { + _id: "690df2a467ea25a1a346d0ae", + author: "huggingfacejs", + datasetsServerInfo: { + formats: expect.any(Array), + libraries: expect.any(Array), + modalities: expect.any(Array), + numRows: expect.any(Number), + viewer: expect.any(String), + }, + downloads: expect.any(Number), + gated: false, + id: "huggingfacejs/tasks", + isBenchmark: false, + isLikedByUser: false, + isTraces: false, + lastModified: expect.any(String), + likes: expect.any(Number), + position: 0, + private: false, + repoType: "dataset", + type: "dataset", + }, + { + _id: "690df2b1954547dac9727da3", + description: "Only used in E2E tests", + id: "690df2897fa1945492b8cf42", + isUpvotedByUser: false, + lastUpdated: expect.any(String), + numberItems: 5, + owner: { + _id: "6414d83b385a75d7790d5a58", + avatarUrl: expect.any(String), + followerCount: expect.any(Number), + fullname: "Huggingface.js", + isHf: false, + isHfAdmin: false, + isMod: false, + isUserFollowing: expect.any(Boolean), + name: "huggingfacejs", + type: "org", + }, + position: 1, + shareUrl: "https://hf.co/collections/huggingfacejs/test-collection", + slug: "huggingfacejs/test-collection-690df2897fa1945492b8cf42", + theme: "pink", + title: "Test Collection", + type: "collection", + upvotes: expect.any(Number), + }, + { + _id: "690df2c49f252aa897a873b2", + ai_category: "Model Benchmarking", + ai_short_description: "Upload ML models to Hugging Face Hub from your browser", + author: "huggingfacejs", + authorData: { + _id: "6414d83b385a75d7790d5a58", + avatarUrl: expect.any(String), + followerCount: expect.any(Number), + fullname: "Huggingface.js", + isHf: false, + isHfAdmin: false, + isMod: false, + isUserFollowing: expect.any(Boolean), + name: "huggingfacejs", + type: "org", + }, + colorFrom: "green", + colorTo: "green", + createdAt: "2023-03-17T21:33:16.000Z", + emoji: "🌎", + featured: false, + id: "huggingfacejs/push-model-from-web", + isLikedByUser: false, + lastModified: expect.any(String), + likes: expect.any(Number), + pinned: false, + position: 2, + private: false, + repoType: "space", + runtime: { + hardware: { + current: null, + requested: null, + }, + replicas: { + current: 1, + requested: 1, + }, + stage: "RUNNING", + }, + sdk: "static", + tags: ["static", "region:us"], + title: "Push Model From Web", + trendingScore: expect.any(Number), + type: "space", + visibility: "public", + }, + { + _id: "690df2d0c9390ed6ab0f88b1", + id: "2510.04871", + isUpvotedByUser: false, + position: 3, + publishedAt: "2025-10-06T14:58:08.000Z", + thumbnailUrl: "https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2510.04871.png", + title: "Less is More: Recursive Reasoning with Tiny Networks", + type: "paper", + upvotes: expect.any(Number), + }, + { + _id: "690df30af2d88fbd705feda5", + author: "huggingfacejs", + authorData: { + _id: "6414d83b385a75d7790d5a58", + avatarUrl: expect.any(String), + followerCount: expect.any(Number), + fullname: "Huggingface.js", + isHf: false, + isHfAdmin: false, + isMod: false, + isUserFollowing: expect.any(Boolean), + name: "huggingfacejs", + type: "org", + }, + availableInferenceProviders: [], + downloads: expect.any(Number), + gated: false, + id: "huggingfacejs/test-model", + isLikedByUser: false, + lastModified: expect.any(String), + likes: expect.any(Number), + position: 4, + private: false, + repoType: "model", + type: "model", + }, + ], + theme: "pink", + private: false, + position: 0, + shareUrl: "https://hf.co/collections/huggingfacejs/test-collection", + upvotes: expect.any(Number), + isUpvotedByUser: false, + }); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/collection-info.ts b/node_modules/@huggingface/hub/src/lib/collection-info.ts new file mode 100644 index 0000000000000000000000000000000000000000..ce46e0ceba872e43a8a25be544f1d7a4919403ec --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/collection-info.ts @@ -0,0 +1,34 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiCollectionInfo } from "../types/api/api-collection"; +import type { CredentialsParams } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; + +export async function collectionInfo( + params: { + /** + * The slug of the collection. + */ + slug: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise { + const accessToken = checkCredentials(params); + + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/collections/${params.slug}`, { + headers: { + "Content-Type": "application/json", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : undefined), + }, + }); + + if (!res.ok) { + throw await createApiError(res); + } + + return await res.json(); +} diff --git a/node_modules/@huggingface/hub/src/lib/commit.spec.ts b/node_modules/@huggingface/hub/src/lib/commit.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..1e59201c3ee24c64c5c5bff3e2efecc21c558e5d --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/commit.spec.ts @@ -0,0 +1,271 @@ +import { assert, it, describe } from "vitest"; + +import { TEST_HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../test/consts"; +import type { RepoId } from "../types/public"; +import type { CommitFile } from "./commit"; +import { commit } from "./commit"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { downloadFile } from "./download-file"; +import { fileDownloadInfo } from "./file-download-info"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { isFrontend } from "../utils/isFrontend"; + +const lfsContent = "O123456789".repeat(100_000); + +describe("commit", () => { + it("should commit to a repo with blobs", async function () { + const tokenizerJsonUrl = new URL( + "https://huggingface.co/spaces/aschen/push-model-from-web/raw/main/mobilenet/model.json", + ); + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo: RepoId = { + name: repoName, + type: "model", + }; + + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo, + license: "mit", + }); + + try { + const readme1 = await downloadFile({ repo, path: "README.md", hubUrl: TEST_HUB_URL }); + assert(readme1, "Readme doesn't exist"); + + const nodeOperation: CommitFile[] = isFrontend + ? [] + : [ + { + operation: "addOrUpdate", + path: "tsconfig.json", + content: (await import("node:url")).pathToFileURL("./tsconfig.json") as URL, + }, + ]; + + await commit({ + repo, + title: "Some commit", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + operations: [ + { + operation: "addOrUpdate", + content: new Blob(["This is me"]), + path: "test.txt", + }, + { + operation: "addOrUpdate", + content: new Blob([lfsContent]), + path: "test.lfs.txt", + }, + ...nodeOperation, + { + operation: "addOrUpdate", + content: tokenizerJsonUrl, + path: "lamaral.json", + }, + { + operation: "delete", + path: "README.md", + }, + ], + // To test web workers in the front-end + useWebWorkers: { minSize: 5_000 }, + }); + + const fileContent = await downloadFile({ repo, path: "test.txt", hubUrl: TEST_HUB_URL }); + assert.strictEqual(await fileContent?.text(), "This is me"); + + const lfsFileContent = await downloadFile({ repo, path: "test.lfs.txt", hubUrl: TEST_HUB_URL }); + assert.strictEqual(await lfsFileContent?.text(), lfsContent); + + const lfsFileUrl = `${TEST_HUB_URL}/${repoName}/raw/main/test.lfs.txt`; + const lfsFilePointer = await fetch(lfsFileUrl); + assert.strictEqual(lfsFilePointer.status, 200); + assert.strictEqual( + (await lfsFilePointer.text()).trim(), + ` +version https://git-lfs.github.com/spec/v1 +oid sha256:a3bbce7ee1df7233d85b5f4d60faa3755f93f537804f8b540c72b0739239ddf8 +size ${lfsContent.length} + `.trim(), + ); + + if (!isFrontend) { + const fileUrlContent = await downloadFile({ repo, path: "tsconfig.json", hubUrl: TEST_HUB_URL }); + assert.strictEqual( + await fileUrlContent?.text(), + (await import("node:fs")).readFileSync("./tsconfig.json", "utf-8"), + ); + } + + const webResourceContent = await downloadFile({ repo, path: "lamaral.json", hubUrl: TEST_HUB_URL }); + assert.strictEqual(await webResourceContent?.text(), await (await fetch(tokenizerJsonUrl)).text()); + + const readme2 = await downloadFile({ repo, path: "README.md", hubUrl: TEST_HUB_URL }); + assert.strictEqual(readme2, null); + } finally { + await deleteRepo({ + repo: { + name: repoName, + type: "model", + }, + hubUrl: TEST_HUB_URL, + credentials: { accessToken: TEST_ACCESS_TOKEN }, + }); + } + }, 60_000); + + it("should commit a full repo from HF with web urls", async function () { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo: RepoId = { + name: repoName, + type: "model", + }; + + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + repo, + hubUrl: TEST_HUB_URL, + }); + + try { + const FILES_TO_UPLOAD = [ + `https://huggingface.co/spaces/huggingfacejs/push-model-from-web/resolve/main/mobilenet/model.json`, + `https://huggingface.co/spaces/huggingfacejs/push-model-from-web/resolve/main/mobilenet/group1-shard1of2`, + `https://huggingface.co/spaces/huggingfacejs/push-model-from-web/resolve/main/mobilenet/group1-shard2of2`, + `https://huggingface.co/spaces/huggingfacejs/push-model-from-web/resolve/main/mobilenet/coffee.jpg`, + `https://huggingface.co/spaces/huggingfacejs/push-model-from-web/resolve/main/mobilenet/README.md`, + ]; + + const operations: CommitFile[] = await Promise.all( + FILES_TO_UPLOAD.map(async (file) => { + return { + operation: "addOrUpdate", + path: file.slice(file.indexOf("main/") + "main/".length), + // upload remote file + content: new URL(file), + }; + }), + ); + await commit({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + title: "upload model", + operations, + }); + + const LFSSize = (await fileDownloadInfo({ repo, path: "mobilenet/group1-shard1of2", hubUrl: TEST_HUB_URL })) + ?.size; + + assert.strictEqual(LFSSize, 4_194_304); + + const pointerFile = await downloadFile({ + repo, + path: "mobilenet/group1-shard1of2", + raw: true, + hubUrl: TEST_HUB_URL, + }); + + // Make sure SHA is computed properly as well + assert.strictEqual( + (await pointerFile?.text())?.trim(), + ` +version https://git-lfs.github.com/spec/v1 +oid sha256:3fb621eb9b37478239504ee083042d5b18699e8b8618e569478b03b119a85a69 +size 4194304 + `.trim(), + ); + } finally { + await deleteRepo({ + repo: { + name: repoName, + type: "model", + }, + hubUrl: TEST_HUB_URL, + credentials: { accessToken: TEST_ACCESS_TOKEN }, + }); + } + // https://huggingfacejs-push-model-from-web.hf.space/ + }, 60_000); + + it("should be able to create a PR and then commit to it", async function () { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo: RepoId = { + name: repoName, + type: "model", + }; + + await createRepo({ + credentials: { + accessToken: TEST_ACCESS_TOKEN, + }, + repo, + hubUrl: TEST_HUB_URL, + }); + + try { + const pr = await commit({ + repo, + credentials: { + accessToken: TEST_ACCESS_TOKEN, + }, + hubUrl: TEST_HUB_URL, + title: "Create PR", + isPullRequest: true, + operations: [ + { + operation: "addOrUpdate", + content: new Blob(["This is me"]), + path: "test.txt", + }, + ], + }); + + if (!pr) { + throw new Error("PR creation failed"); + } + + if (!pr.pullRequestUrl) { + throw new Error("No pull request url"); + } + + const prNumber = pr.pullRequestUrl.split("/").pop(); + const prRef = `refs/pr/${prNumber}`; + + await commit({ + repo, + credentials: { + accessToken: TEST_ACCESS_TOKEN, + }, + hubUrl: TEST_HUB_URL, + branch: prRef, + title: "Some commit", + operations: [ + { + operation: "addOrUpdate", + content: new URL( + `https://huggingface.co/spaces/huggingfacejs/push-model-from-web/resolve/main/mobilenet/group1-shard1of2`, + ), + path: "mobilenet/group1-shard1of2", + }, + ], + }); + + assert(commit, "PR commit failed"); + } finally { + await deleteRepo({ + repo: { + name: repoName, + type: "model", + }, + hubUrl: TEST_HUB_URL, + credentials: { accessToken: TEST_ACCESS_TOKEN }, + }); + } + }, 60_000); +}); diff --git a/node_modules/@huggingface/hub/src/lib/commit.ts b/node_modules/@huggingface/hub/src/lib/commit.ts new file mode 100644 index 0000000000000000000000000000000000000000..56f660463d4e9475d5309e61c4d9ebfde8f14726 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/commit.ts @@ -0,0 +1,1067 @@ +import { HUB_URL } from "../consts"; +import { HubApiError, createApiError, InvalidApiResponseFormatError } from "../error"; +import type { + ApiBucketBatchResponse, + ApiCommitHeader, + ApiCommitLfsFile, + ApiCommitOperation, + ApiLfsBatchRequest, + ApiLfsBatchResponse, + ApiLfsCompleteMultipartRequest, + ApiPreuploadRequest, + ApiPreuploadResponse, +} from "../types/api/api-commit"; +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { chunk } from "../utils/chunk"; +import { promisesQueue } from "../utils/promisesQueue"; +import { promisesQueueStreaming } from "../utils/promisesQueueStreaming"; +import { sha256 } from "../utils/sha256"; +import { toRepoId } from "../utils/toRepoId"; +import { WebBlob } from "../utils/WebBlob"; +import { eventToGenerator } from "../utils/eventToGenerator"; +import { base64FromBytes } from "../utils/base64FromBytes"; +import { isFrontend } from "../utils/isFrontend"; +import { createBlobs } from "../utils/createBlobs"; +import type { XetTokenParams } from "../utils/uploadShards"; +import { uploadShards } from "../utils/uploadShards"; +import { splitAsyncGenerator } from "../utils/splitAsyncGenerator"; +import { SplicedBlob } from "../utils/SplicedBlob"; + +const CONCURRENT_SHAS = 5; +const CONCURRENT_LFS_UPLOADS = 5; +const MULTIPART_PARALLEL_UPLOAD = 5; + +export interface CommitDeletedEntry { + operation: "delete"; + path: string; +} + +export type ContentSource = Blob | URL; + +export interface CommitFile { + operation: "addOrUpdate"; + path: string; + content: ContentSource; + // forceLfs?: boolean +} + +/** + * Opitmized when only the beginning or the end of the file is replaced + * + * todo: handle other cases + */ +export interface CommitEditFile { + operation: "edit"; + path: string; + /** Later, will be ContentSource. For now simpler to just handle blobs */ + originalContent: Blob; + edits: Array<{ + /** + * Later, will be ContentSource. For now simpler to just handle blobs + * + * originalContent from [start, end) will be replaced by this + */ + content: Blob; + /** + * The start position of the edit in the original content + */ + start: number; + /** + * The end position of the edit in the original content + * + * originalContent from [start, end) will be replaced by the edit + */ + end: number; + }>; +} + +type CommitBlob = Omit & { content: Blob }; + +// TODO: find a nice way to handle LFS & non-LFS files in an uniform manner, see https://github.com/huggingface/moon-landing/issues/4370 +// export type CommitRenameFile = { +// operation: "rename"; +// path: string; +// oldPath: string; +// content?: ContentSource; +// }; + +/** + * Server-side copy of a file from a source repo/bucket to the destination repo. + * + * Only supported when the destination repo is a bucket. The source file must be xet-backed, + * so the caller is responsible for resolving the source path to its {@link sourceXetHash} + * (typically via {@link pathsInfo} or {@link listFiles}). + * + * For higher-level helpers that perform the resolution and handle non-xet source files, + * see {@link copyFile}, {@link copyFiles} and {@link copyFolder}. + */ +export interface CommitCopyFile { + operation: "copy"; + path: string; + sourceXetHash: string; + sourceRepo: RepoDesignation; +} + +export type CommitOperation = + | CommitDeletedEntry + | CommitFile + | CommitEditFile + | CommitCopyFile /* | CommitRenameFile */; +type CommitBlobOperation = Exclude | CommitBlob; + +export type CommitParams = { + title: string; + description?: string; + repo: RepoDesignation; + operations: CommitOperation[]; + /** @default "main" */ + branch?: string; + /** + * Parent commit. Optional + * + * - When opening a PR: will use parentCommit as the parent commit + * - When committing on a branch: Will make sure that there were no intermediate commits + */ + parentCommit?: string; + isPullRequest?: boolean; + hubUrl?: string; + /** + * Whether to use web workers to compute SHA256 hashes. + * + * @default false + */ + useWebWorkers?: boolean | { minSize?: number; poolSize?: number }; + /** + * Maximum depth of folders to upload. Files deeper than this will be ignored + * + * @default 5 + */ + maxFolderDepth?: number; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + abortSignal?: AbortSignal; + /** + * @default true + * + * Use xet protocol: https://huggingface.co/blog/xet-on-the-hub to upload, rather than a basic S3 PUT + */ + useXet?: boolean; + // Credentials are optional due to custom fetch functions or cookie auth +} & Partial; + +export interface CommitOutput { + pullRequestUrl?: string; + commit: { + oid: string; + url: string; + }; + hookOutput: string; +} + +function isFileOperation(op: CommitOperation): op is CommitBlob { + const ret = op.operation === "addOrUpdate"; + + if (ret && !(op.content instanceof Blob)) { + throw new TypeError("Precondition failed: op.content should be a Blob"); + } + + return ret; +} + +export type CommitProgressEvent = + | { + event: "phase"; + phase: "preuploading" | "uploadingLargeFiles" | "committing"; + } + | { + event: "fileProgress"; + path: string; + progress: number; + state: "hashing" | "uploading" | "error"; + }; + +/** + * Internal function for now, used by commit. + * + * Can be exposed later to offer fine-tuned progress info + * + * CommitOutput is not present for bucket commits + */ +export async function* commitIter(params: CommitParams): AsyncGenerator { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + + if (repoId.type === "bucket") { + return yield* commitIterBucket(params); + } + + if (params.operations.some((op) => op.operation === "copy")) { + throw new Error("'copy' operations are only supported when the destination repo is a bucket"); + } + + yield { event: "phase", phase: "preuploading" }; + + let useXet = params.useXet ?? true; + + const lfsShas = new Map(); + + const abortController = new AbortController(); + const abortSignal = abortController.signal; + + // Polyfill see https://discuss.huggingface.co/t/why-cant-i-upload-a-parquet-file-to-my-dataset-error-o-throwifaborted-is-not-a-function/62245 + if (!abortSignal.throwIfAborted) { + abortSignal.throwIfAborted = () => { + if (abortSignal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + }; + } + + if (params.abortSignal) { + params.abortSignal.addEventListener("abort", () => abortController.abort()); + } + + try { + const allOperations = ( + await Promise.all( + params.operations.map(async (operation) => { + if (operation.operation === "edit") { + // Convert EditFile operation to a file operation with SplicedBlob + const splicedBlob = SplicedBlob.create( + operation.originalContent, + operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })), + ); + return { + operation: "addOrUpdate" as const, + path: operation.path, + content: splicedBlob, + }; + } + + if (operation.operation !== "addOrUpdate") { + return operation; + } + + if (!(operation.content instanceof URL)) { + /** TS trick to enforce `content` to be a `Blob` */ + return { ...operation, content: operation.content }; + } + + const lazyBlobs = await createBlobs(operation.content, operation.path, { + fetch: params.fetch, + maxFolderDepth: params.maxFolderDepth, + }); + + abortSignal?.throwIfAborted(); + + return lazyBlobs.map((blob) => ({ + ...operation, + content: blob.blob, + path: blob.path, + })); + }), + ) + ).flat(1); + + const gitAttributes = allOperations.filter(isFileOperation).find((op) => op.path === ".gitattributes")?.content; + + for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { + const payload: ApiPreuploadRequest = { + gitAttributes: gitAttributes && (await gitAttributes.text()), + files: await Promise.all( + operations.map(async (operation) => ({ + path: operation.path, + size: operation.content.size, + sample: base64FromBytes(new Uint8Array(await operation.content.slice(0, 512).arrayBuffer())), + })), + ), + }; + + abortSignal?.throwIfAborted(); + + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/preupload/${encodeURIComponent( + params.branch ?? "main", + )}` + (params.isPullRequest ? "?create_pr=1" : ""), + { + method: "POST", + headers: { + ...(accessToken && { Authorization: `Bearer ${accessToken}` }), + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), + signal: abortSignal, + }, + ); + + if (!res.ok) { + throw await createApiError(res); + } + + const json: ApiPreuploadResponse = await res.json(); + + for (const file of json.files) { + if (file.uploadMode === "lfs") { + lfsShas.set(file.path, null); + } + } + } + + yield { event: "phase", phase: "uploadingLargeFiles" }; + + for (const operations of chunk( + allOperations.filter(isFileOperation).filter((op) => lfsShas.has(op.path)), + 100, + )) { + const shas = yield* eventToGenerator< + { event: "fileProgress"; state: "hashing"; path: string; progress: number }, + string[] + >((yieldCallback, returnCallback, rejectCallack) => { + return promisesQueue( + operations.map((op) => async () => { + const iterator = sha256(op.content, { useWebWorker: params.useWebWorkers, abortSignal: abortSignal }); + let res: IteratorResult; + do { + res = await iterator.next(); + if (!res.done) { + yieldCallback({ event: "fileProgress", path: op.path, progress: res.value, state: "hashing" }); + } + } while (!res.done); + const sha = res.value; + lfsShas.set(op.path, res.value); + return sha; + }), + CONCURRENT_SHAS, + ).then(returnCallback, rejectCallack); + }); + + abortSignal?.throwIfAborted(); + + const payload: ApiLfsBatchRequest = { + operation: "upload", + // multipart is a custom protocol for HF + transfers: ["basic", "multipart", ...(useXet ? ["xet" as const] : [])], + hash_algo: "sha_256", + ...(!params.isPullRequest && { + ref: { + name: params.branch ?? "main", + }, + }), + objects: operations.map((op, i) => ({ + oid: shas[i], + size: op.content.size, + })), + }; + + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/${repoId.type === "model" ? "" : repoId.type + "s/"}${ + repoId.name + }.git/info/lfs/objects/batch`, + { + method: "POST", + headers: { + ...(accessToken && { Authorization: `Bearer ${accessToken}` }), + Accept: "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json", + }, + body: JSON.stringify(payload), + signal: abortSignal, + }, + ); + + if (!res.ok) { + throw await createApiError(res); + } + + const json: ApiLfsBatchResponse = await res.json(); + const batchRequestId = res.headers.get("X-Request-Id") || undefined; + + const shaToOperation = new Map(operations.map((op, i) => [shas[i], op])); + + if (useXet && json.transfer !== "xet") { + useXet = false; + } + + let xetParams: XetTokenParams | null = null; + + if (useXet) { + // First get all the files that are already uploaded out of the way + for (const obj of json.objects) { + const op = shaToOperation.get(obj.oid); + if (!op) { + throw new InvalidApiResponseFormatError("Unrequested object ID in response"); + } + + if (obj.error) { + const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${ + obj.error.message + }${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; + throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); + } + + if (!obj.actions?.upload) { + // Already uploaded + yield { + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading", + }; + } else { + const headers = new Headers(obj.actions.upload.header); + + xetParams = { + sessionId: headers.get("X-Xet-Session-Id") ?? undefined, + casUrl: headers.get("X-Xet-Cas-Url") ?? undefined, + accessToken: headers.get("X-Xet-Access-Token") ?? undefined, + expiresAt: headers.get("X-Xet-Token-Expiration") + ? new Date(parseInt(headers.get("X-Xet-Token-Expiration") ?? "0") * 1000) + : undefined, + refreshWriteTokenUrl: obj.actions.upload.href, + }; + } + } + const source = (async function* () { + for (const obj of json.objects) { + const op = shaToOperation.get(obj.oid); + if (!op || !obj.actions?.upload) { + continue; + } + abortSignal?.throwIfAborted(); + yield { content: op.content, path: op.path, sha256: obj.oid }; + } + })(); + if (xetParams) { + const fixedXetParams = xetParams; + const sources = splitAsyncGenerator(source, 5); + yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => + Promise.all( + sources.map(async function (source) { + for await (const event of uploadShards(source, { + fetch: params.fetch, + accessToken, + hubUrl: params.hubUrl ?? HUB_URL, + repo: repoId, + xetParams: fixedXetParams, + // todo: maybe leave empty if PR? + rev: params.branch ?? "main", + isPullRequest: params.isPullRequest, + yieldCallback: (event) => yieldCallback({ ...event, state: "uploading" }), + })) { + if (event.event === "file") { + yieldCallback({ + event: "fileProgress" as const, + path: event.path, + progress: 1, + state: "uploading" as const, + }); + } else if (event.event === "fileProgress") { + yieldCallback({ + event: "fileProgress" as const, + path: event.path, + progress: event.progress, + state: "uploading" as const, + }); + } + } + }), + ).then(() => returnCallback(undefined), rejectCallback), + ); + } else { + // No LFS file to upload + } + } else { + yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + return promisesQueueStreaming( + json.objects.map((obj) => async () => { + const op = shaToOperation.get(obj.oid); + + if (!op) { + throw new InvalidApiResponseFormatError("Unrequested object ID in response"); + } + + abortSignal?.throwIfAborted(); + + if (obj.error) { + const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${ + obj.error.message + }${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; + throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); + } + if (!obj.actions?.upload) { + // Already uploaded + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading", + }); + return; + } + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 0, + state: "uploading", + }); + const content = op.content; + + const header = obj.actions.upload.header; + if (header?.chunk_size) { + const chunkSize = parseInt(header.chunk_size); + + // multipart upload + // parts are in upload.header['00001'] to upload.header['99999'] + + const completionUrl = obj.actions.upload.href; + const parts = Object.keys(header).filter((key) => /^[0-9]+$/.test(key)); + + if (parts.length !== Math.ceil(content.size / chunkSize)) { + throw new Error("Invalid server response to upload large LFS file, wrong number of parts"); + } + + const completeReq: ApiLfsCompleteMultipartRequest = { + oid: obj.oid, + parts: parts.map((part) => ({ + partNumber: +part, + etag: "", + })), + }; + + // Defined here so that it's not redefined at each iteration (and the caller can tell it's for the same file) + const progressCallback = (progress: number) => + yieldCallback({ event: "fileProgress", path: op.path, progress, state: "uploading" }); + + await promisesQueueStreaming( + parts.map((part) => async () => { + abortSignal?.throwIfAborted(); + + const index = parseInt(part) - 1; + const slice = content.slice(index * chunkSize, (index + 1) * chunkSize); + + const res = await (params.fetch ?? fetch)(header[part], { + method: "PUT", + /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ + body: slice instanceof WebBlob && isFrontend ? await slice.arrayBuffer() : slice, + signal: abortSignal, + ...({ + progressHint: { + path: op.path, + part: index, + numParts: parts.length, + progressCallback, + }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any), + }); + + if (!res.ok) { + throw await createApiError(res, { + requestId: batchRequestId, + message: `Error while uploading part ${part} of ${ + operations[shas.indexOf(obj.oid)].path + } to LFS storage`, + }); + } + + const eTag = res.headers.get("ETag"); + + if (!eTag) { + throw new Error("Cannot get ETag of part during multipart upload"); + } + + completeReq.parts[Number(part) - 1].etag = eTag; + }), + MULTIPART_PARALLEL_UPLOAD, + ); + + abortSignal?.throwIfAborted(); + + const res = await (params.fetch ?? fetch)(completionUrl, { + method: "POST", + body: JSON.stringify(completeReq), + headers: { + Accept: "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json", + }, + signal: abortSignal, + }); + + if (!res.ok) { + throw await createApiError(res, { + requestId: batchRequestId, + message: `Error completing multipart upload of ${ + operations[shas.indexOf(obj.oid)].path + } to LFS storage`, + }); + } + + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading", + }); + } else { + const res = await (params.fetch ?? fetch)(obj.actions.upload.href, { + method: "PUT", + headers: { + ...(batchRequestId ? { "X-Request-Id": batchRequestId } : undefined), + }, + /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ + body: content instanceof WebBlob && isFrontend ? await content.arrayBuffer() : content, + signal: abortSignal, + ...({ + progressHint: { + path: op.path, + progressCallback: (progress: number) => + yieldCallback({ + event: "fileProgress", + path: op.path, + progress, + state: "uploading", + }), + }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any), + }); + + if (!res.ok) { + throw await createApiError(res, { + requestId: batchRequestId, + message: `Error while uploading ${operations[shas.indexOf(obj.oid)].path} to LFS storage`, + }); + } + + yieldCallback({ + event: "fileProgress", + path: op.path, + progress: 1, + state: "uploading", + }); + } + }), + CONCURRENT_LFS_UPLOADS, + ).then(returnCallback, rejectCallback); + }); + } + } + + abortSignal?.throwIfAborted(); + + yield { event: "phase", phase: "committing" }; + + return yield* eventToGenerator( + async (yieldCallback, returnCallback, rejectCallback) => + (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commit/${encodeURIComponent( + params.branch ?? "main", + )}` + (params.isPullRequest ? "?create_pr=1" : ""), + { + method: "POST", + headers: { + ...(accessToken && { Authorization: `Bearer ${accessToken}` }), + "Content-Type": "application/x-ndjson", + }, + body: [ + { + key: "header", + value: { + summary: params.title, + description: params.description, + parentCommit: params.parentCommit, + } satisfies ApiCommitHeader, + }, + ...((await Promise.all( + allOperations.map((operation) => { + if (isFileOperation(operation)) { + const sha = lfsShas.get(operation.path); + if (sha) { + return { + key: "lfsFile", + value: { + path: operation.path, + algo: "sha256", + size: operation.content.size, + oid: sha, + } satisfies ApiCommitLfsFile, + }; + } + } + + return convertOperationToNdJson(operation); + }), + )) satisfies ApiCommitOperation[]), + ] + .map((x) => JSON.stringify(x)) + .join("\n"), + signal: abortSignal, + ...({ + progressHint: { + progressCallback: (progress: number) => { + // For now, we display equal progress for all files + // We could compute the progress based on the size of `convertOperationToNdJson` for each of the files instead + for (const op of allOperations) { + if (isFileOperation(op) && !lfsShas.has(op.path)) { + yieldCallback({ + event: "fileProgress", + path: op.path, + progress, + state: "uploading", + }); + } + } + }, + }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any), + }, + ) + .then(async (res) => { + if (!res.ok) { + throw await createApiError(res); + } + + const json = await res.json(); + + returnCallback({ + pullRequestUrl: json.pullRequestUrl, + commit: { + oid: json.commitOid, + url: json.commitUrl, + }, + hookOutput: json.hookOutput, + }); + }) + .catch(rejectCallback), + ); + } catch (err) { + // For parallel requests, cancel them all if one fails + abortController.abort(); + throw err; + } +} + +export async function* commitIterBucket(params: CommitParams): AsyncGenerator { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + + if (params.useXet === false) { + throw new Error("useXet must be true or undefined for buckets"); + } + + const abortController = new AbortController(); + const abortSignal = abortController.signal; + + // Polyfill see https://discuss.huggingface.co/t/why-cant-i-upload-a-parquet-file-to-my-dataset-error-o-throwifaborted-is-not-a-function/62245 + if (!abortSignal.throwIfAborted) { + abortSignal.throwIfAborted = () => { + if (abortSignal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + }; + } + + if (params.abortSignal) { + params.abortSignal.addEventListener("abort", () => abortController.abort()); + } + + try { + const allOperations = ( + await Promise.all( + params.operations.map(async (operation) => { + if (operation.operation === "edit") { + // Convert EditFile operation to a file operation with SplicedBlob + const splicedBlob = SplicedBlob.create( + operation.originalContent, + operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })), + ); + return { + operation: "addOrUpdate" as const, + path: operation.path, + content: splicedBlob, + }; + } + + if (operation.operation !== "addOrUpdate") { + return operation; + } + + if (!(operation.content instanceof URL)) { + /** TS trick to enforce `content` to be a `Blob` */ + return { ...operation, content: operation.content }; + } + + const lazyBlobs = await createBlobs(operation.content, operation.path, { + fetch: params.fetch, + maxFolderDepth: params.maxFolderDepth, + }); + + abortSignal?.throwIfAborted(); + + return lazyBlobs.map((blob) => ({ + ...operation, + content: blob.blob, + path: blob.path, + })); + }), + ) + ).flat(1); + + yield { event: "phase", phase: "uploadingLargeFiles" }; + + for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { + const xetHashes = new Map(); + abortSignal?.throwIfAborted(); + + // First get all the files that are already uploaded out of the way + + const source = (async function* () { + for (const operation of operations) { + abortSignal?.throwIfAborted(); + yield { content: operation.content, path: operation.path }; + } + })(); + + const xetParams: XetTokenParams = { + sessionId: crypto.randomUUID(), + refreshWriteTokenUrl: `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/xet-write-token`, + }; + const sources = splitAsyncGenerator(source, 5); + yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => + Promise.all( + sources.map(async function (source) { + for await (const event of uploadShards(source, { + fetch: params.fetch, + accessToken, + hubUrl: params.hubUrl ?? HUB_URL, + repo: repoId, + xetParams, + rev: params.branch ?? "main", + yieldCallback: (event) => yieldCallback({ ...event, state: "uploading" }), + })) { + if (event.event === "file") { + yieldCallback({ + event: "fileProgress" as const, + path: event.path, + progress: 1, + state: "uploading" as const, + }); + xetHashes.set(event.path, event.xetHash); + } else if (event.event === "fileProgress") { + yieldCallback({ + event: "fileProgress" as const, + path: event.path, + progress: event.progress, + state: "uploading" as const, + }); + } + } + }), + ).then(() => returnCallback(undefined), rejectCallback), + ); + + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...(accessToken && { Authorization: `Bearer ${accessToken}` }), + "Content-Type": "application/x-ndjson", + }, + body: [...xetHashes.entries()] + .map(([path, xetHash]) => + JSON.stringify({ + type: "addFile", + path, + xetHash, + }), + ) + .join("\n"), + signal: abortSignal, + }, + ); + + if (!resp.ok && resp.status !== 422) { + throw await createApiError(resp); + } + + const json = (await resp.json()) as ApiBucketBatchResponse; + + for (const failed of json.failed) { + yield { + event: "fileProgress", + path: failed.path, + progress: 0, + state: "error", + }; + } + } + + abortSignal?.throwIfAborted(); + + const copyOperations = allOperations.filter( + (operation): operation is CommitCopyFile => operation.operation === "copy", + ); + + for (const copyChunk of chunk(copyOperations, 100)) { + abortSignal?.throwIfAborted(); + + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...(accessToken && { Authorization: `Bearer ${accessToken}` }), + "Content-Type": "application/x-ndjson", + }, + body: copyChunk + .map((op) => { + const sourceRepoId = toRepoId(op.sourceRepo); + return JSON.stringify({ + type: "copyFile", + path: op.path, + xetHash: op.sourceXetHash, + sourceRepoType: sourceRepoId.type, + sourceRepoId: sourceRepoId.name, + }); + }) + .join("\n"), + signal: abortSignal, + }, + ); + + if (!resp.ok && resp.status !== 422) { + throw await createApiError(resp); + } + + const json = (await resp.json()) as ApiBucketBatchResponse; + + for (const failed of json.failed) { + yield { + event: "fileProgress", + path: failed.path, + progress: 0, + state: "error", + }; + } + } + + abortSignal?.throwIfAborted(); + + const deletedOperations = allOperations.filter((operation) => operation.operation === "delete"); + + if (deletedOperations.length > 0) { + const resp = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/batch`, + { + method: "POST", + headers: { + ...(accessToken && { Authorization: `Bearer ${accessToken}` }), + "Content-Type": "application/x-ndjson", + }, + body: deletedOperations + .map((operation) => + JSON.stringify({ + type: "deleteFile", + path: operation.path, + }), + ) + .join("\n"), + signal: abortSignal, + }, + ); + + if (!resp.ok) { + throw await createApiError(resp); + } + + const json = await resp.json(); + + if (json.failed.length > 0) { + const failedPaths = json.failed.slice(0, 5).map((f: { path: string }) => f.path); + throw new Error( + `Failed to delete ${json.failed.length} file(s): ${failedPaths.join(", ")}${json.failed.length > 5 ? "..." : ""}, request ID: ${resp.headers.get("X-Request-Id")}`, + ); + } + } + + abortSignal?.throwIfAborted(); + } catch (err) { + // For parallel requests, cancel them all if one fails + abortController.abort(); + throw err; + } +} + +/** + * @returns undefined for bucket uploads, CommitOutput otherwise + */ +export async function commit(params: CommitParams): Promise { + const iterator = commitIter(params); + const failedPaths: string[] = []; + let failedCount = 0; + let res = await iterator.next(); + while (!res.done) { + if (res.value.event === "fileProgress" && res.value.state === "error") { + failedCount++; + if (failedPaths.length < 5) { + failedPaths.push(res.value.path); + } + } + res = await iterator.next(); + } + if (failedCount > 0) { + throw new Error( + `Failed to upload ${failedCount} file(s): ${failedPaths.join(", ")}${failedCount > 5 ? "..." : ""}`, + ); + } + return res.value; +} + +async function convertOperationToNdJson(operation: CommitBlobOperation): Promise { + switch (operation.operation) { + case "addOrUpdate": { + // todo: handle LFS + return { + key: "file", + value: { + content: base64FromBytes(new Uint8Array(await operation.content.arrayBuffer())), + path: operation.path, + encoding: "base64", + }, + }; + } + // case "rename": { + // // todo: detect when remote file is already LFS, and in that case rename as LFS + // return { + // key: "file", + // value: { + // content: operation.content, + // path: operation.path, + // oldPath: operation.oldPath + // } + // }; + // } + case "delete": { + return { + key: "deletedFile", + value: { + path: operation.path, + }, + }; + } + case "edit": { + // Note: By the time we get here, splice operations should have been converted to addOrUpdate operations with SplicedBlob + // But we handle this case for completeness + throw new Error( + "Edit operations should be converted to addOrUpdate operations before reaching convertOperationToNdJson", + ); + } + default: + throw new TypeError("Unknown operation: " + (operation as { operation: string }).operation); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/copy-files.spec.ts b/node_modules/@huggingface/hub/src/lib/copy-files.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..69637694a75b2d4fb52ff5c5b86132a3c44f786b --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/copy-files.spec.ts @@ -0,0 +1,271 @@ +import { describe, it, expect } from "vitest"; +import { TEST_HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../test/consts"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { copyFile, copyFiles, copyFolder, relativeUnderFolder } from "./copy-files"; +import { listFiles } from "./list-files"; +import { commit } from "./commit"; + +describe("relativeUnderFolder", () => { + it("returns the basename when filePath equals folderPath (single-file folder)", () => { + expect(relativeUnderFolder("data/train.parquet", "data/train.parquet")).toBe("train.parquet"); + }); + + it("returns the path relative to the folder", () => { + expect(relativeUnderFolder("data/2024/train.parquet", "data")).toBe("2024/train.parquet"); + }); + + it("returns the filePath unchanged when folderPath is empty", () => { + expect(relativeUnderFolder("a/b/c.txt", "")).toBe("a/b/c.txt"); + }); + + it("throws when filePath is not under folderPath", () => { + expect(() => relativeUnderFolder("foo/bar", "baz")).toThrow("not inside folder"); + }); +}); + +describe("copyFiles (mocked)", () => { + it("rejects copy ops on a non-bucket destination", async () => { + await expect( + copyFiles({ + destination: { type: "model", name: "ns/repo" } as never, + files: [ + { + source: { repo: { type: "bucket", name: "ns/bucket" }, path: "file.bin" }, + destinationPath: "file.bin", + }, + ], + accessToken: TEST_ACCESS_TOKEN, + hubUrl: "https://example.invalid", + fetch: mockFetch({ + "/api/buckets/ns/bucket/paths-info": () => + jsonResponse([{ path: "file.bin", type: "file", size: 5, xetHash: "abc123" }]), + }), + }), + ).rejects.toThrow("'copy' operations are only supported when the destination repo is a bucket"); + }); + + it("throws when the source path is a folder", async () => { + await expect( + copyFiles({ + destination: { type: "bucket", name: "ns/dst" }, + files: [ + { + source: { repo: { type: "model", name: "ns/model" }, path: "data" }, + destinationPath: "data", + }, + ], + accessToken: TEST_ACCESS_TOKEN, + hubUrl: "https://example.invalid", + fetch: mockFetch({ + "/api/models/ns/model/paths-info": () => jsonResponse([{ path: "data", type: "directory", size: 0 }]), + }), + }), + ).rejects.toThrow("is a folder; use copyFolder()"); + }); + + it("throws a clear error when the source file is missing", async () => { + await expect( + copyFiles({ + destination: { type: "bucket", name: "ns/dst" }, + files: [ + { + source: { repo: { type: "model", name: "ns/model" }, path: "missing.txt" }, + destinationPath: "missing.txt", + }, + ], + accessToken: TEST_ACCESS_TOKEN, + hubUrl: "https://example.invalid", + fetch: mockFetch({ + "/api/models/ns/model/paths-info": () => jsonResponse([]), + }), + }), + ).rejects.toThrow("Source file not found"); + }); + + it("refuses to copy unmigrated LFS files and reports their size", async () => { + await expect( + copyFiles({ + destination: { type: "bucket", name: "ns/dst" }, + files: [ + { + source: { repo: { type: "model", name: "ns/model" }, path: "model.safetensors" }, + destinationPath: "model.safetensors", + }, + ], + accessToken: TEST_ACCESS_TOKEN, + hubUrl: "https://example.invalid", + fetch: mockFetch({ + "/api/models/ns/model/paths-info": () => + jsonResponse([ + { + path: "model.safetensors", + type: "file", + size: 5_300_000_000, + lfs: { oid: "deadbeef", size: 5_300_000_000, pointerSize: 134 }, + }, + ]), + }), + }), + ).rejects.toThrow(/LFS file\(s\).*'model\.safetensors' \(5\.30 GB\).*Migrate these files to xet/s); + }); +}); + +describe("copyFile / copyFiles / copyFolder (integration)", () => { + it("copies a single file from one bucket to another", async () => { + const srcBucketName = `${TEST_USER}/TEST-src-${insecureRandomString()}`; + const dstBucketName = `${TEST_USER}/TEST-dst-${insecureRandomString()}`; + + try { + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + repo: { name: srcBucketName, type: "bucket" }, + hubUrl: TEST_HUB_URL, + }); + + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + repo: { name: dstBucketName, type: "bucket" }, + hubUrl: TEST_HUB_URL, + }); + + await commit({ + repo: { type: "bucket", name: srcBucketName }, + operations: [ + { + operation: "addOrUpdate", + path: "test-file.txt", + content: new Blob(["hello world"]), + }, + ], + title: "Add test file", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + await copyFile({ + source: { repo: { type: "bucket", name: srcBucketName }, path: "test-file.txt" }, + destination: { repo: { type: "bucket", name: dstBucketName }, path: "test-file.txt" }, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + const files: string[] = []; + for await (const file of listFiles({ + repo: { type: "bucket", name: dstBucketName }, + recursive: true, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + })) { + if (file.type === "file") { + files.push(file.path); + } + } + + expect(files).toContain("test-file.txt"); + } finally { + await deleteRepo({ + repo: { name: srcBucketName, type: "bucket" }, + credentials: { accessToken: TEST_ACCESS_TOKEN }, + hubUrl: TEST_HUB_URL, + }).catch(() => {}); + + await deleteRepo({ + repo: { name: dstBucketName, type: "bucket" }, + credentials: { accessToken: TEST_ACCESS_TOKEN }, + hubUrl: TEST_HUB_URL, + }).catch(() => {}); + } + }); + + it("copies a folder from a model repo to a bucket", async () => { + const srcRepoName = `${TEST_USER}/TEST-repo-${insecureRandomString()}`; + const dstBucketName = `${TEST_USER}/TEST-dst-${insecureRandomString()}`; + + try { + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + repo: { name: srcRepoName, type: "model" }, + hubUrl: TEST_HUB_URL, + }); + + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + repo: { name: dstBucketName, type: "bucket" }, + hubUrl: TEST_HUB_URL, + }); + + await commit({ + repo: { type: "model", name: srcRepoName }, + operations: [ + { + operation: "addOrUpdate", + path: "config.json", + content: new Blob(['{"model_type": "test"}']), + }, + ], + title: "Add config", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + await copyFolder({ + source: { repo: { type: "model", name: srcRepoName } }, + destination: { + repo: { type: "bucket", name: dstBucketName }, + path: "models/test/", + }, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + const files: string[] = []; + for await (const file of listFiles({ + repo: { type: "bucket", name: dstBucketName }, + recursive: true, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + })) { + if (file.type === "file") { + files.push(file.path); + } + } + + expect(files).toContain("models/test/config.json"); + } finally { + await deleteRepo({ + repo: { name: srcRepoName, type: "model" }, + credentials: { accessToken: TEST_ACCESS_TOKEN }, + hubUrl: TEST_HUB_URL, + }).catch(() => {}); + + await deleteRepo({ + repo: { name: dstBucketName, type: "bucket" }, + credentials: { accessToken: TEST_ACCESS_TOKEN }, + hubUrl: TEST_HUB_URL, + }).catch(() => {}); + } + }); +}); + +function jsonResponse(body: unknown, init: ResponseInit = {}): Response { + return new Response(JSON.stringify(body), { + status: init.status ?? 200, + headers: { "Content-Type": "application/json", ...(init.headers ?? {}) }, + }); +} + +type MockHandler = (req: Request) => Response | Promise; + +function mockFetch(routes: Record): typeof fetch { + return async (input, init) => { + const req = new Request(input as RequestInfo, init); + const url = new URL(req.url); + const matched = Object.entries(routes).find(([prefix]) => url.pathname.startsWith(prefix)); + if (!matched) { + throw new Error(`Unexpected request to ${req.method} ${url.pathname}`); + } + return matched[1](req); + }; +} diff --git a/node_modules/@huggingface/hub/src/lib/copy-files.ts b/node_modules/@huggingface/hub/src/lib/copy-files.ts new file mode 100644 index 0000000000000000000000000000000000000000..479bf2a39a06ee1ccb3283bf9d4fbf6fe4e31336 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/copy-files.ts @@ -0,0 +1,598 @@ +import type { BucketDesignation, CredentialsParams, RepoDesignation, RepoId } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { formatBytes } from "../utils/formatBytes"; +import { promisesQueue } from "../utils/promisesQueue"; +import { toRepoId } from "../utils/toRepoId"; +import { eventToGenerator } from "../utils/eventToGenerator"; +import type { CommitOperation, CommitParams } from "./commit"; +import { commit } from "./commit"; +import { downloadFile } from "./download-file"; +import type { ListFileEntry } from "./list-files"; +import { listFiles } from "./list-files"; +import type { PathInfo } from "./paths-info"; +import { pathsInfo } from "./paths-info"; + +/** + * Progress events yielded by {@link copyFileIter} / {@link copyFilesIter} / {@link copyFolderIter}. + * + * Currently only `fileDownloaded` is emitted: one event per source file that had to be downloaded + * (small git-stored files that can't be copied server-side). Xet-backed files are copied + * server-side and do not produce events. + */ +export interface CopyProgressEvent { + event: "fileDownloaded"; + /** Source path of the file that was just downloaded. */ + path: string; + /** Number of files downloaded so far (including this one). */ + downloaded: number; + /** Total number of files that will be downloaded. */ + total: number; +} +const DOWNLOAD_CONCURRENCY = 5; +const PATHS_INFO_BATCH_SIZE = 100; +const MAX_REPORTED_LFS_PATHS = 5; + +/** + * Source location of a file in {@link copyFile} / {@link copyFiles} / {@link copyFolder}. + */ +export interface CopySource { + repo: RepoDesignation; + /** + * Path of the file (or folder, for {@link copyFolder}) inside the source repo. + * Leave empty in {@link copyFolder} to copy the whole repo. + */ + path: string; + /** + * Git revision to read the source from. Ignored for bucket sources. + * + * @default "main" + */ + revision?: string; +} + +/** + * Destination location for {@link copyFile} / {@link copyFolder}. + * + * The destination repo must be a bucket — server-side copy is currently only supported + * towards buckets. + */ +export interface CopyDestination { + repo: BucketDesignation; + /** + * Exact destination path within the destination bucket. For {@link copyFolder}, + * acts as a prefix; leave empty to copy under the bucket root. + */ + path: string; +} + +/** + * One file to copy in a {@link copyFiles} call. + */ +export interface CopyFilesEntry { + source: CopySource; + /** + * Exact path within the destination bucket. The bucket itself is shared with the + * other entries via the top-level {@link copyFiles} `destination` parameter. + */ + destinationPath: string; +} + +type SharedParams = { + hubUrl?: CommitParams["hubUrl"]; + fetch?: CommitParams["fetch"]; + abortSignal?: CommitParams["abortSignal"]; +} & Partial; + +/** + * Copy a single file from a source repo/bucket to the destination bucket. + * + * The copy is server-side (no data transfer) when the source file is xet-backed. + * For small non-xet repo files (e.g. `config.json`) the file is downloaded and + * re-uploaded to the destination bucket in the same commit. + * + * LFS pointer files that have not been migrated to xet are rejected up front + * (they would otherwise require downloading the full LFS blob). + * + * @example + * ```ts + * await copyFile({ + * source: { + * repo: { type: "model", name: "username/my-model" }, + * path: "model.safetensors", + * }, + * destination: { + * repo: { type: "bucket", name: "username/my-bucket" }, + * path: "models/my-model/model.safetensors", + * }, + * accessToken: "hf_...", + * }); + * ``` + */ +export function copyFile( + params: { + source: CopySource; + destination: CopyDestination; + } & SharedParams, +): Promise { + return copyFiles({ + ...(params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }), + destination: params.destination.repo, + files: [ + { + source: params.source, + destinationPath: params.destination.path, + }, + ], + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal, + }); +} + +/** + * Async-iterator variant of {@link copyFile} that yields {@link CopyProgressEvent}s while + * downloading non-xet source files (xet-backed files are copied server-side and do not + * emit events). See {@link copyFile} for the semantics. + * + * @example + * ```ts + * for await (const event of copyFileIter({ source, destination, accessToken })) { + * console.log(`downloaded ${event.path} (${event.downloaded}/${event.total})`); + * } + * ``` + */ +export function copyFileIter( + params: { + source: CopySource; + destination: CopyDestination; + } & SharedParams, +): AsyncGenerator { + return copyFilesIter({ + ...(params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }), + destination: params.destination.repo, + files: [ + { + source: params.source, + destinationPath: params.destination.path, + }, + ], + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal, + }); +} + +/** + * Copy multiple files (potentially from different source repos/buckets) to the destination + * bucket in a single commit. + * + * For xet-backed source files, the copy is performed server-side with no data transfer. + * For non-xet source files (typically small git-stored repo files), the file is + * downloaded and re-uploaded as part of the same commit. + * + * LFS pointer files that have not been migrated to xet are rejected up front. + * + * @example + * ```ts + * await copyFiles({ + * destination: { type: "bucket", name: "username/my-bucket" }, + * files: [ + * { + * source: { + * repo: { type: "bucket", name: "username/other-bucket" }, + * path: "data.bin", + * }, + * destinationPath: "data.bin", + * }, + * { + * source: { + * repo: { type: "model", name: "username/my-model" }, + * path: "model.safetensors", + * }, + * destinationPath: "models/my-model/model.safetensors", + * }, + * ], + * accessToken: "hf_...", + * }); + * ``` + */ +export async function copyFiles( + params: { + destination: BucketDesignation; + files: CopyFilesEntry[]; + } & SharedParams, +): Promise { + const iterator = copyFilesIter(params); + while (true) { + const res = await iterator.next(); + if (res.done) { + return undefined; + } + } +} + +/** + * Async-iterator variant of {@link copyFiles} that yields {@link CopyProgressEvent}s while + * downloading non-xet source files (xet-backed files are copied server-side and do not + * emit events). See {@link copyFiles} for the semantics. + */ +export async function* copyFilesIter( + params: { + destination: BucketDesignation; + files: CopyFilesEntry[]; + } & SharedParams, +): AsyncGenerator { + if (params.files.length === 0) { + return undefined; + } + + const operations = yield* resolveCopyOperationsIter(params, params.files); + + await commit({ + ...(params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }), + repo: params.destination, + operations, + title: "", + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal, + }); + return undefined; +} + +/** + * Copy a folder (recursively) from a source repo/bucket to the destination bucket + * in a single commit. + * + * Per-file paths are resolved relative to {@link CopySource.path}; the source folder + * itself is not preserved in the destination unless {@link CopyDestination.path} + * keeps it. + * + * @example + * ```ts + * // Copy an entire dataset under "datasets/my-dataset/" in the bucket + * await copyFolder({ + * source: { repo: { type: "dataset", name: "username/my-dataset" } }, + * destination: { + * repo: { type: "bucket", name: "username/my-bucket" }, + * path: "datasets/my-dataset/", + * }, + * accessToken: "hf_...", + * }); + * + * // Copy a subfolder + * await copyFolder({ + * source: { + * repo: { type: "bucket", name: "username/src-bucket" }, + * path: "models/", + * }, + * destination: { + * repo: { type: "bucket", name: "username/dst-bucket" }, + * path: "backup/", + * }, + * accessToken: "hf_...", + * }); + * ``` + */ +export async function copyFolder( + params: { + source: Omit & { path?: string }; + destination: Omit & { path?: string }; + } & SharedParams, +): Promise { + const iterator = copyFolderIter(params); + while (true) { + const res = await iterator.next(); + if (res.done) { + return undefined; + } + } +} + +/** + * Async-iterator variant of {@link copyFolder} that yields {@link CopyProgressEvent}s while + * downloading non-xet source files (xet-backed files are copied server-side and do not + * emit events). See {@link copyFolder} for the semantics. + */ +export async function* copyFolderIter( + params: { + source: Omit & { path?: string }; + destination: Omit & { path?: string }; + } & SharedParams, +): AsyncGenerator { + const accessToken = checkCredentials(params); + const sourceRepoId = toRepoId(params.source.repo); + const sourcePath = (params.source.path ?? "").replace(/\/+$/, ""); + const destinationPrefix = (params.destination.path ?? "").replace(/\/+$/, ""); + const sourceRevision = sourceRepoId.type === "bucket" ? undefined : (params.source.revision ?? "main"); + + const operations: CommitOperation[] = []; + const pendingDownloads: PendingDownload[] = []; + const lfsOffenders: Array<{ path: string; size: number }> = []; + + for await (const item of listFiles({ + repo: sourceRepoId, + path: sourcePath || undefined, + recursive: true, + revision: sourceRevision, + accessToken, + hubUrl: params.hubUrl, + fetch: params.fetch, + })) { + if (item.type !== "file") { + continue; + } + + const relPath = relativeUnderFolder(item.path, sourcePath); + const destPath = destinationPrefix ? `${destinationPrefix}/${relPath}` : relPath; + + switch (classifySourceFile(item)) { + case "copy": + operations.push({ + operation: "copy", + path: destPath, + sourceXetHash: item.xetHash as string, + sourceRepo: sourceRepoId, + }); + continue; + case "lfs": + lfsOffenders.push({ path: item.path, size: item.lfs?.size ?? item.size }); + continue; + case "download": + // Regular git-stored file (small): download + re-upload in the same commit. + pendingDownloads.push({ + index: operations.length, + repoId: sourceRepoId, + revision: sourceRevision, + sourcePath: item.path, + }); + operations.push({ + operation: "addOrUpdate", + path: destPath, + content: new Blob([]), + }); + continue; + } + } + + if (lfsOffenders.length > 0) { + throwUnmigratedLfsError(sourceRepoId, lfsOffenders); + } + + if (operations.length === 0) { + return undefined; + } + + yield* downloadAndFillBlobsIter({ + pendingDownloads, + operations, + accessToken, + hubUrl: params.hubUrl, + fetch: params.fetch, + }); + + await commit({ + ...(params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }), + repo: params.destination.repo, + operations, + title: "", + hubUrl: params.hubUrl, + fetch: params.fetch, + abortSignal: params.abortSignal, + }); + return undefined; +} + +/** + * Resolve a list of {@link CopyFilesEntry} entries into `CommitOperation`s, batching + * `pathsInfo` calls per source repo and parallelizing downloads for non-xet files. + * Yields one {@link CopyProgressEvent} per downloaded file. + */ +async function* resolveCopyOperationsIter( + shared: SharedParams, + files: CopyFilesEntry[], +): AsyncGenerator { + const accessToken = checkCredentials(shared); + + // Group files by (source repo, source revision) so we can batch pathsInfo calls. + const groups = new Map< + string, + { + repoId: RepoId; + revision: string | undefined; + entries: Array<{ index: number; file: CopyFilesEntry }>; + } + >(); + + for (let i = 0; i < files.length; i++) { + const file = files[i]; + const repoId = toRepoId(file.source.repo); + const revision = repoId.type === "bucket" ? undefined : (file.source.revision ?? "main"); + const key = `${repoId.type}\0${repoId.name}\0${revision ?? ""}`; + + let group = groups.get(key); + if (!group) { + group = { repoId, revision, entries: [] }; + groups.set(key, group); + } + group.entries.push({ index: i, file }); + } + + const operations: CommitOperation[] = new Array(files.length); + const pendingDownloads: PendingDownload[] = []; + + for (const group of groups.values()) { + const paths = group.entries.map((e) => e.file.source.path); + + const infos: Awaited> = []; + for (let offset = 0; offset < paths.length; offset += PATHS_INFO_BATCH_SIZE) { + const slice = paths.slice(offset, offset + PATHS_INFO_BATCH_SIZE); + const res = await pathsInfo({ + repo: group.repoId, + paths: slice, + revision: group.revision, + accessToken, + hubUrl: shared.hubUrl, + fetch: shared.fetch, + }); + infos.push(...res); + } + + const infoByPath = new Map(infos.map((i) => [i.path, i])); + const lfsOffenders: Array<{ path: string; size: number }> = []; + + for (const { index, file } of group.entries) { + const info = infoByPath.get(file.source.path); + if (!info) { + throw new Error(`Source file not found: '${file.source.path}' in ${group.repoId.type}s/${group.repoId.name}`); + } + if (info.type !== "file") { + throw new Error( + `Source path '${file.source.path}' in ${group.repoId.type}s/${group.repoId.name} is a folder; use copyFolder() instead.`, + ); + } + + switch (classifySourceFile(info)) { + case "copy": + operations[index] = { + operation: "copy", + path: file.destinationPath, + sourceXetHash: info.xetHash as string, + sourceRepo: group.repoId, + }; + continue; + case "lfs": + lfsOffenders.push({ path: file.source.path, size: info.lfs?.size ?? info.size }); + continue; + case "download": + pendingDownloads.push({ + index, + repoId: group.repoId, + revision: group.revision, + sourcePath: file.source.path, + }); + operations[index] = { + operation: "addOrUpdate", + path: file.destinationPath, + content: new Blob([]), + }; + continue; + } + } + + if (lfsOffenders.length > 0) { + throwUnmigratedLfsError(group.repoId, lfsOffenders); + } + } + + yield* downloadAndFillBlobsIter({ + pendingDownloads, + operations, + accessToken, + hubUrl: shared.hubUrl, + fetch: shared.fetch, + }); + + return operations; +} + +interface PendingDownload { + index: number; + repoId: RepoId; + revision: string | undefined; + sourcePath: string; +} + +/** + * Download all `pendingDownloads` in parallel and fill the matching `addOrUpdate` + * placeholder ops in `operations` with the downloaded blob. Yields one + * {@link CopyProgressEvent} per file as it completes. No-op if the list is empty. + */ +function downloadAndFillBlobsIter(args: { + pendingDownloads: PendingDownload[]; + operations: CommitOperation[]; + accessToken: string | undefined; + hubUrl: string | undefined; + fetch: typeof fetch | undefined; +}): AsyncGenerator { + const total = args.pendingDownloads.length; + return eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + if (total === 0) { + returnCallback(); + return; + } + let downloaded = 0; + promisesQueue( + args.pendingDownloads.map(({ index, repoId, revision, sourcePath }) => async () => { + const blob = await downloadFile({ + repo: repoId, + path: sourcePath, + revision, + accessToken: args.accessToken, + hubUrl: args.hubUrl, + fetch: args.fetch, + }); + if (!blob) { + throw new Error(`Failed to download '${sourcePath}' from ${repoId.type}s/${repoId.name}`); + } + const op = args.operations[index]; + if (op.operation !== "addOrUpdate") { + throw new Error("Internal: expected addOrUpdate placeholder operation"); + } + op.content = blob; + downloaded++; + yieldCallback({ event: "fileDownloaded", path: sourcePath, downloaded, total }); + }), + DOWNLOAD_CONCURRENCY, + ).then( + () => returnCallback(), + (err) => rejectCallback(err), + ); + }); +} + +/** + * Compute the path of `filePath` relative to `folderPath`. Used to map source paths + * under a folder being copied to destination paths under the new prefix. + */ +export function relativeUnderFolder(filePath: string, folderPath: string): string { + if (!folderPath) { + return filePath; + } + if (filePath === folderPath) { + return filePath.split("/").pop() ?? filePath; + } + if (filePath.startsWith(folderPath + "/")) { + return filePath.slice(folderPath.length + 1); + } + throw new Error(`Path '${filePath}' is not inside folder '${folderPath}'`); +} + +/** + * Decide how to handle a source file in the copy pipeline: + * - `"copy"`: xet-backed, can be copied server-side. + * - `"download"`: regular git-stored file, safe to download + re-upload. + * - `"lfs"`: LFS pointer file that has not been migrated to xet. We refuse to copy these + * because they can be arbitrarily large; the caller should migrate them to xet first. + */ +function classifySourceFile(file: ListFileEntry | PathInfo): "copy" | "download" | "lfs" { + if (file.xetHash) { + return "copy"; + } + if (file.lfs) { + return "lfs"; + } + return "download"; +} + +function throwUnmigratedLfsError(repoId: RepoId, entries: Array<{ path: string; size: number }>): never { + const head = entries + .slice(0, MAX_REPORTED_LFS_PATHS) + .map((e) => `'${e.path}' (${formatBytes(e.size)})`) + .join(", "); + const more = entries.length > MAX_REPORTED_LFS_PATHS ? ` (and ${entries.length - MAX_REPORTED_LFS_PATHS} more)` : ""; + throw new Error( + `Cannot copy ${entries.length} LFS file(s) from ${repoId.type}s/${repoId.name} that have not been migrated to xet: ${head}${more}. ` + + `Migrate these files to xet before copying.`, + ); +} diff --git a/node_modules/@huggingface/hub/src/lib/count-commits.spec.ts b/node_modules/@huggingface/hub/src/lib/count-commits.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..f60754789b7a1b6f1c99765e9fa5d77689478eb8 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/count-commits.spec.ts @@ -0,0 +1,16 @@ +import { assert, it, describe } from "vitest"; +import { countCommits } from "./count-commits"; + +describe("countCommits", () => { + it("should fetch paginated commits from the repo", async () => { + const count = await countCommits({ + repo: { + name: "openai-community/gpt2", + type: "model", + }, + revision: "607a30d783dfa663caf39e06633721c8d4cfcd7e", + }); + + assert.equal(count, 26); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/count-commits.ts b/node_modules/@huggingface/hub/src/lib/count-commits.ts new file mode 100644 index 0000000000000000000000000000000000000000..cdfad30740287de0451543ebf6b28c2ad60de1f2 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/count-commits.ts @@ -0,0 +1,35 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { toRepoId } from "../utils/toRepoId"; + +export async function countCommits( + params: { + repo: RepoDesignation; + /** + * Revision to list commits from. Defaults to the default branch. + */ + revision?: string; + hubUrl?: string; + fetch?: typeof fetch; + } & Partial, +): Promise { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + + // Could upgrade to 1000 commits per page + const url: string | undefined = `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commits/${ + params.revision ?? "main" + }?limit=1`; + + const res: Response = await (params.fetch ?? fetch)(url, { + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {}, + }); + + if (!res.ok) { + throw await createApiError(res); + } + + return parseInt(res.headers.get("x-total-count") ?? "0", 10); +} diff --git a/node_modules/@huggingface/hub/src/lib/create-branch.spec.ts b/node_modules/@huggingface/hub/src/lib/create-branch.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..b616fb4cebfd888a23c33bfe3a7b7334d575fd2d --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/create-branch.spec.ts @@ -0,0 +1,159 @@ +import { assert, it, describe } from "vitest"; +import { TEST_ACCESS_TOKEN, TEST_HUB_URL, TEST_USER } from "../test/consts"; +import type { RepoId } from "../types/public"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { createBranch } from "./create-branch"; +import { uploadFile } from "./upload-file"; +import { downloadFile } from "./download-file"; + +describe("createBranch", () => { + it("should create a new branch from the default branch", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo, + }); + + await uploadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + file: { + path: "file.txt", + content: new Blob(["file content"]), + }, + }); + + await createBranch({ + repo, + branch: "new-branch", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + const content = await downloadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + path: "file.txt", + revision: "new-branch", + }); + + assert.equal(await content?.text(), "file content"); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); + + it("should create an empty branch", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo, + }); + + await uploadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + file: { + path: "file.txt", + content: new Blob(["file content"]), + }, + }); + + await createBranch({ + repo, + branch: "empty-branch", + empty: true, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + const content = await downloadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + path: "file.txt", + revision: "empty-branch", + }); + + assert.equal(content, null); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); + + it("should overwrite an existing branch", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo, + }); + + await uploadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + file: { + path: "file.txt", + content: new Blob(["file content"]), + }, + }); + + await createBranch({ + repo, + branch: "overwrite-branch", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + await createBranch({ + repo, + branch: "overwrite-branch", + overwrite: true, + empty: true, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + const content = await downloadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + path: "file.txt", + revision: "overwrite-branch", + }); + + assert.equal(content, null); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/create-branch.ts b/node_modules/@huggingface/hub/src/lib/create-branch.ts new file mode 100644 index 0000000000000000000000000000000000000000..41dd204ee38e26d2d2d87b10e98dd85be2eba66d --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/create-branch.ts @@ -0,0 +1,54 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { AccessToken, RepoDesignation } from "../types/public"; +import { toRepoId } from "../utils/toRepoId"; + +export async function createBranch(params: { + repo: RepoDesignation; + /** + * Revision to create the branch from. Defaults to the default branch. + * + * Use empty: true to create an empty branch. + */ + revision?: string; + hubUrl?: string; + accessToken?: AccessToken; + fetch?: typeof fetch; + /** + * The name of the branch to create + */ + branch: string; + /** + * Use this to create an empty branch, with no commits. + */ + empty?: boolean; + /** + * Use this to overwrite the branch if it already exists. + * + * If you only specify `overwrite` and no `revision`/`empty`, and the branch already exists, it will be a no-op. + */ + overwrite?: boolean; +}): Promise { + const repoId = toRepoId(params.repo); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(params.accessToken && { + Authorization: `Bearer ${params.accessToken}`, + }), + }, + body: JSON.stringify({ + startingPoint: params.revision, + ...(params.empty && { emptyBranch: true }), + overwrite: params.overwrite, + }), + }, + ); + + if (!res.ok) { + throw await createApiError(res); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/create-collection.spec.ts b/node_modules/@huggingface/hub/src/lib/create-collection.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..bf31e59ed915b9681797da5d81558eed1a806f82 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/create-collection.spec.ts @@ -0,0 +1,38 @@ +import { it, describe, expect } from "vitest"; + +import { TEST_HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../test/consts"; +import { createCollection } from "./create-collection"; +import { deleteCollection } from "./delete-collection"; + +describe("createCollection", () => { + it("should create a collection", async () => { + let slug: string = ""; + const randomString = crypto.randomUUID(); + const title = `Test Collection ${randomString}`; + + try { + const result = await createCollection({ + collection: { + title, + namespace: TEST_USER, + description: "This is a test collection", + private: false, + }, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + expect(result.slug.startsWith(`${TEST_USER}/test-collection-${randomString}`)).toBe(true); + + slug = result.slug; + } finally { + if (slug) { + await deleteCollection({ + slug, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + } + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/create-collection.ts b/node_modules/@huggingface/hub/src/lib/create-collection.ts new file mode 100644 index 0000000000000000000000000000000000000000..97f1a626d99127414d2e94da9e812e60cde6cdef --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/create-collection.ts @@ -0,0 +1,35 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiCreateCollectionPayload } from "../types/api/api-create-collection"; +import type { CredentialsParams } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; + +export async function createCollection( + params: { + collection: ApiCreateCollectionPayload; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise<{ slug: string }> { + const accessToken = checkCredentials(params); + + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/collections`, { + method: "POST", + body: JSON.stringify(params.collection), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!res.ok) { + throw await createApiError(res); + } + + const output = await res.json(); + + return { slug: output.slug }; +} diff --git a/node_modules/@huggingface/hub/src/lib/create-repo.spec.ts b/node_modules/@huggingface/hub/src/lib/create-repo.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..f91e398e040c7714a75cb148f3571e81ecf745c1 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/create-repo.spec.ts @@ -0,0 +1,106 @@ +import { assert, it, describe, expect } from "vitest"; + +import { TEST_HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../test/consts"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { downloadFile } from "./download-file"; + +describe("createRepo", () => { + it("should create a repo", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + + const result = await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + repo: { + name: repoName, + type: "model", + }, + hubUrl: TEST_HUB_URL, + files: [{ path: ".gitattributes", content: new Blob(["*.html filter=lfs diff=lfs merge=lfs -text"]) }], + }); + + expect(result).toEqual({ + repoUrl: `${TEST_HUB_URL}/${repoName}`, + id: expect.any(String), + }); + + const content = await downloadFile({ + repo: { + name: repoName, + type: "model", + }, + path: ".gitattributes", + hubUrl: TEST_HUB_URL, + }); + + assert(content); + assert.strictEqual(await content.text(), "*.html filter=lfs diff=lfs merge=lfs -text"); + + await deleteRepo({ + repo: { + name: repoName, + type: "model", + }, + credentials: { accessToken: TEST_ACCESS_TOKEN }, + hubUrl: TEST_HUB_URL, + }); + }); + + it("should throw a client error when trying to create a repo without a fully-qualified name", async () => { + const tryCreate = createRepo({ + repo: { name: "canonical", type: "model" }, + credentials: { accessToken: TEST_ACCESS_TOKEN }, + hubUrl: TEST_HUB_URL, + }); + + await expect(tryCreate).rejects.toBeInstanceOf(TypeError); + }); + + it("should create a model with a string as name", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + + const result = await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo: repoName, + files: [{ path: ".gitattributes", content: new Blob(["*.html filter=lfs diff=lfs merge=lfs -text"]) }], + }); + + expect(result).toEqual({ + repoUrl: `${TEST_HUB_URL}/${repoName}`, + id: expect.any(String), + }); + + await deleteRepo({ + repo: { + name: repoName, + type: "model", + }, + hubUrl: TEST_HUB_URL, + credentials: { accessToken: TEST_ACCESS_TOKEN }, + }); + }); + + it("should create a dataset with a string as name", async () => { + const repoName = `datasets/${TEST_USER}/TEST-${insecureRandomString()}`; + + const result = await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo: repoName, + files: [{ path: ".gitattributes", content: new Blob(["*.html filter=lfs diff=lfs merge=lfs -text"]) }], + }); + + expect(result).toEqual({ + repoUrl: `${TEST_HUB_URL}/${repoName}`, + id: expect.any(String), + }); + + await deleteRepo({ + repo: repoName, + hubUrl: TEST_HUB_URL, + credentials: { accessToken: TEST_ACCESS_TOKEN }, + }); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/create-repo.ts b/node_modules/@huggingface/hub/src/lib/create-repo.ts new file mode 100644 index 0000000000000000000000000000000000000000..88358227e24eee49ca7ffad3ce7a4426c365a79d --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/create-repo.ts @@ -0,0 +1,102 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiCreateRepoPayload } from "../types/api/api-create-repo"; +import type { CredentialsParams, RepoDesignation, SpaceSdk } from "../types/public"; +import { base64FromBytes } from "../utils/base64FromBytes"; +import { checkCredentials } from "../utils/checkCredentials"; +import { toRepoId } from "../utils/toRepoId"; + +export async function createRepo( + params: { + repo: RepoDesignation; + /** + * If unset, will follow the organization's default setting. (typically public, except for some Enterprise organizations) + */ + visibility?: "public" | "private" | "protected"; + /** + * @deprecated Use {@link visibility} instead. + */ + private?: boolean; + resourceGroupId?: string; + /** + * Does not work for buckets + */ + license?: string; + /** + * Only a few lightweight files are supported at repo creation - and not for buckets + */ + files?: Array<{ content: ArrayBuffer | Blob; path: string }>; + /** @required for when {@link repo.type} === "space" */ + sdk?: SpaceSdk; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): Promise<{ repoUrl: string; id: string }> { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const [namespace, repoName] = repoId.name.split("/"); + const visibility = + params.visibility ?? (params.private !== undefined ? (params.private ? "private" : "public") : undefined); + + if (!namespace || !repoName) { + throw new TypeError( + `"${repoId.name}" is not a fully qualified repo name. It should be of the form "{namespace}/{repoName}".`, + ); + } + + const res = + repoId.type === "bucket" + ? await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/buckets/${namespace}/${repoName}`, { + method: "POST", + body: JSON.stringify({ + visibility, + resourceGroupId: params.resourceGroupId, + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }) + : await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/repos/create`, { + method: "POST", + body: JSON.stringify({ + name: repoName, + visibility, + organization: namespace, + resourceGroupId: params.resourceGroupId, + license: params.license, + ...(repoId.type === "space" + ? { + type: "space", + sdk: params.sdk ?? "static", + } + : { + type: repoId.type, + }), + files: params.files + ? await Promise.all( + params.files.map(async (file) => ({ + encoding: "base64", + path: file.path, + content: base64FromBytes( + new Uint8Array(file.content instanceof Blob ? await file.content.arrayBuffer() : file.content), + ), + })), + ) + : undefined, + } satisfies ApiCreateRepoPayload), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!res.ok) { + throw await createApiError(res); + } + const output = await res.json(); + return { repoUrl: output.url, id: output.id }; +} diff --git a/node_modules/@huggingface/hub/src/lib/dataset-info.spec.ts b/node_modules/@huggingface/hub/src/lib/dataset-info.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..ae235e5e83a308a19c0e4f60692651424d02a902 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/dataset-info.spec.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { datasetInfo } from "./dataset-info"; +import type { DatasetEntry } from "./list-datasets"; +import type { ApiDatasetInfo } from "../types/api/api-dataset"; + +describe("datasetInfo", () => { + it("should return the dataset info", async () => { + const info = await datasetInfo({ + name: "nyu-mll/glue", + }); + expect(info).toEqual({ + id: "621ffdd236468d709f181e3f", + downloads: expect.any(Number), + gated: false, + name: "nyu-mll/glue", + updatedAt: expect.any(Date), + likes: expect.any(Number), + private: false, + }); + }); + + it("should return the dataset info with author", async () => { + const info: DatasetEntry & Pick = await datasetInfo({ + name: "nyu-mll/glue", + additionalFields: ["author"], + }); + expect(info).toEqual({ + id: "621ffdd236468d709f181e3f", + downloads: expect.any(Number), + gated: false, + name: "nyu-mll/glue", + updatedAt: expect.any(Date), + likes: expect.any(Number), + private: false, + author: "nyu-mll", + }); + }); + + it("should return the dataset info for a specific revision", async () => { + const info: DatasetEntry & Pick = await datasetInfo({ + name: "nyu-mll/glue", + revision: "cb2099c76426ff97da7aa591cbd317d91fb5fcb7", + additionalFields: ["sha"], + }); + expect(info).toEqual({ + id: "621ffdd236468d709f181e3f", + downloads: expect.any(Number), + gated: false, + name: "nyu-mll/glue", + updatedAt: expect.any(Date), + likes: expect.any(Number), + private: false, + sha: "cb2099c76426ff97da7aa591cbd317d91fb5fcb7", + }); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/dataset-info.ts b/node_modules/@huggingface/hub/src/lib/dataset-info.ts new file mode 100644 index 0000000000000000000000000000000000000000..fe34334d69c42caaa9f07b907acac1e6a849bf8e --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/dataset-info.ts @@ -0,0 +1,60 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiDatasetInfo } from "../types/api/api-dataset"; +import type { CredentialsParams } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { pick } from "../utils/pick"; +import { type DATASET_EXPANDABLE_KEYS, DATASET_EXPAND_KEYS, type DatasetEntry } from "./list-datasets"; + +export async function datasetInfo< + const T extends Exclude<(typeof DATASET_EXPANDABLE_KEYS)[number], (typeof DATASET_EXPAND_KEYS)[number]> = never, +>( + params: { + name: string; + hubUrl?: string; + additionalFields?: T[]; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + */ + revision?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise> { + const accessToken = params && checkCredentials(params); + + const search = new URLSearchParams([ + ...DATASET_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), + ...(params?.additionalFields?.map((val) => ["expand", val] satisfies [string, string]) ?? []), + ]).toString(); + + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/datasets/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD", + )}?${search.toString()}`, + { + headers: { + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }, + ); + + if (!response.ok) { + throw await createApiError(response); + } + + const data = await response.json(); + + return { + ...(params?.additionalFields && pick(data, params.additionalFields)), + id: data._id, + name: data.id, + private: data.private, + downloads: data.downloads, + likes: data.likes, + gated: data.gated, + updatedAt: new Date(data.lastModified), + } as DatasetEntry & Pick; +} diff --git a/node_modules/@huggingface/hub/src/lib/delete-branch.spec.ts b/node_modules/@huggingface/hub/src/lib/delete-branch.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..dcd253214b671d2393a343bd74447495442e659f --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/delete-branch.spec.ts @@ -0,0 +1,43 @@ +import { it, describe } from "vitest"; +import { TEST_ACCESS_TOKEN, TEST_HUB_URL, TEST_USER } from "../test/consts"; +import type { RepoId } from "../types/public"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { createBranch } from "./create-branch"; +import { deleteBranch } from "./delete-branch"; + +describe("deleteBranch", () => { + it("should delete an existing branch", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo, + }); + + await createBranch({ + repo, + branch: "branch-to-delete", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + await deleteBranch({ + repo, + branch: "branch-to-delete", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/delete-branch.ts b/node_modules/@huggingface/hub/src/lib/delete-branch.ts new file mode 100644 index 0000000000000000000000000000000000000000..76072f2fe2f852f26565dc123c0507f0e9c445bb --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/delete-branch.ts @@ -0,0 +1,32 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { AccessToken, RepoDesignation } from "../types/public"; +import { toRepoId } from "../utils/toRepoId"; + +export async function deleteBranch(params: { + repo: RepoDesignation; + /** + * The name of the branch to delete + */ + branch: string; + hubUrl?: string; + accessToken?: AccessToken; + fetch?: typeof fetch; +}): Promise { + const repoId = toRepoId(params.repo); + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, + { + method: "DELETE", + headers: { + ...(params.accessToken && { + Authorization: `Bearer ${params.accessToken}`, + }), + }, + }, + ); + + if (!res.ok) { + throw await createApiError(res); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/delete-collection-item.ts b/node_modules/@huggingface/hub/src/lib/delete-collection-item.ts new file mode 100644 index 0000000000000000000000000000000000000000..4678514f1a92ae20a0b3963d00a5babdafd10bb7 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/delete-collection-item.ts @@ -0,0 +1,40 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { CredentialsParams } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; + +export async function deleteCollectionItem( + params: { + /** + * The slug of the collection to delete the item from. + */ + slug: string; + /** + * The item object id which is different from the repo_id/paper_id provided when adding the item to the collection. + * This should be the _id property of the item. + */ + itemId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise { + const accessToken = checkCredentials(params); + + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/collections/${params.slug}/items/${params.itemId}`, + { + method: "DELETE", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }, + ); + + if (!res.ok) { + throw await createApiError(res); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/delete-collection.ts b/node_modules/@huggingface/hub/src/lib/delete-collection.ts new file mode 100644 index 0000000000000000000000000000000000000000..1bffe064ac43afe20db7f0b58a03da0e865fd2cb --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/delete-collection.ts @@ -0,0 +1,36 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { CredentialsParams } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; + +export async function deleteCollection( + params: { + /** + * The slug of the collection to delete. + */ + slug: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise { + if (!params.slug) { + throw new TypeError("slug is required"); + } + + const accessToken = checkCredentials(params); + + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/collections/${params.slug}`, { + method: "DELETE", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!res.ok) { + throw await createApiError(res); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/delete-file.spec.ts b/node_modules/@huggingface/hub/src/lib/delete-file.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..67ce0d2dfe1f3169c05bbd66568ade9d73167d8f --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/delete-file.spec.ts @@ -0,0 +1,65 @@ +import { assert, it, describe, expect } from "vitest"; + +import { TEST_ACCESS_TOKEN, TEST_HUB_URL, TEST_USER } from "../test/consts"; +import type { RepoId } from "../types/public"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { deleteFile } from "./delete-file"; +import { downloadFile } from "./download-file"; + +describe("deleteFile", () => { + it("should delete a file", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + const result = await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo, + files: [ + { path: "file1", content: new Blob(["file1"]) }, + { path: "file2", content: new Blob(["file2"]) }, + ], + }); + + expect(result).toEqual({ + repoUrl: `${TEST_HUB_URL}/${repoName}`, + id: expect.any(String), + }); + + let content = await downloadFile({ + hubUrl: TEST_HUB_URL, + repo, + path: "file1", + }); + + assert.strictEqual(await content?.text(), "file1"); + + await deleteFile({ path: "file1", repo, accessToken: TEST_ACCESS_TOKEN, hubUrl: TEST_HUB_URL }); + + content = await downloadFile({ + repo, + path: "file1", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual(content, null); + + content = await downloadFile({ + repo, + path: "file2", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual(await content?.text(), "file2"); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/delete-file.ts b/node_modules/@huggingface/hub/src/lib/delete-file.ts new file mode 100644 index 0000000000000000000000000000000000000000..36c550dfc88e2afdbaf14c5402290c8e859353e5 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/delete-file.ts @@ -0,0 +1,35 @@ +import type { CredentialsParams } from "../types/public"; +import type { CommitOutput, CommitParams } from "./commit"; +import { commit } from "./commit"; + +export function deleteFile( + params: { + repo: CommitParams["repo"]; + path: string; + commitTitle?: CommitParams["title"]; + commitDescription?: CommitParams["description"]; + hubUrl?: CommitParams["hubUrl"]; + fetch?: CommitParams["fetch"]; + branch?: CommitParams["branch"]; + isPullRequest?: CommitParams["isPullRequest"]; + parentCommit?: CommitParams["parentCommit"]; + } & CredentialsParams, +): Promise { + return commit({ + ...(params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }), + repo: params.repo, + operations: [ + { + operation: "delete", + path: params.path, + }, + ], + title: params.commitTitle ?? `Delete ${params.path}`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch, + }); +} diff --git a/node_modules/@huggingface/hub/src/lib/delete-files.spec.ts b/node_modules/@huggingface/hub/src/lib/delete-files.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..4d3f0a993c2de26ef9fa4669698aed4b6fcf344f --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/delete-files.spec.ts @@ -0,0 +1,82 @@ +import { assert, it, describe, expect } from "vitest"; + +import { TEST_HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../test/consts"; +import type { RepoId } from "../types/public"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { deleteFiles } from "./delete-files"; +import { downloadFile } from "./download-file"; + +describe("deleteFiles", () => { + it("should delete multiple files", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + const result = await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + repo, + files: [ + { path: "file1", content: new Blob(["file1"]) }, + { path: "file2", content: new Blob(["file2"]) }, + { path: "file3", content: new Blob(["file3"]) }, + ], + hubUrl: TEST_HUB_URL, + }); + + expect(result).toEqual({ + repoUrl: `${TEST_HUB_URL}/${repoName}`, + id: expect.any(String), + }); + + let content = await downloadFile({ + repo, + path: "file1", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual(await content?.text(), "file1"); + + content = await downloadFile({ + repo, + path: "file2", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual(await content?.text(), "file2"); + + await deleteFiles({ paths: ["file1", "file2"], repo, accessToken: TEST_ACCESS_TOKEN, hubUrl: TEST_HUB_URL }); + + content = await downloadFile({ + repo, + path: "file1", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual(content, null); + + content = await downloadFile({ + repo, + path: "file2", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual(content, null); + + content = await downloadFile({ + repo, + path: "file3", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual(await content?.text(), "file3"); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/delete-files.ts b/node_modules/@huggingface/hub/src/lib/delete-files.ts new file mode 100644 index 0000000000000000000000000000000000000000..6242c4a99b5f14b30521c8efcc117e5223088b55 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/delete-files.ts @@ -0,0 +1,33 @@ +import type { CredentialsParams } from "../types/public"; +import type { CommitOutput, CommitParams } from "./commit"; +import { commit } from "./commit"; + +export function deleteFiles( + params: { + repo: CommitParams["repo"]; + paths: string[]; + commitTitle?: CommitParams["title"]; + commitDescription?: CommitParams["description"]; + hubUrl?: CommitParams["hubUrl"]; + branch?: CommitParams["branch"]; + isPullRequest?: CommitParams["isPullRequest"]; + parentCommit?: CommitParams["parentCommit"]; + fetch?: CommitParams["fetch"]; + } & CredentialsParams, +): Promise { + return commit({ + ...(params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }), + repo: params.repo, + operations: params.paths.map((path) => ({ + operation: "delete", + path, + })), + title: params.commitTitle ?? `Deletes ${params.paths.length} files`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch, + }); +} diff --git a/node_modules/@huggingface/hub/src/lib/delete-repo.ts b/node_modules/@huggingface/hub/src/lib/delete-repo.ts new file mode 100644 index 0000000000000000000000000000000000000000..efa2fe985b1bfdbf4b040dc10099cd394eae7167 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/delete-repo.ts @@ -0,0 +1,46 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { toRepoId } from "../utils/toRepoId"; + +export async function deleteRepo( + params: { + repo: RepoDesignation; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): Promise { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const [namespace, repoName] = repoId.name.split("/"); + + const res = + repoId.type === "bucket" + ? await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/buckets/${namespace}/${repoName}`, { + method: "DELETE", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }) + : await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/repos/delete`, { + method: "DELETE", + body: JSON.stringify({ + name: repoName, + organization: namespace, + type: repoId.type, + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!res.ok) { + throw await createApiError(res); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/download-file-to-cache-dir.spec.ts b/node_modules/@huggingface/hub/src/lib/download-file-to-cache-dir.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..22213418dc966378e02356d11e5bda68485b47db --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/download-file-to-cache-dir.spec.ts @@ -0,0 +1,306 @@ +import { expect, test, describe, vi, beforeEach } from "vitest"; +import type { RepoDesignation, RepoId } from "../types/public"; +import { dirname, join } from "node:path"; +import { lstat, mkdir, stat, symlink, rename } from "node:fs/promises"; +import { pathsInfo } from "./paths-info"; +import { createWriteStream, type Stats } from "node:fs"; +import { getHFHubCachePath, getRepoFolderName } from "./cache-management"; +import { toRepoId } from "../utils/toRepoId"; +import { downloadFileToCacheDir } from "./download-file-to-cache-dir"; +import { createSymlink } from "../utils/symlink"; + +vi.mock("node:fs/promises", () => ({ + rename: vi.fn(), + symlink: vi.fn(), + lstat: vi.fn(), + mkdir: vi.fn(), + stat: vi.fn(), +})); + +vi.mock("node:fs", () => ({ + createWriteStream: vi.fn(), +})); + +vi.mock("./paths-info", () => ({ + pathsInfo: vi.fn(), +})); + +vi.mock("../utils/symlink", () => ({ + createSymlink: vi.fn(), +})); + +const DUMMY_REPO: RepoId = { + name: "hello-world", + type: "model", +}; + +const DUMMY_ETAG = "dummy-etag"; + +// utility test method to get blob file path +function _getBlobFile(params: { + repo: RepoDesignation; + etag: string; + cacheDir?: string; // default to {@link getHFHubCache} +}) { + return join(params.cacheDir ?? getHFHubCachePath(), getRepoFolderName(toRepoId(params.repo)), "blobs", params.etag); +} + +// utility test method to get snapshot file path +function _getSnapshotFile(params: { + repo: RepoDesignation; + path: string; + revision: string; + cacheDir?: string; // default to {@link getHFHubCache} +}) { + return join( + params.cacheDir ?? getHFHubCachePath(), + getRepoFolderName(toRepoId(params.repo)), + "snapshots", + params.revision, + params.path, + ); +} + +describe("downloadFileToCacheDir", () => { + const fetchMock: typeof fetch = vi.fn(); + beforeEach(() => { + vi.resetAllMocks(); + // mock 200 request + vi.mocked(fetchMock).mockResolvedValue( + new Response("dummy-body", { + status: 200, + headers: { + etag: DUMMY_ETAG, + "Content-Range": "bytes 0-54/55", + }, + }), + ); + + // prevent to use caching + vi.mocked(stat).mockRejectedValue(new Error("Do not exists")); + vi.mocked(lstat).mockRejectedValue(new Error("Do not exists")); + }); + + test("should throw an error if fileDownloadInfo return nothing", async () => { + await expect(async () => { + await downloadFileToCacheDir({ + repo: DUMMY_REPO, + path: "/README.md", + fetch: fetchMock, + }); + }).rejects.toThrowError("cannot get path info for /README.md"); + + expect(pathsInfo).toHaveBeenCalledWith( + expect.objectContaining({ + repo: DUMMY_REPO, + paths: ["/README.md"], + fetch: fetchMock, + }), + ); + }); + + test("existing symlinked and blob should not re-download it", async () => { + // ///snapshots/README.md + const expectPointer = _getSnapshotFile({ + repo: DUMMY_REPO, + path: "/README.md", + revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", + }); + // stat ensure a symlink and the pointed file exists + vi.mocked(stat).mockResolvedValue({} as Stats); // prevent default mocked reject + + const output = await downloadFileToCacheDir({ + repo: DUMMY_REPO, + path: "/README.md", + fetch: fetchMock, + revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", + }); + + expect(stat).toHaveBeenCalledOnce(); + // Get call argument for stat + const starArg = vi.mocked(stat).mock.calls[0][0]; + + expect(starArg).toBe(expectPointer); + expect(fetchMock).not.toHaveBeenCalledWith(); + + expect(output).toBe(expectPointer); + }); + + test("existing symlinked and blob with default revision should not re-download it", async () => { + // ///snapshots/README.md + const expectPointer = _getSnapshotFile({ + repo: DUMMY_REPO, + path: "/README.md", + revision: "main", + }); + // stat ensure a symlink and the pointed file exists + vi.mocked(stat).mockResolvedValue({} as Stats); // prevent default mocked reject + vi.mocked(lstat).mockResolvedValue({} as Stats); + vi.mocked(pathsInfo).mockResolvedValue([ + { + oid: DUMMY_ETAG, + size: 55, + path: "README.md", + type: "file", + lastCommit: { + date: new Date(), + id: "main", + title: "Commit msg", + }, + }, + ]); + + const output = await downloadFileToCacheDir({ + repo: DUMMY_REPO, + path: "/README.md", + fetch: fetchMock, + }); + + expect(stat).toHaveBeenCalledOnce(); + expect(symlink).not.toHaveBeenCalledOnce(); + // Get call argument for stat + const starArg = vi.mocked(stat).mock.calls[0][0]; + + expect(starArg).toBe(expectPointer); + expect(fetchMock).not.toHaveBeenCalledWith(); + + expect(output).toBe(expectPointer); + }); + + test("existing blob should only create the symlink", async () => { + // ///snapshots/README.md + const expectPointer = _getSnapshotFile({ + repo: DUMMY_REPO, + path: "/README.md", + revision: "dummy-commit-hash", + }); + // //blobs/ + const expectedBlob = _getBlobFile({ + repo: DUMMY_REPO, + etag: DUMMY_ETAG, + }); + + // mock existing blob only no symlink + vi.mocked(lstat).mockResolvedValue({} as Stats); + // mock pathsInfo resolve content + vi.mocked(pathsInfo).mockResolvedValue([ + { + oid: DUMMY_ETAG, + size: 55, + path: "README.md", + type: "file", + lastCommit: { + date: new Date(), + id: "dummy-commit-hash", + title: "Commit msg", + }, + }, + ]); + + const output = await downloadFileToCacheDir({ + repo: DUMMY_REPO, + path: "/README.md", + fetch: fetchMock, + }); + + // should have check for the blob + expect(lstat).toHaveBeenCalled(); + expect(vi.mocked(lstat).mock.calls[0][0]).toBe(expectedBlob); + + // symlink should have been created + expect(createSymlink).toHaveBeenCalledOnce(); + // no download done + expect(fetchMock).not.toHaveBeenCalled(); + + expect(output).toBe(expectPointer); + }); + + test("expect resolve value to be the pointer path of downloaded file", async () => { + // ///snapshots/README.md + const expectPointer = _getSnapshotFile({ + repo: DUMMY_REPO, + path: "/README.md", + revision: "dummy-commit-hash", + }); + // //blobs/ + const expectedBlob = _getBlobFile({ + repo: DUMMY_REPO, + etag: DUMMY_ETAG, + }); + + vi.mocked(pathsInfo).mockResolvedValue([ + { + oid: DUMMY_ETAG, + size: 55, + path: "README.md", + type: "file", + lastCommit: { + date: new Date(), + id: "dummy-commit-hash", + title: "Commit msg", + }, + }, + ]); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + vi.mocked(createWriteStream).mockReturnValue(async function* () {} as any); + + const output = await downloadFileToCacheDir({ + repo: DUMMY_REPO, + path: "/README.md", + fetch: fetchMock, + }); + + // expect blobs and snapshots folder to have been mkdir + expect(vi.mocked(mkdir).mock.calls[0][0]).toBe(dirname(expectedBlob)); + expect(vi.mocked(mkdir).mock.calls[1][0]).toBe(dirname(expectPointer)); + + expect(output).toBe(expectPointer); + }); + + test("should write fetch response to blob", async () => { + // ///snapshots/README.md + const expectPointer = _getSnapshotFile({ + repo: DUMMY_REPO, + path: "/README.md", + revision: "dummy-commit-hash", + }); + // //blobs/ + const expectedBlob = _getBlobFile({ + repo: DUMMY_REPO, + etag: DUMMY_ETAG, + }); + + // mock pathsInfo resolve content + vi.mocked(pathsInfo).mockResolvedValue([ + { + oid: DUMMY_ETAG, + size: 55, + path: "README.md", + type: "file", + lastCommit: { + date: new Date(), + id: "dummy-commit-hash", + title: "Commit msg", + }, + }, + ]); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + vi.mocked(createWriteStream).mockReturnValue(async function* () {} as any); + + await downloadFileToCacheDir({ + repo: DUMMY_REPO, + path: "/README.md", + fetch: fetchMock, + }); + + const incomplete = `${expectedBlob}.incomplete`; + // 1. should write fetch#response#body to incomplete file + expect(createWriteStream).toHaveBeenCalledWith(incomplete); + // 2. should rename the incomplete to the blob expected name + expect(rename).toHaveBeenCalledWith(incomplete, expectedBlob); + // 3. should create symlink pointing to blob + expect(createSymlink).toHaveBeenCalledWith({ sourcePath: expectedBlob, finalPath: expectPointer }); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/download-file-to-cache-dir.ts b/node_modules/@huggingface/hub/src/lib/download-file-to-cache-dir.ts new file mode 100644 index 0000000000000000000000000000000000000000..ba125a1fe934da183ee10a48493d8b2d8b59415e --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/download-file-to-cache-dir.ts @@ -0,0 +1,148 @@ +import { getHFHubCachePath, getRepoFolderName } from "./cache-management"; +import { dirname, join } from "node:path"; +import { rename, lstat, mkdir, stat } from "node:fs/promises"; +import type { PathInfo } from "./paths-info"; +import { pathsInfo } from "./paths-info"; +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import { toRepoId } from "../utils/toRepoId"; +import { downloadFile } from "./download-file"; +import { createSymlink } from "../utils/symlink"; +import { Readable } from "node:stream"; +import type { ReadableStream } from "node:stream/web"; +import { pipeline } from "node:stream/promises"; +import { createWriteStream } from "node:fs"; + +export const REGEX_COMMIT_HASH: RegExp = new RegExp("^[0-9a-f]{40}$"); + +function getFilePointer(storageFolder: string, revision: string, relativeFilename: string): string { + const snapshotPath = join(storageFolder, "snapshots"); + return join(snapshotPath, revision, relativeFilename); +} + +/** + * handy method to check if a file exists, or the pointer of a symlinks exists + * @param path + * @param followSymlinks + */ +async function exists(path: string, followSymlinks?: boolean): Promise { + try { + if (followSymlinks) { + await stat(path); + } else { + await lstat(path); + } + return true; + } catch (err: unknown) { + return false; + } +} + +/** + * Download a given file if it's not already present in the local cache. + * @param params + * @return the symlink to the blob object + */ +export async function downloadFileToCacheDir( + params: { + repo: RepoDesignation; + path: string; + /** + * If true, will download the raw git file. + * + * For example, when calling on a file stored with Git LFS, the pointer file will be downloaded instead. + */ + raw?: boolean; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + * + * @default "main" + */ + revision?: string; + hubUrl?: string; + cacheDir?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise { + const repoId = toRepoId(params.repo); + if (repoId.type === "bucket") { + throw new Error("downloadFileToCacheDir is not supported for bucket repos."); + } + const revision = params.revision ?? "main"; + const cacheDir = params.cacheDir ?? getHFHubCachePath(); + const storageFolder = join(cacheDir, getRepoFolderName(repoId)); + + let commitHash: string | undefined; + + if (revision && REGEX_COMMIT_HASH.test(revision)) { + commitHash = revision; + const pointerPath = getFilePointer(storageFolder, revision, params.path); + if (await exists(pointerPath, true)) { + return pointerPath; + } + } + + const pathsInformation: PathInfo[] = await pathsInfo({ + ...params, + paths: [params.path], + revision, + expand: true, + }); + if (!pathsInformation || pathsInformation.length !== 1) { + throw new Error(`cannot get path info for ${params.path}`); + } + + const info = pathsInformation[0]; + let etag: string; + if (info.lfs) { + etag = info.lfs.oid; + } else if (info.xetHash) { + etag = info.xetHash; + } else if (info.oid) { + etag = info.oid; + } else { + throw new Error(`cannot determine etag for ${params.path}`); + } + + const snapshotId = commitHash ?? info.lastCommit?.id ?? etag; + const pointerPath = getFilePointer(storageFolder, snapshotId, params.path); + const blobPath = join(storageFolder, "blobs", etag); + + if (await exists(pointerPath, true)) { + return pointerPath; + } + + // mkdir blob and pointer path parent directory + await mkdir(dirname(blobPath), { recursive: true }); + await mkdir(dirname(pointerPath), { recursive: true }); + + // We might already have the blob but not the pointer + // shortcut the download if needed + if (await exists(blobPath)) { + // create symlinks in snapshot folder to blob object + await createSymlink({ sourcePath: blobPath, finalPath: pointerPath }); + return pointerPath; + } + + const incomplete = `${blobPath}.incomplete`; + console.debug(`Downloading ${params.path} to ${incomplete}`); + + const blob: Blob | null = await downloadFile({ + ...params, + revision, + }); + + if (!blob) { + throw new Error(`invalid response for file ${params.path}`); + } + + await pipeline(Readable.fromWeb(blob.stream() as ReadableStream), createWriteStream(incomplete)); + + // rename .incomplete file to expect blob + await rename(incomplete, blobPath); + // create symlinks in snapshot folder to blob object + await createSymlink({ sourcePath: blobPath, finalPath: pointerPath }); + return pointerPath; +} diff --git a/node_modules/@huggingface/hub/src/lib/download-file.spec.ts b/node_modules/@huggingface/hub/src/lib/download-file.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..53ac648140d3bc8042abec5d0b4ad05815cc5b1d --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/download-file.spec.ts @@ -0,0 +1,83 @@ +import { expect, test, describe, assert } from "vitest"; +import { downloadFile } from "./download-file"; +import { deleteRepo } from "./delete-repo"; +import { createRepo } from "./create-repo"; +import { TEST_ACCESS_TOKEN, TEST_HUB_URL, TEST_USER } from "../test/consts"; +import { insecureRandomString } from "../utils/insecureRandomString"; + +describe("downloadFile", () => { + test("should download regular file", async () => { + const blob = await downloadFile({ + repo: { + type: "model", + name: "openai-community/gpt2", + }, + path: "README.md", + }); + + const text = await blob?.slice(0, 1000).text(); + assert( + text?.includes(`--- +language: en +tags: +- exbert + +license: mit +--- + + +# GPT-2 + +Test the whole generation capabilities here: https://transformer.huggingface.co/doc/gpt2-large`), + ); + }); + test("should downoad xet file", async () => { + const blob = await downloadFile({ + repo: { + type: "model", + name: "celinah/xet-experiments", + }, + path: "large_text.txt", + }); + + const text = await blob?.slice(0, 100).text(); + expect(text).toMatch("this is a text file.".repeat(10).slice(0, 100)); + }); + + test("should download private file", async () => { + const repoName = `datasets/${TEST_USER}/TEST-${insecureRandomString()}`; + + const result = await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + visibility: "private", + repo: repoName, + files: [{ path: ".gitattributes", content: new Blob(["*.html filter=lfs diff=lfs merge=lfs -text"]) }], + }); + + expect(result).toEqual({ + repoUrl: `${TEST_HUB_URL}/${repoName}`, + id: expect.any(String), + }); + + try { + const blob = await downloadFile({ + repo: repoName, + path: ".gitattributes", + hubUrl: TEST_HUB_URL, + accessToken: TEST_ACCESS_TOKEN, + }); + + assert(blob, "File should be found"); + + const text = await blob?.text(); + assert.strictEqual(text, "*.html filter=lfs diff=lfs merge=lfs -text"); + } finally { + await deleteRepo({ + repo: repoName, + hubUrl: TEST_HUB_URL, + accessToken: TEST_ACCESS_TOKEN, + }); + } + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/download-file.ts b/node_modules/@huggingface/hub/src/lib/download-file.ts new file mode 100644 index 0000000000000000000000000000000000000000..264cff2ca20aae53e8de918d788cf814806875cf --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/download-file.ts @@ -0,0 +1,78 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { WebBlob } from "../utils/WebBlob"; +import { XetBlob } from "../utils/XetBlob"; +import type { XetReadToken } from "../utils/XetBlob"; +import type { FileDownloadInfoOutput } from "./file-download-info"; +import { fileDownloadInfo } from "./file-download-info"; + +/** + * @returns null when the file doesn't exist + */ +export async function downloadFile( + params: { + repo: RepoDesignation; + path: string; + /** + * If true, will download the raw git file. + * + * For example, when calling on a file stored with Git LFS, the pointer file will be downloaded instead. + */ + raw?: boolean; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + * + * @default "main" + */ + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + /** + * Whether to use the xet protocol to download the file (if applicable). + * + * When an object with `readToken` is provided along with `downloadInfo`, + * the xet download can skip the token refresh roundtrip. + * + * @default true + */ + xet?: boolean | { readToken: XetReadToken }; + /** + * Can save an http request if provided + */ + downloadInfo?: FileDownloadInfoOutput; + } & Partial, +): Promise { + const accessToken = checkCredentials(params); + + const info = + params.downloadInfo ?? + (await fileDownloadInfo({ + accessToken, + repo: params.repo, + path: params.path, + revision: params.revision, + hubUrl: params.hubUrl, + fetch: params.fetch, + raw: params.raw, + })); + + if (!info) { + return null; + } + + if (info.xet && params.xet !== false) { + return new XetBlob({ + refreshUrl: info.xet.refreshUrl.href, + reconstructionUrl: info.xet.reconstructionUrl.href, + fetch: params.fetch, + accessToken, + size: info.size, + readToken: typeof params.xet === "object" ? params.xet.readToken : undefined, + }); + } + + return new WebBlob(new URL(info.url), 0, info.size, "", true, params.fetch ?? fetch, accessToken); +} diff --git a/node_modules/@huggingface/hub/src/lib/file-download-info.spec.ts b/node_modules/@huggingface/hub/src/lib/file-download-info.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..0a18cc757384513a08f8b2cf23ae6593e191b970 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/file-download-info.spec.ts @@ -0,0 +1,59 @@ +import { assert, it, describe } from "vitest"; +import { fileDownloadInfo } from "./file-download-info"; + +describe("fileDownloadInfo", () => { + it("should fetch LFS file info", async () => { + const info = await fileDownloadInfo({ + repo: { + name: "google-bert/bert-base-uncased", + type: "model", + }, + path: "tf_model.h5", + revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", + }); + + assert.strictEqual(info?.size, 536063208); + assert.strictEqual(info?.etag, '"a7a17d6d844b5de815ccab5f42cad6d24496db3850a2a43d8258221018ce87d2"'); + }); + + it("should fetch raw LFS pointer info", async () => { + const info = await fileDownloadInfo({ + repo: { + name: "google-bert/bert-base-uncased", + type: "model", + }, + path: "tf_model.h5", + revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", + raw: true, + }); + + assert.strictEqual(info?.size, 134); + assert.strictEqual(info?.etag, '"9eb98c817f04b051b3bcca591bcd4e03cec88018"'); + }); + + it("should fetch non-LFS file info", async () => { + const info = await fileDownloadInfo({ + repo: { + name: "google-bert/bert-base-uncased", + type: "model", + }, + path: "tokenizer_config.json", + revision: "1a7dd4986e3dab699c24ca19b2afd0f5e1a80f37", + }); + + assert.strictEqual(info?.size, 28); + assert.strictEqual(info?.etag, '"a661b1a138dac6dc5590367402d100765010ffd6"'); + }); + + it("should fetch xet file info", async () => { + const info = await fileDownloadInfo({ + repo: { + type: "model", + name: "celinah/xet-experiments", + }, + path: "large_text.txt", + }); + assert.strictEqual(info?.size, 62914580); + assert.strictEqual(info?.etag, '"c27f98578d9363b27db0bc1cbd9c692f8e6e90ae98c38cee7bc0a88829debd17"'); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/file-download-info.ts b/node_modules/@huggingface/hub/src/lib/file-download-info.ts new file mode 100644 index 0000000000000000000000000000000000000000..1a507854fa92ac2cdb4a587e24aafadb019c015e --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/file-download-info.ts @@ -0,0 +1,152 @@ +import { HUB_URL } from "../consts"; +import { createApiError, InvalidApiResponseFormatError } from "../error"; +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { parseLinkHeader } from "../utils/parseLinkHeader"; +import { toRepoId } from "../utils/toRepoId"; + +export interface XetFileInfo { + hash: string; + refreshUrl: URL; + /** + * Can be directly used instead of the hash. + */ + reconstructionUrl: URL; +} + +export interface FileDownloadInfoOutput { + size: number; + etag: string; + xet?: XetFileInfo; + // URL to fetch (with the access token if private file) + url: string; +} +/** + * @returns null when the file doesn't exist + */ +export async function fileDownloadInfo( + params: { + repo: RepoDesignation; + path: string; + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + /** + * To get the raw pointer file behind a LFS file + */ + raw?: boolean; + /** + * To avoid the content-disposition header in the `downloadLink` for LFS files + * + * So that on browsers you can use the URL in an iframe for example + */ + noContentDisposition?: boolean; + } & Partial, +): Promise { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + + const hubUrl = params.hubUrl ?? HUB_URL; + const revision = repoId.type === "bucket" ? undefined : (params.revision ?? "main"); + const url = + `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${ + params.raw ? "raw" : "resolve" + }${revision ? `/${encodeURIComponent(revision)}` : ""}/${params.path}` + + (params.noContentDisposition ? "?noContentDisposition=1" : ""); + + const resp = await (params.fetch ?? fetch)(url, { + method: "GET", + headers: { + ...(accessToken && { + Authorization: `Bearer ${accessToken}`, + }), + Range: "bytes=0-0", + Accept: "application/vnd.xet-fileinfo+json, */*", + }, + }); + + if (resp.status === 404 && resp.headers.get("X-Error-Code") === "EntryNotFound") { + return null; + } + + if (!resp.ok) { + throw await createApiError(resp); + } + + let size: number | undefined; + let xetInfo: XetFileInfo | undefined; + + if (resp.headers.get("Content-Type")?.includes("application/vnd.xet-fileinfo+json")) { + size = parseInt(resp.headers.get("X-Linked-Size") ?? "invalid"); + if (isNaN(size)) { + throw new InvalidApiResponseFormatError("Invalid file size received in X-Linked-Size header"); + } + + const hash = resp.headers.get("X-Xet-Hash"); + const links = parseLinkHeader(resp.headers.get("Link") ?? ""); + + const reconstructionUrl = (() => { + try { + return new URL(links["xet-reconstruction-info"]); + } catch { + return null; + } + })(); + const refreshUrl = (() => { + try { + return new URL(links["xet-auth"]); + } catch { + return null; + } + })(); + + if (!hash) { + throw new InvalidApiResponseFormatError("No hash received in X-Xet-Hash header"); + } + + if (!reconstructionUrl || !refreshUrl) { + throw new InvalidApiResponseFormatError("No xet-reconstruction-info or xet-auth link header"); + } + xetInfo = { + hash, + refreshUrl, + reconstructionUrl, + }; + } + + if (size === undefined || isNaN(size)) { + const contentRangeHeader = resp.headers.get("content-range"); + + if (!contentRangeHeader) { + throw new InvalidApiResponseFormatError("Expected size information"); + } + + const [, parsedSize] = contentRangeHeader.split("/"); + size = parseInt(parsedSize); + + if (isNaN(size)) { + throw new InvalidApiResponseFormatError("Invalid file size received"); + } + } + + const etag = resp.headers.get("X-Linked-ETag") ?? resp.headers.get("ETag") ?? undefined; + + if (!etag) { + throw new InvalidApiResponseFormatError("Expected ETag"); + } + + return { + etag, + size, + xet: xetInfo, + // Cannot use resp.url in case it's a S3 url and the user adds an Authorization header to it. + url: + resp.url && + (new URL(resp.url).origin === new URL(hubUrl).origin || resp.headers.get("X-Cache")?.endsWith(" cloudfront")) + ? resp.url + : url, + }; +} diff --git a/node_modules/@huggingface/hub/src/lib/file-exists.spec.ts b/node_modules/@huggingface/hub/src/lib/file-exists.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..54c8ccd90e2623a2790377146128a949006a539f --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/file-exists.spec.ts @@ -0,0 +1,30 @@ +import { assert, it, describe } from "vitest"; +import { fileExists } from "./file-exists"; + +describe("fileExists", () => { + it("should return true for file that exists", async () => { + const info = await fileExists({ + repo: { + name: "google-bert/bert-base-uncased", + type: "model", + }, + path: "tf_model.h5", + revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", + }); + + assert(info, "file should exist"); + }); + + it("should return false for file that does not exist", async () => { + const info = await fileExists({ + repo: { + name: "google-bert/bert-base-uncased", + type: "model", + }, + path: "tf_model.h5dadazdzazd", + revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", + }); + + assert(!info, "file should not exist"); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/file-exists.ts b/node_modules/@huggingface/hub/src/lib/file-exists.ts new file mode 100644 index 0000000000000000000000000000000000000000..70bd742a62ff06d42b23c68c53506b2d2ceb7bcb --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/file-exists.ts @@ -0,0 +1,43 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { toRepoId } from "../utils/toRepoId"; + +export async function fileExists( + params: { + repo: RepoDesignation; + path: string; + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + + const hubUrl = params.hubUrl ?? HUB_URL; + const revision = repoId.type === "bucket" ? undefined : (params.revision ?? "main"); + const endpoint = repoId.type === "bucket" ? "resolve" : "raw"; + const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${endpoint}${ + revision ? `/${encodeURIComponent(revision)}` : "" + }/${params.path}`; + + const resp = await (params.fetch ?? fetch)(url, { + method: "HEAD", + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {}, + }); + + if (resp.status === 404) { + return false; + } + + if (!resp.ok) { + throw await createApiError(resp); + } + + return true; +} diff --git a/node_modules/@huggingface/hub/src/lib/index.ts b/node_modules/@huggingface/hub/src/lib/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..125787f500c66ae72def7d28b7cfe0cad71d2f53 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/index.ts @@ -0,0 +1,37 @@ +export * from "./cache-management"; +export * from "./check-repo-access"; +export * from "./commit"; +export * from "./copy-files"; +export * from "./count-commits"; +export * from "./create-repo"; +export * from "./create-branch"; +export * from "./create-collection"; +export * from "./dataset-info"; +export * from "./delete-branch"; +export * from "./delete-file"; +export * from "./delete-files"; +export * from "./delete-repo"; +export * from "./delete-collection"; +export * from "./download-file"; +export * from "./download-file-to-cache-dir"; +export * from "./file-download-info"; +export * from "./file-exists"; +export * from "./jobs"; +export * from "./list-commits"; +export * from "./list-datasets"; +export * from "./list-files"; +export * from "./list-models"; +export * from "./list-spaces"; +export * from "./list-collections"; +export * from "./model-info"; +export * from "./oauth-handle-redirect"; +export * from "./oauth-login-url"; +export * from "./parse-safetensors-metadata"; +export * from "./paths-info"; +export * from "./repo-exists"; +export * from "./snapshot-download"; +export * from "./space-info"; +export * from "./upload-file"; +export * from "./upload-files"; +export * from "./upload-files-with-progress"; +export * from "./who-am-i"; diff --git a/node_modules/@huggingface/hub/src/lib/jobs/cancel-job.ts b/node_modules/@huggingface/hub/src/lib/jobs/cancel-job.ts new file mode 100644 index 0000000000000000000000000000000000000000..52a6a80378445b448fde338e4dccd3e595b20237 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/cancel-job.ts @@ -0,0 +1,45 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; +import type { ApiJob } from "../../types/api/api-jobs"; + +/** + * Cancel a job. + */ +export async function cancelJob( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): Promise { + const accessToken = checkCredentials(params); + + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/cancel`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }, + ); + + if (!response.ok) { + throw await createApiError(response); + } + + return await response.json(); +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/create-scheduled-job.ts b/node_modules/@huggingface/hub/src/lib/jobs/create-scheduled-job.ts new file mode 100644 index 0000000000000000000000000000000000000000..5b426be6e6293b4ec0fdea42797af64845a3fa73 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/create-scheduled-job.ts @@ -0,0 +1,92 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; +import { toRepoId } from "../../utils/toRepoId"; +import type { ApiScheduledJob, CreateScheduledJobOptions } from "../../types/api/api-jobs"; + +/** + * Create a scheduled job. + */ +export async function createScheduledJob( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CreateScheduledJobOptions & + CredentialsParams, +): Promise { + const accessToken = checkCredentials(params); + + const { namespace, hubUrl, fetch: customFetch, ...rest } = params; + + if (!rest.jobSpec.dockerImage && !rest.jobSpec.spaceId) { + throw new Error("Either dockerImage or spaceId must be provided in jobSpec"); + } + + if (rest.jobSpec.dockerImage && rest.jobSpec.spaceId) { + throw new Error("Cannot provide both dockerImage and spaceId in jobSpec"); + } + + const body: Record = { + jobSpec: { + flavor: rest.jobSpec.flavor, + }, + schedule: rest.schedule, + suspend: rest.suspend ?? false, + concurrency: rest.concurrency ?? false, + }; + + if (rest.jobSpec.dockerImage) { + (body.jobSpec as Record).dockerImage = rest.jobSpec.dockerImage; + } + if (rest.jobSpec.spaceId) { + (body.jobSpec as Record).spaceId = rest.jobSpec.spaceId; + } + if (rest.jobSpec.command) { + (body.jobSpec as Record).command = rest.jobSpec.command; + } + (body.jobSpec as Record).environment = rest.jobSpec.environment || {}; + if (rest.jobSpec.secrets) { + (body.jobSpec as Record).secrets = rest.jobSpec.secrets; + } + if (rest.jobSpec.arch) { + (body.jobSpec as Record).arch = rest.jobSpec.arch; + } + if (rest.jobSpec.timeoutSeconds !== undefined) { + (body.jobSpec as Record).timeoutSeconds = rest.jobSpec.timeoutSeconds; + } + if (rest.jobSpec.attempts !== undefined) { + (body.jobSpec as Record).attempts = rest.jobSpec.attempts; + } + if (rest.jobSpec.labels) { + (body.jobSpec as Record).labels = rest.jobSpec.labels; + } + if (rest.jobSpec.volumes?.length) { + (body.jobSpec as Record).volumes = rest.jobSpec.volumes.map(({ source, ...rest }) => { + const repoId = toRepoId(source); + return { type: repoId.type, source: repoId.name, ...rest }; + }); + } + + const response = await (customFetch || fetch)(`${hubUrl || HUB_URL}/api/scheduled-jobs/${namespace}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + throw await createApiError(response); + } + + return await response.json(); +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/delete-scheduled-job.ts b/node_modules/@huggingface/hub/src/lib/jobs/delete-scheduled-job.ts new file mode 100644 index 0000000000000000000000000000000000000000..03bfee16fd1a4c1241bcd9e9225c46b944eb21e8 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/delete-scheduled-job.ts @@ -0,0 +1,42 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; + +/** + * Delete a scheduled job. + */ +export async function deleteScheduledJob( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The scheduled job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): Promise { + const accessToken = checkCredentials(params); + + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}`, + { + method: "DELETE", + headers: { + "Content-Type": "application/json", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }, + ); + + if (!response.ok) { + throw await createApiError(response); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/duplicate-job.ts b/node_modules/@huggingface/hub/src/lib/jobs/duplicate-job.ts new file mode 100644 index 0000000000000000000000000000000000000000..9e5e444e9d4ca3e051f80801e1f45d842bdf5991 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/duplicate-job.ts @@ -0,0 +1,45 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; +import type { ApiJob } from "../../types/api/api-jobs"; + +/** + * Duplicate a job (re-run with the same spec). + */ +export async function duplicateJob( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The job ID to duplicate + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): Promise { + const accessToken = checkCredentials(params); + + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/duplicate`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }, + ); + + if (!response.ok) { + throw await createApiError(response); + } + + return await response.json(); +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/get-job.ts b/node_modules/@huggingface/hub/src/lib/jobs/get-job.ts new file mode 100644 index 0000000000000000000000000000000000000000..8a027516ad93867e89bb526f288a813f90f2d80d --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/get-job.ts @@ -0,0 +1,43 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; +import type { ApiJob } from "../../types/api/api-jobs"; + +/** + * Get a specific job by ID. + */ +export async function getJob( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): Promise { + const accessToken = checkCredentials(params); + + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}`, + { + headers: { + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }, + ); + + if (!response.ok) { + throw await createApiError(response); + } + + return await response.json(); +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/get-scheduled-job.ts b/node_modules/@huggingface/hub/src/lib/jobs/get-scheduled-job.ts new file mode 100644 index 0000000000000000000000000000000000000000..de8d152443188a876a1794ed938ce22bd72514c2 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/get-scheduled-job.ts @@ -0,0 +1,43 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; +import type { ApiScheduledJob } from "../../types/api/api-jobs"; + +/** + * Get a specific scheduled job by ID. + */ +export async function getScheduledJob( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The scheduled job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): Promise { + const accessToken = checkCredentials(params); + + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}`, + { + headers: { + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }, + ); + + if (!response.ok) { + throw await createApiError(response); + } + + return await response.json(); +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/index.ts b/node_modules/@huggingface/hub/src/lib/jobs/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..c7f9aad3d09cedcc45554cb88f65c90582b990ec --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/index.ts @@ -0,0 +1,16 @@ +export * from "./cancel-job"; +export * from "./create-scheduled-job"; +export * from "./delete-scheduled-job"; +export * from "./duplicate-job"; +export * from "./get-job"; +export * from "./get-scheduled-job"; +export * from "./list-job-hardware"; +export * from "./list-jobs"; +export * from "./list-scheduled-jobs"; +export * from "./resume-scheduled-job"; +export * from "./run-job"; +export * from "./run-scheduled-job"; +export * from "./stream-job-events"; +export * from "./stream-job-logs"; +export * from "./stream-job-metrics"; +export * from "./suspend-scheduled-job"; diff --git a/node_modules/@huggingface/hub/src/lib/jobs/list-job-hardware.ts b/node_modules/@huggingface/hub/src/lib/jobs/list-job-hardware.ts new file mode 100644 index 0000000000000000000000000000000000000000..a6d129a2fc868a8b27fdffc12ce56c1b709ef506 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/list-job-hardware.ts @@ -0,0 +1,37 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; +import type { ApiJobHardware } from "../../types/api/api-jobs"; + +/** + * Get the list of available hardware for jobs. + * This endpoint is public and does not require authentication, but authentication is optional. + */ +export async function listJobHardware( + params?: { + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise { + const accessToken = checkCredentials(params ?? {}); + + const headers: Record = {}; + + if (accessToken) { + headers.Authorization = `Bearer ${accessToken}`; + } + + const response = await (params?.fetch || fetch)(`${params?.hubUrl || HUB_URL}/api/jobs/hardware`, { + headers, + }); + + if (!response.ok) { + throw await createApiError(response); + } + + return await response.json(); +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/list-jobs.ts b/node_modules/@huggingface/hub/src/lib/jobs/list-jobs.ts new file mode 100644 index 0000000000000000000000000000000000000000..7b214b24f037d7825bea9f518c2763df2aee917b --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/list-jobs.ts @@ -0,0 +1,36 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; +import type { ApiJob } from "../../types/api/api-jobs"; + +/** + * List jobs for a namespace (user or organization). + */ +export async function listJobs( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): Promise { + const accessToken = checkCredentials(params); + + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { + headers: { + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }); + + if (!response.ok) { + throw await createApiError(response); + } + + return await response.json(); +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/list-scheduled-jobs.ts b/node_modules/@huggingface/hub/src/lib/jobs/list-scheduled-jobs.ts new file mode 100644 index 0000000000000000000000000000000000000000..47693868326a976977327a6b1f896472c9e71c97 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/list-scheduled-jobs.ts @@ -0,0 +1,36 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; +import type { ApiScheduledJob } from "../../types/api/api-jobs"; + +/** + * List scheduled jobs for a namespace. + */ +export async function listScheduledJobs( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): Promise { + const accessToken = checkCredentials(params); + + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}`, { + headers: { + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }); + + if (!response.ok) { + throw await createApiError(response); + } + + return await response.json(); +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/resume-scheduled-job.ts b/node_modules/@huggingface/hub/src/lib/jobs/resume-scheduled-job.ts new file mode 100644 index 0000000000000000000000000000000000000000..c7c0aa331b6549e32a9b59d027c5aefe3fa18729 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/resume-scheduled-job.ts @@ -0,0 +1,41 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; + +/** + * Resume a scheduled job. + */ +export async function resumeScheduledJob( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The scheduled job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): Promise { + const accessToken = checkCredentials(params); + + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/resume`, + { + method: "POST", + headers: { + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }, + ); + + if (!response.ok) { + throw await createApiError(response); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/run-job.ts b/node_modules/@huggingface/hub/src/lib/jobs/run-job.ts new file mode 100644 index 0000000000000000000000000000000000000000..44f8568fcfa74b17b9d64e5707f6fe0d42b48cb1 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/run-job.ts @@ -0,0 +1,89 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; +import { toRepoId } from "../../utils/toRepoId"; +import type { ApiJob, CreateJobOptions } from "../../types/api/api-jobs"; +export type { JobVolume } from "../../types/api/api-jobs"; + +/** + * Run a new job. + */ +export async function runJob( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CreateJobOptions & + CredentialsParams, +): Promise { + const accessToken = checkCredentials(params); + + if (!params.dockerImage && !params.spaceId) { + throw new Error("Either dockerImage or spaceId must be provided"); + } + + if (params.dockerImage && params.spaceId) { + throw new Error("Cannot provide both dockerImage and spaceId"); + } + + const body: Record = { + flavor: params.flavor, + environment: params.environment || {}, + }; + + if (params.dockerImage) { + body.dockerImage = params.dockerImage; + } + if (params.spaceId) { + body.spaceId = params.spaceId; + } + if (params.command) { + body.command = params.command; + } + if (params.arguments) { + body.arguments = params.arguments; + } + if (params.secrets) { + body.secrets = params.secrets; + } + if (params.arch) { + body.arch = params.arch; + } + if (params.timeoutSeconds !== undefined) { + body.timeoutSeconds = params.timeoutSeconds; + } + if (params.attempts !== undefined) { + body.attempts = params.attempts; + } + if (params.labels) { + body.labels = params.labels; + } + if (params.volumes?.length) { + body.volumes = params.volumes.map(({ source, ...rest }) => { + const repoId = toRepoId(source); + return { type: repoId.type, source: repoId.name, ...rest }; + }); + } + + const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + throw await createApiError(response); + } + + return await response.json(); +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/run-scheduled-job.ts b/node_modules/@huggingface/hub/src/lib/jobs/run-scheduled-job.ts new file mode 100644 index 0000000000000000000000000000000000000000..3224830b90509ba3c9f88cbb1504e5b075026c72 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/run-scheduled-job.ts @@ -0,0 +1,50 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; +import type { ApiJob } from "../../types/api/api-jobs"; + +/** + * Trigger a scheduled job to run immediately. + * Returns the job that was triggered, or null if another instance is already running. + */ +export async function runScheduledJob( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The scheduled job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): Promise { + const accessToken = checkCredentials(params); + + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/run`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }, + ); + + if (!response.ok) { + if (response.status === 409) { + // Another instance is already running + return null; + } + throw await createApiError(response); + } + + return await response.json(); +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/stream-job-events.ts b/node_modules/@huggingface/hub/src/lib/jobs/stream-job-events.ts new file mode 100644 index 0000000000000000000000000000000000000000..df3efebab07f66d65cc4741f25b91d0a3bb754a0 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/stream-job-events.ts @@ -0,0 +1,81 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; + +/** + * Stream job events using Server-Sent Events (SSE). + * Returns an async iterable of event chunks. + */ +export async function* streamJobEvents( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): AsyncGenerator { + const accessToken = checkCredentials(params); + + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/events`, + { + headers: { + Accept: "text/event-stream", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }, + ); + + if (!response.ok) { + throw await createApiError(response); + } + + if (!response.body) { + return; + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + + // Process remaining buffer + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + } finally { + reader.releaseLock(); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/stream-job-logs.ts b/node_modules/@huggingface/hub/src/lib/jobs/stream-job-logs.ts new file mode 100644 index 0000000000000000000000000000000000000000..45cda71901c7941a240afef08f282375e6a12bcd --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/stream-job-logs.ts @@ -0,0 +1,91 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; + +/** + * Stream job logs using Server-Sent Events (SSE). + * Returns an async iterable of log chunks. + */ +export async function* streamJobLogs( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): AsyncGenerator<{ message: string; timestamp: Date }, void, unknown> { + const accessToken = checkCredentials(params); + + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/logs`, + { + headers: { + Accept: "text/event-stream", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }, + ); + + if (!response.ok) { + throw await createApiError(response); + } + + if (!response.body) { + return; + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const data = JSON.parse(line.slice(6)); + yield { message: data.data, timestamp: new Date(data.timestamp) }; + } catch { + yield { message: line.slice(6), timestamp: new Date() }; + } + } + } + } + + // Process remaining buffer + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + try { + const data = JSON.parse(line.slice(6)); + yield { message: data.data, timestamp: new Date(data.timestamp) }; + } catch { + yield { message: line.slice(6), timestamp: new Date() }; + } + } + } + } + } finally { + reader.releaseLock(); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/stream-job-metrics.ts b/node_modules/@huggingface/hub/src/lib/jobs/stream-job-metrics.ts new file mode 100644 index 0000000000000000000000000000000000000000..72eeceaf1a6932a69b6fa2beabac5c3ade54cab1 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/stream-job-metrics.ts @@ -0,0 +1,81 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; + +/** + * Stream job metrics using Server-Sent Events (SSE). + * Returns an async iterable of metric chunks. + */ +export async function* streamJobMetrics( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): AsyncGenerator { + const accessToken = checkCredentials(params); + + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/metrics`, + { + headers: { + Accept: "text/event-stream", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }, + ); + + if (!response.ok) { + throw await createApiError(response); + } + + if (!response.body) { + return; + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + + // Process remaining buffer + if (buffer) { + const lines = buffer.split("\n"); + for (const line of lines) { + if (line.startsWith("data: ")) { + yield line.slice(6); + } + } + } + } finally { + reader.releaseLock(); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/jobs/suspend-scheduled-job.ts b/node_modules/@huggingface/hub/src/lib/jobs/suspend-scheduled-job.ts new file mode 100644 index 0000000000000000000000000000000000000000..dcf4a5e3dded753073aa094a9c12cbd03a46112f --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/jobs/suspend-scheduled-job.ts @@ -0,0 +1,42 @@ +import { HUB_URL } from "../../consts"; +import { createApiError } from "../../error"; +import type { CredentialsParams } from "../../types/public"; +import { checkCredentials } from "../../utils/checkCredentials"; + +/** + * Suspend (pause) a scheduled job. + */ +export async function suspendScheduledJob( + params: { + /** + * The namespace (username or organization name) + */ + namespace: string; + /** + * The scheduled job ID + */ + jobId: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): Promise { + const accessToken = checkCredentials(params); + + const response = await (params.fetch || fetch)( + `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/suspend`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }, + ); + + if (!response.ok) { + throw await createApiError(response); + } +} diff --git a/node_modules/@huggingface/hub/src/lib/list-collections.spec.ts b/node_modules/@huggingface/hub/src/lib/list-collections.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..c07af809b8a4c42cf0efe40ba931b4dc96fecdc1 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/list-collections.spec.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from "vitest"; +import { listCollections } from "./list-collections"; +import type { ApiCollectionInfo } from "../types/api/api-collection"; + +describe("listCollections", () => { + it("should list collections", async () => { + const results: ApiCollectionInfo[] = []; + + for await (const entry of listCollections({ + search: { owner: ["huggingfacejs"] }, + })) { + results.push(entry); + } + + expect(results.length).toBe(1); + + const itemTypes = results[0].items.map((item) => item.type); + expect(itemTypes).toEqual(["dataset", "collection", "space", "paper"]); + + expect(results).toEqual([ + { + slug: "huggingfacejs/test-collection-690df2897fa1945492b8cf42", + title: "Test Collection", + description: "Only used in E2E tests", + gating: false, + lastUpdated: expect.any(String), + owner: { + _id: "6414d83b385a75d7790d5a58", + avatarUrl: expect.any(String), + fullname: "Huggingface.js", + name: "huggingfacejs", + type: "org", + followerCount: expect.any(Number), + isHf: false, + isHfAdmin: false, + isMod: false, + isUserFollowing: expect.any(Boolean), + }, + items: [ + { + _id: "690df2a467ea25a1a346d0ae", + author: "huggingfacejs", + datasetsServerInfo: { + formats: expect.any(Array), + libraries: expect.any(Array), + modalities: expect.any(Array), + numRows: expect.any(Number), + viewer: expect.any(String), + }, + downloads: expect.any(Number), + gated: false, + id: "huggingfacejs/tasks", + isBenchmark: false, + isLikedByUser: false, + isTraces: false, + lastModified: expect.any(String), + likes: expect.any(Number), + position: 0, + private: false, + repoType: "dataset", + type: "dataset", + }, + { + _id: "690df2b1954547dac9727da3", + description: "Only used in E2E tests", + id: "690df2897fa1945492b8cf42", + isUpvotedByUser: false, + lastUpdated: expect.any(String), + numberItems: 5, + owner: { + _id: "6414d83b385a75d7790d5a58", + avatarUrl: expect.any(String), + followerCount: expect.any(Number), + fullname: "Huggingface.js", + isHf: false, + isHfAdmin: false, + isMod: false, + isUserFollowing: expect.any(Boolean), + name: "huggingfacejs", + type: "org", + }, + position: 1, + shareUrl: "https://hf.co/collections/huggingfacejs/test-collection", + slug: "huggingfacejs/test-collection-690df2897fa1945492b8cf42", + theme: "pink", + title: "Test Collection", + type: "collection", + upvotes: expect.any(Number), + }, + { + _id: "690df2c49f252aa897a873b2", + ai_category: "Model Benchmarking", + ai_short_description: "Upload ML models to Hugging Face Hub from your browser", + author: "huggingfacejs", + authorData: { + _id: "6414d83b385a75d7790d5a58", + avatarUrl: expect.any(String), + followerCount: expect.any(Number), + fullname: "Huggingface.js", + isHf: false, + isHfAdmin: false, + isMod: false, + isUserFollowing: expect.any(Boolean), + name: "huggingfacejs", + type: "org", + }, + colorFrom: "green", + colorTo: "green", + createdAt: "2023-03-17T21:33:16.000Z", + emoji: "🌎", + featured: false, + id: "huggingfacejs/push-model-from-web", + isLikedByUser: false, + lastModified: expect.any(String), + likes: expect.any(Number), + pinned: false, + position: 2, + private: false, + repoType: "space", + runtime: { + hardware: { + current: null, + requested: null, + }, + replicas: { + current: 1, + requested: 1, + }, + stage: "RUNNING", + }, + sdk: "static", + tags: ["static", "region:us"], + title: "Push Model From Web", + trendingScore: expect.any(Number), + type: "space", + visibility: "public", + }, + { + _id: "690df2d0c9390ed6ab0f88b1", + id: "2510.04871", + isUpvotedByUser: false, + position: 3, + publishedAt: "2025-10-06T14:58:08.000Z", + thumbnailUrl: "https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2510.04871.png", + title: "Less is More: Recursive Reasoning with Tiny Networks", + type: "paper", + upvotes: expect.any(Number), + }, + ], + theme: "pink", + private: false, + upvotes: expect.any(Number), + isUpvotedByUser: false, + }, + ]); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/list-collections.ts b/node_modules/@huggingface/hub/src/lib/list-collections.ts new file mode 100644 index 0000000000000000000000000000000000000000..613499cad3c00f76bd4c02a1a4c71c6f3dd8a701 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/list-collections.ts @@ -0,0 +1,102 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { CredentialsParams } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { parseLinkHeader } from "../utils/parseLinkHeader"; +import type { ApiCollectionInfo } from "../types/api/api-collection"; + +/* + * When listing collections, the item list per collection is truncated to 4 items maximum. + * To retrieve all items from a collection, you need to make an additional call using its collection slug. + */ +export async function* listCollections( + params?: { + search?: { + /** + * Filter collections created by specific owners (users or organizations). + */ + owner?: string[]; + /** + * Filter collections containing specific items. + * Value must be the item_type and item_id concatenated. + * Example: "models/teknium/OpenHermes-2.5-Mistral-7B", "datasets/rajpurkar/squad" or "papers/2311.12983". + */ + item?: string[]; + /** + * Filter based on substrings for titles & descriptions. + */ + q?: string; + }; + /** + * Sort the returned collections. Supported values are "lastModified", "trending" (default) and "upvotes". + */ + sort?: "lastModified" | "trending" | "upvotes"; + /** + * Set to limit the number of collections returned. + */ + limit?: number; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): AsyncGenerator { + const accessToken = params && checkCredentials(params); + + const searchParams = new URLSearchParams(); + + let totalToFetch = params?.limit ?? Infinity; + searchParams.append("limit", String(Math.min(totalToFetch, 100))); + + if (params?.sort) { + searchParams.append("sort", params.sort); + } + + if (params?.search?.owner) { + for (const owner of params.search.owner) { + searchParams.append("owner", owner); + } + } + + if (params?.search?.item) { + for (const item of params.search.item) { + searchParams.append("item", item); + } + } + + if (params?.search?.q) { + searchParams.append("q", params.search.q); + } + + let url: string | undefined = `${params?.hubUrl || HUB_URL}/api/collections?${searchParams}`; + + while (url) { + const res: Response = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : undefined), + }, + }); + + if (!res.ok) { + throw await createApiError(res); + } + + const collections: ApiCollectionInfo[] = await res.json(); + + for (const collection of collections) { + yield collection; + + totalToFetch--; + + if (totalToFetch <= 0) { + return; + } + } + + const linkHeader = res.headers.get("Link"); + + url = linkHeader ? parseLinkHeader(linkHeader).next : undefined; + } +} diff --git a/node_modules/@huggingface/hub/src/lib/list-commits.spec.ts b/node_modules/@huggingface/hub/src/lib/list-commits.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..a1f4dd5e5555923c40d02fc2024d765fea07b4ff --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/list-commits.spec.ts @@ -0,0 +1,117 @@ +import { assert, it, describe } from "vitest"; +import type { CommitData } from "./list-commits"; +import { listCommits } from "./list-commits"; + +describe("listCommits", () => { + it("should fetch paginated commits from the repo", async () => { + const commits: CommitData[] = []; + for await (const commit of listCommits({ + repo: { + name: "openai-community/gpt2", + type: "model", + }, + revision: "607a30d783dfa663caf39e06633721c8d4cfcd7e", + batchSize: 5, + })) { + commits.push(commit); + } + + assert.equal(commits.length, 26); + assert.deepEqual(commits.slice(0, 6), [ + { + oid: "607a30d783dfa663caf39e06633721c8d4cfcd7e", + title: "Adds the tokenizer configuration file (#80)", + message: "\n\n\n- Adds tokenizer_config.json file (db6d57930088fb63e52c010bd9ac77c955ac55e7)\n\n", + authors: [ + { + username: "lysandre", + avatarUrl: + "https://cdn-avatars.huggingface.co/v1/production/uploads/5e3aec01f55e2b62848a5217/PMKS0NNB4MJQlTSFzh918.jpeg", + }, + ], + date: new Date("2024-02-19T10:57:45.000Z"), + }, + { + oid: "11c5a3d5811f50298f278a704980280950aedb10", + title: "Adding ONNX file of this model (#60)", + message: "\n\n\n- Adding ONNX file of this model (9411f419c589519e1a46c94ac7789ea20fd7c322)\n\n", + authors: [ + { + username: "fxmarty", + avatarUrl: + "https://cdn-avatars.huggingface.co/v1/production/uploads/1651743336129-624c60cba8ec93a7ac188b56.png", + }, + ], + date: new Date("2023-06-30T02:19:43.000Z"), + }, + { + oid: "e7da7f221d5bf496a48136c0cd264e630fe9fcc8", + title: "Update generation_config.json", + message: "", + authors: [ + { + username: "joaogante", + avatarUrl: "https://cdn-avatars.huggingface.co/v1/production/uploads/1641203017724-noauth.png", + }, + ], + date: new Date("2022-12-16T15:44:21.000Z"), + }, + { + oid: "f27b190eeac4c2302d24068eabf5e9d6044389ae", + title: "Add note that this is the smallest version of the model (#18)", + message: + "\n\n\n- Add note that this is the smallest version of the model (611838ef095a5bb35bf2027d05e1194b7c9d37ac)\n\n\nCo-authored-by: helen \n", + authors: [ + { + username: "sgugger", + avatarUrl: + "https://cdn-avatars.huggingface.co/v1/production/uploads/1593126474392-5ef50182b71947201082a4e5.jpeg", + }, + { + username: "mathemakitten", + avatarUrl: + "https://cdn-avatars.huggingface.co/v1/production/uploads/1658248499901-6079afe2d2cd8c150e6ae05e.jpeg", + }, + ], + date: new Date("2022-11-23T12:55:26.000Z"), + }, + { + oid: "0dd7bcc7a64e4350d8859c9a2813132fbf6ae591", + title: "Our very first generation_config.json (#17)", + message: + "\n\n\n- Our very first generation_config.json (671851b7e9d56ef062890732065d7bd5f4628bd6)\n\n\nCo-authored-by: Joao Gante \n", + authors: [ + { + username: "sgugger", + avatarUrl: + "https://cdn-avatars.huggingface.co/v1/production/uploads/1593126474392-5ef50182b71947201082a4e5.jpeg", + }, + { + username: "joaogante", + avatarUrl: "https://cdn-avatars.huggingface.co/v1/production/uploads/1641203017724-noauth.png", + }, + ], + date: new Date("2022-11-18T18:19:30.000Z"), + }, + { + oid: "75e09b43581151bd1d9ef6700faa605df408979f", + title: "Upload model.safetensors with huggingface_hub (#12)", + message: + "\n\n\n- Upload model.safetensors with huggingface_hub (ba2f794b2e4ea09ef932a6628fa0815dfaf09661)\n\n\nCo-authored-by: Nicolas Patry \n", + authors: [ + { + username: "julien-c", + avatarUrl: + "https://cdn-avatars.huggingface.co/v1/production/uploads/5dd96eb166059660ed1ee413/NQtzmrDdbG0H8qkZvRyGk.jpeg", + }, + { + username: "Narsil", + avatarUrl: + "https://cdn-avatars.huggingface.co/v1/production/uploads/1608285816082-5e2967b819407e3277369b95.png", + }, + ], + date: new Date("2022-10-20T09:34:54.000Z"), + }, + ]); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/list-commits.ts b/node_modules/@huggingface/hub/src/lib/list-commits.ts new file mode 100644 index 0000000000000000000000000000000000000000..4d641eb63039581d3d02cc7514def7d36feb9739 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/list-commits.ts @@ -0,0 +1,70 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiCommitData } from "../types/api/api-commit"; +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { parseLinkHeader } from "../utils/parseLinkHeader"; +import { toRepoId } from "../utils/toRepoId"; + +export interface CommitData { + oid: string; + title: string; + message: string; + authors: Array<{ username: string; avatarUrl: string }>; + date: Date; +} + +export async function* listCommits( + params: { + repo: RepoDesignation; + /** + * Revision to list commits from. Defaults to the default branch. + */ + revision?: string; + hubUrl?: string; + /** + * Number of commits to fetch from the hub each http call. Defaults to 100. Can be set to 1000. + */ + batchSize?: number; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): AsyncGenerator { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + + // Could upgrade to 1000 commits per page + let url: string | undefined = `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/commits/${ + params.revision ?? "main" + }?limit=${params.batchSize ?? 100}`; + + while (url) { + const res: Response = await (params.fetch ?? fetch)(url, { + headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {}, + }); + + if (!res.ok) { + throw await createApiError(res); + } + + const resJson: ApiCommitData[] = await res.json(); + for (const commit of resJson) { + yield { + oid: commit.id, + title: commit.title, + message: commit.message, + authors: commit.authors.map((author) => ({ + username: author.user, + avatarUrl: author.avatar, + })), + date: new Date(commit.date), + }; + } + + const linkHeader = res.headers.get("Link"); + + url = linkHeader ? parseLinkHeader(linkHeader).next : undefined; + } +} diff --git a/node_modules/@huggingface/hub/src/lib/list-datasets.spec.ts b/node_modules/@huggingface/hub/src/lib/list-datasets.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..9b8c42cdd97840488f01a472febba8661312b319 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/list-datasets.spec.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import type { DatasetEntry } from "./list-datasets"; +import { listDatasets } from "./list-datasets"; + +describe("listDatasets", () => { + it("should list datasets from hf-doc-builder", async () => { + const results: DatasetEntry[] = []; + + for await (const entry of listDatasets({ search: { owner: "hf-doc-build" } })) { + if (entry.name !== "hf-doc-build/doc-build" && entry.name !== "hf-doc-build/doc-build-dev") { + continue; + } + if (typeof entry.downloads === "number") { + entry.downloads = 0; + } + if (typeof entry.likes === "number") { + entry.likes = 0; + } + if (entry.updatedAt instanceof Date && !isNaN(entry.updatedAt.getTime())) { + entry.updatedAt = new Date(0); + } + + results.push(entry); + } + + expect(results.sort((a, b) => a.id.localeCompare(b.id))).to.deep.equal([ + { + id: "6356b19985da6f13863228bd", + name: "hf-doc-build/doc-build", + private: false, + gated: false, + downloads: 0, + likes: 0, + updatedAt: new Date(0), + }, + { + id: "636a1b69f2f9ec4289c4c19e", + name: "hf-doc-build/doc-build-dev", + gated: false, + private: false, + downloads: 0, + likes: 0, + updatedAt: new Date(0), + }, + ]); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/list-datasets.ts b/node_modules/@huggingface/hub/src/lib/list-datasets.ts new file mode 100644 index 0000000000000000000000000000000000000000..f71280cf072538cf2b68b42d1d8fdd8fa7d7f646 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/list-datasets.ts @@ -0,0 +1,135 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiDatasetInfo } from "../types/api/api-dataset"; +import type { CredentialsParams } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { parseLinkHeader } from "../utils/parseLinkHeader"; +import { pick } from "../utils/pick"; + +export const DATASET_EXPAND_KEYS = [ + "private", + "downloads", + "gated", + "likes", + "lastModified", +] as const satisfies readonly (keyof ApiDatasetInfo)[]; + +export const DATASET_EXPANDABLE_KEYS = [ + "author", + "cardData", + "citation", + "createdAt", + "disabled", + "description", + "downloads", + "downloadsAllTime", + "gated", + "gitalyUid", + "lastModified", + "likes", + "paperswithcode_id", + "private", + // "siblings", + "sha", + "tags", +] as const satisfies readonly (keyof ApiDatasetInfo)[]; + +export interface DatasetEntry { + id: string; + name: string; + private: boolean; + downloads: number; + gated: false | "auto" | "manual"; + likes: number; + updatedAt: Date; +} + +export async function* listDatasets< + const T extends Exclude<(typeof DATASET_EXPANDABLE_KEYS)[number], (typeof DATASET_EXPAND_KEYS)[number]> = never, +>( + params?: { + search?: { + /** + * Will search in the dataset name for matches + */ + query?: string; + owner?: string; + tags?: string[]; + }; + hubUrl?: string; + additionalFields?: T[]; + /** + * Set to limit the number of datasets returned. + */ + limit?: number; + /** + * Sort datasets by a specific field. + */ + sort?: + | "createdAt" + | "downloads" + | "likes" + | "lastModified" + | "likes30d" + | "trendingScore" + | "datasetsServerInfo.numRows" + | "mainSize" + | "id"; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): AsyncGenerator> { + const accessToken = params && checkCredentials(params); + let totalToFetch = params?.limit ?? Infinity; + const search = new URLSearchParams([ + ...Object.entries({ + limit: String(Math.min(totalToFetch, 500)), + ...(params?.search?.owner ? { author: params.search.owner } : undefined), + ...(params?.search?.query ? { search: params.search.query } : undefined), + ...(params?.sort ? { sort: params.sort } : undefined), + }), + ...(params?.search?.tags?.map((tag) => ["filter", tag]) ?? []), + ...DATASET_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), + ...(params?.additionalFields?.map((val) => ["expand", val] satisfies [string, string]) ?? []), + ]).toString(); + let url: string | undefined = `${params?.hubUrl || HUB_URL}/api/datasets` + (search ? "?" + search : ""); + + while (url) { + const res: Response = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : undefined), + }, + }); + + if (!res.ok) { + throw await createApiError(res); + } + + const items: ApiDatasetInfo[] = await res.json(); + + for (const item of items) { + yield { + ...(params?.additionalFields && pick(item, params.additionalFields)), + id: item._id, + name: item.id, + private: item.private, + downloads: item.downloads, + likes: item.likes, + gated: item.gated, + updatedAt: new Date(item.lastModified), + } as DatasetEntry & Pick; + totalToFetch--; + if (totalToFetch <= 0) { + return; + } + } + + const linkHeader = res.headers.get("Link"); + + url = linkHeader ? parseLinkHeader(linkHeader).next : undefined; + // Could update limit in url to fetch less items if not all items of next page are needed. + } +} diff --git a/node_modules/@huggingface/hub/src/lib/list-files.spec.ts b/node_modules/@huggingface/hub/src/lib/list-files.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..00d3777de8f05a034db5d589c1e2113795598d55 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/list-files.spec.ts @@ -0,0 +1,173 @@ +import { assert, it, describe } from "vitest"; +import type { ListFileEntry } from "./list-files"; +import { listFiles } from "./list-files"; + +describe("listFiles", () => { + it("should fetch the list of files from the repo", async () => { + const cursor = listFiles({ + repo: { + name: "google-bert/bert-base-uncased", + type: "model", + }, + revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", + }); + + const files: ListFileEntry[] = []; + + for await (const entry of cursor) { + files.push(entry); + } + + assert.deepStrictEqual(files, [ + { + oid: "dc08351d4dc0732d9c8af04070ced089b201ce2f", + path: ".gitattributes", + size: 345, + type: "file", + }, + { + oid: "fca794a5f07ff8f963fe8b61e3694b0fb7f955df", + path: "config.json", + size: 313, + type: "file", + }, + { + lfs: { + oid: "097417381d6c7230bd9e3557456d726de6e83245ec8b24f529f60198a67b203a", + size: 440473133, + pointerSize: 134, + }, + xetHash: "2d8408d3a894d02517d04956e2f7546ff08362594072f3527ce144b5212a3296", + oid: "ba5d19791be1dd7992e33bd61f20207b0f7f50a5", + path: "pytorch_model.bin", + size: 440473133, + type: "file", + }, + { + lfs: { + oid: "a7a17d6d844b5de815ccab5f42cad6d24496db3850a2a43d8258221018ce87d2", + size: 536063208, + pointerSize: 134, + }, + xetHash: "879c5715c18a0b7f051dd33f70f0a5c8dd1522e0a43f6f75520f16167f29279b", + oid: "9eb98c817f04b051b3bcca591bcd4e03cec88018", + path: "tf_model.h5", + size: 536063208, + type: "file", + }, + { + oid: "fb140275c155a9c7c5a3b3e0e77a9e839594a938", + path: "vocab.txt", + size: 231508, + type: "file", + }, + ]); + }); + + it("should fetch the list of files from the repo, including last commit", async () => { + const cursor = listFiles({ + repo: { + name: "google-bert/bert-base-uncased", + type: "model", + }, + revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", + expand: true, + }); + + const files: ListFileEntry[] = []; + + for await (const entry of cursor) { + delete entry.securityFileStatus; // flaky + files.push(entry); + } + + assert.deepStrictEqual(files, [ + { + lastCommit: { + date: "2018-11-14T23:35:08.000Z", + id: "504939aa53e8ce310dba3dd2296dbe266c575de4", + title: "initial commit", + }, + oid: "dc08351d4dc0732d9c8af04070ced089b201ce2f", + path: ".gitattributes", + size: 345, + type: "file", + }, + { + lastCommit: { + date: "2019-06-18T09:06:51.000Z", + id: "bb3c1c3256d2598217df9889a14a2e811587891d", + title: "Update config.json", + }, + oid: "fca794a5f07ff8f963fe8b61e3694b0fb7f955df", + path: "config.json", + size: 313, + type: "file", + }, + { + lastCommit: { + date: "2019-06-18T09:06:34.000Z", + id: "3d2477d72b675a999d1b13ca822aaaf4908634ad", + title: "Update pytorch_model.bin", + }, + lfs: { + oid: "097417381d6c7230bd9e3557456d726de6e83245ec8b24f529f60198a67b203a", + size: 440473133, + pointerSize: 134, + }, + xetHash: "2d8408d3a894d02517d04956e2f7546ff08362594072f3527ce144b5212a3296", + oid: "ba5d19791be1dd7992e33bd61f20207b0f7f50a5", + path: "pytorch_model.bin", + size: 440473133, + type: "file", + }, + { + lastCommit: { + date: "2019-09-23T19:48:44.000Z", + id: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", + title: "Update tf_model.h5", + }, + lfs: { + oid: "a7a17d6d844b5de815ccab5f42cad6d24496db3850a2a43d8258221018ce87d2", + size: 536063208, + pointerSize: 134, + }, + xetHash: "879c5715c18a0b7f051dd33f70f0a5c8dd1522e0a43f6f75520f16167f29279b", + oid: "9eb98c817f04b051b3bcca591bcd4e03cec88018", + path: "tf_model.h5", + size: 536063208, + type: "file", + }, + { + lastCommit: { + date: "2018-11-14T23:35:08.000Z", + id: "2f07d813ca87c8c709147704c87210359ccf2309", + title: "Update vocab.txt", + }, + oid: "fb140275c155a9c7c5a3b3e0e77a9e839594a938", + path: "vocab.txt", + size: 231508, + type: "file", + }, + ]); + }); + + it("should fetch the list of files from the repo, including subfolders", async () => { + const cursor = listFiles({ + repo: { + name: "xsum", + type: "dataset", + }, + revision: "0f3ea2f2b55fcb11e71fb1e3aec6822e44ddcb0f", + recursive: true, + }); + + const files: ListFileEntry[] = []; + + for await (const entry of cursor) { + files.push(entry); + } + + assert(files.some((file) => file.path === "data/XSUM-EMNLP18-Summary-Data-Original.tar.gz")); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/list-files.ts b/node_modules/@huggingface/hub/src/lib/list-files.ts new file mode 100644 index 0000000000000000000000000000000000000000..70544c40f42838ae2c8f88e54f0bf23b9c5abbe4 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/list-files.ts @@ -0,0 +1,106 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiIndexTreeEntry } from "../types/api/api-index-tree"; +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { parseLinkHeader } from "../utils/parseLinkHeader"; +import { toRepoId } from "../utils/toRepoId"; + +export interface ListFileEntry { + type: "file" | "directory" | "unknown"; + size: number; + path: string; + /** + * Not available for bucket repos. + */ + oid?: string; + lfs?: { + oid: string; + size: number; + /** Size of the raw pointer file, 100~200 bytes */ + pointerSize: number; + }; + /** + * Xet-backed hash, a new protocol replacing LFS for big files. + */ + xetHash?: string; + /** + * Only fetched if `expand` is set to `true` in the `listFiles` call. + * + * Not available for bucket repos, use {@link uploadedAt} instead. + */ + lastCommit?: { + date: string; + id: string; + title: string; + }; + /** + * Only fetched if `expand` is set to `true` in the `listFiles` call. + * + * Only available for bucket repos. + */ + uploadedAt?: string; + /** + * Only fetched if `expand` is set to `true` in the `listFiles` call. + */ + securityFileStatus?: unknown; +} + +/** + * List files in a folder. To list ALL files in the directory, call it + * with {@link params.recursive} set to `true`. + */ +export async function* listFiles( + params: { + repo: RepoDesignation; + /** + * Do we want to list files in subdirectories? + */ + recursive?: boolean; + /** + * Eg 'data' for listing all files in the 'data' folder. Leave it empty to list all + * files in the repo. + */ + path?: string; + /** + * Fetch `lastCommit` and `securityFileStatus` for each file. + */ + expand?: boolean; + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): AsyncGenerator { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + const revision = repoId.type === "bucket" ? undefined : params.revision || "main"; + let url: string | undefined = `${params.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}/tree${ + revision ? `/${revision}` : "" + }${params.path ? "/" + params.path : ""}?recursive=${!!params.recursive}&expand=${!!params.expand}`; + + while (url) { + const res: Response = await (params.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : undefined), + }, + }); + + if (!res.ok) { + throw await createApiError(res); + } + + const items: ApiIndexTreeEntry[] = await res.json(); + + for (const item of items) { + yield item; + } + + const linkHeader = res.headers.get("Link"); + + url = linkHeader ? parseLinkHeader(linkHeader).next : undefined; + } +} diff --git a/node_modules/@huggingface/hub/src/lib/list-models.spec.ts b/node_modules/@huggingface/hub/src/lib/list-models.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..83c6830c02326853519260209a5de956dd142bc8 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/list-models.spec.ts @@ -0,0 +1,163 @@ +import { describe, expect, it } from "vitest"; +import type { ModelEntry } from "./list-models"; +import { listModels } from "./list-models"; + +describe("listModels", () => { + it("should list models for depth estimation", async () => { + const results: ModelEntry[] = []; + + for await (const entry of listModels({ + search: { owner: "Intel", task: "depth-estimation" }, + })) { + if (typeof entry.downloads === "number") { + entry.downloads = 0; + } + if (typeof entry.likes === "number") { + entry.likes = 0; + } + if (entry.updatedAt instanceof Date && !isNaN(entry.updatedAt.getTime())) { + entry.updatedAt = new Date(0); + } + + if (!["Intel/dpt-large", "Intel/dpt-hybrid-midas"].includes(entry.name)) { + expect(entry.task).to.equal("depth-estimation"); + continue; + } + + results.push(entry); + } + + results.sort((a, b) => a.id.localeCompare(b.id)); + + expect(results).deep.equal([ + { + id: "621ffdc136468d709f17e709", + name: "Intel/dpt-large", + private: false, + gated: false, + downloads: 0, + likes: 0, + task: "depth-estimation", + updatedAt: new Date(0), + }, + { + id: "638f07977559bf9a2b2b04ac", + name: "Intel/dpt-hybrid-midas", + gated: false, + private: false, + downloads: 0, + likes: 0, + task: "depth-estimation", + updatedAt: new Date(0), + }, + ]); + }); + + it("should list indonesian models with gguf format", async () => { + let count = 0; + for await (const entry of listModels({ + search: { tags: ["gguf", "id"] }, + additionalFields: ["tags"], + limit: 2, + })) { + count++; + expect(entry.tags).to.include("gguf"); + expect(entry.tags).to.include("id"); + } + + expect(count).to.equal(2); + }); + + it("should search model by name", async () => { + let count = 0; + for await (const entry of listModels({ + search: { query: "t5" }, + limit: 10, + })) { + count++; + expect(entry.name.toLocaleLowerCase()).to.include("t5"); + } + + expect(count).to.equal(10); + }); + + it("should search model by inference provider", async () => { + let count = 0; + for await (const entry of listModels({ + search: { inferenceProviders: ["together"] }, + additionalFields: ["inferenceProviderMapping"], + limit: 10, + })) { + count++; + if (Array.isArray(entry.inferenceProviderMapping)) { + expect(entry.inferenceProviderMapping.map(({ provider }) => provider)).to.include("together"); + } + } + + expect(count).to.equal(10); + }); + + it("should search model by several inference providers", async () => { + let count = 0; + const inferenceProviders = ["together", "replicate"]; + for await (const entry of listModels({ + search: { inferenceProviders }, + additionalFields: ["inferenceProviderMapping"], + limit: 10, + })) { + count++; + if (Array.isArray(entry.inferenceProviderMapping)) { + expect( + entry.inferenceProviderMapping.filter(({ provider }) => inferenceProviders.includes(provider)).length, + ).toBeGreaterThan(0); + } + } + + expect(count).to.equal(10); + }); + + it("should list meta-llama models with inference provider mapping", async () => { + let count = 0; + for await (const entry of listModels({ + search: { owner: "meta-llama" }, + additionalFields: ["inferenceProviderMapping"], + limit: 1, + })) { + count++; + expect(entry.inferenceProviderMapping).to.be.an("array").that.is.not.empty; + for (const item of entry.inferenceProviderMapping ?? []) { + expect(item).to.have.property("provider").that.is.a("string").and.is.not.empty; + expect(item).to.have.property("hfModelId").that.is.a("string").and.is.not.empty; + expect(item).to.have.property("providerId").that.is.a("string").and.is.not.empty; + } + } + + expect(count).to.equal(1); + }); + + it("should search models by apps", async () => { + let count = 0; + for await (const entry of listModels({ + search: { apps: ["mlx-lm"] }, + additionalFields: ["tags"], + limit: 10, + })) { + count++; + expect(entry.tags).to.include("mlx"); + } + + expect(count).to.equal(10); + }); + + it("should list models with filePaths", async () => { + for await (const entry of listModels({ + search: { owner: "huggingfacejs" }, + additionalFields: ["filePaths"], + })) { + expect(entry.name).to.equal("huggingfacejs/test-model"); + expect(entry.filePaths).to.be.an("array"); + expect(entry.filePaths).to.include(".gitattributes"); + expect(entry.filePaths).to.include("README.md"); + } + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/list-models.ts b/node_modules/@huggingface/hub/src/lib/list-models.ts new file mode 100644 index 0000000000000000000000000000000000000000..fae7b042e50d56548a5fc77d306f43b627e745c7 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/list-models.ts @@ -0,0 +1,190 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiModelInfo } from "../types/api/api-model"; +import type { CredentialsParams, PipelineType } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { parseLinkHeader } from "../utils/parseLinkHeader"; +import { normalizeInferenceProviderMapping } from "../utils/normalizeInferenceProviderMapping"; + +export const MODEL_EXPAND_KEYS = [ + "pipeline_tag", + "private", + "gated", + "downloads", + "likes", + "lastModified", +] as const satisfies readonly (keyof ApiModelInfo)[]; + +export const MODEL_EXPANDABLE_KEYS = [ + "author", + "cardData", + "config", + "createdAt", + "disabled", + "downloads", + "downloadsAllTime", + "gated", + "gitalyUid", + "inferenceProviderMapping", + "lastModified", + "library_name", + "likes", + "model-index", + "pipeline_tag", + "private", + "safetensors", + "sha", + "spaces", + "tags", + "transformersInfo", +] as const satisfies readonly (keyof ApiModelInfo)[]; + +export interface ModelDerivedFields { + filePaths: string[]; +} + +export const MODEL_DERIVED_FIELD_TO_API_KEY: Record = { + filePaths: "siblings", +}; + +export type ModelAdditionalField = + | Exclude<(typeof MODEL_EXPANDABLE_KEYS)[number], (typeof MODEL_EXPAND_KEYS)[number]> + | keyof ModelDerivedFields; + +export type ResolveModelAdditionalFields = Pick & + Pick; + +export interface ModelEntry { + id: string; + name: string; + private: boolean; + gated: false | "auto" | "manual"; + task?: PipelineType; + likes: number; + downloads: number; + updatedAt: Date; +} + +export async function* listModels( + params?: { + search?: { + /** + * Will search in the model name for matches + */ + query?: string; + owner?: string; + task?: PipelineType; + tags?: string[]; + /** + * Will search for models that have one of the inference providers in the list. + */ + inferenceProviders?: string[]; + /** + * Will search for models that support at least one of those local apps (eg "lmstudio", "mlx-lm", ...) + */ + apps?: string[]; + }; + hubUrl?: string; + additionalFields?: T[]; + /** + * Set to limit the number of models returned. + */ + limit?: number; + /** + * Sort models by a specific field. + */ + sort?: + | "createdAt" + | "downloads" + | "likes" + | "lastModified" + | "likes30d" + | "trendingScore" + | "num_parameters" + | "mainSize" + | "id"; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): AsyncGenerator> { + const accessToken = params && checkCredentials(params); + let totalToFetch = params?.limit ?? Infinity; + const additionalExpandKeys = + params?.additionalFields?.map( + (field) => MODEL_DERIVED_FIELD_TO_API_KEY[field as keyof ModelDerivedFields] ?? field, + ) ?? []; + const search = new URLSearchParams([ + ...Object.entries({ + limit: String(Math.min(totalToFetch, 500)), + ...(params?.search?.owner ? { author: params.search.owner } : undefined), + ...(params?.search?.task ? { pipeline_tag: params.search.task } : undefined), + ...(params?.search?.query ? { search: params.search.query } : undefined), + ...(params?.search?.inferenceProviders + ? { inference_provider: params.search.inferenceProviders.join(",") } + : undefined), + ...(params?.search?.apps ? { apps: params.search.apps.join(",") } : undefined), + ...(params?.sort ? { sort: params.sort } : undefined), + }), + ...(params?.search?.tags?.map((tag) => ["filter", tag]) ?? []), + ...MODEL_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), + ...additionalExpandKeys.map((val) => ["expand", val] satisfies [string, string]), + ]).toString(); + let url: string | undefined = `${params?.hubUrl || HUB_URL}/api/models?${search}`; + + while (url) { + const res: Response = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : undefined), + }, + }); + + if (!res.ok) { + throw await createApiError(res); + } + + const items: ApiModelInfo[] = await res.json(); + + for (const item of items) { + const additional: Record = {}; + if (params?.additionalFields) { + for (const field of params.additionalFields) { + if (field === "filePaths") { + additional.filePaths = (item.siblings ?? []).map((s) => s.rfilename); + } else if (field === "inferenceProviderMapping" && item.inferenceProviderMapping) { + additional.inferenceProviderMapping = normalizeInferenceProviderMapping( + item.id, + item.inferenceProviderMapping, + ); + } else { + additional[field] = item[field as keyof ApiModelInfo]; + } + } + } + + yield { + ...additional, + id: item._id, + name: item.id, + private: item.private, + task: item.pipeline_tag, + downloads: item.downloads, + gated: item.gated, + likes: item.likes, + updatedAt: new Date(item.lastModified), + } as ModelEntry & ResolveModelAdditionalFields; + totalToFetch--; + + if (totalToFetch <= 0) { + return; + } + } + + const linkHeader = res.headers.get("Link"); + + url = linkHeader ? parseLinkHeader(linkHeader).next : undefined; + // Could update url to reduce the limit if we don't need the whole 500 of the next batch. + } +} diff --git a/node_modules/@huggingface/hub/src/lib/list-spaces.spec.ts b/node_modules/@huggingface/hub/src/lib/list-spaces.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..3cc5999137a7837ae4312374a7a89ec46985dff3 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/list-spaces.spec.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import type { SpaceEntry } from "./list-spaces"; +import { listSpaces } from "./list-spaces"; + +describe("listSpaces", () => { + it("should list spaces for Microsoft", async () => { + const results: SpaceEntry[] = []; + + for await (const entry of listSpaces({ + search: { owner: "microsoft" }, + additionalFields: ["subdomain"], + })) { + if (entry.name !== "microsoft/visual_chatgpt") { + continue; + } + if (typeof entry.likes === "number") { + entry.likes = 0; + } + if (entry.updatedAt instanceof Date && !isNaN(entry.updatedAt.getTime())) { + entry.updatedAt = new Date(0); + } + + results.push(entry); + } + + results.sort((a, b) => a.id.localeCompare(b.id)); + + expect(results).deep.equal([ + { + id: "6409a392bbc73d022c58c980", + name: "microsoft/visual_chatgpt", + private: false, + likes: 0, + sdk: "gradio", + subdomain: "microsoft-visual-chatgpt", + updatedAt: new Date(0), + }, + ]); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/list-spaces.ts b/node_modules/@huggingface/hub/src/lib/list-spaces.ts new file mode 100644 index 0000000000000000000000000000000000000000..d1bd4d7fa86662dc12c19e58327b4f8d47124abc --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/list-spaces.ts @@ -0,0 +1,116 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiSpaceInfo } from "../types/api/api-space"; +import type { CredentialsParams, SpaceSdk } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { parseLinkHeader } from "../utils/parseLinkHeader"; +import { pick } from "../utils/pick"; + +export const SPACE_EXPAND_KEYS = [ + "sdk", + "likes", + "private", + "lastModified", +] as const satisfies readonly (keyof ApiSpaceInfo)[]; +export const SPACE_EXPANDABLE_KEYS = [ + "author", + "cardData", + "datasets", + "disabled", + "gitalyUid", + "lastModified", + "createdAt", + "likes", + "private", + "runtime", + "sdk", + // "siblings", + "sha", + "subdomain", + "tags", + "models", +] as const satisfies readonly (keyof ApiSpaceInfo)[]; + +export interface SpaceEntry { + id: string; + name: string; + sdk?: SpaceSdk; + likes: number; + private: boolean; + updatedAt: Date; + // Use additionalFields to fetch the fields from ApiSpaceInfo +} + +export async function* listSpaces< + const T extends Exclude<(typeof SPACE_EXPANDABLE_KEYS)[number], (typeof SPACE_EXPAND_KEYS)[number]> = never, +>( + params?: { + search?: { + /** + * Will search in the space name for matches + */ + query?: string; + owner?: string; + tags?: string[]; + }; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + /** + * Additional fields to fetch from huggingface.co. + */ + additionalFields?: T[]; + /** + * Sort spaces by a specific field. + */ + sort?: "createdAt" | "downloads" | "likes" | "lastModified" | "likes30d" | "trendingScore" | "mainSize" | "id"; + } & Partial, +): AsyncGenerator> { + const accessToken = params && checkCredentials(params); + const search = new URLSearchParams([ + ...Object.entries({ + limit: "500", + ...(params?.search?.owner ? { author: params.search.owner } : undefined), + ...(params?.search?.query ? { search: params.search.query } : undefined), + ...(params?.sort ? { sort: params.sort } : undefined), + }), + ...(params?.search?.tags?.map((tag) => ["filter", tag]) ?? []), + ...[...SPACE_EXPAND_KEYS, ...(params?.additionalFields ?? [])].map( + (val) => ["expand", val] satisfies [string, string], + ), + ]).toString(); + let url: string | undefined = `${params?.hubUrl || HUB_URL}/api/spaces?${search}`; + + while (url) { + const res: Response = await (params?.fetch ?? fetch)(url, { + headers: { + accept: "application/json", + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : undefined), + }, + }); + + if (!res.ok) { + throw await createApiError(res); + } + + const items: ApiSpaceInfo[] = await res.json(); + + for (const item of items) { + yield { + ...(params?.additionalFields && pick(item, params.additionalFields)), + id: item._id, + name: item.id, + sdk: item.sdk, + likes: item.likes, + private: item.private, + updatedAt: new Date(item.lastModified), + } as SpaceEntry & Pick; + } + + const linkHeader = res.headers.get("Link"); + + url = linkHeader ? parseLinkHeader(linkHeader).next : undefined; + } +} diff --git a/node_modules/@huggingface/hub/src/lib/model-info.spec.ts b/node_modules/@huggingface/hub/src/lib/model-info.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..b57294dce10c89bcdb01facbfeb717077a4c82ff --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/model-info.spec.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +import { modelInfo } from "./model-info"; +import type { ModelEntry } from "./list-models"; +import type { ApiModelInfo } from "../types/api/api-model"; + +describe("modelInfo", () => { + it("should return the model info", async () => { + const info = await modelInfo({ + name: "openai-community/gpt2", + }); + expect(info).toEqual({ + id: "621ffdc036468d709f17434d", + downloads: expect.any(Number), + gated: false, + name: "openai-community/gpt2", + updatedAt: expect.any(Date), + likes: expect.any(Number), + task: "text-generation", + private: false, + }); + }); + + it("should return the model info with author", async () => { + const info: ModelEntry & Pick = await modelInfo({ + name: "openai-community/gpt2", + additionalFields: ["author"], + }); + expect(info).toEqual({ + id: "621ffdc036468d709f17434d", + downloads: expect.any(Number), + author: "openai-community", + gated: false, + name: "openai-community/gpt2", + updatedAt: expect.any(Date), + likes: expect.any(Number), + task: "text-generation", + private: false, + }); + }); + + it("should return the model info for a specific revision", async () => { + const info: ModelEntry & Pick = await modelInfo({ + name: "openai-community/gpt2", + additionalFields: ["sha"], + revision: "f27b190eeac4c2302d24068eabf5e9d6044389ae", + }); + expect(info).toEqual({ + id: "621ffdc036468d709f17434d", + downloads: expect.any(Number), + gated: false, + name: "openai-community/gpt2", + updatedAt: expect.any(Date), + likes: expect.any(Number), + task: "text-generation", + private: false, + sha: "f27b190eeac4c2302d24068eabf5e9d6044389ae", + }); + }); + + it("should return model info with filePaths", async () => { + const info = await modelInfo({ + name: "huggingfacejs/test-model", + additionalFields: ["filePaths"], + }); + expect(info.filePaths).to.be.an("array"); + expect(info.filePaths).to.include(".gitattributes"); + expect(info.filePaths).to.include("README.md"); + }); + + it("should return model info deepseek-ai models with inference provider mapping", async () => { + const info = await modelInfo({ + name: "deepseek-ai/DeepSeek-R1-0528", + additionalFields: ["inferenceProviderMapping"], + }); + + expect(info.inferenceProviderMapping).toBeDefined(); + expect(info.inferenceProviderMapping).toBeInstanceOf(Array); + expect(info.inferenceProviderMapping?.length).toBeGreaterThan(0); + info.inferenceProviderMapping?.forEach((item) => { + expect(item).toHaveProperty("provider"); + expect(item).toHaveProperty("hfModelId", "deepseek-ai/DeepSeek-R1-0528"); + expect(item).toHaveProperty("providerId"); + }); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/model-info.ts b/node_modules/@huggingface/hub/src/lib/model-info.ts new file mode 100644 index 0000000000000000000000000000000000000000..598e5a03a67ffe2efc4d003f81c1d298c9e338be --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/model-info.ts @@ -0,0 +1,84 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiModelInfo } from "../types/api/api-model"; +import type { CredentialsParams } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { normalizeInferenceProviderMapping } from "../utils/normalizeInferenceProviderMapping"; +import { + MODEL_EXPAND_KEYS, + MODEL_DERIVED_FIELD_TO_API_KEY, + type ModelAdditionalField, + type ModelDerivedFields, + type ResolveModelAdditionalFields, + type ModelEntry, +} from "./list-models"; + +export async function modelInfo( + params: { + name: string; + hubUrl?: string; + additionalFields?: T[]; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + */ + revision?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise> { + const accessToken = params && checkCredentials(params); + + const additionalExpandKeys = + params?.additionalFields?.map( + (field) => MODEL_DERIVED_FIELD_TO_API_KEY[field as keyof ModelDerivedFields] ?? field, + ) ?? []; + + const search = new URLSearchParams([ + ...MODEL_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), + ...additionalExpandKeys.map((val) => ["expand", val] satisfies [string, string]), + ]).toString(); + + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/models/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD", + )}?${search.toString()}`, + { + headers: { + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }, + ); + + if (!response.ok) { + throw await createApiError(response); + } + + const data: ApiModelInfo = await response.json(); + + const additional: Record = {}; + if (params?.additionalFields) { + for (const field of params.additionalFields) { + if (field === "filePaths") { + additional.filePaths = (data.siblings ?? []).map((s) => s.rfilename); + } else if (field === "inferenceProviderMapping" && data.inferenceProviderMapping) { + additional.inferenceProviderMapping = normalizeInferenceProviderMapping(data.id, data.inferenceProviderMapping); + } else { + additional[field] = data[field as keyof ApiModelInfo]; + } + } + } + + return { + ...additional, + id: data._id, + name: data.id, + private: data.private, + task: data.pipeline_tag, + downloads: data.downloads, + gated: data.gated, + likes: data.likes, + updatedAt: new Date(data.lastModified), + } as ModelEntry & ResolveModelAdditionalFields; +} diff --git a/node_modules/@huggingface/hub/src/lib/oauth-handle-redirect.spec.ts b/node_modules/@huggingface/hub/src/lib/oauth-handle-redirect.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..f06f6a40e0dcdb6621ea75dedd369c3771b11561 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/oauth-handle-redirect.spec.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { TEST_COOKIE, TEST_HUB_URL } from "../test/consts"; +import { oauthLoginUrl } from "./oauth-login-url"; +import { oauthHandleRedirect } from "./oauth-handle-redirect"; + +describe("oauthHandleRedirect", () => { + it("should work", async () => { + const localStorage = { + nonce: undefined, + codeVerifier: undefined, + }; + const url = await oauthLoginUrl({ + clientId: "dummy-app", + redirectUrl: "http://localhost:3000", + localStorage, + scopes: "openid profile email", + hubUrl: TEST_HUB_URL, + }); + const resp = await fetch(url, { + method: "POST", + headers: { + Cookie: `token=${TEST_COOKIE}`, + }, + redirect: "manual", + }); + if (resp.status !== 303) { + throw new Error(`Failed to fetch url ${url}: ${resp.status} ${resp.statusText}`); + } + const location = resp.headers.get("Location"); + if (!location) { + throw new Error(`No location header in response`); + } + const result = await oauthHandleRedirect({ + redirectedUrl: location, + codeVerifier: localStorage.codeVerifier, + nonce: localStorage.nonce, + hubUrl: TEST_HUB_URL, + }); + + if (!result) { + throw new Error("Expected result to be defined"); + } + expect(result.accessToken).toEqual(expect.any(String)); + expect(result.accessTokenExpiresAt).toBeInstanceOf(Date); + expect(result.accessTokenExpiresAt.getTime()).toBeGreaterThan(Date.now()); + expect(result.scope).toEqual(expect.any(String)); + expect(result.userInfo).toEqual({ + sub: "62f264b9f3c90f4b6514a269", + name: "@huggingface/hub CI bot", + preferred_username: "hub.js", + email_verified: true, + email: "eliott@huggingface.co", + isPro: false, + picture: "https://hub-ci.huggingface.co/avatars/934b830e9fdaa879487852f79eef7165.svg", + profile: "https://hub-ci.huggingface.co/hub.js", + website: "https://github.com/huggingface/hub.js", + orgs: [], + }); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/oauth-handle-redirect.ts b/node_modules/@huggingface/hub/src/lib/oauth-handle-redirect.ts new file mode 100644 index 0000000000000000000000000000000000000000..f5ed66bae8289edc5b1480f0d3870a966ed573d5 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/oauth-handle-redirect.ts @@ -0,0 +1,343 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; + +export interface UserInfo { + /** + * OpenID Connect field. Unique identifier for the user, even in case of rename. + */ + sub: string; + /** + * OpenID Connect field. The user's full name. + */ + name: string; + /** + * OpenID Connect field. The user's username. + */ + preferred_username: string; + /** + * OpenID Connect field, available if scope "email" was granted. + */ + email_verified?: boolean; + /** + * OpenID Connect field, available if scope "email" was granted. + */ + email?: string; + /** + * OpenID Connect field. The user's profile picture URL. + */ + picture: string; + /** + * OpenID Connect field. The user's profile URL. + */ + profile: string; + /** + * OpenID Connect field. The user's website URL. + */ + website?: string; + + /** + * Hugging Face field. Whether the user is a pro user. + */ + isPro: boolean; + /** + * Hugging Face field. Whether the user has a payment method set up. Needs "read-billing" scope. + */ + canPay?: boolean; + /** + * Hugging Face field. The user's orgs + */ + orgs?: Array<{ + /** + * OpenID Connect field. Unique identifier for the org. + */ + sub: string; + /** + * OpenID Connect field. The org's full name. + */ + name: string; + /** + * OpenID Connect field. The org's username. + */ + preferred_username: string; + /** + * OpenID Connect field. The org's profile picture URL. + */ + picture: string; + + /** + * Hugging Face field. The org's plan (e.g., "enterprise", "team"). + */ + plan?: string; + /** + * Hugging Face field. Whether the org has a payment method set up. Needs "read-billing" scope, and the user needs to approve access to the org in the OAuth page. + */ + canPay?: boolean; + /** + * Hugging Face field. The user's role in the org. The user needs to approve access to the org in the OAuth page. + */ + roleInOrg?: string; + /** + * @deprecated Use securityRestrictions instead with "sso" + * HuggingFace field. When the user granted the oauth app access to the org, but didn't complete SSO. + * + * Should never happen directly after the oauth flow. + */ + pendingSSO?: boolean; + /** + * @deprecated Use securityRestrictions instead with "mfa" + * + * HuggingFace field. When the user granted the oauth app access to the org, but didn't complete MFA. + * + * Should never happen directly after the oauth flow. + */ + missingMFA?: boolean; + /** + * HuggingFace field. When the user granted the oauth app access to the org, but didn't complete following security restrictions. + * + * Should never happen directly after the oauth flow. + */ + securityRestrictions?: ("mfa" | "sso" | "ip" | "token-policy")[]; + }>; +} + +export interface OAuthResult { + accessToken: string; + accessTokenExpiresAt: Date; + userInfo: UserInfo; + /** + * State passed to the OAuth provider in the original request to the OAuth provider. + */ + state?: string; + /** + * Granted scope + */ + scope: string; +} + +/** + * To call after the OAuth provider redirects back to the app. + * + * There is also a helper function {@link oauthHandleRedirectIfPresent}, which will call `oauthHandleRedirect` if the URL contains an oauth code + * in the query parameters and return `false` otherwise. + */ +export async function oauthHandleRedirect(opts?: { + /** + * The URL of the hub. Defaults to {@link HUB_URL}. + */ + hubUrl?: string; + /** + * The URL to analyze. + * + * @default window.location.href + */ + redirectedUrl?: string; + /** + * nonce generated by oauthLoginUrl + * + * @default localStorage.getItem("huggingface.co:oauth:nonce") + */ + nonce?: string; + /** + * codeVerifier generated by oauthLoginUrl + * + * @default localStorage.getItem("huggingface.co:oauth:code_verifier") + */ + codeVerifier?: string; +}): Promise { + if (typeof window === "undefined" && !opts?.redirectedUrl) { + throw new Error("oauthHandleRedirect is only available in the browser, unless you provide redirectedUrl"); + } + if (typeof localStorage === "undefined" && (!opts?.nonce || !opts?.codeVerifier)) { + throw new Error( + "oauthHandleRedirect requires localStorage to be available, unless you provide nonce and codeVerifier", + ); + } + + const redirectedUrl = opts?.redirectedUrl ?? window.location.href; + const searchParams = (() => { + try { + return new URL(redirectedUrl).searchParams; + } catch (err) { + throw new Error("Failed to parse redirected URL: " + redirectedUrl); + } + })(); + + const [error, errorDescription] = [searchParams.get("error"), searchParams.get("error_description")]; + + if (error) { + throw new Error(`${error}: ${errorDescription}`); + } + + const code = searchParams.get("code"); + const nonce = opts?.nonce ?? localStorage.getItem("huggingface.co:oauth:nonce"); + + if (!code) { + throw new Error("Missing oauth code from query parameters in redirected URL: " + redirectedUrl); + } + + if (!nonce) { + throw new Error("Missing oauth nonce from localStorage"); + } + + const codeVerifier = opts?.codeVerifier ?? localStorage.getItem("huggingface.co:oauth:code_verifier"); + + if (!codeVerifier) { + throw new Error("Missing oauth code_verifier from localStorage"); + } + + const state = searchParams.get("state"); + + if (!state) { + throw new Error("Missing oauth state from query parameters in redirected URL"); + } + + let parsedState: { nonce: string; redirectUri: string; state?: string }; + + try { + parsedState = JSON.parse(state); + } catch { + throw new Error("Invalid oauth state in redirected URL, unable to parse JSON: " + state); + } + + if (parsedState.nonce !== nonce) { + throw new Error("Invalid oauth state in redirected URL"); + } + + const hubUrl = opts?.hubUrl || HUB_URL; + + const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; + const openidConfigRes = await fetch(openidConfigUrl, { + headers: { + Accept: "application/json", + }, + }); + + if (!openidConfigRes.ok) { + throw await createApiError(openidConfigRes); + } + + const openidConfig: { + authorization_endpoint: string; + token_endpoint: string; + userinfo_endpoint: string; + } = await openidConfigRes.json(); + + const tokenRes = await fetch(openidConfig.token_endpoint, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ + grant_type: "authorization_code", + code, + redirect_uri: parsedState.redirectUri, + code_verifier: codeVerifier, + }).toString(), + }); + + if (!opts?.codeVerifier) { + localStorage.removeItem("huggingface.co:oauth:code_verifier"); + } + if (!opts?.nonce) { + localStorage.removeItem("huggingface.co:oauth:nonce"); + } + + if (!tokenRes.ok) { + throw await createApiError(tokenRes); + } + + const token: { + access_token: string; + expires_in: number; + id_token: string; + // refresh_token: string; + scope: string; + token_type: string; + } = await tokenRes.json(); + + const accessTokenExpiresAt = new Date(Date.now() + token.expires_in * 1000); + + const userInfoRes = await fetch(openidConfig.userinfo_endpoint, { + headers: { + Authorization: `Bearer ${token.access_token}`, + }, + }); + + if (!userInfoRes.ok) { + throw await createApiError(userInfoRes); + } + + const userInfo: UserInfo = await userInfoRes.json(); + + return { + accessToken: token.access_token, + accessTokenExpiresAt, + userInfo: userInfo, + state: parsedState.state, + scope: token.scope, + }; +} + +// if (code && !nonce) { +// console.warn("Missing oauth nonce from localStorage"); +// } + +/** + * To call after the OAuth provider redirects back to the app. + * + * It returns false if the URL does not contain an oauth code in the query parameters, otherwise + * it calls {@link oauthHandleRedirect}. + * + * Depending on your app, you may want to call {@link oauthHandleRedirect} directly instead. + */ +export async function oauthHandleRedirectIfPresent(opts?: { + /** + * The URL of the hub. Defaults to {@link HUB_URL}. + */ + hubUrl?: string; + /** + * The URL to analyze. + * + * @default window.location.href + */ + redirectedUrl?: string; + /** + * nonce generated by oauthLoginUrl + * + * @default localStorage.getItem("huggingface.co:oauth:nonce") + */ + nonce?: string; + /** + * codeVerifier generated by oauthLoginUrl + * + * @default localStorage.getItem("huggingface.co:oauth:code_verifier") + */ + codeVerifier?: string; +}): Promise { + if (typeof window === "undefined" && !opts?.redirectedUrl) { + throw new Error("oauthHandleRedirect is only available in the browser, unless you provide redirectedUrl"); + } + if (typeof localStorage === "undefined" && (!opts?.nonce || !opts?.codeVerifier)) { + throw new Error( + "oauthHandleRedirect requires localStorage to be available, unless you provide nonce and codeVerifier", + ); + } + const searchParams = new URLSearchParams(opts?.redirectedUrl ?? window.location.search); + + if (searchParams.has("error")) { + return oauthHandleRedirect(opts); + } + + if (searchParams.has("code")) { + if (!localStorage.getItem("huggingface.co:oauth:nonce")) { + console.warn( + "Missing oauth nonce from localStorage. This can happen when the user refreshes the page after logging in, without changing the URL.", + ); + return false; + } + + return oauthHandleRedirect(opts); + } + + return false; +} diff --git a/node_modules/@huggingface/hub/src/lib/oauth-login-url.ts b/node_modules/@huggingface/hub/src/lib/oauth-login-url.ts new file mode 100644 index 0000000000000000000000000000000000000000..a7ba0b1cdab559be1fef5c8001eb9faacd810ab5 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/oauth-login-url.ts @@ -0,0 +1,166 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import { base64FromBytes } from "../utils/base64FromBytes"; + +/** + * Use "Sign in with Hub" to authenticate a user, and get oauth user info / access token. + * + * Returns an url to redirect to. After the user is redirected back to your app, call `oauthHandleRedirect` to get the oauth user info / access token. + * + * When called from inside a static Space with OAuth enabled, it will load the config from the space, otherwise you need to at least specify + * the client ID of your OAuth App. + * + * @example + * ```ts + * import { oauthLoginUrl, oauthHandleRedirectIfPresent } from "@huggingface/hub"; + * + * const oauthResult = await oauthHandleRedirectIfPresent(); + * + * if (!oauthResult) { + * // If the user is not logged in, redirect to the login page + * window.location.href = await oauthLoginUrl(); + * } + * + * // You can use oauthResult.accessToken, oauthResult.accessTokenExpiresAt and oauthResult.userInfo + * console.log(oauthResult); + * ``` + * + * (Theoretically, this function could be used to authenticate a user for any OAuth provider supporting PKCE and OpenID Connect by changing `hubUrl`, + * but it is currently only tested with the Hugging Face Hub.) + */ +export async function oauthLoginUrl(opts?: { + /** + * OAuth client ID. + * + * For static Spaces, you can omit this and it will be loaded from the Space config, as long as `hf_oauth: true` is present in the README.md's metadata. + * For other Spaces, it is available to the backend in the OAUTH_CLIENT_ID environment variable, as long as `hf_oauth: true` is present in the README.md's metadata. + * + * You can also create a Developer Application at https://huggingface.co/settings/connected-applications and use its client ID. + */ + clientId?: string; + hubUrl?: string; + /** + * OAuth scope, a list of space-separated scopes. + * + * For static Spaces, you can omit this and it will be loaded from the Space config, as long as `hf_oauth: true` is present in the README.md's metadata. + * For other Spaces, it is available to the backend in the OAUTH_SCOPES environment variable, as long as `hf_oauth: true` is present in the README.md's metadata. + * + * Defaults to "openid profile". + * + * You can also create a Developer Application at https://huggingface.co/settings/connected-applications and use its scopes. + * + * See https://huggingface.co/docs/hub/oauth for a list of available scopes. + */ + scopes?: string; + /** + * Redirect URI, defaults to the current URL. + * + * For Spaces, any URL within the Space is allowed. + * + * For Developer Applications, you can add any URL you want to the list of allowed redirect URIs at https://huggingface.co/settings/connected-applications. + */ + redirectUrl?: string; + /** + * State to pass to the OAuth provider, which will be returned in the call to `oauthLogin` after the redirect. + */ + state?: string; + /** + * If provided, will be filled with the code verifier and nonce used for the OAuth flow, + * instead of using localStorage. + * + * When calling {@link `oauthHandleRedirectIfPresent`} or {@link `oauthHandleRedirect`} you will need to provide the same values. + */ + localStorage?: { + codeVerifier?: string; + nonce?: string; + }; +}): Promise { + if (typeof window === "undefined" && (!opts?.redirectUrl || !opts?.clientId)) { + throw new Error("oauthLogin is only available in the browser, unless you provide clientId and redirectUrl"); + } + if (typeof localStorage === "undefined" && !opts?.localStorage) { + throw new Error( + "oauthLogin requires localStorage to be available in the context, unless you provide a localStorage empty object as argument", + ); + } + + const hubUrl = opts?.hubUrl || HUB_URL; + const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; + const openidConfigRes = await fetch(openidConfigUrl, { + headers: { + Accept: "application/json", + }, + }); + + if (!openidConfigRes.ok) { + throw await createApiError(openidConfigRes); + } + + const opendidConfig: { + authorization_endpoint: string; + token_endpoint: string; + userinfo_endpoint: string; + } = await openidConfigRes.json(); + + const newNonce = globalThis.crypto.randomUUID(); + // Two random UUIDs concatenated together, because min length is 43 and max length is 128 + const newCodeVerifier = globalThis.crypto.randomUUID() + globalThis.crypto.randomUUID(); + + if (opts?.localStorage) { + if (opts.localStorage.codeVerifier !== undefined && opts.localStorage.codeVerifier !== null) { + throw new Error( + "localStorage.codeVerifier must be initially set to null or undefined, and will be filled by oauthLoginUrl", + ); + } + if (opts.localStorage.nonce !== undefined && opts.localStorage.nonce !== null) { + throw new Error( + "localStorage.nonce must be initially set to null or undefined, and will be filled by oauthLoginUrl", + ); + } + opts.localStorage.codeVerifier = newCodeVerifier; + opts.localStorage.nonce = newNonce; + } else { + localStorage.setItem("huggingface.co:oauth:nonce", newNonce); + localStorage.setItem("huggingface.co:oauth:code_verifier", newCodeVerifier); + } + + const redirectUri = opts?.redirectUrl || (typeof window !== "undefined" ? window.location.href : undefined); + if (!redirectUri) { + throw new Error("Missing redirectUrl"); + } + const state = JSON.stringify({ + nonce: newNonce, + redirectUri, + state: opts?.state, + }); + + const variables: Record | null = + // @ts-expect-error window.huggingface is defined inside static Spaces. + typeof window !== "undefined" ? (window.huggingface?.variables ?? null) : null; + + const clientId = opts?.clientId || variables?.OAUTH_CLIENT_ID; + + if (!clientId) { + if (variables) { + throw new Error("Missing clientId, please add hf_oauth: true to the README.md's metadata in your static Space"); + } + throw new Error("Missing clientId"); + } + + const challenge = base64FromBytes( + new Uint8Array(await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(newCodeVerifier))), + ) + .replace(/[+]/g, "-") + .replace(/[/]/g, "_") + .replace(/=/g, ""); + + return `${opendidConfig.authorization_endpoint}?${new URLSearchParams({ + client_id: clientId, + scope: opts?.scopes || variables?.OAUTH_SCOPES || "openid profile", + response_type: "code", + redirect_uri: redirectUri, + state, + code_challenge: challenge, + code_challenge_method: "S256", + }).toString()}`; +} diff --git a/node_modules/@huggingface/hub/src/lib/parse-safetensors-metadata.spec.ts b/node_modules/@huggingface/hub/src/lib/parse-safetensors-metadata.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..665f3c022c764c4d1b9d96efcd2cb928fd186bd5 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/parse-safetensors-metadata.spec.ts @@ -0,0 +1,477 @@ +import { assert, it, describe } from "vitest"; +import { + parseSafetensorsMetadata, + parseSafetensorsShardFilename, + globMatch, + isQuantizedTensor, + matchesCompressedTensorsTarget, +} from "./parse-safetensors-metadata"; +import { sum } from "../utils/sum"; + +describe("parseSafetensorsMetadata", () => { + it("fetch info for single-file (with the default conventional filename)", async () => { + const parse = await parseSafetensorsMetadata({ + repo: "google-bert/bert-base-uncased", + computeParametersCount: true, + revision: "86b5e0934494bd15c9632b12f734a8a67f723594", + }); + + assert(!parse.sharded); + assert.deepStrictEqual(parse.header.__metadata__, { format: "pt" }); + + // Example of one tensor (the header contains many tensors) + + assert.deepStrictEqual(parse.header["bert.embeddings.LayerNorm.beta"], { + dtype: "F32", + shape: [768], + data_offsets: [0, 3072], + }); + + assert.deepStrictEqual(parse.parameterCount, { F32: 110_106_428 }); + assert.deepStrictEqual(sum(Object.values(parse.parameterCount)), 110_106_428); + // total params = 110m + + assert.deepStrictEqual(parse.filepaths, ["model.safetensors"]); + }); + + it("fetch info for sharded (with the default conventional filename)", async () => { + const parse = await parseSafetensorsMetadata({ + repo: "bigscience/bloom", + computeParametersCount: true, + revision: "053d9cd9fbe814e091294f67fcfedb3397b954bb", + }); + + assert(parse.sharded); + + assert.strictEqual(Object.keys(parse.headers).length, 72); + // This model has 72 shards! + + // Example of one tensor inside one file + + assert.deepStrictEqual(parse.headers["model_00012-of-00072.safetensors"]["h.10.input_layernorm.weight"], { + dtype: "BF16", + shape: [14336], + data_offsets: [3288649728, 3288678400], + }); + + assert.deepStrictEqual(parse.parameterCount, { BF16: 176_247_271_424 }); + assert.deepStrictEqual(sum(Object.values(parse.parameterCount)), 176_247_271_424); + // total params = 176B + + assert.strictEqual(parse.filepaths[0], "model.safetensors.index.json"); + assert.strictEqual(parse.filepaths.length, 73); // 1 index + 72 shards + assert.ok(parse.filepaths.includes("model_00012-of-00072.safetensors")); + }); + + it("fetch info for single-file with multiple dtypes", async () => { + const parse = await parseSafetensorsMetadata({ + repo: "roberta-base", + computeParametersCount: true, + revision: "e2da8e2f811d1448a5b465c236feacd80ffbac7b", + }); + + assert(!parse.sharded); + + assert.deepStrictEqual(parse.parameterCount, { F32: 124_697_433, I64: 514 }); + assert.deepStrictEqual(sum(Object.values(parse.parameterCount)), 124_697_947); + // total params = 124m + }); + + it("fetch info for single-file with file path", async () => { + const parse = await parseSafetensorsMetadata({ + repo: "CompVis/stable-diffusion-v1-4", + computeParametersCount: true, + path: "unet/diffusion_pytorch_model.safetensors", + revision: "133a221b8aa7292a167afc5127cb63fb5005638b", + }); + + assert(!parse.sharded); + assert.deepStrictEqual(parse.header.__metadata__, { format: "pt" }); + + // Example of one tensor (the header contains many tensors) + + assert.deepStrictEqual(parse.header["up_blocks.3.resnets.0.norm2.bias"], { + dtype: "F32", + shape: [320], + data_offsets: [3_409_382_416, 3_409_383_696], + }); + + assert.deepStrictEqual(parse.parameterCount, { F32: 859_520_964 }); + assert.deepStrictEqual(sum(Object.values(parse.parameterCount)), 859_520_964); + + assert.deepStrictEqual(parse.filepaths, ["unet/diffusion_pytorch_model.safetensors"]); + }); + + it("fetch info for sharded with file path", async () => { + const parse = await parseSafetensorsMetadata({ + repo: "Alignment-Lab-AI/ALAI-gemma-7b", + computeParametersCount: true, + path: "7b/1/model.safetensors.index.json", + revision: "37e307261fe97bbf8b2463d61dbdd1a10daa264c", + }); + + assert(parse.sharded); + + assert.strictEqual(Object.keys(parse.headers).length, 4); + + assert.deepStrictEqual(parse.headers["model-00004-of-00004.safetensors"]["model.layers.24.mlp.up_proj.weight"], { + dtype: "BF16", + shape: [24576, 3072], + data_offsets: [301996032, 452990976], + }); + + assert.deepStrictEqual(parse.parameterCount, { BF16: 8_537_680_896 }); + assert.deepStrictEqual(sum(Object.values(parse.parameterCount)), 8_537_680_896); + + assert.strictEqual(parse.filepaths[0], "7b/1/model.safetensors.index.json"); + assert.strictEqual(parse.filepaths.length, 5); // 1 index + 4 shards + assert.ok(parse.filepaths.includes("7b/1/model-00001-of-00004.safetensors")); + assert.ok(parse.filepaths.includes("7b/1/model-00004-of-00004.safetensors")); + }); + + it("fetch info for sharded, but get param count directly from metadata", async () => { + const parse = await parseSafetensorsMetadata({ + repo: "hf-internal-testing/sharded-model-metadata-num-parameters", + computeParametersCount: true, + revision: "999395eb3db277f3d7a0393402b02486ca91cef8", + }); + + assert(parse.sharded); + assert.deepStrictEqual(parse.parameterTotal, 109_482_240); + // total params = 109M + }); + + it("fetch info for single-file, but get param count directly from metadata", async () => { + const parse = await parseSafetensorsMetadata({ + repo: "hf-internal-testing/single-file-model", + computeParametersCount: true, + revision: "75fcd3fed0285ac7f1092897ff2aefdf24bf872e", + }); + + assert(!parse.sharded); + assert.deepStrictEqual(parse.parameterTotal, 109_482_240); + }); + + it("should detect sharded safetensors filename", async () => { + const safetensorsFilename = "model_00005-of-00072.safetensors"; // https://huggingface.co/bigscience/bloom/blob/4d8e28c67403974b0f17a4ac5992e4ba0b0dbb6f/model_00005-of-00072.safetensors + const safetensorsShardFileInfo = parseSafetensorsShardFilename(safetensorsFilename); + + assert.strictEqual(safetensorsShardFileInfo?.prefix, "model_"); + assert.strictEqual(safetensorsShardFileInfo?.basePrefix, "model"); + assert.strictEqual(safetensorsShardFileInfo?.shard, "00005"); + assert.strictEqual(safetensorsShardFileInfo?.total, "00072"); + }); + + it("should detect sharded safetensors filename with 6 digits", async () => { + const safetensorsFilename = "model-00001-of-000163.safetensors"; // https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp/blob/main/model-00001-of-000163.safetensors + const safetensorsShardFileInfo = parseSafetensorsShardFilename(safetensorsFilename); + + assert.strictEqual(safetensorsShardFileInfo?.prefix, "model-"); + assert.strictEqual(safetensorsShardFileInfo?.basePrefix, "model"); + assert.strictEqual(safetensorsShardFileInfo?.shard, "00001"); + assert.strictEqual(safetensorsShardFileInfo?.total, "000163"); + }); + + it("should support sub-byte data types", async () => { + const newDataTypes: Array<"F4" | "F6_E2M3" | "F6_E3M2" | "E8M0"> = ["F4", "F6_E2M3", "F6_E3M2", "E8M0"]; + + for (const dtype of newDataTypes) { + const tensorInfo = { + dtype, + shape: [1, 2], + data_offsets: [0, 1] as [number, number], + }; + + assert.ok(typeof tensorInfo.dtype === "string"); + assert.ok(["F4", "F6_E2M3", "F6_E3M2", "E8M0"].includes(tensorInfo.dtype)); + } + }); + + it("should handle parameter counting with sub-byte data types", () => { + const mockHeader = { + tensor_f4: { + dtype: "F4" as const, + shape: [10, 20], + data_offsets: [0, 100] as [number, number], + }, + tensor_f6_e2m3: { + dtype: "F6_E2M3" as const, + shape: [5, 10], + data_offsets: [100, 150] as [number, number], + }, + tensor_f6_e3m2: { + dtype: "F6_E3M2" as const, + shape: [8, 12], + data_offsets: [150, 246] as [number, number], + }, + tensor_e8m0: { + dtype: "E8M0" as const, + shape: [4, 6], + data_offsets: [246, 270] as [number, number], + }, + __metadata__: { format: "pt" }, + }; + + const computeNumOfParamsByDtypeSingleFile = (header: typeof mockHeader) => { + const counter: Partial> = {}; + const tensors = Object.fromEntries(Object.entries(header).filter(([key]) => key !== "__metadata__")); + + for (const [, v] of Object.entries(tensors) as [ + string, + { dtype: string; shape: number[]; data_offsets: [number, number] }, + ][]) { + if (v.shape.length === 0) { + continue; + } + counter[v.dtype] = (counter[v.dtype] ?? 0) + v.shape.reduce((a: number, b: number) => a * b); + } + return counter; + }; + + const parameterCount = computeNumOfParamsByDtypeSingleFile(mockHeader); + + assert.strictEqual(parameterCount.F4, 200); + assert.strictEqual(parameterCount.F6_E2M3, 50); + assert.strictEqual(parameterCount.F6_E3M2, 96); + assert.strictEqual(parameterCount.E8M0, 24); + }); + + it("fetch info for GPTQ quantized 8B model", async () => { + const parse = await parseSafetensorsMetadata({ + repo: "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w4a16", + revision: "3921b6aee65496a708b0af456c964ceca7423193", + computeParametersCount: true, + }); + + const parameterCount = parse.parameterCount; + assert.ok(parameterCount); + assert.ok(parameterCount.I32); + assert.ok(parameterCount.F16); + assert.strictEqual(parameterCount.I32, 6_979_321_856); + assert.strictEqual(parameterCount.F16, 1_052_315_648); + + const parameterCountTotal = + parse.parameterTotal ?? + sum( + Object.entries(parameterCount) + .filter(([, value]) => typeof value === "number") + .map(([, value]) => value as number), + ); + + assert.strictEqual(parameterCountTotal, 8_031_637_504); + }); + + it("fetch info for openai/gpt-oss-20b (large sharded model)", async () => { + const parse = await parseSafetensorsMetadata({ + repo: "openai/gpt-oss-20b", + computeParametersCount: true, + revision: "bbf09307421df45099c1e7dcbd64e3106ce5b403", + }); + + assert(parse.sharded); + + assert.ok(Object.keys(parse.headers).length > 1); + assert.ok(parse.parameterCount); + + const totalParams = parse.parameterTotal || sum(Object.values(parse.parameterCount)); + + assert.strictEqual(totalParams, 21_511_953_984); // 21.5B + + assert.ok(parse.parameterCount.BF16 && parse.parameterCount.U8); + + assert.strictEqual(Object.keys(parse.headers).length, 3); + }); + + it("should support FP4 and UE8 data types in type system", () => { + const newDataTypes: Array<"FP4" | "UE8"> = ["FP4", "UE8"]; + + for (const dtype of newDataTypes) { + const tensorInfo = { + dtype, + shape: [1, 2], + data_offsets: [0, 1] as [number, number], + }; + + assert.ok(typeof tensorInfo.dtype === "string"); + assert.ok(["FP4", "UE8"].includes(tensorInfo.dtype)); + } + + const mockHeader = { + tensor_fp4: { + dtype: "FP4" as const, + shape: [100, 200], + data_offsets: [0, 5000] as [number, number], + }, + tensor_ue8: { + dtype: "UE8" as const, + shape: [50, 100], + data_offsets: [5000, 10000] as [number, number], + }, + __metadata__: { format: "pt" }, + }; + + const computeNumOfParamsByDtypeSingleFile = (header: typeof mockHeader) => { + const counter: Partial> = {}; + const tensors = Object.fromEntries(Object.entries(header).filter(([key]) => key !== "__metadata__")); + + for (const [, v] of Object.entries(tensors) as [ + string, + { dtype: string; shape: number[]; data_offsets: [number, number] }, + ][]) { + if (v.shape.length === 0) { + continue; + } + counter[v.dtype] = (counter[v.dtype] ?? 0) + v.shape.reduce((a: number, b: number) => a * b); + } + return counter; + }; + + const parameterCount = computeNumOfParamsByDtypeSingleFile(mockHeader); + + assert.strictEqual(parameterCount.FP4, 20000); + assert.strictEqual(parameterCount.UE8, 5000); + }); + + describe("globMatch", () => { + it("exact match when no wildcard", () => { + assert.strictEqual(globMatch("foo", "foo"), true); + assert.strictEqual(globMatch("foo", "foobar"), false); + assert.strictEqual(globMatch("foo", "xfoo"), false); + assert.strictEqual(globMatch("foo", "xfoox"), false); + }); + + it("single leading wildcard (*.ext)", () => { + assert.strictEqual(globMatch("*.txt", "file.txt"), true); + assert.strictEqual(globMatch("*.txt", ".txt"), true); + assert.strictEqual(globMatch("*.txt", "file.txt.bak"), false); + assert.strictEqual(globMatch("*.txt", "txt"), false); + }); + + it("single trailing wildcard (prefix.*)", () => { + assert.strictEqual(globMatch("model.*", "model.bin"), true); + assert.strictEqual(globMatch("model.*", "model."), true); + assert.strictEqual(globMatch("model.*", "my_model.bin"), false); + }); + + it("wildcard on both sides (*mid*)", () => { + assert.strictEqual(globMatch("*layer*", "model.layer.weight"), true); + assert.strictEqual(globMatch("*layer*", "layer"), true); + assert.strictEqual(globMatch("*layer*", "no_match"), false); + }); + + it("multiple wildcards", () => { + assert.strictEqual(globMatch("a*b*c", "abc"), true); + assert.strictEqual(globMatch("a*b*c", "aXXbYYc"), true); + assert.strictEqual(globMatch("a*b*c", "aXXbYY"), false); + assert.strictEqual(globMatch("a*b*c", "XXbYYc"), false); + }); + + it("wildcard-only pattern matches anything", () => { + assert.strictEqual(globMatch("*", "anything"), true); + assert.strictEqual(globMatch("*", ""), true); + }); + + it("typical quantization config patterns", () => { + assert.strictEqual(globMatch("lm_head", "lm_head"), true); + assert.strictEqual(globMatch("lm_head", "model.lm_head"), false); + assert.strictEqual(globMatch("*lm_head*", "model.lm_head.weight"), true); + }); + + it("bare module names match via substring in isQuantizedTensor context", () => { + // globMatch itself is a strict glob matcher — no wildcard means exact match + assert.strictEqual(globMatch("lm_head", "model.lm_head.weight"), false); + // But isQuantizedTensor uses substring matching for bare names (no *) + // to match Python transformers behavior. See isQuantizedTensor tests below. + }); + }); + + describe("isQuantizedTensor", () => { + const makeConfig = (modules: string[]) => ({ + quant_method: "bitsandbytes" as const, + modules_to_not_convert: modules, + }); + + it("returns false when no quantization config", () => { + assert.strictEqual(isQuantizedTensor("model.layer.weight", undefined), false); + }); + + it("returns true when modules_to_not_convert is empty", () => { + assert.strictEqual(isQuantizedTensor("model.layer.weight", makeConfig([])), true); + }); + + it("bare module name excludes tensors containing that substring (Python compat)", () => { + const config = makeConfig(["lm_head"]); + assert.strictEqual(isQuantizedTensor("model.lm_head.weight", config), false); + assert.strictEqual(isQuantizedTensor("lm_head", config), false); + assert.strictEqual(isQuantizedTensor("lm_head.weight", config), false); + assert.strictEqual(isQuantizedTensor("model.embed_tokens.weight", config), true); + }); + + it("glob pattern with wildcards uses globMatch", () => { + const config = makeConfig(["*lm_head*"]); + assert.strictEqual(isQuantizedTensor("model.lm_head.weight", config), false); + assert.strictEqual(isQuantizedTensor("model.embed_tokens.weight", config), true); + }); + + it("multiple exclusion patterns", () => { + const config = makeConfig(["lm_head", "embed_tokens"]); + assert.strictEqual(isQuantizedTensor("model.lm_head.weight", config), false); + assert.strictEqual(isQuantizedTensor("model.embed_tokens.weight", config), false); + assert.strictEqual(isQuantizedTensor("model.layers.0.self_attn.q_proj.weight", config), true); + }); + }); + + describe("matchesCompressedTensorsTarget", () => { + it("exact module name match", () => { + assert.strictEqual( + matchesCompressedTensorsTarget("model.language_model.embed_tokens", "model.language_model.embed_tokens"), + true, + ); + assert.strictEqual( + matchesCompressedTensorsTarget( + "model.language_model.embed_tokens", + "model.language_model.embed_tokens_per_layer", + ), + false, + ); + }); + + it("class-name targets do not match module names", () => { + assert.strictEqual(matchesCompressedTensorsTarget("Linear", "model.layers.0.mlp.down_proj"), false); + }); + + it("re: targets with .* wildcard and $ anchor", () => { + assert.strictEqual(matchesCompressedTensorsTarget("re:.*lm_head$", "model.lm_head"), true); + assert.strictEqual(matchesCompressedTensorsTarget("re:.*lm_head$", "model.lm_head.weight"), false); + assert.strictEqual(matchesCompressedTensorsTarget("re:.*lm_head$", "lm_head"), true); + }); + + it("re: targets are anchored at the start, open-ended without $", () => { + assert.strictEqual(matchesCompressedTensorsTarget("re:model\\.layers.*", "model.layers.0.mlp.gate_proj"), true); + assert.strictEqual(matchesCompressedTensorsTarget("re:model\\.layers.*", "lm.model.layers.0"), false); + assert.strictEqual(matchesCompressedTensorsTarget("re:^model\\.layers.*", "model.layers.0"), true); + }); + + it("re: targets with unsupported regex syntax never match", () => { + assert.strictEqual( + matchesCompressedTensorsTarget("re:.*mlp\\.(gate|up)_proj.*", "model.layers.0.mlp.gate_proj"), + false, + ); + assert.strictEqual(matchesCompressedTensorsTarget("re:(a+)+$", "aaaaaaaaaaaaaaaaaaaaab"), false); + }); + }); + + it("fetch info for moonshotai/Kimi-K2.5 (large index file >20MB)", async () => { + // This model has a ~23.5MB index file due to having many experts + const parse = await parseSafetensorsMetadata({ + repo: "moonshotai/Kimi-K2.5", + revision: "2426b45b6af0da48d0dcce71bbce6225e5c73adc", + computeParametersCount: true, + }); + + assert(parse.sharded); + assert.strictEqual(Object.keys(parse.headers).length, 64); + assert.deepStrictEqual(parse.parameterCount, { F32: 23_040, I32: 1_014_687_129_600, BF16: 43_902_267_888 }); + assert.deepStrictEqual(sum(Object.values(parse.parameterCount)), 1_058_589_420_528); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/parse-safetensors-metadata.ts b/node_modules/@huggingface/hub/src/lib/parse-safetensors-metadata.ts new file mode 100644 index 0000000000000000000000000000000000000000..14243ab1850d2853387f7f13683f13dd09c84ad5 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/parse-safetensors-metadata.ts @@ -0,0 +1,607 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import { omit } from "../utils/omit"; +import { toRepoId } from "../utils/toRepoId"; +import { typedEntries } from "../utils/typedEntries"; +import { downloadFile } from "./download-file"; +import { fileExists } from "./file-exists"; +import { promisesQueue } from "../utils/promisesQueue"; +import type { SetRequired } from "../vendor/type-fest/set-required"; + +export const SAFETENSORS_FILE = "model.safetensors"; +export const SAFETENSORS_INDEX_FILE = "model.safetensors.index.json"; +/// We advise model/library authors to use the filenames above for convention inside model repos, +/// but in some situations safetensors weights have different filenames. +export const RE_SAFETENSORS_FILE = /\.safetensors$/; +export const RE_SAFETENSORS_INDEX_FILE = /\.safetensors\.index\.json$/; +export const RE_SAFETENSORS_SHARD_FILE = + /^(?(?.*?)[_-])(?\d{5,6})-of-(?\d{5,6})\.safetensors$/; +export interface SafetensorsShardFileInfo { + prefix: string; + basePrefix: string; + shard: string; + total: string; +} +export function parseSafetensorsShardFilename(filename: string): SafetensorsShardFileInfo | null { + const match = RE_SAFETENSORS_SHARD_FILE.exec(filename); + if (match && match.groups) { + return { + prefix: match.groups["prefix"], + basePrefix: match.groups["basePrefix"], + shard: match.groups["shard"], + total: match.groups["total"], + }; + } + return null; +} + +const PARALLEL_DOWNLOADS = 20; +const MAX_HEADER_LENGTH = 25_000_000; // 25MB +const MAX_CONFIG_LENGTH = 10_000_000; // 10MB — config.json is typically small; cap to avoid large memory use +const MAX_SHARD_COUNT = 10_000; // well above any real sharded model; blocks crafted index with millions of entries +const GPTQ_QWEIGHT_SUFFIX = "qweight"; +const GPTQ_AWQ_AUXILIARY_SUFFIXES = ["qzeros", "g_idx", "scales"]; + +class SafetensorParseError extends Error {} + +type FileName = string; + +export type TensorName = string; +export type Dtype = + | "F64" + | "F32" + | "C64" + | "F16" + | "F8_E4M3" + | "F8_E4M3FNUZ" + | "F8_E5M2" + | "F8_E5M2FNUZ" + | "F8_E8M0" + | "E8M0" + | "F6_E3M2" + | "F6_E2M3" + | "F4" + | "FP4" + | "BF16" + | "I64" + | "U64" + | "I32" + | "U32" + | "I16" + | "I8" + | "U16" + | "U8" + | "UE8" + | "BOOL"; + +export interface TensorInfo { + dtype: Dtype; + shape: number[]; + data_offsets: [number, number]; +} + +export type SafetensorsFileHeader = Record & { + __metadata__?: { total_parameters?: string | number } & Record; +}; + +export interface SafetensorsIndexJson { + dtype?: string; + /// ^there's sometimes a dtype but it looks inconsistent. + metadata?: { total_parameters?: string | number } & Record; + /// ^ why the naming inconsistency? + weight_map: Record; +} + +export type SafetensorsShardedHeaders = Record; + +export type SafetensorsParseFromRepo = + | { + sharded: false; + header: SafetensorsFileHeader; + parameterCount?: Partial>; + parameterTotal?: number; + filepaths: string[]; + } + | { + sharded: true; + index: SafetensorsIndexJson; + headers: SafetensorsShardedHeaders; + parameterCount?: Partial>; + parameterTotal?: number; + filepaths: string[]; + }; + +/** + * Fetches and parses model config.json + */ +async function fetchModelConfig( + params: { + repo: RepoDesignation; + revision?: string; + hubUrl?: string; + fetch?: typeof fetch; + } & Partial, +): Promise { + try { + const configBlob = await downloadFile({ + ...params, + path: "config.json", + }); + + if (!configBlob) { + return null; + } + + const config = JSON.parse(await configBlob.slice(0, MAX_CONFIG_LENGTH).text()); + return config as ModelConfig; + } catch (error) { + // Config file might not exist or be inaccessible + return null; + } +} + +async function parseSingleFile( + path: string, + params: { + repo: RepoDesignation; + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise { + const blob = await downloadFile({ ...params, path }); + + if (!blob) { + throw new SafetensorParseError(`Failed to parse file ${path}: failed to fetch safetensors header length.`); + } + + const bufLengthOfHeaderLE = await blob.slice(0, 8).arrayBuffer(); + const lengthOfHeader = new DataView(bufLengthOfHeaderLE).getBigUint64(0, true); + // ^little-endian + if (lengthOfHeader <= 0) { + throw new SafetensorParseError(`Failed to parse file ${path}: safetensors header is malformed.`); + } + if (lengthOfHeader > MAX_HEADER_LENGTH) { + throw new SafetensorParseError( + `Failed to parse file ${path}: safetensor header is too big. Maximum supported size is ${MAX_HEADER_LENGTH} bytes.`, + ); + } + + try { + // no validation for now, we assume it's a valid FileHeader. + const header: SafetensorsFileHeader = JSON.parse(await blob.slice(8, 8 + Number(lengthOfHeader)).text()); + return header; + } catch (err) { + throw new SafetensorParseError(`Failed to parse file ${path}: safetensors header is not valid JSON.`); + } +} + +async function parseShardedIndex( + path: string, + params: { + repo: RepoDesignation; + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise { + const indexBlob = await downloadFile({ + ...params, + path, + }); + + if (!indexBlob) { + throw new SafetensorParseError(`Failed to parse file ${path}: failed to fetch safetensors index.`); + } + + try { + // no validation for now, we assume it's a valid IndexJson. + const index = JSON.parse(await indexBlob.slice(0, MAX_HEADER_LENGTH).text()); + return index; + } catch (error) { + throw new SafetensorParseError(`Failed to parse file ${path}: not a valid JSON.`); + } +} + +async function fetchAllHeaders( + path: string, + index: SafetensorsIndexJson, + params: { + repo: RepoDesignation; + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise { + const pathPrefix = path.slice(0, path.lastIndexOf("/") + 1); + const filenames = [...new Set(Object.values(index.weight_map))]; + if (filenames.length > MAX_SHARD_COUNT) { + throw new SafetensorParseError( + `Too many shard files (${filenames.length}). Maximum supported is ${MAX_SHARD_COUNT}.`, + ); + } + for (const filename of filenames) { + if (filename.includes("..") || filename.startsWith("/") || filename.includes("://")) { + throw new SafetensorParseError(`Unsafe shard filename in weight_map: "${filename}"`); + } + } + const shardedMap: SafetensorsShardedHeaders = Object.fromEntries( + await promisesQueue( + filenames.map( + (filename) => async () => + [filename, await parseSingleFile(pathPrefix + filename, params)] satisfies [string, SafetensorsFileHeader], + ), + PARALLEL_DOWNLOADS, + ), + ); + return shardedMap; +} + +function parseTotalParameters(value: string | number | undefined): number | undefined { + if (!value) { + return undefined; + } + if (typeof value === "number") { + return value; + } + return parseInt(value); +} + +/** + * Analyze model.safetensors.index.json or model.safetensors from a model hosted + * on Hugging Face using smart range requests to extract its metadata. + */ +export async function parseSafetensorsMetadata( + params: { + /** Only models are supported */ + repo: RepoDesignation; + /** + * Relative file path to safetensors file inside `repo`. Defaults to `SAFETENSORS_FILE` or `SAFETENSORS_INDEX_FILE` (whichever one exists). + */ + path?: string; + /** + * Will include SafetensorsParseFromRepo["parameterCount"], an object containing the number of parameters for each DType + * + * @default false + */ + computeParametersCount: true; + hubUrl?: string; + revision?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise>; +export async function parseSafetensorsMetadata( + params: { + /** Only models are supported */ + repo: RepoDesignation; + path?: string; + /** + * Will include SafetensorsParseFromRepo["parameterCount"], an object containing the number of parameters for each DType + * + * @default false + */ + computeParametersCount?: boolean; + hubUrl?: string; + revision?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise; +export async function parseSafetensorsMetadata( + params: { + repo: RepoDesignation; + path?: string; + computeParametersCount?: boolean; + hubUrl?: string; + revision?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise { + const repoId = toRepoId(params.repo); + + if (repoId.type !== "model") { + throw new TypeError("Only model repos should contain safetensors files."); + } + + // Fetch model config for quantization information + const modelConfig = params.computeParametersCount ? await fetchModelConfig(params) : null; + const quantConfig = modelConfig?.quantization_config ?? modelConfig?.text_config?.quantization_config; + + if ( + (params.path && RE_SAFETENSORS_FILE.test(params.path)) || + (await fileExists({ ...params, path: SAFETENSORS_FILE })) + ) { + const header = await parseSingleFile(params.path ?? SAFETENSORS_FILE, params); + const paramStats = params.computeParametersCount + ? { + parameterCount: computeNumOfParamsByDtypeSingleFile(header, quantConfig), + /// shortcut: get param count directly from metadata + parameterTotal: parseTotalParameters(header.__metadata__?.total_parameters), + } + : undefined; + return { + sharded: false, + header, + ...paramStats, + filepaths: [params.path ?? SAFETENSORS_FILE], + }; + } else if ( + (params.path && RE_SAFETENSORS_INDEX_FILE.test(params.path)) || + (await fileExists({ ...params, path: SAFETENSORS_INDEX_FILE })) + ) { + const path = params.path ?? SAFETENSORS_INDEX_FILE; + const index = await parseShardedIndex(path, params); + const shardedMap = await fetchAllHeaders(path, index, params); + const pathPrefix = path.slice(0, path.lastIndexOf("/") + 1); + + const paramStats = params.computeParametersCount + ? { + parameterCount: computeNumOfParamsByDtypeSharded(shardedMap, quantConfig), + /// shortcut: get param count directly from metadata + parameterTotal: parseTotalParameters(index.metadata?.total_parameters), + } + : undefined; + return { + sharded: true, + index, + headers: shardedMap, + ...paramStats, + filepaths: [path, ...Object.keys(shardedMap).map((filename) => pathPrefix + filename)], + }; + } else { + throw new Error("model id does not seem to contain safetensors weights"); + } +} + +export interface QuantizationConfig { + quant_method?: string; + modules_to_not_convert?: string[]; + bits?: number; + load_in_4bit?: boolean; + load_in_8bit?: boolean; + // compressed-tensors specific + format?: string; + config_groups?: Record; +} + +export interface ModelConfig { + quantization_config?: QuantizationConfig; + text_config?: { quantization_config?: QuantizationConfig }; +} + +/** + * @internal + * Glob match without RegExp: splits pattern on `*` and checks that each literal + * segment appears in order within `str`. Avoids RegExp entirely (no ReDoS risk, + * no SyntaxError from attacker-controlled patterns in config.json). + */ +export function globMatch(pattern: string, str: string): boolean { + const parts = pattern.split("*"); + + if (parts.length === 1) { + return pattern === str; + } + + if (!str.startsWith(parts[0])) { + return false; + } + let pos = parts[0].length; + + const lastPart = parts[parts.length - 1]; + if (!str.endsWith(lastPart)) { + return false; + } + const end = str.length - lastPart.length; + + for (let i = 1; i < parts.length - 1; i++) { + const idx = str.indexOf(parts[i], pos); + if (idx === -1 || idx + parts[i].length > end) { + return false; + } + pos = idx + parts[i].length; + } + + return pos <= end; +} + +/** + * Determines if a tensor is quantized based on quantization config and tensor name. + * + * Python's transformers uses plain substring matching for `modules_to_not_convert`, + * so bare names like `"lm_head"` must match `"model.lm_head.weight"`. When the + * pattern contains a `*` we fall back to proper glob matching for flexibility. + */ +export function isQuantizedTensor(tensorName: string, quantConfig?: QuantizationConfig): boolean { + if (!quantConfig) { + return false; + } + const patterns = quantConfig.modules_to_not_convert; + if (!patterns?.length) { + return true; + } + return !patterns.some((pattern) => + pattern.includes("*") ? globMatch(pattern, tensorName) : tensorName.includes(pattern), + ); +} + +/** + * @internal + * Matches a module name against a compressed-tensors target. + * + * Targets are either exact module names, class names (e.g. `"Linear"`, which we + * cannot resolve from tensor names and therefore ignore), or Python regexes + * prefixed with `re:`. To avoid evaluating attacker-controlled RegExp from + * config.json (ReDoS, SyntaxError — see globMatch), we translate the common + * regex subset (`.*` wildcard, `^`/`$` anchors, `\.` escapes) to globMatch and + * treat targets using any other regex syntax as non-matching. + */ +export function matchesCompressedTensorsTarget(target: string, moduleName: string): boolean { + if (!target.startsWith("re:")) { + return target === moduleName; + } + let pattern = target.slice(3); + // Python's re.match anchors at the start; only `$` anchors the end. + if (pattern.startsWith("^")) { + pattern = pattern.slice(1); + } + if (pattern.endsWith("$")) { + pattern = pattern.slice(0, -1); + } else { + pattern += ".*"; + } + const glob = pattern.replaceAll(".*", "*").replaceAll("\\.", "."); + if (/[\\+?()[\]{}|^$]/.test(glob)) { + // unsupported regex syntax — skip this target rather than risk a wrong match + return false; + } + return globMatch(glob, moduleName); +} + +/** + * Gets the parameter multiplier for a quantized tensor based on quantization method + */ +function getQuantizationMultiplier(tensorName: string, dtype: Dtype, quantConfig?: QuantizationConfig): number { + if (!quantConfig || !isQuantizedTensor(tensorName, quantConfig)) { + return 1; + } + + const quantMethod = quantConfig.quant_method?.toLowerCase(); + + switch (quantMethod) { + case "mxfp4": + if (dtype === "U8" && tensorName.includes("_blocks")) { + return 2; + } + return 1; + + case "gptq": + case "awq": + if (getTensorSuffix(tensorName) === GPTQ_QWEIGHT_SUFFIX) { + const bits = quantConfig.bits && quantConfig.bits > 0 ? quantConfig.bits : 4; + return Math.max(1, Math.floor(32 / bits)); + } + if (quantConfig.bits === 4 && dtype === "U8") { + return 2; + } + if (quantConfig.bits === 2 && dtype === "U8") { + return 4; + } + return 1; + + case "compressed-tensors": + if (dtype === "I32") { + const groups = Object.values(quantConfig.config_groups ?? {}); + // Mixed-precision models pack different modules at different bit widths + // (one config group per width), so resolve the group whose targets + // match this tensor's module name (e.g. "model.lm_head.weight_packed" + // -> "model.lm_head") instead of assuming a single global num_bits. + const suffixIndex = tensorName.lastIndexOf(".weight"); + const moduleName = suffixIndex === -1 ? tensorName : tensorName.slice(0, suffixIndex); + const group = groups.find((g) => + g.targets?.some((target) => matchesCompressedTensorsTarget(target, moduleName)), + ); + if (group) { + if ((group.format ?? quantConfig.format) !== "pack-quantized") { + return 1; + } + const numBits = group.weights?.num_bits ?? 4; + return Math.max(1, Math.floor(32 / numBits)); + } + // fallback when no group targets match: first group's num_bits + if (quantConfig.format === "pack-quantized") { + const numBits = groups.find((g) => g.weights?.num_bits)?.weights?.num_bits ?? 4; + return Math.max(1, Math.floor(32 / numBits)); + } + } + return 1; + + case "bitsandbytes": + if (quantConfig.load_in_4bit && dtype === "U8") { + return 2; + } + return 1; + + default: + if (dtype === "U8" && (quantConfig.load_in_4bit || quantConfig.bits === 4)) { + return 2; + } + return 1; + } +} + +function computeNumOfParamsByDtypeSingleFile( + header: SafetensorsFileHeader, + quantConfig?: QuantizationConfig, +): Partial> { + const counter: Partial> = {}; + const tensors = omit(header, "__metadata__"); + + for (const [tensorName, v] of typedEntries(tensors)) { + if (shouldSkipTensor(tensorName, quantConfig)) { + continue; + } + if (v.shape.length === 0) { + continue; + } + + const elements = v.shape.reduce((a, b) => a * b); + if (!Number.isFinite(elements)) { + continue; + } + const multiplier = quantConfig ? getQuantizationMultiplier(tensorName, v.dtype, quantConfig) : 1; + if (multiplier === 0) { + continue; + } + counter[v.dtype] = (counter[v.dtype] ?? 0) + elements * multiplier; + } + return counter; +} + +function computeNumOfParamsByDtypeSharded( + shardedMap: SafetensorsShardedHeaders, + quantConfig?: QuantizationConfig, +): Partial> { + const counter: Partial> = {}; + for (const header of Object.values(shardedMap)) { + for (const [k, v] of typedEntries(computeNumOfParamsByDtypeSingleFile(header, quantConfig))) { + counter[k] = (counter[k] ?? 0) + (v ?? 0); + } + } + return counter; +} + +function getTensorSuffix(tensorName: string): string { + const lastDotIndex = tensorName.lastIndexOf("."); + return lastDotIndex === -1 ? tensorName : tensorName.slice(lastDotIndex + 1); +} + +function shouldSkipTensor(tensorName: string, quantConfig?: QuantizationConfig): boolean { + if (!quantConfig) { + return false; + } + const quantMethod = quantConfig.quant_method?.toLowerCase(); + if (quantMethod !== "gptq" && quantMethod !== "awq") { + return false; + } + if (!isQuantizedTensor(tensorName, quantConfig)) { + return false; + } + const suffix = getTensorSuffix(tensorName); + return suffix !== GPTQ_QWEIGHT_SUFFIX && GPTQ_AWQ_AUXILIARY_SUFFIXES.includes(suffix); +} diff --git a/node_modules/@huggingface/hub/src/lib/paths-info.spec.ts b/node_modules/@huggingface/hub/src/lib/paths-info.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..7a888251b387f1244bcf4238abcb35a86c0db0e4 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/paths-info.spec.ts @@ -0,0 +1,73 @@ +import { expect, it, describe } from "vitest"; +import type { CommitInfo, PathInfo } from "./paths-info"; +import { pathsInfo } from "./paths-info"; + +describe("pathsInfo", () => { + it("should fetch LFS path info", async () => { + const result: PathInfo[] = await pathsInfo({ + repo: { + name: "google-bert/bert-base-uncased", + type: "model", + }, + paths: ["tf_model.h5"], + revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", + }); + + expect(result).toHaveLength(1); + + const modelPathInfo = result[0]; + expect(modelPathInfo.path).toBe("tf_model.h5"); + expect(modelPathInfo.type).toBe("file"); + // lfs pointer, therefore lfs should be defined + expect(modelPathInfo?.lfs).toBeDefined(); + expect(modelPathInfo?.lfs?.oid).toBe("a7a17d6d844b5de815ccab5f42cad6d24496db3850a2a43d8258221018ce87d2"); + expect(modelPathInfo?.lfs?.size).toBe(536063208); + expect(modelPathInfo?.lfs?.pointerSize).toBe(134); + + // should not include expand info + expect(modelPathInfo.lastCommit).toBeUndefined(); + expect(modelPathInfo.securityFileStatus).toBeUndefined(); + }); + + it("expand params should fetch lastCommit", async () => { + const result: (PathInfo & { + lastCommit: CommitInfo; + })[] = await pathsInfo({ + repo: { + name: "google-bert/bert-base-uncased", + type: "model", + }, + paths: ["tf_model.h5"], + revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", + expand: true, // include + }); + + expect(result).toHaveLength(1); + + const modelPathInfo = result[0]; + + // should include expand info + expect(modelPathInfo.lastCommit).toBeDefined(); + + expect(modelPathInfo.lastCommit.id).toBe("dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7"); + expect(modelPathInfo.lastCommit.title).toBe("Update tf_model.h5"); + expect(modelPathInfo.lastCommit.date.getTime()).toBe(1569268124000); // 2019-09-23T19:48:44.000Z + }); + + it("non-LFS pointer should have lfs undefined", async () => { + const result: PathInfo[] = await pathsInfo({ + repo: { + name: "google-bert/bert-base-uncased", + type: "model", + }, + paths: ["config.json"], + revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", + }); + + expect(result).toHaveLength(1); + + const modelPathInfo = result[0]; + expect(modelPathInfo.path).toBe("config.json"); + expect(modelPathInfo.lfs).toBeUndefined(); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/paths-info.ts b/node_modules/@huggingface/hub/src/lib/paths-info.ts new file mode 100644 index 0000000000000000000000000000000000000000..f665317addf14c7f8e5eb76b02bc0ca8cf8fe186 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/paths-info.ts @@ -0,0 +1,142 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { toRepoId } from "../utils/toRepoId"; +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; + +export interface LfsPathInfo { + oid: string; + size: number; + pointerSize: number; +} + +export interface CommitInfo { + id: string; + title: string; + date: Date; +} + +export interface SecurityFileStatus { + status: string; +} + +export interface PathInfo { + path: string; + type: string; + /** + * Not available for bucket repos. + */ + oid?: string; + size: number; + /** + * Only defined when path is LFS pointer. Not available for bucket repos. + */ + lfs?: LfsPathInfo; + /** + * Xet-backed hash. Always present for bucket file entries. + */ + xetHash?: string; + /** + * Not available for bucket repos, use {@link uploadedAt} instead. + */ + lastCommit?: CommitInfo; + /** + * Only available for bucket repos. + */ + uploadedAt?: string; + securityFileStatus?: SecurityFileStatus; +} + +// Define the overloaded signatures +export function pathsInfo( + params: { + repo: RepoDesignation; + paths: string[]; + expand: true; // if expand true + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise<(PathInfo & { lastCommit: CommitInfo; securityFileStatus: SecurityFileStatus })[]>; +export function pathsInfo( + params: { + repo: RepoDesignation; + paths: string[]; + expand?: boolean; + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise; + +export async function pathsInfo( + params: { + repo: RepoDesignation; + paths: string[]; + expand?: boolean; + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise { + const accessToken = checkCredentials(params); + const repoId = toRepoId(params.repo); + + const hubUrl = params.hubUrl ?? HUB_URL; + + const revision = repoId.type === "bucket" ? undefined : (params.revision ?? "main"); + const url = `${hubUrl}/api/${repoId.type}s/${repoId.name}/paths-info${ + revision ? `/${encodeURIComponent(revision)}` : "" + }`; + + const resp = await (params.fetch ?? fetch)(url, { + method: "POST", + headers: { + ...(accessToken && { + Authorization: `Bearer ${accessToken}`, + }), + Accept: "application/json", + "Content-Type": "application/json", + }, + body: JSON.stringify({ + paths: params.paths, + expand: params.expand, + }), + }); + + if (!resp.ok) { + throw await createApiError(resp); + } + + const json: unknown = await resp.json(); + if (!Array.isArray(json)) { + throw new Error("malformed response: expected array"); + } + + return json.map((item: PathInfo) => ({ + path: item.path, + lfs: item.lfs, + type: item.type, + oid: item.oid, + size: item.size, + xetHash: item.xetHash, + uploadedAt: item.uploadedAt, + securityFileStatus: item.securityFileStatus, + lastCommit: item.lastCommit + ? { + date: new Date(item.lastCommit.date), + title: item.lastCommit.title, + id: item.lastCommit.id, + } + : undefined, + })); +} diff --git a/node_modules/@huggingface/hub/src/lib/repo-exists.spec.ts b/node_modules/@huggingface/hub/src/lib/repo-exists.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..c4bbb192f236db689fbed854df5609d5d6c091e9 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/repo-exists.spec.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { repoExists } from "./repo-exists"; + +describe("repoExists", () => { + it("should check if a repo exists", async () => { + const exists1 = await repoExists({ repo: { type: "model", name: "openai-community/gpt2" } }); + + expect(exists1).toBe(true); + + const exists2 = await repoExists({ repo: { type: "model", name: "openai-community/gpt9000" } }); + expect(exists2).toBe(false); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/repo-exists.ts b/node_modules/@huggingface/hub/src/lib/repo-exists.ts new file mode 100644 index 0000000000000000000000000000000000000000..8be149df4337adfd02675e2b3f2682c1552f9de8 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/repo-exists.ts @@ -0,0 +1,43 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { RepoDesignation } from "../types/public"; +import { toRepoId } from "../utils/toRepoId"; + +export async function repoExists(params: { + repo: RepoDesignation; + + hubUrl?: string; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + */ + revision?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + accessToken?: string; +}): Promise { + const repoId = toRepoId(params.repo); + + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}?expand[]=likes`, + { + method: "GET", + headers: { + ...(params.accessToken && { + Authorization: `Bearer ${params.accessToken}`, + }), + }, + }, + ); + + if (res.status === 404 || res.status === 401) { + return false; + } + + if (!res.ok) { + throw await createApiError(res); + } + + return true; +} diff --git a/node_modules/@huggingface/hub/src/lib/snapshot-download.spec.ts b/node_modules/@huggingface/hub/src/lib/snapshot-download.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..69988d7ea4006255c7676d70dac67a983b972f23 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/snapshot-download.spec.ts @@ -0,0 +1,273 @@ +import { expect, test, describe, vi, beforeEach } from "vitest"; +import { dirname, join } from "node:path"; +import { mkdir, writeFile } from "node:fs/promises"; +import { getHFHubCachePath } from "./cache-management"; +import { downloadFileToCacheDir } from "./download-file-to-cache-dir"; +import { snapshotDownload } from "./snapshot-download"; +import type { ListFileEntry } from "./list-files"; +import { listFiles } from "./list-files"; +import { modelInfo } from "./model-info"; +import { datasetInfo } from "./dataset-info"; +import type { DatasetEntry } from "./list-datasets"; +import type { ApiDatasetInfo } from "../types/api/api-dataset"; +import { spaceInfo } from "./space-info"; +import type { SpaceEntry } from "./list-spaces"; +import type { ApiSpaceInfo } from "../types/api/api-space"; + +vi.mock("node:fs/promises", () => ({ + writeFile: vi.fn(), + mkdir: vi.fn(), +})); + +vi.mock("./space-info", () => ({ + spaceInfo: vi.fn(), +})); + +vi.mock("./dataset-info", () => ({ + datasetInfo: vi.fn(), +})); + +vi.mock("./model-info", () => ({ + modelInfo: vi.fn(), +})); + +vi.mock("./list-files", () => ({ + listFiles: vi.fn(), +})); + +vi.mock("./download-file-to-cache-dir", () => ({ + downloadFileToCacheDir: vi.fn(), +})); + +const DUMMY_SHA = "dummy-sha"; + +// utility method to transform an array of ListFileEntry to an AsyncGenerator +async function* toAsyncGenerator(content: ListFileEntry[]): AsyncGenerator { + for (const entry of content) { + yield Promise.resolve(entry); + } +} + +beforeEach(() => { + vi.resetAllMocks(); + vi.mocked(listFiles).mockReturnValue(toAsyncGenerator([])); + + // mock repo info + vi.mocked(modelInfo).mockResolvedValue({ + sha: DUMMY_SHA, + } as unknown as Awaited>); + vi.mocked(datasetInfo).mockResolvedValue({ + sha: DUMMY_SHA, + } as DatasetEntry & ApiDatasetInfo); + vi.mocked(spaceInfo).mockResolvedValue({ + sha: DUMMY_SHA, + } as SpaceEntry & ApiSpaceInfo); +}); + +describe("snapshotDownload", () => { + test("empty AsyncGenerator should not call downloadFileToCacheDir", async () => { + await snapshotDownload({ + repo: { + name: "foo/bar", + type: "space", + }, + }); + + expect(downloadFileToCacheDir).not.toHaveBeenCalled(); + }); + + test("repo type model should use modelInfo", async () => { + await snapshotDownload({ + repo: { + name: "foo/bar", + type: "model", + }, + }); + expect(modelInfo).toHaveBeenCalledOnce(); + expect(modelInfo).toHaveBeenCalledWith({ + name: "foo/bar", + additionalFields: ["sha"], + revision: "main", + repo: { + name: "foo/bar", + type: "model", + }, + }); + }); + + test("repo type dataset should use datasetInfo", async () => { + await snapshotDownload({ + repo: { + name: "foo/bar", + type: "dataset", + }, + }); + expect(datasetInfo).toHaveBeenCalledOnce(); + expect(datasetInfo).toHaveBeenCalledWith({ + name: "foo/bar", + additionalFields: ["sha"], + revision: "main", + repo: { + name: "foo/bar", + type: "dataset", + }, + }); + }); + + test("repo type space should use spaceInfo", async () => { + await snapshotDownload({ + repo: { + name: "foo/bar", + type: "space", + }, + }); + expect(spaceInfo).toHaveBeenCalledOnce(); + expect(spaceInfo).toHaveBeenCalledWith({ + name: "foo/bar", + additionalFields: ["sha"], + revision: "main", + repo: { + name: "foo/bar", + type: "space", + }, + }); + }); + + test("commitHash should be saved to ref folder", async () => { + await snapshotDownload({ + repo: { + name: "foo/bar", + type: "space", + }, + revision: "dummy-revision", + }); + + // cross-platform testing + const expectedPath = join(getHFHubCachePath(), "spaces--foo--bar", "refs", "dummy-revision"); + expect(mkdir).toHaveBeenCalledWith(dirname(expectedPath), { recursive: true }); + expect(writeFile).toHaveBeenCalledWith(expectedPath, DUMMY_SHA); + }); + + test("directory ListFileEntry should mkdir it", async () => { + vi.mocked(listFiles).mockReturnValue( + toAsyncGenerator([ + { + oid: "dummy-etag", + type: "directory", + path: "potatoes", + size: 0, + lastCommit: { + date: new Date().toISOString(), + id: DUMMY_SHA, + title: "feat: best commit", + }, + }, + ]), + ); + + await snapshotDownload({ + repo: { + name: "foo/bar", + type: "space", + }, + }); + + // cross-platform testing + const expectedPath = join(getHFHubCachePath(), "spaces--foo--bar", "snapshots", DUMMY_SHA, "potatoes"); + expect(mkdir).toHaveBeenCalledWith(expectedPath, { recursive: true }); + }); + + test("files in ListFileEntry should download them", async () => { + const entries: ListFileEntry[] = Array.from({ length: 10 }, (_, i) => ({ + oid: `dummy-etag-${i}`, + type: "file", + path: `file-${i}.txt`, + size: i, + lastCommit: { + date: new Date().toISOString(), + id: DUMMY_SHA, + title: "feat: best commit", + }, + })); + vi.mocked(listFiles).mockReturnValue(toAsyncGenerator(entries)); + + await snapshotDownload({ + repo: { + name: "foo/bar", + type: "space", + }, + }); + + for (const entry of entries) { + expect(downloadFileToCacheDir).toHaveBeenCalledWith( + expect.objectContaining({ + repo: { + name: "foo/bar", + type: "space", + }, + path: entry.path, + revision: DUMMY_SHA, + }), + ); + } + }); + + test("custom params should be propagated", async () => { + // fetch mock + const fetchMock: typeof fetch = vi.fn(); + const hubMock = "https://foor.bar"; + const accessTokenMock = "dummy-access-token"; + + vi.mocked(listFiles).mockReturnValue( + toAsyncGenerator([ + { + oid: `dummy-etag`, + type: "file", + path: `file.txt`, + size: 10, + lastCommit: { + date: new Date().toISOString(), + id: DUMMY_SHA, + title: "feat: best commit", + }, + }, + ]), + ); + + await snapshotDownload({ + repo: { + name: "foo/bar", + type: "space", + }, + hubUrl: hubMock, + fetch: fetchMock, + accessToken: accessTokenMock, + }); + + expect(spaceInfo).toHaveBeenCalledWith( + expect.objectContaining({ + fetch: fetchMock, + hubUrl: hubMock, + accessToken: accessTokenMock, + }), + ); + + // list files should receive custom fetch + expect(listFiles).toHaveBeenCalledWith( + expect.objectContaining({ + fetch: fetchMock, + hubUrl: hubMock, + accessToken: accessTokenMock, + }), + ); + + // download file to cache should receive custom fetch + expect(downloadFileToCacheDir).toHaveBeenCalledWith( + expect.objectContaining({ + fetch: fetchMock, + hubUrl: hubMock, + accessToken: accessTokenMock, + }), + ); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/snapshot-download.ts b/node_modules/@huggingface/hub/src/lib/snapshot-download.ts new file mode 100644 index 0000000000000000000000000000000000000000..57a7c929f085257114b02aa3e822336edc3c99f6 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/snapshot-download.ts @@ -0,0 +1,121 @@ +import type { CredentialsParams, RepoDesignation } from "../types/public"; +import { listFiles } from "./list-files"; +import { getHFHubCachePath, getRepoFolderName } from "./cache-management"; +import { spaceInfo } from "./space-info"; +import { datasetInfo } from "./dataset-info"; +import { modelInfo } from "./model-info"; +import { toRepoId } from "../utils/toRepoId"; +import { join, dirname } from "node:path"; +import { mkdir, writeFile } from "node:fs/promises"; +import { downloadFileToCacheDir } from "./download-file-to-cache-dir"; + +export const DEFAULT_REVISION = "main"; + +/** + * Downloads an entire repository at a given revision in the cache directory {@link getHFHubCachePath}. + * You can list all cached repositories using {@link scanCachedRepo} + * @remarks It uses internally {@link downloadFileToCacheDir}. + */ +export async function snapshotDownload( + params: { + repo: RepoDesignation; + cacheDir?: string; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + * + * @default "main" + */ + revision?: string; + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise { + let cacheDir: string; + if (params.cacheDir) { + cacheDir = params.cacheDir; + } else { + cacheDir = getHFHubCachePath(); + } + + let revision: string; + if (params.revision) { + revision = params.revision; + } else { + revision = DEFAULT_REVISION; + } + + const repoId = toRepoId(params.repo); + const storageFolder = join(cacheDir, getRepoFolderName(repoId)); + + let repoInfo: { sha: string }; + switch (repoId.type) { + case "space": + repoInfo = await spaceInfo({ + ...params, + name: repoId.name, + additionalFields: ["sha"], + revision: revision, + }); + break; + case "dataset": + repoInfo = await datasetInfo({ + ...params, + name: repoId.name, + additionalFields: ["sha"], + revision: revision, + }); + break; + case "model": + repoInfo = await modelInfo({ + ...params, + name: repoId.name, + additionalFields: ["sha"], + revision: revision, + }); + break; + default: + throw new Error( + `Unsupported repository type: ${repoId.type}. snapshotDownload is not supported for bucket repos.`, + ); + } + + const commitHash = repoInfo.sha; + + if (revision !== commitHash) { + const refPath = join(storageFolder, "refs", revision); + await mkdir(dirname(refPath), { recursive: true }); + await writeFile(refPath, commitHash); + } + + const snapshotFolder = join(storageFolder, "snapshots", commitHash); + + const cursor = listFiles({ + ...params, + repo: params.repo, + recursive: true, + revision: commitHash, + }); + + for await (const entry of cursor) { + switch (entry.type) { + case "file": + await downloadFileToCacheDir({ + ...params, + path: entry.path, + revision: commitHash, + cacheDir: cacheDir, + }); + break; + case "directory": + await mkdir(join(snapshotFolder, entry.path), { recursive: true }); + break; + default: + throw new Error(`unknown entry type: ${entry.type}`); + } + } + + return snapshotFolder; +} diff --git a/node_modules/@huggingface/hub/src/lib/space-info.spec.ts b/node_modules/@huggingface/hub/src/lib/space-info.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..ea966f98bceea2577e268164f8512bc822a8f82e --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/space-info.spec.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import { spaceInfo } from "./space-info"; +import type { SpaceEntry } from "./list-spaces"; +import type { ApiSpaceInfo } from "../types/api/api-space"; + +describe("spaceInfo", () => { + it("should return the space info", async () => { + const info = await spaceInfo({ + name: "huggingfacejs/client-side-oauth", + }); + expect(info).toEqual({ + id: "659835e689010f9c7aed608d", + name: "huggingfacejs/client-side-oauth", + updatedAt: expect.any(Date), + likes: expect.any(Number), + private: false, + sdk: "static", + }); + }); + + it("should return the space info with author", async () => { + const info: SpaceEntry & Pick = await spaceInfo({ + name: "huggingfacejs/client-side-oauth", + additionalFields: ["author"], + }); + expect(info).toEqual({ + id: "659835e689010f9c7aed608d", + name: "huggingfacejs/client-side-oauth", + updatedAt: expect.any(Date), + likes: expect.any(Number), + private: false, + sdk: "static", + author: "huggingfacejs", + }); + }); + + it("should return the space info for a given revision", async () => { + const info: SpaceEntry & Pick = await spaceInfo({ + name: "huggingfacejs/client-side-oauth", + additionalFields: ["sha"], + revision: "e410a9ff348e6bed393b847711e793282d7c672e", + }); + expect(info).toEqual({ + id: "659835e689010f9c7aed608d", + name: "huggingfacejs/client-side-oauth", + updatedAt: expect.any(Date), + likes: expect.any(Number), + private: false, + sdk: "static", + sha: "e410a9ff348e6bed393b847711e793282d7c672e", + }); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/space-info.ts b/node_modules/@huggingface/hub/src/lib/space-info.ts new file mode 100644 index 0000000000000000000000000000000000000000..cbf0fb8555300fd0c77d81d3f4dce0c5563f083e --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/space-info.ts @@ -0,0 +1,60 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiSpaceInfo } from "../types/api/api-space"; +import type { CredentialsParams } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { pick } from "../utils/pick"; +import type { SPACE_EXPANDABLE_KEYS, SpaceEntry } from "./list-spaces"; +import { SPACE_EXPAND_KEYS } from "./list-spaces"; + +export async function spaceInfo< + const T extends Exclude<(typeof SPACE_EXPANDABLE_KEYS)[number], (typeof SPACE_EXPAND_KEYS)[number]> = never, +>( + params: { + name: string; + hubUrl?: string; + additionalFields?: T[]; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + */ + revision?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial, +): Promise> { + const accessToken = params && checkCredentials(params); + + const search = new URLSearchParams([ + ...SPACE_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), + ...(params?.additionalFields?.map((val) => ["expand", val] satisfies [string, string]) ?? []), + ]).toString(); + + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/spaces/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD", + )}?${search.toString()}`, + { + headers: { + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }, + ); + + if (!response.ok) { + throw await createApiError(response); + } + + const data = await response.json(); + + return { + ...(params?.additionalFields && pick(data, params.additionalFields)), + id: data._id, + name: data.id, + sdk: data.sdk, + likes: data.likes, + private: data.private, + updatedAt: new Date(data.lastModified), + } as SpaceEntry & Pick; +} diff --git a/node_modules/@huggingface/hub/src/lib/upload-file.spec.ts b/node_modules/@huggingface/hub/src/lib/upload-file.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..e99e0961593949d4375bb98b6790b9c291e72a5b --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/upload-file.spec.ts @@ -0,0 +1,99 @@ +import { assert, it, describe, expect } from "vitest"; + +import { TEST_ACCESS_TOKEN, TEST_HUB_URL, TEST_USER } from "../test/consts"; +import type { RepoId } from "../types/public"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { downloadFile } from "./download-file"; +import { uploadFile } from "./upload-file"; + +describe("uploadFile", () => { + it("should upload a file", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + const result = await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + repo, + hubUrl: TEST_HUB_URL, + }); + + expect(result).toEqual({ + repoUrl: `${TEST_HUB_URL}/${repoName}`, + id: expect.any(String), + }); + + await uploadFile({ + accessToken: TEST_ACCESS_TOKEN, + repo, + file: { content: new Blob(["file1"]), path: "file1" }, + hubUrl: TEST_HUB_URL, + }); + await uploadFile({ + accessToken: TEST_ACCESS_TOKEN, + repo, + file: new URL("https://huggingface.co/gpt2/raw/main/config.json"), + hubUrl: TEST_HUB_URL, + }); + + let content = await downloadFile({ + repo, + path: "file1", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual(await content?.text(), "file1"); + + content = await downloadFile({ + repo, + path: "config.json", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual( + (await content?.text())?.trim(), + ` +{ + "activation_function": "gelu_new", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 50256, + "embd_pdrop": 0.1, + "eos_token_id": 50256, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 1024, + "n_embd": 768, + "n_head": 12, + "n_layer": 12, + "n_positions": 1024, + "resid_pdrop": 0.1, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "task_specific_params": { + "text-generation": { + "do_sample": true, + "max_length": 50 + } + }, + "vocab_size": 50257 +} + `.trim(), + ); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/upload-file.ts b/node_modules/@huggingface/hub/src/lib/upload-file.ts new file mode 100644 index 0000000000000000000000000000000000000000..e02f5ce38ac54adb8ce72e3d3faf90ceb54e75f6 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/upload-file.ts @@ -0,0 +1,49 @@ +import type { CredentialsParams } from "../types/public"; +import type { CommitOutput, CommitParams, ContentSource } from "./commit"; +import { commit } from "./commit"; + +export function uploadFile( + params: { + repo: CommitParams["repo"]; + file: URL | File | { path: string; content: ContentSource }; + commitTitle?: CommitParams["title"]; + commitDescription?: CommitParams["description"]; + hubUrl?: CommitParams["hubUrl"]; + branch?: CommitParams["branch"]; + isPullRequest?: CommitParams["isPullRequest"]; + parentCommit?: CommitParams["parentCommit"]; + fetch?: CommitParams["fetch"]; + useWebWorkers?: CommitParams["useWebWorkers"]; + abortSignal?: CommitParams["abortSignal"]; + useXet?: CommitParams["useXet"]; + } & Partial, +): Promise { + const path = + params.file instanceof URL + ? (params.file.pathname.split("/").at(-1) ?? "file") + : "path" in params.file + ? params.file.path + : params.file.name; + + return commit({ + ...(params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }), + repo: params.repo, + operations: [ + { + operation: "addOrUpdate", + path, + content: "content" in params.file ? params.file.content : params.file, + }, + ], + title: params.commitTitle ?? `Add ${path}`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet, + }); +} diff --git a/node_modules/@huggingface/hub/src/lib/upload-files-with-progress.spec.ts b/node_modules/@huggingface/hub/src/lib/upload-files-with-progress.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..a5f19bb48600d9fe2dd6bbf8c5bf5458970f5668 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/upload-files-with-progress.spec.ts @@ -0,0 +1,178 @@ +import { assert, it, describe, expect } from "vitest"; + +import { TEST_HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../test/consts"; +import type { RepoId } from "../types/public"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { downloadFile } from "./download-file"; +import { uploadFilesWithProgress } from "./upload-files-with-progress"; +import type { CommitOutput, CommitProgressEvent } from "./commit"; + +describe("uploadFilesWithProgress", () => { + for (const useXet of [false, true]) { + describe(`useXet: ${useXet}`, () => { + it("should upload files", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + const lfsContent = "O123456789".repeat(100_000); + + try { + const result = await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + repo, + hubUrl: TEST_HUB_URL, + }); + + expect(result).toEqual({ + repoUrl: `${TEST_HUB_URL}/${repoName}`, + id: expect.any(String), + }); + + const it = uploadFilesWithProgress({ + accessToken: TEST_ACCESS_TOKEN, + repo, + files: [ + { content: new Blob(["file1"]), path: "file1" }, + new URL("https://huggingface.co/gpt2/raw/main/config.json"), + // Large file + { + content: new Blob([lfsContent]), + path: "test.lfs.txt", + }, + ], + useWebWorkers: { + minSize: 1_000, + }, + hubUrl: TEST_HUB_URL, + useXet, + }); + + let res: IteratorResult; + let progressEvents: CommitProgressEvent[] = []; + + do { + res = await it.next(); + if (!res.done) { + progressEvents.push(res.value); + } + } while (!res.done); + + // const intermediateHashingEvents = progressEvents.filter( + // (e) => e.event === "fileProgress" && e.type === "hashing" && e.progress !== 0 && e.progress !== 1 + // ); + // if (isFrontend) { + // assert(intermediateHashingEvents.length > 0); + // } + // const intermediateUploadEvents = progressEvents.filter( + // (e) => e.event === "fileProgress" && e.type === "uploading" && e.progress !== 0 && e.progress !== 1 + // ); + // if (isFrontend) { + // assert(intermediateUploadEvents.length > 0, "There should be at least one intermediate upload event"); + // } + progressEvents = progressEvents.filter( + (e, i) => + (e.event !== "fileProgress" || e.progress === 0 || e.progress === 1) && + (i === 0 || JSON.stringify(e) !== JSON.stringify(progressEvents[i - 1])), + ); + + assert.deepStrictEqual(progressEvents, [ + { + event: "phase", + phase: "preuploading", + }, + { + event: "phase", + phase: "uploadingLargeFiles", + }, + { + event: "fileProgress", + path: "test.lfs.txt", + progress: 0, + state: "hashing", + }, + { + event: "fileProgress", + path: "test.lfs.txt", + progress: 1, + state: "hashing", + }, + { + event: "fileProgress", + path: "test.lfs.txt", + progress: 0, + state: "uploading", + }, + { + event: "fileProgress", + path: "test.lfs.txt", + progress: 1, + state: "uploading", + }, + { + event: "phase", + phase: "committing", + }, + ]); + + let content = await downloadFile({ + repo, + path: "file1", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual(await content?.text(), "file1"); + + content = await downloadFile({ + repo, + path: "config.json", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual( + (await content?.text())?.trim(), + ` +{ + "activation_function": "gelu_new", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 50256, + "embd_pdrop": 0.1, + "eos_token_id": 50256, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 1024, + "n_embd": 768, + "n_head": 12, + "n_layer": 12, + "n_positions": 1024, + "resid_pdrop": 0.1, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "task_specific_params": { + "text-generation": { + "do_sample": true, + "max_length": 50 + } + }, + "vocab_size": 50257 +} + `.trim(), + ); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); + }); + } +}); diff --git a/node_modules/@huggingface/hub/src/lib/upload-files-with-progress.ts b/node_modules/@huggingface/hub/src/lib/upload-files-with-progress.ts new file mode 100644 index 0000000000000000000000000000000000000000..364aabdb9ecf0418bdce4b70425155135c6b03ba --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/upload-files-with-progress.ts @@ -0,0 +1,159 @@ +import type { CredentialsParams } from "../types/public"; +import { typedInclude } from "../utils/typedInclude"; +import type { CommitOutput, CommitParams, CommitProgressEvent, ContentSource } from "./commit"; +import { commitIter } from "./commit"; + +const multipartUploadTracking = new WeakMap< + (progress: number) => void, + { + numParts: number; + partsProgress: Record; + } +>(); + +/** + * Uploads with progress + * + * Needs XMLHttpRequest to be available for progress events for uploads on models, datasets and spaces. + * Set useWebWorkers to true in order to have progress events for hashing for models, datasets and spaces. + */ +export async function* uploadFilesWithProgress( + params: { + repo: CommitParams["repo"]; + files: Array; + commitTitle?: CommitParams["title"]; + commitDescription?: CommitParams["description"]; + hubUrl?: CommitParams["hubUrl"]; + branch?: CommitParams["branch"]; + isPullRequest?: CommitParams["isPullRequest"]; + parentCommit?: CommitParams["parentCommit"]; + abortSignal?: CommitParams["abortSignal"]; + maxFolderDepth?: CommitParams["maxFolderDepth"]; + useXet?: CommitParams["useXet"]; + /** + * Set this to true in order to have progress events for hashing + */ + useWebWorkers?: CommitParams["useWebWorkers"]; + } & Partial, +): AsyncGenerator { + return yield* commitIter({ + ...(params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }), + repo: params.repo, + operations: params.files.map((file) => ({ + operation: "addOrUpdate", + path: file instanceof URL ? (file.pathname.split("/").at(-1) ?? "file") : "path" in file ? file.path : file.name, + content: "content" in file ? file.content : file, + })), + title: params.commitTitle ?? `Add ${params.files.length} files`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet, + fetch: async (input, init) => { + if (!init) { + return fetch(input); + } + + if ( + !typedInclude(["PUT", "POST"], init.method) || + !("progressHint" in init) || + !init.progressHint || + typeof XMLHttpRequest === "undefined" || + typeof input !== "string" || + (!(init.body instanceof ArrayBuffer) && + !(init.body instanceof Blob) && + !(init.body instanceof File) && + typeof init.body !== "string") + ) { + return fetch(input, init); + } + + const progressHint = init.progressHint as { + progressCallback: (progress: number) => void; + } & (Record | { part: number; numParts: number }); + const progressCallback = progressHint.progressCallback; + + const xhr = new XMLHttpRequest(); + + xhr.upload.addEventListener("progress", (event) => { + if (event.lengthComputable) { + if (progressHint.part !== undefined) { + let tracking = multipartUploadTracking.get(progressCallback); + if (!tracking) { + tracking = { numParts: progressHint.numParts, partsProgress: {} }; + multipartUploadTracking.set(progressCallback, tracking); + } + tracking.partsProgress[progressHint.part] = event.loaded / event.total; + let totalProgress = 0; + for (const partProgress of Object.values(tracking.partsProgress)) { + totalProgress += partProgress; + } + if (totalProgress === tracking.numParts) { + progressCallback(0.9999999999); + } else { + progressCallback(totalProgress / tracking.numParts); + } + } else { + if (event.loaded === event.total) { + progressCallback(0.9999999999); + } else { + progressCallback(event.loaded / event.total); + } + } + } + }); + + xhr.open(init.method, input, true); + + if (init.headers) { + const headers = new Headers(init.headers); + headers.forEach((value, key) => { + xhr.setRequestHeader(key, value); + }); + } + + init.signal?.throwIfAborted(); + xhr.send(init.body); + + return new Promise((resolve, reject) => { + xhr.addEventListener("load", () => { + resolve( + new Response(xhr.responseText, { + status: xhr.status, + statusText: xhr.statusText, + headers: Object.fromEntries( + xhr + .getAllResponseHeaders() + .trim() + .split("\n") + .map((header) => [ + header.slice(0, header.indexOf(":")), + header.slice(header.indexOf(":") + 1).trim(), + ]), + ), + }), + ); + }); + xhr.addEventListener("error", () => { + reject(new Error(xhr.statusText)); + }); + + if (init.signal) { + init.signal.addEventListener("abort", () => { + xhr.abort(); + + try { + init.signal?.throwIfAborted(); + } catch (err) { + reject(err); + } + }); + } + }); + }, + }); +} diff --git a/node_modules/@huggingface/hub/src/lib/upload-files.fs.spec.ts b/node_modules/@huggingface/hub/src/lib/upload-files.fs.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..6692c67d33d01d2f94e9a12ad7d863636a4b8853 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/upload-files.fs.spec.ts @@ -0,0 +1,72 @@ +import { assert, it, describe, expect } from "vitest"; + +import { TEST_ACCESS_TOKEN, TEST_HUB_URL, TEST_USER } from "../test/consts"; +import type { RepoId } from "../types/public"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { downloadFile } from "./download-file"; +import { uploadFiles } from "./upload-files"; +import { mkdir } from "fs/promises"; +import { writeFile } from "fs/promises"; +import { pathToFileURL } from "url"; +import { tmpdir } from "os"; + +describe("uploadFiles", () => { + it("should upload local folder", async () => { + const tmpDir = tmpdir(); + + await mkdir(`${tmpDir}/test-folder/sub`, { recursive: true }); + + await writeFile(`${tmpDir}/test-folder/sub/file1.txt`, "file1"); + await writeFile(`${tmpDir}/test-folder/sub/file2.txt`, "file2"); + + await writeFile(`${tmpDir}/test-folder/file3.txt`, "file3"); + await writeFile(`${tmpDir}/test-folder/file4.txt`, "file4"); + + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + const result = await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + repo, + hubUrl: TEST_HUB_URL, + }); + + expect(result).toEqual({ + repoUrl: `${TEST_HUB_URL}/${repoName}`, + id: expect.any(String), + }); + + await uploadFiles({ + accessToken: TEST_ACCESS_TOKEN, + repo, + files: [pathToFileURL(`${tmpDir}/test-folder`)], + hubUrl: TEST_HUB_URL, + }); + + let content = await downloadFile({ + repo, + path: "test-folder/sub/file1.txt", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual(await content?.text(), "file1"); + + content = await downloadFile({ + repo, + path: "test-folder/file3.txt", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual(await content?.text(), `file3`); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/upload-files.spec.ts b/node_modules/@huggingface/hub/src/lib/upload-files.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..58cb24fe3476d295bd198e1299bcee8ecededa27 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/upload-files.spec.ts @@ -0,0 +1,96 @@ +import { assert, it, describe, expect } from "vitest"; + +import { TEST_ACCESS_TOKEN, TEST_HUB_URL, TEST_USER } from "../test/consts"; +import type { RepoId } from "../types/public"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { downloadFile } from "./download-file"; +import { uploadFiles } from "./upload-files"; + +describe("uploadFiles", () => { + it("should upload files", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + const result = await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + repo, + hubUrl: TEST_HUB_URL, + }); + + expect(result).toEqual({ + repoUrl: `${TEST_HUB_URL}/${repoName}`, + id: expect.any(String), + }); + + await uploadFiles({ + accessToken: TEST_ACCESS_TOKEN, + repo, + files: [ + { content: new Blob(["file1"]), path: "file1" }, + new URL("https://huggingface.co/gpt2/raw/main/config.json"), + ], + hubUrl: TEST_HUB_URL, + }); + + let content = await downloadFile({ + repo, + path: "file1", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual(await content?.text(), "file1"); + + content = await downloadFile({ + repo, + path: "config.json", + hubUrl: TEST_HUB_URL, + }); + + assert.strictEqual( + (await content?.text())?.trim(), + ` +{ + "activation_function": "gelu_new", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 50256, + "embd_pdrop": 0.1, + "eos_token_id": 50256, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 1024, + "n_embd": 768, + "n_head": 12, + "n_layer": 12, + "n_positions": 1024, + "resid_pdrop": 0.1, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "task_specific_params": { + "text-generation": { + "do_sample": true, + "max_length": 50 + } + }, + "vocab_size": 50257 +} + `.trim(), + ); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/upload-files.ts b/node_modules/@huggingface/hub/src/lib/upload-files.ts new file mode 100644 index 0000000000000000000000000000000000000000..f400cf298a14d4f9ebb435056d1d18300d020e51 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/upload-files.ts @@ -0,0 +1,41 @@ +import type { CredentialsParams } from "../types/public"; +import type { CommitOutput, CommitParams, ContentSource } from "./commit"; +import { commit } from "./commit"; + +export function uploadFiles( + params: { + repo: CommitParams["repo"]; + files: Array; + commitTitle?: CommitParams["title"]; + commitDescription?: CommitParams["description"]; + hubUrl?: CommitParams["hubUrl"]; + branch?: CommitParams["branch"]; + isPullRequest?: CommitParams["isPullRequest"]; + parentCommit?: CommitParams["parentCommit"]; + fetch?: CommitParams["fetch"]; + useWebWorkers?: CommitParams["useWebWorkers"]; + maxFolderDepth?: CommitParams["maxFolderDepth"]; + abortSignal?: CommitParams["abortSignal"]; + useXet?: CommitParams["useXet"]; + } & Partial, +): Promise { + return commit({ + ...(params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }), + repo: params.repo, + operations: params.files.map((file) => ({ + operation: "addOrUpdate", + path: file instanceof URL ? (file.pathname.split("/").at(-1) ?? "file") : "path" in file ? file.path : file.name, + content: "content" in file ? file.content : file, + })), + title: params.commitTitle ?? `Add ${params.files.length} files`, + description: params.commitDescription, + hubUrl: params.hubUrl, + branch: params.branch, + isPullRequest: params.isPullRequest, + parentCommit: params.parentCommit, + fetch: params.fetch, + useWebWorkers: params.useWebWorkers, + abortSignal: params.abortSignal, + useXet: params.useXet, + }); +} diff --git a/node_modules/@huggingface/hub/src/lib/who-am-i.spec.ts b/node_modules/@huggingface/hub/src/lib/who-am-i.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..9996727293ae4b5ca656d8fb949d05c64a7fd3c4 --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/who-am-i.spec.ts @@ -0,0 +1,36 @@ +import { assert, it, describe } from "vitest"; +import { TEST_ACCESS_TOKEN, TEST_HUB_URL } from "../test/consts"; +import { whoAmI } from "./who-am-i"; + +describe("whoAmI", () => { + it("should fetch identity info", async () => { + const info = await whoAmI({ accessToken: TEST_ACCESS_TOKEN, hubUrl: TEST_HUB_URL }); + + if (info.auth.accessToken?.createdAt instanceof Date) { + info.auth.accessToken.createdAt = new Date(0); + } + + assert.deepStrictEqual(info, { + type: "user", + id: "62f264b9f3c90f4b6514a269", + name: "hub.js", + fullname: "@huggingface/hub CI bot", + email: "eliott@huggingface.co", + emailVerified: true, + canPay: false, + isPro: false, + periodEnd: null, + avatarUrl: "/avatars/934b830e9fdaa879487852f79eef7165.svg", + billingMode: "postpaid", + orgs: [], + auth: { + type: "access_token", + accessToken: { + createdAt: new Date(0), + displayName: "ci-hub.js", + role: "write", + }, + }, + }); + }); +}); diff --git a/node_modules/@huggingface/hub/src/lib/who-am-i.ts b/node_modules/@huggingface/hub/src/lib/who-am-i.ts new file mode 100644 index 0000000000000000000000000000000000000000..1c50650bc957c834d1efa5973e17472e4037a4bd --- /dev/null +++ b/node_modules/@huggingface/hub/src/lib/who-am-i.ts @@ -0,0 +1,92 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiWhoAmIReponse } from "../types/api/api-who-am-i"; +import type { AccessTokenRole, AuthType, CredentialsParams } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; + +export interface WhoAmIUser { + /** Unique ID persistent across renames */ + id: string; + type: "user"; + email: string; + emailVerified: boolean; + isPro: boolean; + orgs: WhoAmIOrg[]; + name: string; + fullname: string; + canPay: boolean; + avatarUrl: string; + /** + * Unix timestamp in seconds + */ + periodEnd: number | null; + billingMode: "postpaid" | "prepaid"; +} + +export interface WhoAmIOrg { + /** Unique ID persistent across renames */ + id: string; + type: "org"; + name: string; + fullname: string; + email: string | null; + canPay: boolean; + avatarUrl: string; + /** + * Unix timestamp in seconds + */ + periodEnd: number | null; +} + +export interface WhoAmIApp { + id: string; + type: "app"; + name: string; + scope?: { + entities: string[]; + role: "admin" | "write" | "contributor" | "read"; + }; +} + +export type WhoAmI = WhoAmIApp | WhoAmIOrg | WhoAmIUser; +export interface AuthInfo { + type: AuthType; + accessToken?: { + displayName: string; + role: AccessTokenRole; + createdAt: Date; + }; + expiresAt?: Date; +} + +export async function whoAmI( + params: { + hubUrl?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & CredentialsParams, +): Promise { + const accessToken = checkCredentials(params); + + const res = await (params.fetch ?? fetch)(`${params.hubUrl ?? HUB_URL}/api/whoami-v2`, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }); + + if (!res.ok) { + throw await createApiError(res); + } + + const response: ApiWhoAmIReponse & { + auth: AuthInfo; + } = await res.json(); + + if (typeof response.auth.accessToken?.createdAt === "string") { + response.auth.accessToken.createdAt = new Date(response.auth.accessToken.createdAt); + } + + return response; +} diff --git a/node_modules/@huggingface/hub/src/test/consts.ts b/node_modules/@huggingface/hub/src/test/consts.ts new file mode 100644 index 0000000000000000000000000000000000000000..6b8b7983d62f140076851c03c901ab2da6c6bff3 --- /dev/null +++ b/node_modules/@huggingface/hub/src/test/consts.ts @@ -0,0 +1,4 @@ +export const TEST_HUB_URL = "https://hub-ci.huggingface.co"; +export const TEST_USER = "hub.js"; +export const TEST_ACCESS_TOKEN = "hf_hub.js"; +export const TEST_COOKIE = "huggingface-hub.js-cookie"; diff --git a/node_modules/@huggingface/hub/src/types/api/api-author.ts b/node_modules/@huggingface/hub/src/types/api/api-author.ts new file mode 100644 index 0000000000000000000000000000000000000000..b9e562a01ff9b2342a23e990318f6b5d53eddfaf --- /dev/null +++ b/node_modules/@huggingface/hub/src/types/api/api-author.ts @@ -0,0 +1,26 @@ +export type ApiAuthor = + | { + avatarUrl: string; + fullname: string; + name: string; + isHf: boolean; + isHfAdmin: boolean; + isMod: boolean; + followerCount?: number; + type: "org"; + plan?: string; + isUserFollowing?: boolean; + } + | { + avatarUrl: string; + fullname: string; + name: string; + isHf: boolean; + isHfAdmin: boolean; + isMod: boolean; + followerCount?: number; + type: "user"; + isPro: boolean; + _id: string; + isUserFollowing?: boolean; + }; diff --git a/node_modules/@huggingface/hub/src/types/api/api-collection.ts b/node_modules/@huggingface/hub/src/types/api/api-collection.ts new file mode 100644 index 0000000000000000000000000000000000000000..52b497ebb57929d6497f397996ec523427b403db --- /dev/null +++ b/node_modules/@huggingface/hub/src/types/api/api-collection.ts @@ -0,0 +1,329 @@ +import type { ApiAuthor } from "./api-author"; + +export interface ApiCollectionInfo { + slug: string; + title: string; + description?: string; + lastUpdated: string; + gating: + | true + | ( + | false + | { + mode: "auto"; + } + | { + mode: "manual"; + notifications: { + mode: "bulk" | "real-time"; + email?: string; + }; + } + ); + owner: ApiAuthor; + /** + * Note that it's limited to 4 items when the listing endpoint is used. + */ + items: ApiCollectionItem[]; + theme: "orange" | "blue" | "green" | "purple" | "pink" | "indigo"; + private: boolean; + upvotes: number; + isUpvotedByUser: boolean; +} + +interface ApiCollectionItemBase { + _id: string; + position: number; + note?: { + html: string; + text: string; + }; + gallery?: string[]; +} + +interface ApiCollectionItemModel extends ApiCollectionItemBase { + type: "model"; + author: string; + downloads: number; + id: string; + availableInferenceProviders: { + provider: + | "black-forest-labs" + | "cerebras" + | "cohere" + | "fal-ai" + | "featherless-ai" + | "fireworks-ai" + | "groq" + | "hf-inference" + | "hyperbolic" + | "nebius" + | "novita" + | "nscale" + | "openai" + | "ovhcloud" + | "replicate" + | "sambanova" + | "together"; + providerStatus: "live" | "staging" | "error"; + modelStatus: "live" | "staging" | "error"; + providerId: string; + task: + | "text-classification" + | "token-classification" + | "table-question-answering" + | "question-answering" + | "zero-shot-classification" + | "translation" + | "summarization" + | "feature-extraction" + | "text-generation" + | "text2text-generation" + | "fill-mask" + | "sentence-similarity" + | "text-to-speech" + | "text-to-audio" + | "automatic-speech-recognition" + | "audio-to-audio" + | "audio-classification" + | "audio-text-to-text" + | "voice-activity-detection" + | "depth-estimation" + | "image-classification" + | "object-detection" + | "image-segmentation" + | "text-to-image" + | "image-to-text" + | "image-to-image" + | "image-to-video" + | "unconditional-image-generation" + | "video-classification" + | "reinforcement-learning" + | "robotics" + | "tabular-classification" + | "tabular-regression" + | "tabular-to-text" + | "table-to-text" + | "multiple-choice" + | "text-ranking" + | "text-retrieval" + | "time-series-forecasting" + | "text-to-video" + | "image-text-to-text" + | "visual-question-answering" + | "document-question-answering" + | "zero-shot-image-classification" + | "graph-ml" + | "mask-generation" + | "zero-shot-object-detection" + | "text-to-3d" + | "image-to-3d" + | "image-feature-extraction" + | "video-text-to-text" + | "keypoint-detection" + | "visual-document-retrieval" + | "any-to-any" + | "video-to-video" + | "other" + | "conversational"; + adapterType?: "lora"; + adapterWeightsPath?: string; + }[]; + isLikedByUser: boolean; + lastModified: string; + likes: number; + pipeline_tag?: string; + private: boolean; + repoType: "model"; + gated: false | ("auto" | "manual"); + resourceGroup?: { + id: string; + name: string; + numUsers: number; + }; + numParameters?: number; + authorData?: ApiAuthor; + widgetOutputUrls?: string[]; +} + +interface ApiCollectionItemDataset extends ApiCollectionItemBase { + type: "dataset"; + author: string; + id: string; + isLikedByUser: boolean; + likes: number; + datasetsServerInfo?: { + viewer: "preview" | "viewer-partial" | "viewer"; + numRows: number | null; + libraries: ( + | "mlcroissant" + | "webdataset" + | "datasets" + | "pandas" + | "dask" + | "distilabel" + | "fiftyone" + | "argilla" + | "polars" + | "duckdb" + )[]; + formats: ("json" | "csv" | "parquet" | "imagefolder" | "audiofolder" | "webdataset" | "text" | "arrow")[]; + modalities: ("3d" | "audio" | "document" | "geospatial" | "image" | "tabular" | "text" | "timeseries" | "video")[]; + }; + private: boolean; + repoType: "dataset"; + downloads: number; + gated: false | ("auto" | "manual"); + lastModified: string; + resourceGroup?: { + id: string; + name: string; + numUsers: number; + }; +} + +interface ApiCollectionItemSpace extends ApiCollectionItemBase { + type: "space"; + author: string; + colorFrom: string; + colorTo: string; + createdAt: string; + emoji: string; + id: string; + isLikedByUser: boolean; + lastModified: string; + likes: number; + pinned: boolean; + private: boolean; + featured: boolean; + repoType: "space"; + title: string; + sdk?: "gradio" | "docker" | "static" | "streamlit"; + runtime: { + stage: + | "NO_APP_FILE" + | "CONFIG_ERROR" + | "BUILDING" + | "BUILD_ERROR" + | "APP_STARTING" + | "RUNNING" + | "RUNNING_BUILDING" + | "RUNNING_APP_STARTING" + | "RUNTIME_ERROR" + | "DELETING" + | "STOPPED" + | "PAUSED" + | "SLEEPING"; + hardware: { + current: + | ( + | "cpu-basic" + | "cpu-upgrade" + | "cpu-performance" + | "cpu-xl" + | "zero-a10g" + | "t4-small" + | "t4-medium" + | "l4x1" + | "l4x4" + | "l40sx1" + | "l40sx4" + | "l40sx8" + | "a10g-small" + | "a10g-large" + | "a10g-largex2" + | "a10g-largex4" + | "a100-large" + | "h100" + | "h100x8" + ) + | null; + requested: + | ( + | "cpu-basic" + | "cpu-upgrade" + | "cpu-performance" + | "cpu-xl" + | "zero-a10g" + | "t4-small" + | "t4-medium" + | "l4x1" + | "l4x4" + | "l40sx1" + | "l40sx4" + | "l40sx8" + | "a10g-small" + | "a10g-large" + | "a10g-largex2" + | "a10g-largex4" + | "a100-large" + | "h100" + | "h100x8" + ) + | null; + }; + storage: ("small" | "medium" | "large") | null; + errorMessage?: string; + gcTimeout?: number | null; + replicas: { + current?: number | null; + requested: number | "auto"; + }; + devMode?: boolean; + domains?: { + domain: string; + isCustom?: boolean | null; + stage: "READY" | "PENDING"; + }[]; + sha?: string; + }; + originSpace?: { + author: ApiAuthor; + name: string; + }; + ai_short_description?: string; + ai_category?: string; + trendingScore?: number; + resourceGroup?: { + id: string; + name: string; + numUsers: number; + }; + tags: string[]; + authorData?: ApiAuthor; + shortDescription?: string; + semanticRelevancyScore?: number; + visibility?: "public" | "private" | "protected"; +} + +interface ApiCollectionItemPaper extends ApiCollectionItemBase { + type: "paper"; + id: string; + title: string; + upvotes: number; + publishedAt: string; + thumbnailUrl?: string; + isUpvotedByUser?: boolean; +} + +interface ApiCollectionItemCollection extends ApiCollectionItemBase { + type: "collection"; + slug: string; + lastUpdated: string; + description?: string; + owner: ApiAuthor; + title: string; + theme: "orange" | "blue" | "green" | "purple" | "pink" | "indigo"; + upvotes: number; + isUpvotedByUser: boolean; + id: string; + numberItems: number; + shareUrl: string; +} + +type ApiCollectionItem = + | ApiCollectionItemModel + | ApiCollectionItemDataset + | ApiCollectionItemSpace + | ApiCollectionItemPaper + | ApiCollectionItemCollection; diff --git a/node_modules/@huggingface/hub/src/types/api/api-commit.ts b/node_modules/@huggingface/hub/src/types/api/api-commit.ts new file mode 100644 index 0000000000000000000000000000000000000000..7ef67730da786d83814a279df8f86fb26f4dc4ce --- /dev/null +++ b/node_modules/@huggingface/hub/src/types/api/api-commit.ts @@ -0,0 +1,204 @@ +export interface ApiLfsBatchRequest { + /// github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md + operation: "download" | "upload"; + transfers?: Array; + /** + * Optional object describing the server ref that the objects belong to. Note: Added in v2.4. + * + * We use this object for QOL and to fail early for users when they're trying to push to the wrong reference. + * But it does nothing for security. + */ + ref?: { + name: string; + } | null; + objects: { + oid: string; + /** + * Integer byte size of the LFS object. Must be at least zero. + */ + size: number; + }[]; + /** + * The hash algorithm used to name Git LFS objects. Optional; defaults to sha256 if not specified. + * */ + hash_algo?: string; +} + +export interface ApiLfsBatchResponse { + transfer?: ApiLfsResponseTransfer; + objects: ApiLfsResponseObject[]; +} + +export type ApiLfsResponseTransfer = "basic" | "multipart" | "xet"; + +export interface ApiLfsCompleteMultipartRequest { + oid: string; + parts: { etag: string; partNumber: number }[]; +} + +export interface ApiLfsResponseObject { + /** + * Optional boolean specifying whether the request + * for this specific object is authenticated. + * If omitted or false, Git LFS will attempt to find credentials for this URL. + */ + authenticated?: boolean; + oid: string; + /** + * Integer byte size of the LFS object. Must be at least zero. + */ + size: number; + /** + * Applicable actions depend on which `operation` is specified in the request. + * How these properties are interpreted depends on which transfer adapter + * the client will be using. + */ + actions?: { + /** + * Download operations MUST specify a download action, + * or an object error if the object cannot be downloaded for some reason + */ + download?: ApiLfsAction; + /** + * Upload operations can specify an upload and a verify action. + * The upload action describes how to upload the object. + */ + upload?: ApiLfsAction; + /** + * The LFS client will hit this URL after a successful upload. + * Servers can use this for extra verification, if needed. + */ + verify?: ApiLfsAction; + }; + /** + * If there are problems accessing individual objects, servers should continue + * to return a 200 status code, and provide per-object errors + */ + error?: { + code: number; + message: string; + }; +} + +export interface ApiLfsAction { + href: string; + /** + * Optional hash of String HTTP header key/value pairs to apply to the request + */ + header?: { [key: string]: string } & { chunk_size?: string }; + /** + * Whole number of seconds after local client time when transfer will expire. + * Preferred over `expires_at` if both are provided. + * Maximum of 2147483647, minimum of -2147483647. + */ + expires_in?: number; + /** + * String uppercase RFC 3339-formatted timestamp with second precision + * for when the given action expires (usually due to a temporary token). + */ + expires_at?: string; +} + +export interface ApiPreuploadRequest { + /** + * Optional, otherwise takes the existing content of `.gitattributes` for the revision. + * + * Provide this parameter if you plan to modify `.gitattributes` yourself at the same + * time as uploading LFS files. + * + * Note that this is not needed if you solely rely on automatic LFS detection from HF: the commit endpoint + * will automatically edit the `.gitattributes` file to track the files passed to its `lfsFiles` param. + */ + gitAttributes?: string; + files: Array<{ + /** + * Path of the LFS file + */ + path: string; + /** + * Full size of the LFS file + */ + size: number; + /** + * Base64-encoded sample of the first 512 bytes of the file + */ + sample: string; + }>; +} +export interface ApiBucketBatchResponse { + /** True if all files were successfully added */ + success: boolean; + /** Total number of operations attempted */ + processed: number; + /** Number of successful operations */ + succeeded: number; + /** List of failed operations */ + failed: Array<{ + path: string; + error: string; + }>; +} + +export interface ApiPreuploadResponse { + files: Array<{ + path: string; + uploadMode: "lfs" | "regular"; + }>; +} + +export interface ApiCommitHeader { + summary: string; + description?: string; + /** + * Parent commit. Optional + * + * - When opening a PR: will use parentCommit as the parent commit + * - When committing on a branch: Will make sure that there were no intermediate commits + */ + parentCommit?: string; +} + +export interface ApiCommitDeletedEntry { + path: string; +} + +export interface ApiCommitLfsFile { + path: string; + oldPath?: string; + /** Required if {@link oldPath} is not set */ + algo?: "sha256"; + /** Required if {@link oldPath} is not set */ + oid?: string; + size?: number; +} + +export interface ApiCommitFile { + /** Required if {@link oldPath} is not set */ + content?: string; + path: string; + oldPath?: string; + encoding?: "utf-8" | "base64"; +} + +export type ApiCommitOperation = + | { + key: "file"; + value: ApiCommitFile; + } + | { + key: "lfsFile"; + value: ApiCommitLfsFile; + } + | { + key: "deletedFile"; + value: ApiCommitDeletedEntry; + }; + +export interface ApiCommitData { + id: string; + title: string; + message: string; + authors: Array<{ user: string; avatar: string }>; + date: string; + formatted?: string; +} diff --git a/node_modules/@huggingface/hub/src/types/api/api-create-collection.ts b/node_modules/@huggingface/hub/src/types/api/api-create-collection.ts new file mode 100644 index 0000000000000000000000000000000000000000..9d246b7561b869a6f0f7ea506c25e1e159e93993 --- /dev/null +++ b/node_modules/@huggingface/hub/src/types/api/api-create-collection.ts @@ -0,0 +1,19 @@ +export interface ApiCreateCollectionPayload { + /** + * Title of the collection to create. + */ + title: string; + /** + * Namespace of the collection to create (username or org). + */ + namespace: string; + /** + * Description of the collection to create. + */ + description?: string; + /** + * Whether the collection should be private or not. Defaults to False (i.e. public collection). + * @default false + */ + private?: boolean; +} diff --git a/node_modules/@huggingface/hub/src/types/api/api-create-repo.ts b/node_modules/@huggingface/hub/src/types/api/api-create-repo.ts new file mode 100644 index 0000000000000000000000000000000000000000..0d6a9adccc09a9ffad31ff6859e801935dbb592a --- /dev/null +++ b/node_modules/@huggingface/hub/src/types/api/api-create-repo.ts @@ -0,0 +1,25 @@ +import type { SetRequired } from "../../vendor/type-fest/set-required"; +import type { RepoType, SpaceHardwareFlavor, SpaceSdk } from "../public"; +import type { ApiCommitFile } from "./api-commit"; + +export type ApiCreateRepoPayload = { + name: string; + canonical?: boolean; + license?: string; + resourceGroupId?: string; + template?: string; + organization?: string; + visibility?: "public" | "private" | "protected"; + lfsmultipartthresh?: number; + files?: SetRequired[]; +} & ( + | { + type: Exclude; + } + | { + type: "space"; + hardware?: SpaceHardwareFlavor; + sdk: SpaceSdk; + sdkVersion?: string; + } +); diff --git a/node_modules/@huggingface/hub/src/types/api/api-dataset.ts b/node_modules/@huggingface/hub/src/types/api/api-dataset.ts new file mode 100644 index 0000000000000000000000000000000000000000..43b0978537d8855745fd668443b3fc102060b3e4 --- /dev/null +++ b/node_modules/@huggingface/hub/src/types/api/api-dataset.ts @@ -0,0 +1,89 @@ +import type { License } from "../public"; + +export interface ApiDatasetInfo { + _id: string; + id: string; + arxivIds?: string[]; + author?: string; + cardExists?: true; + cardError?: unknown; + cardData?: ApiDatasetMetadata; + contributors?: Array<{ user: string; _id: string }>; + disabled: boolean; + discussionsDisabled: boolean; + gated: false | "auto" | "manual"; + gitalyUid: string; + lastAuthor: { email: string; user?: string }; + lastModified: string; // date + likes: number; + likesRecent: number; + private: boolean; + updatedAt: string; // date + createdAt: string; // date + tags: string[]; + paperswithcode_id?: string; + sha: string; + files?: string[]; + citation?: string; + description?: string; + downloads: number; + downloadsAllTime: number; + previewable?: boolean; + doi?: { id: string; commit: string }; +} + +export interface ApiDatasetMetadata { + licenses?: undefined; + license?: License | License[]; + license_name?: string; + license_link?: "LICENSE" | "LICENSE.md" | string; + license_details?: string; + languages?: undefined; + language?: string | string[]; + language_bcp47?: string[]; + language_details?: string; + tags?: string[]; + task_categories?: string[]; + task_ids?: string[]; + config_names?: string[]; + configs?: { + config_name: string; + data_files?: + | string + | string[] + | { + split: string; + path: string | string[]; + }[]; + data_dir?: string; + }[]; + benchmark?: string; + paperswithcode_id?: string | null; + pretty_name?: string; + viewer?: boolean; + viewer_display_urls?: boolean; + thumbnail?: string | null; + description?: string | null; + annotations_creators?: string[]; + language_creators?: string[]; + multilinguality?: string[]; + size_categories?: string[]; + source_datasets?: string[]; + extra_gated_prompt?: string; + extra_gated_fields?: { + /** + * "text" | "checkbox" | "date_picker" | "country" | "ip_location" | { type: "text" | "checkbox" | "date_picker" | "country" | "ip_location" } | { type: "select", options: Array } Property + */ + [x: string]: + | "text" + | "checkbox" + | "date_picker" + | "country" + | "ip_location" + | { type: "text" | "checkbox" | "date_picker" | "country" | "ip_location" } + | { type: "select"; options: Array }; + }; + extra_gated_heading?: string; + extra_gated_description?: string; + extra_gated_button_content?: string; +} diff --git a/node_modules/@huggingface/hub/src/types/api/api-index-tree.ts b/node_modules/@huggingface/hub/src/types/api/api-index-tree.ts new file mode 100644 index 0000000000000000000000000000000000000000..196a27e37b90f7b3b9bd31324a5cd964f21d96a0 --- /dev/null +++ b/node_modules/@huggingface/hub/src/types/api/api-index-tree.ts @@ -0,0 +1,51 @@ +export interface ApiIndexTreeEntry { + type: "file" | "directory" | "unknown"; + size: number; + path: string; + oid: string; + lfs?: { + oid: string; + size: number; + /** Size of the raw pointer file, 100~200 bytes */ + pointerSize: number; + }; + /** + * Xet content hash. Set for bucket file entries (always) and for repo LFS entries + * that have been migrated to xet. + */ + xetHash?: string; + lastCommit?: { + date: string; + id: string; + title: string; + }; + security?: ApiFileScanResult; +} + +export interface ApiFileScanResult { + /** namespaced by repo type (models/, datasets/, spaces/) */ + repositoryId: string; + blobId: string; + name: string; + safe: boolean; + avScan?: ApiAVScan; + pickleImportScan?: ApiPickleImportScan; +} + +interface ApiAVScan { + virusFound: boolean; + virusNames?: string[]; +} + +type ApiSafetyLevel = "innocuous" | "suspicious" | "dangerous"; + +interface ApiPickleImport { + module: string; + name: string; + safety: ApiSafetyLevel; +} + +interface ApiPickleImportScan { + highestSafetyLevel: ApiSafetyLevel; + imports: ApiPickleImport[]; +} diff --git a/node_modules/@huggingface/hub/src/types/api/api-jobs.ts b/node_modules/@huggingface/hub/src/types/api/api-jobs.ts new file mode 100644 index 0000000000000000000000000000000000000000..65bacaf2a4d23a084e2ac27ec925b771596cfa05 --- /dev/null +++ b/node_modules/@huggingface/hub/src/types/api/api-jobs.ts @@ -0,0 +1,174 @@ +import type { RepoDesignation, RepoType, SpaceHardwareFlavor } from "../public"; + +export interface ApiJobVolume { + type: RepoType; + source: string; + mountPath: string; + revision?: string; + readOnly?: boolean; + path?: string; +} + +export interface JobVolume { + /** Source repo, e.g. "datasets/user/my-dataset", "user/my-model", or { type: "dataset", name: "user/my-dataset" } */ + source: RepoDesignation; + /** Mount path inside the container, e.g. "/data" */ + mountPath: string; + /** Git revision (only for repos, defaults to "main") */ + revision?: string; + /** Read-only mount (forced true for repos, defaults to false for buckets) */ + readOnly?: boolean; + /** Subfolder prefix inside the bucket/repo to mount, e.g. "path/to/dir" */ + path?: string; +} + +export interface ApiJobHardware { + name: string; + prettyName: string; + cpu: string; + ram: string; + accelerator: { + type: "gpu" | "neuron"; + model: string; + quantity: string; + vram: string; + manufacturer: "Nvidia" | "AWS"; + } | null; + unitCostMicroUSD: number; + unitCostUSD: number; + unitLabel: string; +} + +export type JobStatusStage = "DELETING" | "RUNNING" | "PAUSED" | "STOPPED" | "UPDATING" | "ERROR"; + +export interface ApiJobStatus { + stage: JobStatusStage; + message?: string | null; + failureCount: number; +} + +export interface ApiJobUser { + id: string; + name: string; + type?: "user" | "org"; + avatarUrl?: string; +} + +export interface ApiJob { + type: "job"; + id: string; + status: ApiJobStatus; + createdAt: string; + updatedAt?: string; + startedAt?: string | null; + finishedAt?: string | null; + createdBy?: { + id: string; + name: string; + }; + dockerImage?: string | null; + spaceId?: string | null; + command?: string[] | null; + arguments?: string[] | null; + environment?: Record | null; + flavor: SpaceHardwareFlavor; + arch?: "amd64" | "arm64" | null; + timeoutSeconds?: number | null; + attempts?: number; + owner?: ApiJobUser; + initiator?: ApiJobUser; + secrets?: string[]; + labels?: Record | null; + volumes?: ApiJobVolume[] | null; +} + +export interface ApiScheduledJob { + id: string; + schedule: string; + suspend: boolean; + concurrency: boolean; + createdAt: string; + updatedAt: string; + jobSpec: { + dockerImage?: string | null; + spaceId?: string | null; + command?: string[] | null; + environment?: Record | null; + flavor: SpaceHardwareFlavor; + arch?: "amd64" | "arm64" | null; + timeoutSeconds?: number | null; + attempts?: number; + labels?: Record | null; + volumes?: ApiJobVolume[] | null; + }; +} + +export interface CreateJobOptions { + /** + * The Docker image to run (e.g., "python:3.12" or "pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel") + */ + dockerImage?: string; + /** + * The Space ID to run (e.g., "username/space-name") + */ + spaceId?: string; + /** + * The command to run (array of strings) + */ + command?: string[]; + /** + * Additional arguments to pass to the command + */ + arguments?: string[]; + /** + * Environment variables to set + */ + environment?: Record; + /** + * Secrets to pass (will be encrypted server-side) + */ + secrets?: Record; + /** + * Hardware flavor to use + */ + flavor: SpaceHardwareFlavor; + /** + * Architecture (defaults to "amd64") + */ + arch?: "amd64" | "arm64"; + /** + * Timeout in seconds + */ + timeoutSeconds?: number | null; + /** + * Maximum number of attempts (defaults to 1) + */ + attempts?: number; + /** + * Labels to attach to the job (key-value pairs) + */ + labels?: Record; + /** + * HuggingFace Buckets or Repos to mount as volumes in the job container + */ + volumes?: JobVolume[]; +} + +export interface CreateScheduledJobOptions { + /** + * The job specification + */ + jobSpec: Omit; + /** + * CRON schedule expression (e.g., "0 9 * * 1" for 9 AM every Monday) or shortcuts like "@hourly", "@daily" + */ + schedule: string; + /** + * Whether the scheduled job is suspended (paused) + */ + suspend?: boolean; + /** + * Whether multiple instances of this job can run concurrently + */ + concurrency?: boolean; +} diff --git a/node_modules/@huggingface/hub/src/types/api/api-model.ts b/node_modules/@huggingface/hub/src/types/api/api-model.ts new file mode 100644 index 0000000000000000000000000000000000000000..314fda393b4906ebf4268c779dc102b4db4bb345 --- /dev/null +++ b/node_modules/@huggingface/hub/src/types/api/api-model.ts @@ -0,0 +1,283 @@ +import type { ModelLibraryKey, TransformersInfo, WidgetType } from "@huggingface/tasks"; +import type { License, PipelineType } from "../public"; + +export interface ApiModelInfo { + _id: string; + id: string; + arxivIds: string[]; + author?: string; + cardData?: ApiModelMetadata; + cardError: unknown; + cardExists?: true; + config: unknown; + contributors: Array<{ user: string; _id: string }>; + disabled: boolean; + discussionsDisabled: boolean; + doi?: { id: string; commit: string }; + downloads: number; + downloadsAllTime: number; + files: string[]; + gitalyUid: string; + inferenceProviderMapping?: ApiModelInferenceProviderMappingEntry[]; + lastAuthor: { email: string; user?: string }; + lastModified: string; // convert to date + library_name?: ModelLibraryKey; + likes: number; + likesRecent: number; + private: boolean; + gated: false | "auto" | "manual"; + sha: string; + spaces: string[]; + updatedAt: string; // convert to date + createdAt: string; // convert to date + pipeline_tag: PipelineType; + tags: string[]; + "model-index": unknown; + safetensors?: { + parameters: Record; + total: number; + }; + siblings: Array<{ rfilename: string }>; + transformersInfo?: TransformersInfo; +} + +export interface ApiModelIndex { + name: string; + results: { + task: { + /** + * Example: automatic-speech-recognition +Use task id from https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/tasksData.ts + */ + type: string; + /** + * Example: Speech Recognition + */ + name?: string; + }; + /** + * This will switch to required at some point. +in any case, we need them to link to PWC + */ + dataset?: { + /** + * Example: common_voice. Use dataset id from https://hf.co/datasets + */ + type: string; + /** + * A pretty name for the dataset. Example: Common Voice zh-CN +Also encode config params into the name if relevant. + */ + name: string; + /** + * Optional. The name of the dataset configuration used in `load_dataset()` + */ + config?: string; + /** + * Optional. Example: test + */ + split?: string; + /** + * Optional. Example: 5503434ddd753f426f4b38109466949a1217c2bb + */ + revision?: string; + args?: + | string + | { + /** + * String Property + */ + [x: string]: string; + }; + }; + metrics: { + /** + * Example: wer. Use metric id from https://hf.co/metrics + */ + type: string; + /** + * Required. Example: 20.0 or "20.0 ± 1.2" + */ + value: unknown; + /** + * Example: Test WER + */ + name?: string; + /** + * Optional. The name of the metric configuration used in `load_metric()`. + */ + config?: string; + args?: + | string + | { + /** + * String Property + */ + [x: string]: string; + }; + /** + * [Automatically computed, do not set] Dynamically overridden by huggingface in API calls to indicate if it was verified by Hugging Face. + */ + verified?: boolean; + /** + * Generated by Hugging Face to prove the results are valid. + */ + verifyToken?: string; + }[]; + /** + * The source for this evaluation result. + */ + source?: { + /** + * Example: Open LLM Leaderboard + */ + name?: string; + /** + * Example: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard + */ + url: string; + }; + }[]; +} + +export interface ApiWidgetExampleFromModelcard { + example_title?: string; + group?: string; + text?: string; + src?: string; + table?: { + /** + * (string | number)[] Property + */ + [x: string]: (string | number)[]; + }; + structured_data?: { + /** + * (string | number)[] Property + */ + [x: string]: (string | number)[]; + }; + candidate_labels?: string; + messages?: { + role: "system" | "user" | "assistant"; + content: string; + }[]; + multi_class?: boolean; + source_sentence?: string; + sentences?: string[]; + parameters?: { + aggregation_strategy?: string; + top_k?: number; + top_p?: number; + temperature?: number; + max_new_tokens?: number; + do_sample?: boolean; + negative_prompt?: string; + guidance_scale?: number; + num_inference_steps?: number; + }; + output?: + | { + label: string; + score: number; + }[] + | { + answer: string; + score: number; + } + | { + text: string; + } + | { + url: string; + }; +} + +export interface ApiModelMetadata { + datasets?: string | string[]; + license?: License | License[]; + license_name?: string; + license_link?: "LICENSE" | "LICENSE.md" | string; + license_details?: string; + inference?: + | boolean + | { + parameters?: { + aggregation_strategy?: string; + top_k?: number; + top_p?: number; + temperature?: number; + max_new_tokens?: number; + do_sample?: boolean; + negative_prompt?: string; + guidance_scale?: number; + num_inference_steps?: number; + }; + }; + language?: string | string[]; + language_bcp47?: string[]; + language_details?: string; + tags?: string[]; + pipeline_tag?: string; + co2_eq_emissions?: + | number + | { + /** + * Emissions in grams of CO2 + */ + emissions: number; + /** + * source of the information, either directly from AutoTrain, code carbon or from a scientific article documenting the model + */ + source?: string; + /** + * pre-training or fine-tuning + */ + training_type?: string; + /** + * as granular as possible, for instance Quebec, Canada or Brooklyn, NY, USA + */ + geographical_location?: string; + /** + * how much compute and what kind, e.g. 8 v100 GPUs + */ + hardware_used?: string; + }; + library_name?: string; + thumbnail?: string | null; + description?: string | null; + mask_token?: string; + widget?: ApiWidgetExampleFromModelcard[]; + "model-index"?: ApiModelIndex[]; + finetuned_from?: string; + base_model?: string | string[]; + instance_prompt?: string | null; + extra_gated_prompt?: string; + extra_gated_fields?: { + /** + * "text" | "checkbox" | "date_picker" | "country" | "ip_location" | { type: "text" | "checkbox" | "date_picker" | "country" | "ip_location" } | { type: "select", options: Array } Property + */ + [x: string]: + | "text" + | "checkbox" + | "date_picker" + | "country" + | "ip_location" + | { type: "text" | "checkbox" | "date_picker" | "country" | "ip_location" } + | { type: "select"; options: Array }; + }; + extra_gated_heading?: string; + extra_gated_description?: string; + extra_gated_button_content?: string; +} + +export interface ApiModelInferenceProviderMappingEntry { + provider: string; // Provider name + hfModelId: string; // ID of the model on the Hugging Face Hub + providerId: string; // ID of the model on the provider's side + status: "live" | "staging"; + task: WidgetType; + adapter?: string; + adapterWeightsPath?: string; + type?: "single-file" | "tag-filter"; +} diff --git a/node_modules/@huggingface/hub/src/types/api/api-space.ts b/node_modules/@huggingface/hub/src/types/api/api-space.ts new file mode 100644 index 0000000000000000000000000000000000000000..151677f4b36aa268f6ec95340f838a58b4a0f0f5 --- /dev/null +++ b/node_modules/@huggingface/hub/src/types/api/api-space.ts @@ -0,0 +1,93 @@ +import type { License, SpaceRuntime, SpaceSdk } from "../public"; + +type Color = "red" | "yellow" | "green" | "blue" | "indigo" | "purple" | "pink" | "gray"; + +export interface ApiSpaceInfo { + _id: string; + id: string; + arxivIds?: string[]; + author: string; + cardExists?: true; + cardError?: unknown; + cardData?: unknown; + contributors?: Array<{ user: string; _id: string }>; + disabled: boolean; + discussionsDisabled: boolean; + duplicationDisabled: boolean; + gated: false | "auto" | "manual"; + gitalyUid: string; + lastAuthor: { email: string; user?: string }; + lastModified: string; // date + likes: number; + likesRecent: number; + private: boolean; + updatedAt: string; // date + createdAt: string; // date + tags: string[]; + sha: string; + subdomain: string; + title: string; + emoji: string; + colorFrom: Color; + colorTo: Color; + pinned: boolean; + siblings: Array<{ rfilename: string }>; + sdk?: SpaceSdk; + runtime?: SpaceRuntime; + models?: string[]; + datasets?: string[]; + originSpace?: { _id: string; authorId: string }; +} + +export interface ApiSpaceMetadata { + license?: License | License[]; + tags?: string[]; + title?: string; + colorFrom?: "red" | "yellow" | "green" | "blue" | "indigo" | "purple" | "pink" | "gray"; + colorTo?: "red" | "yellow" | "green" | "blue" | "indigo" | "purple" | "pink" | "gray"; + emoji?: string; + sdk?: "streamlit" | "gradio" | "docker" | "static"; + sdk_version?: string | string; + python_version?: string | string; + fullWidth?: boolean; + header?: "mini" | "default"; + app_file?: string; + app_port?: number; + base_path?: string; + models?: string[]; + datasets?: string[]; + pinned?: boolean; + metaTitle?: string; + description?: string; + thumbnail?: string; + /** + * If enabled, will associate an oauth app to the Space, adding variables and secrets to the Space's environment + */ + hf_oauth?: boolean; + /** + * The expiration of access tokens for your oauth app in minutes. max 30 days (43,200 minutes). Defaults to 8 hours (480 minutes) + */ + hf_oauth_expiration_minutes?: number; + /** + * OAuth scopes to request. By default you have access to the user's profile, you can request access to their repos or inference-api. + */ + hf_oauth_scopes?: ("email" | "read-repos" | "write-repos" | "manage-repos" | "inference-api")[]; + suggested_hardware?: + | "cpu-basic" + | "zero-a10g" + | "cpu-upgrade" + | "cpu-xl" + | "t4-small" + | "t4-medium" + | "a10g-small" + | "a10g-large" + | "a10g-largex2" + | "a10g-largex4" + | "a100-large"; + suggested_storage?: "small" | "medium" | "large"; + custom_headers?: { + "cross-origin-embedder-policy"?: "unsafe-none" | "require-corp" | "credentialless"; + "cross-origin-opener-policy"?: "same-origin" | "same-origin-allow-popups" | "unsafe-none"; + "cross-origin-resource-policy"?: "same-site" | "same-origin" | "cross-origin"; + }; +} diff --git a/node_modules/@huggingface/hub/src/types/api/api-who-am-i.ts b/node_modules/@huggingface/hub/src/types/api/api-who-am-i.ts new file mode 100644 index 0000000000000000000000000000000000000000..5ece69e5a25d0b79ef5bd9c937a87c554f1c2156 --- /dev/null +++ b/node_modules/@huggingface/hub/src/types/api/api-who-am-i.ts @@ -0,0 +1,52 @@ +import type { AccessTokenRole, AuthType } from "../public"; + +interface ApiWhoAmIBase { + /** Unique ID persistent across renames */ + id: string; + type: "user" | "org" | "app"; + name: string; +} + +interface ApiWhoAmIEntityBase extends ApiWhoAmIBase { + fullname: string; + email: string | null; + canPay: boolean; + avatarUrl: string; + /** + * Unix timestamp in seconds + */ + periodEnd: number | null; +} + +interface ApiWhoAmIOrg extends ApiWhoAmIEntityBase { + type: "org"; +} + +interface ApiWhoAmIUser extends ApiWhoAmIEntityBase { + type: "user"; + email: string; + emailVerified: boolean; + isPro: boolean; + orgs: ApiWhoAmIOrg[]; + billingMode: "postpaid" | "prepaid"; +} + +interface ApiWhoAmIApp extends ApiWhoAmIBase { + type: "app"; + name: string; + scope?: { + entities: string[]; + role: AccessTokenRole; + }; +} + +export type ApiWhoAmIReponse = ApiWhoAmIUser | ApiWhoAmIOrg | ApiWhoAmIApp; + +export interface ApiWhoAmIAuthInfo { + type: AuthType; + accessToken?: { + displayName: string; + expiration?: string; + role: AccessTokenRole; + }; +} diff --git a/node_modules/@huggingface/hub/src/types/public.ts b/node_modules/@huggingface/hub/src/types/public.ts new file mode 100644 index 0000000000000000000000000000000000000000..8f03fda731d9ba708fa5ad1d9a52109a0aed2f06 --- /dev/null +++ b/node_modules/@huggingface/hub/src/types/public.ts @@ -0,0 +1,204 @@ +import type { PipelineType } from "@huggingface/tasks"; + +export type RepoType = "space" | "dataset" | "model" | "bucket" | "kernel"; + +export interface RepoId { + name: string; + type: RepoType; +} + +export type RepoFullName = + | string + | `spaces/${string}` + | `datasets/${string}` + | `buckets/${string}` + | `kernels/${string}`; + +export type RepoDesignation = RepoId | RepoFullName; + +/** + * A {@link RepoDesignation} narrowed to bucket repos. + * + * Used by APIs that only operate on buckets (e.g. {@link copyFile}, {@link copyFiles}, + * {@link copyFolder}). + */ +export type BucketDesignation = { type: "bucket"; name: string } | `buckets/${string}`; + +/** Actually `hf_${string}`, but for convenience, using the string type */ +export type AccessToken = string; + +/** + * @deprecated Use `AccessToken` instead. Pass { accessToken: "hf_..." } instead of { credentials: { accessToken: "hf_..." } } + */ +export interface Credentials { + accessToken: AccessToken; +} + +export type CredentialsParams = + | { + accessToken?: undefined; + /** + * @deprecated Use `accessToken` instead + */ + credentials: Credentials; + } + | { + accessToken: AccessToken; + /** + * @deprecated Use `accessToken` instead + */ + credentials?: undefined; + }; + +export type SpaceHardwareFlavor = + | "cpu-basic" + | "cpu-upgrade" + | "cpu-performance" + | "cpu-xl" + | "sprx8" + | "zero-a10g" + | "inf2x6" + | "t4-small" + | "t4-medium" + | "l4x1" + | "l4x4" + | "l40sx1" + | "l40sx4" + | "l40sx8" + | "a10g-small" + | "a10g-large" + | "a10g-largex2" + | "a10g-largex4" + | "a100-large" + | "a100x4" + | "a100x8"; + +export type SpaceSdk = "streamlit" | "gradio" | "docker" | "static"; + +export type SpaceStage = + | "NO_APP_FILE" + | "CONFIG_ERROR" + | "BUILDING" + | "BUILD_ERROR" + | "RUNNING" + | "RUNNING_BUILDING" + | "RUNTIME_ERROR" + | "DELETING" + | "PAUSED" + | "SLEEPING"; + +export type AccessTokenRole = "admin" | "write" | "contributor" | "read"; + +export type AuthType = "access_token" | "app_token" | "app_token_as_user"; + +export type { PipelineType }; + +export interface SpaceRuntime { + stage: SpaceStage; + sdk?: SpaceSdk; + sdkVersion?: string; + errorMessage?: string; + hardware?: { + current: SpaceHardwareFlavor | null; + currentPrettyName?: string; + requested: SpaceHardwareFlavor | null; + requestedPrettyName?: string; + }; + /** when calling /spaces, those props are only fetched if ?full=true */ + resources?: SpaceResourceConfig; + /** in seconds */ + gcTimeout?: number | null; +} + +export interface SpaceResourceRequirement { + cpu?: string; + memory?: string; + gpu?: string; + gpuModel?: string; + ephemeral?: string; +} + +export interface SpaceResourceConfig { + requests: SpaceResourceRequirement; + limits: SpaceResourceRequirement; + replicas?: number; + throttled?: boolean; + is_custom?: boolean; +} + +export type License = + | "apache-2.0" + | "mit" + | "openrail" + | "bigscience-openrail-m" + | "creativeml-openrail-m" + | "bigscience-bloom-rail-1.0" + | "bigcode-openrail-m" + | "afl-3.0" + | "artistic-2.0" + | "bsl-1.0" + | "bsd" + | "bsd-2-clause" + | "bsd-3-clause" + | "bsd-3-clause-clear" + | "c-uda" + | "cc" + | "cc0-1.0" + | "cc-by-2.0" + | "cc-by-2.5" + | "cc-by-3.0" + | "cc-by-4.0" + | "cc-by-sa-3.0" + | "cc-by-sa-4.0" + | "cc-by-nc-2.0" + | "cc-by-nc-3.0" + | "cc-by-nc-4.0" + | "cc-by-nd-4.0" + | "cc-by-nc-nd-3.0" + | "cc-by-nc-nd-4.0" + | "cc-by-nc-sa-2.0" + | "cc-by-nc-sa-3.0" + | "cc-by-nc-sa-4.0" + | "cdla-sharing-1.0" + | "cdla-permissive-1.0" + | "cdla-permissive-2.0" + | "wtfpl" + | "ecl-2.0" + | "epl-1.0" + | "epl-2.0" + | "etalab-2.0" + | "eupl-1.1" + | "agpl-3.0" + | "gfdl" + | "gpl" + | "gpl-2.0" + | "gpl-3.0" + | "lgpl" + | "lgpl-2.1" + | "lgpl-3.0" + | "isc" + | "lppl-1.3c" + | "ms-pl" + | "mpl-2.0" + | "odc-by" + | "odbl" + | "openrail++" + | "osl-3.0" + | "postgresql" + | "ofl-1.1" + | "ncsa" + | "unlicense" + | "zlib" + | "pddl" + | "lgpl-lr" + | "deepfloyd-if-license" + | "llama2" + | "llama3" + | "llama3.1" + | "llama3.2" + | "llama3.3" + | "gemma" + | "apple-ascl" + | "apple-amlr" + | "unknown" + | "other"; diff --git a/node_modules/@huggingface/hub/src/utils/ChunkCache.spec.ts b/node_modules/@huggingface/hub/src/utils/ChunkCache.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..f7901501e9479c81fc7c43d685a72eb41603ec11 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/ChunkCache.spec.ts @@ -0,0 +1,265 @@ +import { describe, it, expect } from "vitest"; +import { ChunkCache } from "./ChunkCache"; + +describe("ChunkCache", () => { + describe("basic operations", () => { + it("should create a cache with specified max size", () => { + const cache = new ChunkCache(5); + expect(cache.maxSize).toBe(5); + expect(cache.index).toBe(0); + }); + + it("should add and retrieve chunks", () => { + const cache = new ChunkCache(5); + + cache.addChunkToCache("hash1", 100, 10, null); + cache.addChunkToCache("hash2", 200, 20, null); + + const chunk1 = cache.getChunk("hash1", null); + const chunk2 = cache.getChunk("hash2", null); + + expect(chunk1).toEqual({ xorbIndex: 100, chunkIndex: 10 }); + expect(chunk2).toEqual({ xorbIndex: 200, chunkIndex: 20 }); + }); + + it("should return undefined for non-existent chunks", () => { + const cache = new ChunkCache(5); + + const chunk = cache.getChunk("nonexistent", null); + expect(chunk).toBeUndefined(); + }); + + it("should remove chunks from cache", () => { + const cache = new ChunkCache(5); + + cache.addChunkToCache("hash1", 100, 10, null); + expect(cache.getChunk("hash1", null)).toBeDefined(); + + cache.removeChunkFromCache("hash1"); + expect(cache.getChunk("hash1", null)).toBeUndefined(); + }); + }); + + describe("duplicate handling", () => { + it("should ignore duplicate hashes", () => { + const cache = new ChunkCache(5); + + // Add initial chunk + cache.addChunkToCache("hash1", 100, 10, null); + expect(cache.index).toBe(1); + expect(cache.map.size).toBe(1); + + // Try to add same hash again - should be ignored + cache.addChunkToCache("hash1", 999, 99, null); + expect(cache.index).toBe(1); // index should not increment + expect(cache.map.size).toBe(1); // map size should not increase + + // Original data should be preserved + const chunk = cache.getChunk("hash1", null); + expect(chunk).toEqual({ xorbIndex: 100, chunkIndex: 10 }); + }); + + it("should maintain consistency when adding duplicates mixed with new hashes", () => { + const cache = new ChunkCache(5); + + // Add some chunks + cache.addChunkToCache("hash1", 100, 10, null); + cache.addChunkToCache("hash2", 200, 20, null); + cache.addChunkToCache("hash3", 300, 30, null); + + expect(cache.index).toBe(3); + expect(cache.map.size).toBe(3); + + // Try to add duplicates + cache.addChunkToCache("hash1", 999, 99, null); // duplicate + cache.addChunkToCache("hash4", 400, 40, null); // new + cache.addChunkToCache("hash2", 888, 88, null); // duplicate + + expect(cache.index).toBe(4); // only incremented for hash4 + expect(cache.map.size).toBe(4); + + // Verify all chunks are accessible and have correct data + expect(cache.getChunk("hash1", null)).toEqual({ xorbIndex: 100, chunkIndex: 10 }); + expect(cache.getChunk("hash2", null)).toEqual({ xorbIndex: 200, chunkIndex: 20 }); + expect(cache.getChunk("hash3", null)).toEqual({ xorbIndex: 300, chunkIndex: 30 }); + expect(cache.getChunk("hash4", null)).toEqual({ xorbIndex: 400, chunkIndex: 40 }); + }); + }); + + describe("cache overflow and circular buffer behavior", () => { + it("should handle cache overflow correctly", () => { + const cache = new ChunkCache(3); // Small cache size + + // Fill the cache to capacity + cache.addChunkToCache("hash1", 100, 10, null); + cache.addChunkToCache("hash2", 200, 20, null); + cache.addChunkToCache("hash3", 300, 30, null); + + expect(cache.index).toBe(0); // wrapped around (3 % 3 = 0) + expect(cache.map.size).toBe(3); + + // All chunks should be accessible + expect(cache.getChunk("hash1", null)).toBeDefined(); + expect(cache.getChunk("hash2", null)).toBeDefined(); + expect(cache.getChunk("hash3", null)).toBeDefined(); + + // Add one more chunk - should evict the oldest (hash1) + cache.addChunkToCache("hash4", 400, 40, null); + + expect(cache.index).toBe(1); // wrapped around (4 % 3 = 1) + expect(cache.map.size).toBe(3); // size should remain the same + + // hash1 should be evicted, others should remain + expect(cache.getChunk("hash1", null)).toBeUndefined(); + expect(cache.getChunk("hash2", null)).toEqual({ xorbIndex: 200, chunkIndex: 20 }); + expect(cache.getChunk("hash3", null)).toEqual({ xorbIndex: 300, chunkIndex: 30 }); + expect(cache.getChunk("hash4", null)).toEqual({ xorbIndex: 400, chunkIndex: 40 }); + }); + + it("should continue evicting oldest entries as new ones are added", () => { + const cache = new ChunkCache(3); + + // Fill cache + cache.addChunkToCache("hash1", 100, 10, null); + cache.addChunkToCache("hash2", 200, 20, null); + cache.addChunkToCache("hash3", 300, 30, null); + + // Add more chunks to test multiple evictions + cache.addChunkToCache("hash4", 400, 40, null); // evicts hash1 + cache.addChunkToCache("hash5", 500, 50, null); // evicts hash2 + cache.addChunkToCache("hash6", 600, 60, null); // evicts hash3 + + expect(cache.map.size).toBe(3); + + // Only the last 3 should remain + expect(cache.getChunk("hash1", null)).toBeUndefined(); + expect(cache.getChunk("hash2", null)).toBeUndefined(); + expect(cache.getChunk("hash3", null)).toBeUndefined(); + expect(cache.getChunk("hash4", null)).toEqual({ xorbIndex: 400, chunkIndex: 40 }); + expect(cache.getChunk("hash5", null)).toEqual({ xorbIndex: 500, chunkIndex: 50 }); + expect(cache.getChunk("hash6", null)).toEqual({ xorbIndex: 600, chunkIndex: 60 }); + }); + + it("should handle removals during overflow correctly", () => { + const cache = new ChunkCache(3); + + // Fill cache + cache.addChunkToCache("hash1", 100, 10, null); + cache.addChunkToCache("hash2", 200, 20, null); + cache.addChunkToCache("hash3", 300, 30, null); + + // Remove middle element + cache.removeChunkFromCache("hash2"); + expect(cache.map.size).toBe(2); + + // Add new elements + cache.addChunkToCache("hash4", 400, 40, null); + cache.addChunkToCache("hash5", 500, 50, null); + + // The removal should not affect the eviction logic + expect(cache.getChunk("hash1", null)).toBeUndefined(); // evicted + expect(cache.getChunk("hash2", null)).toBeUndefined(); // removed + expect(cache.getChunk("hash3", null)).toEqual({ xorbIndex: 300, chunkIndex: 30 }); + expect(cache.getChunk("hash4", null)).toEqual({ xorbIndex: 400, chunkIndex: 40 }); + expect(cache.getChunk("hash5", null)).toEqual({ xorbIndex: 500, chunkIndex: 50 }); + }); + }); + + describe("consistency after operations", () => { + it("should maintain consistent state after mixed operations", () => { + const cache = new ChunkCache(4); + + // Add initial chunks + cache.addChunkToCache("a", 1, 10, null); + cache.addChunkToCache("b", 2, 20, null); + cache.addChunkToCache("c", 3, 30, null); + + // Mix of operations + cache.addChunkToCache("a", 999, 999, null); // duplicate (ignored) + cache.removeChunkFromCache("b"); // removal + cache.addChunkToCache("d", 4, 40, null); // new addition + cache.addChunkToCache("e", 5, 50, null); // new addition - this triggers overflow + cache.addChunkToCache("c", 888, 888, null); // duplicate (ignored) + + // Verify final state + // With cache size 4: a(0), b(1, removed), c(2), d(3), e(4 -> 0, wraps and evicts a) + expect(cache.getChunk("a", null)).toBeUndefined(); // evicted by e + expect(cache.getChunk("b", null)).toBeUndefined(); // removed + expect(cache.getChunk("c", null)).toEqual({ xorbIndex: 3, chunkIndex: 30 }); + expect(cache.getChunk("d", null)).toEqual({ xorbIndex: 4, chunkIndex: 40 }); + expect(cache.getChunk("e", null)).toEqual({ xorbIndex: 5, chunkIndex: 50 }); + + // Map size should be 3 (a evicted, b removed) + expect(cache.map.size).toBe(3); + }); + + it("should maintain consistency after cache overflow with mixed operations", () => { + const cache = new ChunkCache(3); + + // Fill cache + cache.addChunkToCache("first", 1, 1, null); + cache.addChunkToCache("second", 2, 2, null); + cache.addChunkToCache("third", 3, 3, null); + + // Cause overflow with duplicates and removals mixed in + cache.addChunkToCache("fourth", 4, 4, null); // evicts "first" + cache.addChunkToCache("second", 999, 999, null); // duplicate (ignored) + cache.removeChunkFromCache("third"); // removal + cache.addChunkToCache("fifth", 5, 5, null); // evicts "second" + + // Final state verification + expect(cache.getChunk("first", null)).toBeUndefined(); // evicted + expect(cache.getChunk("second", null)).toBeUndefined(); // evicted + expect(cache.getChunk("third", null)).toBeUndefined(); // removed + expect(cache.getChunk("fourth", null)).toEqual({ xorbIndex: 4, chunkIndex: 4 }); + expect(cache.getChunk("fifth", null)).toEqual({ xorbIndex: 5, chunkIndex: 5 }); + + cache.addChunkToCache("sixth", 6, 6, null); // new, takes "third" place + expect(cache.getChunk("sixth", null)).toEqual({ xorbIndex: 6, chunkIndex: 6 }); + expect(cache.getChunk("fourth", null)).toEqual({ xorbIndex: 4, chunkIndex: 4 }); + expect(cache.getChunk("fifth", null)).toEqual({ xorbIndex: 5, chunkIndex: 5 }); + + cache.addChunkToCache("seventh", 7, 7, null); // new, takes "fourth" place + expect(cache.getChunk("seventh", null)).toEqual({ xorbIndex: 7, chunkIndex: 7 }); + expect(cache.getChunk("fourth", null)).toBeUndefined(); // evicted + expect(cache.getChunk("fifth", null)).toEqual({ xorbIndex: 5, chunkIndex: 5 }); + + expect(cache.map.size).toBe(3); + }); + }); + + describe("negative xorbIndex handling", () => { + it("should handle negative xorbIndex values (remote xorbs)", () => { + const cache = new ChunkCache(5); + + cache.addChunkToCache("remote1", -1, 10, null); + cache.addChunkToCache("local1", 1, 20, null); + cache.addChunkToCache("remote2", -100, 30, null); + + expect(cache.getChunk("remote1", null)).toEqual({ xorbIndex: -1, chunkIndex: 10 }); + expect(cache.getChunk("local1", null)).toEqual({ xorbIndex: 1, chunkIndex: 20 }); + expect(cache.getChunk("remote2", null)).toEqual({ xorbIndex: -100, chunkIndex: 30 }); + }); + }); + + describe("edge cases", () => { + it("should handle cache with max size of 1", () => { + const cache = new ChunkCache(1); + + cache.addChunkToCache("hash1", 1, 1, null); + expect(cache.getChunk("hash1", null)).toBeDefined(); + + cache.addChunkToCache("hash2", 2, 2, null); + expect(cache.getChunk("hash1", null)).toBeUndefined(); // evicted + expect(cache.getChunk("hash2", null)).toEqual({ xorbIndex: 2, chunkIndex: 2 }); + }); + + it("should handle empty cache operations", () => { + const cache = new ChunkCache(5); + + expect(cache.getChunk("nonexistent", null)).toBeUndefined(); + cache.removeChunkFromCache("nonexistent"); // should not throw + expect(cache.map.size).toBe(0); + }); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/ChunkCache.ts b/node_modules/@huggingface/hub/src/utils/ChunkCache.ts new file mode 100644 index 0000000000000000000000000000000000000000..cb64defb06f77e1a9938c8215a91ac190858aeb2 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/ChunkCache.ts @@ -0,0 +1,99 @@ +const CHUNK_CACHE_INITIAL_SIZE = 10_000; +const CHUNK_CACHE_GROW_FACTOR = 1.5; +const CHUNK_CACHE_MAX_SIZE = 1_000_000; + +export class ChunkCache { + index = 0; + // Index >= 0 means local xorb, < 0 means remote xorb + xorbIndices: Int32Array; + // Max 8K chunks per xorb, less than 64K uint16_t + chunkIndices: Uint16Array; + map = new Map(); // hash -> chunkCacheIndex. Less overhead that way, empty object is 60+B and empty array is 40+B + hmacs = new Set(); // todo : remove old hmacs + maxSize: number; + + constructor(maxSize: number = CHUNK_CACHE_MAX_SIZE) { + if (maxSize < 1) { + throw new Error("maxSize must be at least 1"); + } + this.maxSize = maxSize; + this.xorbIndices = new Int32Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); + this.chunkIndices = new Uint16Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); + } + + addChunkToCache(hash: string, xorbIndex: number, chunkIndex: number, hmac: string | null): void { + if (this.map.has(hash)) { + // Happens when we receive an existing chunk from remote dedup info (eg duplicate chunk in shard? Or shards with same hmac key + // sharing chunks/xorbs) + + // processing this chunk again would desync the cache, as `this.map.size` would not increase, as opposed to `this.index` + + // We could readd/remove it to "refresh it" + return; + } + if (this.map.values().next().value === this.index) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + this.map.delete(this.map.keys().next().value!); + } + this.map.set(hash, this.index); + if (hmac !== null) { + this.hmacs.add(hmac); + } + + if (this.index >= this.xorbIndices.length) { + // todo: switch to resize() with modern browsers + const oldXorbIndices = this.xorbIndices; + const oldChunkIndices = this.chunkIndices; + this.xorbIndices = new Int32Array(Math.min(this.xorbIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); + this.chunkIndices = new Uint16Array(Math.min(this.chunkIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); + this.xorbIndices.set(oldXorbIndices); + this.chunkIndices.set(oldChunkIndices); + } + + this.xorbIndices[this.index] = xorbIndex; + this.chunkIndices[this.index] = chunkIndex; + this.index = (this.index + 1) % this.maxSize; + } + + getChunk( + hash: string, + /** + * Set to null if you only want to check against locally created chunks, or the hash is already a hmac + */ + hmacFunction: ((hash: string, key: string) => string) | null, + ): + | { + xorbIndex: number; + chunkIndex: number; + } + | undefined { + let index = this.map.get(hash); + if (index === undefined && hmacFunction !== null) { + for (const hmac of this.hmacs) { + index = this.map.get(hmacFunction(hash, hmac)); + if (index !== undefined) { + break; + } + } + } + if (index === undefined) { + return undefined; + } + return { + xorbIndex: this.xorbIndices[index], + chunkIndex: this.chunkIndices[index], + }; + } + + updateChunkIndex(hash: string, chunkIndex: number): void { + const index = this.map.get(hash); + if (index === undefined) { + throw new Error(`Chunk not found in cache: ${hash}`); + } + this.chunkIndices[index] = chunkIndex; + } + + removeChunkFromCache(hash: string): void { + this.map.delete(hash); + } +} diff --git a/node_modules/@huggingface/hub/src/utils/FileBlob.spec.ts b/node_modules/@huggingface/hub/src/utils/FileBlob.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..2ed51d8e38e817bc69d0924e3f4b47885e85c5d9 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/FileBlob.spec.ts @@ -0,0 +1,45 @@ +import { open, stat } from "node:fs/promises"; +import { TextDecoder } from "node:util"; +import { describe, expect, it } from "vitest"; +import { FileBlob } from "./FileBlob"; + +describe("FileBlob", () => { + it("should create a FileBlob with a slice on the entire file", async () => { + const file = await open("package.json", "r"); + const { size } = await stat("package.json"); + + const fileBlob = await FileBlob.create("package.json"); + + expect(fileBlob).toMatchObject({ + path: "package.json", + start: 0, + end: size, + }); + expect(fileBlob.size).toBe(size); + expect(fileBlob.type).toBe(""); + const text = await fileBlob.text(); + const expectedText = (await file.read(Buffer.alloc(size), 0, size)).buffer.toString("utf8"); + expect(text).toBe(expectedText); + const result = await fileBlob.stream().getReader().read(); + expect(new TextDecoder().decode(result.value)).toBe(expectedText); + }); + + it("should create a slice on the file", async () => { + const file = await open("package.json", "r"); + const fileBlob = await FileBlob.create("package.json"); + + const slice = fileBlob.slice(10, 20); + + expect(slice).toMatchObject({ + path: "package.json", + start: 10, + end: 20, + }); + expect(slice.size).toBe(10); + const sliceText = await slice.text(); + const expectedText = (await file.read(Buffer.alloc(10), 0, 10, 10)).buffer.toString("utf8"); + expect(sliceText).toBe(expectedText); + const result = await slice.stream().getReader().read(); + expect(new TextDecoder().decode(result.value)).toBe(expectedText); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/FileBlob.ts b/node_modules/@huggingface/hub/src/utils/FileBlob.ts new file mode 100644 index 0000000000000000000000000000000000000000..504efcf6ed7ae89a5b4ef957072377ebb8a10045 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/FileBlob.ts @@ -0,0 +1,122 @@ +import { createReadStream } from "node:fs"; +import { open, stat } from "node:fs/promises"; +import { Readable } from "node:stream"; +import type { FileHandle } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; + +/** + * @internal + * + * A FileBlob is a replacement for the Blob class that allows to lazy read files + * in order to preserve memory. + * + * It is a drop-in replacement for the Blob class, so you can use it as a Blob. + * + * The main difference is the instantiation, which is done asynchronously using the `FileBlob.create` method. + * + * @example + * const fileBlob = await FileBlob.create("path/to/package.json"); + * + * await fetch("https://aschen.tech", { method: "POST", body: fileBlob }); + */ +export class FileBlob extends Blob { + /** + * Creates a new FileBlob on the provided file. + * + * @param path Path to the file to be lazy readed + */ + static async create(path: string | URL): Promise { + path = path instanceof URL ? fileURLToPath(path) : path; + + const { size } = await stat(path); + + const fileBlob = new FileBlob(path, 0, size); + + return fileBlob; + } + + private path: string; + private start: number; + private end: number; + + private constructor(path: string, start: number, end: number) { + super(); + + this.path = path; + this.start = start; + this.end = end; + } + + /** + * Returns the size of the blob. + */ + override get size(): number { + return this.end - this.start; + } + + /** + * Returns a new instance of FileBlob that is a slice of the current one. + * + * The slice is inclusive of the start and exclusive of the end. + * + * The slice method does not supports negative start/end. + * + * @param start beginning of the slice + * @param end end of the slice + */ + override slice(start = 0, end = this.size): FileBlob { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on FileBlob.slice"); + } + + const slice = new FileBlob(this.path, this.start + start, Math.min(this.start + end, this.end)); + + return slice; + } + + /** + * Read the part of the file delimited by the FileBlob and returns it as an ArrayBuffer. + */ + override async arrayBuffer(): Promise { + const slice = await this.execute((file) => file.read(Buffer.alloc(this.size), 0, this.size, this.start)); + + return slice.buffer; + } + + /** + * Read the part of the file delimited by the FileBlob and returns it as a string. + */ + override async text(): Promise { + const buffer = (await this.arrayBuffer()) as Buffer; + + return buffer.toString("utf8"); + } + + /** + * Returns a stream around the part of the file delimited by the FileBlob. + */ + override stream(): ReturnType { + if (this.start === this.end) { + return new Blob([]).stream(); + } + + return Readable.toWeb(createReadStream(this.path, { start: this.start, end: this.end - 1 })) as ReturnType< + Blob["stream"] + >; + } + + /** + * We are opening and closing the file for each action to prevent file descriptor leaks. + * + * It is an intended choice of developer experience over performances. + */ + private async execute(action: (file: FileHandle) => Promise) { + const file = await open(this.path, "r"); + + try { + return await action(file); + } finally { + await file.close(); + } + } +} diff --git a/node_modules/@huggingface/hub/src/utils/RangeList.spec.ts b/node_modules/@huggingface/hub/src/utils/RangeList.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..e05f85a8fa14b41b2c4549738dd9b8d67e5b30f1 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/RangeList.spec.ts @@ -0,0 +1,96 @@ +import { describe, it, expect } from "vitest"; +import { RangeList } from "./RangeList"; + +describe("RangeList", () => { + it("should add a single range", () => { + const rangeList = new RangeList(); + rangeList.add(1, 100); + + const ranges = rangeList.getAllRanges(); + expect(ranges).toHaveLength(1); + expect(ranges[0]).toEqual({ + start: 1, + end: 100, + refCount: 1, + data: null, + }); + }); + + it("should handle overlapping ranges", () => { + const rangeList = new RangeList(); + rangeList.add(1, 100); + rangeList.add(30, 50); + + const ranges = rangeList.getAllRanges(); + expect(ranges).toHaveLength(3); + expect(ranges).toEqual([ + { start: 1, end: 30, refCount: 1, data: null }, + { start: 30, end: 50, refCount: 2, data: null }, + { start: 50, end: 100, refCount: 1, data: null }, + ]); + }); + + it("should remove a range at existing boundaries", () => { + const rangeList = new RangeList(); + rangeList.add(1, 100); + rangeList.add(30, 50); + rangeList.remove(30, 50); + + const ranges = rangeList.getAllRanges(); + expect(ranges).toHaveLength(3); + expect(ranges).toEqual([ + { start: 1, end: 30, refCount: 1, data: null }, + { start: 30, end: 50, refCount: 1, data: null }, + { start: 50, end: 100, refCount: 1, data: null }, + ]); + }); + + it("should throw error when removing range at non-existing boundaries", () => { + const rangeList = new RangeList(); + rangeList.add(1, 100); + rangeList.add(30, 50); + + expect(() => rangeList.remove(2, 50)).toThrow("Range boundaries must match existing boundaries"); + }); + + it("should get ranges within boundaries", () => { + const rangeList = new RangeList(); + rangeList.add(1, 100); + rangeList.add(30, 50); + + const ranges = rangeList.getRanges(30, 100); + expect(ranges).toHaveLength(2); + expect(ranges).toEqual([ + { start: 30, end: 50, refCount: 2, data: null }, + { start: 50, end: 100, refCount: 1, data: null }, + ]); + }); + + it("should throw error when end is less than or equal to start", () => { + const rangeList = new RangeList(); + + expect(() => rangeList.add(100, 1)).toThrow("End must be greater than start"); + expect(() => rangeList.add(1, 1)).toThrow("End must be greater than start"); + expect(() => rangeList.remove(100, 1)).toThrow("End must be greater than start"); + expect(() => rangeList.remove(1, 1)).toThrow("End must be greater than start"); + expect(() => rangeList.getRanges(100, 1)).toThrow("End must be greater than start"); + expect(() => rangeList.getRanges(1, 1)).toThrow("End must be greater than start"); + }); + + it("should handle multiple overlapping ranges", () => { + const rangeList = new RangeList(); + rangeList.add(1, 100); + rangeList.add(30, 50); + rangeList.add(40, 60); + + const ranges = rangeList.getAllRanges(); + expect(ranges).toHaveLength(5); + expect(ranges).toEqual([ + { start: 1, end: 30, refCount: 1, data: null }, + { start: 30, end: 40, refCount: 2, data: null }, + { start: 40, end: 50, refCount: 3, data: null }, + { start: 50, end: 60, refCount: 2, data: null }, + { start: 60, end: 100, refCount: 1, data: null }, + ]); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/RangeList.ts b/node_modules/@huggingface/hub/src/utils/RangeList.ts new file mode 100644 index 0000000000000000000000000000000000000000..9665500890ee1c34806168707a64e42af7ed8074 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/RangeList.ts @@ -0,0 +1,179 @@ +/** + * Code generated with this prompt by Cursor: + * + * I want to build a class to manage ranges + * + * I can add ranges to it with a start& an end (both integer, end > start). It should store those ranges efficiently. + * + * When several ranges overlap, eg [1, 100] and [30, 50], I want the class to split the range into non-overlapping ranges, and add a "ref counter" to the ranges. For example, [1, 30], [30, 50] * 2, [50, 100] + * + * I also want to be able to remove ranges, it will decrease the ref counter or remove the range altogether. I can only remove ranges at existing boundaries. For example, with the [1, 30], [30, 50] * 2, [50, 100] configuration + * + * - removing [1, 100] => the only range remaning is [30, 50] + * - removing [2, 50] => error, because "2' is not a boundary + * - removing [30, 50] => [1, 30], [30, 50], [50, 100] (do not "merge" the ranges back together) + * + * I want to be able to associate data to each range. And I want to be able to get the ranges inside boundaries. For example , with [1, 30], [30, 50] * 2, [50, 100] configuration + * + * - getting [30, 100] => I receive [30, 50] * 2, [50, 100], and I can get / modify the data associated to each range by accessing their data prop. Note the "*2" is just the ref counter, there is onlly one range object for the interval returned + * - getting [2, 50] => I get [30, 50] * 2 + * + * ---- + * + * Could optimize with binary search, but the ranges we want to handle are not that many. + */ +interface Range { + start: number; + end: number; + refCount: number; + data: T | null; +} + +export class RangeList { + private ranges: Range[] = []; + + /** + * Add a range to the list. If it overlaps with existing ranges, + * it will split them and increment reference counts accordingly. + */ + add(start: number, end: number): void { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + + // Find all ranges that overlap with the new range + const overlappingRanges: { index: number; range: Range }[] = []; + for (let i = 0; i < this.ranges.length; i++) { + const range = this.ranges[i]; + if (start < range.end && end > range.start) { + overlappingRanges.push({ index: i, range }); + } + if (range.data !== null) { + throw new Error("Overlapping range already has data"); + } + } + + if (overlappingRanges.length === 0) { + // No overlaps, just add the new range + this.ranges.push({ start, end, refCount: 1, data: null }); + this.ranges.sort((a, b) => a.start - b.start); + return; + } + + // Handle overlaps by splitting ranges + const newRanges: Range[] = []; + let currentPos = start; + + for (let i = 0; i < overlappingRanges.length; i++) { + const { range } = overlappingRanges[i]; + + // Add range before overlap if exists + if (currentPos < range.start) { + newRanges.push({ + start: currentPos, + end: range.start, + refCount: 1, + data: null, + }); + } else if (range.start < currentPos) { + newRanges.push({ + start: range.start, + end: currentPos, + refCount: range.refCount, + data: null, + }); + } + + // Add overlapping part with increased ref count + newRanges.push({ + start: Math.max(currentPos, range.start), + end: Math.min(end, range.end), + refCount: range.refCount + 1, + data: null, + }); + + // Add remaining part of existing range if exists + if (range.end > end) { + newRanges.push({ + start: end, + end: range.end, + refCount: range.refCount, + data: null, + }); + } + + currentPos = Math.max(currentPos, range.end); + } + + // Add remaining part after last overlap if exists + if (currentPos < end) { + newRanges.push({ + start: currentPos, + end, + refCount: 1, + data: null, + }); + } + + // Remove old overlapping ranges and insert new ones + const firstIndex = overlappingRanges[0].index; + const lastIndex = overlappingRanges[overlappingRanges.length - 1].index; + this.ranges.splice(firstIndex, lastIndex - firstIndex + 1, ...newRanges); + this.ranges.sort((a, b) => a.start - b.start); + } + + /** + * Remove a range from the list. The range must start and end at existing boundaries. + */ + remove(start: number, end: number): void { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + + // Find ranges that need to be modified + const affectedRanges: { index: number; range: Range }[] = []; + for (let i = 0; i < this.ranges.length; i++) { + const range = this.ranges[i]; + if (start < range.end && end > range.start) { + affectedRanges.push({ index: i, range }); + } + } + + if (affectedRanges.length === 0) { + throw new Error("No ranges found to remove"); + } + + // Verify boundaries match + if (start !== affectedRanges[0].range.start || end !== affectedRanges[affectedRanges.length - 1].range.end) { + throw new Error("Range boundaries must match existing boundaries"); + } + + // Todo: also check if there's a gap in the middle but it should not happen with our usage + + for (let i = 0; i < affectedRanges.length; i++) { + const { range } = affectedRanges[i]; + + range.refCount--; + } + + this.ranges = this.ranges.filter((range) => range.refCount > 0); + } + + /** + * Get all ranges within the specified boundaries. + */ + getRanges(start: number, end: number): Range[] { + if (end <= start) { + throw new TypeError("End must be greater than start"); + } + + return this.ranges.filter((range) => start < range.end && end > range.start); + } + + /** + * Get all ranges in the list + */ + getAllRanges(): Range[] { + return [...this.ranges]; + } +} diff --git a/node_modules/@huggingface/hub/src/utils/SplicedBlob.spec.ts b/node_modules/@huggingface/hub/src/utils/SplicedBlob.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..cf8520f15f2d9c7ae37a5df208b31c286da3b622 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/SplicedBlob.spec.ts @@ -0,0 +1,454 @@ +import { describe, expect, it, beforeEach } from "vitest"; +import { SplicedBlob } from "./SplicedBlob"; + +describe("SplicedBlob", () => { + let originalBlob: Blob; + let insertBlob: Blob; + let replaceBlob: Blob; + + beforeEach(() => { + // originalBlob: "0123456789" (10 chars) + originalBlob = new Blob(["0123456789"]); + // insertBlob: "ABC" (3 chars) - used in tests where we insert something into the blob + insertBlob = new Blob(["ABC"]); + // replaceBlob: "XY" (2 chars) - used in tests where part of the blob is replaced + replaceBlob = new Blob(["XY"]); + }); + + describe("create", () => { + it("should create a SplicedBlob with valid parameters", () => { + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + expect(splicedBlob).toBeInstanceOf(SplicedBlob); + }); + + it("should throw error for negative start", () => { + expect(() => SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: -1, end: 5 }])).toThrow( + "Invalid start/end positions for SplicedBlob", + ); + }); + + it("should throw error for negative end", () => { + expect(() => SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: -1 }])).toThrow( + "Invalid start/end positions for SplicedBlob", + ); + }); + + it("should throw error for start > original.size", () => { + expect(() => SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 15, end: 5 }])).toThrow( + "Invalid start/end positions for SplicedBlob", + ); + }); + + it("should throw error for end > original.size", () => { + expect(() => SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 15 }])).toThrow( + "Invalid start/end positions for SplicedBlob", + ); + }); + + it("should throw error for start > end", () => { + expect(() => SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 7, end: 5 }])).toThrow( + "Invalid start/end positions for SplicedBlob", + ); + }); + }); + + describe("size and type", () => { + it("should calculate size correctly for insertion", () => { + // Insert "ABC" at position 5: "01234ABC56789" = 13 chars + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + expect(splicedBlob.size).toBe(13); + }); + + it("should calculate size correctly for replacement", () => { + // Replace "345" with "XY": "012XY6789" = 9 chars + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: replaceBlob, start: 3, end: 6 }]); + expect(splicedBlob.size).toBe(9); + }); + + it("should return original blob type", () => { + const typedBlob = new Blob(["test"], { type: "text/plain" }); + const splicedBlob = SplicedBlob.create(typedBlob, [{ insert: insertBlob, start: 2, end: 2 }]); + expect(splicedBlob.type).toBe("text/plain"); + }); + }); + + describe("text method", () => { + it("should insert at beginning", async () => { + // Insert "ABC" at start: "ABC0123456789" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 0, end: 0 }]); + const text = await splicedBlob.text(); + expect(text).toBe("ABC0123456789"); + }); + + it("should insert at end", async () => { + // Insert "ABC" at end: "0123456789ABC" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 10, end: 10 }]); + const text = await splicedBlob.text(); + expect(text).toBe("0123456789ABC"); + }); + + it("should insert in middle", async () => { + // Insert "ABC" at position 5: "01234ABC56789" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const text = await splicedBlob.text(); + expect(text).toBe("01234ABC56789"); + }); + + it("should replace content", async () => { + // Replace "345" with "XY": "012XY6789" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: replaceBlob, start: 3, end: 6 }]); + const text = await splicedBlob.text(); + expect(text).toBe("012XY6789"); + }); + + it("should replace everything", async () => { + // Replace entire content with "ABC": "ABC" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 0, end: 10 }]); + const text = await splicedBlob.text(); + expect(text).toBe("ABC"); + }); + }); + + describe("slice method - basic cases", () => { + it("should return empty blob for start >= end", async () => { + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(5, 5); + expect(slice.size).toBe(0); + expect(await slice.text()).toBe(""); + }); + + it("should handle slice beyond size", async () => { + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(10, 20); + const text = await slice.text(); + expect(text).toBe("789"); // Only gets what's available + }); + + it("should throw error for negative start/end", () => { + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + expect(() => splicedBlob.slice(-1, 5)).toThrow("Unsupported negative start/end on SplicedBlob.slice"); + expect(() => splicedBlob.slice(0, -1)).toThrow("Unsupported negative start/end on SplicedBlob.slice"); + }); + }); + + describe("slice method - before segment only", () => { + it("should slice entirely in before segment", async () => { + // SplicedBlob: "01234ABC56789", slice(1, 4) = "123" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(1, 4); + const text = await slice.text(); + expect(text).toBe("123"); + }); + + it("should slice from start of before segment", async () => { + // SplicedBlob: "01234ABC56789", slice(0, 3) = "012" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(0, 3); + const text = await slice.text(); + expect(text).toBe("012"); + }); + + it("should slice to end of before segment", async () => { + // SplicedBlob: "01234ABC56789", slice(2, 5) = "234" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(2, 5); + const text = await slice.text(); + expect(text).toBe("234"); + }); + }); + + describe("slice method - insert segment only", () => { + it("should slice entirely in insert segment", async () => { + // SplicedBlob: "01234ABC56789", slice(6, 7) = "B" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(6, 7); + const text = await slice.text(); + expect(text).toBe("B"); + }); + + it("should slice entire insert segment", async () => { + // SplicedBlob: "01234ABC56789", slice(5, 8) = "ABC" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(5, 8); + const text = await slice.text(); + expect(text).toBe("ABC"); + }); + + it("should slice from start of insert segment", async () => { + // SplicedBlob: "01234ABC56789", slice(5, 7) = "AB" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(5, 7); + const text = await slice.text(); + expect(text).toBe("AB"); + }); + + it("should slice to end of insert segment", async () => { + // SplicedBlob: "01234ABC56789", slice(6, 8) = "BC" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(6, 8); + const text = await slice.text(); + expect(text).toBe("BC"); + }); + }); + + describe("slice method - after segment only", () => { + it("should slice entirely in after segment", async () => { + // SplicedBlob: "01234ABC56789", slice(9, 12) = "678" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(9, 12); + const text = await slice.text(); + expect(text).toBe("678"); + }); + + it("should slice from start of after segment", async () => { + // SplicedBlob: "01234ABC56789", slice(8, 11) = "567" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(8, 11); + const text = await slice.text(); + expect(text).toBe("567"); + }); + + it("should slice to end of after segment", async () => { + // SplicedBlob: "01234ABC56789", slice(10, 13) = "789" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(10, 13); + const text = await slice.text(); + expect(text).toBe("789"); + }); + }); + + describe("slice method - spanning before and insert", () => { + it("should slice spanning before and insert segments", async () => { + // SplicedBlob: "01234ABC56789", slice(3, 7) = "34AB" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(3, 7); + const text = await slice.text(); + expect(text).toBe("34AB"); + }); + + it("should slice from start spanning before and insert", async () => { + // SplicedBlob: "01234ABC56789", slice(0, 6) = "01234A" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(0, 6); + const text = await slice.text(); + expect(text).toBe("01234A"); + }); + + it("should slice to end of insert spanning before and insert", async () => { + // SplicedBlob: "01234ABC56789", slice(4, 8) = "4ABC" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(4, 8); + const text = await slice.text(); + expect(text).toBe("4ABC"); + }); + }); + + describe("slice method - spanning insert and after", () => { + it("should slice spanning insert and after segments", async () => { + // SplicedBlob: "01234ABC56789", slice(6, 10) = "BC56" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(6, 10); + const text = await slice.text(); + expect(text).toBe("BC56"); + }); + + it("should slice from start of insert to end", async () => { + // SplicedBlob: "01234ABC56789", slice(5, 13) = "ABC56789" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(5, 13); + const text = await slice.text(); + expect(text).toBe("ABC56789"); + }); + + it("should slice from middle of insert spanning to after", async () => { + // SplicedBlob: "01234ABC56789", slice(7, 11) = "C567" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(7, 11); + const text = await slice.text(); + expect(text).toBe("C567"); + }); + }); + + describe("slice method - spanning all three segments", () => { + it("should slice spanning all three segments", async () => { + // SplicedBlob: "01234ABC56789", slice(3, 10) = "34ABC56" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(3, 10); + const text = await slice.text(); + expect(text).toBe("34ABC56"); + }); + + it("should slice entire spliced blob", async () => { + // SplicedBlob: "01234ABC56789", slice(0, 13) = "01234ABC56789" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(0, 13); + const text = await slice.text(); + expect(text).toBe("01234ABC56789"); + }); + + it("should slice most of spliced blob", async () => { + // SplicedBlob: "01234ABC56789", slice(1, 12) = "1234ABC5678" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice = splicedBlob.slice(1, 12); + const text = await slice.text(); + expect(text).toBe("1234ABC5678"); + }); + }); + + describe("slice method - with replacement", () => { + it("should slice before replacement", async () => { + // Replace "345" with "XY": "012XY6789", slice(0, 3) = "012" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: replaceBlob, start: 3, end: 6 }]); + const slice = splicedBlob.slice(0, 3); + const text = await slice.text(); + expect(text).toBe("012"); + }); + + it("should slice replacement only", async () => { + // Replace "345" with "XY": "012XY6789", slice(3, 5) = "XY" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: replaceBlob, start: 3, end: 6 }]); + const slice = splicedBlob.slice(3, 5); + const text = await slice.text(); + expect(text).toBe("XY"); + }); + + it("should slice after replacement", async () => { + // Replace "345" with "XY": "012XY6789", slice(5, 9) = "6789" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: replaceBlob, start: 3, end: 6 }]); + const slice = splicedBlob.slice(5, 9); + const text = await slice.text(); + expect(text).toBe("6789"); + }); + + it("should slice spanning before and replacement", async () => { + // Replace "345" with "XY": "012XY6789", slice(1, 4) = "12X" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: replaceBlob, start: 3, end: 6 }]); + const slice = splicedBlob.slice(1, 4); + const text = await slice.text(); + expect(text).toBe("12X"); + }); + + it("should slice spanning replacement and after", async () => { + // Replace "345" with "XY": "012XY6789", slice(4, 7) = "Y67" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: replaceBlob, start: 3, end: 6 }]); + const slice = splicedBlob.slice(4, 7); + const text = await slice.text(); + expect(text).toBe("Y67"); + }); + + it("should slice spanning all segments with replacement", async () => { + // Replace "345" with "XY": "012XY6789", slice(1, 8) = "12XY678" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: replaceBlob, start: 3, end: 6 }]); + const slice = splicedBlob.slice(1, 8); + const text = await slice.text(); + expect(text).toBe("12XY678"); + }); + }); + + describe("slice method - edge cases", () => { + it("should handle empty insert blob", async () => { + const emptyBlob = new Blob([""]); + // Replace "345" with "": "0126789" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: emptyBlob, start: 3, end: 6 }]); + const text = await splicedBlob.text(); + expect(text).toBe("0126789"); + + const slice = splicedBlob.slice(2, 5); + expect(await slice.text()).toBe("267"); + }); + + it("should handle slice at segment boundaries", async () => { + // SplicedBlob: "01234ABC56789", exact boundary slices + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + + // End of before segment + expect(await splicedBlob.slice(4, 5).text()).toBe("4"); + // Start of insert segment + expect(await splicedBlob.slice(5, 6).text()).toBe("A"); + // End of insert segment + expect(await splicedBlob.slice(7, 8).text()).toBe("C"); + // Start of after segment + expect(await splicedBlob.slice(8, 9).text()).toBe("5"); + }); + + it("should handle insert at beginning with slice", async () => { + // Insert "ABC" at start: "ABC0123456789" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 0, end: 0 }]); + const slice = splicedBlob.slice(1, 5); + expect(await slice.text()).toBe("BC01"); + }); + + it("should handle insert at end with slice", async () => { + // Insert "ABC" at end: "0123456789ABC" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 10, end: 10 }]); + const slice = splicedBlob.slice(8, 12); + expect(await slice.text()).toBe("89AB"); + }); + }); + + describe("arrayBuffer method", () => { + it("should return correct ArrayBuffer for insertion", async () => { + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const buffer = await splicedBlob.arrayBuffer(); + const text = new TextDecoder().decode(buffer); + expect(text).toBe("01234ABC56789"); + expect(buffer.byteLength).toBe(13); + }); + + it("should return correct ArrayBuffer for replacement", async () => { + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: replaceBlob, start: 3, end: 6 }]); + const buffer = await splicedBlob.arrayBuffer(); + const text = new TextDecoder().decode(buffer); + expect(text).toBe("012XY6789"); + expect(buffer.byteLength).toBe(9); + }); + }); + + describe("stream method", () => { + it("should return correct stream for insertion", async () => { + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const stream = splicedBlob.stream(); + const response = new Response(stream); + const text = await response.text(); + expect(text).toBe("01234ABC56789"); + }); + + it("should return correct stream for replacement", async () => { + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: replaceBlob, start: 3, end: 6 }]); + const stream = splicedBlob.stream(); + const response = new Response(stream); + const text = await response.text(); + expect(text).toBe("012XY6789"); + }); + + it("should handle empty segments in stream", async () => { + const emptyBlob = new Blob([""]); + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: emptyBlob, start: 5, end: 5 }]); + const stream = splicedBlob.stream(); + const response = new Response(stream); + const text = await response.text(); + expect(text).toBe("0123456789"); + }); + }); + + describe("nested slicing", () => { + it("should allow slicing of sliced blob", async () => { + // SplicedBlob: "01234ABC56789", slice(3, 10) = "34ABC56", then slice(2, 5) = "ABC" + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const firstSlice = splicedBlob.slice(3, 10); + const secondSlice = firstSlice.slice(2, 5); + const text = await secondSlice.text(); + expect(text).toBe("ABC"); + }); + + it("should handle multiple levels of slicing", async () => { + // Complex slicing chain + const splicedBlob = SplicedBlob.create(originalBlob, [{ insert: insertBlob, start: 5, end: 5 }]); + const slice1 = splicedBlob.slice(2, 11); // "234ABC567" + const slice2 = slice1.slice(1, 7); // "34ABC5" + const slice3 = slice2.slice(2, 5); // "ABC" + const text = await slice3.text(); + expect(text).toBe("ABC"); + }); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/SplicedBlob.ts b/node_modules/@huggingface/hub/src/utils/SplicedBlob.ts new file mode 100644 index 0000000000000000000000000000000000000000..28877bcc47b7efb17024371d13ee1e04dd2e9e23 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/SplicedBlob.ts @@ -0,0 +1,246 @@ +import { sum } from "./sum"; + +/** + * Represents a single splice operation + */ +interface SpliceOperation { + insert: Blob; + start: number; + end: number; +} + +/** + * @internal + * + * A SplicedBlob is a Blob that represents the result of splicing one or more insert blobs + * into an original blob at specified positions, replacing content between start and end. + * + * It is a drop-in replacement for the Blob class, so you can use it as a Blob. + * The splicing is done virtually without copying data until accessed. + * + * @example + * const originalBlob = new Blob(["Hello, World!"]); + * const insertBlob = new Blob(["Beautiful "]); + * const splicedBlob = SplicedBlob.create(originalBlob, insertBlob, 7, 7); + * // Result represents: "Hello, Beautiful World!" + */ +export class SplicedBlob extends Blob { + public originalBlob: Blob; + public spliceOperations: SpliceOperation[]; + + private constructor(originalBlob: Blob, spliceOperations: SpliceOperation[]) { + super(); + + this.originalBlob = originalBlob; + this.spliceOperations = spliceOperations; // Create a copy to prevent external mutation + } + + static create(originalBlob: Blob, operations: SpliceOperation[]): SplicedBlob { + // Validate all operations + for (const op of operations) { + if (op.start < 0 || op.end < 0) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + if (op.start > originalBlob.size || op.end > originalBlob.size) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + if (op.start > op.end) { + throw new Error("Invalid start/end positions for SplicedBlob"); + } + } + + // Sort operations by start position and validate no overlaps + const sortedOps = [...operations].sort((a, b) => a.start - b.start); + for (let i = 0; i < sortedOps.length - 1; i++) { + if (sortedOps[i].end > sortedOps[i + 1].start) { + throw new Error("Overlapping splice operations are not supported"); + } + } + + return new SplicedBlob(originalBlob, sortedOps); + } + + /** + * Returns the size of the spliced blob. + * Size = original size - total replaced size + total insert size + */ + override get size(): number { + let totalReplacedSize = 0; + let totalInsertSize = 0; + + for (const op of this.spliceOperations) { + totalReplacedSize += op.end - op.start; + totalInsertSize += op.insert.size; + } + + return this.originalBlob.size - totalReplacedSize + totalInsertSize; + } + + /** + * Returns the MIME type of the original blob. + */ + override get type(): string { + return this.originalBlob.type; + } + + /** + * Returns a new instance of SplicedBlob that is a slice of the current one. + * + * The slice is inclusive of the start and exclusive of the end. + * The slice method does not support negative start/end. + * + * @param start beginning of the slice + * @param end end of the slice + */ + override slice(start = 0, end = this.size): Blob { + if (start < 0 || end < 0) { + throw new TypeError("Unsupported negative start/end on SplicedBlob.slice"); + } + + start = Math.min(start, this.size); + end = Math.min(end, this.size); + + if (start >= end) { + return new Blob([]); + } + + // Get all segments and calculate their cumulative positions + const segments = this.segments; + const segmentBoundaries: number[] = [0]; + let cumulativeSize = 0; + + for (const segment of segments) { + cumulativeSize += segment.size; + segmentBoundaries.push(cumulativeSize); + } + + // Find which segments the slice spans + const resultSegments: Blob[] = []; + + for (let i = 0; i < segments.length; i++) { + const segmentStart = segmentBoundaries[i]; + const segmentEnd = segmentBoundaries[i + 1]; + + // Skip segments that are entirely before the slice + if (segmentEnd <= start) { + continue; + } + + // Skip segments that are entirely after the slice + if (segmentStart >= end) { + break; + } + + // Calculate slice bounds within this segment + const sliceStart = Math.max(0, start - segmentStart); + const sliceEnd = Math.min(segments[i].size, end - segmentStart); + + if (sliceStart < sliceEnd) { + resultSegments.push(segments[i].slice(sliceStart, sliceEnd)); + } + } + + return new Blob(resultSegments); + } + + get firstSpliceIndex(): number { + return this.spliceOperations[0]?.start ?? Infinity; + } + + /** + * Read the spliced blob content and returns it as an ArrayBuffer. + */ + override async arrayBuffer(): Promise { + const segments = this.segments; + const buffers = await Promise.all(segments.map((segment) => segment.arrayBuffer())); + + // Concatenate all buffers + const totalSize = sum(buffers.map((buffer) => buffer.byteLength)); + const result = new Uint8Array(totalSize); + + let offset = 0; + for (const buffer of buffers) { + result.set(new Uint8Array(buffer), offset); + offset += buffer.byteLength; + } + + return result.buffer; + } + + /** + * Read the spliced blob content and returns it as a string. + */ + override async text(): Promise { + const buffer = await this.arrayBuffer(); + return new TextDecoder().decode(buffer); + } + + /** + * Returns a stream around the spliced blob content. + */ + override stream(): ReturnType { + const readable = new ReadableStream({ + start: async (controller) => { + try { + const segments = this.segments; + + for (const segment of segments) { + const reader = segment.stream().getReader(); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + controller.enqueue(value); + } + } finally { + reader.releaseLock(); + } + } + + controller.close(); + } catch (error) { + controller.error(error); + } + }, + }); + + return readable; + } + + /** + * Get all segments that make up the spliced blob. + * This includes original blob segments between splice operations and insert blobs. + */ + private get segments(): Blob[] { + const segments: Blob[] = []; + let currentPosition = 0; + + // Sort operations by start position to ensure correct order + const sortedOps = [...this.spliceOperations].sort((a, b) => a.start - b.start); + + for (const op of sortedOps) { + // Add segment from current position to start of this operation + if (currentPosition < op.start) { + segments.push(this.originalBlob.slice(currentPosition, op.start)); + } + + // Add the insert blob (if it has content) + if (op.insert.size > 0) { + segments.push(op.insert); + } + + // Move current position to end of this operation + currentPosition = op.end; + } + + // Add remaining segment after last operation + if (currentPosition < this.originalBlob.size) { + segments.push(this.originalBlob.slice(currentPosition)); + } + + return segments; + } +} diff --git a/node_modules/@huggingface/hub/src/utils/WebBlob.spec.ts b/node_modules/@huggingface/hub/src/utils/WebBlob.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..853076b04088fd6ead29f60ec3d64e57a33fb2c7 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/WebBlob.spec.ts @@ -0,0 +1,99 @@ +import { describe, expect, it, beforeAll } from "vitest"; +import { WebBlob } from "./WebBlob"; + +describe("WebBlob", () => { + const resourceUrl = new URL("https://huggingface.co/spaces/aschen/push-model-from-web/raw/main/mobilenet/model.json"); + let fullText: string; + let size: number; + let contentType: string; + + beforeAll(async () => { + // Compute the reference size from the response body itself; in browsers + // `Content-Length` is not reliably exposed when the response is gzipped + // on the fly by CloudFront. + const response = await fetch(resourceUrl); + const blob = await response.blob(); + size = blob.size; + fullText = await blob.text(); + contentType = response.headers.get("content-type") || ""; + }); + + it("should create a WebBlob with a slice on the entire resource", async () => { + const webBlob = await WebBlob.create(resourceUrl, { cacheBelow: 0, accessToken: undefined }); + + expect(webBlob).toMatchObject({ + url: resourceUrl, + start: 0, + end: size, + contentType, + }); + expect(webBlob).toBeInstanceOf(WebBlob); + expect(webBlob.size).toBe(size); + expect(webBlob.type).toBe(contentType); + + const text = await webBlob.text(); + expect(text).toBe(fullText); + + const streamText = await new Response(webBlob.stream()).text(); + expect(streamText).toBe(fullText); + }); + + it("should create a WebBlob with a slice on the entire resource, cached", async () => { + const webBlob = await WebBlob.create(resourceUrl, { cacheBelow: 1_000_000, accessToken: undefined }); + + expect(webBlob).not.toBeInstanceOf(WebBlob); + expect(webBlob.size).toBe(size); + expect(webBlob.type.replace(/;\s*charset=utf-8/, "")).toBe(contentType.replace(/;\s*charset=utf-8/, "")); + + const text = await webBlob.text(); + expect(text).toBe(fullText); + + const streamText = await new Response(webBlob.stream()).text(); + expect(streamText).toBe(fullText); + }); + + it("should lazy load a LFS file hosted on Hugging Face", async () => { + const zephyrUrl = + "https://huggingface.co/HuggingFaceH4/zephyr-7b-alpha/resolve/main/model-00001-of-00008.safetensors"; + const url = new URL(zephyrUrl); + const webBlob = await WebBlob.create(url); + + expect(webBlob.size).toBe(1_889_587_040); + expect(webBlob).toBeInstanceOf(WebBlob); + expect(webBlob).toMatchObject({ url }); + expect(await webBlob.slice(10, 22).text()).toBe("__metadata__"); + }); + + it("should lazy load a Xet file hosted on Hugging Face", async () => { + const stableDiffusionUrl = + "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/unet/diffusion_pytorch_model.fp16.safetensors"; + const url = new URL(stableDiffusionUrl); + const webBlob = await WebBlob.create(url); + + expect(webBlob.size).toBe(5_135_149_760); + expect(webBlob).toBeInstanceOf(WebBlob); + expect(webBlob).toMatchObject({ url }); + expect(await webBlob.slice(10, 22).text()).toBe("__metadata__"); + }); + + it("should create a slice on the file", async () => { + const expectedText = fullText.slice(10, 20); + + const slice = (await WebBlob.create(resourceUrl, { cacheBelow: 0, accessToken: undefined })).slice(10, 20); + + expect(slice).toMatchObject({ + url: resourceUrl, + start: 10, + end: 20, + contentType, + }); + expect(slice.size).toBe(10); + expect(slice.type).toBe(contentType); + + const sliceText = await slice.text(); + expect(sliceText).toBe(expectedText); + + const streamText = await new Response(slice.stream()).text(); + expect(streamText).toBe(expectedText); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/WebBlob.ts b/node_modules/@huggingface/hub/src/utils/WebBlob.ts new file mode 100644 index 0000000000000000000000000000000000000000..0c4e6da35d8b55a30bf0e81de5eb198511c138f7 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/WebBlob.ts @@ -0,0 +1,170 @@ +/** + * WebBlob is a Blob implementation for web resources that supports range requests. + */ + +import { createApiError } from "../error"; + +interface WebBlobCreateOptions { + /** + * @default 1_000_000 + * + * Objects below that size will immediately be fetched and put in RAM, rather + * than streamed ad-hoc + */ + cacheBelow?: number; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + accessToken: string | undefined; +} + +export class WebBlob extends Blob { + static async create(url: URL, opts?: WebBlobCreateOptions): Promise { + const customFetch = opts?.fetch ?? fetch; + + // Probe with `Range: bytes=0-0` rather than `HEAD` to learn the file size + // and confirm range support in a single round trip. + // + // In browsers, when CloudFront gzips a response on the fly (typical for + // small text/JSON files behind `/api/resolve-cache/...`), the cached + // response loses both `Content-Length` and `Accept-Ranges`. Subsequent + // HEAD requests served from that cache inherit the missing headers, so + // the lib could not tell either the file size or whether ranges were + // supported, and silently fell back to buffering the whole blob in RAM. + // + // Range responses are never content-encoded, so `Content-Range` + // (carrying the total size) and the strong `ETag` always survive, + // regardless of the cached encoding state. + const probe = await customFetch(url, { + headers: { + Range: "bytes=0-0", + ...(opts?.accessToken && { Authorization: `Bearer ${opts.accessToken}` }), + }, + }); + + if (!probe.ok) { + throw await createApiError(probe); + } + + const contentType = probe.headers.get("content-type") || ""; + + // 206 → server honored the range request; total size is in `Content-Range`. + if (probe.status === 206) { + const totalSize = Number(probe.headers.get("content-range")?.split("/").pop()); + await probe.body?.cancel(); + + if (Number.isFinite(totalSize) && totalSize >= (opts?.cacheBelow ?? 1_000_000)) { + return new WebBlob(url, 0, totalSize, contentType, true, customFetch, opts?.accessToken); + } + + // Small file (or unknown total) → buffer it in RAM. + const full = await customFetch(url, { + ...(opts?.accessToken && { headers: { Authorization: `Bearer ${opts.accessToken}` } }), + }); + if (!full.ok) { + throw await createApiError(full); + } + return full.blob(); + } + + // 200 → server ignored `Range`; we've already started downloading the + // full body, so just consume it. + return probe.blob(); + } + + private url: URL; + private start: number; + private end: number; + private contentType: string; + private full: boolean; + private fetch: typeof fetch; + private accessToken: string | undefined; + + constructor( + url: URL, + start: number, + end: number, + contentType: string, + full: boolean, + customFetch: typeof fetch, + accessToken: string | undefined, + ) { + super([]); + + this.url = url; + this.start = start; + this.end = end; + this.contentType = contentType; + this.full = full; + this.fetch = customFetch; + this.accessToken = accessToken; + } + + override get size(): number { + return this.end - this.start; + } + + override get type(): string { + return this.contentType; + } + + override slice(start = 0, end = this.size): WebBlob { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on WebBlob.slice"); + } + + const slice = new WebBlob( + this.url, + this.start + start, + Math.min(this.start + end, this.end), + this.contentType, + start === 0 && end === this.size ? this.full : false, + this.fetch, + this.accessToken, + ); + + return slice; + } + + override async arrayBuffer(): Promise { + const result = await this.fetchRange(); + + return result.arrayBuffer(); + } + + override async text(): Promise { + const result = await this.fetchRange(); + + return result.text(); + } + + override stream(): ReturnType { + const stream = new TransformStream(); + + this.fetchRange() + .then((response) => response.body?.pipeThrough(stream)) + .catch((error) => stream.writable.abort(error.message)); + + return stream.readable; + } + + private fetchRange(): Promise { + const fetch = this.fetch; // to avoid this.fetch() which is bound to the instance instead of globalThis + if (this.full) { + return fetch(this.url, { + ...(this.accessToken && { + headers: { + Authorization: `Bearer ${this.accessToken}`, + }, + }), + }).then((resp) => (resp.ok ? resp : createApiError(resp))); + } + return fetch(this.url, { + headers: { + Range: `bytes=${this.start}-${this.end - 1}`, + ...(this.accessToken && { Authorization: `Bearer ${this.accessToken}` }), + }, + }).then((resp) => (resp.ok ? resp : createApiError(resp))); + } +} diff --git a/node_modules/@huggingface/hub/src/utils/XetBlob.spec.ts b/node_modules/@huggingface/hub/src/utils/XetBlob.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..cc3f5ce4c3d6f93c507185349433cc1b074dab42 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/XetBlob.spec.ts @@ -0,0 +1,953 @@ +import { describe, expect, it } from "vitest"; +import type { ReconstructionInfo } from "./XetBlob"; +import { bg4_regroup_bytes, bg4_split_bytes, XetBlob } from "./XetBlob"; +import { sum } from "./sum"; + +describe("XetBlob", () => { + it("should handle empty files (size 0) without making network requests", async () => { + let fetchCount = 0; + + const blob = new XetBlob({ + hash: "test", + size: 0, + refreshUrl: "https://huggingface.co", + fetch: async () => { + fetchCount++; + return new Response(); + }, + }); + + const text = await blob.text(); + expect(text).toBe(""); + expect(fetchCount).toBe(0); + + const arrayBuffer = await blob.arrayBuffer(); + expect(arrayBuffer.byteLength).toBe(0); + expect(fetchCount).toBe(0); + + const stream = blob.stream(); + const reader = stream.getReader(); + const result = await reader.read(); + expect(result.done).toBe(true); + expect(result.value).toBeUndefined(); + expect(fetchCount).toBe(0); + }); + + it("should lazy load the first 22 bytes", async () => { + const blob = new XetBlob({ + hash: "7b3b6d07673a88cf467e67c1f7edef1a8c268cbf66e9dd9b0366322d4ab56d9b", + size: 5_234_139_343, + refreshUrl: "https://huggingface.co/api/models/celinah/xet-experiments/xet-read-token/main", + }); + + expect(await blob.slice(10, 22).text()).toBe("__metadata__"); + }); + + it("should load the first chunk correctly", async () => { + let xorbCount = 0; + const blob = new XetBlob({ + refreshUrl: "https://huggingface.co/api/models/celinah/xet-experiments/xet-read-token/main", + hash: "7b3b6d07673a88cf467e67c1f7edef1a8c268cbf66e9dd9b0366322d4ab56d9b", + size: 5_234_139_343, + fetch: async (url, opts) => { + if (typeof url === "string" && url.includes("/xorbs/")) { + xorbCount++; + } + return fetch(url, opts); + }, + }); + + const xetDownload = await blob.slice(0, 29928).arrayBuffer(); + const bridgeDownload = await fetch( + "https://huggingface.co/celinah/xet-experiments/resolve/main/model5GB.safetensors", + { + headers: { + Range: "bytes=0-29927", + }, + }, + ).then((res) => res.arrayBuffer()); + + expect(new Uint8Array(xetDownload)).toEqual(new Uint8Array(bridgeDownload)); + expect(xorbCount).toBe(1); + }); + + it("should load just past the first chunk correctly", async () => { + let xorbCount = 0; + const blob = new XetBlob({ + refreshUrl: "https://huggingface.co/api/models/celinah/xet-experiments/xet-read-token/main", + hash: "7b3b6d07673a88cf467e67c1f7edef1a8c268cbf66e9dd9b0366322d4ab56d9b", + size: 5_234_139_343, + fetch: async (url, opts) => { + if (typeof url === "string" && url.includes("/xorbs/")) { + xorbCount++; + } + return fetch(url, opts); + }, + }); + + const xetDownload = await blob.slice(0, 29929).arrayBuffer(); + const bridgeDownload = await fetch( + "https://huggingface.co/celinah/xet-experiments/resolve/main/model5GB.safetensors", + { + headers: { + Range: "bytes=0-29928", + }, + }, + ).then((res) => res.arrayBuffer()); + + expect(xetDownload.byteLength).toBe(29929); + expect(new Uint8Array(xetDownload)).toEqual(new Uint8Array(bridgeDownload)); + expect(xorbCount).toBe(2); + }); + + it("should load the first 200kB correctly", async () => { + let xorbCount = 0; + const blob = new XetBlob({ + refreshUrl: "https://huggingface.co/api/models/celinah/xet-experiments/xet-read-token/main", + hash: "7b3b6d07673a88cf467e67c1f7edef1a8c268cbf66e9dd9b0366322d4ab56d9b", + size: 5_234_139_343, + fetch: async (url, opts) => { + if (typeof url === "string" && url.includes("/xorbs/")) { + xorbCount++; + } + return fetch(url, opts); + }, + // internalLogging: true, + }); + + const xetDownload = await blob.slice(0, 200_000).arrayBuffer(); + const bridgeDownload = await fetch( + "https://huggingface.co/celinah/xet-experiments/resolve/main/model5GB.safetensors", + { + headers: { + Range: "bytes=0-199999", + }, + }, + ).then((res) => res.arrayBuffer()); + + expect(xetDownload.byteLength).toBe(200_000); + expect(new Uint8Array(xetDownload)).toEqual(new Uint8Array(bridgeDownload)); + expect(xorbCount).toBe(2); + }, 60_000); + + it("should load correctly when loading far into a chunk range", async () => { + const blob = new XetBlob({ + refreshUrl: "https://huggingface.co/api/models/celinah/xet-experiments/xet-read-token/main", + hash: "7b3b6d07673a88cf467e67c1f7edef1a8c268cbf66e9dd9b0366322d4ab56d9b", + size: 5_234_139_343, + // internalLogging: true, + }); + + const xetDownload = await blob.slice(10_000_000, 10_100_000).arrayBuffer(); + const bridgeDownload = await fetch( + "https://huggingface.co/celinah/xet-experiments/resolve/main/model5GB.safetensors", + { + headers: { + Range: "bytes=10000000-10099999", + }, + }, + ).then((res) => res.arrayBuffer()); + + console.log("xet", xetDownload.byteLength, "bridge", bridgeDownload.byteLength); + expect(new Uint8Array(xetDownload).length).toEqual(100_000); + expect(new Uint8Array(xetDownload)).toEqual(new Uint8Array(bridgeDownload)); + }); + + it("should load text correctly when offset_into_range starts in a chunk further than the first", async () => { + const blob = new XetBlob({ + refreshUrl: "https://huggingface.co/api/models/celinah/xet-experiments/xet-read-token/main", + hash: "794efea76d8cb372bbe1385d9e51c3384555f3281e629903ecb6abeff7d54eec", + size: 62_914_580, + }); + + // Reconstruction info + // { + // "offset_into_first_range": 600000, + // "terms": + // [ + // { + // "hash": "be748f77930d5929cabd510a15f2c30f2f460b639804ef79dea46affa04fd8b2", + // "unpacked_length": 655360, + // "range": { "start": 0, "end": 5 }, + // }, + // { + // "hash": "be748f77930d5929cabd510a15f2c30f2f460b639804ef79dea46affa04fd8b2", + // "unpacked_length": 655360, + // "range": { "start": 0, "end": 5 }, + // }, + // ], + // "fetch_info": + // { + // "be748f77930d5929cabd510a15f2c30f2f460b639804ef79dea46affa04fd8b2": + // [ + // { + // "range": { "start": 0, "end": 5 }, + // "url": "...", + // "url_range": { "start": 0, "end": 2839 }, + // }, + // ], + // }, + // } + + const text = await blob.slice(600_000, 700_000).text(); + const bridgeDownload = await fetch("https://huggingface.co/celinah/xet-experiments/resolve/main/large_text.txt", { + headers: { + Range: "bytes=600000-699999", + }, + }).then((res) => res.text()); + + console.log("xet", text.length, "bridge", bridgeDownload.length); + expect(text.length).toBe(bridgeDownload.length); + }); + + describe("bg4_regoup_bytes", () => { + it("should regroup bytes when the array is %4 length", () => { + expect(bg4_regroup_bytes(new Uint8Array([1, 5, 2, 6, 3, 7, 4, 8]))).toEqual( + new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]), + ); + }); + + it("should regroup bytes when the array is %4 + 1 length", () => { + expect(bg4_regroup_bytes(new Uint8Array([1, 5, 9, 2, 6, 3, 7, 4, 8]))).toEqual( + new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9]), + ); + }); + + it("should regroup bytes when the array is %4 + 2 length", () => { + expect(bg4_regroup_bytes(new Uint8Array([1, 5, 9, 2, 6, 10, 3, 7, 4, 8]))).toEqual( + new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), + ); + }); + + it("should regroup bytes when the array is %4 + 3 length", () => { + expect(bg4_regroup_bytes(new Uint8Array([1, 5, 9, 2, 6, 10, 3, 7, 11, 4, 8]))).toEqual( + new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]), + ); + }); + }); + + describe("bg4_split_bytes", () => { + it("should split bytes when the array is %4 length", () => { + expect(bg4_split_bytes(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]))).toEqual( + new Uint8Array([1, 5, 2, 6, 3, 7, 4, 8]), + ); + }); + + it("should split bytes when the array is %4 + 1 length", () => { + expect(bg4_split_bytes(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9]))).toEqual( + new Uint8Array([1, 5, 9, 2, 6, 3, 7, 4, 8]), + ); + }); + + it("should split bytes when the array is %4 + 2 length", () => { + expect(bg4_split_bytes(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))).toEqual( + new Uint8Array([1, 5, 9, 2, 6, 10, 3, 7, 4, 8]), + ); + }); + + it("should split bytes when the array is %4 + 3 length", () => { + expect(bg4_split_bytes(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]))).toEqual( + new Uint8Array([1, 5, 9, 2, 6, 10, 3, 7, 11, 4, 8]), + ); + }); + + it("should be the inverse of bg4_regroup_bytes", () => { + const testArrays = [ + new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]), + new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9]), + new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), + new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]), + new Uint8Array([42]), + new Uint8Array([1, 2]), + new Uint8Array([1, 2, 3]), + ]; + + testArrays.forEach((arr) => { + expect(bg4_regroup_bytes(bg4_split_bytes(arr))).toEqual(arr); + }); + }); + }); + + describe("when mocked", () => { + describe("loading many chunks every read", () => { + it("should load different slices", async () => { + const chunk1Content = "hello"; + const chunk2Content = "world!"; + const debugged: Array<{ event: "read" | string } & Record> = []; + + const chunks = Array(1000) + .fill(0) + .flatMap(() => [makeChunk(chunk1Content), makeChunk(chunk2Content)]); + + const mergedChunks = await new Blob(chunks).arrayBuffer(); + const wholeText = (chunk1Content + chunk2Content).repeat(1000); + + const totalSize = wholeText.length; + let fetchCount = 0; + + const blob = new XetBlob({ + hash: "test", + size: totalSize, + refreshUrl: "https://huggingface.co", + listener: (e) => debugged.push(e), + fetch: async function (_url, opts) { + const url = new URL(_url as string); + const headers = opts?.headers as Record | undefined; + + switch (url.hostname) { + case "huggingface.co": { + // This is a token + return new Response( + JSON.stringify({ + casUrl: "https://cas.co", + accessToken: "boo", + exp: 1_000_000, + }), + ); + } + case "cas.co": { + // This is the reconstruction info + const range = headers?.["Range"]?.slice("bytes=".length).split("-").map(Number); + + const start = range?.[0] ?? 0; + // const end = range?.[1] ?? (totalSize - 1); + + return new Response( + JSON.stringify({ + terms: Array(1000) + .fill(0) + .map(() => ({ + hash: "test", + range: { + start: 0, + end: 2, + }, + unpacked_length: chunk1Content.length + chunk2Content.length, + })), + fetch_info: { + test: [ + { + url: "https://fetch.co", + range: { start: 0, end: 2 }, + url_range: { + start: 0, + end: mergedChunks.byteLength / 1000 - 1, + }, + }, + ], + }, + offset_into_first_range: start, + } satisfies ReconstructionInfo), + ); + } + case "fetch.co": { + fetchCount++; + return new Response( + new ReadableStream({ + pull(controller) { + controller.enqueue(new Uint8Array(mergedChunks)); + controller.close(); + }, + }), + //mergedChunks + ); + } + default: + throw new Error("Unhandled URL"); + } + }, + }); + + const startIndexes = [0, 5, 11, 6, 12, 100, 2000, totalSize - 12, totalSize - 2]; + + for (const index of startIndexes) { + console.log("slice", index); + const content = await blob.slice(index).text(); + expect(content.length).toBe(wholeText.length - index); + expect(content.slice(0, 1000)).toEqual(wholeText.slice(index).slice(0, 1000)); + expect(debugged.filter((e) => e.event === "read").length).toBe(2); // 1 read + 1 undefined + expect(fetchCount).toEqual(1); + + fetchCount = 0; + debugged.length = 0; + } + }); + + it("should load different slices when working with different XORBS", async () => { + const chunk1Content = "hello"; + const chunk2Content = "world!"; + const debugged: Array<{ event: "read" | string } & Record> = []; + + const chunks = Array(1000) + .fill(0) + .flatMap(() => [makeChunk(chunk1Content), makeChunk(chunk2Content)]); + + const mergedChunks = await new Blob(chunks).arrayBuffer(); + const wholeText = (chunk1Content + chunk2Content).repeat(1000); + + const totalSize = wholeText.length; + let fetchCount = 0; + + const blob = new XetBlob({ + hash: "test", + size: totalSize, + refreshUrl: "https://huggingface.co", + listener: (e) => debugged.push(e), + fetch: async function (_url, opts) { + const url = new URL(_url as string); + const headers = opts?.headers as Record | undefined; + + switch (url.hostname) { + case "huggingface.co": { + // This is a token + return new Response( + JSON.stringify({ + casUrl: "https://cas.co", + accessToken: "boo", + exp: 1_000_000, + }), + ); + } + case "cas.co": { + // This is the reconstruction info + const range = headers?.["Range"]?.slice("bytes=".length).split("-").map(Number); + + const start = range?.[0] ?? 0; + // const end = range?.[1] ?? (totalSize - 1); + + return new Response( + JSON.stringify({ + terms: Array(1000) + .fill(0) + .map((_, i) => ({ + hash: "test" + (i % 2), + range: { + start: 0, + end: 2, + }, + unpacked_length: chunk1Content.length + chunk2Content.length, + })), + fetch_info: { + test0: [ + { + url: "https://fetch.co", + range: { start: 0, end: 2 }, + url_range: { + start: 0, + end: mergedChunks.byteLength - 1, + }, + }, + ], + test1: [ + { + url: "https://fetch.co", + range: { start: 0, end: 2 }, + url_range: { + start: 0, + end: mergedChunks.byteLength - 1, + }, + }, + ], + }, + offset_into_first_range: start, + } satisfies ReconstructionInfo), + ); + } + case "fetch.co": { + fetchCount++; + return new Response( + new ReadableStream({ + pull(controller) { + controller.enqueue(new Uint8Array(mergedChunks)); + controller.close(); + }, + }), + //mergedChunks + ); + } + default: + throw new Error("Unhandled URL"); + } + }, + }); + + const startIndexes = [0, 5, 11, 6, 12, 100, 2000, totalSize - 12, totalSize - 2]; + + for (const index of startIndexes) { + console.log("slice", index); + const content = await blob.slice(index).text(); + expect(content.length).toBe(wholeText.length - index); + expect(content.slice(0, 1000)).toEqual(wholeText.slice(index).slice(0, 1000)); + expect(debugged.filter((e) => e.event === "read").length).toBe(4); // 1 read + 1 undefined + expect(fetchCount).toEqual(2); + + fetchCount = 0; + debugged.length = 0; + } + }); + }); + + describe("loading one chunk at a time", () => { + it("should load different slices but not till the end", async () => { + const chunk1Content = "hello"; + const chunk2Content = "world!"; + const debugged: Array<{ event: "read" | string } & Record> = []; + + const chunks = Array(1000) + .fill(0) + .flatMap(() => [makeChunk(chunk1Content), makeChunk(chunk2Content)]); + + const totalChunkLength = sum(chunks.map((x) => x.byteLength)); + const wholeText = (chunk1Content + chunk2Content).repeat(1000); + + const totalSize = wholeText.length; + let fetchCount = 0; + + const blob = new XetBlob({ + hash: "test", + size: totalSize, + refreshUrl: "https://huggingface.co", + listener: (e) => debugged.push(e), + fetch: async function (_url, opts) { + const url = new URL(_url as string); + const headers = opts?.headers as Record | undefined; + + switch (url.hostname) { + case "huggingface.co": { + // This is a token + return new Response( + JSON.stringify({ + casUrl: "https://cas.co", + accessToken: "boo", + exp: 1_000_000, + }), + ); + } + case "cas.co": { + // This is the reconstruction info + const range = headers?.["Range"]?.slice("bytes=".length).split("-").map(Number); + + const start = range?.[0] ?? 0; + // const end = range?.[1] ?? (totalSize - 1); + + return new Response( + JSON.stringify({ + terms: [ + { + hash: "test", + range: { + start: 0, + end: 2000, + }, + unpacked_length: chunk1Content.length + chunk2Content.length, + }, + ], + fetch_info: { + test: [ + { + url: "https://fetch.co", + range: { start: 0, end: 2000 }, + url_range: { + start: 0, + end: totalChunkLength - 1, + }, + }, + ], + }, + offset_into_first_range: start, + } satisfies ReconstructionInfo), + ); + } + case "fetch.co": { + fetchCount++; + return new Response( + new ReadableStream({ + pull(controller) { + for (const chunk of chunks) { + controller.enqueue(chunk); + } + controller.close(); + }, + }), + { + headers: { + "Content-Range": `bytes 0-${totalChunkLength - 1}/${totalChunkLength}`, + ETag: `"test"`, + "Content-Length": `${totalChunkLength}`, + }, + }, + ); + } + default: + throw new Error("Unhandled URL"); + } + }, + }); + + const startIndexes = [0, 5, 11, 6, 12, 100, 2000]; + + for (const index of startIndexes) { + console.log("slice", index); + const content = await blob.slice(index, 4000).text(); + expect(content.length).toBe(4000 - index); + expect(content.slice(0, 1000)).toEqual(wholeText.slice(index).slice(0, 1000)); + expect(fetchCount).toEqual(1); + + fetchCount = 0; + debugged.length = 0; + } + }); + + it("should load different slices", async () => { + const chunk1Content = "hello"; + const chunk2Content = "world!"; + const debugged: Array<{ event: "read" | string } & Record> = []; + + const chunks = Array(1000) + .fill(0) + .flatMap(() => [makeChunk(chunk1Content), makeChunk(chunk2Content)]); + + const totalChunkLength = sum(chunks.map((x) => x.byteLength)); + const wholeText = (chunk1Content + chunk2Content).repeat(1000); + + const totalSize = wholeText.length; + let fetchCount = 0; + + const blob = new XetBlob({ + hash: "test", + size: totalSize, + refreshUrl: "https://huggingface.co", + listener: (e) => debugged.push(e), + fetch: async function (_url, opts) { + const url = new URL(_url as string); + const headers = opts?.headers as Record | undefined; + + switch (url.hostname) { + case "huggingface.co": { + // This is a token + return new Response( + JSON.stringify({ + casUrl: "https://cas.co", + accessToken: "boo", + exp: 1_000_000, + }), + ); + } + case "cas.co": { + // This is the reconstruction info + const range = headers?.["Range"]?.slice("bytes=".length).split("-").map(Number); + + const start = range?.[0] ?? 0; + // const end = range?.[1] ?? (totalSize - 1); + + return new Response( + JSON.stringify({ + terms: Array(1000) + .fill(0) + .map(() => ({ + hash: "test", + range: { + start: 0, + end: 2, + }, + unpacked_length: chunk1Content.length + chunk2Content.length, + })), + fetch_info: { + test: [ + { + url: "https://fetch.co", + range: { start: 0, end: 2 }, + url_range: { + start: 0, + end: totalChunkLength - 1, + }, + }, + ], + }, + offset_into_first_range: start, + } satisfies ReconstructionInfo), + ); + } + case "fetch.co": { + fetchCount++; + return new Response( + new ReadableStream({ + pull(controller) { + for (const chunk of chunks) { + controller.enqueue(chunk); + } + controller.close(); + }, + }), + ); + } + default: + throw new Error("Unhandled URL"); + } + }, + }); + + const startIndexes = [0, 5, 11, 6, 12, 100, 2000, totalSize - 12, totalSize - 2]; + + for (const index of startIndexes) { + console.log("slice", index); + const content = await blob.slice(index).text(); + expect(content.length).toBe(wholeText.length - index); + expect(content.slice(0, 1000)).toEqual(wholeText.slice(index).slice(0, 1000)); + expect(debugged.filter((e) => e.event === "read").length).toBe(2000 + 1); // 1 read for each chunk + 1 undefined + expect(fetchCount).toEqual(1); + + fetchCount = 0; + debugged.length = 0; + } + }); + }); + + describe("loading at 29 bytes intervals", () => { + it("should load different slices", async () => { + const chunk1Content = "hello"; + const chunk2Content = "world!"; + const debugged: Array<{ event: "read" | string } & Record> = []; + + const chunks = Array(1000) + .fill(0) + .flatMap(() => [makeChunk(chunk1Content), makeChunk(chunk2Content)]); + const mergedChunks = await new Blob(chunks).arrayBuffer(); + const splitChunks = splitChunk(new Uint8Array(mergedChunks), 29); + + const totalChunkLength = sum(chunks.map((x) => x.byteLength)); + const wholeText = (chunk1Content + chunk2Content).repeat(1000); + + const totalSize = wholeText.length; + let fetchCount = 0; + + const blob = new XetBlob({ + hash: "test", + size: totalSize, + refreshUrl: "https://huggingface.co", + listener: (e) => debugged.push(e), + fetch: async function (_url, opts) { + const url = new URL(_url as string); + const headers = opts?.headers as Record | undefined; + + switch (url.hostname) { + case "huggingface.co": { + // This is a token + return new Response( + JSON.stringify({ + casUrl: "https://cas.co", + accessToken: "boo", + exp: 1_000_000, + }), + ); + } + case "cas.co": { + // This is the reconstruction info + const range = headers?.["Range"]?.slice("bytes=".length).split("-").map(Number); + + const start = range?.[0] ?? 0; + // const end = range?.[1] ?? (totalSize - 1); + + return new Response( + JSON.stringify({ + terms: Array(1000) + .fill(0) + .map(() => ({ + hash: "test", + range: { + start: 0, + end: 2, + }, + unpacked_length: chunk1Content.length + chunk2Content.length, + })), + fetch_info: { + test: [ + { + url: "https://fetch.co", + range: { start: 0, end: 2 }, + url_range: { + start: 0, + end: totalChunkLength - 1, + }, + }, + ], + }, + offset_into_first_range: start, + } satisfies ReconstructionInfo), + ); + } + case "fetch.co": { + fetchCount++; + return new Response( + new ReadableStream({ + pull(controller) { + for (const chunk of splitChunks) { + controller.enqueue(chunk); + } + controller.close(); + }, + }), + ); + } + default: + throw new Error("Unhandled URL"); + } + }, + }); + + const startIndexes = [0, 5, 11, 6, 12, 100, 2000, totalSize - 12, totalSize - 2]; + + for (const index of startIndexes) { + console.log("slice", index); + const content = await blob.slice(index).text(); + expect(content.length).toBe(wholeText.length - index); + expect(content.slice(0, 1000)).toEqual(wholeText.slice(index).slice(0, 1000)); + expect(debugged.filter((e) => e.event === "read").length).toBe(Math.ceil(totalChunkLength / 29) + 1); // 1 read for each chunk + 1 undefined + expect(fetchCount).toEqual(1); + + fetchCount = 0; + debugged.length = 0; + } + }); + }); + + describe("loading one byte at a time", () => { + it("should load different slices", async () => { + const chunk1Content = "hello"; + const chunk2Content = "world!"; + const debugged: Array<{ event: "read" | string } & Record> = []; + + const chunks = Array(100) + .fill(0) + .flatMap(() => [makeChunk(chunk1Content), makeChunk(chunk2Content)]) + .flatMap((x) => splitChunk(x, 1)); + + const totalChunkLength = sum(chunks.map((x) => x.byteLength)); + const wholeText = (chunk1Content + chunk2Content).repeat(100); + + const totalSize = wholeText.length; + let fetchCount = 0; + + const blob = new XetBlob({ + hash: "test", + size: totalSize, + refreshUrl: "https://huggingface.co", + listener: (e) => debugged.push(e), + fetch: async function (_url, opts) { + const url = new URL(_url as string); + const headers = opts?.headers as Record | undefined; + + switch (url.hostname) { + case "huggingface.co": { + // This is a token + return new Response( + JSON.stringify({ + casUrl: "https://cas.co", + accessToken: "boo", + exp: 1_000_000, + }), + ); + } + case "cas.co": { + // This is the reconstruction info + const range = headers?.["Range"]?.slice("bytes=".length).split("-").map(Number); + + const start = range?.[0] ?? 0; + // const end = range?.[1] ?? (totalSize - 1); + + return new Response( + JSON.stringify({ + terms: Array(100) + .fill(0) + .map(() => ({ + hash: "test", + range: { + start: 0, + end: 2, + }, + unpacked_length: chunk1Content.length + chunk2Content.length, + })), + fetch_info: { + test: [ + { + url: "https://fetch.co", + range: { start: 0, end: 2 }, + url_range: { + start: 0, + end: totalChunkLength - 1, + }, + }, + ], + }, + offset_into_first_range: start, + } satisfies ReconstructionInfo), + ); + } + case "fetch.co": { + fetchCount++; + return new Response( + new ReadableStream({ + pull(controller) { + for (const chunk of chunks) { + controller.enqueue(chunk); + } + controller.close(); + }, + }), + ); + } + default: + throw new Error("Unhandled URL"); + } + }, + }); + + const startIndexes = [0, 5, 11, 6, 12, 100, totalSize - 12, totalSize - 2]; + + for (const index of startIndexes) { + console.log("slice", index); + const content = await blob.slice(index).text(); + expect(content.length).toBe(wholeText.length - index); + expect(content.slice(0, 1000)).toEqual(wholeText.slice(index).slice(0, 1000)); + expect(debugged.filter((e) => e.event === "read").length).toBe(totalChunkLength + 1); // 1 read for each chunk + 1 undefined + expect(fetchCount).toEqual(1); + + fetchCount = 0; + debugged.length = 0; + } + }); + }); + }); +}); + +function makeChunk(content: string) { + const encoded = new TextEncoder().encode(content); + + const array = new Uint8Array(encoded.length + 8); + + const dataView = new DataView(array.buffer); + dataView.setUint8(0, 0); // version + dataView.setUint8(1, encoded.length % 256); // Compressed length + dataView.setUint8(2, (encoded.length >> 8) % 256); // Compressed length + dataView.setUint8(3, (encoded.length >> 16) % 256); // Compressed length + dataView.setUint8(4, 0); // Compression scheme + dataView.setUint8(5, encoded.length % 256); // Uncompressed length + dataView.setUint8(6, (encoded.length >> 8) % 256); // Uncompressed length + dataView.setUint8(7, (encoded.length >> 16) % 256); // Uncompressed length + + array.set(encoded, 8); + + return array; +} + +function splitChunk(chunk: Uint8Array, toLength: number): Uint8Array[] { + const dataView = new DataView(chunk.buffer); + return new Array(Math.ceil(chunk.byteLength / toLength)).fill(0).map((_, i) => { + const array = new Uint8Array(Math.min(toLength, chunk.byteLength - i * toLength)); + + for (let j = 0; j < array.byteLength; j++) { + array[j] = dataView.getUint8(i * toLength + j); + } + return array; + }); +} diff --git a/node_modules/@huggingface/hub/src/utils/XetBlob.ts b/node_modules/@huggingface/hub/src/utils/XetBlob.ts new file mode 100644 index 0000000000000000000000000000000000000000..d0f6ba35b5f87b81f91bc61c7172cc2ee2769989 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/XetBlob.ts @@ -0,0 +1,729 @@ +import { createApiError } from "../error"; +import type { CredentialsParams } from "../types/public"; +import { checkCredentials } from "./checkCredentials"; +import { combineUint8Arrays } from "./combineUint8Arrays"; +import { decompress as lz4_decompress } from "../vendor/lz4js"; +import { RangeList } from "./RangeList"; + +const JWT_SAFETY_PERIOD = 60_000; +const JWT_CACHE_SIZE = 1_000; + +export interface XetReadToken { + accessToken: string; + casUrl: string; + exp: number; +} + +type XetBlobCreateOptions = { + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + // URL to get the access token from + refreshUrl: string; + size: number; + listener?: (arg: { event: "read" } | { event: "progress"; progress: { read: number; total: number } }) => void; + internalLogging?: boolean; + /** + * Pre-fetched read token to avoid the refresh URL roundtrip. + */ + readToken?: XetReadToken; +} & ({ hash: string; reconstructionUrl?: string } | { hash?: string; reconstructionUrl: string }) & + Partial; + +export interface ReconstructionInfo { + /** + * List of CAS blocks + */ + terms: Array<{ + /** Hash of the CAS block */ + hash: string; + /** Total uncompressed length of data of the chunks from range.start to range.end - 1 */ + unpacked_length: number; + /** Chunks. Eg start: 10, end: 100 = chunks 10-99 */ + range: { start: number; end: number }; + }>; + + /** + * Dictionnary of CAS block hash => list of ranges in the block + url to fetch it + */ + fetch_info: Record< + string, + Array<{ + url: string; + /** Chunk range */ + range: { start: number; end: number }; + /** + * Byte range, when making the call to the URL. + * + * We assume that we're given non-overlapping ranges for each hash + */ + url_range: { start: number; end: number }; + }> + >; + /** + * When doing a range request, the offset into the term's uncompressed data. Can be multiple chunks' worth of data. + */ + offset_into_first_range: number; +} + +export enum XetChunkCompressionScheme { + None = 0, + LZ4 = 1, + ByteGroupingLZ4 = 2, +} + +const compressionSchemeLabels: Record = { + [XetChunkCompressionScheme.None]: "None", + [XetChunkCompressionScheme.LZ4]: "LZ4", + [XetChunkCompressionScheme.ByteGroupingLZ4]: "ByteGroupingLZ4", +}; + +interface ChunkHeader { + version: number; // u8, 1 byte + compressed_length: number; // 3 * u8, 3 bytes + compression_scheme: XetChunkCompressionScheme; // u8, 1 byte + uncompressed_length: number; // 3 * u8, 3 bytes +} + +export const XET_CHUNK_HEADER_BYTES = 8; + +/** + * XetBlob is a blob implementation that fetches data directly from the Xet storage + */ +export class XetBlob extends Blob { + fetch: typeof fetch; + accessToken?: string; + refreshUrl: string; + reconstructionUrl?: string; + hash?: string; + start = 0; + end = 0; + internalLogging = false; + reconstructionInfo: ReconstructionInfo | undefined; + listener: XetBlobCreateOptions["listener"]; + + constructor(params: XetBlobCreateOptions) { + super([]); + + this.fetch = params.fetch ?? fetch.bind(globalThis); + this.accessToken = checkCredentials(params); + this.refreshUrl = params.refreshUrl; + this.end = params.size; + this.reconstructionUrl = params.reconstructionUrl; + this.hash = params.hash; + this.listener = params.listener; + this.internalLogging = params.internalLogging ?? false; + + if (params.readToken) { + const key = cacheKey({ refreshUrl: this.refreshUrl, initialAccessToken: this.accessToken }); + jwts.set(key, { + accessToken: params.readToken.accessToken, + expiresAt: new Date(params.readToken.exp * 1000), + casUrl: params.readToken.casUrl, + }); + } + } + + override get size(): number { + return this.end - this.start; + } + + #clone() { + const blob = new XetBlob({ + fetch: this.fetch, + hash: this.hash, + refreshUrl: this.refreshUrl, + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + reconstructionUrl: this.reconstructionUrl!, + size: this.size, + }); + + blob.accessToken = this.accessToken; + blob.start = this.start; + blob.end = this.end; + blob.reconstructionInfo = this.reconstructionInfo; + blob.listener = this.listener; + blob.internalLogging = this.internalLogging; + + return blob; + } + + override slice(start = 0, end = this.size): XetBlob { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on XetBlob.slice"); + } + + const slice = this.#clone(); + + slice.start = this.start + start; + slice.end = Math.min(this.start + end, this.end); + + if (slice.start !== this.start || slice.end !== this.end) { + slice.reconstructionInfo = undefined; + } + + return slice; + } + + #reconstructionInfoPromise?: Promise; + + #loadReconstructionInfo() { + if (this.#reconstructionInfoPromise) { + return this.#reconstructionInfoPromise; + } + + this.#reconstructionInfoPromise = (async () => { + const connParams = await getAccessToken(this.accessToken, this.fetch, this.refreshUrl); + + // debug( + // `curl '${connParams.casUrl}/v1/reconstructions/${this.hash}' -H 'Authorization: Bearer ${connParams.accessToken}'` + // ); + + const resp = await this.fetch(this.reconstructionUrl ?? `${connParams.casUrl}/v1/reconstructions/${this.hash}`, { + headers: { + Authorization: `Bearer ${connParams.accessToken}`, + Range: `bytes=${this.start}-${this.end - 1}`, + }, + }); + + if (!resp.ok) { + throw await createApiError(resp); + } + + this.reconstructionInfo = (await resp.json()) as ReconstructionInfo; + + return this.reconstructionInfo; + })().finally(() => (this.#reconstructionInfoPromise = undefined)); + + return this.#reconstructionInfoPromise; + } + + async #fetch(): Promise> { + if (this.size === 0) { + return new ReadableStream({ + start(controller) { + controller.close(); + }, + }); + } + + if (!this.reconstructionInfo) { + await this.#loadReconstructionInfo(); + } + + const rangeLists = new Map>(); + + if (!this.reconstructionInfo) { + throw new Error("Failed to load reconstruction info"); + } + + for (const term of this.reconstructionInfo.terms) { + let rangeList = rangeLists.get(term.hash); + if (!rangeList) { + rangeList = new RangeList(); + rangeLists.set(term.hash, rangeList); + } + + rangeList.add(term.range.start, term.range.end); + } + const listener = this.listener; + const log = this.internalLogging ? (...args: unknown[]) => console.log(...args) : () => {}; + + async function* readData( + reconstructionInfo: ReconstructionInfo, + customFetch: typeof fetch, + maxBytes: number, + reloadReconstructionInfo: () => Promise, + ) { + let totalBytesRead = 0; + let readBytesToSkip = reconstructionInfo.offset_into_first_range; + + for (const term of reconstructionInfo.terms) { + if (totalBytesRead >= maxBytes) { + break; + } + + const rangeList = rangeLists.get(term.hash); + if (!rangeList) { + throw new Error(`Failed to find range list for term ${term.hash}`); + } + + { + const termRanges = rangeList.getRanges(term.range.start, term.range.end); + + if (termRanges.every((range) => range.data)) { + log("all data available for term", term.hash, readBytesToSkip); + rangeLoop: for (const range of termRanges) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + for (let chunk of range.data!) { + if (readBytesToSkip) { + const skipped = Math.min(readBytesToSkip, chunk.byteLength); + chunk = chunk.slice(skipped); + readBytesToSkip -= skipped; + if (!chunk.byteLength) { + continue; + } + } + if (chunk.byteLength > maxBytes - totalBytesRead) { + chunk = chunk.slice(0, maxBytes - totalBytesRead); + } + totalBytesRead += chunk.byteLength; + // The stream consumer can decide to transfer ownership of the chunk, so we need to return a clone + // if there's more than one range for the same term + yield range.refCount > 1 ? chunk.slice() : chunk; + listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } }); + + if (totalBytesRead >= maxBytes) { + break rangeLoop; + } + } + } + rangeList.remove(term.range.start, term.range.end); + continue; + } + } + + let fetchInfo = reconstructionInfo.fetch_info[term.hash].find( + (info) => info.range.start <= term.range.start && info.range.end >= term.range.end, + ); + + if (!fetchInfo) { + throw new Error( + `Failed to find fetch info for term ${term.hash} and range ${term.range.start}-${term.range.end}`, + ); + } + + log("term", term); + log("fetchinfo", fetchInfo); + log("readBytesToSkip", readBytesToSkip); + + let resp = await customFetch(fetchInfo.url, { + headers: { + Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}`, + }, + }); + + if (resp.status === 403) { + // In case it's expired + reconstructionInfo = await reloadReconstructionInfo(); + fetchInfo = reconstructionInfo.fetch_info[term.hash]?.find( + (info) => info.range.start <= term.range.start && info.range.end >= term.range.end, + ); + if (!fetchInfo) { + throw new Error( + `Failed to find fetch info for term ${term.hash} and range ${term.range.start}-${term.range.end} after refresh`, + ); + } + resp = await customFetch(fetchInfo.url, { + headers: { + Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}`, + }, + }); + } + + if (!resp.ok) { + throw await createApiError(resp); + } + + log( + "expected content length", + resp.headers.get("content-length"), + "range", + fetchInfo.url_range, + resp.headers.get("content-range"), + ); + + const reader = resp.body?.getReader(); + if (!reader) { + throw new Error("Failed to get reader from response body"); + } + + let done = false; + let chunkIndex = fetchInfo.range.start; + const ranges = rangeList.getRanges(fetchInfo.range.start, fetchInfo.range.end); + + let leftoverBytes: Uint8Array | undefined = undefined; + let totalFetchBytes = 0; + + fetchData: while (!done && totalBytesRead < maxBytes) { + const result = await reader.read(); + listener?.({ event: "read" }); + + done = result.done; + + log("read", result.value?.byteLength, "bytes", "total read", totalBytesRead, "toSkip", readBytesToSkip); + + if (!result.value) { + log("no data in result, cancelled", result); + continue; + } + + totalFetchBytes += result.value.byteLength; + + if (leftoverBytes) { + result.value = combineUint8Arrays(leftoverBytes, result.value); + leftoverBytes = undefined; + } + + while (totalBytesRead < maxBytes && result.value?.byteLength) { + if (result.value.byteLength < 8) { + // We need 8 bytes to parse the chunk header + leftoverBytes = result.value; + continue fetchData; + } + + const header = new DataView(result.value.buffer, result.value.byteOffset, XET_CHUNK_HEADER_BYTES); + const chunkHeader: ChunkHeader = { + version: header.getUint8(0), + compressed_length: header.getUint8(1) | (header.getUint8(2) << 8) | (header.getUint8(3) << 16), + compression_scheme: header.getUint8(4), + uncompressed_length: header.getUint8(5) | (header.getUint8(6) << 8) | (header.getUint8(7) << 16), + }; + + log("chunk header", chunkHeader, "to skip", readBytesToSkip); + + if (chunkHeader.version !== 0) { + throw new Error(`Unsupported chunk version ${chunkHeader.version}`); + } + + if ( + chunkHeader.compression_scheme !== XetChunkCompressionScheme.None && + chunkHeader.compression_scheme !== XetChunkCompressionScheme.LZ4 && + chunkHeader.compression_scheme !== XetChunkCompressionScheme.ByteGroupingLZ4 + ) { + throw new Error( + `Unsupported compression scheme ${ + compressionSchemeLabels[chunkHeader.compression_scheme] ?? chunkHeader.compression_scheme + }`, + ); + } + + if (result.value.byteLength < chunkHeader.compressed_length + XET_CHUNK_HEADER_BYTES) { + // We need more data to read the full chunk + leftoverBytes = result.value; + continue fetchData; + } + + result.value = result.value.slice(XET_CHUNK_HEADER_BYTES); + + let uncompressed = + chunkHeader.compression_scheme === XetChunkCompressionScheme.LZ4 + ? lz4_decompress(result.value.slice(0, chunkHeader.compressed_length), chunkHeader.uncompressed_length) + : chunkHeader.compression_scheme === XetChunkCompressionScheme.ByteGroupingLZ4 + ? bg4_regroup_bytes( + lz4_decompress( + result.value.slice(0, chunkHeader.compressed_length), + chunkHeader.uncompressed_length, + ), + ) + : result.value.slice(0, chunkHeader.compressed_length); + + const range = ranges.find((range) => chunkIndex >= range.start && chunkIndex < range.end); + const shouldYield = chunkIndex >= term.range.start && chunkIndex < term.range.end; + const minRefCountToStore = shouldYield ? 2 : 1; + let stored = false; + + // Assuming non-overlapping fetch_info ranges for the same hash + if (range && range.refCount >= minRefCountToStore) { + range.data ??= []; + range.data.push(uncompressed); + stored = true; + } + + if (shouldYield) { + if (readBytesToSkip) { + const skipped = Math.min(readBytesToSkip, uncompressed.byteLength); + uncompressed = uncompressed.slice(readBytesToSkip); + readBytesToSkip -= skipped; + } + + if (uncompressed.byteLength > maxBytes - totalBytesRead) { + uncompressed = uncompressed.slice(0, maxBytes - totalBytesRead); + } + + if (uncompressed.byteLength) { + log( + "yield", + uncompressed.byteLength, + "bytes", + result.value.byteLength, + "total read", + totalBytesRead, + stored, + ); + totalBytesRead += uncompressed.byteLength; + yield stored ? uncompressed.slice() : uncompressed; + listener?.({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } }); + } + } + + chunkIndex++; + result.value = result.value.slice(chunkHeader.compressed_length); + } + } + + if ( + done && + totalBytesRead < maxBytes && + totalFetchBytes < fetchInfo.url_range.end - fetchInfo.url_range.start + 1 + ) { + log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes); + log("failed to fetch all data for term", term.hash); + throw new Error( + `Failed to fetch all data for term ${term.hash}, fetched ${totalFetchBytes} bytes out of ${ + fetchInfo.url_range.end - fetchInfo.url_range.start + 1 + }`, + ); + } + + log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes); + + // Release the reader + log("cancel reader"); + await reader.cancel(); + } + } + + const iterator = readData( + this.reconstructionInfo, + this.fetch, + this.end - this.start, + this.#loadReconstructionInfo.bind(this), + ); + + // todo: when Chrome/Safari support it, use ReadableStream.from(readData) + return new ReadableStream( + { + // todo: when Safari supports it, type controller as ReadableByteStreamController + async pull(controller) { + const result = await iterator.next(); + + if (result.value) { + controller.enqueue(result.value); + } + + if (result.done) { + controller.close(); + } + }, + type: "bytes", + // todo: when Safari supports it, add autoAllocateChunkSize param + }, + // todo : use ByteLengthQueuingStrategy when there's good support for it, currently in Node.js it fails due to size being a function + { + highWaterMark: 1_000, // 1_000 chunks for ~1MB of RAM + }, + ); + } + + override async arrayBuffer(): Promise { + const result = await this.#fetch(); + + return new Response(result).arrayBuffer(); + } + + override async text(): Promise { + const result = await this.#fetch(); + + return new Response(result).text(); + } + + async response(): Promise { + const result = await this.#fetch(); + + return new Response(result); + } + + override stream(): ReturnType { + const stream = new TransformStream(); + + this.#fetch() + .then((response) => response.pipeThrough(stream)) + .catch((error) => stream.writable.abort(error.message)); + + return stream.readable; + } +} + +const jwtPromises: Map> = new Map(); +/** + * Cache to store JWTs, to avoid making many auth requests when downloading multiple files from the same repo + */ +const jwts: Map< + string, + { + accessToken: string; + expiresAt: Date; + casUrl: string; + } +> = new Map(); + +function cacheKey(params: { refreshUrl: string; initialAccessToken: string | undefined }): string { + return JSON.stringify([params.refreshUrl, params.initialAccessToken]); +} + +// exported for testing purposes +export function bg4_regroup_bytes(bytes: Uint8Array): Uint8Array { + // python code + + // split = len(x) // 4 + // rem = len(x) % 4 + // g1_pos = split + (1 if rem >= 1 else 0) + // g2_pos = g1_pos + split + (1 if rem >= 2 else 0) + // g3_pos = g2_pos + split + (1 if rem == 3 else 0) + // ret = bytearray(len(x)) + // ret[0::4] = x[:g1_pos] + // ret[1::4] = x[g1_pos:g2_pos] + // ret[2::4] = x[g2_pos:g3_pos] + // ret[3::4] = x[g3_pos:] + + // todo: optimize to do it in-place + + const split = Math.floor(bytes.byteLength / 4); + const rem = bytes.byteLength % 4; + const g1_pos = split + (rem >= 1 ? 1 : 0); + const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); + const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); + + const ret = new Uint8Array(bytes.byteLength); + for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + + for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + + for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + + for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { + ret[i] = bytes[j]; + } + + return ret; + + // alternative implementation (to benchmark which one is faster) + // for (let i = 0; i < bytes.byteLength - 3; i += 4) { + // ret[i] = bytes[i / 4]; + // ret[i + 1] = bytes[g1_pos + i / 4]; + // ret[i + 2] = bytes[g2_pos + i / 4]; + // ret[i + 3] = bytes[g3_pos + i / 4]; + // } + + // if (rem === 1) { + // ret[bytes.byteLength - 1] = bytes[g1_pos - 1]; + // } else if (rem === 2) { + // ret[bytes.byteLength - 2] = bytes[g1_pos - 1]; + // ret[bytes.byteLength - 1] = bytes[g2_pos - 1]; + // } else if (rem === 3) { + // ret[bytes.byteLength - 3] = bytes[g1_pos - 1]; + // ret[bytes.byteLength - 2] = bytes[g2_pos - 1]; + // ret[bytes.byteLength - 1] = bytes[g3_pos - 1]; + // } +} + +export function bg4_split_bytes(bytes: Uint8Array): Uint8Array { + // This function does the opposite of bg4_regroup_bytes + // It takes interleaved bytes and groups them by 4 + + const ret = new Uint8Array(bytes.byteLength); + const split = Math.floor(bytes.byteLength / 4); + const rem = bytes.byteLength % 4; + + // Calculate group positions in the output array + const g1_pos = split + (rem >= 1 ? 1 : 0); + const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); + const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); + + // Extract every 4th byte starting from position 0, 1, 2, 3 + // and place them in their respective groups + for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + + for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + + for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + + for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { + ret[j] = bytes[i]; + } + + return ret; +} + +async function getAccessToken( + initialAccessToken: string | undefined, + customFetch: typeof fetch, + refreshUrl: string, +): Promise<{ accessToken: string; casUrl: string }> { + const key = cacheKey({ refreshUrl, initialAccessToken }); + + const jwt = jwts.get(key); + + if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD)) { + return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; + } + + // If we already have a promise for this repo, return it + const existingPromise = jwtPromises.get(key); + if (existingPromise) { + return existingPromise; + } + + const promise = (async () => { + const resp = await customFetch(refreshUrl, { + headers: { + ...(initialAccessToken + ? { + Authorization: `Bearer ${initialAccessToken}`, + } + : {}), + }, + }); + + if (!resp.ok) { + throw new Error(`Failed to get JWT token: ${resp.status} ${await resp.text()}`); + } + + const json: { accessToken: string; casUrl: string; exp: number } = await resp.json(); + const jwt = { + accessToken: json.accessToken, + expiresAt: new Date(json.exp * 1000), + casUrl: json.casUrl, + }; + + jwtPromises.delete(key); + + for (const [key, value] of jwts.entries()) { + if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD)) { + jwts.delete(key); + } else { + break; + } + } + if (jwts.size >= JWT_CACHE_SIZE) { + const keyToDelete = jwts.keys().next().value; + if (keyToDelete) { + jwts.delete(keyToDelete); + } + } + jwts.set(key, jwt); + + return { + accessToken: json.accessToken, + casUrl: json.casUrl, + }; + })(); + + jwtPromises.set(key, promise); + + return promise; +} diff --git a/node_modules/@huggingface/hub/src/utils/base64FromBytes.ts b/node_modules/@huggingface/hub/src/utils/base64FromBytes.ts new file mode 100644 index 0000000000000000000000000000000000000000..5327bbfe25838372cc7e25123d1cc9beaf80ceda --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/base64FromBytes.ts @@ -0,0 +1,11 @@ +export function base64FromBytes(arr: Uint8Array): string { + if (globalThis.Buffer) { + return globalThis.Buffer.from(arr).toString("base64"); + } else { + const bin: string[] = []; + arr.forEach((byte) => { + bin.push(String.fromCharCode(byte)); + }); + return globalThis.btoa(bin.join("")); + } +} diff --git a/node_modules/@huggingface/hub/src/utils/checkCredentials.ts b/node_modules/@huggingface/hub/src/utils/checkCredentials.ts new file mode 100644 index 0000000000000000000000000000000000000000..0e1717054b5dd1d03f4bd0b9c8f986a2ea4560f7 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/checkCredentials.ts @@ -0,0 +1,18 @@ +import type { CredentialsParams } from "../types/public"; + +export function checkAccessToken(accessToken: string): void { + if (!accessToken.startsWith("hf_")) { + throw new TypeError("Your access token must start with 'hf_'"); + } +} + +export function checkCredentials(params: Partial): string | undefined { + if (params.accessToken) { + checkAccessToken(params.accessToken); + return params.accessToken; + } + if (params.credentials?.accessToken) { + checkAccessToken(params.credentials.accessToken); + return params.credentials.accessToken; + } +} diff --git a/node_modules/@huggingface/hub/src/utils/chunk.ts b/node_modules/@huggingface/hub/src/utils/chunk.ts new file mode 100644 index 0000000000000000000000000000000000000000..20718cc89e5e32fd99d99be93dead0e4e24949f4 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/chunk.ts @@ -0,0 +1,25 @@ +import { range } from "./range"; + +/** + * Chunk array into arrays of length at most `chunkSize` + * + * @param chunkSize must be greater than or equal to 1 + */ +export function chunk(arr: T, chunkSize: number): T[] { + if (isNaN(chunkSize) || chunkSize < 1) { + throw new RangeError("Invalid chunk size: " + chunkSize); + } + + if (!arr.length) { + return []; + } + + /// Small optimization to not chunk buffers unless needed + if (arr.length <= chunkSize) { + return [arr]; + } + + return range(Math.ceil(arr.length / chunkSize)).map((i) => { + return arr.slice(i * chunkSize, (i + 1) * chunkSize); + }) as T[]; +} diff --git a/node_modules/@huggingface/hub/src/utils/combineUint8Arrays.spec.ts b/node_modules/@huggingface/hub/src/utils/combineUint8Arrays.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..96c806a2882bd6b7cccefb3b338cf8f4818dbd2a --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/combineUint8Arrays.spec.ts @@ -0,0 +1,15 @@ +import { describe, it, expect } from "vitest"; +import { combineUint8Arrays } from "./combineUint8Arrays"; + +describe("combineUint8Arrays", () => { + it.each([ + { a: [], b: [], expected: [] }, + { a: [], b: [1, 2, 3], expected: [1, 2, 3] }, + { a: [4, 5, 6], b: [], expected: [4, 5, 6] }, + { a: [7, 8], b: [9, 10], expected: [7, 8, 9, 10] }, + { a: [1], b: [2, 3, 4], expected: [1, 2, 3, 4] }, + ])("combines $a and $b to $expected", ({ a, b, expected }) => { + const result = combineUint8Arrays(new Uint8Array(a), new Uint8Array(b)); + expect(result).toEqual(new Uint8Array(expected)); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/combineUint8Arrays.ts b/node_modules/@huggingface/hub/src/utils/combineUint8Arrays.ts new file mode 100644 index 0000000000000000000000000000000000000000..f0be769b85e44e22963a6c6976707a0a1fc25902 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/combineUint8Arrays.ts @@ -0,0 +1,10 @@ +export function combineUint8Arrays( + a: Uint8Array, + b: Uint8Array, +): Uint8Array { + const aLength = a.length; + const combinedBytes = new Uint8Array(aLength + b.length); + combinedBytes.set(a); + combinedBytes.set(b, aLength); + return combinedBytes; +} diff --git a/node_modules/@huggingface/hub/src/utils/createBlob.ts b/node_modules/@huggingface/hub/src/utils/createBlob.ts new file mode 100644 index 0000000000000000000000000000000000000000..5d5f200a66ec1946d2a770cfd7ccde1b9f724ec4 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/createBlob.ts @@ -0,0 +1,30 @@ +import { WebBlob } from "./WebBlob"; +import { isFrontend } from "./isFrontend"; + +/** + * This function allow to retrieve either a FileBlob or a WebBlob from a URL. + * + * From the backend: + * - support local files + * - support http resources with absolute URLs + * + * From the frontend: + * - support http resources with absolute or relative URLs + */ +export async function createBlob(url: URL, opts?: { fetch?: typeof fetch; accessToken?: string }): Promise { + if (url.protocol === "http:" || url.protocol === "https:") { + return WebBlob.create(url, { fetch: opts?.fetch, accessToken: opts?.accessToken }); + } + + if (isFrontend) { + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); + } + + if (url.protocol === "file:") { + const { FileBlob } = await import("./FileBlob"); + + return FileBlob.create(url); + } + + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); +} diff --git a/node_modules/@huggingface/hub/src/utils/createBlobs.ts b/node_modules/@huggingface/hub/src/utils/createBlobs.ts new file mode 100644 index 0000000000000000000000000000000000000000..93314ebf06232806279e0c90cfe532f7da1dbf68 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/createBlobs.ts @@ -0,0 +1,51 @@ +import { WebBlob } from "./WebBlob"; +import { isFrontend } from "./isFrontend"; + +/** + * This function allow to retrieve either a FileBlob or a WebBlob from a URL. + * + * From the backend: + * - support local files + * - support local folders + * - support http resources with absolute URLs + * + * From the frontend: + * - support http resources with absolute or relative URLs + */ +export async function createBlobs( + url: URL, + destPath: string, + opts?: { fetch?: typeof fetch; maxFolderDepth?: number; accessToken?: string }, +): Promise> { + if (url.protocol === "http:" || url.protocol === "https:") { + const blob = await WebBlob.create(url, { fetch: opts?.fetch, accessToken: opts?.accessToken }); + return [{ path: destPath, blob }]; + } + + if (isFrontend) { + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); + } + + if (url.protocol === "file:") { + const { FileBlob } = await import("./FileBlob"); + const { subPaths } = await import("./sub-paths"); + const paths = await subPaths(url, opts?.maxFolderDepth); + + if (paths.length === 1 && paths[0].relativePath === ".") { + const blob = await FileBlob.create(url); + return [{ path: destPath, blob }]; + } + + return Promise.all( + paths.map(async (path) => ({ + path: `${destPath}/${path.relativePath}` + .replace(/\/[.]$/, "") + .replaceAll("//", "/") + .replace(/^[.]?\//, ""), + blob: await FileBlob.create(new URL(path.path)), + })), + ); + } + + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); +} diff --git a/node_modules/@huggingface/hub/src/utils/createXorb.spec.ts b/node_modules/@huggingface/hub/src/utils/createXorb.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..80229554be14a9a64d8976fe66de13a5c90cc258 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/createXorb.spec.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; +import { backtrackDedup, CurrentXorbInfo } from "./createXorbs"; +import type { ShardData } from "./shardParser"; +import { ChunkCache } from "./ChunkCache"; + +describe("createXorb", () => { + describe("backtrackDedup", () => { + it("should update cache info for chunks that go back due to previous chunks being erased", () => { + const xorb = new CurrentXorbInfo(); + + const chunkMetadata = [ + { + xorbId: xorb.id, + chunkIndex: 0, + length: 101, + }, + { + xorbId: xorb.id, + chunkIndex: 1, + length: 101, + }, + ]; + xorb.chunks = [ + { + hash: "chunk1", + length: 101, + offset: 0, + }, + { + hash: "chunk2", + length: 101, + offset: 101, + }, + ]; + const shardData: ShardData = { + hmacKey: "shard1", + xorbs: [ + { + hash: "remoteXorb1", + chunks: [ + { + hash: "chunk0:shard1", + startOffset: 0, + unpackedLength: 100, + }, + { + hash: "chunk1:shard1", + startOffset: 100, + unpackedLength: 101, + }, + ], + }, + ], + }; + + const computeHmac = (hash: string, key: string) => { + return hash + ":" + key; + }; + + const chunkCache = new ChunkCache(); + let chunkIndex = 0; + for (const chunk of xorb.chunks) { + chunkCache.addChunkToCache(chunk.hash, xorb.id, chunkIndex++, shardData.hmacKey); + } + let xorbIndex = 0; + for (const xorb of shardData.xorbs) { + xorbIndex--; + for (let i = 0; i < xorb.chunks.length; i++) { + chunkCache.addChunkToCache(xorb.chunks[i].hash, xorbIndex, i, shardData.hmacKey); + } + } + const dedup = backtrackDedup(xorb, computeHmac, shardData, chunkCache, chunkMetadata, 0); + expect(dedup).toBe(101); + expect(xorb.chunks).toEqual([{ hash: "chunk2", length: 101, offset: 0 }]); + expect(chunkMetadata).toEqual([ + { + xorbId: -1, + chunkIndex: 1, + length: 101, + }, + { + xorbId: 0, + chunkIndex: 0, + length: 101, + }, + ]); + // chunk1 should use remote hash now + expect(chunkCache.getChunk("chunk1", computeHmac)).toEqual({ xorbIndex: -1, chunkIndex: 1 }); + // The xorb index for chunk2 should be 0 now that the previous chunk was erased from the xorb + expect(chunkCache.getChunk("chunk2", computeHmac)).toEqual({ xorbIndex: 0, chunkIndex: 0 }); + }); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/createXorbs.ts b/node_modules/@huggingface/hub/src/utils/createXorbs.ts new file mode 100644 index 0000000000000000000000000000000000000000..ed41d11872933e167420e839552e36aef77873f7 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/createXorbs.ts @@ -0,0 +1,779 @@ +import { bg4_split_bytes, XET_CHUNK_HEADER_BYTES, XetChunkCompressionScheme } from "./XetBlob"; +import { compress as lz4_compress } from "../vendor/lz4js"; +import { ChunkCache } from "./ChunkCache"; +import { xetWriteToken, type XetWriteTokenParams } from "./xetWriteToken"; +import type { ShardData } from "./shardParser"; +import { parseShardData } from "./shardParser"; +import { SplicedBlob } from "./SplicedBlob"; +import { + createChunker, + nextBlock, + finalize, + hashToHex, + hexToBytes, + xorbHash, + fileHash, + hmac, + verificationHash, + type Chunk, +} from "@huggingface/xetchunk-wasm"; + +const TARGET_CHUNK_SIZE = 64 * 1024; +const MAX_CHUNK_SIZE = 2 * TARGET_CHUNK_SIZE; +const XORB_SIZE = 64 * 1024 * 1024; +const MAX_XORB_CHUNKS = 8 * 1024; +const INTERVAL_BETWEEN_REMOTE_DEDUP = 4_000_000; // 4MB +/** + * 0 = only show progress when uploading the xorb + * 1 = only show progress when processing the file + * 0.5 = show progress when uploading the xorb and when processing the file + */ +const PROCESSING_PROGRESS_RATIO = 0.1; +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const UPLOADING_PROGRESS_RATIO = 1 - PROCESSING_PROGRESS_RATIO; + +function computeXorbHashHex(chunks: { hash: string; length: number }[]): string { + const chunkObjs: Chunk[] = chunks.map((c) => ({ hash: hexToBytes(c.hash), length: c.length })); + return hashToHex(xorbHash(chunkObjs)); +} + +function computeHmacHex(hash: string, key: string): string { + return hashToHex(hmac(hexToBytes(hash), hexToBytes(key))); +} + +function computeVerificationHashHex(hashes: string[]): string { + return hashToHex(verificationHash(hashes.map(hexToBytes))); +} + +function computeFileHashHex(chunks: { hash: string; length: number }[]): string { + const chunkObjs: Chunk[] = chunks.map((c) => ({ hash: hexToBytes(c.hash), length: c.length })); + return hashToHex(fileHash(chunkObjs)); +} + +function addDataToChunker( + data: Uint8Array, + chunker: ReturnType, +): { hash: string; length: number; dedup: boolean }[] { + return nextBlock(chunker, data).map((c) => ({ hash: hashToHex(c.hash), length: c.length, dedup: false })); +} + +function finalizeChunker( + chunker: ReturnType, +): { hash: string; length: number; dedup: boolean }[] { + const last = finalize(chunker); + if (!last) { + return []; + } + return [{ hash: hashToHex(last.hash), length: last.length, dedup: false }]; +} + +interface XorbEvent { + event: "xorb"; + xorb: Uint8Array; + hash: string; + id: number; + chunks: Array<{ hash: string; length: number }>; + files: Array<{ + path: string; + progress: number; + lastSentProgress: number; + }>; +} + +export class CurrentXorbInfo { + id: number; + offset: number; + chunks: Array<{ hash: string; length: number; offset: number }>; + + fileProcessedBytes: Record; + fileUploadedBytes: Record; + fileSize: Record; + data: Uint8Array; + immutableData: { + chunkIndex: number; + offset: number; + } | null; + + constructor() { + this.id = 0; + this.offset = 0; + this.chunks = []; + this.fileProcessedBytes = {}; + this.fileUploadedBytes = {}; + this.fileSize = {}; + this.data = new Uint8Array(XORB_SIZE); + this.immutableData = null; + } + + event(computeXorbHash: (chunks: { hash: string; length: number }[]) => string): XorbEvent { + const xorbChunksCleaned = this.chunks.map((chunk) => ({ + hash: chunk.hash, + length: chunk.length, + })); + + return { + event: "xorb" as const, + xorb: this.data.subarray(0, this.offset), + hash: computeXorbHash(xorbChunksCleaned), + chunks: xorbChunksCleaned, + id: this.id, + files: Object.entries(this.fileProcessedBytes).map(([path, processedBytes]) => ({ + path, + progress: processedBytes / this.fileSize[path], + lastSentProgress: + ((this.fileUploadedBytes[path] ?? 0) + + (processedBytes - (this.fileUploadedBytes[path] ?? 0)) * PROCESSING_PROGRESS_RATIO) / + this.fileSize[path], + })), + }; + } +} + +export async function* createXorbs( + fileSources: AsyncGenerator<{ content: Blob; path: string; sha256?: string }>, + params: XetWriteTokenParams & { + yieldCallback?: (event: { event: "fileProgress"; path: string; progress: number }) => void; + }, +): AsyncGenerator< + | XorbEvent + | { + event: "file"; + path: string; + hash: string; + sha256?: string; + /** Percentage of file bytes that were deduplicated (0-1) */ + dedupRatio: number; + representation: Array<{ + xorbId: number | string; // either xorb id (for local xorbs) or xorb hash (for remote xorbs) + indexStart: number; + indexEnd: number; + /** Unpacked length */ + length: number; + rangeHash: string; + }>; + }, + void, + undefined +> { + const alreadyDoneFileSha256s: Set = new Set(); + let xorbId = 0; + const chunkCache = new ChunkCache(); + let xorb = new CurrentXorbInfo(); + + const nextXorb = (currentFile: { path: string; uploadedBytes: number; size: number }): XorbEvent => { + const event = xorb.event(computeXorbHashHex); + + xorbId++; + xorb = new CurrentXorbInfo(); + xorb.id = xorbId; + xorb.fileUploadedBytes = { + [currentFile.path]: currentFile.uploadedBytes, + }; + xorb.fileSize[currentFile.path] = currentFile.size; + + return event; + }; + + const pendingFileEvents: Array<{ + event: "file"; + path: string; + hash: string; + dedupRatio: number; + sha256?: string; + representation: Array<{ + xorbId: number | string; + indexStart: number; + indexEnd: number; + length: number; + rangeHash: string; + }>; + }> = []; + + const remoteXorbHashes: string[] = [""]; // starts at index 1 (to simplify implem a bit) + + for await (const fileSource of fileSources) { + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: 0, + }); + if (fileSource.sha256 && alreadyDoneFileSha256s.has(fileSource.sha256)) { + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: 1, + }); + continue; + } + if (fileSource.sha256) { + alreadyDoneFileSha256s.add(fileSource.sha256); + } + + const chunker = createChunker(TARGET_CHUNK_SIZE); + { + xorb.fileSize[fileSource.path] = fileSource.content.size; + + // Load dedup info for the first chunk of the file, if it's potentially modified by the splice + if (fileSource.content instanceof SplicedBlob && fileSource.content.firstSpliceIndex < MAX_CHUNK_SIZE) { + await loadDedupInfoToCache( + fileSource.content.originalBlob.slice(0, MAX_CHUNK_SIZE), + remoteXorbHashes, + params, + chunkCache, + computeHmacHex, + { + maxChunks: 1, + isAtBeginning: true, + }, + ); + } + let bytesSinceRemoteDedup = Infinity; + let bytesSinceLastProgressEvent = 0; + let isFirstFileChunk = true; + const sourceChunks: Array = []; + + const reader = fileSource.content.stream().getReader(); + let processedBytes = 0; + let dedupedBytes = 0; // Track bytes that were deduplicated + // Needed to compute the final file hash + // todo: have the wasm function to compute file hash be able to take data chunk by chunk instead of all at once + const fileChunks: Array<{ hash: string; length: number }> = []; + // Collect chunk metadata to build representation at the end + // todo: build partial representation at the end of each xorb, to avoid having to store all chunks in memory + const chunkMetadata: Array<{ + xorbId: number | string; + chunkIndex: number; + length: number; + }> = []; + + const addChunks = async function* (chunks: Array<{ hash: string; length: number; dedup: boolean }>) { + for (const chunk of chunks) { + if (isFirstFileChunk) { + chunk.dedup = true; + isFirstFileChunk = false; + } + let chunkIndex = xorb.chunks.length; + let chunkXorbId = xorbId; + + // Remove chunks from source data + const chunkToCopy = removeChunkFromSourceData(sourceChunks, chunk.length); + + let cacheData = chunkCache.getChunk(chunk.hash, computeHmacHex); + if (cacheData === undefined && chunk.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { + const token = await xetWriteToken(params); + bytesSinceRemoteDedup = 0; + + const shardResp = await (params.fetch ?? fetch)(token.casUrl + "/v1/chunks/default/" + chunk.hash, { + headers: { + Authorization: `Bearer ${token.accessToken}`, + }, + }); + + // todo: handle non-404 non-429 errors, eg throw error + if (shardResp.ok) { + const shard = await shardResp.blob(); + const shardData = await parseShardData(shard); + + for (const xorb of shardData.xorbs) { + const remoteXorbId = -remoteXorbHashes.length; + remoteXorbHashes.push(xorb.hash); + let i = 0; + for (const chunk of xorb.chunks) { + chunkCache.addChunkToCache(chunk.hash, remoteXorbId, i++, shardData.hmacKey); + } + } + cacheData = chunkCache.getChunk(chunk.hash, computeHmacHex); + + // We backtrack a bit to check if new dedup info contains older chunks + const oldDedupedBytes = dedupedBytes; + dedupedBytes = backtrackDedup(xorb, computeHmacHex, shardData, chunkCache, chunkMetadata, dedupedBytes); + + if (dedupedBytes > oldDedupedBytes) { + xorb.fileUploadedBytes[fileSource.path] ??= 0; + xorb.fileUploadedBytes[fileSource.path] += dedupedBytes - oldDedupedBytes; + } + } + } + if (cacheData === undefined) { + if (!writeChunk(xorb, chunkToCopy, chunk.hash)) { + // Failure to write chunk, maybe because it went over xorb size limit + yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); + + chunkIndex = 0; + chunkXorbId = xorbId; + + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: (rep.xorbId as number) >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId], + })); + yield event; + } + pendingFileEvents.length = 0; + + if (!writeChunk(xorb, chunkToCopy, chunk.hash)) { + throw new Error("Failed to write chunk into xorb"); + } + } + + chunkCache.addChunkToCache(chunk.hash, xorbId, chunkIndex, null); + } else { + chunkXorbId = cacheData.xorbIndex; + chunkIndex = cacheData.chunkIndex; + dedupedBytes += chunk.length; // Track deduplicated bytes + xorb.fileUploadedBytes[fileSource.path] ??= 0; + xorb.fileUploadedBytes[fileSource.path] += chunk.length; + } + + bytesSinceRemoteDedup += chunk.length; + bytesSinceLastProgressEvent += chunk.length; + + // Collect metadata for building representation at the end + fileChunks.push({ hash: chunk.hash, length: chunk.length }); + chunkMetadata.push({ + xorbId: chunkXorbId, + chunkIndex: chunkIndex, + length: chunk.length, + }); + + xorb.fileProcessedBytes[fileSource.path] = processedBytes; + + if (bytesSinceLastProgressEvent >= 1_000_000) { + // Emit half of the progress when processed locally, other half when uploading the xorb + bytesSinceLastProgressEvent = 0; + params.yieldCallback?.({ + event: "fileProgress", + path: fileSource.path, + progress: + ((xorb.fileUploadedBytes[fileSource.path] ?? 0) + + (xorb.fileProcessedBytes[fileSource.path] - (xorb.fileUploadedBytes[fileSource.path] ?? 0)) * + PROCESSING_PROGRESS_RATIO) / + fileSource.content.size, + }); + } + + if (xorb.chunks.length >= MAX_XORB_CHUNKS) { + yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); + + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: (rep.xorbId as number) >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId], + })); + yield event; + } + pendingFileEvents.length = 0; + } + } + }; + + while (true) { + const { done, value } = await reader.read(); + if (done) { + yield* addChunks(finalizeChunker(chunker)); + break; + } + processedBytes += value.length; + sourceChunks.push(value); + yield* addChunks(addDataToChunker(value, chunker)); + } + + const fileRepresentation = buildFileRepresentation(chunkMetadata, fileChunks, computeVerificationHashHex); + xorb.immutableData = { + chunkIndex: xorb.chunks.length, + offset: xorb.offset, + }; + const dedupRatio = fileSource.content.size > 0 ? dedupedBytes / fileSource.content.size : 0; + + pendingFileEvents.push({ + event: "file" as const, + path: fileSource.path, + hash: computeFileHashHex(fileChunks), + sha256: fileSource.sha256, + dedupRatio, + representation: fileRepresentation, + }); + } + } + + if (xorb.offset > 0) { + yield xorb.event(computeXorbHashHex); + } + + for (const event of pendingFileEvents) { + event.representation = event.representation.map((rep) => ({ + ...rep, + xorbId: (rep.xorbId as number) >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId], + })); + yield event; + } +} + +export function backtrackDedup( + xorb: CurrentXorbInfo, + computeHmac: (hash: string, key: string) => string, + shardData: ShardData, + chunkCache: ChunkCache, + chunkMetadata: { xorbId: number | string; chunkIndex: number; length: number }[], + dedupedBytes: number, +): number { + const chunkIndexesToBacktrackFor = new Map(); + for ( + let chunkToRecheckIndex = xorb.immutableData?.chunkIndex ?? 0; + chunkToRecheckIndex < xorb.chunks.length; + chunkToRecheckIndex++ + ) { + const chunk = xorb.chunks[chunkToRecheckIndex]; + const hmacHash = computeHmac(chunk.hash, shardData.hmacKey); + const cacheData = chunkCache.getChunk(hmacHash, null); + if (cacheData !== undefined) { + chunkIndexesToBacktrackFor.set(chunkToRecheckIndex, { + xorbId: cacheData.xorbIndex, + chunkIndex: cacheData.chunkIndex, + }); + chunkCache.removeChunkFromCache(chunk.hash); + } + } + + // Use remote dedup info to update chunk metadata for file representation + for (const metadata of chunkMetadata) { + if (metadata.xorbId === xorb.id && chunkIndexesToBacktrackFor.has(metadata.chunkIndex)) { + const backtrackData = chunkIndexesToBacktrackFor.get(metadata.chunkIndex); + if (backtrackData !== undefined) { + metadata.xorbId = backtrackData.xorbId; + metadata.chunkIndex = backtrackData.chunkIndex; + dedupedBytes += metadata.length; + } + } + } + + // Remove chunks that were backtracked from xorbChunks + const xorbRangesToErase: Array<{ start: number; end: number }> = []; + for (let i = 0; i < xorb.chunks.length; i++) { + const chunk = xorb.chunks[i]; + if (chunkIndexesToBacktrackFor.has(i)) { + xorbRangesToErase.push({ + start: chunk.offset, + end: i < xorb.chunks.length - 1 ? xorb.chunks[i + 1].offset : xorb.offset, + }); + } + } + const xorbRangesToKeep: Array<{ start: number; end: number }> = []; + let currentStart = 0; + for (let i = 0; i < xorbRangesToErase.length; i++) { + const range = xorbRangesToErase[i]; + if (currentStart !== range.start) { + xorbRangesToKeep.push({ start: currentStart, end: range.start }); + } + currentStart = range.end; + } + if (currentStart !== xorb.offset) { + xorbRangesToKeep.push({ start: currentStart, end: xorb.offset }); + } + + let currentOffset = 0; + for (const range of xorbRangesToKeep) { + if (range.start !== currentOffset) { + xorb.data.set(xorb.data.subarray(range.start, range.end), currentOffset); + } + currentOffset += range.end - range.start; + } + const newXorbChunks: Array<{ hash: string; length: number; offset: number }> = []; + const oldIndexToNewIndex = new Map(); + let erasedOffset = 0; + for (let i = 0; i < xorb.chunks.length; i++) { + const chunk = xorb.chunks[i]; + if (chunkIndexesToBacktrackFor.has(i)) { + if (i < xorb.chunks.length - 1) { + erasedOffset += xorb.chunks[i + 1].offset - chunk.offset; + } + } else { + newXorbChunks.push({ + hash: chunk.hash, + length: chunk.length, + offset: chunk.offset - erasedOffset, + }); + // Only need a mapping if index changed (at least one previous chunk was erased) + if (erasedOffset > 0) { + oldIndexToNewIndex.set(i, newXorbChunks.length - 1); + } + } + } + xorb.chunks = newXorbChunks; + xorb.offset = currentOffset; + // Update chunkMetadata and chunkCache with new chunk indexes for the current xorb chunks + for (const chunk of chunkMetadata) { + if (chunk.xorbId === xorb.id) { + const newIndex = oldIndexToNewIndex.get(chunk.chunkIndex); + if (newIndex !== undefined) { + const cached = chunkCache.getChunk(xorb.chunks[newIndex].hash, null); + if (cached !== undefined && cached.xorbIndex === chunk.xorbId && cached.chunkIndex === chunk.chunkIndex) { + chunkCache.updateChunkIndex(xorb.chunks[newIndex].hash, newIndex); + } + chunk.chunkIndex = newIndex; + } + } + } + return dedupedBytes; +} + +/** + * Removes and returns a chunk of the specified length from the sourceChunks array. + */ +function removeChunkFromSourceData(sourceChunks: Array, chunkLength: number): Uint8Array { + if (chunkLength === sourceChunks[0].length) { + const chunkToCopy = sourceChunks[0]; + sourceChunks.shift(); + return chunkToCopy; + } else if (chunkLength < sourceChunks[0].length) { + const chunkToCopy = sourceChunks[0].subarray(0, chunkLength); + sourceChunks[0] = sourceChunks[0].subarray(chunkLength); + return chunkToCopy; + } else { + const chunkToCopy = new Uint8Array(chunkLength); + let copyOffset = 0; + let index = 0; + let toSlice = -1; + while (copyOffset < chunkLength) { + const nToCopy = Math.min(sourceChunks[index].length, chunkLength - copyOffset); + chunkToCopy.set(sourceChunks[index].subarray(0, nToCopy), copyOffset); + copyOffset += nToCopy; + + if (nToCopy === sourceChunks[index].length) { + index++; + } else { + toSlice = nToCopy; + } + } + sourceChunks.splice(0, index); + if (toSlice !== -1) { + sourceChunks[0] = sourceChunks[0].subarray(toSlice); + } + return chunkToCopy; + } +} + +/** + * Write a chunk header to the xorb and return the offset of where to write the next chunk + * + * If it returns 0, it means there wasn't enough space in the xorb + */ +function writeChunk(xorb: CurrentXorbInfo, chunk: Uint8Array, hash: string): boolean { + const regularCompressedChunk = lz4_compress(chunk); + const bgCompressedChunk = lz4_compress(bg4_split_bytes(chunk)); + const compressedChunk = + bgCompressedChunk.length < regularCompressedChunk.length ? bgCompressedChunk : regularCompressedChunk; + const chunkToWrite = compressedChunk.length < chunk.length ? compressedChunk : chunk; + + if (xorb.offset + XET_CHUNK_HEADER_BYTES + chunkToWrite.length > XORB_SIZE) { + return false; + } + + xorb.data[xorb.offset] = 0; + xorb.data[xorb.offset + 1] = chunkToWrite.length & 0xff; + xorb.data[xorb.offset + 2] = (chunkToWrite.length >> 8) & 0xff; + xorb.data[xorb.offset + 3] = (chunkToWrite.length >> 16) & 0xff; + xorb.data[xorb.offset + 4] = + chunkToWrite.length < chunk.length + ? bgCompressedChunk.length < regularCompressedChunk.length + ? XetChunkCompressionScheme.ByteGroupingLZ4 + : XetChunkCompressionScheme.LZ4 + : XetChunkCompressionScheme.None; + xorb.data[xorb.offset + 5] = chunk.length & 0xff; + xorb.data[xorb.offset + 6] = (chunk.length >> 8) & 0xff; + xorb.data[xorb.offset + 7] = (chunk.length >> 16) & 0xff; + + xorb.data.set(chunkToWrite, xorb.offset + XET_CHUNK_HEADER_BYTES); + + xorb.chunks.push({ hash, length: chunk.length, offset: xorb.offset }); + xorb.offset += XET_CHUNK_HEADER_BYTES + chunkToWrite.length; + return true; +} + +// Build file representation from collected metadata +const buildFileRepresentation = ( + metadata: Array<{ xorbId: number | string; chunkIndex: number; length: number }>, + chunks: Array<{ hash: string; length: number }>, + computeVerificationHash: (hashes: string[]) => string, +): Array<{ + xorbId: number | string; + indexStart: number; + indexEnd: number; + length: number; + rangeHash: string; +}> => { + if (metadata.length === 0) { + return []; + } + + const representation: Array<{ + xorbId: number | string; + indexStart: number; + indexEnd: number; + length: number; + rangeHash: string; + }> = []; + + let currentRange = { + xorbId: metadata[0].xorbId, + indexStart: metadata[0].chunkIndex, + indexEnd: metadata[0].chunkIndex + 1, + length: metadata[0].length, + chunkHashStart: 0, + }; + + for (let i = 1; i < metadata.length; i++) { + const chunk = metadata[i]; + + // Check if this chunk continues the current range + if (currentRange.xorbId === chunk.xorbId && currentRange.indexEnd === chunk.chunkIndex) { + // Extend current range + currentRange.indexEnd = chunk.chunkIndex + 1; + currentRange.length += chunk.length; + } else { + // Finalize current range and start a new one + const rangeHash = computeVerificationHash(chunks.slice(currentRange.chunkHashStart, i).map((x) => x.hash)); + representation.push({ + xorbId: currentRange.xorbId, + indexStart: currentRange.indexStart, + indexEnd: currentRange.indexEnd, + length: currentRange.length, + rangeHash, + }); + + currentRange = { + xorbId: chunk.xorbId, + indexStart: chunk.chunkIndex, + indexEnd: chunk.chunkIndex + 1, + length: chunk.length, + chunkHashStart: i, + }; + } + } + + // Finalize the last range + const rangeHash = computeVerificationHash(chunks.slice(currentRange.chunkHashStart).map((x) => x.hash)); + representation.push({ + xorbId: currentRange.xorbId, + indexStart: currentRange.indexStart, + indexEnd: currentRange.indexEnd, + length: currentRange.length, + rangeHash, + }); + + return representation; +}; + +/** + * Helper to load dedup info for blob contents into cache. + * Processes the blob's contents, chunks it, and loads dedup info into cache without writing to xorb. + * + * For now this is optimized for when the replacement data is at the very beginning of the file + * + * todo: handle when it's not at the beginning of the file by backingtracking xorb contents + * todo: handle when it's not at the beginning of the file by using previous content to chunk at the same boundaries as it would have in the original file + */ +async function loadDedupInfoToCache( + content: Blob, + /** Will be mutated */ + remoteXorbHashes: string[], + params: XetWriteTokenParams, + chunkCache: ChunkCache, + computeHmacHex: (hash: string, key: string) => string, + + opts?: { + isAtBeginning?: boolean; + /** + * The end position of the content to process + * + * Will process content up to the end of the chunk after this position + */ + end?: number; + /** + * The maximum number of chunks to process + * + * Will process content up to the end of the chunk after this position + */ + maxChunks?: number; + }, +): Promise { + const chunker = createChunker(TARGET_CHUNK_SIZE); + const cache = chunkCache; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + let dedupedBytes = 0; + let chunksProcessed = 0; + let totalBytes = 0; + let bytesSinceRemoteDedup = Infinity; + const sourceChunks: Array = []; + + const reader = content.stream().getReader(); + + const processChunks = async (chunks: Array<{ hash: string; length: number; dedup: boolean }>) => { + for (const chunk of chunks) { + chunksProcessed++; + if (opts?.isAtBeginning && chunksProcessed === 1) { + chunk.dedup = true; + } + totalBytes += chunk.length; + + removeChunkFromSourceData(sourceChunks, chunk.length); + + let cacheData = cache.getChunk(chunk.hash, computeHmacHex); + + if (cacheData !== undefined) { + dedupedBytes += chunk.length; + bytesSinceRemoteDedup += chunk.length; + continue; + } + + if (chunk.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { + const token = await xetWriteToken(params); + bytesSinceRemoteDedup = 0; + + const shardResp = await (params.fetch ?? fetch)(token.casUrl + "/v1/chunks/default/" + chunk.hash, { + headers: { + Authorization: `Bearer ${token.accessToken}`, + }, + }); + + if (shardResp.ok) { + const shard = await shardResp.blob(); + const shardData = await parseShardData(shard); + + for (const xorb of shardData.xorbs) { + const remoteXorbId = -remoteXorbHashes.length; + remoteXorbHashes.push(xorb.hash); + let i = 0; + for (const xorbChunk of xorb.chunks) { + cache.addChunkToCache(xorbChunk.hash, remoteXorbId, i++, shardData.hmacKey); + } + } + cacheData = cache.getChunk(chunk.hash, computeHmacHex); + } + } + + if (cacheData !== undefined) { + dedupedBytes += chunk.length; + } + + bytesSinceRemoteDedup += chunk.length; + } + }; + + while (true) { + if (opts?.end !== undefined && totalBytes >= opts.end) { + break; + } + if (opts?.maxChunks !== undefined && chunksProcessed >= opts.maxChunks) { + break; + } + const { done, value } = await reader.read(); + if (done) { + await processChunks(finalizeChunker(chunker)); + break; + } + sourceChunks.push(value); + await processChunks(addDataToChunker(value, chunker)); + } +} diff --git a/node_modules/@huggingface/hub/src/utils/eventToGenerator.spec.ts b/node_modules/@huggingface/hub/src/utils/eventToGenerator.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..59ed182a4a2394f6662afeee1d4f42fe4692e0b5 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/eventToGenerator.spec.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { eventToGenerator } from "./eventToGenerator"; + +describe("eventToGenerator", () => { + it("should handle synchronous events", async () => { + const it = eventToGenerator((yieldCallback, returnCallback) => { + yieldCallback(1); + yieldCallback(2); + returnCallback(3); + }); + + const results = []; + let res: IteratorResult; + do { + res = await it.next(); + if (!res.done) { + results.push(res.value); + } + } while (!res.done); + + expect(results).toEqual([1, 2]); + expect(res.value).toBe(3); + }); + + it("should handle asynchronous events", async () => { + const it = eventToGenerator((yieldCallback, returnCallback) => { + setTimeout(() => yieldCallback(1), 100); + setTimeout(() => yieldCallback(2), 200); + setTimeout(() => returnCallback(3), 300); + }); + + const results = []; + let res: IteratorResult; + do { + res = await it.next(); + if (!res.done) { + results.push(res.value); + } + } while (!res.done); + + expect(results).toEqual([1, 2]); + expect(res.value).toBe(3); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/eventToGenerator.ts b/node_modules/@huggingface/hub/src/utils/eventToGenerator.ts new file mode 100644 index 0000000000000000000000000000000000000000..1bbc84f997fe142cf2b22a80ba0927d3d11bbfc0 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/eventToGenerator.ts @@ -0,0 +1,64 @@ +export async function* eventToGenerator( + cb: ( + yieldCallback: (y: YieldType) => void, + returnCallback: (r: ReturnType) => void, + rejectCallack: (reason: unknown) => void, + ) => unknown, +): AsyncGenerator { + const promises: Array<{ + p: Promise<{ done: true; value: ReturnType } | { done: false; value: YieldType }>; + resolve: (value: { done: true; value: ReturnType } | { done: false; value: YieldType }) => void; + reject: (reason?: unknown) => void; + }> = []; + + function addPromise() { + let resolve: (value: { done: true; value: ReturnType } | { done: false; value: YieldType }) => void; + let reject: (reason?: unknown) => void; + const p = new Promise<{ done: true; value: ReturnType } | { done: false; value: YieldType }>((res, rej) => { + resolve = res; + reject = rej; + }); + // @ts-expect-error TS doesn't know that promise callback is executed immediately + promises.push({ p, resolve, reject }); + } + + addPromise(); + + const callbackRes = Promise.resolve() + .then(() => + cb( + (y) => { + addPromise(); + promises.at(-2)?.resolve({ done: false, value: y }); + }, + (r) => { + addPromise(); + promises.at(-2)?.resolve({ done: true, value: r }); + }, + (err) => promises.shift()?.reject(err), + ), + ) + .catch((err) => promises.shift()?.reject(err)); + + while (1) { + const p = promises[0]; + if (!p) { + throw new Error("Logic error in eventGenerator, promises should never be empty"); + } + const result = await p.p; + promises.shift(); + if (result.done) { + await callbackRes; // Clean up, may be removed in the future + // // Cleanup promises - shouldn't be needed due to above await + // for (const promise of promises) { + // promise.resolve(result); + // await promise.p; + // } + return result.value; + } + yield result.value; + } + + // So TS doesn't complain + throw new Error("Unreachable"); +} diff --git a/node_modules/@huggingface/hub/src/utils/formatBytes.spec.ts b/node_modules/@huggingface/hub/src/utils/formatBytes.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..20dfa1e57d7382f6a4218fed04ace55725e2ecdb --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/formatBytes.spec.ts @@ -0,0 +1,23 @@ +import { describe, it, expect } from "vitest"; +import { formatBytes } from "./formatBytes"; + +describe("formatBytes", () => { + it("uses SI units (multiples of 1000)", () => { + expect(formatBytes(0)).toBe("0 B"); + expect(formatBytes(999)).toBe("999 B"); + expect(formatBytes(1000)).toBe("1.00 kB"); + expect(formatBytes(1_500)).toBe("1.50 kB"); + expect(formatBytes(1_000_000)).toBe("1.00 MB"); + expect(formatBytes(5_300_000_000)).toBe("5.30 GB"); + }); + + it("adjusts precision based on magnitude", () => { + expect(formatBytes(12_300)).toBe("12.3 kB"); + expect(formatBytes(123_000)).toBe("123 kB"); + }); + + it("handles invalid inputs gracefully", () => { + expect(formatBytes(NaN)).toBe("NaN B"); + expect(formatBytes(-1)).toBe("-1 B"); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/formatBytes.ts b/node_modules/@huggingface/hub/src/utils/formatBytes.ts new file mode 100644 index 0000000000000000000000000000000000000000..d704eb6a4645857fb60177bf3fd895f1946ad824 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/formatBytes.ts @@ -0,0 +1,19 @@ +/** + * Format a byte count using SI units (multiples of 1000, e.g. `1.2 GB`). + * + * Negative or non-finite inputs are returned as `" B"` without unit conversion. + */ +export function formatBytes(bytes: number): string { + if (!Number.isFinite(bytes) || bytes < 0) { + return `${bytes} B`; + } + const units = ["B", "kB", "MB", "GB", "TB", "PB"]; + let value = bytes; + let i = 0; + while (value >= 1000 && i < units.length - 1) { + value /= 1000; + i++; + } + const formatted = i === 0 ? value.toString() : value.toFixed(value >= 100 ? 0 : value >= 10 ? 1 : 2); + return `${formatted} ${units[i]}`; +} diff --git a/node_modules/@huggingface/hub/src/utils/hexFromBytes.ts b/node_modules/@huggingface/hub/src/utils/hexFromBytes.ts new file mode 100644 index 0000000000000000000000000000000000000000..a6c331c09525a0906333e7007e5724858795aa35 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/hexFromBytes.ts @@ -0,0 +1,11 @@ +export function hexFromBytes(arr: Uint8Array): string { + if (globalThis.Buffer) { + return globalThis.Buffer.from(arr).toString("hex"); + } else { + const bin: string[] = []; + arr.forEach((byte) => { + bin.push(byte.toString(16).padStart(2, "0")); + }); + return bin.join(""); + } +} diff --git a/node_modules/@huggingface/hub/src/utils/insecureRandomString.ts b/node_modules/@huggingface/hub/src/utils/insecureRandomString.ts new file mode 100644 index 0000000000000000000000000000000000000000..f9954d431b26374eb47d4b81f10a5837990a6e90 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/insecureRandomString.ts @@ -0,0 +1,3 @@ +export function insecureRandomString(): string { + return Math.random().toString(36).slice(2); +} diff --git a/node_modules/@huggingface/hub/src/utils/isBackend.ts b/node_modules/@huggingface/hub/src/utils/isBackend.ts new file mode 100644 index 0000000000000000000000000000000000000000..1e6f27998645f2971dcdd92503d78de521273a26 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/isBackend.ts @@ -0,0 +1,6 @@ +const isBrowser = typeof window !== "undefined" && typeof window.document !== "undefined"; + +const isWebWorker = + typeof self === "object" && self.constructor && self.constructor.name === "DedicatedWorkerGlobalScope"; + +export const isBackend = !isBrowser && !isWebWorker; diff --git a/node_modules/@huggingface/hub/src/utils/isFrontend.ts b/node_modules/@huggingface/hub/src/utils/isFrontend.ts new file mode 100644 index 0000000000000000000000000000000000000000..0b9bab392e71f315704c210bc0e8ff210379703d --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/isFrontend.ts @@ -0,0 +1,3 @@ +import { isBackend } from "./isBackend"; + +export const isFrontend = !isBackend; diff --git a/node_modules/@huggingface/hub/src/utils/mergeAsyncGenerators.spec.ts b/node_modules/@huggingface/hub/src/utils/mergeAsyncGenerators.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..dc1bfdc6d87a1a4b4283fd13e24852cb132926c2 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/mergeAsyncGenerators.spec.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from "vitest"; +import { mergeAsyncGenerators } from "./mergeAsyncGenerators"; +import { splitAsyncGenerator } from "./splitAsyncGenerator"; + +describe("mergeAsyncGenerators", () => { + const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + it("should merge multiple async generators", async () => { + const generator1 = (async function* () { + yield 1; + yield 2; + await sleep(250); + yield 3; + })(); + const generator2 = (async function* () { + await sleep(100); + yield 4; + yield 5; + yield 6; + })(); + const generator3 = (async function* () { + await sleep(200); + yield 7; + yield 8; + yield 9; + })(); + + const results: number[] = []; + + for await (const result of mergeAsyncGenerators([generator1, generator2, generator3])) { + results.push(result); + } + expect(results).toEqual([1, 2, 4, 5, 6, 7, 8, 9, 3]); + }); + + it("should merge multiple async generators from a single source", async () => { + const source = (async function* () { + yield 1; + yield 2; + yield 3; + yield 4; + yield 5; + yield 6; + yield 7; + yield 8; + yield 9; + })(); + const sources = splitAsyncGenerator(source, 3); + + const generator1 = (async function* () { + for await (const result of sources[0]) { + yield { result, gen: 1 }; + await sleep(100); + } + })(); + + const generator2 = (async function* () { + await sleep(50); + for await (const result of sources[1]) { + yield { result, gen: 2 }; + await sleep(100); + } + })(); + + const generator3 = (async function* () { + await sleep(80); + let count = 0; + for await (const result of sources[2]) { + yield { result, gen: 3 }; + count++; + + if (count >= 2) { + return; + } + } + })(); + + const results: { result: number; gen: number }[] = []; + for await (const result of mergeAsyncGenerators([generator1, generator2, generator3])) { + results.push(result); + } + expect(results.length).toBe(9); + expect(results).toEqual([ + { result: 1, gen: 1 }, + { result: 2, gen: 2 }, + { result: 3, gen: 3 }, + { result: 4, gen: 3 }, + { result: 5, gen: 1 }, + { result: 6, gen: 2 }, + { result: 7, gen: 1 }, + { result: 8, gen: 2 }, + { result: 9, gen: 1 }, + ]); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/mergeAsyncGenerators.ts b/node_modules/@huggingface/hub/src/utils/mergeAsyncGenerators.ts new file mode 100644 index 0000000000000000000000000000000000000000..984549f26c87046474c510565b81436e128355a1 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/mergeAsyncGenerators.ts @@ -0,0 +1,34 @@ +/** + * Merge outputs of multiple async generators. + */ +export async function* mergeAsyncGenerators(generators: AsyncGenerator[]): AsyncGenerator { + const executing: Promise<{ result: IteratorResult; gen: AsyncGenerator }>[] = []; + + const generatorSymbol = Symbol("generator"); + + for (const gen of generators) { + const p = gen.next().then((result) => ({ result, gen })); + Object.defineProperty(p, generatorSymbol, { + value: gen, + }); + executing.push(p); + } + + while (executing.length > 0) { + const next = await Promise.race(executing); + const { result, gen } = next; + + const index = executing.findIndex((p) => Object.getOwnPropertyDescriptor(p, generatorSymbol)?.value === gen); + + if (result.done) { + executing.splice(index, 1); + continue; + } + + yield result.value; + executing[index] = gen.next().then((result) => ({ result, gen })); + Object.defineProperty(executing[index], generatorSymbol, { + value: gen, + }); + } +} diff --git a/node_modules/@huggingface/hub/src/utils/normalizeInferenceProviderMapping.ts b/node_modules/@huggingface/hub/src/utils/normalizeInferenceProviderMapping.ts new file mode 100644 index 0000000000000000000000000000000000000000..471997f2de0cce1b826a18c5df6b7713bfa6be95 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/normalizeInferenceProviderMapping.ts @@ -0,0 +1,36 @@ +import type { WidgetType } from "@huggingface/tasks"; +import type { ApiModelInferenceProviderMappingEntry } from "../types/api/api-model"; + +/** + * Normalize inferenceProviderMapping to always return an array format. + * + * Little hack to simplify Inference Providers logic and make it backward and forward compatible. + * Right now, API returns a dict on model-info and a list on list-models. Let's harmonize to list. + */ +export function normalizeInferenceProviderMapping( + hfModelId: string, + inferenceProviderMapping?: + | ApiModelInferenceProviderMappingEntry[] + | Record, +): ApiModelInferenceProviderMappingEntry[] { + if (!inferenceProviderMapping) { + return []; + } + + // If it's already an array, return it as is + if (Array.isArray(inferenceProviderMapping)) { + return inferenceProviderMapping.map((entry) => ({ + ...entry, + hfModelId, + })); + } + + // Convert mapping to array format + return Object.entries(inferenceProviderMapping).map(([provider, mapping]) => ({ + provider, + hfModelId, + providerId: mapping.providerId, + status: mapping.status, + task: mapping.task, + })); +} diff --git a/node_modules/@huggingface/hub/src/utils/omit.ts b/node_modules/@huggingface/hub/src/utils/omit.ts new file mode 100644 index 0000000000000000000000000000000000000000..e37f54b8844415b1d0573ae51117906c2e339ccc --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/omit.ts @@ -0,0 +1,14 @@ +import { pick } from "./pick"; +import { typedInclude } from "./typedInclude"; + +/** + * Return copy of object, omitting blacklisted array of props + */ +export function omit, K extends keyof T>( + o: T, + props: K[] | K, +): Pick> { + const propsArr = Array.isArray(props) ? props : [props]; + const letsKeep = (Object.keys(o) as (keyof T)[]).filter((prop) => !typedInclude(propsArr, prop)); + return pick(o, letsKeep); +} diff --git a/node_modules/@huggingface/hub/src/utils/parseLinkHeader.ts b/node_modules/@huggingface/hub/src/utils/parseLinkHeader.ts new file mode 100644 index 0000000000000000000000000000000000000000..6939a89be6701240097f8b2f7f6c581b4ab6c739 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/parseLinkHeader.ts @@ -0,0 +1,8 @@ +/** + * Parse Link HTTP header, eg `; rel="next"` + */ +export function parseLinkHeader(header: string): Record { + const regex = /<(https?:[/][/][^>]+)>;\s+rel="([^"]+)"/g; + + return Object.fromEntries([...header.matchAll(regex)].map(([, url, rel]) => [rel, url])); +} diff --git a/node_modules/@huggingface/hub/src/utils/pick.ts b/node_modules/@huggingface/hub/src/utils/pick.ts new file mode 100644 index 0000000000000000000000000000000000000000..a47a57f861c00d3917d2065c49d09016a14b25e7 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/pick.ts @@ -0,0 +1,13 @@ +/** + * Return copy of object, only keeping whitelisted properties. + */ +export function pick(o: T, props: K[] | ReadonlyArray): Pick { + return Object.assign( + {}, + ...props.map((prop) => { + if (o[prop] !== undefined) { + return { [prop]: o[prop] }; + } + }), + ); +} diff --git a/node_modules/@huggingface/hub/src/utils/promisesQueue.spec.ts b/node_modules/@huggingface/hub/src/utils/promisesQueue.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..3e9ea6d124ff5cc290eb6aefffa4ce8148b3c46a --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/promisesQueue.spec.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { promisesQueue } from "./promisesQueue"; + +describe("promisesQueue", () => { + it("should handle multiple errors without triggering an uncaughtException", async () => { + const factories = [ + () => Promise.reject(new Error("error 1")), + () => Promise.reject(new Error("error 2")), + () => Promise.reject(new Error("error 3")), + ]; + + try { + await promisesQueue(factories, 10); + } catch (err) { + if (!(err instanceof Error)) { + throw err; + } + } + + try { + await promisesQueue(factories, 1); + } catch (err) { + if (!(err instanceof Error)) { + throw err; + } + expect(err.message).toBe("error 1"); + } + }); + + it("should return ordered results", async () => { + const factories = [ + () => Promise.resolve(1), + () => Promise.resolve(2), + () => Promise.resolve(3), + () => Promise.resolve(4), + () => Promise.resolve(5), + () => Promise.resolve(6), + () => Promise.resolve(7), + () => Promise.resolve(8), + () => Promise.resolve(9), + () => Promise.resolve(10), + ]; + + const results = await promisesQueue(factories, 3); + + expect(results).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/promisesQueue.ts b/node_modules/@huggingface/hub/src/utils/promisesQueue.ts new file mode 100644 index 0000000000000000000000000000000000000000..35d2d06907e3aad537e96f99509297f79d5ee547 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/promisesQueue.ts @@ -0,0 +1,23 @@ +/** + * Execute queue of promises. + * + * Inspired by github.com/rxaviers/async-pool + */ +export async function promisesQueue(factories: (() => Promise)[], concurrency: number): Promise { + const results: T[] = []; + const executing: Set> = new Set(); + let index = 0; + for (const factory of factories) { + const closureIndex = index++; + const e = factory().then((r) => { + results[closureIndex] = r; + executing.delete(e); + }); + executing.add(e); + if (executing.size >= concurrency) { + await Promise.race(executing); + } + } + await Promise.all(executing); + return results; +} diff --git a/node_modules/@huggingface/hub/src/utils/promisesQueueStreaming.ts b/node_modules/@huggingface/hub/src/utils/promisesQueueStreaming.ts new file mode 100644 index 0000000000000000000000000000000000000000..72f441afbce2538c22b77a5a9d9e89f0d8e0ba22 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/promisesQueueStreaming.ts @@ -0,0 +1,25 @@ +/** + * Execute queue of promises in a streaming fashion. + * + * Optimized for streaming: + * - Expects an iterable as input + * - Does not return a list of all results + * + * Inspired by github.com/rxaviers/async-pool + */ +export async function promisesQueueStreaming( + factories: AsyncIterable<() => Promise> | Iterable<() => Promise>, + concurrency: number, +): Promise { + const executing: Promise[] = []; + for await (const factory of factories) { + const e = factory().then(() => { + executing.splice(executing.indexOf(e), 1); + }); + executing.push(e); + if (executing.length >= concurrency) { + await Promise.race(executing); + } + } + await Promise.all(executing); +} diff --git a/node_modules/@huggingface/hub/src/utils/range.ts b/node_modules/@huggingface/hub/src/utils/range.ts new file mode 100644 index 0000000000000000000000000000000000000000..d7ebababf2ba06366899f5d192b08e849b708f9a --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/range.ts @@ -0,0 +1,13 @@ +/** + * One param: create list of integers from 0 (inclusive) to n (exclusive) + * Two params: create list of integers from a (inclusive) to b (exclusive) + */ +export function range(n: number, b?: number): number[] { + return b + ? Array(b - n) + .fill(0) + .map((_, i) => n + i) + : Array(n) + .fill(0) + .map((_, i) => i); +} diff --git a/node_modules/@huggingface/hub/src/utils/sha256-node.ts b/node_modules/@huggingface/hub/src/utils/sha256-node.ts new file mode 100644 index 0000000000000000000000000000000000000000..a7c75ad7765db2246c327de8fcc2d7096b68033f --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/sha256-node.ts @@ -0,0 +1,26 @@ +import { Readable } from "node:stream"; +import type { ReadableStream } from "node:stream/web"; +import { createHash } from "node:crypto"; + +export async function* sha256Node( + buffer: ArrayBuffer | Blob, + opts?: { + abortSignal?: AbortSignal; + }, +): AsyncGenerator { + const sha256Stream = createHash("sha256"); + const size = buffer instanceof Blob ? buffer.size : buffer.byteLength; + let done = 0; + const readable = + buffer instanceof Blob ? Readable.fromWeb(buffer.stream() as ReadableStream) : Readable.from(Buffer.from(buffer)); + + for await (const buffer of readable) { + sha256Stream.update(buffer); + done += buffer.length; + yield done / size; + + opts?.abortSignal?.throwIfAborted(); + } + + return sha256Stream.digest("hex"); +} diff --git a/node_modules/@huggingface/hub/src/utils/sha256.spec.ts b/node_modules/@huggingface/hub/src/utils/sha256.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..8e62b936b60d94debe9b91103f30eeedc1826712 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/sha256.spec.ts @@ -0,0 +1,50 @@ +import { describe, it, expect } from "vitest"; +import { sha256 } from "./sha256"; + +const smallContent = "hello world"; +const smallContentSHA256 = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"; +const bigContent = "O123456789".repeat(100_000); +const bigContentSHA256 = "a3bbce7ee1df7233d85b5f4d60faa3755f93f537804f8b540c72b0739239ddf8"; +const biggerContent = "0123456789".repeat(1_000_000); +const biggerContentSHA256 = "d52fcc26b48dbd4d79b125eb0a29b803ade07613c67ac7c6f2751aefef008486"; + +describe("sha256", () => { + async function calcSHA256(content: string, useWebWorker: boolean) { + const iterator = sha256(new Blob([content]), { useWebWorker }); + let res: IteratorResult; + do { + res = await iterator.next(); + } while (!res.done); + return res.value; + } + + it("Calculate hash of a small file", async () => { + const sha = await calcSHA256(smallContent, false); + expect(sha).toBe(smallContentSHA256); + }); + + it("Calculate hash of a big file", async () => { + const sha = await calcSHA256(bigContent, false); + expect(sha).toBe(bigContentSHA256); + }); + + it("Calculate hash of a bigger file", async () => { + const sha = await calcSHA256(biggerContent, false); + expect(sha).toBe(biggerContentSHA256); + }); + + it("Calculate hash of a small file (+ web worker)", async () => { + const sha = await calcSHA256(smallContent, true); + expect(sha).toBe(smallContentSHA256); + }); + + it("Calculate hash of a big file (+ web worker)", async () => { + const sha = await calcSHA256(bigContent, true); + expect(sha).toBe(bigContentSHA256); + }); + + it("Calculate hash of a bigger file (+ web worker)", async () => { + const sha = await calcSHA256(biggerContent, true); + expect(sha).toBe(biggerContentSHA256); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/sha256.ts b/node_modules/@huggingface/hub/src/utils/sha256.ts new file mode 100644 index 0000000000000000000000000000000000000000..0458d0495c3489e5f1bebf706282404b128c65dd --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/sha256.ts @@ -0,0 +1,206 @@ +import { eventToGenerator } from "./eventToGenerator"; +import { hexFromBytes } from "./hexFromBytes"; +import { isFrontend } from "./isFrontend"; + +async function getWebWorkerCode() { + const sha256Module = await import("../vendor/hash-wasm/sha256-wrapper"); + return URL.createObjectURL(new Blob([sha256Module.createSHA256WorkerCode()])); +} + +const pendingWorkers: Worker[] = []; +const runningWorkers: Set = new Set(); + +let resolve: () => void; +let waitPromise: Promise = new Promise((r) => { + resolve = r; +}); + +async function getWorker(poolSize?: number): Promise { + { + const worker = pendingWorkers.pop(); + if (worker) { + runningWorkers.add(worker); + return worker; + } + } + if (!poolSize) { + const worker = new Worker(await getWebWorkerCode()); + runningWorkers.add(worker); + return worker; + } + + if (poolSize <= 0) { + throw new TypeError("Invalid webworker pool size: " + poolSize); + } + + while (runningWorkers.size >= poolSize) { + await waitPromise; + } + + const worker = new Worker(await getWebWorkerCode()); + runningWorkers.add(worker); + return worker; +} + +async function freeWorker(worker: Worker, poolSize: number | undefined): Promise { + if (!poolSize) { + return destroyWorker(worker); + } + runningWorkers.delete(worker); + pendingWorkers.push(worker); + const r = resolve; + waitPromise = new Promise((r) => { + resolve = r; + }); + r(); +} + +function destroyWorker(worker: Worker): void { + runningWorkers.delete(worker); + worker.terminate(); + const r = resolve; + waitPromise = new Promise((r) => { + resolve = r; + }); + r(); +} + +/** + * @returns hex-encoded sha + * @yields progress (0-1) + */ +export async function* sha256( + buffer: Blob, + opts?: { useWebWorker?: boolean | { minSize?: number; poolSize?: number }; abortSignal?: AbortSignal }, +): AsyncGenerator { + yield 0; + + const maxCryptoSize = + typeof opts?.useWebWorker === "object" && opts?.useWebWorker.minSize !== undefined + ? opts.useWebWorker.minSize + : 10_000_000; + if (buffer.size < maxCryptoSize && globalThis.crypto?.subtle) { + const res = hexFromBytes( + new Uint8Array( + await globalThis.crypto.subtle.digest("SHA-256", buffer instanceof Blob ? await buffer.arrayBuffer() : buffer), + ), + ); + + yield 1; + + return res; + } + + if (isFrontend) { + if (opts?.useWebWorker) { + try { + const poolSize = typeof opts?.useWebWorker === "object" ? opts.useWebWorker.poolSize : undefined; + const worker = await getWorker(poolSize); + + // Define handlers to allow removal + let messageHandler: (event: MessageEvent) => void; + let errorHandler: (event: ErrorEvent) => void; + + const cleanup = () => { + worker.removeEventListener("message", messageHandler); + worker.removeEventListener("error", errorHandler); + }; + + return yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { + messageHandler = (event: MessageEvent) => { + if (event.data.sha256) { + cleanup(); + freeWorker(worker, poolSize); + returnCallback(event.data.sha256); + } else if (event.data.progress) { + yieldCallback(event.data.progress); + + try { + opts.abortSignal?.throwIfAborted(); + } catch (err) { + cleanup(); + destroyWorker(worker); + rejectCallback(err); + } + } else { + cleanup(); + destroyWorker(worker); + rejectCallback(event); + } + }; + + errorHandler = (event: ErrorEvent) => { + cleanup(); + destroyWorker(worker); + rejectCallback(event.error); + }; + + // Handle external abort signal if it aborts before any worker message + if (opts?.abortSignal) { + try { + opts.abortSignal?.throwIfAborted(); + } catch (err) { + cleanup(); + destroyWorker(worker); + rejectCallback(opts.abortSignal.reason ?? new DOMException("Aborted", "AbortError")); + return; + } + + const abortListener = () => { + cleanup(); + destroyWorker(worker); + + rejectCallback(opts.abortSignal?.reason ?? new DOMException("Aborted", "AbortError")); + opts.abortSignal?.removeEventListener("abort", abortListener); + }; + + opts.abortSignal.addEventListener("abort", abortListener); + } + + worker.addEventListener("message", messageHandler); + worker.addEventListener("error", errorHandler); + worker.postMessage({ file: buffer }); + }); + } catch (err) { + console.warn("Failed to use web worker for sha256", err); + } + } + if (!wasmModule) { + wasmModule = await import("../vendor/hash-wasm/sha256-wrapper"); + } + + const sha256 = await wasmModule.createSHA256(); + sha256.init(); + + const reader = buffer.stream().getReader(); + const total = buffer.size; + let bytesDone = 0; + + while (true) { + const { done, value } = await reader.read(); + + if (done) { + break; + } + + sha256.update(value); + bytesDone += value.length; + yield bytesDone / total; + + opts?.abortSignal?.throwIfAborted(); + } + + return sha256.digest("hex"); + } + + if (!cryptoModule) { + cryptoModule = await import("./sha256-node"); + } + + return yield* cryptoModule.sha256Node(buffer, { abortSignal: opts?.abortSignal }); +} + +// eslint-disable-next-line @typescript-eslint/consistent-type-imports +let cryptoModule: typeof import("./sha256-node"); +// eslint-disable-next-line @typescript-eslint/consistent-type-imports +let wasmModule: typeof import("../vendor/hash-wasm/sha256-wrapper"); diff --git a/node_modules/@huggingface/hub/src/utils/shardParser.spec.ts b/node_modules/@huggingface/hub/src/utils/shardParser.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..76ab7de588b7a64c23777879b85f4d1d97684e35 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/shardParser.spec.ts @@ -0,0 +1,25 @@ +import { parseShardData } from "./shardParser"; +import { readFile } from "fs/promises"; +import { expect, describe, it } from "vitest"; +import { hmac, hashToHex, hexToBytes } from "@huggingface/xetchunk-wasm"; + +describe("shardParser", () => { + it("should parse a shard", async () => { + const buffer = await readFile("tests/gpt2-64-8bits.tflite.shard"); + const shard = await parseShardData(new Blob([buffer])); + const expectedJson = JSON.parse(await readFile("tests/gpt2-64-8bits.tflite.shard.json", "utf-8")); + + expect(shard.hmacKey).toBe(expectedJson.hmac_key); + expect(shard.xorbs.length).toEqual(expectedJson.xorbs.length); + for (let i = 0; i < shard.xorbs.length; i++) { + expect(shard.xorbs[i].hash).toEqual(expectedJson.xorbs[i].hash); + expect(shard.xorbs[i].chunks.length).toEqual(expectedJson.xorbs[i].chunk_hashes.length); + for (let j = 0; j < shard.xorbs[i].chunks.length; j++) { + expect(shard.xorbs[i].chunks[j].hash).toEqual(expectedJson.xorbs[i].chunk_hashes[j]); + } + } + + const chunkHash = "9502eec19d4b0c9f7b389228fa801f68ecdf15d69ccd1da2f9ddbd0219898335"; + expect(hashToHex(hmac(hexToBytes(chunkHash), hexToBytes(shard.hmacKey)))).toEqual(shard.xorbs[1].chunks[0].hash); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/shardParser.ts b/node_modules/@huggingface/hub/src/utils/shardParser.ts new file mode 100644 index 0000000000000000000000000000000000000000..e1a45d40c9a4474db3b7758b72bf37aa6da67d21 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/shardParser.ts @@ -0,0 +1,149 @@ +import { SHARD_FOOTER_VERSION, SHARD_HEADER_VERSION, SHARD_MAGIC_TAG } from "./uploadShards"; + +const HASH_LENGTH = 32; +const XORB_HASH_BOOKEND = "ff".repeat(HASH_LENGTH); + +// Read 4 uint64 in little endian and convert to hex +function readHashFromArray(array: Uint8Array, offset: number): string { + let hash = ""; + for (let i = 0; i < HASH_LENGTH; i += 8) { + hash += `${array[offset + i + 7].toString(16).padStart(2, "0")}${array[offset + i + 6] + .toString(16) + .padStart(2, "0")}${array[offset + i + 5].toString(16).padStart(2, "0")}${array[offset + i + 4] + .toString(16) + .padStart(2, "0")}${array[offset + i + 3].toString(16).padStart(2, "0")}${array[offset + i + 2] + .toString(16) + .padStart(2, "0")}${array[offset + i + 1].toString(16).padStart(2, "0")}${array[offset + i] + .toString(16) + .padStart(2, "0")}`; + } + return hash; +} + +export interface ShardData { + hmacKey: string; + xorbs: Array<{ + hash: string; + chunks: Array<{ + hash: string; + startOffset: number; + unpackedLength: number; + }>; + }>; +} + +export async function parseShardData(shardBlob: Blob): Promise { + const shard = new Uint8Array(await shardBlob.arrayBuffer()); + const shardView = new DataView(shard.buffer); + + const magicTag = shard.slice(0, SHARD_MAGIC_TAG.length); + if (!magicTag.every((byte, i) => byte === SHARD_MAGIC_TAG[i])) { + throw new Error("Invalid shard magic tag"); + } + + const version = shardView.getBigUint64(SHARD_MAGIC_TAG.length, true); + if (version !== SHARD_HEADER_VERSION) { + throw new Error(`Invalid shard version: ${version}`); + } + + const footerSize = Number(shardView.getBigUint64(SHARD_MAGIC_TAG.length + 8, true)); + + // Read footer to get section offsets + const footerStart = shard.length - footerSize; + const footerVersion = shardView.getBigUint64(footerStart, true); + if (footerVersion !== SHARD_FOOTER_VERSION) { + throw new Error(`Invalid shard footer version: ${footerVersion}`); + } + + // version: u64, // Footer version (must be 1) + // file_info_offset: u64, // Offset to file info section + // cas_info_offset: u64, // Offset to CAS info section + // file_lookup_offset: u64, // Offset to file lookup table + // file_lookup_num_entry: u64, // Number of file lookup entries + // cas_lookup_offset: u64, // Offset to CAS lookup table + // cas_lookup_num_entry: u64, // Number of CAS lookup entries + // chunk_lookup_offset: u64, // Offset to chunk lookup table + // chunk_lookup_num_entry: u64, // Number of chunk lookup entries + // chunk_hash_hmac_key: [u64; 4], // HMAC key for chunk hashes (32 bytes) + // shard_creation_timestamp: u64, // Creation time (seconds since epoch) + // shard_key_expiry: u64, // Expiry time (seconds since epoch) + // _buffer: [u64; 6], // Reserved space (48 bytes) + // stored_bytes_on_disk: u64, // Total bytes stored on disk + // materialized_bytes: u64, // Total materialized bytes + // stored_bytes: u64, // Total stored bytes + // footer_offset: u64, + + // const fileInfoStart = Number(shardView.getBigUint64(footerStart + 8, true)); + const xorbInfoStart = Number(shardView.getBigUint64(footerStart + 16, true)); + const fileLookupStart = Number(shardView.getBigUint64(footerStart + 24, true)); + // const numFileLookups = Number(shardView.getBigUint64(footerStart + 32, true)); + // const xorbLookupStart = Number(shardView.getBigUint64(footerStart + 40, true)); + // const numXorbLookups = Number(shardView.getBigUint64(footerStart + 48, true)); + // const chunkLookupStart = Number(shardView.getBigUint64(footerStart + 56, true)); + // const numChunkLookups = Number(shardView.getBigUint64(footerStart + 64, true)); + const hmacKey = readHashFromArray(shard, footerStart + 72); + // const shardCreationTimestamp = Number(shardView.getBigUint64(footerStart + 104, true)); + // const shardKeyExpiry = Number(shardView.getBigUint64(footerStart + 112, true)); + // const storedBytesOnDisk = Number(shardView.getBigUint64(footerStart + 168, true)); + // const materializedBytes = Number(shardView.getBigUint64(footerStart + 176, true)); + // const storedBytes = Number(shardView.getBigUint64(footerStart + 184, true)); + // const footerOffset = Number(shardView.getBigUint64(footerStart + 192, true)); + + // Parse XORB Info Section + const xorbs: ShardData["xorbs"] = []; + let offset = xorbInfoStart; + + while (offset < fileLookupStart) { + // Read xorb entry + const xorbHash = readHashFromArray(shard, offset); + offset += HASH_LENGTH; + + if (xorbHash === XORB_HASH_BOOKEND) { + break; + } + + // const flags = shardView.getUint32(offset, true); + offset += 4; + + const chunkCount = shardView.getUint32(offset, true); + offset += 4; + + // const numBytesInXorb = shardView.getUint32(offset, true); + offset += 4; + + // const numBytesUnpacked = shardView.getUint32(offset, true); + offset += 4; + + // Read chunks for this xorb + const chunks: ShardData["xorbs"][number]["chunks"] = []; + for (let i = 0; i < chunkCount; i++) { + const chunkHash = readHashFromArray(shard, offset); + offset += HASH_LENGTH; + + const startOffset = shardView.getUint32(offset, true); + offset += 4; + + const length = shardView.getUint32(offset, true); + offset += 4; + + // Skip reserved 8 bytes + offset += 8; + + chunks.push({ + hash: chunkHash, + startOffset, + unpackedLength: length, + }); + } + + xorbs.push({ + hash: xorbHash, + chunks, + }); + } + + return { + hmacKey, + xorbs, + }; +} diff --git a/node_modules/@huggingface/hub/src/utils/splitAsyncGenerator.ts b/node_modules/@huggingface/hub/src/utils/splitAsyncGenerator.ts new file mode 100644 index 0000000000000000000000000000000000000000..b077e9abfcb476a5a5785a113fcf7e1098194daa --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/splitAsyncGenerator.ts @@ -0,0 +1,43 @@ +/** + * Split an async generator into multiple async generators, all drawing from the same source. + */ +export function splitAsyncGenerator(source: AsyncGenerator, n: number): Array> { + if (n <= 0) { + return []; + } + + const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + + let takenIndex: number | null = null; + const generators: AsyncGenerator[] = []; + let remaining = n; + for (let i = 0; i < n; i++) { + generators.push({ + next: async () => { + while (takenIndex !== null) { + await sleep(1); + } + takenIndex = i; + return source.next().then((r) => { + takenIndex = null; + return r; + }); + }, + return: async () => { + remaining--; + if (remaining === 0) { + return source.return(undefined); + } + return { + done: true, + value: undefined, + }; + }, + throw: async (error) => { + return source.throw(error); + }, + [Symbol.asyncIterator]: () => generators[i], + }); + } + return generators; +} diff --git a/node_modules/@huggingface/hub/src/utils/sub-paths.spec.ts b/node_modules/@huggingface/hub/src/utils/sub-paths.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..6dcb773ab7f106e8bcf1d09751ff01b1e0daceba --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/sub-paths.spec.ts @@ -0,0 +1,39 @@ +import { mkdir, writeFile } from "fs/promises"; +import { tmpdir } from "os"; +import { describe, expect, it } from "vitest"; +import { subPaths } from "./sub-paths"; +import { pathToFileURL } from "url"; + +describe("sub-paths", () => { + it("should retrieve all sub-paths of a directory", async () => { + const tmpDir = tmpdir(); + + await mkdir(`${tmpDir}/test-dir/sub`, { recursive: true }); + + await writeFile(`${tmpDir}/test-dir/sub/file1.txt`, "file1"); + await writeFile(`${tmpDir}/test-dir/sub/file2.txt`, "file2"); + await writeFile(`${tmpDir}/test-dir/file3.txt`, "file3"); + await writeFile(`${tmpDir}/test-dir/file4.txt`, "file4"); + const result = await subPaths(pathToFileURL(`${tmpDir}/test-dir`)); + + expect(result).toEqual([ + { + path: pathToFileURL(`${tmpDir}/test-dir/file3.txt`), + relativePath: "file3.txt", + }, + { + path: pathToFileURL(`${tmpDir}/test-dir/file4.txt`), + relativePath: "file4.txt", + }, + + { + path: pathToFileURL(`${tmpDir}/test-dir/sub/file1.txt`), + relativePath: "sub/file1.txt", + }, + { + path: pathToFileURL(`${tmpDir}/test-dir/sub/file2.txt`), + relativePath: "sub/file2.txt", + }, + ]); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/sub-paths.ts b/node_modules/@huggingface/hub/src/utils/sub-paths.ts new file mode 100644 index 0000000000000000000000000000000000000000..c15e73393ad2a78e782e5f6b5eb0d1e37d6e9a6f --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/sub-paths.ts @@ -0,0 +1,38 @@ +import { readdir, stat } from "node:fs/promises"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +/** + * Recursively retrieves all sub-paths of a given directory up to a specified depth. + */ +export async function subPaths( + path: URL, + maxDepth = 10, +): Promise< + Array<{ + path: URL; + relativePath: string; + }> +> { + const state = await stat(path); + if (!state.isDirectory()) { + return [{ path, relativePath: "." }]; + } + + const files = await readdir(path, { withFileTypes: true }); + const ret: Array<{ path: URL; relativePath: string }> = []; + for (const file of files) { + const filePath = pathToFileURL(fileURLToPath(path) + "/" + file.name); + if (file.isDirectory()) { + ret.push( + ...(await subPaths(filePath, maxDepth - 1)).map((subPath) => ({ + ...subPath, + relativePath: `${file.name}/${subPath.relativePath}`, + })), + ); + } else { + ret.push({ path: filePath, relativePath: file.name }); + } + } + + return ret; +} diff --git a/node_modules/@huggingface/hub/src/utils/sum.ts b/node_modules/@huggingface/hub/src/utils/sum.ts new file mode 100644 index 0000000000000000000000000000000000000000..9d3fe6f15960f0f373a4b8669cace09c41c4b02f --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/sum.ts @@ -0,0 +1,6 @@ +/** + * Sum of elements in array + */ +export function sum(arr: number[]): number { + return arr.reduce((a, b) => a + b, 0); +} diff --git a/node_modules/@huggingface/hub/src/utils/symlink.spec.ts b/node_modules/@huggingface/hub/src/utils/symlink.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..56edaf4f33c170389d4799cdec7b9c59686f1a63 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/symlink.spec.ts @@ -0,0 +1,89 @@ +/* eslint-disable @typescript-eslint/consistent-type-imports */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { describe, expect, it, vi } from "vitest"; +import { createSymlink } from "./symlink"; +import { readFileSync, writeFileSync } from "node:fs"; +import { lstat, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let failSymlink = false; +vi.mock("node:fs/promises", async (importOriginal) => ({ + ...(await importOriginal()), + symlink: async (...args: any[]) => { + if (failSymlink) { + failSymlink = false; + throw new Error("Symlink not supported"); + } + + // @ts-expect-error - ignore + return (await importOriginal()).symlink(...args); + }, +})); + +describe("utils/symlink", () => { + it("should create a symlink", async () => { + writeFileSync(join(tmpdir(), "test.txt"), "hello world"); + await createSymlink({ + sourcePath: join(tmpdir(), "test.txt"), + finalPath: join(tmpdir(), "test-symlink.txt"), + }); + + const stats = await lstat(join(tmpdir(), "test-symlink.txt")); + expect(stats.isSymbolicLink()).toBe(process.platform !== "win32"); + + // Test file content + const content = readFileSync(join(tmpdir(), "test-symlink.txt"), "utf8"); + expect(content).toBe("hello world"); + + // Cleanup + await rm(join(tmpdir(), "test-symlink.txt")); + await rm(join(tmpdir(), "test.txt")); + }); + + it("should work when symlinking twice", async () => { + writeFileSync(join(tmpdir(), "test.txt"), "hello world"); + writeFileSync(join(tmpdir(), "test2.txt"), "hello world2"); + await createSymlink({ + sourcePath: join(tmpdir(), "test.txt"), + finalPath: join(tmpdir(), "test-symlink.txt"), + }); + await createSymlink({ + sourcePath: join(tmpdir(), "test2.txt"), + finalPath: join(tmpdir(), "test-symlink.txt"), + }); + + const stats = await lstat(join(tmpdir(), "test-symlink.txt")); + expect(stats.isSymbolicLink()).toBe(process.platform !== "win32"); + + // Test file content + const content = readFileSync(join(tmpdir(), "test-symlink.txt"), "utf8"); + expect(content).toBe("hello world2"); + + // Cleanup + await rm(join(tmpdir(), "test-symlink.txt")); + await rm(join(tmpdir(), "test.txt")); + await rm(join(tmpdir(), "test2.txt")); + }); + + it("should work when symlink doesn't work (windows)", async () => { + writeFileSync(join(tmpdir(), "test.txt"), "hello world"); + + failSymlink = true; + await createSymlink({ + sourcePath: join(tmpdir(), "test.txt"), + finalPath: join(tmpdir(), "test-symlink.txt"), + }); + + const stats = await lstat(join(tmpdir(), "test-symlink.txt")); + expect(stats.isSymbolicLink()).toBe(false); + + // Test file content + const content = readFileSync(join(tmpdir(), "test-symlink.txt"), "utf8"); + expect(content).toBe("hello world"); + + // Cleanup + await rm(join(tmpdir(), "test-symlink.txt")); + await rm(join(tmpdir(), "test.txt")); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/symlink.ts b/node_modules/@huggingface/hub/src/utils/symlink.ts new file mode 100644 index 0000000000000000000000000000000000000000..17cedf66ad17629435dcd492238fc9a258068961 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/symlink.ts @@ -0,0 +1,65 @@ +/** + * Heavily inspired by https://github.com/huggingface/huggingface_hub/blob/fcfd14361bd03f23f82efced1aa65a7cbfa4b922/src/huggingface_hub/file_download.py#L517 + */ + +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import * as os from "node:os"; + +function expandUser(path: string): string { + if (path.startsWith("~")) { + return path.replace("~", os.homedir()); + } + return path; +} + +/** + * Create a symbolic link named dst pointing to src. + * + * By default, it will try to create a symlink using a relative path. Relative paths have 2 advantages: + * - If the cache_folder is moved (example: back-up on a shared drive), relative paths within the cache folder will + * not break. + * - Relative paths seems to be better handled on Windows. Issue was reported 3 times in less than a week when + * changing from relative to absolute paths. See https://github.com/huggingface/huggingface_hub/issues/1398, + * https://github.com/huggingface/diffusers/issues/2729 and https://github.com/huggingface/transformers/pull/22228. + * NOTE: The issue with absolute paths doesn't happen on admin mode. + * When creating a symlink from the cache to a local folder, it is possible that a relative path cannot be created. + * This happens when paths are not on the same volume. In that case, we use absolute paths. + * + * The result layout looks something like + * └── [ 128] snapshots + * ├── [ 128] 2439f60ef33a0d46d85da5001d52aeda5b00ce9f + * │ ├── [ 52] README.md -> ../../../blobs/d7edf6bd2a681fb0175f7735299831ee1b22b812 + * │ └── [ 76] pytorch_model.bin -> ../../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd + * + * If symlinks cannot be created on this platform (most likely to be Windows), the workaround is to avoid symlinks by + * having the actual file in `dst`. If it is a new file (`new_blob=True`), we move it to `dst`. If it is not a new file + * (`new_blob=False`), we don't know if the blob file is already referenced elsewhere. To avoid breaking existing + * cache, the file is duplicated on the disk. + */ +export async function createSymlink(params: { + /** + * The path to the symlink. + */ + finalPath: string; + /** + * The path the symlink should point to. + */ + sourcePath: string; +}): Promise { + const abs_src = path.resolve(expandUser(params.sourcePath)); + const abs_dst = path.resolve(expandUser(params.finalPath)); + + try { + await fs.rm(abs_dst); + } catch { + // ignore + } + + try { + await fs.symlink(path.relative(path.dirname(abs_dst), abs_src), abs_dst); + } catch { + console.info(`Symlink not supported. Copying file from ${abs_src} to ${abs_dst}`); + await fs.copyFile(abs_src, abs_dst); + } +} diff --git a/node_modules/@huggingface/hub/src/utils/toRepoId.ts b/node_modules/@huggingface/hub/src/utils/toRepoId.ts new file mode 100644 index 0000000000000000000000000000000000000000..7b332a455bdfa9d442ecfe13c9eaf98586987f58 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/toRepoId.ts @@ -0,0 +1,84 @@ +import type { RepoDesignation, RepoId } from "../types/public"; + +export function toRepoId(repo: RepoDesignation): RepoId { + if (typeof repo !== "string") { + return repo; + } + + if (repo.startsWith("model/") || repo.startsWith("models/")) { + throw new TypeError( + "A repo designation for a model should not start with 'models/', directly specify the model namespace / name", + ); + } + + if (repo.startsWith("space/")) { + throw new TypeError("Spaces should start with 'spaces/', plural, not 'space/'"); + } + + if (repo.startsWith("dataset/")) { + throw new TypeError("Datasets should start with 'datasets/', plural, not 'dataset/'"); + } + + if (repo.startsWith("bucket/")) { + throw new TypeError("Buckets should start with 'buckets/', plural, not 'bucket/'"); + } + + if (repo.startsWith("kernel/")) { + throw new TypeError("Kernels should start with 'kernels/', plural, not 'kernel/'"); + } + + const slashes = repo.split("/").length - 1; + + if (repo.startsWith("spaces/")) { + if (slashes !== 2) { + throw new TypeError("Space Id must include namespace and name of the space"); + } + + return { + type: "space", + name: repo.slice("spaces/".length), + }; + } + + if (repo.startsWith("datasets/")) { + if (slashes > 2) { + throw new TypeError("Too many slashes in repo designation: " + repo); + } + + return { + type: "dataset", + name: repo.slice("datasets/".length), + }; + } + + if (repo.startsWith("buckets/")) { + if (slashes !== 2) { + throw new TypeError("Bucket Id must include namespace and name of the bucket"); + } + + return { + type: "bucket", + name: repo.slice("buckets/".length), + }; + } + + if (repo.startsWith("kernels/")) { + if (slashes !== 2) { + throw new TypeError("Kernel Id must include namespace and name of the kernel"); + } + + return { + type: "kernel", + name: repo.slice("kernels/".length), + }; + } + + if (slashes > 1) { + throw new TypeError("Too many slashes in repo designation: " + repo); + } + + return { + type: "model", + name: repo, + }; +} diff --git a/node_modules/@huggingface/hub/src/utils/typedEntries.ts b/node_modules/@huggingface/hub/src/utils/typedEntries.ts new file mode 100644 index 0000000000000000000000000000000000000000..031ba7daa0cc381ce650224c040f67261fbd56c3 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/typedEntries.ts @@ -0,0 +1,5 @@ +import type { Entries } from "../vendor/type-fest/entries"; + +export function typedEntries>(obj: T): Entries { + return Object.entries(obj) as Entries; +} diff --git a/node_modules/@huggingface/hub/src/utils/typedInclude.ts b/node_modules/@huggingface/hub/src/utils/typedInclude.ts new file mode 100644 index 0000000000000000000000000000000000000000..71e2f7a7e111995a744589dd34cb090d9743ea16 --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/typedInclude.ts @@ -0,0 +1,3 @@ +export function typedInclude(arr: readonly T[], v: V): v is T { + return arr.includes(v as T); +} diff --git a/node_modules/@huggingface/hub/src/utils/uploadShards.spec.ts b/node_modules/@huggingface/hub/src/utils/uploadShards.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..8dd7d332ac3a000d07eac84956bf8985c02e622c --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/uploadShards.spec.ts @@ -0,0 +1,230 @@ +import { describe, expect, it, vi } from "vitest"; +import { SHARD_MAGIC_TAG, uploadShards } from "./uploadShards"; + +const MDB_FILE_FLAG_WITH_VERIFICATION = 0x80000000; +const MDB_FILE_FLAG_WITH_METADATA_EXT = 0x40000000; + +const HASH_LENGTH = 32; +const FILE_BOOKEND_LENGTH = 48; +const FILE_ENTRY_BASE_SIZE = 48; // hash + flags + rep length + reserved +const REPRESENTATION_ENTRY_SIZE = 48; +const VERIFICATION_ENTRY_SIZE = 48; +const METADATA_ENTRY_SIZE = 48; + +vi.mock("./createXorbs", () => ({ + createXorbs: vi.fn(async function* ( + source: AsyncGenerator<{ content: Blob; path: string; sha256?: string }>, + ): AsyncGenerator< + | { + event: "xorb"; + xorb: Uint8Array; + hash: string; + id: number; + chunks: Array<{ hash: string; length: number }>; + files: Array<{ path: string; progress: number; lastSentProgress: number }>; + } + | { + event: "file"; + path: string; + hash: string; + sha256?: string; + dedupRatio: number; + representation: Array<{ + xorbId: number | string; + indexStart: number; + indexEnd: number; + length: number; + rangeHash: string; + }>; + } + > { + for await (const file of source) { + yield { + event: "xorb", + xorb: new Uint8Array([1, 2, 3]), + hash: "1".repeat(64), + id: 0, + chunks: [{ hash: "2".repeat(64), length: 3 }], + files: [], + }; + + yield { + event: "file", + path: file.path, + hash: "3".repeat(64), + sha256: file.sha256, + dedupRatio: 0, + representation: [ + { + xorbId: 0, + indexStart: 0, + indexEnd: 1, + length: 3, + rangeHash: "4".repeat(64), + }, + ], + }; + } + }), +})); + +function readFileEntryInfo(shard: Uint8Array): { flags: number; fileEntryLength: number } { + const shardView = new DataView(shard.buffer, shard.byteOffset, shard.byteLength); + const footerSize = Number(shardView.getBigUint64(SHARD_MAGIC_TAG.length + 8, true)); + const footerStart = shard.length - footerSize; + const fileInfoOffset = Number(shardView.getBigUint64(footerStart + 8, true)); + const xorbInfoOffset = Number(shardView.getBigUint64(footerStart + 16, true)); + + const fileEntryLength = xorbInfoOffset - fileInfoOffset - FILE_BOOKEND_LENGTH; + const flags = shardView.getUint32(fileInfoOffset + HASH_LENGTH, true); + + return { flags, fileEntryLength }; +} + +function toSource(sha256?: string): AsyncGenerator<{ content: Blob; path: string; sha256?: string }> { + return (async function* () { + yield { + content: new Blob(["content"]), + path: "file.bin", + ...(sha256 !== undefined ? { sha256 } : {}), + }; + })(); +} + +function toMultiSource(paths: string[]): AsyncGenerator<{ content: Blob; path: string; sha256?: string }> { + return (async function* () { + for (const path of paths) { + yield { + content: new Blob(["content"]), + path, + }; + } + })(); +} + +describe("uploadShards", () => { + it("omits metadata flag and metadata section when sha256 is missing", async () => { + const uploadedShards: Uint8Array[] = []; + const fetchMock: typeof fetch = vi.fn(async (input, init) => { + const url = String(input); + + if (url.endsWith("/v1/shards")) { + if (!(init?.body instanceof Uint8Array)) { + throw new Error("Expected Uint8Array shard body"); + } + uploadedShards.push(new Uint8Array(init.body)); + } + + return new Response(null, { status: 200 }); + }); + + for await (const event of uploadShards(toSource(), { + accessToken: "test-token", + hubUrl: "https://hub.local", + fetch: fetchMock, + repo: { type: "model", name: "user/repo" }, + rev: "main", + xetParams: { + casUrl: "https://cas.local", + accessToken: "cas-token", + expiresAt: new Date(Date.now() + 600_000), + refreshWriteTokenUrl: "https://hub.local/xet-write-token", + }, + })) { + void event; + } + + expect(uploadedShards).toHaveLength(1); + expect(readFileEntryInfo(uploadedShards[0])).toEqual({ + flags: MDB_FILE_FLAG_WITH_VERIFICATION, + fileEntryLength: FILE_ENTRY_BASE_SIZE + REPRESENTATION_ENTRY_SIZE + VERIFICATION_ENTRY_SIZE, + }); + }); + + it("keeps metadata flag and metadata section when sha256 is provided", async () => { + const uploadedShards: Uint8Array[] = []; + const fetchMock: typeof fetch = vi.fn(async (input, init) => { + const url = String(input); + + if (url.endsWith("/v1/shards")) { + if (!(init?.body instanceof Uint8Array)) { + throw new Error("Expected Uint8Array shard body"); + } + uploadedShards.push(new Uint8Array(init.body)); + } + + return new Response(null, { status: 200 }); + }); + + for await (const event of uploadShards(toSource("5".repeat(64)), { + accessToken: "test-token", + hubUrl: "https://hub.local", + fetch: fetchMock, + repo: { type: "model", name: "user/repo" }, + rev: "main", + xetParams: { + casUrl: "https://cas.local", + accessToken: "cas-token", + expiresAt: new Date(Date.now() + 600_000), + refreshWriteTokenUrl: "https://hub.local/xet-write-token", + }, + })) { + void event; + } + + expect(uploadedShards).toHaveLength(1); + expect(readFileEntryInfo(uploadedShards[0])).toEqual({ + flags: MDB_FILE_FLAG_WITH_VERIFICATION + MDB_FILE_FLAG_WITH_METADATA_EXT, + fileEntryLength: FILE_ENTRY_BASE_SIZE + REPRESENTATION_ENTRY_SIZE + VERIFICATION_ENTRY_SIZE + METADATA_ENTRY_SIZE, + }); + }); + + it("dedupes file entries with the same xet hash within a shard", async () => { + const uploadedShards: Uint8Array[] = []; + const fetchMock: typeof fetch = vi.fn(async (input, init) => { + const url = String(input); + + if (url.endsWith("/v1/shards")) { + if (!(init?.body instanceof Uint8Array)) { + throw new Error("Expected Uint8Array shard body"); + } + uploadedShards.push(new Uint8Array(init.body)); + } + + return new Response(null, { status: 200 }); + }); + + const fileEvents: Array<{ path: string; xetHash: string }> = []; + for await (const event of uploadShards(toMultiSource(["a.bin", "b.bin", "c.bin"]), { + accessToken: "test-token", + hubUrl: "https://hub.local", + fetch: fetchMock, + repo: { type: "model", name: "user/repo" }, + rev: "main", + xetParams: { + casUrl: "https://cas.local", + accessToken: "cas-token", + expiresAt: new Date(Date.now() + 600_000), + refreshWriteTokenUrl: "https://hub.local/xet-write-token", + }, + })) { + if (event.event === "file") { + fileEvents.push({ path: event.path, xetHash: event.xetHash }); + } + } + + // Each path still gets its file event yielded so callers can map path -> hash. + expect(fileEvents).toEqual([ + { path: "a.bin", xetHash: "3".repeat(64) }, + { path: "b.bin", xetHash: "3".repeat(64) }, + { path: "c.bin", xetHash: "3".repeat(64) }, + ]); + + // But only one file entry is written into the shard. + expect(uploadedShards).toHaveLength(1); + expect(readFileEntryInfo(uploadedShards[0])).toEqual({ + flags: MDB_FILE_FLAG_WITH_VERIFICATION, + fileEntryLength: FILE_ENTRY_BASE_SIZE + REPRESENTATION_ENTRY_SIZE + VERIFICATION_ENTRY_SIZE, + }); + }); +}); diff --git a/node_modules/@huggingface/hub/src/utils/uploadShards.ts b/node_modules/@huggingface/hub/src/utils/uploadShards.ts new file mode 100644 index 0000000000000000000000000000000000000000..4cd11cc24375e70b79534fc5945b8e361dd41b0f --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/uploadShards.ts @@ -0,0 +1,440 @@ +import { createApiError } from "../error"; +import type { RepoId } from "../types/public"; +import { createXorbs } from "./createXorbs"; +import { sum } from "./sum"; +import { xetWriteToken } from "./xetWriteToken"; + +const SHARD_MAX_SIZE = 64 * 1024 * 1024; +const SHARD_HEADER_SIZE = 48; +const SHARD_FOOTER_SIZE = 200; +const HASH_LENGTH = 32; +const XORB_FOOTER_LENGTH = 48; +const FILE_FOOTER_LENGTH = 48; +export const SHARD_HEADER_VERSION = 2n; +export const SHARD_FOOTER_VERSION = 1n; + +const MDB_FILE_FLAG_WITH_VERIFICATION = 0x80000000; // Cannot define as 1 << 31 because it becomes a negative number +const MDB_FILE_FLAG_WITH_METADATA_EXT = 0x40000000; + +export const SHARD_MAGIC_TAG = new Uint8Array([ + "H".charCodeAt(0), + "F".charCodeAt(0), + "R".charCodeAt(0), + "e".charCodeAt(0), + "p".charCodeAt(0), + "o".charCodeAt(0), + "M".charCodeAt(0), + "e".charCodeAt(0), + "t".charCodeAt(0), + "a".charCodeAt(0), + "D".charCodeAt(0), + "a".charCodeAt(0), + "t".charCodeAt(0), + "a".charCodeAt(0), + 0, + 85, + 105, + 103, + 69, + 106, + 123, + 129, + 87, + 131, + 165, + 189, + 217, + 92, + 205, + 209, + 74, + 169, +]); + +export interface XetTokenParams { + sessionId?: string; + casUrl?: string; + accessToken?: string; + expiresAt?: Date; + refreshWriteTokenUrl: string; +} + +interface UploadShardsParams { + accessToken: string | undefined; + hubUrl: string; + xetParams: XetTokenParams; + fetch?: typeof fetch; + repo: RepoId; + rev: string; + isPullRequest?: boolean; + yieldCallback?: (event: { event: "fileProgress"; path: string; progress: number }) => void; +} + +/** + * Outputs the file sha256 after their xorbs/shards have been uploaded. + */ +export async function* uploadShards( + source: AsyncGenerator<{ content: Blob; path: string; sha256?: string }>, + params: UploadShardsParams, +): AsyncGenerator< + | { + event: "file"; + path: string; + xetHash: string; + sha256: string | undefined; + dedupRatio: number; + } + | { event: "fileProgress"; path: string; progress: number } +> { + const xorbHashes: Array = []; + const seenFileXetHashes = new Set(); + + const fileInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.25); + const xorbInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.75); + + const xorbView = new DataView(xorbInfoSection.buffer); + let xorbViewOffset = 0; + const fileInfoView = new DataView(fileInfoSection.buffer); + let fileViewOffset = 0; + let xorbTotalSize = 0n; + let fileTotalSize = 0n; + let xorbTotalUnpackedSize = 0n; + + for await (const output of createXorbs(source, params)) { + switch (output.event) { + case "xorb": { + xorbHashes.push(output.hash); + + // Calculate space needed for this xorb entry + const xorbEntrySize = HASH_LENGTH + 4 + 4 + 4 + 4; // hash + flags + count + unpacked + packed + const chunksSize = output.chunks.length * (HASH_LENGTH + 4 + 4 + 8); // per chunk: hash + length + offset + reserved + const totalXorbSize = xorbEntrySize + chunksSize; + + // Check if adding this xorb would exceed buffer capacity + if (xorbViewOffset + totalXorbSize > xorbInfoSection.length) { + // Upload current shard and reset buffers + if (xorbViewOffset > 0 || fileViewOffset > 0) { + await uploadShard(createShard(), params); + } + } + + // todo: handle when going out of bounds + writeHashToArray(output.hash, xorbInfoSection, xorbViewOffset); + xorbViewOffset += HASH_LENGTH; + xorbView.setUint32(xorbViewOffset, 0, true); // flags + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, output.chunks.length, true); + xorbViewOffset += 4; + const xorbUnpackedSize = sum(output.chunks.map((x) => x.length)); + xorbView.setUint32(xorbViewOffset, xorbUnpackedSize, true); + xorbTotalUnpackedSize += BigInt(xorbUnpackedSize); + xorbTotalSize += BigInt(output.xorb.byteLength); + xorbViewOffset += 4; + xorbView.setUint32(xorbViewOffset, output.xorb.byteLength, true); + xorbViewOffset += 4; + + let chunkBytes = 0; + for (const chunk of output.chunks) { + writeHashToArray(chunk.hash, xorbInfoSection, xorbViewOffset); + xorbViewOffset += HASH_LENGTH; + // start offset + xorbView.setUint32(xorbViewOffset, chunkBytes, true); + xorbViewOffset += 4; + // chunk length + xorbView.setUint32(xorbViewOffset, chunk.length, true); + xorbViewOffset += 4; + xorbView.setBigUint64(xorbViewOffset, 0n, true); // reserved + xorbViewOffset += 8; + chunkBytes += chunk.length; + } + + for (const file of output.files) { + yield { + event: "fileProgress", + path: file.path, + progress: file.lastSentProgress, + }; + } + + await uploadXorb(output, params); + //^ Todo: queue it and do not await it + + for (const file of output.files) { + yield { event: "fileProgress", path: file.path, progress: file.progress }; + } + break; + } + case "file": { + yield { + event: "file", + path: output.path, + xetHash: output.hash, + sha256: output.sha256, + dedupRatio: output.dedupRatio, + }; // Maybe wait until shard is uploaded before yielding. + + if (seenFileXetHashes.has(output.hash)) { + break; + } + seenFileXetHashes.add(output.hash); + + // Calculate space needed for this file entry + const fileHeaderSize = HASH_LENGTH + 4 + 4 + 8; // hash + flags + rep length + reserved + const representationSize = output.representation.length * (HASH_LENGTH + 4 + 4 + 4 + 4); // per rep: xorb hash + flags + length + offset + endOffset + const verificationSize = output.representation.length * (HASH_LENGTH + 16); // per rep: range hash + reserved + const fileSha256 = output.sha256; + const hasMetadataExt = fileSha256 !== undefined; + const metadataSize = hasMetadataExt ? HASH_LENGTH + 16 : 0; // sha256 + reserved + const totalFileSize = fileHeaderSize + representationSize + verificationSize + metadataSize; + + // Check if adding this file would exceed buffer capacity + if (fileViewOffset + totalFileSize > fileInfoSection.length) { + // Upload current shard and reset buffers + if (xorbViewOffset > 0 || fileViewOffset > 0) { + await uploadShard(createShard(), params); + } + } + + writeHashToArray(output.hash, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH; + // Cannot use | binary operator since it works with int32 not uint32 and one of the flags is 1 << 31 + fileInfoView.setUint32( + fileViewOffset, + MDB_FILE_FLAG_WITH_VERIFICATION + (hasMetadataExt ? MDB_FILE_FLAG_WITH_METADATA_EXT : 0), + true, + ); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, output.representation.length, true); + fileViewOffset += 4; + fileInfoView.setBigUint64(fileViewOffset, 0n, true); // reserved + fileViewOffset += 8; + + for (const repItem of output.representation) { + writeHashToArray( + typeof repItem.xorbId === "number" ? xorbHashes[repItem.xorbId] : repItem.xorbId, + fileInfoSection, + fileViewOffset, + ); + fileViewOffset += HASH_LENGTH; + fileInfoView.setUint32(fileViewOffset, 0, true); // Xorb flags + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.length, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.indexStart, true); + fileViewOffset += 4; + fileInfoView.setUint32(fileViewOffset, repItem.indexEnd, true); + fileViewOffset += 4; + } + + // File verification data + for (const repItem of output.representation) { + writeHashToArray(repItem.rangeHash, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH; + // reserved in file verification data + for (let i = 0; i < 16; i++) { + fileInfoSection[fileViewOffset + i] = 0; + } + fileViewOffset += 16; + } + + if (hasMetadataExt) { + // File metadata ext + writeHashToArray(fileSha256, fileInfoSection, fileViewOffset); + fileViewOffset += HASH_LENGTH; + + // reserved in file metadata ext + for (let i = 0; i < 16; i++) { + fileInfoSection[fileViewOffset + i] = 0; + } + fileViewOffset += 16; + } + + break; + } + } + } + + function createShard(): Uint8Array { + const shard = new Uint8Array( + SHARD_HEADER_SIZE + SHARD_FOOTER_SIZE + xorbViewOffset + XORB_FOOTER_LENGTH + fileViewOffset + FILE_FOOTER_LENGTH, + ); + + const shardView = new DataView(shard.buffer); + let shardOffset = 0; + + // Header + shard.set(SHARD_MAGIC_TAG, shardOffset); + shardOffset += SHARD_MAGIC_TAG.length; + + shardView.setBigUint64(shardOffset, SHARD_HEADER_VERSION, true); + shardOffset += 8; + + shardView.setBigUint64(shardOffset, BigInt(SHARD_FOOTER_SIZE), true); + shardOffset += 8; + + // File Info Section + shard.set(fileInfoSection.slice(0, fileViewOffset), shardOffset); + shardOffset += fileViewOffset; + + // File info bookend + for (let i = 0; i < 32; i++) { + shard[shardOffset + i] = 0xff; + } + shardOffset += 32; + for (let i = 0; i < 16; i++) { + shard[shardOffset + i] = 0; + } + shardOffset += 16; + + // XORB Info Section + const xorbInfoOffset = shardOffset; + shard.set(xorbInfoSection.slice(0, xorbViewOffset), shardOffset); + shardOffset += xorbViewOffset; + + // Xorb info bookend + for (let i = 0; i < 32; i++) { + shard[shardOffset + i] = 0xff; + } + shardOffset += 32; + for (let i = 0; i < 16; i++) { + shard[shardOffset + i] = 0; + } + shardOffset += 16; + + // Footer + const footerOffset = shardOffset; + + // version: u64, // Footer version (must be 1) + // file_info_offset: u64, // Offset to file info section + // cas_info_offset: u64, // Offset to CAS info section + // reserved 48 bytes + // chunk_hash_hmac_key: [u64; 4], // HMAC key for chunk hashes (32 bytes) + // shard_creation_timestamp: u64, // Creation time (seconds since epoch) + // shard_key_expiry: u64, // Expiry time (seconds since epoch) + // _buffer: [u64; 6], // Reserved space (48 bytes) + // stored_bytes_on_disk: u64, // Total bytes stored on disk + // materialized_bytes: u64, // Total materialized bytes + // stored_bytes: u64, // Total stored bytes + // footer_offset: u64, + + shardView.setBigUint64(shardOffset, SHARD_FOOTER_VERSION, true); + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(SHARD_HEADER_SIZE), true); // beginning of fileinfo section + shardOffset += 8; + shardView.setBigUint64(shardOffset, BigInt(xorbInfoOffset), true); // beginning of xorbinfo section + shardOffset += 8; + + for (let i = 0; i < 48; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 48; + + // Chunk HMAC + for (let i = 0; i < 32; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 32; + + shardView.setBigUint64(shardOffset, BigInt(Math.floor(Date.now() / 1000)), true); + shardOffset += 8; + + // Shard key expiration + shardView.setBigUint64(shardOffset, 0n, true); + shardOffset += 8; + + // Reserved space (48 bytes) + for (let i = 0; i < 48; i++) { + shardView.setUint8(shardOffset + i, 0); + } + shardOffset += 48; + + shardView.setBigUint64(shardOffset, xorbTotalSize, true); + shardOffset += 8; + + shardView.setBigUint64(shardOffset, fileTotalSize, true); + shardOffset += 8; + + shardView.setBigUint64(shardOffset, xorbTotalUnpackedSize, true); + shardOffset += 8; + + shardView.setBigUint64(shardOffset, BigInt(footerOffset), true); + + xorbViewOffset = 0; + fileViewOffset = 0; + xorbTotalSize = 0n; + xorbTotalUnpackedSize = 0n; + fileTotalSize = 0n; + + return shard; + } + + // If un-uploaded data remains, upload it + if (xorbViewOffset || fileViewOffset) { + await uploadShard(createShard(), params); + } +} + +// Todo: switch from hex to non-hex when WASM switches. For now consider hash is hex +function writeHashToArray(hash: string, array: Uint8Array, offset: number) { + for (let i = 0; i < hash.length; i += 16) { + // Write a uint64 in little endian + array[offset + i / 2] = parseInt(hash.substring(i + 2 * 7, i + 2 * 8), 16); + array[offset + i / 2 + 1] = parseInt(hash.substring(i + 2 * 6, i + 2 * 7), 16); + array[offset + i / 2 + 2] = parseInt(hash.substring(i + 2 * 5, i + 2 * 6), 16); + array[offset + i / 2 + 3] = parseInt(hash.substring(i + 2 * 4, i + 2 * 5), 16); + array[offset + i / 2 + 4] = parseInt(hash.substring(i + 2 * 3, i + 2 * 4), 16); + array[offset + i / 2 + 5] = parseInt(hash.substring(i + 2 * 2, i + 2 * 3), 16); + array[offset + i / 2 + 6] = parseInt(hash.substring(i + 2 * 1, i + 2 * 2), 16); + array[offset + i / 2 + 7] = parseInt(hash.substring(i + 2 * 0, i + 2 * 1), 16); + } +} + +async function uploadXorb( + xorb: { hash: string; xorb: Uint8Array; files: Array<{ path: string; progress: number; lastSentProgress: number }> }, + params: UploadShardsParams, +) { + const token = await xetWriteToken(params); + + const resp = await (params.fetch ?? fetch)(`${token.casUrl}/v1/xorbs/default/${xorb.hash}`, { + method: "POST", + body: xorb.xorb, + headers: { + Authorization: `Bearer ${token.accessToken}`, + ...(params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {}), + }, + ...{ + progressHint: { + progressCallback: (progress: number) => { + for (const file of xorb.files) { + params.yieldCallback?.({ + event: "fileProgress", + path: file.path, + progress: file.lastSentProgress + (file.progress - file.lastSentProgress) * progress, + }); + } + }, + }, + }, + }); + + if (!resp.ok) { + throw await createApiError(resp); + } +} + +async function uploadShard(shard: Uint8Array, params: UploadShardsParams) { + const token = await xetWriteToken(params); + + const resp = await (params.fetch ?? fetch)(`${token.casUrl}/v1/shards`, { + method: "POST", + body: shard, + headers: { + Authorization: `Bearer ${token.accessToken}`, + ...(params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {}), + }, + }); + + if (!resp.ok) { + throw await createApiError(resp); + } +} diff --git a/node_modules/@huggingface/hub/src/utils/xetWriteToken.ts b/node_modules/@huggingface/hub/src/utils/xetWriteToken.ts new file mode 100644 index 0000000000000000000000000000000000000000..e877dc55ae288c930572fdbac9bfbaff6ae41ace --- /dev/null +++ b/node_modules/@huggingface/hub/src/utils/xetWriteToken.ts @@ -0,0 +1,98 @@ +import { createApiError } from "../error"; +import type { XetTokenParams } from "./uploadShards"; + +export interface XetWriteTokenParams { + accessToken: string | undefined; + fetch?: typeof fetch; + xetParams: XetTokenParams; +} + +const JWT_SAFETY_PERIOD = 60_000; +const JWT_CACHE_SIZE = 1_000; + +const jwtPromises: Map> = new Map(); +/** + * Cache to store JWTs, to avoid making many auth requests when downloading multiple files from the same repo + */ +const jwts: Map< + string, + { + accessToken: string; + expiresAt: Date; + casUrl: string; + } +> = new Map(); + +export async function xetWriteToken(params: XetWriteTokenParams): Promise<{ accessToken: string; casUrl: string }> { + if ( + params.xetParams.expiresAt && + params.xetParams.casUrl && + params.xetParams.accessToken && + params.xetParams.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD) + ) { + return { accessToken: params.xetParams.accessToken, casUrl: params.xetParams.casUrl }; + } + const key = params.xetParams.refreshWriteTokenUrl; + + const jwt = jwts.get(key); + + if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD)) { + return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; + } + + // If we already have a promise for this repo, return it + const existingPromise = jwtPromises.get(key); + if (existingPromise) { + return existingPromise; + } + + const promise = (async () => { + const resp = await (params.fetch ?? fetch)(params.xetParams.refreshWriteTokenUrl, { + headers: { + ...(params.accessToken + ? { + Authorization: `Bearer ${params.accessToken}`, + } + : {}), + ...(params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {}), + }, + }); + + if (!resp.ok) { + throw await createApiError(resp); + } + + const json: { accessToken: string; casUrl: string; exp: number } = await resp.json(); + const jwt = { + accessToken: json.accessToken, + expiresAt: new Date(json.exp * 1000), + casUrl: json.casUrl, + }; + + jwtPromises.delete(key); + + for (const [key, value] of jwts.entries()) { + if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD)) { + jwts.delete(key); + } else { + break; + } + } + if (jwts.size >= JWT_CACHE_SIZE) { + const keyToDelete = jwts.keys().next().value; + if (keyToDelete) { + jwts.delete(keyToDelete); + } + } + jwts.set(key, jwt); + + return { + accessToken: json.accessToken, + casUrl: json.casUrl, + }; + })(); + + jwtPromises.set(key, promise); + + return promise; +} diff --git a/node_modules/@huggingface/hub/src/vendor/hash-wasm/LICENSE b/node_modules/@huggingface/hub/src/vendor/hash-wasm/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..689ccbe872a76ddfeb0196e0e4b9d60d67480220 --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/hash-wasm/LICENSE @@ -0,0 +1,39 @@ +MIT License + +Copyright (c) 2020 Dani Biró + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Embedded C implementations might use other, similarly permissive licenses. +Check the beginning of the files from the /src directory. + +Special thank you to the authors of original C algorithms: + +- Alexander Peslyak +- Aleksey Kravchenko +- Colin Percival +- Stephan Brumme +- Steve Reid +- Samuel Neves +- Solar Designer +- Project Nayuki +- ARM Limited +- Yanbo Li dreamfly281@gmail.com, goldboar@163.comYanbo Li +- Mark Adler +- Yann Collet diff --git a/node_modules/@huggingface/hub/src/vendor/hash-wasm/build.sh b/node_modules/@huggingface/hub/src/vendor/hash-wasm/build.sh new file mode 100644 index 0000000000000000000000000000000000000000..18424dfbc296bf3418e77fce35fbde5298a4a6a3 --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/hash-wasm/build.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +CURRENT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +cd $CURRENT_PATH + +# Clean up +docker kill hash-wasm-builder +docker rm hash-wasm-builder + +# Start container +docker run -it -d --name hash-wasm-builder emscripten/emsdk:3.1.55 bash + +# Copy & compile +docker exec hash-wasm-builder bash -c "mkdir /source" +docker cp ./sha256.c hash-wasm-builder:/source +docker exec hash-wasm-builder bash -c "\ + cd /source && \ + emcc sha256.c -o sha256.js -msimd128 -sSINGLE_FILE -sMODULARIZE=1 -sENVIRONMENT=web,worker -sEXPORTED_FUNCTIONS=_Hash_Init,_Hash_Update,_Hash_Final,_GetBufferPtr -sFILESYSTEM=0 -fno-rtti -fno-exceptions -O1 -sMODULARIZE=1 -sEXPORT_ES6=1 \ + " +# Patch "_scriptDir" variable +docker exec hash-wasm-builder bash -c "\ + cd /source && \ + sed -i 's\var _scriptDir\var _unused\g' ./sha256.js && \ + sed -i 's\_scriptDir\false\g' ./sha256.js \ + " + +# Copy back compiled file +docker cp hash-wasm-builder:/source/sha256.js . + + +# Clean up +docker kill hash-wasm-builder +docker rm hash-wasm-builder diff --git a/node_modules/@huggingface/hub/src/vendor/hash-wasm/sha256-wrapper.ts b/node_modules/@huggingface/hub/src/vendor/hash-wasm/sha256-wrapper.ts new file mode 100644 index 0000000000000000000000000000000000000000..3a897696ed4af0dc0a03c9b3d968f9b10aab1f23 --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/hash-wasm/sha256-wrapper.ts @@ -0,0 +1,62 @@ +import WasmModule from "./sha256"; + +export async function createSHA256(isInsideWorker = false): Promise<{ + init(): void; + update(data: Uint8Array): void; + digest(method: "hex"): string; +}> { + const BUFFER_MAX_SIZE = 8 * 1024 * 1024; + const wasm: Awaited> = isInsideWorker + ? // @ts-expect-error WasmModule will be populated inside self object + await self["SHA256WasmModule"]() + : await WasmModule(); + const heap = wasm.HEAPU8.subarray(wasm._GetBufferPtr()); + return { + init() { + wasm._Hash_Init(256); + }, + update(data: Uint8Array) { + let byteUsed = 0; + while (byteUsed < data.byteLength) { + const bytesLeft = data.byteLength - byteUsed; + const length = Math.min(bytesLeft, BUFFER_MAX_SIZE); + heap.set(data.subarray(byteUsed, byteUsed + length)); + wasm._Hash_Update(length); + byteUsed += length; + } + }, + digest(method: "hex") { + if (method !== "hex") { + throw new Error("Only digest hex is supported"); + } + wasm._Hash_Final(); + const result = Array.from(heap.slice(0, 32)); + return result.map((b) => b.toString(16).padStart(2, "0")).join(""); + }, + }; +} + +export function createSHA256WorkerCode(): string { + return ` + self.addEventListener('message', async (event) => { + const { file } = event.data; + const sha256 = await self.createSHA256(true); + sha256.init(); + const reader = file.stream().getReader(); + const total = file.size; + let bytesDone = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + sha256.update(value); + bytesDone += value.length; + postMessage({ progress: bytesDone / total }); + } + postMessage({ sha256: sha256.digest('hex') }); + }); + self.SHA256WasmModule = ${WasmModule.toString()}; + self.createSHA256 = ${createSHA256.toString()}; + `; +} diff --git a/node_modules/@huggingface/hub/src/vendor/hash-wasm/sha256.c b/node_modules/@huggingface/hub/src/vendor/hash-wasm/sha256.c new file mode 100644 index 0000000000000000000000000000000000000000..b8c0cb7eaf322cb4f74c42908f2d2542e5894233 --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/hash-wasm/sha256.c @@ -0,0 +1,432 @@ +/* sha256.c - an implementation of SHA-256/224 hash functions + * based on FIPS 180-3 (Federal Information Processing Standart). + * + * Copyright (c) 2010, Aleksey Kravchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + + * Modified for hash-wasm by Dani Biró + */ + +#define WITH_BUFFER + + +////////////////////////////////////////////////////////////////////////// + +#include +#include + +#ifndef NULL +#define NULL 0 +#endif + +#ifdef _MSC_VER +#define WASM_EXPORT +#define __inline__ +#else +#define WASM_EXPORT __attribute__((visibility("default"))) +#endif + +#ifdef WITH_BUFFER + +#define MAIN_BUFFER_SIZE 8 * 1024 * 1024 +alignas(128) uint8_t main_buffer[MAIN_BUFFER_SIZE]; + +WASM_EXPORT +uint8_t *Hash_GetBuffer() { + return main_buffer; +} + +#endif + +// Sometimes LLVM emits these functions during the optimization step +// even with -nostdlib -fno-builtin flags +static __inline__ void* memcpy(void* dst, const void* src, uint32_t cnt) { + uint8_t *destination = dst; + const uint8_t *source = src; + while (cnt) { + *(destination++)= *(source++); + --cnt; + } + return dst; +} + +static __inline__ void* memset(void* dst, const uint8_t value, uint32_t cnt) { + uint8_t *p = dst; + while (cnt--) { + *p++ = value; + } + return dst; +} + +static __inline__ void* memcpy2(void* dst, const void* src, uint32_t cnt) { + uint64_t *destination64 = dst; + const uint64_t *source64 = src; + while (cnt >= 8) { + *(destination64++)= *(source64++); + cnt -= 8; + } + + uint8_t *destination = (uint8_t*)destination64; + const uint8_t *source = (uint8_t*)source64; + while (cnt) { + *(destination++)= *(source++); + --cnt; + } + return dst; +} + +static __inline__ void memcpy16(void* dst, const void* src) { + uint64_t* dst64 = (uint64_t*)dst; + uint64_t* src64 = (uint64_t*)src; + + dst64[0] = src64[0]; + dst64[1] = src64[1]; +} + +static __inline__ void memcpy32(void* dst, const void* src) { + uint64_t* dst64 = (uint64_t*)dst; + uint64_t* src64 = (uint64_t*)src; + + #pragma clang loop unroll(full) + for (int i = 0; i < 4; i++) { + dst64[i] = src64[i]; + } +} + +static __inline__ void memcpy64(void* dst, const void* src) { + uint64_t* dst64 = (uint64_t*)dst; + uint64_t* src64 = (uint64_t*)src; + + #pragma clang loop unroll(full) + for (int i = 0; i < 8; i++) { + dst64[i] = src64[i]; + } +} + +static __inline__ uint64_t widen8to64(const uint8_t value) { + return value | (value << 8) | (value << 16) | (value << 24); +} + +static __inline__ void memset16(void* dst, const uint8_t value) { + uint64_t val = widen8to64(value); + uint64_t* dst64 = (uint64_t*)dst; + + dst64[0] = val; + dst64[1] = val; +} + +static __inline__ void memset32(void* dst, const uint8_t value) { + uint64_t val = widen8to64(value); + uint64_t* dst64 = (uint64_t*)dst; + + #pragma clang loop unroll(full) + for (int i = 0; i < 4; i++) { + dst64[i] = val; + } +} + +static __inline__ void memset64(void* dst, const uint8_t value) { + uint64_t val = widen8to64(value); + uint64_t* dst64 = (uint64_t*)dst; + + #pragma clang loop unroll(full) + for (int i = 0; i < 8; i++) { + dst64[i] = val; + } +} + +static __inline__ void memset128(void* dst, const uint8_t value) { + uint64_t val = widen8to64(value); + uint64_t* dst64 = (uint64_t*)dst; + + #pragma clang loop unroll(full) + for (int i = 0; i < 16; i++) { + dst64[i] = val; + } +} + + +////////////////////////////////////////////////////////////////////////// + +#define sha256_block_size 64 +#define sha256_hash_size 32 +#define sha224_hash_size 28 +#define ROTR32(dword, n) ((dword) >> (n) ^ ((dword) << (32 - (n)))) +#define bswap_32(x) __builtin_bswap32(x) + +struct sha256_ctx { + uint32_t message[16]; /* 512-bit buffer for leftovers */ + uint64_t length; /* number of processed bytes */ + uint32_t hash[8]; /* 256-bit algorithm internal hashing state */ + uint32_t digest_length; /* length of the algorithm digest in bytes */ +}; + +struct sha256_ctx sctx; +struct sha256_ctx* ctx = &sctx; + +/* SHA-224 and SHA-256 constants for 64 rounds. These words represent + * the first 32 bits of the fractional parts of the cube + * roots of the first 64 prime numbers. */ +static const uint32_t rhash_k256[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, + 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, + 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, + 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +}; + +/* The SHA256/224 functions defined by FIPS 180-3, 4.1.2 */ +/* Optimized version of Ch(x,y,z)=((x & y) | (~x & z)) */ +#define Ch(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) +/* Optimized version of Maj(x,y,z)=((x & y) ^ (x & z) ^ (y & z)) */ +#define Maj(x, y, z) (((x) & (y)) ^ ((z) & ((x) ^ (y)))) + +#define Sigma0(x) (ROTR32((x), 2) ^ ROTR32((x), 13) ^ ROTR32((x), 22)) +#define Sigma1(x) (ROTR32((x), 6) ^ ROTR32((x), 11) ^ ROTR32((x), 25)) +#define sigma0(x) (ROTR32((x), 7) ^ ROTR32((x), 18) ^ ((x) >> 3)) +#define sigma1(x) (ROTR32((x), 17) ^ ROTR32((x), 19) ^ ((x) >> 10)) + +/* Recalculate element n-th of circular buffer W using formula + * W[n] = sigma1(W[n - 2]) + W[n - 7] + sigma0(W[n - 15]) + W[n - 16]; */ +#define RECALCULATE_W(W, n) \ + (W[n] += \ + (sigma1(W[(n - 2) & 15]) + W[(n - 7) & 15] + sigma0(W[(n - 15) & 15]))) + +#define ROUND(a, b, c, d, e, f, g, h, k, data) \ + { \ + uint32_t T1 = h + Sigma1(e) + Ch(e, f, g) + k + (data); \ + d += T1, h = T1 + Sigma0(a) + Maj(a, b, c); \ + } +#define ROUND_1_16(a, b, c, d, e, f, g, h, n) \ + ROUND(a, b, c, d, e, f, g, h, rhash_k256[n], W[n] = bswap_32(block[n])) +#define ROUND_17_64(a, b, c, d, e, f, g, h, n) \ + ROUND(a, b, c, d, e, f, g, h, k[n], RECALCULATE_W(W, n)) + +/** + * Initialize context before calculaing hash. + * + */ +void sha256_init() { + /* Initial values. These words were obtained by taking the first 32 + * bits of the fractional parts of the square roots of the first + * eight prime numbers. */ + static const uint32_t SHA256_H0[8] = { + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19 + }; + + ctx->length = 0; + ctx->digest_length = sha256_hash_size; + + /* initialize algorithm state */ + + #pragma clang loop vectorize(enable) + for (uint8_t i = 0; i < 8; i += 2) { + *(uint64_t*)&ctx->hash[i] = *(uint64_t*)&SHA256_H0[i]; + } +} + +/** + * Initialize context before calculaing hash. + * + */ +void sha224_init() { + /* Initial values from FIPS 180-3. These words were obtained by taking + * bits from 33th to 64th of the fractional parts of the square + * roots of ninth through sixteenth prime numbers. */ + static const uint32_t SHA224_H0[8] = { + 0xc1059ed8, 0x367cd507, 0x3070dd17, 0xf70e5939, + 0xffc00b31, 0x68581511, 0x64f98fa7, 0xbefa4fa4 + }; + + ctx->length = 0; + ctx->digest_length = sha224_hash_size; + + #pragma clang loop vectorize(enable) + for (uint8_t i = 0; i < 8; i += 2) { + *(uint64_t*)&ctx->hash[i] = *(uint64_t*)&SHA224_H0[i]; + } +} + +/** + * The core transformation. Process a 512-bit block. + * + * @param hash algorithm state + * @param block the message block to process + */ +static void sha256_process_block(uint32_t hash[8], uint32_t block[16]) { + uint32_t A, B, C, D, E, F, G, H; + uint32_t W[16]; + const uint32_t* k; + int i; + + A = hash[0], B = hash[1], C = hash[2], D = hash[3]; + E = hash[4], F = hash[5], G = hash[6], H = hash[7]; + + /* Compute SHA using alternate Method: FIPS 180-3 6.1.3 */ + ROUND_1_16(A, B, C, D, E, F, G, H, 0); + ROUND_1_16(H, A, B, C, D, E, F, G, 1); + ROUND_1_16(G, H, A, B, C, D, E, F, 2); + ROUND_1_16(F, G, H, A, B, C, D, E, 3); + ROUND_1_16(E, F, G, H, A, B, C, D, 4); + ROUND_1_16(D, E, F, G, H, A, B, C, 5); + ROUND_1_16(C, D, E, F, G, H, A, B, 6); + ROUND_1_16(B, C, D, E, F, G, H, A, 7); + ROUND_1_16(A, B, C, D, E, F, G, H, 8); + ROUND_1_16(H, A, B, C, D, E, F, G, 9); + ROUND_1_16(G, H, A, B, C, D, E, F, 10); + ROUND_1_16(F, G, H, A, B, C, D, E, 11); + ROUND_1_16(E, F, G, H, A, B, C, D, 12); + ROUND_1_16(D, E, F, G, H, A, B, C, 13); + ROUND_1_16(C, D, E, F, G, H, A, B, 14); + ROUND_1_16(B, C, D, E, F, G, H, A, 15); + + #pragma clang loop vectorize(enable) + for (i = 16, k = &rhash_k256[16]; i < 64; i += 16, k += 16) { + ROUND_17_64(A, B, C, D, E, F, G, H, 0); + ROUND_17_64(H, A, B, C, D, E, F, G, 1); + ROUND_17_64(G, H, A, B, C, D, E, F, 2); + ROUND_17_64(F, G, H, A, B, C, D, E, 3); + ROUND_17_64(E, F, G, H, A, B, C, D, 4); + ROUND_17_64(D, E, F, G, H, A, B, C, 5); + ROUND_17_64(C, D, E, F, G, H, A, B, 6); + ROUND_17_64(B, C, D, E, F, G, H, A, 7); + ROUND_17_64(A, B, C, D, E, F, G, H, 8); + ROUND_17_64(H, A, B, C, D, E, F, G, 9); + ROUND_17_64(G, H, A, B, C, D, E, F, 10); + ROUND_17_64(F, G, H, A, B, C, D, E, 11); + ROUND_17_64(E, F, G, H, A, B, C, D, 12); + ROUND_17_64(D, E, F, G, H, A, B, C, 13); + ROUND_17_64(C, D, E, F, G, H, A, B, 14); + ROUND_17_64(B, C, D, E, F, G, H, A, 15); + } + + hash[0] += A, hash[1] += B, hash[2] += C, hash[3] += D; + hash[4] += E, hash[5] += F, hash[6] += G, hash[7] += H; +} + +/** + * Calculate message hash. + * Can be called repeatedly with chunks of the message to be hashed. + * + * @param size length of the message chunk + */ +WASM_EXPORT +void Hash_Update(uint32_t size) { + const uint8_t* msg = main_buffer; + uint32_t index = (uint32_t)ctx->length & 63; + ctx->length += size; + + /* fill partial block */ + if (index) { + uint32_t left = sha256_block_size - index; + uint32_t end = size < left ? size : left; + uint8_t* message8 = (uint8_t*)ctx->message; + for (uint8_t i = 0; i < end; i++) { + *(message8 + index + i) = msg[i]; + } + if (size < left) return; + + /* process partial block */ + sha256_process_block(ctx->hash, (uint32_t*)ctx->message); + msg += left; + size -= left; + } + + while (size >= sha256_block_size) { + uint32_t* aligned_message_block = (uint32_t*)msg; + + sha256_process_block(ctx->hash, aligned_message_block); + msg += sha256_block_size; + size -= sha256_block_size; + } + + if (size) { + /* save leftovers */ + for (uint8_t i = 0; i < size; i++) { + *(((uint8_t*)ctx->message) + i) = msg[i]; + } + } +} + +/** + * Store calculated hash into the given array. + * + */ +WASM_EXPORT +void Hash_Final() { + uint32_t index = ((uint32_t)ctx->length & 63) >> 2; + uint32_t shift = ((uint32_t)ctx->length & 3) * 8; + + /* pad message and run for last block */ + + /* append the byte 0x80 to the message */ + ctx->message[index] &= ~(0xFFFFFFFFu << shift); + ctx->message[index++] ^= 0x80u << shift; + + /* if no room left in the message to store 64-bit message length */ + if (index > 14) { + /* then fill the rest with zeros and process it */ + while (index < 16) { + ctx->message[index++] = 0; + } + sha256_process_block(ctx->hash, ctx->message); + index = 0; + } + + while (index < 14) { + ctx->message[index++] = 0; + } + + ctx->message[14] = bswap_32((uint32_t)(ctx->length >> 29)); + ctx->message[15] = bswap_32((uint32_t)(ctx->length << 3)); + sha256_process_block(ctx->hash, ctx->message); + + #pragma clang loop vectorize(enable) + for (int32_t i = 7; i >= 0; i--) { + ctx->hash[i] = bswap_32(ctx->hash[i]); + } + + for (uint8_t i = 0; i < ctx->digest_length; i++) { + main_buffer[i] = *(((uint8_t*)ctx->hash) + i); + } +} + +WASM_EXPORT +uint32_t Hash_Init(uint32_t bits) { + if (bits == 224) { + sha224_init(); + } else { + sha256_init(); + } + return 0; +} + +WASM_EXPORT +const uint32_t STATE_SIZE = sizeof(*ctx); + +WASM_EXPORT +uint8_t* Hash_GetState() { + return (uint8_t*) ctx; +} + +WASM_EXPORT +uint32_t GetBufferPtr() { + return (uint32_t) main_buffer; +} diff --git a/node_modules/@huggingface/hub/src/vendor/hash-wasm/sha256.d.ts b/node_modules/@huggingface/hub/src/vendor/hash-wasm/sha256.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b6d0f51481360ced2c39d6140091d7da6681f150 --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/hash-wasm/sha256.d.ts @@ -0,0 +1,8 @@ +declare function Module(): Promise<{ + HEAPU8: Uint8Array; + _Hash_Init(type: number): void; + _Hash_Update(length: number): void; + _Hash_Final(): void; + _GetBufferPtr(): number; +}>; +export default Module; diff --git a/node_modules/@huggingface/hub/src/vendor/hash-wasm/sha256.js b/node_modules/@huggingface/hub/src/vendor/hash-wasm/sha256.js new file mode 100644 index 0000000000000000000000000000000000000000..7ff85e582d49404ba7565cef5428115398baac3f --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/hash-wasm/sha256.js @@ -0,0 +1,685 @@ + +var Module = (() => { + var _unused = import.meta.url; + + return ( +function(moduleArg = {}) { + +// include: shell.js +// The Module object: Our interface to the outside world. We import +// and export values on it. There are various ways Module can be used: +// 1. Not defined. We create it here +// 2. A function parameter, function(Module) { ..generated code.. } +// 3. pre-run appended it, var Module = {}; ..generated code.. +// 4. External script tag defines var Module. +// We need to check if Module already exists (e.g. case 3 above). +// Substitution will be replaced with actual code on later stage of the build, +// this way Closure Compiler will not mangle it (e.g. case 4. above). +// Note that if you want to run closure, and also to use Module +// after the generated code, you will need to define var Module = {}; +// before the code. Then that object will be used in the code, and you +// can continue to use Module afterwards as well. +var Module = moduleArg; + +// Set up the promise that indicates the Module is initialized +var readyPromiseResolve, readyPromiseReject; +Module['ready'] = new Promise((resolve, reject) => { + readyPromiseResolve = resolve; + readyPromiseReject = reject; +}); + +// --pre-jses are emitted after the Module integration code, so that they can +// refer to Module (if they choose; they can also define Module) + + +// Sometimes an existing Module object exists with properties +// meant to overwrite the default module functionality. Here +// we collect those properties and reapply _after_ we configure +// the current environment's defaults to avoid having to be so +// defensive during initialization. +var moduleOverrides = Object.assign({}, Module); + +var arguments_ = []; +var thisProgram = './this.program'; +var quit_ = (status, toThrow) => { + throw toThrow; +}; + +// Determine the runtime environment we are in. You can customize this by +// setting the ENVIRONMENT setting at compile time (see settings.js). + +// Attempt to auto-detect the environment +var ENVIRONMENT_IS_WEB = typeof window == 'object'; +var ENVIRONMENT_IS_WORKER = typeof importScripts == 'function'; +// N.b. Electron.js environment is simultaneously a NODE-environment, but +// also a web environment. +var ENVIRONMENT_IS_NODE = typeof process == 'object' && typeof process.versions == 'object' && typeof process.versions.node == 'string'; +var ENVIRONMENT_IS_SHELL = !ENVIRONMENT_IS_WEB && !ENVIRONMENT_IS_NODE && !ENVIRONMENT_IS_WORKER; + +// `/` should be present at the end if `scriptDirectory` is not empty +var scriptDirectory = ''; +function locateFile(path) { + if (Module['locateFile']) { + return Module['locateFile'](path, scriptDirectory); + } + return scriptDirectory + path; +} + +// Hooks that are implemented differently in different runtime environments. +var read_, + readAsync, + readBinary; + +// Note that this includes Node.js workers when relevant (pthreads is enabled). +// Node.js workers are detected as a combination of ENVIRONMENT_IS_WORKER and +// ENVIRONMENT_IS_NODE. +if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { + if (ENVIRONMENT_IS_WORKER) { // Check worker, not web, since window could be polyfilled + scriptDirectory = self.location.href; + } else if (typeof document != 'undefined' && document.currentScript) { // web + scriptDirectory = document.currentScript.src; + } + // When MODULARIZE, this JS may be executed later, after document.currentScript + // is gone, so we saved it, and we use it here instead of any other info. + if (false) { + scriptDirectory = false; + } + // blob urls look like blob:http://site.com/etc/etc and we cannot infer anything from them. + // otherwise, slice off the final part of the url to find the script directory. + // if scriptDirectory does not contain a slash, lastIndexOf will return -1, + // and scriptDirectory will correctly be replaced with an empty string. + // If scriptDirectory contains a query (starting with ?) or a fragment (starting with #), + // they are removed because they could contain a slash. + if (scriptDirectory.startsWith('blob:')) { + scriptDirectory = ''; + } else { + scriptDirectory = scriptDirectory.substr(0, scriptDirectory.replace(/[?#].*/, '').lastIndexOf('/')+1); + } + + // Differentiate the Web Worker from the Node Worker case, as reading must + // be done differently. + { +// include: web_or_worker_shell_read.js +read_ = (url) => { + var xhr = new XMLHttpRequest(); + xhr.open('GET', url, false); + xhr.send(null); + return xhr.responseText; + } + + if (ENVIRONMENT_IS_WORKER) { + readBinary = (url) => { + var xhr = new XMLHttpRequest(); + xhr.open('GET', url, false); + xhr.responseType = 'arraybuffer'; + xhr.send(null); + return new Uint8Array(/** @type{!ArrayBuffer} */(xhr.response)); + }; + } + + readAsync = (url, onload, onerror) => { + var xhr = new XMLHttpRequest(); + xhr.open('GET', url, true); + xhr.responseType = 'arraybuffer'; + xhr.onload = () => { + if (xhr.status == 200 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0 + onload(xhr.response); + return; + } + onerror(); + }; + xhr.onerror = onerror; + xhr.send(null); + } + +// end include: web_or_worker_shell_read.js + } +} else +{ +} + +var out = Module['print'] || console.log.bind(console); +var err = Module['printErr'] || console.error.bind(console); + +// Merge back in the overrides +Object.assign(Module, moduleOverrides); +// Free the object hierarchy contained in the overrides, this lets the GC +// reclaim data used. +moduleOverrides = null; + +// Emit code to handle expected values on the Module object. This applies Module.x +// to the proper local x. This has two benefits: first, we only emit it if it is +// expected to arrive, and second, by using a local everywhere else that can be +// minified. + +if (Module['arguments']) arguments_ = Module['arguments']; + +if (Module['thisProgram']) thisProgram = Module['thisProgram']; + +if (Module['quit']) quit_ = Module['quit']; + +// perform assertions in shell.js after we set up out() and err(), as otherwise if an assertion fails it cannot print the message +// end include: shell.js + +// include: preamble.js +// === Preamble library stuff === + +// Documentation for the public APIs defined in this file must be updated in: +// site/source/docs/api_reference/preamble.js.rst +// A prebuilt local version of the documentation is available at: +// site/build/text/docs/api_reference/preamble.js.txt +// You can also build docs locally as HTML or other formats in site/ +// An online HTML version (which may be of a different version of Emscripten) +// is up at http://kripken.github.io/emscripten-site/docs/api_reference/preamble.js.html + +var wasmBinary; +if (Module['wasmBinary']) wasmBinary = Module['wasmBinary']; + +if (typeof WebAssembly != 'object') { + abort('no native wasm support detected'); +} + +// include: base64Utils.js +// Converts a string of base64 into a byte array (Uint8Array). +function intArrayFromBase64(s) { + + var decoded = atob(s); + var bytes = new Uint8Array(decoded.length); + for (var i = 0 ; i < decoded.length ; ++i) { + bytes[i] = decoded.charCodeAt(i); + } + return bytes; +} + +// If filename is a base64 data URI, parses and returns data (Buffer on node, +// Uint8Array otherwise). If filename is not a base64 data URI, returns undefined. +function tryParseAsDataURI(filename) { + if (!isDataURI(filename)) { + return; + } + + return intArrayFromBase64(filename.slice(dataURIPrefix.length)); +} +// end include: base64Utils.js +// Wasm globals + +var wasmMemory; + +//======================================== +// Runtime essentials +//======================================== + +// whether we are quitting the application. no code should run after this. +// set in exit() and abort() +var ABORT = false; + +// set by exit() and abort(). Passed to 'onExit' handler. +// NOTE: This is also used as the process return code code in shell environments +// but only when noExitRuntime is false. +var EXITSTATUS; + +// In STRICT mode, we only define assert() when ASSERTIONS is set. i.e. we +// don't define it at all in release modes. This matches the behaviour of +// MINIMAL_RUNTIME. +// TODO(sbc): Make this the default even without STRICT enabled. +/** @type {function(*, string=)} */ +function assert(condition, text) { + if (!condition) { + // This build was created without ASSERTIONS defined. `assert()` should not + // ever be called in this configuration but in case there are callers in + // the wild leave this simple abort() implementation here for now. + abort(text); + } +} + +// Memory management + +var HEAP, +/** @type {!Int8Array} */ + HEAP8, +/** @type {!Uint8Array} */ + HEAPU8, +/** @type {!Int16Array} */ + HEAP16, +/** @type {!Uint16Array} */ + HEAPU16, +/** @type {!Int32Array} */ + HEAP32, +/** @type {!Uint32Array} */ + HEAPU32, +/** @type {!Float32Array} */ + HEAPF32, +/** @type {!Float64Array} */ + HEAPF64; + +// include: runtime_shared.js +function updateMemoryViews() { + var b = wasmMemory.buffer; + Module['HEAP8'] = HEAP8 = new Int8Array(b); + Module['HEAP16'] = HEAP16 = new Int16Array(b); + Module['HEAPU8'] = HEAPU8 = new Uint8Array(b); + Module['HEAPU16'] = HEAPU16 = new Uint16Array(b); + Module['HEAP32'] = HEAP32 = new Int32Array(b); + Module['HEAPU32'] = HEAPU32 = new Uint32Array(b); + Module['HEAPF32'] = HEAPF32 = new Float32Array(b); + Module['HEAPF64'] = HEAPF64 = new Float64Array(b); +} +// end include: runtime_shared.js +// include: runtime_stack_check.js +// end include: runtime_stack_check.js +// include: runtime_assertions.js +// end include: runtime_assertions.js +var __ATPRERUN__ = []; // functions called before the runtime is initialized +var __ATINIT__ = []; // functions called during startup +var __ATEXIT__ = []; // functions called during shutdown +var __ATPOSTRUN__ = []; // functions called after the main() is called + +var runtimeInitialized = false; + +function preRun() { + if (Module['preRun']) { + if (typeof Module['preRun'] == 'function') Module['preRun'] = [Module['preRun']]; + while (Module['preRun'].length) { + addOnPreRun(Module['preRun'].shift()); + } + } + callRuntimeCallbacks(__ATPRERUN__); +} + +function initRuntime() { + runtimeInitialized = true; + + + callRuntimeCallbacks(__ATINIT__); +} + +function postRun() { + + if (Module['postRun']) { + if (typeof Module['postRun'] == 'function') Module['postRun'] = [Module['postRun']]; + while (Module['postRun'].length) { + addOnPostRun(Module['postRun'].shift()); + } + } + + callRuntimeCallbacks(__ATPOSTRUN__); +} + +function addOnPreRun(cb) { + __ATPRERUN__.unshift(cb); +} + +function addOnInit(cb) { + __ATINIT__.unshift(cb); +} + +function addOnExit(cb) { +} + +function addOnPostRun(cb) { + __ATPOSTRUN__.unshift(cb); +} + +// include: runtime_math.js +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/imul + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/fround + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/clz32 + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/trunc + +// end include: runtime_math.js +// A counter of dependencies for calling run(). If we need to +// do asynchronous work before running, increment this and +// decrement it. Incrementing must happen in a place like +// Module.preRun (used by emcc to add file preloading). +// Note that you can add dependencies in preRun, even though +// it happens right before run - run will be postponed until +// the dependencies are met. +var runDependencies = 0; +var runDependencyWatcher = null; +var dependenciesFulfilled = null; // overridden to take different actions when all run dependencies are fulfilled + +function getUniqueRunDependency(id) { + return id; +} + +function addRunDependency(id) { + runDependencies++; + + Module['monitorRunDependencies']?.(runDependencies); + +} + +function removeRunDependency(id) { + runDependencies--; + + Module['monitorRunDependencies']?.(runDependencies); + + if (runDependencies == 0) { + if (runDependencyWatcher !== null) { + clearInterval(runDependencyWatcher); + runDependencyWatcher = null; + } + if (dependenciesFulfilled) { + var callback = dependenciesFulfilled; + dependenciesFulfilled = null; + callback(); // can add another dependenciesFulfilled + } + } +} + +/** @param {string|number=} what */ +function abort(what) { + Module['onAbort']?.(what); + + what = 'Aborted(' + what + ')'; + // TODO(sbc): Should we remove printing and leave it up to whoever + // catches the exception? + err(what); + + ABORT = true; + EXITSTATUS = 1; + + what += '. Build with -sASSERTIONS for more info.'; + + // Use a wasm runtime error, because a JS error might be seen as a foreign + // exception, which means we'd run destructors on it. We need the error to + // simply make the program stop. + // FIXME This approach does not work in Wasm EH because it currently does not assume + // all RuntimeErrors are from traps; it decides whether a RuntimeError is from + // a trap or not based on a hidden field within the object. So at the moment + // we don't have a way of throwing a wasm trap from JS. TODO Make a JS API that + // allows this in the wasm spec. + + // Suppress closure compiler warning here. Closure compiler's builtin extern + // definition for WebAssembly.RuntimeError claims it takes no arguments even + // though it can. + // TODO(https://github.com/google/closure-compiler/pull/3913): Remove if/when upstream closure gets fixed. + /** @suppress {checkTypes} */ + var e = new WebAssembly.RuntimeError(what); + + readyPromiseReject(e); + // Throw the error whether or not MODULARIZE is set because abort is used + // in code paths apart from instantiation where an exception is expected + // to be thrown when abort is called. + throw e; +} + +// include: memoryprofiler.js +// end include: memoryprofiler.js +// include: URIUtils.js +// Prefix of data URIs emitted by SINGLE_FILE and related options. +var dataURIPrefix = 'data:application/octet-stream;base64,'; + +/** + * Indicates whether filename is a base64 data URI. + * @noinline + */ +var isDataURI = (filename) => filename.startsWith(dataURIPrefix); + +/** + * Indicates whether filename is delivered via file protocol (as opposed to http/https) + * @noinline + */ +var isFileURI = (filename) => filename.startsWith('file://'); +// end include: URIUtils.js +// include: runtime_exceptions.js +// end include: runtime_exceptions.js +var wasmBinaryFile; + wasmBinaryFile = 'data:application/octet-stream;base64,AGFzbQEAAAABHQZgAX8AYAABf2AAAGABfwF/YAJ/fwBgA39/fwF/Aw0MAgAEAgMBBQABAQADBAUBcAEBAQUGAQGAAoACBg4CfwFB8IuEBAt/AUEACweYAQoGbWVtb3J5AgARX193YXNtX2NhbGxfY3RvcnMAAAtIYXNoX1VwZGF0ZQABCkhhc2hfRmluYWwAAwlIYXNoX0luaXQABAxHZXRCdWZmZXJQdHIABRlfX2luZGlyZWN0X2Z1bmN0aW9uX3RhYmxlAQAJc3RhY2tTYXZlAAkMc3RhY2tSZXN0b3JlAAoKc3RhY2tBbGxvYwALCossDAIAC+4CAgV/AX5BACgCwAoiASABKQNAIgYgAK18NwNAAkACQAJAIAanQT9xIgINAEGACyEBIAAhAgwBC0HAACACayEDAkAgAEUNACADIAAgAyAASRshBCABIAJqIQVBACEBA0AgBSABIgFqQYALIAFqLQAAOgAAIAFBAWoiAiEBIAIgBEcNAAsLAkACQCAAIANJIgRFDQBBgAshASAAIQIMAQtBACgCwAoiAUHIAGogARACQYALIANqIQEgACADayECCyABIQEgAiECIAQNAQsgASEBAkACQCACIgJBwABPDQAgASEFIAIhAAwBCyACIQIgASEEA0BBACgCwApByABqIAQiBBACIAJBQGoiASECIARBwABqIgUhBCAFIQUgASEAIAFBP0sNAAsLIAUhBSAAIgBFDQBBACEBQQAhAgNAQQAoAsAKIAEiAWogBSABai0AADoAACACQQFqIgJB/wFxIgQhASACIQIgACAESw0ACwsLqCEBK38gACgCCCICIAAoAgQiAyAAKAIAIgRzcSADIARxcyAEQR53IARBE3dzIARBCndzaiAAKAIQIgVBGncgBUEVd3MgBUEHd3MgACgCHCIGaiAAKAIYIgcgACgCFCIIcyAFcSAHc2ogASgCACIJQRh0IAlBgP4DcUEIdHIgCUEIdkGA/gNxIAlBGHZyciIKakGY36iUBGoiC2oiCSAEcyADcSAJIARxcyAJQR53IAlBE3dzIAlBCndzaiAHIAEoAgQiDEEYdCAMQYD+A3FBCHRyIAxBCHZBgP4DcSAMQRh2cnIiDWogCyAAKAIMIg5qIg8gCCAFc3EgCHNqIA9BGncgD0EVd3MgD0EHd3NqQZGJ3YkHaiIQaiIMIAlzIARxIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAggASgCCCILQRh0IAtBgP4DcUEIdHIgC0EIdkGA/gNxIAtBGHZyciIRaiAQIAJqIhIgDyAFc3EgBXNqIBJBGncgEkEVd3MgEkEHd3NqQc/3g657aiITaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAUgASgCDCIQQRh0IBBBgP4DcUEIdHIgEEEIdkGA/gNxIBBBGHZyciIUaiATIANqIhMgEiAPc3EgD3NqIBNBGncgE0EVd3MgE0EHd3NqQaW3181+aiIVaiIQIAtzIAxxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIA8gASgCECIWQRh0IBZBgP4DcUEIdHIgFkEIdkGA/gNxIBZBGHZyciIXaiAVIARqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQduE28oDaiIYaiIPIBBzIAtxIA8gEHFzIA9BHncgD0ETd3MgD0EKd3NqIAEoAhQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGSASaiAYIAlqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQfGjxM8FaiIYaiIJIA9zIBBxIAkgD3FzIAlBHncgCUETd3MgCUEKd3NqIAEoAhgiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGiATaiAYIAxqIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQaSF/pF5aiIYaiIMIAlzIA9xIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAEoAhwiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiGyAWaiAYIAtqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQdW98dh6aiIYaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAEoAiAiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHCASaiAYIBBqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQZjVnsB9aiIYaiIQIAtzIAxxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAiQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHSATaiAYIA9qIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQYG2jZQBaiIYaiIPIBBzIAtxIA8gEHFzIA9BHncgD0ETd3MgD0EKd3NqIAEoAigiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHiAWaiAYIAlqIhYgEyASc3EgEnNqIBZBGncgFkEVd3MgFkEHd3NqQb6LxqECaiIYaiIJIA9zIBBxIAkgD3FzIAlBHncgCUETd3MgCUEKd3NqIAEoAiwiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiHyASaiAYIAxqIhIgFiATc3EgE3NqIBJBGncgEkEVd3MgEkEHd3NqQcP7sagFaiIYaiIMIAlzIA9xIAwgCXFzIAxBHncgDEETd3MgDEEKd3NqIAEoAjAiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiICATaiAYIAtqIhMgEiAWc3EgFnNqIBNBGncgE0EVd3MgE0EHd3NqQfS6+ZUHaiIYaiILIAxzIAlxIAsgDHFzIAtBHncgC0ETd3MgC0EKd3NqIAEoAjQiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiISAWaiAYIBBqIhAgEyASc3EgEnNqIBBBGncgEEEVd3MgEEEHd3NqQf7j+oZ4aiIYaiIWIAtzIAxxIBYgC3FzIBZBHncgFkETd3MgFkEKd3NqIAEoAjgiFUEYdCAVQYD+A3FBCHRyIBVBCHZBgP4DcSAVQRh2cnIiIiASaiAYIA9qIg8gECATc3EgE3NqIA9BGncgD0EVd3MgD0EHd3NqQaeN8N55aiIVaiISIBZzIAtxIBIgFnFzIBJBHncgEkETd3MgEkEKd3NqIAEoAjwiAUEYdCABQYD+A3FBCHRyIAFBCHZBgP4DcSABQRh2cnIiIyATaiAVIAlqIgEgDyAQc3EgEHNqIAFBGncgAUEVd3MgAUEHd3NqQfTi74x8aiIJaiEVIBIhGCAWISQgCyElIAkgDGohJiABIScgDyEoIBAhKSAjISMgIiEiICEhISAgISAgHyEfIB4hHiAdIR0gHCEcIBshGyAaIRogGSEZIBchFyAUIRQgESERIA0hECAKIQxBgAkhAUEQISoDQCAVIgkgGCIKcyAkIitxIAkgCnFzIAlBHncgCUETd3MgCUEKd3NqIBAiEEEZdyAQQQ53cyAQQQN2cyAMaiAdIh1qICIiFkEPdyAWQQ13cyAWQQp2c2oiDCApaiAmIhIgJyIPICgiE3NxIBNzaiASQRp3IBJBFXdzIBJBB3dzaiABIgEoAgBqIiRqIgsgCXMgCnEgCyAJcXMgC0EedyALQRN3cyALQQp3c2ogESIYQRl3IBhBDndzIBhBA3ZzIBBqIB4iHmogIyIVQQ93IBVBDXdzIBVBCnZzaiINIBNqIAEoAgRqICQgJWoiEyASIA9zcSAPc2ogE0EadyATQRV3cyATQQd3c2oiJWoiECALcyAJcSAQIAtxcyAQQR53IBBBE3dzIBBBCndzaiAUIiRBGXcgJEEOd3MgJEEDdnMgGGogHyIfaiAMQQ93IAxBDXdzIAxBCnZzaiIRIA9qIAEoAghqICUgK2oiGCATIBJzcSASc2ogGEEadyAYQRV3cyAYQQd3c2oiJWoiDyAQcyALcSAPIBBxcyAPQR53IA9BE3dzIA9BCndzaiAXIhdBGXcgF0EOd3MgF0EDdnMgJGogICIgaiANQQ93IA1BDXdzIA1BCnZzaiIUIBJqIAEoAgxqICUgCmoiCiAYIBNzcSATc2ogCkEadyAKQRV3cyAKQQd3c2oiJWoiEiAPcyAQcSASIA9xcyASQR53IBJBE3dzIBJBCndzaiATIBkiJEEZdyAkQQ53cyAkQQN2cyAXaiAhIiFqIBFBD3cgEUENd3MgEUEKdnNqIhdqIAEoAhBqICUgCWoiEyAKIBhzcSAYc2ogE0EadyATQRV3cyATQQd3c2oiJWoiCSAScyAPcSAJIBJxcyAJQR53IAlBE3dzIAlBCndzaiABKAIUIBoiGkEZdyAaQQ53cyAaQQN2cyAkaiAWaiAUQQ93IBRBDXdzIBRBCnZzaiIZaiAYaiAlIAtqIhggEyAKc3EgCnNqIBhBGncgGEEVd3MgGEEHd3NqIiVqIgsgCXMgEnEgCyAJcXMgC0EedyALQRN3cyALQQp3c2ogASgCGCAbIiRBGXcgJEEOd3MgJEEDdnMgGmogFWogF0EPdyAXQQ13cyAXQQp2c2oiGmogCmogJSAQaiIKIBggE3NxIBNzaiAKQRp3IApBFXdzIApBB3dzaiIlaiIQIAtzIAlxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAhwgHCIcQRl3IBxBDndzIBxBA3ZzICRqIAxqIBlBD3cgGUENd3MgGUEKdnNqIhtqIBNqICUgD2oiJCAKIBhzcSAYc2ogJEEadyAkQRV3cyAkQQd3c2oiE2oiDyAQcyALcSAPIBBxcyAPQR53IA9BE3dzIA9BCndzaiABKAIgIB1BGXcgHUEOd3MgHUEDdnMgHGogDWogGkEPdyAaQQ13cyAaQQp2c2oiHGogGGogEyASaiIYICQgCnNxIApzaiAYQRp3IBhBFXdzIBhBB3dzaiITaiISIA9zIBBxIBIgD3FzIBJBHncgEkETd3MgEkEKd3NqIAEoAiQgHkEZdyAeQQ53cyAeQQN2cyAdaiARaiAbQQ93IBtBDXdzIBtBCnZzaiIdaiAKaiATIAlqIgkgGCAkc3EgJHNqIAlBGncgCUEVd3MgCUEHd3NqIgpqIhMgEnMgD3EgEyAScXMgE0EedyATQRN3cyATQQp3c2ogASgCKCAfQRl3IB9BDndzIB9BA3ZzIB5qIBRqIBxBD3cgHEENd3MgHEEKdnNqIh5qICRqIAogC2oiCiAJIBhzcSAYc2ogCkEadyAKQRV3cyAKQQd3c2oiJGoiCyATcyAScSALIBNxcyALQR53IAtBE3dzIAtBCndzaiABKAIsICBBGXcgIEEOd3MgIEEDdnMgH2ogF2ogHUEPdyAdQQ13cyAdQQp2c2oiH2ogGGogJCAQaiIYIAogCXNxIAlzaiAYQRp3IBhBFXdzIBhBB3dzaiIkaiIQIAtzIBNxIBAgC3FzIBBBHncgEEETd3MgEEEKd3NqIAEoAjAgIUEZdyAhQQ53cyAhQQN2cyAgaiAZaiAeQQ93IB5BDXdzIB5BCnZzaiIgaiAJaiAkIA9qIiQgGCAKc3EgCnNqICRBGncgJEEVd3MgJEEHd3NqIg9qIgkgEHMgC3EgCSAQcXMgCUEedyAJQRN3cyAJQQp3c2ogASgCNCAWQRl3IBZBDndzIBZBA3ZzICFqIBpqIB9BD3cgH0ENd3MgH0EKdnNqIiFqIApqIA8gEmoiDyAkIBhzcSAYc2ogD0EadyAPQRV3cyAPQQd3c2oiCmoiEiAJcyAQcSASIAlxcyASQR53IBJBE3dzIBJBCndzaiABKAI4IBVBGXcgFUEOd3MgFUEDdnMgFmogG2ogIEEPdyAgQQ13cyAgQQp2c2oiImogGGogCiATaiITIA8gJHNxICRzaiATQRp3IBNBFXdzIBNBB3dzaiIYaiIWIBJzIAlxIBYgEnFzIBZBHncgFkETd3MgFkEKd3NqIAEoAjwgDEEZdyAMQQ53cyAMQQN2cyAVaiAcaiAhQQ93ICFBDXdzICFBCnZzaiIKaiAkaiAYIAtqIgsgEyAPc3EgD3NqIAtBGncgC0EVd3MgC0EHd3NqIiZqIishFSAWIRggEiEkIAkhJSAmIBBqIiwhJiALIScgEyEoIA8hKSAKISMgIiEiICEhISAgISAgHyEfIB4hHiAdIR0gHCEcIBshGyAaIRogGSEZIBchFyAUIRQgESERIA0hECAMIQwgAUHAAGohASAqIgpBEGohKiAKQTBJDQALIAAgDyAGajYCHCAAIBMgB2o2AhggACALIAhqNgIUIAAgLCAFajYCECAAIAkgDmo2AgwgACASIAJqNgIIIAAgFiADajYCBCAAICsgBGo2AgAL1AMDBX8BfgF7QQAoAsAKIgAgACgCQCIBQQJ2QQ9xIgJBAnRqIgMgAygCAEF/IAFBA3QiAXRBf3NxQYABIAF0czYCAAJAAkAgAkEOTw0AIAJBAWohAAwBCwJAIAJBDkcNACAAQQA2AjwLIABByABqIAAQAkEAIQALAkAgACIAQQ1LDQBBACgCwAogAEECdCIAakEAQTggAGsQBhoLQQAoAsAKIgAgACkDQCIFpyICQRt0IAJBC3RBgID8B3FyIAJBBXZBgP4DcSACQQN0QRh2cnI2AjwgACAFQh2IpyICQRh0IAJBgP4DcUEIdHIgAkEIdkGA/gNxIAJBGHZycjYCOCAAQcgAaiAAEAJBACgCwApBPGohAUEAIQADQCABQQcgACIAa0ECdGoiAiAC/QACACAG/Q0MDQ4PCAkKCwQFBgcAAQIDIAb9DQMCAQAHBgUECwoJCA8ODQwgBv0NDA0ODwgJCgsEBQYHAAECA/0LAgAgAEEEaiICIQAgAkEIRw0ACwJAQQAoAsAKIgMoAmhFDQAgA0HIAGohBEEAIQBBACECA0BBgAsgACIAaiAEIABqLQAAOgAAIAJBAWoiAkH/AXEiASEAIAIhAiADKAJoIAFLDQALCwtxAQJ/QQAoAsAKIgFCADcDQCABQcgAaiECAkAgAEHgAUcNACABQRw2AmggAkEQakEA/QAEsAj9CwIAIAJBAP0ABKAI/QsCAEEADwsgAUEgNgJoIAJBEGpBAP0ABJAI/QsCACACQQD9AASACP0LAgBBAAsFAEGACwvyAgIDfwF+AkAgAkUNACAAIAE6AAAgACACaiIDQX9qIAE6AAAgAkEDSQ0AIAAgAToAAiAAIAE6AAEgA0F9aiABOgAAIANBfmogAToAACACQQdJDQAgACABOgADIANBfGogAToAACACQQlJDQAgAEEAIABrQQNxIgRqIgMgAUH/AXFBgYKECGwiATYCACADIAIgBGtBfHEiBGoiAkF8aiABNgIAIARBCUkNACADIAE2AgggAyABNgIEIAJBeGogATYCACACQXRqIAE2AgAgBEEZSQ0AIAMgATYCGCADIAE2AhQgAyABNgIQIAMgATYCDCACQXBqIAE2AgAgAkFsaiABNgIAIAJBaGogATYCACACQWRqIAE2AgAgBCADQQRxQRhyIgVrIgJBIEkNACABrUKBgICAEH4hBiADIAVqIQEDQCABIAY3AxggASAGNwMQIAEgBjcDCCABIAY3AwAgAUEgaiEBIAJBYGoiAkEfSw0ACwsgAAsGACAAJAELBAAjAQsEACMACwYAIAAkAAsSAQJ/IwAgAGtBcHEiASQAIAELC9ICAgBBgAgLwAJn5glqha5nu3Lzbjw69U+lf1IOUYxoBZur2YMfGc3gW9ieBcEH1Xw2F91wMDlZDvcxC8D/ERVYaKeP+WSkT/q+mC+KQpFEN3HP+8C1pdu16VvCVjnxEfFZpII/ktVeHKuYqgfYAVuDEr6FMSTDfQxVdF2+cv6x3oCnBtybdPGbwcFpm+SGR77vxp3BD8yhDCRvLOktqoR0StypsFzaiPl2UlE+mG3GMajIJwOwx39Zv/ML4MZHkafVUWPKBmcpKRSFCrcnOCEbLvxtLE0TDThTVHMKZbsKanYuycKBhSxykqHov6JLZhqocItLwqNRbMcZ6JLRJAaZ1oU1DvRwoGoQFsGkGQhsNx5Md0gntbywNLMMHDlKqthOT8qcW/NvLmjugo90b2OleBR4yIQIAseM+v++kOtsUKT3o/m+8nhxxgBBwAoLBIAFgAA='; + if (!isDataURI(wasmBinaryFile)) { + wasmBinaryFile = locateFile(wasmBinaryFile); + } + +function getBinarySync(file) { + if (file == wasmBinaryFile && wasmBinary) { + return new Uint8Array(wasmBinary); + } + var binary = tryParseAsDataURI(file); + if (binary) { + return binary; + } + if (readBinary) { + return readBinary(file); + } + throw 'both async and sync fetching of the wasm failed'; +} + +function getBinaryPromise(binaryFile) { + + // Otherwise, getBinarySync should be able to get it synchronously + return Promise.resolve().then(() => getBinarySync(binaryFile)); +} + +function instantiateArrayBuffer(binaryFile, imports, receiver) { + return getBinaryPromise(binaryFile).then((binary) => { + return WebAssembly.instantiate(binary, imports); + }).then(receiver, (reason) => { + err(`failed to asynchronously prepare wasm: ${reason}`); + + abort(reason); + }); +} + +function instantiateAsync(binary, binaryFile, imports, callback) { + return instantiateArrayBuffer(binaryFile, imports, callback); +} + +// Create the wasm instance. +// Receives the wasm imports, returns the exports. +function createWasm() { + // prepare imports + var info = { + 'env': wasmImports, + 'wasi_snapshot_preview1': wasmImports, + }; + // Load the wasm module and create an instance of using native support in the JS engine. + // handle a generated wasm instance, receiving its exports and + // performing other necessary setup + /** @param {WebAssembly.Module=} module*/ + function receiveInstance(instance, module) { + wasmExports = instance.exports; + + + + wasmMemory = wasmExports['memory']; + + updateMemoryViews(); + + addOnInit(wasmExports['__wasm_call_ctors']); + + removeRunDependency('wasm-instantiate'); + return wasmExports; + } + // wait for the pthread pool (if any) + addRunDependency('wasm-instantiate'); + + // Prefer streaming instantiation if available. + function receiveInstantiationResult(result) { + // 'result' is a ResultObject object which has both the module and instance. + // receiveInstance() will swap in the exports (to Module.asm) so they can be called + // TODO: Due to Closure regression https://github.com/google/closure-compiler/issues/3193, the above line no longer optimizes out down to the following line. + // When the regression is fixed, can restore the above PTHREADS-enabled path. + receiveInstance(result['instance']); + } + + // User shell pages can write their own Module.instantiateWasm = function(imports, successCallback) callback + // to manually instantiate the Wasm module themselves. This allows pages to + // run the instantiation parallel to any other async startup actions they are + // performing. + // Also pthreads and wasm workers initialize the wasm instance through this + // path. + if (Module['instantiateWasm']) { + + try { + return Module['instantiateWasm'](info, receiveInstance); + } catch(e) { + err(`Module.instantiateWasm callback failed with error: ${e}`); + // If instantiation fails, reject the module ready promise. + readyPromiseReject(e); + } + } + + // If instantiation fails, reject the module ready promise. + instantiateAsync(wasmBinary, wasmBinaryFile, info, receiveInstantiationResult).catch(readyPromiseReject); + return {}; // no exports yet; we'll fill them in later +} + +// Globals used by JS i64 conversions (see makeSetValue) +var tempDouble; +var tempI64; + +// include: runtime_debug.js +// end include: runtime_debug.js +// === Body === +// end include: preamble.js + + + /** @constructor */ + function ExitStatus(status) { + this.name = 'ExitStatus'; + this.message = `Program terminated with exit(${status})`; + this.status = status; + } + + var callRuntimeCallbacks = (callbacks) => { + while (callbacks.length > 0) { + // Pass the module as the first argument. + callbacks.shift()(Module); + } + }; + + + /** + * @param {number} ptr + * @param {string} type + */ + function getValue(ptr, type = 'i8') { + if (type.endsWith('*')) type = '*'; + switch (type) { + case 'i1': return HEAP8[ptr]; + case 'i8': return HEAP8[ptr]; + case 'i16': return HEAP16[((ptr)>>1)]; + case 'i32': return HEAP32[((ptr)>>2)]; + case 'i64': abort('to do getValue(i64) use WASM_BIGINT'); + case 'float': return HEAPF32[((ptr)>>2)]; + case 'double': return HEAPF64[((ptr)>>3)]; + case '*': return HEAPU32[((ptr)>>2)]; + default: abort(`invalid type for getValue: ${type}`); + } + } + + var noExitRuntime = Module['noExitRuntime'] || true; + + + /** + * @param {number} ptr + * @param {number} value + * @param {string} type + */ + function setValue(ptr, value, type = 'i8') { + if (type.endsWith('*')) type = '*'; + switch (type) { + case 'i1': HEAP8[ptr] = value; break; + case 'i8': HEAP8[ptr] = value; break; + case 'i16': HEAP16[((ptr)>>1)] = value; break; + case 'i32': HEAP32[((ptr)>>2)] = value; break; + case 'i64': abort('to do setValue(i64) use WASM_BIGINT'); + case 'float': HEAPF32[((ptr)>>2)] = value; break; + case 'double': HEAPF64[((ptr)>>3)] = value; break; + case '*': HEAPU32[((ptr)>>2)] = value; break; + default: abort(`invalid type for setValue: ${type}`); + } + } +var wasmImports = { + +}; +var wasmExports = createWasm(); +var ___wasm_call_ctors = () => (___wasm_call_ctors = wasmExports['__wasm_call_ctors'])(); +var _Hash_Update = Module['_Hash_Update'] = (a0) => (_Hash_Update = Module['_Hash_Update'] = wasmExports['Hash_Update'])(a0); +var _Hash_Final = Module['_Hash_Final'] = () => (_Hash_Final = Module['_Hash_Final'] = wasmExports['Hash_Final'])(); +var _Hash_Init = Module['_Hash_Init'] = (a0) => (_Hash_Init = Module['_Hash_Init'] = wasmExports['Hash_Init'])(a0); +var _GetBufferPtr = Module['_GetBufferPtr'] = () => (_GetBufferPtr = Module['_GetBufferPtr'] = wasmExports['GetBufferPtr'])(); +var stackSave = () => (stackSave = wasmExports['stackSave'])(); +var stackRestore = (a0) => (stackRestore = wasmExports['stackRestore'])(a0); +var stackAlloc = (a0) => (stackAlloc = wasmExports['stackAlloc'])(a0); + + +// include: postamble.js +// === Auto-generated postamble setup entry stuff === + + + + +var calledRun; + +dependenciesFulfilled = function runCaller() { + // If run has never been called, and we should call run (INVOKE_RUN is true, and Module.noInitialRun is not false) + if (!calledRun) run(); + if (!calledRun) dependenciesFulfilled = runCaller; // try this again later, after new deps are fulfilled +}; + +function run() { + + if (runDependencies > 0) { + return; + } + + preRun(); + + // a preRun added a dependency, run will be called later + if (runDependencies > 0) { + return; + } + + function doRun() { + // run may have just been called through dependencies being fulfilled just in this very frame, + // or while the async setStatus time below was happening + if (calledRun) return; + calledRun = true; + Module['calledRun'] = true; + + if (ABORT) return; + + initRuntime(); + + readyPromiseResolve(Module); + if (Module['onRuntimeInitialized']) Module['onRuntimeInitialized'](); + + postRun(); + } + + if (Module['setStatus']) { + Module['setStatus']('Running...'); + setTimeout(function() { + setTimeout(function() { + Module['setStatus'](''); + }, 1); + doRun(); + }, 1); + } else + { + doRun(); + } +} + +if (Module['preInit']) { + if (typeof Module['preInit'] == 'function') Module['preInit'] = [Module['preInit']]; + while (Module['preInit'].length > 0) { + Module['preInit'].pop()(); + } +} + +run(); + +// end include: postamble.js + + + + return moduleArg.ready +} +); +})(); +export default Module; \ No newline at end of file diff --git a/node_modules/@huggingface/hub/src/vendor/lz4js/LICENSE b/node_modules/@huggingface/hub/src/vendor/lz4js/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..7dfda0d31577cba1033826adacb3c799c234790a --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/lz4js/LICENSE @@ -0,0 +1,11 @@ +ISC License + +Copyright 2019 John Chadwick + +Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +--- + +Note: this license is not actually included in https://github.com/Benzinga/lz4js, but the package.json specifies the ISC license diff --git a/node_modules/@huggingface/hub/src/vendor/lz4js/index.ts b/node_modules/@huggingface/hub/src/vendor/lz4js/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..150805de6f593a353bc282dba723794182145d0e --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/lz4js/index.ts @@ -0,0 +1,537 @@ +// lz4.js - An implementation of Lz4 in plain JavaScript. +// +// TODO: +// - Unify header parsing/writing. +// - Support options (block size, checksums) +// - Support streams +// - Better error handling (handle bad offset, etc.) +// - HC support (better search algorithm) +// - Tests/benchmarking + +import * as xxhash from "./xxh32.js"; +import * as util from "./util.js"; + +// Constants +// -- + +// Compression format parameters/constants. +const minMatch = 4; +const matchSearchLimit = 12; +const minTrailingLitterals = 5; +const skipTrigger = 6; +const hashSize = 1 << 16; + +// Token constants. +const mlBits = 4; +const mlMask = (1 << mlBits) - 1; +const runBits = 4; +const runMask = (1 << runBits) - 1; + +// Shared buffers +const blockBuf = makeBuffer(5 << 20); +const hashTable = makeHashTable(); + +// Frame constants. +const magicNum = 0x184d2204; + +// Frame descriptor flags. +const fdContentChksum = 0x4; +const fdContentSize = 0x8; +const fdBlockChksum = 0x10; +// var fdBlockIndep = 0x20; +const fdVersion = 0x40; +const fdVersionMask = 0xc0; + +// Block sizes. +const bsUncompressed = 0x80000000; +const bsDefault = 7; +const bsShift = 4; +const bsMask = 7; +const bsMap: Record = { + 4: 0x10000, + 5: 0x40000, + 6: 0x100000, + 7: 0x400000, +}; + +// Utility functions/primitives +// -- + +// Makes our hashtable. On older browsers, may return a plain array. +function makeHashTable() { + try { + return new Uint32Array(hashSize); + } catch (error) { + const hashTable = new Array(hashSize); + + for (let i = 0; i < hashSize; i++) { + hashTable[i] = 0; + } + + return hashTable; + } +} + +// Clear hashtable. +function clearHashTable(table: Uint32Array | number[]) { + for (let i = 0; i < hashSize; i++) { + table[i] = 0; + } +} + +// Makes a byte buffer. On older browsers, may return a plain array. +function makeBuffer(size: number) { + return new Uint8Array(size); +} + +function sliceArray(array: Uint8Array, start: number, end: number) { + return array.slice(start, end); +} + +// Implementation +// -- + +// Calculates an upper bound for lz4 compression. +export function compressBound(n: number) { + return (n + n / 255 + 16) | 0; +} + +// Calculates an upper bound for lz4 decompression, by reading the data. +export function decompressBound(src: Uint8Array) { + let sIndex = 0; + + // Read magic number + if (util.readU32(src, sIndex) !== magicNum) { + throw new Error("invalid magic number"); + } + + sIndex += 4; + + // Read descriptor + const descriptor = src[sIndex++]; + + // Check version + if ((descriptor & fdVersionMask) !== fdVersion) { + throw new Error("incompatible descriptor version " + (descriptor & fdVersionMask)); + } + + // Read flags + const useBlockSum = (descriptor & fdBlockChksum) !== 0; + const useContentSize = (descriptor & fdContentSize) !== 0; + + // Read block size + const bsIdx = (src[sIndex++] >> bsShift) & bsMask; + + if (bsMap[bsIdx] === undefined) { + throw new Error("invalid block size " + bsIdx); + } + + const maxBlockSize = bsMap[bsIdx]; + + // Get content size + if (useContentSize) { + return util.readU64(src, sIndex); + } + + // Checksum + sIndex++; + + // Read blocks. + let maxSize = 0; + while (true) { + let blockSize = util.readU32(src, sIndex); + sIndex += 4; + + if (blockSize & bsUncompressed) { + blockSize &= ~bsUncompressed; + maxSize += blockSize; + } else if (blockSize > 0) { + maxSize += maxBlockSize; + } + + if (blockSize === 0) { + return maxSize; + } + + if (useBlockSum) { + sIndex += 4; + } + + sIndex += blockSize; + } +} + +// Decompresses a block of Lz4. +export function decompressBlock(src: Uint8Array, dst: Uint8Array, sIndex: number, sLength: number, dIndex: number) { + let mLength, mOffset, sEnd, n, i; + const hasCopyWithin = dst.copyWithin !== undefined && dst.fill !== undefined; + + // Setup initial state. + sEnd = sIndex + sLength; + + // Consume entire input block. + while (sIndex < sEnd) { + const token = src[sIndex++]; + + // Copy literals. + let literalCount = token >> 4; + if (literalCount > 0) { + // Parse length. + if (literalCount === 0xf) { + while (true) { + literalCount += src[sIndex]; + if (src[sIndex++] !== 0xff) { + break; + } + } + } + + // Copy literals + for (n = sIndex + literalCount; sIndex < n; ) { + dst[dIndex++] = src[sIndex++]; + } + } + + if (sIndex >= sEnd) { + break; + } + + // Copy match. + mLength = token & 0xf; + + // Parse offset. + mOffset = src[sIndex++] | (src[sIndex++] << 8); + + // Parse length. + if (mLength === 0xf) { + while (true) { + mLength += src[sIndex]; + if (src[sIndex++] !== 0xff) { + break; + } + } + } + + mLength += minMatch; + + // Copy match + // prefer to use typedarray.copyWithin for larger matches + // NOTE: copyWithin doesn't work as required by LZ4 for overlapping sequences + // e.g. mOffset=1, mLength=30 (repeach char 30 times) + // we special case the repeat char w/ array.fill + if (hasCopyWithin && mOffset === 1) { + dst.fill(dst[dIndex - 1] | 0, dIndex, dIndex + mLength); + dIndex += mLength; + } else if (hasCopyWithin && mOffset > mLength && mLength > 31) { + dst.copyWithin(dIndex, dIndex - mOffset, dIndex - mOffset + mLength); + dIndex += mLength; + } else { + for (i = dIndex - mOffset, n = i + mLength; i < n; ) { + dst[dIndex++] = dst[i++] | 0; + } + } + } + + return dIndex; +} + +// Compresses a block with Lz4. +export function compressBlock( + src: Uint8Array, + dst: Uint8Array, + sIndex: number, + sLength: number, + hashTable: Uint32Array | number[], +) { + let mIndex, mAnchor, mLength, mOffset, mStep; + let literalCount, dIndex, sEnd, n; + + // Setup initial state. + dIndex = 0; + sEnd = sLength + sIndex; + mAnchor = sIndex; + + let searchMatchCount = (1 << skipTrigger) + 3; + + // Search for matches with a limit of matchSearchLimit bytes + // before the end of block (Lz4 spec limitation.) + while (sIndex <= sEnd - matchSearchLimit) { + const seq = util.readU32(src, sIndex); + let hash = util.hashU32(seq) >>> 0; + + // Crush hash to 16 bits. + hash = (((hash >> 16) ^ hash) >>> 0) & 0xffff; + + // Look for a match in the hashtable. NOTE: remove one; see below. + mIndex = hashTable[hash] - 1; + + // Put pos in hash table. NOTE: add one so that zero = invalid. + hashTable[hash] = sIndex + 1; + + // Determine if there is a match (within range.) + if (mIndex < 0 || (sIndex - mIndex) >>> 16 > 0 || util.readU32(src, mIndex) !== seq) { + mStep = searchMatchCount++ >> skipTrigger; + sIndex += mStep; + continue; + } + + searchMatchCount = (1 << skipTrigger) + 3; + + // Calculate literal count and offset. + literalCount = sIndex - mAnchor; + mOffset = sIndex - mIndex; + + // We've already matched one word, so get that out of the way. + sIndex += minMatch; + mIndex += minMatch; + + // Determine match length. + // N.B.: mLength does not include minMatch, Lz4 adds it back + // in decoding. + mLength = sIndex; + while (sIndex < sEnd - minTrailingLitterals && src[sIndex] === src[mIndex]) { + sIndex++; + mIndex++; + } + mLength = sIndex - mLength; + + // Write token + literal count. + const token = mLength < mlMask ? mLength : mlMask; + if (literalCount >= runMask) { + dst[dIndex++] = (runMask << mlBits) + token; + for (n = literalCount - runMask; n >= 0xff; n -= 0xff) { + dst[dIndex++] = 0xff; + } + dst[dIndex++] = n; + } else { + dst[dIndex++] = (literalCount << mlBits) + token; + } + + // Write literals. + for (let i = 0; i < literalCount; i++) { + dst[dIndex++] = src[mAnchor + i]; + } + + // Write offset. + dst[dIndex++] = mOffset; + dst[dIndex++] = mOffset >> 8; + + // Write match length. + if (mLength >= mlMask) { + for (n = mLength - mlMask; n >= 0xff; n -= 0xff) { + dst[dIndex++] = 0xff; + } + dst[dIndex++] = n; + } + + // Move the anchor. + mAnchor = sIndex; + } + + // Nothing was encoded. + if (mAnchor === 0) { + return 0; + } + + // Write remaining literals. + // Write literal token+count. + literalCount = sEnd - mAnchor; + if (literalCount >= runMask) { + dst[dIndex++] = runMask << mlBits; + for (n = literalCount - runMask; n >= 0xff; n -= 0xff) { + dst[dIndex++] = 0xff; + } + dst[dIndex++] = n; + } else { + dst[dIndex++] = literalCount << mlBits; + } + + // Write literals. + sIndex = mAnchor; + while (sIndex < sEnd) { + dst[dIndex++] = src[sIndex++]; + } + + return dIndex; +} + +// Decompresses a frame of Lz4 data. +export function decompressFrame(src: Uint8Array, dst: Uint8Array) { + let useBlockSum, useContentSum, useContentSize, descriptor; + let sIndex = 0; + let dIndex = 0; + + // Read magic number + if (util.readU32(src, sIndex) !== magicNum) { + throw new Error("invalid magic number"); + } + + sIndex += 4; + + // Read descriptor + descriptor = src[sIndex++]; + + // Check version + if ((descriptor & fdVersionMask) !== fdVersion) { + throw new Error("incompatible descriptor version"); + } + + // Read flags + useBlockSum = (descriptor & fdBlockChksum) !== 0; + useContentSum = (descriptor & fdContentChksum) !== 0; + useContentSize = (descriptor & fdContentSize) !== 0; + + // Read block size + const bsIdx = (src[sIndex++] >> bsShift) & bsMask; + + if (bsMap[bsIdx] === undefined) { + throw new Error("invalid block size"); + } + + if (useContentSize) { + // TODO: read content size + sIndex += 8; + } + + sIndex++; + + // Read blocks. + while (true) { + var compSize; + + compSize = util.readU32(src, sIndex); + sIndex += 4; + + if (compSize === 0) { + break; + } + + if (useBlockSum) { + // TODO: read block checksum + sIndex += 4; + } + + // Check if block is compressed + if ((compSize & bsUncompressed) !== 0) { + // Mask off the 'uncompressed' bit + compSize &= ~bsUncompressed; + + // Copy uncompressed data into destination buffer. + for (let j = 0; j < compSize; j++) { + dst[dIndex++] = src[sIndex++]; + } + } else { + // Decompress into blockBuf + dIndex = decompressBlock(src, dst, sIndex, compSize, dIndex); + sIndex += compSize; + } + } + + if (useContentSum) { + // TODO: read content checksum + sIndex += 4; + } + + return dIndex; +} + +// Compresses data to an Lz4 frame. +export function compressFrame(src: Uint8Array, dst: Uint8Array) { + let dIndex = 0; + + // Write magic number. + util.writeU32(dst, dIndex, magicNum); + dIndex += 4; + + // Descriptor flags. + dst[dIndex++] = fdVersion; + dst[dIndex++] = bsDefault << bsShift; + + // Descriptor checksum. + dst[dIndex] = xxhash.hash(0, dst, 4, dIndex - 4) >> 8; + dIndex++; + + // Write blocks. + const maxBlockSize = bsMap[bsDefault]; + let remaining = src.length; + let sIndex = 0; + + // Clear the hashtable. + clearHashTable(hashTable); + + // Split input into blocks and write. + while (remaining > 0) { + let compSize = 0; + const blockSize = remaining > maxBlockSize ? maxBlockSize : remaining; + + compSize = compressBlock(src, blockBuf, sIndex, blockSize, hashTable); + + if (compSize > blockSize || compSize === 0) { + // Output uncompressed. + util.writeU32(dst, dIndex, 0x80000000 | blockSize); + dIndex += 4; + + for (let z = sIndex + blockSize; sIndex < z; ) { + dst[dIndex++] = src[sIndex++]; + } + + remaining -= blockSize; + } else { + // Output compressed. + util.writeU32(dst, dIndex, compSize); + dIndex += 4; + + for (let j = 0; j < compSize; ) { + dst[dIndex++] = blockBuf[j++]; + } + + sIndex += blockSize; + remaining -= blockSize; + } + } + + // Write blank end block. + util.writeU32(dst, dIndex, 0); + dIndex += 4; + + return dIndex; +} + +// Decompresses a buffer containing an Lz4 frame. maxSize is optional; if not +// provided, a maximum size will be determined by examining the data. The +// buffer returned will always be perfectly-sized. +export function decompress(src: Uint8Array, maxSize: number) { + let dst, size; + + if (maxSize === undefined) { + maxSize = decompressBound(src); + } + dst = makeBuffer(maxSize); + size = decompressFrame(src, dst); + + if (size !== maxSize) { + dst = sliceArray(dst, 0, size); + } + + return dst; +} + +// Compresses a buffer to an Lz4 frame. maxSize is optional; if not provided, +// a buffer will be created based on the theoretical worst output size for a +// given input size. The buffer returned will always be perfectly-sized. +export function compress(src: Uint8Array, maxSize?: number) { + let dst, size; + + if (maxSize === undefined) { + maxSize = compressBound(src.length); + } + + dst = makeBuffer(maxSize); + size = compressFrame(src, dst); + + if (size !== maxSize) { + dst = sliceArray(dst, 0, size); + } + + return dst; +} diff --git a/node_modules/@huggingface/hub/src/vendor/lz4js/util.ts b/node_modules/@huggingface/hub/src/vendor/lz4js/util.ts new file mode 100644 index 0000000000000000000000000000000000000000..5d579b76604d2fe13537423de069f61eca50a2d4 --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/lz4js/util.ts @@ -0,0 +1,54 @@ +// Simple hash function, from: http://burtleburtle.net/bob/hash/integer.html. +// Chosen because it doesn't use multiply and achieves full avalanche. +export function hashU32(a: number): number { + a = a | 0; + a = (a + 2127912214 + (a << 12)) | 0; + a = a ^ -949894596 ^ (a >>> 19); + a = (a + 374761393 + (a << 5)) | 0; + a = (a + -744332180) ^ (a << 9); + a = (a + -42973499 + (a << 3)) | 0; + return (a ^ -1252372727 ^ (a >>> 16)) | 0; +} + +// Reads a 64-bit little-endian integer from an array. +export function readU64(b: Uint8Array, n: number): number { + let x = 0; + x |= b[n++] << 0; + x |= b[n++] << 8; + x |= b[n++] << 16; + x |= b[n++] << 24; + x |= b[n++] << 32; + x |= b[n++] << 40; + x |= b[n++] << 48; + x |= b[n++] << 56; + return x; +} + +// Reads a 32-bit little-endian integer from an array. +export function readU32(b: Uint8Array, n: number): number { + let x = 0; + x |= b[n++] << 0; + x |= b[n++] << 8; + x |= b[n++] << 16; + x |= b[n++] << 24; + return x; +} + +// Writes a 32-bit little-endian integer from an array. +export function writeU32(b: Uint8Array, n: number, x: number): void { + b[n++] = (x >> 0) & 0xff; + b[n++] = (x >> 8) & 0xff; + b[n++] = (x >> 16) & 0xff; + b[n++] = (x >> 24) & 0xff; +} + +// Multiplies two numbers using 32-bit integer multiplication. +// Algorithm from Emscripten. +export function imul(a: number, b: number): number { + const ah = a >>> 16; + const al = a & 65535; + const bh = b >>> 16; + const bl = b & 65535; + + return (al * bl + ((ah * bl + al * bh) << 16)) | 0; +} diff --git a/node_modules/@huggingface/hub/src/vendor/lz4js/xxh32.ts b/node_modules/@huggingface/hub/src/vendor/lz4js/xxh32.ts new file mode 100644 index 0000000000000000000000000000000000000000..b9d135c657c69da01cda9beab2abe1b06b009e2e --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/lz4js/xxh32.ts @@ -0,0 +1,96 @@ +// xxh32.js - implementation of xxhash32 in plain JavaScript +import * as util from "./util.js"; + +// xxhash32 primes +const prime1 = 0x9e3779b1; +const prime2 = 0x85ebca77; +const prime3 = 0xc2b2ae3d; +const prime4 = 0x27d4eb2f; +const prime5 = 0x165667b1; + +// Utility functions/primitives +// -- +function rotl32(x: number, r: number): number { + x = x | 0; + r = r | 0; + + return (x >>> ((32 - r) | 0)) | (x << r) | 0; +} + +function rotmul32(h: number, r: number, m: number): number { + h = h | 0; + r = r | 0; + m = m | 0; + + return util.imul((h >>> ((32 - r) | 0)) | (h << r), m) | 0; +} + +function shiftxor32(h: number, s: number): number { + h = h | 0; + s = s | 0; + + return ((h >>> s) ^ h) | 0; +} + +// Implementation +// -- + +function xxhapply(h: number, src: number, m0: number, s: number, m1: number): number { + return rotmul32(util.imul(src, m0) + h, s, m1); +} + +function xxh1(h: number, src: Uint8Array, index: number): number { + return rotmul32(h + util.imul(src[index], prime5), 11, prime1); +} + +function xxh4(h: number, src: Uint8Array, index: number): number { + return xxhapply(h, util.readU32(src, index), prime3, 17, prime4); +} + +function xxh16(h: number[], src: Uint8Array, index: number): number[] { + return [ + xxhapply(h[0], util.readU32(src, index + 0), prime2, 13, prime1), + xxhapply(h[1], util.readU32(src, index + 4), prime2, 13, prime1), + xxhapply(h[2], util.readU32(src, index + 8), prime2, 13, prime1), + xxhapply(h[3], util.readU32(src, index + 12), prime2, 13, prime1), + ]; +} + +function xxh32(seed: number, src: Uint8Array, index: number, len: number): number { + let h; + const l = len; + if (len >= 16) { + h = [seed + prime1 + prime2, seed + prime2, seed, seed - prime1]; + + while (len >= 16) { + h = xxh16(h, src, index); + + index += 16; + len -= 16; + } + + h = rotl32(h[0], 1) + rotl32(h[1], 7) + rotl32(h[2], 12) + rotl32(h[3], 18) + l; + } else { + h = (seed + prime5 + len) >>> 0; + } + + while (len >= 4) { + h = xxh4(h, src, index); + + index += 4; + len -= 4; + } + + while (len > 0) { + h = xxh1(h, src, index); + + index++; + len--; + } + + h = shiftxor32(util.imul(shiftxor32(util.imul(shiftxor32(h, 15), prime2), 13), prime3), 16); + + return h >>> 0; +} + +export const hash = xxh32; diff --git a/node_modules/@huggingface/hub/src/vendor/type-fest/basic.ts b/node_modules/@huggingface/hub/src/vendor/type-fest/basic.ts new file mode 100644 index 0000000000000000000000000000000000000000..3fa40a039955fec376a3e8cdc42dc425a00a859b --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/type-fest/basic.ts @@ -0,0 +1,31 @@ +/** +Matches a JSON object. + +This type can be useful to enforce some input to be JSON-compatible or as a super-type to be extended from. Don't use this as a direct return type as the user would have to double-cast it: `jsonObject as unknown as CustomResponse`. Instead, you could extend your CustomResponse type from it to ensure your type only uses JSON-compatible types: `interface CustomResponse extends JsonObject { … }`. + +@category JSON +*/ +export type JsonObject = { [Key in string]: JsonValue } & { [Key in string]?: JsonValue | undefined }; + +/** +Matches a JSON array. + +@category JSON +*/ +export type JsonArray = JsonValue[] | readonly JsonValue[]; + +/** +Matches any valid JSON primitive value. + +@category JSON +*/ +export type JsonPrimitive = string | number | boolean | null; + +/** +Matches any valid JSON value. + +@see `Jsonify` if you need to transform a type to one that is assignable to `JsonValue`. + +@category JSON +*/ +export type JsonValue = JsonPrimitive | JsonObject | JsonArray; diff --git a/node_modules/@huggingface/hub/src/vendor/type-fest/entries.ts b/node_modules/@huggingface/hub/src/vendor/type-fest/entries.ts new file mode 100644 index 0000000000000000000000000000000000000000..3975d823ae3297d6bd021dd5aae2568e66dea6d7 --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/type-fest/entries.ts @@ -0,0 +1,66 @@ +import type { ArrayEntry, MapEntry, ObjectEntry, SetEntry } from "./entry"; + +type ArrayEntries = Array>; +type MapEntries = Array>; +type ObjectEntries = Array>; +type SetEntries> = Array>; + +/** +Many collections have an `entries` method which returns an array of a given object's own enumerable string-keyed property [key, value] pairs. The `Entries` type will return the type of that collection's entries. + +For example the {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/entries|`Object`}, {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map/entries|`Map`}, {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/entries|`Array`}, and {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set/entries|`Set`} collections all have this method. Note that `WeakMap` and `WeakSet` do not have this method since their entries are not enumerable. + +@see `Entry` if you want to just access the type of a single entry. + +@example +``` +import type {Entries} from 'type-fest'; + +interface Example { + someKey: number; +} + +const manipulatesEntries = (examples: Entries) => examples.map(example => [ + // Does some arbitrary processing on the key (with type information available) + example[0].toUpperCase(), + + // Does some arbitrary processing on the value (with type information available) + example[1].toFixed() +]); + +const example: Example = {someKey: 1}; +const entries = Object.entries(example) as Entries; +const output = manipulatesEntries(entries); + +// Objects +const objectExample = {a: 1}; +const objectEntries: Entries = [['a', 1]]; + +// Arrays +const arrayExample = ['a', 1]; +const arrayEntries: Entries = [[0, 'a'], [1, 1]]; + +// Maps +const mapExample = new Map([['a', 1]]); +const mapEntries: Entries = [['a', 1]]; + +// Sets +const setExample = new Set(['a', 1]); +const setEntries: Entries = [['a', 'a'], [1, 1]]; +``` + +@category Object +@category Map +@category Set +@category Array +*/ +export type Entries = + BaseType extends Map + ? MapEntries + : BaseType extends Set + ? SetEntries + : BaseType extends readonly unknown[] + ? ArrayEntries + : BaseType extends object + ? ObjectEntries + : never; diff --git a/node_modules/@huggingface/hub/src/vendor/type-fest/entry.ts b/node_modules/@huggingface/hub/src/vendor/type-fest/entry.ts new file mode 100644 index 0000000000000000000000000000000000000000..1f8dd55ccc1d6b750408519e733cc0e848f26f4a --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/type-fest/entry.ts @@ -0,0 +1,69 @@ +type MapKey = BaseType extends Map ? KeyType : never; +type MapValue = BaseType extends Map ? ValueType : never; + +export type ArrayEntry = [number, BaseType[number]]; +export type MapEntry = [MapKey, MapValue]; +export type ObjectEntry = [keyof BaseType, BaseType[keyof BaseType]]; +export type SetEntry = BaseType extends Set ? [ItemType, ItemType] : never; + +/** +Many collections have an `entries` method which returns an array of a given object's own enumerable string-keyed property [key, value] pairs. The `Entry` type will return the type of that collection's entry. + +For example the {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/entries|`Object`}, {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map/entries|`Map`}, {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/entries|`Array`}, and {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set/entries|`Set`} collections all have this method. Note that `WeakMap` and `WeakSet` do not have this method since their entries are not enumerable. + +@see `Entries` if you want to just access the type of the array of entries (which is the return of the `.entries()` method). + +@example +``` +import type {Entry} from 'type-fest'; + +interface Example { + someKey: number; +} + +const manipulatesEntry = (example: Entry) => [ + // Does some arbitrary processing on the key (with type information available) + example[0].toUpperCase(), + + // Does some arbitrary processing on the value (with type information available) + example[1].toFixed(), +]; + +const example: Example = {someKey: 1}; +const entry = Object.entries(example)[0] as Entry; +const output = manipulatesEntry(entry); + +// Objects +const objectExample = {a: 1}; +const objectEntry: Entry = ['a', 1]; + +// Arrays +const arrayExample = ['a', 1]; +const arrayEntryString: Entry = [0, 'a']; +const arrayEntryNumber: Entry = [1, 1]; + +// Maps +const mapExample = new Map([['a', 1]]); +const mapEntry: Entry = ['a', 1]; + +// Sets +const setExample = new Set(['a', 1]); +const setEntryString: Entry = ['a', 'a']; +const setEntryNumber: Entry = [1, 1]; +``` + +@category Object +@category Map +@category Array +@category Set +*/ +export type Entry = + BaseType extends Map + ? MapEntry + : BaseType extends Set + ? SetEntry + : BaseType extends readonly unknown[] + ? ArrayEntry + : BaseType extends object + ? ObjectEntry + : never; diff --git a/node_modules/@huggingface/hub/src/vendor/type-fest/except.ts b/node_modules/@huggingface/hub/src/vendor/type-fest/except.ts new file mode 100644 index 0000000000000000000000000000000000000000..d6a38169ee1fe3239f8b8b561600007ebab6c0f1 --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/type-fest/except.ts @@ -0,0 +1,68 @@ +import type { IsEqual } from "./is-equal"; + +/** +Filter out keys from an object. + +Returns `never` if `Exclude` is strictly equal to `Key`. +Returns `never` if `Key` extends `Exclude`. +Returns `Key` otherwise. + +@example +``` +type Filtered = Filter<'foo', 'foo'>; +//=> never +``` + +@example +``` +type Filtered = Filter<'bar', string>; +//=> never +``` + +@example +``` +type Filtered = Filter<'bar', 'foo'>; +//=> 'bar' +``` + +@see {Except} +*/ +type Filter = + IsEqual extends true ? never : KeyType extends ExcludeType ? never : KeyType; + +/** +Create a type from an object type without certain keys. + +We recommend setting the `requireExactProps` option to `true`. + +This type is a stricter version of [`Omit`](https://www.typescriptlang.org/docs/handbook/release-notes/typescript-3-5.html#the-omit-helper-type). The `Omit` type does not restrict the omitted keys to be keys present on the given type, while `Except` does. The benefits of a stricter type are avoiding typos and allowing the compiler to pick up on rename refactors automatically. + +This type was proposed to the TypeScript team, which declined it, saying they prefer that libraries implement stricter versions of the built-in types ([microsoft/TypeScript#30825](https://github.com/microsoft/TypeScript/issues/30825#issuecomment-523668235)). + +@example +``` +import type {Except} from 'type-fest'; + +type Foo = { + a: number; + b: string; +}; + +type FooWithoutA = Except; +//=> {b: string} + +const fooWithoutA: FooWithoutA = {a: 1, b: '2'}; +//=> errors: 'a' does not exist in type '{ b: string; }' + +type FooWithoutB = Except; +//=> {a: number} & Partial> + +const fooWithoutB: FooWithoutB = {a: 1, b: '2'}; +//=> errors at 'b': Type 'string' is not assignable to type 'undefined'. +``` + +@category Object +*/ +export type Except = { + [KeyType in keyof ObjectType as Filter]: ObjectType[KeyType]; +}; diff --git a/node_modules/@huggingface/hub/src/vendor/type-fest/is-equal.ts b/node_modules/@huggingface/hub/src/vendor/type-fest/is-equal.ts new file mode 100644 index 0000000000000000000000000000000000000000..d6ff2e53c4d25df11584f0a59504e08170e0a3b8 --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/type-fest/is-equal.ts @@ -0,0 +1,27 @@ +/** +Returns a boolean for whether the two given types are equal. + +@link https://github.com/microsoft/TypeScript/issues/27024#issuecomment-421529650 +@link https://stackoverflow.com/questions/68961864/how-does-the-equals-work-in-typescript/68963796#68963796 + +Use-cases: +- If you want to make a conditional branch based on the result of a comparison of two types. + +@example +``` +import type {IsEqual} from 'type-fest'; + +// This type returns a boolean for whether the given array includes the given item. +// `IsEqual` is used to compare the given array at position 0 and the given item and then return true if they are equal. +type Includes = + Value extends readonly [Value[0], ...infer rest] + ? IsEqual extends true + ? true + : Includes + : false; +``` + +@category Type Guard +@category Utilities +*/ +export type IsEqual = (() => G extends A ? 1 : 2) extends () => G extends B ? 1 : 2 ? true : false; diff --git a/node_modules/@huggingface/hub/src/vendor/type-fest/license-cc0 b/node_modules/@huggingface/hub/src/vendor/type-fest/license-cc0 new file mode 100644 index 0000000000000000000000000000000000000000..0e259d42c996742e9e3cba14c677129b2c1b6311 --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/type-fest/license-cc0 @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/node_modules/@huggingface/hub/src/vendor/type-fest/license-mit b/node_modules/@huggingface/hub/src/vendor/type-fest/license-mit new file mode 100644 index 0000000000000000000000000000000000000000..fa7ceba3eb4a9657a9db7f3ffca4e4e97a9019de --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/type-fest/license-mit @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/node_modules/@huggingface/hub/src/vendor/type-fest/set-required.ts b/node_modules/@huggingface/hub/src/vendor/type-fest/set-required.ts new file mode 100644 index 0000000000000000000000000000000000000000..8e4c6417a92e2b600e66cb00d2c4ff9675620269 --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/type-fest/set-required.ts @@ -0,0 +1,34 @@ +import type { Except } from "./except"; +import type { Simplify } from "./simplify"; + +/** +Create a type that makes the given keys required. The remaining keys are kept as is. The sister of the `SetOptional` type. + +Use-case: You want to define a single model where the only thing that changes is whether or not some of the keys are required. + +@example +``` +import type {SetRequired} from 'type-fest'; + +type Foo = { + a?: number; + b: string; + c?: boolean; +} + +type SomeRequired = SetRequired; +// type SomeRequired = { +// a?: number; +// b: string; // Was already required and still is. +// c: boolean; // Is now required. +// } +``` + +@category Object +*/ +export type SetRequired = Simplify< + // Pick just the keys that are optional from the base type. + Except & + // Pick the keys that should be required from the base type and make them required. + Required> +>; diff --git a/node_modules/@huggingface/hub/src/vendor/type-fest/simplify.ts b/node_modules/@huggingface/hub/src/vendor/type-fest/simplify.ts new file mode 100644 index 0000000000000000000000000000000000000000..f4564fe7043dd43ce6ac4de4b332d7c85915910c --- /dev/null +++ b/node_modules/@huggingface/hub/src/vendor/type-fest/simplify.ts @@ -0,0 +1,59 @@ +/** +Useful to flatten the type output to improve type hints shown in editors. And also to transform an interface into a type to aide with assignability. + +@example +``` +import type {Simplify} from 'type-fest'; + +type PositionProps = { + top: number; + left: number; +}; + +type SizeProps = { + width: number; + height: number; +}; + +// In your editor, hovering over `Props` will show a flattened object with all the properties. +type Props = Simplify; +``` + +Sometimes it is desired to pass a value as a function argument that has a different type. At first inspection it may seem assignable, and then you discover it is not because the `value`'s type definition was defined as an interface. In the following example, `fn` requires an argument of type `Record`. If the value is defined as a literal, then it is assignable. And if the `value` is defined as type using the `Simplify` utility the value is assignable. But if the `value` is defined as an interface, it is not assignable because the interface is not sealed and elsewhere a non-string property could be added to the interface. + +If the type definition must be an interface (perhaps it was defined in a third-party npm package), then the `value` can be defined as `const value: Simplify = ...`. Then `value` will be assignable to the `fn` argument. Or the `value` can be cast as `Simplify` if you can't re-declare the `value`. + +@example +``` +import type {Simplify} from 'type-fest'; + +interface SomeInterface { + foo: number; + bar?: string; + baz: number | undefined; +} + +type SomeType = { + foo: number; + bar?: string; + baz: number | undefined; +}; + +const literal = {foo: 123, bar: 'hello', baz: 456}; +const someType: SomeType = literal; +const someInterface: SomeInterface = literal; + +function fn(object: Record): void {} + +fn(literal); // Good: literal object type is sealed +fn(someType); // Good: type is sealed +fn(someInterface); // Error: Index signature for type 'string' is missing in type 'someInterface'. Because `interface` can be re-opened +fn(someInterface as Simplify); // Good: transform an `interface` into a `type` +``` + +@link https://github.com/microsoft/TypeScript/issues/15300 + +@category Object +*/ +// eslint-disable-next-line @typescript-eslint/ban-types +export type Simplify = { [KeyType in keyof T]: T[KeyType] } & {}; diff --git a/node_modules/@huggingface/hub/tsconfig.json b/node_modules/@huggingface/hub/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..39d83f3de17ed66fd4c6afa8dad7491bd1fd356c --- /dev/null +++ b/node_modules/@huggingface/hub/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "allowSyntheticDefaultImports": true, + "lib": ["ES2022", "DOM"], + "module": "CommonJS", + "moduleResolution": "node", + "target": "ES2022", + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "skipLibCheck": true, + "noImplicitOverride": true, + "outDir": "./dist", + "declaration": true, + "declarationMap": true, + "sourceMap": true + }, + "include": ["src", "index.ts", "cli.ts"], + "exclude": ["dist"] +} diff --git a/node_modules/@huggingface/tasks/LICENSE b/node_modules/@huggingface/tasks/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..1e25cde938b8abaa64a25aba817a01b7aba6472d --- /dev/null +++ b/node_modules/@huggingface/tasks/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Hugging Face + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/README.md b/node_modules/@huggingface/tasks/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3d49eb3a8a963177742bf062d844fdf2c7f7890a --- /dev/null +++ b/node_modules/@huggingface/tasks/README.md @@ -0,0 +1,32 @@ +# Tasks + +This package contains the definition files (written in Typescript) for the huggingface.co hub's: + +- **pipeline types** (a.k.a. **task types**) - used to determine which widget to display on the model page, and which inference API to run. +- **default widget inputs** - when they aren't provided in the model card. +- definitions and UI elements for **model and dataset libraries**. + +Please add any missing ones to these definitions by opening a PR. Thanks 🔥 + +⚠️ The hub's definitive doc is at https://huggingface.co/docs/hub. + +## Definition of Tasks + +This package also contains data used to define https://huggingface.co/tasks. + +The Task pages are made to lower the barrier of entry to understand a task that can be solved with machine learning and use or train a model to accomplish it. It's a collaborative documentation effort made to help out software developers, social scientists, or anyone with no background in machine learning that is interested in understanding how machine learning models can be used to solve a problem. + +The task pages avoid jargon to let everyone understand the documentation, and if specific terminology is needed, it is explained on the most basic level possible. This is important to understand before contributing to Tasks: at the end of every task page, the user is expected to be able to find and pull a model from the Hub and use it on their data and see if it works for their use case to come up with a proof of concept. + +## How to Contribute +You can open a pull request to contribute a new documentation about a new task. Under `src/tasks` we have a folder for every task that contains two files, `about.md` and `data.ts`. `about.md` contains the markdown part of the page, use cases, resources and minimal code block to infer a model that belongs to the task. `data.ts` contains redirections to canonical models and datasets, metrics, the schema of the task and the information the inference widget needs. + +![Anatomy of a Task Page](https://huggingface.co/datasets/huggingfacejs/tasks/resolve/main/contribution-guide/anatomy.png) + +We have a [`dataset`](https://huggingface.co/datasets/huggingfacejs/tasks) that contains data used in the inference widget. The last file is `const.ts`, which has the task to library mapping (e.g. spacy to token-classification) where you can add a library. They will look in the top right corner like below. + +![Libraries of a Task](https://huggingface.co/datasets/huggingfacejs/tasks/resolve/main/contribution-guide/libraries.png) + +This might seem overwhelming, but you don't necessarily need to add all of these in one pull request or on your own, you can simply contribute one section. Feel free to ask for help whenever you need. + +## Feedback (feature requests, bugs, etc.) is super welcome 💙💚💛💜♥️🧡 diff --git a/node_modules/@huggingface/tasks/dist/commonjs/agent-harnesses.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/agent-harnesses.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..0438a2882598de35dc1868d839130942dfbc5acf --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/agent-harnesses.d.ts @@ -0,0 +1,264 @@ +/** + * Registry of AI coding agents / harnesses known to use the Hugging Face Hub. + * + * To add your harness, append an entry below keyed by its `id` (the name used + * when reporting Hub activity), and list the environment variable(s) that + * identify it. + */ +export interface AgentHarness { + /** + * Human-readable name of the harness, e.g. displayed in a leaderboard. + */ + prettyLabel: string; + /** + * URL to the harness's code repository (usually on GitHub). + */ + repoUrl?: string; + /** + * URL to the harness's documentation or website. + */ + docsUrl?: string; + /** + * Short description of the harness. + */ + description?: string; + /** + * Environment variable(s) that identify this harness, mapped to the value + * pattern they must match. Detection matches if ANY entry matches. + * + * The value pattern is one of: + * - `"*"`: the variable is set to any (non-empty) value + * - `""`: the variable equals this exact value + * - `"*"`: the variable value starts with `` (fuzzy match, resolved client-side) + * + * If not provided, the harness is detected through the standard AI_AGENT / AGENT variables only. + */ + envVars?: Record; +} +/** + * Standard environment variables that any tool can set to identify itself. + * When one of these is set, its value is used directly as the agent `id` + * (matched against the keys of `AGENT_HARNESSES`); unrecognized values are + * reported as `"unknown"`. + */ +export declare const STANDARD_AGENT_ENV_VARS: readonly ["AI_AGENT", "AGENT"]; +/** + * Add your new agent harness here. + * + * /!\ IMPORTANT + * + * Insertion order matters for detection priority: harnesses are checked from + * top to bottom and the first match wins. In particular, `cowork` must stay + * before `claude-code` so the more specific signal takes priority when both + * `CLAUDE_CODE` and `CLAUDE_CODE_IS_COWORK` are set. + */ +export declare const AGENT_HARNESSES: { + antigravity: { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + ANTIGRAVITY_AGENT: string; + }; + }; + "augment-cli": { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + AUGMENT_AGENT: string; + }; + }; + cline: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + CLINE_ACTIVE: string; + }; + }; + cowork: { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + CLAUDE_CODE_IS_COWORK: string; + }; + }; + "claude-code": { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + CLAUDECODE: string; + CLAUDE_CODE: string; + }; + }; + codex: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + CODEX_SANDBOX: string; + CODEX_CI: string; + CODEX_THREAD_ID: string; + }; + }; + crush: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + CRUSH: string; + }; + }; + "gemini-cli": { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + GEMINI_CLI: string; + }; + }; + "github-copilot": { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + COPILOT_MODEL: string; + COPILOT_ALLOW_ALL: string; + COPILOT_GITHUB_TOKEN: string; + }; + }; + goose: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + GOOSE_TERMINAL: string; + }; + }; + "hermes-agent": { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + HERMES_SESSION_ID: string; + }; + }; + hi: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + }; + "kilo-code": { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + KILOCODE_FEATURE: string; + }; + }; + kiro: { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + AGENT_CONTEXT_OUT: string; + }; + }; + openclaw: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + OPENCLAW_SHELL: string; + }; + }; + opencode: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + OPENCODE_CLIENT: string; + }; + }; + pi: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + PI_CODING_AGENT: string; + }; + }; + replit: { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + REPL_ID: string; + }; + }; + trae: { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + TRAE_AI_SHELL_ID: string; + }; + }; + warp: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + TERM_PROGRAM: string; + }; + }; + zed: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + ZED_TERM: string; + }; + }; + "cursor-cli": { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + CURSOR_AGENT: string; + }; + }; + cursor: { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + CURSOR_TRACE_ID: string; + }; + }; + devin: { + prettyLabel: string; + docsUrl: string; + description: string; + }; +}; +export type AgentHarnessKey = keyof typeof AGENT_HARNESSES; +//# sourceMappingURL=agent-harnesses.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/agent-harnesses.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/agent-harnesses.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..291dd4c88d5c3f53d749702b4313a0ccf1859a78 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/agent-harnesses.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"agent-harnesses.d.ts","sourceRoot":"","sources":["../../src/agent-harnesses.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IAC5B;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAiC,CAAC;AAEtE;;;;;;;;;GASG;AACH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkKY,CAAC;AAGzC,MAAM,MAAM,eAAe,GAAG,MAAM,OAAO,eAAe,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/agent-harnesses.js b/node_modules/@huggingface/tasks/dist/commonjs/agent-harnesses.js new file mode 100644 index 0000000000000000000000000000000000000000..3ef48468a79d58294ba4380c8854937b03a617f6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/agent-harnesses.js @@ -0,0 +1,183 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.AGENT_HARNESSES = exports.STANDARD_AGENT_ENV_VARS = void 0; +/** + * Standard environment variables that any tool can set to identify itself. + * When one of these is set, its value is used directly as the agent `id` + * (matched against the keys of `AGENT_HARNESSES`); unrecognized values are + * reported as `"unknown"`. + */ +exports.STANDARD_AGENT_ENV_VARS = ["AI_AGENT", "AGENT"]; +/** + * Add your new agent harness here. + * + * /!\ IMPORTANT + * + * Insertion order matters for detection priority: harnesses are checked from + * top to bottom and the first match wins. In particular, `cowork` must stay + * before `claude-code` so the more specific signal takes priority when both + * `CLAUDE_CODE` and `CLAUDE_CODE_IS_COWORK` are set. + */ +exports.AGENT_HARNESSES = { + antigravity: { + prettyLabel: "Antigravity", + docsUrl: "https://antigravity.google", + description: "Agentic development platform from Google built around Gemini.", + envVars: { ANTIGRAVITY_AGENT: "*" }, + }, + "augment-cli": { + prettyLabel: "Augment CLI", + repoUrl: "https://github.com/augmentcode/auggie", + docsUrl: "https://www.augmentcode.com", + description: "Auggie, the command-line coding agent from Augment Code.", + envVars: { AUGMENT_AGENT: "*" }, + }, + cline: { + prettyLabel: "Cline", + repoUrl: "https://github.com/cline/cline", + docsUrl: "https://cline.bot", + description: "Open-source autonomous coding agent for VS Code.", + envVars: { CLINE_ACTIVE: "*" }, + }, + cowork: { + // must stay before `claude-code` so the more specific signal takes priority when both `CLAUDE_CODE` and `CLAUDE_CODE_IS_COWORK` are set. + prettyLabel: "Cowork", + docsUrl: "https://claude.com/product/cowork", + description: "Anthropic's agent for autonomous knowledge work, built on top of Claude Code.", + envVars: { CLAUDE_CODE_IS_COWORK: "*" }, + }, + "claude-code": { + prettyLabel: "Claude Code", + repoUrl: "https://github.com/anthropics/claude-code", + docsUrl: "https://code.claude.com/docs", + description: "Anthropic's agentic coding tool that lives in your terminal.", + envVars: { CLAUDECODE: "*", CLAUDE_CODE: "*" }, + }, + codex: { + prettyLabel: "Codex", + repoUrl: "https://github.com/openai/codex", + docsUrl: "https://developers.openai.com/codex", + description: "OpenAI's lightweight coding agent that runs in your terminal.", + envVars: { CODEX_SANDBOX: "*", CODEX_CI: "*", CODEX_THREAD_ID: "*" }, + }, + crush: { + prettyLabel: "Crush", + repoUrl: "https://github.com/charmbracelet/crush", + docsUrl: "https://github.com/charmbracelet/crush", + description: "Charm's open-source AI coding agent for the terminal.", + envVars: { CRUSH: "*" }, + }, + "gemini-cli": { + prettyLabel: "Gemini CLI", + repoUrl: "https://github.com/google-gemini/gemini-cli", + docsUrl: "https://geminicli.com", + description: "Google's open-source terminal AI coding agent powered by Gemini models.", + envVars: { GEMINI_CLI: "*" }, + }, + "github-copilot": { + prettyLabel: "GitHub Copilot", + docsUrl: "https://docs.github.com/copilot", + description: "GitHub's AI coding assistant.", + envVars: { COPILOT_MODEL: "*", COPILOT_ALLOW_ALL: "*", COPILOT_GITHUB_TOKEN: "*" }, + }, + goose: { + prettyLabel: "Goose", + repoUrl: "https://github.com/aaif-goose/goose", + docsUrl: "https://goose-docs.ai/", + description: "Open-source, extensible AI agent, originally from Block and now part of the Agentic AI Foundation.", + envVars: { GOOSE_TERMINAL: "*" }, + }, + "hermes-agent": { + prettyLabel: "Hermes Agent", + repoUrl: "https://github.com/NousResearch/hermes-agent", + docsUrl: "https://hermes-agent.nousresearch.com/docs", + description: "Nous Research's self-improving, multi-provider terminal AI agent.", + envVars: { HERMES_SESSION_ID: "*" }, + }, + hi: { + prettyLabel: "hi", + repoUrl: "https://github.com/PipeNetwork/hi", + docsUrl: "https://github.com/PipeNetwork/hi#readme", + description: "Rust terminal coding agent with verification-in-the-loop.", + }, + "kilo-code": { + prettyLabel: "Kilo Code", + repoUrl: "https://github.com/Kilo-Org/kilocode", + docsUrl: "https://kilocode.ai/docs", + description: "Open-source agentic coding agent for VS Code, JetBrains, and the terminal.", + envVars: { KILOCODE_FEATURE: "*" }, + }, + kiro: { + prettyLabel: "Kiro", + docsUrl: "https://kiro.dev", + description: "AWS's agentic IDE for spec-driven AI software development.", + envVars: { AGENT_CONTEXT_OUT: "*" }, + }, + openclaw: { + prettyLabel: "OpenClaw", + repoUrl: "https://github.com/openclaw/openclaw", + docsUrl: "https://openclaw.ai", + description: "Open-source, self-hosted personal AI assistant that runs on your own devices.", + envVars: { OPENCLAW_SHELL: "*" }, + }, + opencode: { + prettyLabel: "opencode", + repoUrl: "https://github.com/anomalyco/opencode", + docsUrl: "https://opencode.ai", + description: "Open-source AI coding agent built for the terminal.", + envVars: { OPENCODE_CLIENT: "*" }, + }, + pi: { + prettyLabel: "Pi", + repoUrl: "https://github.com/earendil-works/pi", + docsUrl: "https://pi.dev", + description: "Minimal, self-extensible terminal coding agent with a unified multi-provider LLM API.", + envVars: { PI_CODING_AGENT: "*" }, + }, + replit: { + prettyLabel: "Replit", + docsUrl: "https://replit.com", + description: "Cloud development environment with an AI coding agent.", + envVars: { REPL_ID: "*" }, + }, + trae: { + prettyLabel: "Trae", + docsUrl: "https://trae.ai", + description: "AI-powered IDE from ByteDance.", + envVars: { TRAE_AI_SHELL_ID: "*" }, + }, + warp: { + prettyLabel: "Warp", + repoUrl: "https://github.com/warpdotdev/Warp", + docsUrl: "https://docs.warp.dev", + description: "AI-powered terminal with an agentic Agent Mode.", + envVars: { TERM_PROGRAM: "WarpTerminal" }, + }, + zed: { + prettyLabel: "Zed", + repoUrl: "https://github.com/zed-industries/zed", + docsUrl: "https://zed.dev", + description: "High-performance code editor with an integrated AI agent panel and terminal.", + envVars: { ZED_TERM: "*" }, + }, + "cursor-cli": { + // Kept near the bottom (and before `cursor`): when another agent runs inside the Cursor editor's terminal, + // its child processes inherit `CURSOR_TRACE_ID`, so `cursor` must stay a low-priority fallback and lose to + // the agent's own marker. `cursor-cli` is the more specific Cursor signal (`CURSOR_AGENT`), so it comes first. + prettyLabel: "Cursor CLI", + docsUrl: "https://cursor.com/docs/cli/overview", + description: "Cursor's coding agent for the command line.", + envVars: { CURSOR_AGENT: "*" }, + }, + cursor: { + prettyLabel: "Cursor", + docsUrl: "https://cursor.com", + description: "AI-powered code editor.", + envVars: { CURSOR_TRACE_ID: "*" }, + }, + devin: { + prettyLabel: "Devin", + docsUrl: "https://devin.ai", + description: "Autonomous AI software engineer from Cognition.", + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/dataset-libraries.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/dataset-libraries.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..7dc46b97a7ba271fef3a38989e1f0dcd6a5e6af6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/dataset-libraries.d.ts @@ -0,0 +1,99 @@ +/** + * Elements configurable by a dataset library. + */ +export interface DatasetLibraryUiElement { + /** + * Pretty name of the library. + * displayed (in tags?, and) on the main + * call-to-action button on the dataset page. + */ + prettyLabel: string; + /** + * Repo name of the library's (usually on GitHub) code repo + */ + repoName: string; + /** + * URL to library's (usually on GitHub) code repo + */ + repoUrl: string; + /** + * URL to library's docs + */ + docsUrl?: string; +} +export declare const DATASET_LIBRARIES_UI_ELEMENTS: { + mlcroissant: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + webdataset: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + datasets: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + pandas: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + dask: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + distilabel: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + fiftyone: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + lance: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + argilla: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + polars: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + duckdb: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + datadesigner: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; +}; +export type DatasetLibraryKey = keyof typeof DATASET_LIBRARIES_UI_ELEMENTS; +//# sourceMappingURL=dataset-libraries.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/dataset-libraries.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/dataset-libraries.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..8c7553eddf91259a20de8bdc39ec70c1733e4dac --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/dataset-libraries.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"dataset-libraries.d.ts","sourceRoot":"","sources":["../../src/dataset-libraries.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC;;;;OAIG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyES,CAAC;AAGpD,MAAM,MAAM,iBAAiB,GAAG,MAAM,OAAO,6BAA6B,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/dataset-libraries.js b/node_modules/@huggingface/tasks/dist/commonjs/dataset-libraries.js new file mode 100644 index 0000000000000000000000000000000000000000..2fe1f0c33dbca3fe6c3c511730fd43c4b28eb6a9 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/dataset-libraries.js @@ -0,0 +1,77 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.DATASET_LIBRARIES_UI_ELEMENTS = void 0; +exports.DATASET_LIBRARIES_UI_ELEMENTS = { + mlcroissant: { + prettyLabel: "Croissant", + repoName: "croissant", + repoUrl: "https://github.com/mlcommons/croissant/tree/main/python/mlcroissant", + docsUrl: "https://huggingface.co/docs/dataset-viewer/mlcroissant", + }, + webdataset: { + prettyLabel: "WebDataset", + repoName: "webdataset", + repoUrl: "https://github.com/webdataset/webdataset", + docsUrl: "https://huggingface.co/docs/hub/datasets-webdataset", + }, + datasets: { + prettyLabel: "Datasets", + repoName: "datasets", + repoUrl: "https://github.com/huggingface/datasets", + docsUrl: "https://huggingface.co/docs/hub/datasets-usage", + }, + pandas: { + prettyLabel: "pandas", + repoName: "pandas", + repoUrl: "https://github.com/pandas-dev/pandas", + docsUrl: "https://huggingface.co/docs/hub/datasets-pandas", + }, + dask: { + prettyLabel: "Dask", + repoName: "dask", + repoUrl: "https://github.com/dask/dask", + docsUrl: "https://huggingface.co/docs/hub/datasets-dask", + }, + distilabel: { + prettyLabel: "Distilabel", + repoName: "distilabel", + repoUrl: "https://github.com/argilla-io/distilabel", + docsUrl: "https://huggingface.co/docs/hub/datasets-distilabel", + }, + fiftyone: { + prettyLabel: "FiftyOne", + repoName: "fiftyone", + repoUrl: "https://github.com/voxel51/fiftyone", + docsUrl: "https://huggingface.co/docs/hub/datasets-fiftyone", + }, + lance: { + prettyLabel: "Lance", + repoName: "lance", + repoUrl: "https://github.com/lance-format/lance", + docsUrl: "https://huggingface.co/docs/hub/datasets-lance", + }, + argilla: { + prettyLabel: "Argilla", + repoName: "argilla", + repoUrl: "https://github.com/argilla-io/argilla", + docsUrl: "https://huggingface.co/docs/hub/datasets-argilla", + }, + polars: { + prettyLabel: "Polars", + repoName: "polars", + repoUrl: "https://github.com/pola-rs/polars", + docsUrl: "https://huggingface.co/docs/hub/datasets-polars", + }, + duckdb: { + prettyLabel: "DuckDB", + repoName: "duckdb", + repoUrl: "https://github.com/duckdb/duckdb", + docsUrl: "https://huggingface.co/docs/hub/datasets-duckdb", + }, + datadesigner: { + prettyLabel: "NeMo Data Designer", + repoName: "datadesigner", + repoUrl: "https://github.com/NVIDIA-NeMo/DataDesigner", + docsUrl: "https://nvidia-nemo.github.io/DataDesigner/", + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/default-widget-inputs.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/default-widget-inputs.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b9bd77d2be8a451d248b0f8776226127e315a039 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/default-widget-inputs.d.ts @@ -0,0 +1,6 @@ +import type { WidgetExample } from "./widget-example.js"; +import type { WidgetType } from "./pipelines.js"; +type PerLanguageMapping = Map; +export declare const MAPPING_DEFAULT_WIDGET: Map; +export {}; +//# sourceMappingURL=default-widget-inputs.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/default-widget-inputs.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/default-widget-inputs.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e652e43f9eb09082b9409d0ec2c35eca1d52a541 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/default-widget-inputs.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"default-widget-inputs.d.ts","sourceRoot":"","sources":["../../src/default-widget-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAIjD,KAAK,kBAAkB,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,aAAa,EAAE,CAAC,CAAC;AAoqBtE,eAAO,MAAM,sBAAsB,iCAejC,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/default-widget-inputs.js b/node_modules/@huggingface/tasks/dist/commonjs/default-widget-inputs.js new file mode 100644 index 0000000000000000000000000000000000000000..9bfab33a3603c28c68a01d8827e69f21526b670c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/default-widget-inputs.js @@ -0,0 +1,677 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.MAPPING_DEFAULT_WIDGET = void 0; +/// NOTE TO CONTRIBUTORS: +/// +/// When adding sample inputs for a new language, you don't +/// necessarily have to translate the inputs from existing languages. +/// (which were quite random to begin with) +/// +/// i.e. Feel free to be creative and provide better samples. +// +/// The placeholder will be replaced by the correct mask token +/// in the following examples, depending on the model type +/// +/// see [INTERNAL] github.com/huggingface/moon-landing/blob/c5c3d45fe0ab27347b3ab27bdad646ef20732351/server/lib/App.ts#L254 +// +const MAPPING_EN = new Map([ + ["text-classification", [`I like you. I love you`]], + [ + "token-classification", + [ + `My name is Wolfgang and I live in Berlin`, + `My name is Sarah and I live in London`, + `My name is Clara and I live in Berkeley, California.`, + ], + ], + [ + "table-question-answering", + [ + { + text: `How many stars does the transformers repository have?`, + table: { + Repository: ["Transformers", "Datasets", "Tokenizers"], + Stars: [36542, 4512, 3934], + Contributors: [651, 77, 34], + "Programming language": ["Python", "Python", "Rust, Python and NodeJS"], + }, + }, + ], + ], + [ + "question-answering", + [ + { + text: `Where do I live?`, + context: `My name is Wolfgang and I live in Berlin`, + }, + { + text: `Where do I live?`, + context: `My name is Sarah and I live in London`, + }, + { + text: `What's my name?`, + context: `My name is Clara and I live in Berkeley.`, + }, + { + text: `Which name is also used to describe the Amazon rainforest in English?`, + context: `The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain "Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species.`, + }, + ], + ], + [ + "zero-shot-classification", + [ + { + text: "I have a problem with my iphone that needs to be resolved asap!", + candidate_labels: "urgent, not urgent, phone, tablet, computer", + multi_class: true, + }, + { + text: "Last week I upgraded my iOS version and ever since then my phone has been overheating whenever I use your app.", + candidate_labels: "mobile, website, billing, account access", + multi_class: false, + }, + { + text: "A new model offers an explanation for how the Galilean satellites formed around the solar system’s largest world. Konstantin Batygin did not set out to solve one of the solar system’s most puzzling mysteries when he went for a run up a hill in Nice, France. Dr. Batygin, a Caltech researcher, best known for his contributions to the search for the solar system’s missing “Planet Nine,” spotted a beer bottle. At a steep, 20 degree grade, he wondered why it wasn’t rolling down the hill. He realized there was a breeze at his back holding the bottle in place. Then he had a thought that would only pop into the mind of a theoretical astrophysicist: “Oh! This is how Europa formed.” Europa is one of Jupiter’s four large Galilean moons. And in a paper published Monday in the Astrophysical Journal, Dr. Batygin and a co-author, Alessandro Morbidelli, a planetary scientist at the Côte d’Azur Observatory in France, present a theory explaining how some moons form around gas giants like Jupiter and Saturn, suggesting that millimeter-sized grains of hail produced during the solar system’s formation became trapped around these massive worlds, taking shape one at a time into the potentially habitable moons we know today.", + candidate_labels: "space & cosmos, scientific discovery, microbiology, robots, archeology", + multi_class: true, + }, + ], + ], + ["translation", [`My name is Wolfgang and I live in Berlin`, `My name is Sarah and I live in London`]], + [ + "summarization", + [ + `The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct.`, + ], + ], + [ + "conversational", + [ + `Hi, what can you help me with?`, + `What is 84 * 3 / 2?`, + `Tell me an interesting fact about the universe!`, + `Explain quantum computing in simple terms.`, + ], + ], + [ + "text-generation", + [ + `My name is Julien and I like to`, + `I like traveling by train because`, + `Paris is an amazing place to visit,`, + `Once upon a time,`, + ], + ], + ["fill-mask", [`Paris is the of France.`, `The goal of life is .`]], + [ + "sentence-similarity", + [ + { + source_sentence: "That is a happy person", + sentences: ["That is a happy dog", "That is a very happy person", "Today is a sunny day"], + }, + ], + ], +]); +const MAPPING_ZH = new Map([ + ["text-classification", [`我喜欢你。 我爱你`]], + ["token-classification", [`我叫沃尔夫冈，我住在柏林。`, `我叫萨拉，我住在伦敦。`, `我叫克拉拉，我住在加州伯克利。`]], + [ + "question-answering", + [ + { + text: `我住在哪里？`, + context: `我叫沃尔夫冈，我住在柏林。`, + }, + { + text: `我住在哪里？`, + context: `我叫萨拉，我住在伦敦。`, + }, + { + text: `我的名字是什么？`, + context: `我叫克拉拉，我住在伯克利。`, + }, + ], + ], + ["translation", [`我叫沃尔夫冈，我住在柏林。`, `我叫萨拉，我住在伦敦。`]], + [ + "zero-shot-classification", + [ + { + text: "房间干净明亮，非常不错", + candidate_labels: "这是一条差评, 这是一条好评", + }, + ], + ], + [ + "summarization", + [ + `该塔高324米（1063英尺），与一幢81层的建筑物一样高，是巴黎最高的建筑物。 它的底座是方形的，每边长125米（410英尺）。 在建造过程中，艾菲尔铁塔超过了华盛顿纪念碑，成为世界上最高的人造结构，它保持了41年的头衔，直到1930年纽约市的克莱斯勒大楼竣工。这是第一个到达300米高度的结构。 由于1957年在塔顶增加了广播天线，因此它现在比克莱斯勒大厦高5.2米（17英尺）。 除发射器外，艾菲尔铁塔是法国第二高的独立式建筑，仅次于米劳高架桥。`, + ], + ], + [ + "text-generation", + [`我叫朱利安，我喜欢`, `我叫托马斯，我的主要`, `我叫玛丽亚，我最喜欢的`, `我叫克拉拉，我是`, `从前，`], + ], + ["fill-mask", [`巴黎是国的首都。`, `生活的真谛是。`]], + [ + "sentence-similarity", + [ + { + source_sentence: "那是 個快樂的人", + sentences: ["那是 條快樂的狗", "那是 個非常幸福的人", "今天是晴天"], + }, + ], + ], +]); +const MAPPING_FR = new Map([ + ["text-classification", [`Je t'apprécie beaucoup. Je t'aime.`]], + ["token-classification", [`Mon nom est Wolfgang et je vis à Berlin`]], + [ + "question-answering", + [ + { + text: `Où est-ce que je vis?`, + context: `Mon nom est Wolfgang et je vis à Berlin`, + }, + ], + ], + ["translation", [`Mon nom est Wolfgang et je vis à Berlin`]], + [ + "summarization", + [ + `La tour fait 324 mètres (1,063 pieds) de haut, environ la même hauteur qu'un immeuble de 81 étages, et est la plus haute structure de Paris. Sa base est carrée, mesurant 125 mètres (410 pieds) sur chaque côté. Durant sa construction, la tour Eiffel surpassa le Washington Monument pour devenir la plus haute structure construite par l'homme dans le monde, un titre qu'elle conserva pendant 41 ans jusqu'à l'achèvement du Chrysler Building à New-York City en 1930. Ce fut la première structure à atteindre une hauteur de 300 mètres. Avec l'ajout d'une antenne de radiodiffusion au sommet de la tour Eiffel en 1957, celle-ci redevint plus haute que le Chrysler Building de 5,2 mètres (17 pieds). En excluant les transmetteurs, elle est la seconde plus haute structure autoportante de France après le viaduc de Millau.`, + ], + ], + ["text-generation", [`Mon nom est Julien et j'aime`, `Mon nom est Thomas et mon principal`, `Il était une fois`]], + ["fill-mask", [`Paris est la de la France.`]], + [ + "sentence-similarity", + [ + { + source_sentence: "C'est une personne heureuse", + sentences: [ + "C'est un chien heureux", + "C'est une personne très heureuse", + "Aujourd'hui est une journée ensoleillée", + ], + }, + ], + ], +]); +const MAPPING_ES = new Map([ + ["text-classification", [`Te quiero. Te amo.`]], + ["token-classification", [`Me llamo Wolfgang y vivo en Berlin`]], + [ + "question-answering", + [ + { + text: `¿Dónde vivo?`, + context: `Me llamo Wolfgang y vivo en Berlin`, + }, + { + text: `¿Quién inventó el submarino?`, + context: `Isaac Peral fue un murciano que inventó el submarino`, + }, + { + text: `¿Cuántas personas hablan español?`, + context: `El español es el segundo idioma más hablado del mundo con más de 442 millones de hablantes`, + }, + ], + ], + [ + "translation", + [ + `Me llamo Wolfgang y vivo en Berlin`, + `Los ingredientes de una tortilla de patatas son: huevos, patatas y cebolla`, + ], + ], + [ + "summarization", + [ + `La torre tiene 324 metros (1.063 pies) de altura, aproximadamente la misma altura que un edificio de 81 pisos y la estructura más alta de París. Su base es cuadrada, mide 125 metros (410 pies) a cada lado. Durante su construcción, la Torre Eiffel superó al Washington Monument para convertirse en la estructura artificial más alta del mundo, un título que mantuvo durante 41 años hasta que el Chrysler Building en la ciudad de Nueva York se terminó en 1930. Fue la primera estructura en llegar Una altura de 300 metros. Debido a la adición de una antena de transmisión en la parte superior de la torre en 1957, ahora es más alta que el Chrysler Building en 5,2 metros (17 pies). Excluyendo los transmisores, la Torre Eiffel es la segunda estructura independiente más alta de Francia después del Viaducto de Millau.`, + ], + ], + [ + "text-generation", + [ + `Me llamo Julien y me gusta`, + `Me llamo Thomas y mi principal`, + `Me llamo Manuel y trabajo en`, + `Érase una vez,`, + `Si tú me dices ven, `, + ], + ], + ["fill-mask", [`Mi nombre es y vivo en Nueva York.`, `El español es un idioma muy en el mundo.`]], + [ + "sentence-similarity", + [ + { + source_sentence: "Esa es una persona feliz", + sentences: ["Ese es un perro feliz", "Esa es una persona muy feliz", "Hoy es un día soleado"], + }, + ], + ], +]); +const MAPPING_RU = new Map([ + ["text-classification", [`Ты мне нравишься. Я тебя люблю`]], + ["token-classification", [`Меня зовут Вольфганг и я живу в Берлине`]], + [ + "question-answering", + [ + { + text: `Где живу?`, + context: `Меня зовут Вольфганг и я живу в Берлине`, + }, + ], + ], + ["translation", [`Меня зовут Вольфганг и я живу в Берлине`]], + [ + "summarization", + [ + `Высота башни составляет 324 метра (1063 фута), примерно такая же высота, как у 81-этажного здания, и самое высокое сооружение в Париже. Его основание квадратно, размером 125 метров (410 футов) с любой стороны. Во время строительства Эйфелева башня превзошла монумент Вашингтона, став самым высоким искусственным сооружением в мире, и этот титул она удерживала в течение 41 года до завершения строительство здания Крайслер в Нью-Йорке в 1930 году. Это первое сооружение которое достигло высоты 300 метров. Из-за добавления вещательной антенны на вершине башни в 1957 году она сейчас выше здания Крайслер на 5,2 метра (17 футов). За исключением передатчиков, Эйфелева башня является второй самой высокой отдельно стоящей структурой во Франции после виадука Мийо.`, + ], + ], + ["text-generation", [`Меня зовут Жюльен и`, `Меня зовут Томас и мой основной`, `Однажды`]], + ["fill-mask", [`Меня зовут и я инженер живущий в Нью-Йорке.`]], + [ + "sentence-similarity", + [ + { + source_sentence: "Это счастливый человек", + sentences: ["Это счастливая собака", "Это очень счастливый человек", "Сегодня солнечный день"], + }, + ], + ], +]); +const MAPPING_UK = new Map([ + ["translation", [`Мене звати Вольфґанґ і я живу в Берліні.`]], + ["fill-mask", [`Мене звати .`]], +]); +const MAPPING_IT = new Map([ + ["text-classification", [`Mi piaci. Ti amo`]], + [ + "token-classification", + [ + `Mi chiamo Wolfgang e vivo a Berlino`, + `Mi chiamo Sarah e vivo a Londra`, + `Mi chiamo Clara e vivo a Berkeley in California.`, + ], + ], + [ + "question-answering", + [ + { + text: `Dove vivo?`, + context: `Mi chiamo Wolfgang e vivo a Berlino`, + }, + { + text: `Dove vivo?`, + context: `Mi chiamo Sarah e vivo a Londra`, + }, + { + text: `Come mio chiamo?`, + context: `Mi chiamo Clara e vivo a Berkeley.`, + }, + ], + ], + ["translation", [`Mi chiamo Wolfgang e vivo a Berlino`, `Mi chiamo Sarah e vivo a Londra`]], + [ + "summarization", + [ + `La torre degli Asinelli è una delle cosiddette due torri di Bologna, simbolo della città, situate in piazza di porta Ravegnana, all'incrocio tra le antiche strade San Donato (ora via Zamboni), San Vitale, Maggiore e Castiglione. Eretta, secondo la tradizione, fra il 1109 e il 1119 dal nobile Gherardo Asinelli, la torre è alta 97,20 metri, pende verso ovest per 2,23 metri e presenta all'interno una scalinata composta da 498 gradini. Ancora non si può dire con certezza quando e da chi fu costruita la torre degli Asinelli. Si presume che la torre debba il proprio nome a Gherardo Asinelli, il nobile cavaliere di fazione ghibellina al quale se ne attribuisce la costruzione, iniziata secondo una consolidata tradizione l'11 ottobre 1109 e terminata dieci anni dopo, nel 1119.`, + ], + ], + [ + "text-generation", + [ + `Mi chiamo Loreto e mi piace`, + `Mi chiamo Thomas e il mio principale`, + `Mi chiamo Marianna, la mia cosa preferita`, + `Mi chiamo Clara e sono`, + `C'era una volta`, + ], + ], + ["fill-mask", [`Roma è la d'Italia.`, `Lo scopo della vita è .`]], + [ + "sentence-similarity", + [ + { + source_sentence: "Questa è una persona felice", + sentences: ["Questo è un cane felice", "Questa è una persona molto felice", "Oggi è una giornata di sole"], + }, + ], + ], +]); +const MAPPING_FA = new Map([ + [ + "text-classification", + [`پروژه به موقع تحویل شد و همه چیز خوب بود.`, `سیب‌زمینی بی‌کیفیت بود.`, `قیمت و کیفیت عالی`, `خوب نبود اصلا`], + ], + [ + "token-classification", + [ + `این سریال به صورت رسمی در تاریخ دهم می ۲۰۱۱ توسط شبکه فاکس برای پخش رزرو شد.`, + `دفتر مرکزی شرکت پارس‌مینو در شهر اراک در استان مرکزی قرار دارد.`, + `وی در سال ۲۰۱۳ درگذشت و مسئول خاکسپاری و اقوامش برای او مراسم یادبود گرفتند.`, + ], + ], + [ + "question-answering", + [ + { + text: `من کجا زندگی میکنم؟`, + context: `نام من پژمان است و در گرگان زندگی میکنم.`, + }, + { + text: `نامم چیست و کجا زندگی می‌کنم؟`, + context: `اسمم سارا است و در آفریقای جنوبی زندگی میکنم.`, + }, + { + text: `نام من چیست؟`, + context: `من مریم هستم و در تبریز زندگی می‌کنم.`, + }, + { + text: `بیشترین مساحت جنگل آمازون در کدام کشور است؟`, + context: [ + "آمازون نام بزرگ‌ترین جنگل بارانی جهان است که در شمال آمریکای جنوبی قرار گرفته و بیشتر آن در خاک برزیل و پرو", + "جای دارد. بیش از نیمی از همه جنگل‌های بارانی باقی‌مانده در جهان در آمازون قرار دارد.", + "مساحت جنگل‌های آمازون ۵٫۵ میلیون کیلومتر مربع است که بین ۹ کشور تقسیم شده‌است.", + ].join("\n"), + }, + ], + ], + [ + "translation", + [ + "بیشتر مساحت جنگل‌های آمازون در حوضه آبریز رود آمازون و ۱۱۰۰ شاخه آن واقع شده‌است.", + "مردمان نَبَطی از هزاره‌های یکم و دوم پیش از میلاد در این منطقه زندگی می‌کردند.", + ], + ], + [ + "summarization", + [ + [ + "شاهنامه اثر حکیم ابوالقاسم فردوسی توسی، حماسه‌ای منظوم، بر حسب دست نوشته‌های ", + "موجود دربرگیرنده نزدیک به ۵۰٬۰۰۰ بیت تا نزدیک به ۶۱٬۰۰۰ بیت و یکی از ", + "بزرگ‌ترین و برجسته‌ترین سروده‌های حماسی جهان است که سرایش آن دست‌آوردِ ", + "دست‌کم سی سال کارِ پیوستهٔ این سخن‌سرای نامدار ایرانی است. موضوع این شاهکار ادبی،", + " افسانه‌ها و تاریخ ایران از آغاز تا حملهٔ عرب‌ها به ایران در سدهٔ هفتم میلادی است", + " (شاهنامه از سه بخش اسطوره، پهلوانی و تاریخی تشکیل شده‌است) که در چهار", + " دودمان پادشاهیِ پیشدادیان، کیانیان، اشکانیان و ساسانیان گنجانده می‌شود.", + " شاهنامه بر وزن «فَعولُن فعولن فعولن فَعَلْ»، در بحرِ مُتَقارِبِ مثمَّنِ محذوف نگاشته شده‌است.", + "هنگامی که زبان دانش و ادبیات در ایران زبان عربی بود، فردوسی، با سرودن شاهنامه", + " با ویژگی‌های هدف‌مندی که داشت، زبان پارسی را زنده و پایدار کرد. یکی از ", + " بن‌مایه‌های مهمی که فردوسی برای سرودن شاهنامه از آن استفاده کرد،", + " شاهنامهٔ ابومنصوری بود. شاهنامه نفوذ بسیاری در جهت‌گیری ", + " فرهنگ فارسی و نیز بازتاب‌های شکوه‌مندی در ادبیات جهان داشته‌است و شاعران ", + " بزرگی مانند گوته و ویکتور هوگو از آن به نیکی یاد کرده‌اند.", + ].join("\n"), + ], + ], + ["text-generation", ["اسم من نازنین است و من", "روزی روزگاری"]], + [ + "fill-mask", + [ + `زندگی یک سوال است و این که چگونه کنیم پاسخ این سوال!`, + `زندگی از مرگ پرسید: چرا همه من را دارند اما از تو متنفرند؟`, + ], + ], +]); +const MAPPING_AR = new Map([ + ["text-classification", [`أحبك. أهواك`]], + [ + "token-classification", + [`إسمي محمد وأسكن في برلين`, `إسمي ساره وأسكن في لندن`, `إسمي سامي وأسكن في القدس في فلسطين.`], + ], + [ + "question-answering", + [ + { + text: `أين أسكن؟`, + context: `إسمي محمد وأسكن في بيروت`, + }, + { + text: `أين أسكن؟`, + context: `إسمي ساره وأسكن في لندن`, + }, + { + text: `ما اسمي؟`, + context: `اسمي سعيد وأسكن في حيفا.`, + }, + { + text: `ما لقب خالد بن الوليد بالعربية؟`, + context: `خالد بن الوليد من أبطال وقادة الفتح الإسلامي وقد تحدثت عنه اللغات الإنجليزية والفرنسية والإسبانية ولقب بسيف الله المسلول.`, + }, + ], + ], + ["translation", [`إسمي محمد وأسكن في برلين`, `إسمي ساره وأسكن في لندن`]], + [ + "summarization", + [ + `تقع الأهرامات في الجيزة قرب القاهرة في مصر وقد بنيت منذ عدة قرون، وقيل إنها كانت قبورا للفراعنة وتم بناؤها بعملية هندسية رائعة واستقدمت حجارتها من جبل المقطم وتم نقلها بالسفن أو على الرمل، وما تزال شامخة ويقصدها السياح من كافة أرجاء المعمورة.`, + ], + ], + [ + "text-generation", + [ + `إسمي محمد وأحب أن`, + `دع المكارم لا ترحل لبغيتها - واقعد فإنك أنت الطاعم الكاسي.`, + `لماذا نحن هنا؟`, + `القدس مدينة تاريخية، بناها الكنعانيون في`, + `كان يا ما كان في قديم الزمان`, + ], + ], + ["fill-mask", [`باريس فرنسا.`, `فلسفة الحياة هي .`]], + [ + "sentence-similarity", + [ + { + source_sentence: "هذا شخص سعيد", + sentences: ["هذا كلب سعيد", "هذا شخص سعيد جدا", "اليوم هو يوم مشمس"], + }, + ], + ], +]); +const MAPPING_BN = new Map([ + ["text-classification", [`বাঙালির ঘরে ঘরে আজ নবান্ন উৎসব।`]], + [ + "token-classification", + [`আমার নাম জাহিদ এবং আমি ঢাকায় বাস করি।`, `তিনি গুগলে চাকরী করেন।`, `আমার নাম সুস্মিতা এবং আমি কলকাতায় বাস করি।`], + ], + ["translation", [`আমার নাম জাহিদ, আমি রংপুরে বাস করি।`, `আপনি কী আজকে বাসায় আসবেন?`]], + [ + "summarization", + [ + `‘ইকোনমিস্ট’ লিখেছে, অ্যান্টিবডির চার মাস স্থায়ী হওয়ার খবরটি দুই কারণে আনন্দের। অ্যান্টিবডি যত দিন পর্যন্ত শরীরে টিকবে, তত দিন সংক্রমণ থেকে সুরক্ষিত থাকা সম্ভব। অর্থাৎ, এমন এক টিকার প্রয়োজন হবে, যা অ্যান্টিবডির উত্পাদনকে প্ররোচিত করতে পারে এবং দীর্ঘস্থায়ী সুরক্ষা দিতে পারে। এগুলো খুঁজে বের করাও সহজ। এটি আভাস দেয়, ব্যাপক হারে অ্যান্টিবডি শনাক্তকরণ ফলাফল মোটামুটি নির্ভুল হওয়া উচিত। দ্বিতীয় আরেকটি গবেষণার নেতৃত্ব দিয়েছেন যুক্তরাজ্যের মেডিকেল রিসার্চ কাউন্সিলের (এমআরসি) ইমিউনোলজিস্ট তাও দং। তিনি টি-সেল শনাক্তকরণে কাজ করেছেন। টি-সেল শনাক্তকরণের প্রক্রিয়া অবশ্য অ্যান্টিবডির মতো এত আলোচিত নয়। তবে সংক্রমণের বিরুদ্ধে লড়াই এবং দীর্ঘমেয়াদি সুরক্ষায় সমান গুরুত্বপূর্ণ ভূমিকা পালন করে। গবেষণাসংক্রান্ত নিবন্ধ প্রকাশিত হয়েছে ‘নেচার ইমিউনোলজি’ সাময়িকীতে। তাঁরা বলছেন, গবেষণার ক্ষেত্রে কোভিড-১৯ মৃদু সংক্রমণের শিকার ২৮ ব্যক্তির রক্তের নমুনা, ১৪ জন গুরুতর অসুস্থ ও ১৬ জন সুস্থ ব্যক্তির রক্তের নমুনা পরীক্ষা করেছেন। গবেষণা নিবন্ধে বলা হয়, সংক্রমিত ব্যক্তিদের ক্ষেত্রে টি-সেলের তীব্র প্রতিক্রিয়া তাঁরা দেখেছেন। এ ক্ষেত্রে মৃদু ও গুরুতর অসুস্থ ব্যক্তিদের ক্ষেত্রে প্রতিক্রিয়ার ভিন্নতা পাওয়া গেছে।`, + ], + ], + ["text-generation", [`আমি রতন এবং আমি`, `তুমি যদি চাও তবে`, `মিথিলা আজকে বড্ড`]], + ["fill-mask", [`আমি বাংলায় গাই।`, `আমি খুব ভালোবাসি। `]], + [ + "question-answering", + [ + { + text: `প্রথম এশিয়া কাপ ক্রিকেট টুর্নামেন্ট কোথায় অনুষ্ঠিত হয় ?`, + context: `প্রথম টুর্নামেন্ট অনুষ্ঠিত হয় ১৯৮৪ সালে সংযুক্ত আরব আমিরাত এর শারজাহ তে যেখানে কাউন্সিলের মূল অফিস ছিল (১৯৯৫ পর্যন্ত)। ভারত শ্রীলঙ্কার সাথে আন্তরিকতাহীন ক্রিকেট সম্পর্কের কারণে ১৯৮৬ সালের টুর্নামেন্ট বর্জন করে। ১৯৯৩ সালে ভারত ও পাকিস্তান এর মধ্যে রাজনৈতিক অস্থিরতার কারণে এটি বাতিল হয়ে যায়। শ্রীলঙ্কা এশিয়া কাপ শুরু থেকে অংশ গ্রহণ করে আসছে। আন্তর্জাতিক ক্রিকেট কাউন্সিল নিয়ম করে দিয়েছে যে এশিয়া কাপের সকল খেলা অনুষ্ঠিত হবে অফিসিয়াল একদিনের আন্তর্জাতিক ক্রিকেট হিসেবে। এসিসি ঘোষনা অনুযায়ী প্রতি দুই বছর পর পর টুর্নামেন্ট অনুষ্ঠিত হয় ২০০৮ সাল থেকে।`, + }, + { + text: `ভারতীয় বাঙালি কথাসাহিত্যিক মহাশ্বেতা দেবীর মৃত্যু কবে হয় ?`, + context: `২০১৬ সালের ২৩ জুলাই হৃদরোগে আক্রান্ত হয়ে মহাশ্বেতা দেবী কলকাতার বেল ভিউ ক্লিনিকে ভর্তি হন। সেই বছরই ২৮ জুলাই একাধিক অঙ্গ বিকল হয়ে তাঁর মৃত্যু ঘটে। তিনি মধুমেহ, সেপ্টিসেমিয়া ও মূত্র সংক্রমণ রোগেও ভুগছিলেন।`, + }, + { + text: `মাস্টারদা সূর্যকুমার সেনের বাবার নাম কী ছিল ?`, + context: `সূর্য সেন ১৮৯৪ সালের ২২ মার্চ চট্টগ্রামের রাউজান থানার নোয়াপাড়ায় অর্থনৈতিক ভাবে অস্বচ্ছল পরিবারে জন্মগ্রহণ করেন। তাঁর পিতার নাম রাজমনি সেন এবং মাতার নাম শশী বালা সেন। রাজমনি সেনের দুই ছেলে আর চার মেয়ে। সূর্য সেন তাঁদের পরিবারের চতুর্থ সন্তান। দুই ছেলের নাম সূর্য ও কমল। চার মেয়ের নাম বরদাসুন্দরী, সাবিত্রী, ভানুমতী ও প্রমিলা। শৈশবে পিতা মাতাকে হারানো সূর্য সেন কাকা গৌরমনি সেনের কাছে মানুষ হয়েছেন। সূর্য সেন ছেলেবেলা থেকেই খুব মনোযোগী ভাল ছাত্র ছিলেন এবং ধর্মভাবাপন্ন গম্ভীর প্রকৃতির ছিলেন।`, + }, + ], + ], + [ + "sentence-similarity", + [ + { + source_sentence: "সে একজন সুখী ব্যক্তি", + sentences: ["সে হ্যাপি কুকুর", "সে খুব সুখী মানুষ", "আজ একটি রৌদ্রোজ্জ্বল দিন"], + }, + ], + ], +]); +const MAPPING_MN = new Map([ + ["text-classification", [`Би чамд хайртай`]], + [ + "token-classification", + [ + `Намайг Дорж гэдэг. Би Улаанбаатарт амьдардаг.`, + `Намайг Ганбат гэдэг. Би Увс аймагт төрсөн.`, + `Манай улс таван хошуу малтай.`, + ], + ], + [ + "question-answering", + [ + { + text: `Та хаана амьдардаг вэ?`, + context: `Намайг Дорж гэдэг. Би Улаанбаатарт амьдардаг.`, + }, + { + text: `Таныг хэн гэдэг вэ?`, + context: `Намайг Дорж гэдэг. Би Улаанбаатарт амьдардаг.`, + }, + { + text: `Миний нэрийг хэн гэдэг вэ?`, + context: `Намайг Ганбат гэдэг. Би Увс аймагт төрсөн.`, + }, + ], + ], + ["translation", [`Намайг Дорж гэдэг. Би Улаанбаатарт амьдардаг.`, `Намайг Ганбат гэдэг. Би Увс аймагт төрсөн.`]], + [ + "summarization", + [ + `Монгол Улс (1992 оноос хойш) — дорно болон төв Азид оршдог бүрэн эрхт улс. Хойд талаараа Орос, бусад талаараа Хятад улстай хиллэдэг далайд гарцгүй орон. Нийслэл — Улаанбаатар хот. Алтайн нуруунаас Хянган, Соёноос Говь хүрсэн 1 сая 566 мянган км2 уудам нутагтай, дэлхийд нутаг дэвсгэрийн хэмжээгээр 19-рт жагсдаг. 2015 оны эхэнд Монгол Улсын хүн ам 3 сая хүрсэн (135-р олон). Үндсэндээ монгол үндэстэн (95 хувь), мөн хасаг, тува хүн байна. 16-р зуунаас хойш буддын шашин, 20-р зуунаас шашингүй байдал дэлгэрсэн ба албан хэрэгт монгол хэлээр харилцана.`, + ], + ], + [ + "text-generation", + [`Намайг Дорж гэдэг. Би`, `Хамгийн сайн дуучин бол`, `Миний дуртай хамтлаг бол`, `Эрт урьдын цагт`], + ], + ["fill-mask", [`Монгол улсын Улаанбаатар хотоос ярьж байна.`, `Миний амьдралын зорилго бол .`]], + [ + "automatic-speech-recognition", + [ + { + label: `Common Voice Train Example`, + src: `https://cdn-media.huggingface.co/common_voice/train/common_voice_mn_18577472.wav`, + }, + { + label: `Common Voice Test Example`, + src: `https://cdn-media.huggingface.co/common_voice/test/common_voice_mn_18577346.wav`, + }, + ], + ], + [ + "text-to-speech", + [ + `Би Монгол улсын иргэн.`, + `Энэхүү жишээ нь цаанаа ямар ч утга агуулаагүй болно`, + `Сар шинэдээ сайхан шинэлэж байна уу?`, + ], + ], + [ + "sentence-similarity", + [ + { + source_sentence: "Энэ бол аз жаргалтай хүн юм", + sentences: ["Энэ бол аз жаргалтай нохой юм", "Энэ бол маш их аз жаргалтай хүн юм", "Өнөөдөр нарлаг өдөр байна"], + }, + ], + ], +]); +const MAPPING_SI = new Map([ + ["translation", [`සිංහල ඉතා අලංකාර භාෂාවකි.`, `මෙම තාක්ෂණය භාවිතා කරන ඔබට ස්තූතියි.`]], + ["fill-mask", [`මම ගෙදර .`, ` ඉගෙනීමට ගියාය.`]], +]); +const MAPPING_DE = new Map([ + [ + "question-answering", + [ + { + text: `Wo wohne ich?`, + context: `Mein Name ist Wolfgang und ich lebe in Berlin`, + }, + { + text: `Welcher Name wird auch verwendet, um den Amazonas-Regenwald auf Englisch zu beschreiben?`, + context: `Der Amazonas-Regenwald, auf Englisch auch als Amazonien oder Amazonas-Dschungel bekannt, ist ein feuchter Laubwald, der den größten Teil des Amazonas-Beckens Südamerikas bedeckt. Dieses Becken umfasst 7.000.000 Quadratkilometer (2.700.000 Quadratmeilen), von denen 5.500.000 Quadratkilometer (2.100.000 Quadratmeilen) vom Regenwald bedeckt sind. Diese Region umfasst Gebiete von neun Nationen. Der größte Teil des Waldes befindet sich in Brasilien mit 60% des Regenwaldes, gefolgt von Peru mit 13%, Kolumbien mit 10% und geringen Mengen in Venezuela, Ecuador, Bolivien, Guyana, Suriname und Französisch-Guayana. Staaten oder Abteilungen in vier Nationen enthalten "Amazonas" in ihren Namen. Der Amazonas repräsentiert mehr als die Hälfte der verbleibenden Regenwälder des Planeten und umfasst den größten und artenreichsten tropischen Regenwald der Welt mit geschätzten 390 Milliarden Einzelbäumen, die in 16.000 Arten unterteilt sind.`, + }, + ], + ], + [ + "sentence-similarity", + [ + { + source_sentence: "Das ist eine glückliche Person", + sentences: [ + "Das ist ein glücklicher Hund", + "Das ist eine sehr glückliche Person", + "Heute ist ein sonniger Tag", + ], + }, + ], + ], +]); +const MAPPING_DV = new Map([ + ["text-classification", [`އަހަރެން ގަޔާވޭ. އަހަރެން ލޯބިވޭ`]], + [ + "token-classification", + [`އަހަރެންގެ ނަމަކީ އަހުމަދު އަދި އަހަރެން ދިރިއުޅެނީ މާލޭގަ`, `އަހަރެންގެ ނަމަކީ ސާރާ އަދި އަހަރެން ދިރިއުޅެނީ އުތީމުގަ`, `އަހަރެންގެ ނަމަކީ އައިޝާ އަދި އަހަރެން ދިރިއުޅެނީ ފޭދޫ، އައްޑޫގަ`], + ], + [ + "question-answering", + [ + { + text: `އަހަރެން ދިރިއުޅެނީ ކޮންތާކު؟`, + context: `އަހަރެންގެ ނަމަކީ އަހުމަދު އަދި އަހަރެން ދިރިއުޅެނީ މާލޭގަ`, + }, + { + text: `އަހަރެން ދިރިއުޅެނީ ކޮންތާކު؟`, + context: `އަހަރެންގެ ނަމަކީ ސާރާ އަދި އަހަރެން ދިރިއުޅެނީ އުތީމުގަ`, + }, + { + text: `އަހަރެންގެ ނަމަކީ ކޮބާ؟`, + context: `އަހަރެންގެ ނަމަކީ އައިޝާ އަދި އަހަރެން ދިރިއުޅެނީ ފޭދޫގަ`, + }, + { + text: `އެމޭޒަން ރެއިންފޮރެސްޓް ސިފަކޮށްދިނުމަށް އިނގިރޭސި ބަހުން ބޭނުންކުރާނީ ކޮންނަމެއް؟`, + context: `އެމޭޒަން ރެއިންފޮރެސްޓް (ޕޯޗުޖީޒް: ފްލޮރެސްޓާ އެމަސޮނިކާ ނުވަތަ އެމަސޮނިއާ؛ ސްޕެނިޝް: ސެލްވާ އެމަސޮނިކާ, އެމަސޮނިއާ ނޫނީ އާންމުކޮށް އެމަޒޯނިއާ؛ ފްރެންޗް: ފޮރޭ އެމެޒޮނިއެން؛ ޑަޗް: އެމެޒޯންރޭގެވައުޑް)، އިގިރޭސި ބަހުން ބުނާ އެމެޒޯނިއާ ނުވަތަ ދަ އެމޭޒަން ޖަންގަލް އަކީ, ސައުތު އެމެރިކާގެ އެމޭޒަން ބޭސިން ސަރަހައްދުގެ ބޮޑުބައެއްގައި ހިމެނޭ މޮއިސްޓް ބޮރޯޑްލީފް ފޮރެސްޓެއެކެވެ. އެމޭޒަން ބޭސިން ސަރަހައްދުގެ ބޮޑު މިނަކީ 7 މިލިއަން އަކަ ކިލޯމީޓަރ (2.7 މިލިއަން އަކަ މައިލް(. މީގެ ތެރެއިން 5.5 މިލިއަން އަކަ ކިލޯމީޓަރ (2.1 މިލިއަން އަކަ މައިލް) އަކީ މި ފޮރެސްޓެވެ. މި ސަރަހައްދުގައި 9 ގައުމަކަށް ނިސްބަތްވާ ޓެރިޓަރީ ހިމެނެއެވެ. 60% އާއިއެކެ އެންމެ ބޮޑު ބައެއް ނިސްބަތްވަނީ ބްރެޒިލްއަށެވެ. އޭގެ ފަހުތުން 13% އާއެކު ޕެރޫ އާއި 10% އާއެކު ކޮލަމްބިއާ އަދި ކުޑަ ބައެއް ހިމެނޭ ގޮތުން ވެނެޒުއެލާ, އެކްއަޑޯ, ބޮލިވިއާ, ގުޔާނާ, ސުރިނާމް އަދި ފްރެންޗް ގްއާނާ އަށް ވެސް ނިސްބަތްވެއެވެ. މީގެ ތެރެއިން 4 ގައުމެއްގައި "އެމެޒޮނާސް" ހިމަނައިގެން ސްޓޭޓް ނުވަތަ ޑިޕާޓްމަންޓް އަކަށް ނަންދީފައިވެއެވެ. މުޅި ދުނިޔޭގައި ބާކީ ހުރި ރެއިންފޮރެސްޓްގެ ތެރެއިން ދެބައިކުޅަ އެއްބަޔަށްވުރެބޮޑުވަރެއް އެމޭޒޮން ރެއިންފޮރެސްޓް ހިއްސާކުރެއެވެ. މިއީ މުޅި ދުނިޔެއިން އެންމޮ ބޮޑު އަދި އެންމެ ބައޮޑައިވަރސް ރެއިންފޮރެސްޓް ޓްރެކްޓެވެ. ލަފާކުރެވޭ ގޮތުން 16 ހާސް ސްޕީޝީސްއަށް ބެހިގެންވާ 390 މިލިއަން ވައްތަރުގެ ގަސް މިތާގައި ހިމެނެއެވެ`, + }, + ], + ], + ["translation", [`އަހަރެންގެ ނަމަކީ އަހުމަދު އަދި އަހަރެން ދިރިއުޅެނީ މާލޭގަ`, `އަހަރެންގެ ނަމަކީ ސާރާ އަދި އަހަރެން ދިރިއުޅެނީ އުތީމުގަ`]], + [ + "summarization", + [ + `ޓަވަރުގެ އުސްމިނަކީ 324 މީޓަރު، އެއީ ގާތްގަނޑަކަށް 81 ބުރީގެ އިމާރާތަކާއި އެއްވަރެވެ. އެއީ ޕެރިސްގައި ހުރި އެންމެ އުސް އިމާރާތެވެ. އޭގެ ހަތަރެސްކަނަށް ހުރި ބުޑުގެ ދިގުމިނަކީ ކޮންމެ ފަރާތަކުން 125 މީޓަރެވެ. (410 ފޫޓު) އައިފިލް ޓަވަރު ބިނާކުރި އިރު، ވޮޝިންގްޓަން މޮނިއުމެންޓްގެ އުސްމިން ފަހަނައަޅާ ގޮސް، ދުނިޔޭގައި މީހުން އުފެއްދި ތަންތަނުގެ ތެރެއިން އެންމެ އުސް ތަނުގެ ލަގަބު ލިބުނެވެ. އަދި 1930 ގައި ނިއު ޔޯކްގެ ކްރައިސްލަރ ބިލްޑިންގް ބިނާކުރުމާއި ހަމައަށް 41 އަހަރު ވަންދެން މިލަގަބު ހިފެހެއްޓިއެވެ. މިއީ 300 މީޓަރަށް ވުރެ އުސްކޮށް އިމާރާތްކުރެވުނު ފުރަތަމަ ތަނެވެ. 1957 ގައި ޓަވަރުގެ އެންމެ މަތީގައި ހަރުކުރެވުނު ބްރޯޑްކާސްޓިންގ އޭރިއަލްގެ ސަބަބުން މިހާރު މި ޓަވަރު ކްރައިސްލަރ ބިލްޑިންގއަށް ވުރެ 5.2 މީޓަރ (17 ފޫޓު) އުހެވެ. މި ޓްރާންސްމިޓަރު ނުލާ، އައިފިލް ޓަވަރަކީ، މިލާއު ވިއާޑަކްޓަށް ފަހު ފްރާންސްގައި ހުރި 2 ވަނައަށް އެންމެ އުސް ފްރީސްޓޭންޑިންގ އިމާރާތެވެ`, + ], + ], + [ + "text-generation", + [`އަހަރެންގެ ނަމަކީ ޔޫސުފް އަދި އަހަރެންގެ މައިގަނޑު`, `އަހަރެންގެ ނަމަކީ މަރިއަމް، އަހަރެން އެންމެ ގަޔާވާ`, `އަހަރެންގެ ނަމަކީ ފާތުމަތު އަދި އަހަރެން`, `،އެއް ޒަމާނެއްގައި`], + ], + ["fill-mask", [`. މާލެ އަކީ ދިވެހިރާއްޖޭގެ`, `ގަރުދިޔައަކީ ދިވެހިންގެ މެދުގައި ކެއުމެއް.`]], +]); +exports.MAPPING_DEFAULT_WIDGET = new Map([ + ["en", MAPPING_EN], + ["zh", MAPPING_ZH], + ["fr", MAPPING_FR], + ["es", MAPPING_ES], + ["ru", MAPPING_RU], + ["uk", MAPPING_UK], + ["it", MAPPING_IT], + ["fa", MAPPING_FA], + ["ar", MAPPING_AR], + ["bn", MAPPING_BN], + ["mn", MAPPING_MN], + ["si", MAPPING_SI], + ["de", MAPPING_DE], + ["dv", MAPPING_DV], +]); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/eval.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/eval.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..1fb6f2217d4d13662bcede9660978a954883346c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/eval.d.ts @@ -0,0 +1,146 @@ +/** + * List of supported Evaluation Frameworks supported in the `eval.yaml` file in benchmarks datasets. + */ +export declare const EVALUATION_FRAMEWORKS: { + readonly exgentic: { + readonly name: "exgentic"; + readonly description: "Exgentic is an open evaluation framework for general-purpose AI agents across diverse domains and benchmarks."; + readonly url: "https://github.com/Exgentic/exgentic"; + }; + readonly "inspect-ai": { + readonly name: "inspect-ai"; + readonly description: "Inspect AI is an open-source framework for large language model evaluations."; + readonly url: "https://inspect.aisi.org.uk/"; + }; + readonly "math-arena": { + readonly name: "math-arena"; + readonly description: "MathArena is a platform for evaluation of LLMs on latest math competitions and olympiads."; + readonly url: "https://github.com/eth-sri/matharena"; + }; + readonly mteb: { + readonly name: "mteb"; + readonly description: "Multimodal toolbox for evaluating embeddings and retrieval systems."; + readonly url: "https://github.com/embeddings-benchmark/mteb"; + }; + readonly "olmocr-bench": { + readonly name: "olmocr-bench"; + readonly description: "olmOCR-Bench is a framework for evaluating document-level OCR of various tools."; + readonly url: "https://github.com/allenai/olmocr/tree/main/olmocr/bench"; + }; + readonly harbor: { + readonly name: "harbor"; + readonly description: "Harbor is a framework for evaluating and optimizing agents and language models."; + readonly url: "https://github.com/laude-institute/harbor"; + }; + readonly ifstruct: { + readonly name: "ifstruct"; + readonly description: "IFStruct is a benchmark for structured-output compliance: whether a model produces valid JSON/YAML that follows a requested schema, scored without constrained decoding."; + readonly url: "https://github.com/Liquid4All/ifstruct"; + }; + readonly pier: { + readonly name: "pier"; + readonly description: "Pier is a Harbor fork built for DeepSWE, with stronger support for CLI agents in no-internet tasks and more faithful, consistent agent trajectories."; + readonly url: "https://github.com/datacurve-ai/pier"; + }; + readonly "redline-bench": { + readonly name: "redline-bench"; + readonly description: "RedlineBench measures multi-turn contract redlining: agents produce tracked-change .docx edits that are graded against attorney-authored weighted rubrics by an LLM judge panel across five dimensions. Report: https://intelligence.crosby.ai/benchmark/"; + readonly url: "https://github.com/crosbylegal/redline-bench"; + }; + readonly archipelago: { + readonly name: "archipelago"; + readonly description: "Archipelago is a system for running and evaluating AI agents against MCP applications."; + readonly url: "https://github.com/Mercor-Intelligence/archipelago"; + }; + readonly benchflow: { + readonly name: "benchflow"; + readonly description: "BenchFlow is an evaluation framework for AI agents on professional, skill-aware workflows. It powers SkillsBench and runs containerized agent trials with paired with-skills / without-skills configurations."; + readonly url: "https://github.com/benchflow-ai/benchflow"; + }; + readonly "apex-evals": { + readonly name: "apex-evals"; + readonly description: "APEX Evals is a benchmark suite and evaluation harness for evaluating large language models."; + readonly url: "https://github.com/Mercor-Intelligence/apex-evals"; + }; + readonly "screenspot-pro": { + readonly name: "screenspot-pro"; + readonly description: "ScreenSpot-Pro is a GUI grounding benchmark designed to evaluate how well AI agents can locate and identify UI elements across professional software applications in high-resolution screenshots, covering 1,585 annotated images from 26 professional tools."; + readonly url: "https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding"; + }; + readonly "swe-bench": { + readonly name: "swe-bench"; + readonly description: "SWE Bench is a framework for evaluating the performance of LLMs on software engineering tasks."; + readonly url: "https://github.com/swe-bench/swe-bench"; + }; + readonly "swe-bench-pro": { + readonly name: "swe-bench-pro"; + readonly description: "SWE-Bench Pro is a challenging benchmark evaluating LLMs/Agents on long-horizon software engineering tasks."; + readonly url: "https://github.com/scaleapi/SWE-bench_Pro-os"; + }; + readonly "nemo-evaluator": { + readonly name: "nemo-evaluator"; + readonly description: "NeMo Evaluator is an open-source platform for robust, reproducible, and scalable evaluation of Large Language Models across 100+ benchmarks."; + readonly url: "https://github.com/NVIDIA-NeMo/Evaluator"; + }; + readonly "yc-bench": { + readonly name: "yc-bench"; + readonly description: "YC Bench is a long-horizon deterministic benchmark for LLM agents. The agent plays CEO of an AI startup over a simulated 1–3 year run."; + readonly url: "https://github.com/collinear-ai/yc-bench"; + }; + readonly "open-asr-leaderboard": { + readonly name: "open-asr-leaderboard"; + readonly description: "The Open ASR Leaderboard ranks and evaluates speech recognition models."; + readonly url: "https://github.com/huggingface/open_asr_leaderboard"; + }; + readonly mdpbench: { + readonly name: "mdpbench"; + readonly description: "MDPBench is a benchmark for evaluating multilingual document parsing across digital, photographed, Latin, and non-Latin document subsets."; + readonly url: "https://github.com/Yuliang-Liu/MultimodalOCR"; + }; + readonly parsebench: { + readonly name: "parsebench"; + readonly description: "ParseBench is a benchmark for evaluating document parsing systems on real-world enterprise documents across tables, charts, content faithfulness, semantic formatting, and visual grounding."; + readonly url: "https://github.com/run-llama/ParseBench"; + }; + readonly "video-mme-v2": { + readonly name: "video-mme-v2"; + readonly description: "Video-MME-v2 is a benchmark for evaluating the next stage of video understanding capabilities of multimodal large language models."; + readonly url: "https://github.com/MME-Benchmarks/Video-MME-v2"; + }; + readonly "claw-eval": { + readonly name: "claw-eval"; + readonly description: "CLAW-Eval is an evaluation framework for assessing LLMs as autonomous agents across 300 human-verified tasks covering communication, finance, and productivity domains."; + readonly url: "https://github.com/claw-eval/claw-eval"; + }; + readonly researchclawbench: { + readonly name: "researchclawbench"; + readonly description: "ResearchClawBench is a benchmark for evaluating AI agents on end-to-end scientific research tasks, from reading data and related work to producing code, figures, and publication-style reports."; + readonly url: "https://github.com/InternScience/ResearchClawBench"; + }; + readonly pbench: { + readonly name: "pbench"; + readonly description: "PBench is a multi-level referring expression segmentation benchmark for evaluating vision-language perception across a structured hierarchy of skills."; + readonly url: "https://github.com/tiiuae/Falcon-Perception"; + }; + readonly wildclawbench: { + readonly name: "wildclawbench"; + readonly description: "WildClawBench is an in-the-wild benchmark for evaluating AI agents in the OpenClaw environment across 60 hand-built, end-to-end tasks spanning productivity, code intelligence, social interaction, search, creative synthesis, and safety domains."; + readonly url: "https://github.com/InternLM/WildClawBench"; + }; + readonly wbench: { + readonly name: "wbench"; + readonly description: "WBench is a comprehensive multi-turn benchmark for interactive video world model evaluation, assessing models across 5 dimensions (video quality, setting adherence, interaction adherence, consistency, physics compliance) and 22 metrics over 289 multi-turn interaction cases."; + readonly url: "https://github.com/meituan-longcat/WBench"; + }; + readonly nanofold: { + readonly name: "nanofold"; + readonly description: "nanoFold is a data-efficiency benchmark for protein structure prediction. Its goal is to evaluate models on scenarios with scarce data."; + readonly url: "https://github.com/ChrisHayduk/nanoFold-Competition"; + }; + readonly mmmu: { + readonly name: "mmmu"; + readonly description: "MMMU is a new benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning."; + readonly url: "https://mmmu-benchmark.github.io/"; + }; +}; +//# sourceMappingURL=eval.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/eval.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/eval.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d58ccab984e6b65388afae7024098283d1bbce64 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/eval.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/eval.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgKxB,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/eval.js b/node_modules/@huggingface/tasks/dist/commonjs/eval.js new file mode 100644 index 0000000000000000000000000000000000000000..09d6898bf9d81ef8178bded49e11d476e0dc72b0 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/eval.js @@ -0,0 +1,148 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.EVALUATION_FRAMEWORKS = void 0; +/** + * List of supported Evaluation Frameworks supported in the `eval.yaml` file in benchmarks datasets. + */ +exports.EVALUATION_FRAMEWORKS = { + exgentic: { + name: "exgentic", + description: "Exgentic is an open evaluation framework for general-purpose AI agents across diverse domains and benchmarks.", + url: "https://github.com/Exgentic/exgentic", + }, + "inspect-ai": { + name: "inspect-ai", + description: "Inspect AI is an open-source framework for large language model evaluations.", + url: "https://inspect.aisi.org.uk/", + }, + "math-arena": { + name: "math-arena", + description: "MathArena is a platform for evaluation of LLMs on latest math competitions and olympiads.", + url: "https://github.com/eth-sri/matharena", + }, + mteb: { + name: "mteb", + description: "Multimodal toolbox for evaluating embeddings and retrieval systems.", + url: "https://github.com/embeddings-benchmark/mteb", + }, + "olmocr-bench": { + name: "olmocr-bench", + description: "olmOCR-Bench is a framework for evaluating document-level OCR of various tools.", + url: "https://github.com/allenai/olmocr/tree/main/olmocr/bench", + }, + harbor: { + name: "harbor", + description: "Harbor is a framework for evaluating and optimizing agents and language models.", + url: "https://github.com/laude-institute/harbor", + }, + ifstruct: { + name: "ifstruct", + description: "IFStruct is a benchmark for structured-output compliance: whether a model produces valid JSON/YAML that follows a requested schema, scored without constrained decoding.", + url: "https://github.com/Liquid4All/ifstruct", + }, + pier: { + name: "pier", + description: "Pier is a Harbor fork built for DeepSWE, with stronger support for CLI agents in no-internet tasks and more faithful, consistent agent trajectories.", + url: "https://github.com/datacurve-ai/pier", + }, + "redline-bench": { + name: "redline-bench", + description: "RedlineBench measures multi-turn contract redlining: agents produce tracked-change .docx edits that are graded against attorney-authored weighted rubrics by an LLM judge panel across five dimensions. Report: https://intelligence.crosby.ai/benchmark/", + url: "https://github.com/crosbylegal/redline-bench", + }, + archipelago: { + name: "archipelago", + description: "Archipelago is a system for running and evaluating AI agents against MCP applications.", + url: "https://github.com/Mercor-Intelligence/archipelago", + }, + benchflow: { + name: "benchflow", + description: "BenchFlow is an evaluation framework for AI agents on professional, skill-aware workflows. It powers SkillsBench and runs containerized agent trials with paired with-skills / without-skills configurations.", + url: "https://github.com/benchflow-ai/benchflow", + }, + "apex-evals": { + name: "apex-evals", + description: "APEX Evals is a benchmark suite and evaluation harness for evaluating large language models.", + url: "https://github.com/Mercor-Intelligence/apex-evals", + }, + "screenspot-pro": { + name: "screenspot-pro", + description: "ScreenSpot-Pro is a GUI grounding benchmark designed to evaluate how well AI agents can locate and identify UI elements across professional software applications in high-resolution screenshots, covering 1,585 annotated images from 26 professional tools.", + url: "https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding", + }, + "swe-bench": { + name: "swe-bench", + description: "SWE Bench is a framework for evaluating the performance of LLMs on software engineering tasks.", + url: "https://github.com/swe-bench/swe-bench", + }, + "swe-bench-pro": { + name: "swe-bench-pro", + description: "SWE-Bench Pro is a challenging benchmark evaluating LLMs/Agents on long-horizon software engineering tasks.", + url: "https://github.com/scaleapi/SWE-bench_Pro-os", + }, + "nemo-evaluator": { + name: "nemo-evaluator", + description: "NeMo Evaluator is an open-source platform for robust, reproducible, and scalable evaluation of Large Language Models across 100+ benchmarks.", + url: "https://github.com/NVIDIA-NeMo/Evaluator", + }, + "yc-bench": { + name: "yc-bench", + description: "YC Bench is a long-horizon deterministic benchmark for LLM agents. The agent plays CEO of an AI startup over a simulated 1–3 year run.", + url: "https://github.com/collinear-ai/yc-bench", + }, + "open-asr-leaderboard": { + name: "open-asr-leaderboard", + description: "The Open ASR Leaderboard ranks and evaluates speech recognition models.", + url: "https://github.com/huggingface/open_asr_leaderboard", + }, + mdpbench: { + name: "mdpbench", + description: "MDPBench is a benchmark for evaluating multilingual document parsing across digital, photographed, Latin, and non-Latin document subsets.", + url: "https://github.com/Yuliang-Liu/MultimodalOCR", + }, + parsebench: { + name: "parsebench", + description: "ParseBench is a benchmark for evaluating document parsing systems on real-world enterprise documents across tables, charts, content faithfulness, semantic formatting, and visual grounding.", + url: "https://github.com/run-llama/ParseBench", + }, + "video-mme-v2": { + name: "video-mme-v2", + description: "Video-MME-v2 is a benchmark for evaluating the next stage of video understanding capabilities of multimodal large language models.", + url: "https://github.com/MME-Benchmarks/Video-MME-v2", + }, + "claw-eval": { + name: "claw-eval", + description: "CLAW-Eval is an evaluation framework for assessing LLMs as autonomous agents across 300 human-verified tasks covering communication, finance, and productivity domains.", + url: "https://github.com/claw-eval/claw-eval", + }, + researchclawbench: { + name: "researchclawbench", + description: "ResearchClawBench is a benchmark for evaluating AI agents on end-to-end scientific research tasks, from reading data and related work to producing code, figures, and publication-style reports.", + url: "https://github.com/InternScience/ResearchClawBench", + }, + pbench: { + name: "pbench", + description: "PBench is a multi-level referring expression segmentation benchmark for evaluating vision-language perception across a structured hierarchy of skills.", + url: "https://github.com/tiiuae/Falcon-Perception", + }, + wildclawbench: { + name: "wildclawbench", + description: "WildClawBench is an in-the-wild benchmark for evaluating AI agents in the OpenClaw environment across 60 hand-built, end-to-end tasks spanning productivity, code intelligence, social interaction, search, creative synthesis, and safety domains.", + url: "https://github.com/InternLM/WildClawBench", + }, + wbench: { + name: "wbench", + description: "WBench is a comprehensive multi-turn benchmark for interactive video world model evaluation, assessing models across 5 dimensions (video quality, setting adherence, interaction adherence, consistency, physics compliance) and 22 metrics over 289 multi-turn interaction cases.", + url: "https://github.com/meituan-longcat/WBench", + }, + nanofold: { + name: "nanofold", + description: "nanoFold is a data-efficiency benchmark for protein structure prediction. Its goal is to evaluate models on scenarios with scarce data.", + url: "https://github.com/ChrisHayduk/nanoFold-Competition", + }, + mmmu: { + name: "mmmu", + description: "MMMU is a new benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning.", + url: "https://mmmu-benchmark.github.io/", + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/gguf.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/gguf.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ddf4be14d547aabeae88ee0b398e2e950b020038 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/gguf.d.ts @@ -0,0 +1,91 @@ +export declare enum GGMLFileQuantizationType { + F32 = 0, + F16 = 1, + Q4_0 = 2, + Q4_1 = 3, + Q4_1_SOME_F16 = 4, + Q4_2 = 5, + Q4_3 = 6, + Q8_0 = 7, + Q5_0 = 8, + Q5_1 = 9, + Q2_K = 10, + Q3_K_S = 11, + Q3_K_M = 12, + Q3_K_L = 13, + Q4_K_S = 14, + Q4_K_M = 15, + Q5_K_S = 16, + Q5_K_M = 17, + Q6_K = 18, + IQ2_XXS = 19, + IQ2_XS = 20, + Q2_K_S = 21, + IQ3_XS = 22, + IQ3_XXS = 23, + IQ1_S = 24, + IQ4_NL = 25, + IQ3_S = 26, + IQ3_M = 27, + IQ2_S = 28, + IQ2_M = 29, + IQ4_XS = 30, + IQ1_M = 31, + BF16 = 32, + Q4_0_4_4 = 33, + Q4_0_4_8 = 34, + Q4_0_8_8 = 35, + TQ1_0 = 36, + TQ2_0 = 37, + MXFP4_MOE = 38, + NVFP4 = 39, + Q1_0 = 40, + Q2_K_XL = 1000, + Q3_K_XL = 1001, + Q4_K_XL = 1002, + Q5_K_XL = 1003, + Q6_K_XL = 1004, + Q8_K_XL = 1005 +} +export declare const GGUF_QUANT_RE: RegExp; +export declare const GGUF_QUANT_RE_GLOBAL: RegExp; +export declare function parseGGUFQuantLabel(fname: string): string | undefined; +export declare const GGUF_QUANT_ORDER: GGMLFileQuantizationType[]; +export declare function findNearestQuantType(quant: GGMLFileQuantizationType, availableQuants: GGMLFileQuantizationType[]): GGMLFileQuantizationType | undefined; +export declare enum GGMLQuantizationType { + F32 = 0, + F16 = 1, + Q4_0 = 2, + Q4_1 = 3, + Q5_0 = 6, + Q5_1 = 7, + Q8_0 = 8, + Q8_1 = 9, + Q2_K = 10, + Q3_K = 11, + Q4_K = 12, + Q5_K = 13, + Q6_K = 14, + Q8_K = 15, + IQ2_XXS = 16, + IQ2_XS = 17, + IQ3_XXS = 18, + IQ1_S = 19, + IQ4_NL = 20, + IQ3_S = 21, + IQ2_S = 22, + IQ4_XS = 23, + I8 = 24, + I16 = 25, + I32 = 26, + I64 = 27, + F64 = 28, + IQ1_M = 29, + BF16 = 30, + TQ1_0 = 34, + TQ2_0 = 35, + MXFP4 = 39, + NVFP4 = 40, + Q1_0 = 41 +} +//# sourceMappingURL=gguf.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/gguf.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/gguf.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..76c02d0778cbcc9723a71a01fcad76e6b5944465 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/gguf.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"gguf.d.ts","sourceRoot":"","sources":["../../src/gguf.ts"],"names":[],"mappings":"AAGA,oBAAY,wBAAwB;IACnC,GAAG,IAAI;IACP,GAAG,IAAI;IACP,IAAI,IAAI;IACR,IAAI,IAAI;IACR,aAAa,IAAI;IACjB,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,KAAK;IACT,MAAM,KAAK;IACX,MAAM,KAAK;IACX,MAAM,KAAK;IACX,MAAM,KAAK;IACX,MAAM,KAAK;IACX,MAAM,KAAK;IACX,MAAM,KAAK;IACX,IAAI,KAAK;IACT,OAAO,KAAK;IACZ,MAAM,KAAK;IACX,MAAM,KAAK;IACX,MAAM,KAAK;IACX,OAAO,KAAK;IACZ,KAAK,KAAK;IACV,MAAM,KAAK;IACX,KAAK,KAAK;IACV,KAAK,KAAK;IACV,KAAK,KAAK;IACV,KAAK,KAAK;IACV,MAAM,KAAK;IACX,KAAK,KAAK;IACV,IAAI,KAAK;IACT,QAAQ,KAAK;IACb,QAAQ,KAAK;IACb,QAAQ,KAAK;IACb,KAAK,KAAK;IACV,KAAK,KAAK;IACV,SAAS,KAAK;IACd,KAAK,KAAK;IACV,IAAI,KAAK;IAIT,OAAO,OAAO;IACd,OAAO,OAAO;IACd,OAAO,OAAO;IACd,OAAO,OAAO;IACd,OAAO,OAAO;IACd,OAAO,OAAO;CACd;AAGD,eAAO,MAAM,aAAa,QAEzB,CAAC;AACF,eAAO,MAAM,oBAAoB,QAAiC,CAAC;AAEnE,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAGrE;AAKD,eAAO,MAAM,gBAAgB,EAAE,wBAAwB,EA4DtD,CAAC;AAIF,wBAAgB,oBAAoB,CACnC,KAAK,EAAE,wBAAwB,EAC/B,eAAe,EAAE,wBAAwB,EAAE,GACzC,wBAAwB,GAAG,SAAS,CAmCtC;AAGD,oBAAY,oBAAoB;IAC/B,GAAG,IAAI;IACP,GAAG,IAAI;IACP,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,KAAK;IACT,IAAI,KAAK;IACT,IAAI,KAAK;IACT,IAAI,KAAK;IACT,IAAI,KAAK;IACT,IAAI,KAAK;IACT,OAAO,KAAK;IACZ,MAAM,KAAK;IACX,OAAO,KAAK;IACZ,KAAK,KAAK;IACV,MAAM,KAAK;IACX,KAAK,KAAK;IACV,KAAK,KAAK;IACV,MAAM,KAAK;IACX,EAAE,KAAK;IACP,GAAG,KAAK;IACR,GAAG,KAAK;IACR,GAAG,KAAK;IACR,GAAG,KAAK;IACR,KAAK,KAAK;IACV,IAAI,KAAK;IACT,KAAK,KAAK;IACV,KAAK,KAAK;IACV,KAAK,KAAK;IACV,KAAK,KAAK;IACV,IAAI,KAAK;CACT"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/gguf.js b/node_modules/@huggingface/tasks/dist/commonjs/gguf.js new file mode 100644 index 0000000000000000000000000000000000000000..94a7c26045566b97bc7087a4b8cbbfff399eb9ad --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/gguf.js @@ -0,0 +1,196 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GGMLQuantizationType = exports.GGUF_QUANT_ORDER = exports.GGUF_QUANT_RE_GLOBAL = exports.GGUF_QUANT_RE = exports.GGMLFileQuantizationType = void 0; +exports.parseGGUFQuantLabel = parseGGUFQuantLabel; +exports.findNearestQuantType = findNearestQuantType; +// This list is copied from gguf/types.ts, but will all types available (for backward compatibility) +// NOT to be confused with GGMLQuantizationType, a FileQuantization can contain multiple GGMLQuantizationType +// For example, Q4_K_M model can contains Q4_K and Q6_K tensors +var GGMLFileQuantizationType; +(function (GGMLFileQuantizationType) { + GGMLFileQuantizationType[GGMLFileQuantizationType["F32"] = 0] = "F32"; + GGMLFileQuantizationType[GGMLFileQuantizationType["F16"] = 1] = "F16"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_0"] = 2] = "Q4_0"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_1"] = 3] = "Q4_1"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_1_SOME_F16"] = 4] = "Q4_1_SOME_F16"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_2"] = 5] = "Q4_2"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_3"] = 6] = "Q4_3"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q8_0"] = 7] = "Q8_0"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q5_0"] = 8] = "Q5_0"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q5_1"] = 9] = "Q5_1"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q2_K"] = 10] = "Q2_K"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q3_K_S"] = 11] = "Q3_K_S"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q3_K_M"] = 12] = "Q3_K_M"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q3_K_L"] = 13] = "Q3_K_L"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_K_S"] = 14] = "Q4_K_S"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_K_M"] = 15] = "Q4_K_M"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q5_K_S"] = 16] = "Q5_K_S"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q5_K_M"] = 17] = "Q5_K_M"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q6_K"] = 18] = "Q6_K"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ2_XXS"] = 19] = "IQ2_XXS"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ2_XS"] = 20] = "IQ2_XS"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q2_K_S"] = 21] = "Q2_K_S"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ3_XS"] = 22] = "IQ3_XS"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ3_XXS"] = 23] = "IQ3_XXS"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ1_S"] = 24] = "IQ1_S"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ4_NL"] = 25] = "IQ4_NL"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ3_S"] = 26] = "IQ3_S"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ3_M"] = 27] = "IQ3_M"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ2_S"] = 28] = "IQ2_S"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ2_M"] = 29] = "IQ2_M"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ4_XS"] = 30] = "IQ4_XS"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ1_M"] = 31] = "IQ1_M"; + GGMLFileQuantizationType[GGMLFileQuantizationType["BF16"] = 32] = "BF16"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_0_4_4"] = 33] = "Q4_0_4_4"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_0_4_8"] = 34] = "Q4_0_4_8"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_0_8_8"] = 35] = "Q4_0_8_8"; + GGMLFileQuantizationType[GGMLFileQuantizationType["TQ1_0"] = 36] = "TQ1_0"; + GGMLFileQuantizationType[GGMLFileQuantizationType["TQ2_0"] = 37] = "TQ2_0"; + GGMLFileQuantizationType[GGMLFileQuantizationType["MXFP4_MOE"] = 38] = "MXFP4_MOE"; + GGMLFileQuantizationType[GGMLFileQuantizationType["NVFP4"] = 39] = "NVFP4"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q1_0"] = 40] = "Q1_0"; + // custom quants used by unsloth + // they are not officially a scheme enum value in GGUF, but only here for naming + GGMLFileQuantizationType[GGMLFileQuantizationType["Q2_K_XL"] = 1000] = "Q2_K_XL"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q3_K_XL"] = 1001] = "Q3_K_XL"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_K_XL"] = 1002] = "Q4_K_XL"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q5_K_XL"] = 1003] = "Q5_K_XL"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q6_K_XL"] = 1004] = "Q6_K_XL"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q8_K_XL"] = 1005] = "Q8_K_XL"; +})(GGMLFileQuantizationType || (exports.GGMLFileQuantizationType = GGMLFileQuantizationType = {})); +const ggufQuants = Object.values(GGMLFileQuantizationType).filter((v) => typeof v === "string"); +exports.GGUF_QUANT_RE = new RegExp("(?UD-)?" + `(?${ggufQuants.join("|")})` + "(_(?[A-Z]+))?"); +exports.GGUF_QUANT_RE_GLOBAL = new RegExp(exports.GGUF_QUANT_RE, "g"); +function parseGGUFQuantLabel(fname) { + const quantLabel = fname.toUpperCase().match(exports.GGUF_QUANT_RE_GLOBAL)?.at(-1); // if there is multiple quant substrings in a name, we prefer the last one + return quantLabel; +} +// order of quantization, from biggest to smallest +// this list must be in sync with the order in GGMLFileQuantizationType +// the gguf.spec.ts tests are using verify if the order is correct +exports.GGUF_QUANT_ORDER = [ + GGMLFileQuantizationType.F32, + GGMLFileQuantizationType.BF16, + GGMLFileQuantizationType.F16, + GGMLFileQuantizationType.Q8_K_XL, + GGMLFileQuantizationType.Q8_0, + // 6-bit quantizations + GGMLFileQuantizationType.Q6_K_XL, + GGMLFileQuantizationType.Q6_K, + // 5-bit quantizations + GGMLFileQuantizationType.Q5_K_XL, + GGMLFileQuantizationType.Q5_K_M, + GGMLFileQuantizationType.Q5_K_S, + GGMLFileQuantizationType.Q5_0, + GGMLFileQuantizationType.Q5_1, + // 4-bit quantizations + GGMLFileQuantizationType.Q4_K_XL, + GGMLFileQuantizationType.Q4_K_M, + GGMLFileQuantizationType.Q4_K_S, + GGMLFileQuantizationType.IQ4_NL, + GGMLFileQuantizationType.IQ4_XS, + GGMLFileQuantizationType.Q4_0_4_4, + GGMLFileQuantizationType.Q4_0_4_8, + GGMLFileQuantizationType.Q4_0_8_8, + GGMLFileQuantizationType.Q4_1_SOME_F16, + GGMLFileQuantizationType.Q4_0, + GGMLFileQuantizationType.Q4_1, + GGMLFileQuantizationType.Q4_2, + GGMLFileQuantizationType.Q4_3, + GGMLFileQuantizationType.MXFP4_MOE, + GGMLFileQuantizationType.NVFP4, + // 3-bit quantizations + GGMLFileQuantizationType.Q3_K_XL, + GGMLFileQuantizationType.Q3_K_L, + GGMLFileQuantizationType.Q3_K_M, + GGMLFileQuantizationType.Q3_K_S, + GGMLFileQuantizationType.IQ3_M, + GGMLFileQuantizationType.IQ3_S, + GGMLFileQuantizationType.IQ3_XS, + GGMLFileQuantizationType.IQ3_XXS, + // 2-bit quantizations + GGMLFileQuantizationType.Q2_K_XL, + GGMLFileQuantizationType.Q2_K, + GGMLFileQuantizationType.Q2_K_S, + GGMLFileQuantizationType.IQ2_M, + GGMLFileQuantizationType.IQ2_S, + GGMLFileQuantizationType.IQ2_XS, + GGMLFileQuantizationType.IQ2_XXS, + // 1-bit quantizations + GGMLFileQuantizationType.IQ1_S, + GGMLFileQuantizationType.IQ1_M, + GGMLFileQuantizationType.TQ1_0, + GGMLFileQuantizationType.TQ2_0, + GGMLFileQuantizationType.Q1_0, +]; +// This function finds the nearest quantization type that is less than or equal to the given quantization type. +// It returns undefined if no such quantization type is found. +function findNearestQuantType(quant, availableQuants) { + // Create a map for quick index lookup from the defined order + const orderMap = new Map(); + exports.GGUF_QUANT_ORDER.forEach((q, index) => { + orderMap.set(q, index); + }); + const targetIndex = orderMap.get(quant) ?? 0; // the 0 case should never happen + // Filter the available quantizations to include only those defined in the order map, + // then sort them according to the GGUF_QUANT_ORDER (from largest/index 0 to smallest/highest index). + const sortedAvailable = availableQuants + .filter((q) => orderMap.has(q)) + .sort((a, b) => (orderMap.get(a) ?? Infinity) - (orderMap.get(b) ?? Infinity)); + // If no valid quantizations are available after filtering + if (sortedAvailable.length === 0) { + return undefined; + } + // Iterate through the sorted available quantizations (largest to smallest). + // Find the first one whose order index is >= the target index. + // This means finding the largest quantization that is smaller than or equal to the target. + for (const availableQuant of sortedAvailable) { + // We know the key exists due to the filter above. + const availableIndex = orderMap.get(availableQuant) ?? 0; + if (availableIndex >= targetIndex) { + return availableQuant; + } + } + // If the loop completes, it means all available quantizations are larger (have a smaller index) + // than the target quantization. In this case, return the "smallest" available quantization, + // which is the last element in the sorted list (highest index among available). + return sortedAvailable[sortedAvailable.length - 1]; +} +// This list is only used to calculate the size of the model, NOT to be confused with the quantization FILE type +var GGMLQuantizationType; +(function (GGMLQuantizationType) { + GGMLQuantizationType[GGMLQuantizationType["F32"] = 0] = "F32"; + GGMLQuantizationType[GGMLQuantizationType["F16"] = 1] = "F16"; + GGMLQuantizationType[GGMLQuantizationType["Q4_0"] = 2] = "Q4_0"; + GGMLQuantizationType[GGMLQuantizationType["Q4_1"] = 3] = "Q4_1"; + GGMLQuantizationType[GGMLQuantizationType["Q5_0"] = 6] = "Q5_0"; + GGMLQuantizationType[GGMLQuantizationType["Q5_1"] = 7] = "Q5_1"; + GGMLQuantizationType[GGMLQuantizationType["Q8_0"] = 8] = "Q8_0"; + GGMLQuantizationType[GGMLQuantizationType["Q8_1"] = 9] = "Q8_1"; + GGMLQuantizationType[GGMLQuantizationType["Q2_K"] = 10] = "Q2_K"; + GGMLQuantizationType[GGMLQuantizationType["Q3_K"] = 11] = "Q3_K"; + GGMLQuantizationType[GGMLQuantizationType["Q4_K"] = 12] = "Q4_K"; + GGMLQuantizationType[GGMLQuantizationType["Q5_K"] = 13] = "Q5_K"; + GGMLQuantizationType[GGMLQuantizationType["Q6_K"] = 14] = "Q6_K"; + GGMLQuantizationType[GGMLQuantizationType["Q8_K"] = 15] = "Q8_K"; + GGMLQuantizationType[GGMLQuantizationType["IQ2_XXS"] = 16] = "IQ2_XXS"; + GGMLQuantizationType[GGMLQuantizationType["IQ2_XS"] = 17] = "IQ2_XS"; + GGMLQuantizationType[GGMLQuantizationType["IQ3_XXS"] = 18] = "IQ3_XXS"; + GGMLQuantizationType[GGMLQuantizationType["IQ1_S"] = 19] = "IQ1_S"; + GGMLQuantizationType[GGMLQuantizationType["IQ4_NL"] = 20] = "IQ4_NL"; + GGMLQuantizationType[GGMLQuantizationType["IQ3_S"] = 21] = "IQ3_S"; + GGMLQuantizationType[GGMLQuantizationType["IQ2_S"] = 22] = "IQ2_S"; + GGMLQuantizationType[GGMLQuantizationType["IQ4_XS"] = 23] = "IQ4_XS"; + GGMLQuantizationType[GGMLQuantizationType["I8"] = 24] = "I8"; + GGMLQuantizationType[GGMLQuantizationType["I16"] = 25] = "I16"; + GGMLQuantizationType[GGMLQuantizationType["I32"] = 26] = "I32"; + GGMLQuantizationType[GGMLQuantizationType["I64"] = 27] = "I64"; + GGMLQuantizationType[GGMLQuantizationType["F64"] = 28] = "F64"; + GGMLQuantizationType[GGMLQuantizationType["IQ1_M"] = 29] = "IQ1_M"; + GGMLQuantizationType[GGMLQuantizationType["BF16"] = 30] = "BF16"; + GGMLQuantizationType[GGMLQuantizationType["TQ1_0"] = 34] = "TQ1_0"; + GGMLQuantizationType[GGMLQuantizationType["TQ2_0"] = 35] = "TQ2_0"; + GGMLQuantizationType[GGMLQuantizationType["MXFP4"] = 39] = "MXFP4"; + GGMLQuantizationType[GGMLQuantizationType["NVFP4"] = 40] = "NVFP4"; + GGMLQuantizationType[GGMLQuantizationType["Q1_0"] = 41] = "Q1_0"; +})(GGMLQuantizationType || (exports.GGMLQuantizationType = GGMLQuantizationType = {})); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/hardware-amd.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/hardware-amd.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b134a5211c2b52c3db3a66f9b2d706e3886fb2c1 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/hardware-amd.d.ts @@ -0,0 +1,11 @@ +import type { HardwareSpec } from "./hardware.js"; +export interface AmdGpuHardwareSpec extends HardwareSpec { + /** + * GFX version / LLVM ISA target (AMD GPUs only), e.g. "gfx1100" + * + * potential source https://llvm.org/docs/AMDGPUUsage.html#processors + */ + gfxVersion: string; +} +export declare const AMD_GPU_SKUS: Record; +//# sourceMappingURL=hardware-amd.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/hardware-amd.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/hardware-amd.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..589d94268bc87de1e2aa593fe04f0f64bc29f10e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/hardware-amd.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"hardware-amd.d.ts","sourceRoot":"","sources":["../../src/hardware-amd.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElD,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACvD;;;;OAIG;IACH,UAAU,EAAE,MAAM,CAAC;CACnB;AAID,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAiU3D,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/hardware-amd.js b/node_modules/@huggingface/tasks/dist/commonjs/hardware-amd.js new file mode 100644 index 0000000000000000000000000000000000000000..4a19557ad4675137de0133979d35bb66301fcbda --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/hardware-amd.js @@ -0,0 +1,326 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.AMD_GPU_SKUS = void 0; +const AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS = [16, 24, 32, 48, 64, 96]; +exports.AMD_GPU_SKUS = { + MI300: { + tflops: 383.0, + memory: [192], + gfxVersion: "gfx942", + msrp: 15_000, + power: 750, + releaseYear: 2023, + }, + MI250: { + tflops: 362.1, + memory: [128], + gfxVersion: "gfx90a", + msrp: 10_000, + power: 560, + releaseYear: 2021, + }, + MI210: { + tflops: 181.0, + memory: [64], + gfxVersion: "gfx90a", + msrp: 8_000, + power: 300, + releaseYear: 2022, + }, + MI100: { + tflops: 184.6, + memory: [32], + gfxVersion: "gfx908", + msrp: 6_400, + power: 300, + releaseYear: 2020, + }, + MI60: { + tflops: 29.5, + memory: [32], + gfxVersion: "gfx906", + msrp: 3_000, + power: 300, + releaseYear: 2018, + }, + MI50: { + tflops: 26.5, + memory: [16, 32], + gfxVersion: "gfx906", + msrp: 1_800, + power: 300, + releaseYear: 2018, + }, + "R9700 PRO": { + tflops: 95.7, + memory: [32], + gfxVersion: "gfx1201", + msrp: 1_250, + power: 300, + releaseYear: 2025, + }, + "RX 9070 XT": { + tflops: 97.32, + memory: [16], + gfxVersion: "gfx1201", + msrp: 600, + power: 304, + releaseYear: 2025, + }, + "RX 9070": { + tflops: 72.25, + memory: [16], + gfxVersion: "gfx1201", + msrp: 550, + power: 220, + releaseYear: 2025, + }, + "RX 9060 XT": { + tflops: 51.28, + memory: [8, 16], + gfxVersion: "gfx1200", + msrp: 350, + power: 160, + releaseYear: 2025, + }, + "PRO W7900": { + tflops: 122.6, + memory: [48], + gfxVersion: "gfx1100", + msrp: 4_000, + power: 295, + releaseYear: 2023, + }, + "PRO W7800": { + tflops: 90.5, + memory: [32, 48], + gfxVersion: "gfx1100", + msrp: 2_500, + power: 260, + releaseYear: 2023, + }, + "RX 7900 XTX": { + tflops: 122.8, + memory: [24], + gfxVersion: "gfx1100", + msrp: 1_000, + power: 355, + releaseYear: 2022, + }, + "RX 7900 XT": { + tflops: 103.0, + memory: [20], + gfxVersion: "gfx1100", + msrp: 900, + power: 315, + releaseYear: 2022, + }, + "RX 7900 GRE": { + tflops: 91.96, + memory: [16], + gfxVersion: "gfx1100", + msrp: 550, + power: 260, + releaseYear: 2023, + }, + "RX 7800 XT": { + tflops: 74.65, + memory: [16], + gfxVersion: "gfx1101", + msrp: 500, + power: 263, + releaseYear: 2023, + }, + "RX 7700 XT": { + tflops: 70.34, + memory: [12], + gfxVersion: "gfx1101", + msrp: 450, + power: 245, + releaseYear: 2023, + }, + "RX 7600 XT": { + tflops: 45.14, + memory: [16, 8], + gfxVersion: "gfx1102", + msrp: 350, + power: 190, + releaseYear: 2024, + }, + "RX 6950 XT": { + tflops: 47.31, + memory: [16], + gfxVersion: "gfx1030", + msrp: 1_100, + power: 335, + releaseYear: 2022, + }, + "RX 6800": { + tflops: 32.33, + memory: [16], + gfxVersion: "gfx1030", + msrp: 600, + power: 250, + releaseYear: 2020, + }, + "RX 6700 XT": { + tflops: 26.43, + memory: [12], + gfxVersion: "gfx1031", + msrp: 500, + power: 230, + releaseYear: 2021, + }, + "RX 6700": { + tflops: 22.58, + memory: [10], + gfxVersion: "gfx1031", + msrp: 329, + power: 175, + releaseYear: 2022, + }, + "RX 6650 XT": { + tflops: 21.59, + memory: [8], + gfxVersion: "gfx1032", + msrp: 400, + power: 180, + releaseYear: 2022, + }, + "RX 6600 XT": { + tflops: 21.21, + memory: [8], + gfxVersion: "gfx1032", + msrp: 400, + power: 160, + releaseYear: 2021, + }, + "RX 6600": { + tflops: 17.86, + memory: [8], + gfxVersion: "gfx1032", + msrp: 350, + power: 132, + releaseYear: 2021, + }, + "RX 5700 XT": { + tflops: 19.51, + memory: [8], + gfxVersion: "gfx1010", + msrp: 399, + power: 225, + releaseYear: 2019, + }, + "RX 5700": { + tflops: 15.9, + memory: [8], + gfxVersion: "gfx1010", + msrp: 349, + power: 180, + releaseYear: 2019, + }, + "RX 5500 XT": { + tflops: 10.39, + memory: [4, 8], + gfxVersion: "gfx1012", + msrp: 200, + power: 130, + releaseYear: 2019, + }, + "Radeon Pro V620": { + tflops: 40.55, + memory: [32], + gfxVersion: "gfx1030", + msrp: 3_000, + power: 300, + releaseYear: 2021, + }, + "Radeon Pro VII": { + tflops: 26.11, + memory: [16, 32], + gfxVersion: "gfx906", + msrp: 1_900, + power: 250, + releaseYear: 2020, + }, + "Radeon 610M": { + tflops: 0.97, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1037", + msrp: 300, + power: 15, + releaseYear: 2022, + }, + "Radeon 740M": { + tflops: 5.12, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1103", + msrp: 179, + power: 15, + releaseYear: 2023, + }, + "Radeon 760M": { + tflops: 10.65, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1103", + msrp: 229, + power: 15, + releaseYear: 2023, + }, + "Radeon 780M": { + tflops: 16.59, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1103", + msrp: 329, + power: 15, + releaseYear: 2023, + }, + "Radeon 820M": { + tflops: 1.434, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1152", + msrp: 200, + power: 15, + releaseYear: 2025, + }, + "Radeon 840M": { + tflops: 2.97, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1152", + msrp: 250, + power: 15, + releaseYear: 2025, + }, + "Radeon 860M": { + tflops: 6.14, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1152", + msrp: 300, + power: 15, + releaseYear: 2025, + }, + "Radeon 880M": { + tflops: 8.91, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1150", + msrp: 400, + power: 15, + releaseYear: 2024, + }, + "Radeon 890M": { + tflops: 11.88, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1150", + msrp: 450, + power: 15, + releaseYear: 2024, + }, + "Ryzen AI Max+ 395": { + tflops: 29.7, + memory: [64, 96, 128], + gfxVersion: "gfx1151", + msrp: 1_500, + power: 120, + releaseYear: 2025, + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/hardware-nvidia.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/hardware-nvidia.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..390f423e6b8fb5703a98de527976f81d1f0a5108 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/hardware-nvidia.d.ts @@ -0,0 +1,28 @@ +import type { HardwareSpec } from "./hardware.js"; +export interface NvidiaHardwareSpec extends HardwareSpec { + /** + * CUDA Compute Capability (NVIDIA GPUs only) + * + * potential source https://developer.nvidia.com/cuda/gpus + */ + computeCapability: number; +} +export declare enum NvidiaComputeCapabilities { + BLACKWELL_ULTRA = 12.1, + BLACKWELL_RTX = 12, + BLACKWELL = 10, + HOPPER = 9, + ADA_LOVELACE = 8.9, + ORIN = 8.7, + AMPERE_RTX = 8.6, + AMPERE = 8, + TURING = 7.5, + XAVIER = 7.2, + VOLTA = 7, + PASCAL_TEGRA = 6.2, + PASCAL = 6.1, + PASCAL_DATACENTER = 6, + MAXWELL = 5.3 +} +export declare const NVIDIA_SKUS: Record; +//# sourceMappingURL=hardware-nvidia.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/hardware-nvidia.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/hardware-nvidia.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..0df030e7b56b0cbd874df04b301ad6c6c79d51ab --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/hardware-nvidia.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"hardware-nvidia.d.ts","sourceRoot":"","sources":["../../src/hardware-nvidia.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElD,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACvD;;;;OAIG;IACH,iBAAiB,EAAE,MAAM,CAAC;CAC1B;AAED,oBAAY,yBAAyB;IACpC,eAAe,OAAO;IACtB,aAAa,KAAO;IACpB,SAAS,KAAO;IAChB,MAAM,IAAM;IACZ,YAAY,MAAM;IAClB,IAAI,MAAM;IACV,UAAU,MAAM;IAChB,MAAM,IAAM;IACZ,MAAM,MAAM;IACZ,MAAM,MAAM;IACZ,KAAK,IAAM;IACX,YAAY,MAAM;IAClB,MAAM,MAAM;IACZ,iBAAiB,IAAM;IACvB,OAAO,MAAM;CACb;AAED,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAi9B1D,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/hardware-nvidia.js b/node_modules/@huggingface/tasks/dist/commonjs/hardware-nvidia.js new file mode 100644 index 0000000000000000000000000000000000000000..afdde36b877f40f06bb3ab51542e1341f28dd977 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/hardware-nvidia.js @@ -0,0 +1,999 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.NVIDIA_SKUS = exports.NvidiaComputeCapabilities = void 0; +var NvidiaComputeCapabilities; +(function (NvidiaComputeCapabilities) { + NvidiaComputeCapabilities[NvidiaComputeCapabilities["BLACKWELL_ULTRA"] = 12.1] = "BLACKWELL_ULTRA"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["BLACKWELL_RTX"] = 12] = "BLACKWELL_RTX"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["BLACKWELL"] = 10] = "BLACKWELL"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["HOPPER"] = 9] = "HOPPER"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["ADA_LOVELACE"] = 8.9] = "ADA_LOVELACE"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["ORIN"] = 8.7] = "ORIN"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["AMPERE_RTX"] = 8.6] = "AMPERE_RTX"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["AMPERE"] = 8] = "AMPERE"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["TURING"] = 7.5] = "TURING"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["XAVIER"] = 7.2] = "XAVIER"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["VOLTA"] = 7] = "VOLTA"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["PASCAL_TEGRA"] = 6.2] = "PASCAL_TEGRA"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["PASCAL"] = 6.1] = "PASCAL"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["PASCAL_DATACENTER"] = 6] = "PASCAL_DATACENTER"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["MAXWELL"] = 5.3] = "MAXWELL"; +})(NvidiaComputeCapabilities || (exports.NvidiaComputeCapabilities = NvidiaComputeCapabilities = {})); +exports.NVIDIA_SKUS = { + B300: { + tflops: 1232, + memory: [288], + computeCapability: 10.0, + msrp: 45_000, + power: 1400, + releaseYear: 2026, + }, + B200: { + tflops: 496.6, + memory: [192], + computeCapability: 10.0, + msrp: 40_000, + power: 1000, + releaseYear: 2024, + }, + H200: { + tflops: 241.3, + memory: [141], + computeCapability: 9.0, + msrp: 32_000, + power: 700, + releaseYear: 2024, + }, + H100: { + tflops: 267.6, + memory: [80], + computeCapability: 9.0, + msrp: 30_000, + power: 700, + releaseYear: 2022, + }, + H800: { + tflops: 237.2, + memory: [80], + computeCapability: 9.0, + msrp: 30_000, + power: 700, + releaseYear: 2023, + }, + H20: { + tflops: 148, + memory: [96], + computeCapability: 9.0, + msrp: 13_500, + power: 400, + releaseYear: 2024, + }, + L40s: { + tflops: 91.61, + memory: [48], + computeCapability: 8.9, + msrp: 8_500, + power: 350, + releaseYear: 2023, + }, + L40: { + tflops: 90.52, + memory: [48], + computeCapability: 8.9, + msrp: 7_500, + power: 300, + releaseYear: 2022, + }, + L20: { + tflops: 59.35, + memory: [48], + computeCapability: 8.9, + msrp: 5_000, + power: 275, + releaseYear: 2023, + }, + L4: { + tflops: 30.29, + memory: [24], + computeCapability: 8.9, + msrp: 2_500, + power: 72, + releaseYear: 2023, + }, + GB10: { + tflops: 29.71, + memory: [128], + computeCapability: 12.1, + msrp: 3_999, + power: 140, + releaseYear: 2025, + }, + "RTX PRO 6000 WS": { + tflops: 126, + memory: [96], + computeCapability: 12.0, + msrp: 8_600, + power: 600, + releaseYear: 2025, + }, + "RTX PRO 6000 Max-Q": { + tflops: 116, + memory: [96], + computeCapability: 12.0, + msrp: 8_600, + power: 300, + releaseYear: 2025, + }, + "RTX PRO 5000": { + tflops: 66.94, + memory: [48, 72], + computeCapability: 12.0, + msrp: 4_500, + power: 300, + releaseYear: 2025, + }, + "RTX PRO 4500 WS": { + tflops: 50.53, + memory: [32], + computeCapability: 12.0, + msrp: 2_800, + power: 200, + releaseYear: 2025, + }, + "RTX PRO 4000": { + tflops: 36.83, + memory: [24], + computeCapability: 12.0, + msrp: 1_500, + power: 140, + releaseYear: 2025, + }, + "RTX PRO 4000 SFF": { + tflops: 24.05, + memory: [24], + computeCapability: 12.0, + msrp: 1_500, + power: 70, + releaseYear: 2025, + }, + "RTX PRO 2000": { + tflops: 17.03, + memory: [16], + computeCapability: 12.0, + msrp: 700, + power: 70, + releaseYear: 2025, + }, + "RTX 6000 Ada": { + tflops: 91.1, + memory: [48], + computeCapability: 8.9, + msrp: 6_800, + power: 300, + releaseYear: 2022, + }, + "RTX 5880 Ada": { + tflops: 69.3, + memory: [48], + computeCapability: 8.9, + msrp: 6_000, + power: 285, + releaseYear: 2024, + }, + "RTX 5000 Ada": { + tflops: 65.3, + memory: [32], + computeCapability: 8.9, + msrp: 4_000, + power: 250, + releaseYear: 2023, + }, + "RTX 4500 Ada": { + tflops: 39.6, + memory: [24], + computeCapability: 8.9, + msrp: 2_250, + power: 210, + releaseYear: 2023, + }, + "RTX 4000 Ada": { + tflops: 26.7, + memory: [20], + computeCapability: 8.9, + msrp: 1_250, + power: 130, + releaseYear: 2023, + }, + "RTX 4000 SFF Ada": { + tflops: 19.2, + memory: [20], + computeCapability: 8.9, + msrp: 1_250, + power: 70, + releaseYear: 2023, + }, + "RTX 3500 Ada Mobile": { + tflops: 15.8, + memory: [12], + computeCapability: 8.9, + msrp: 1_500, + power: 150, + releaseYear: 2023, + }, + "RTX 2000 Ada": { + tflops: 12.0, + memory: [16], + computeCapability: 8.9, + msrp: 650, + power: 70, + releaseYear: 2024, + }, + "RTX A6000": { + tflops: 38.7, + memory: [48], + computeCapability: 8.6, + msrp: 4_650, + power: 300, + releaseYear: 2020, + }, + "RTX A5000": { + tflops: 27.77, + memory: [8, 12, 24], + computeCapability: 8.6, + msrp: 2_250, + power: 230, + releaseYear: 2021, + }, + "RTX A5000 Max-Q": { + tflops: 16.59, + memory: [16], + computeCapability: 8.6, + msrp: 2_000, + power: 80, + releaseYear: 2021, + }, + "RTX A5000 Mobile": { + tflops: 19.35, + memory: [16], + computeCapability: 8.6, + msrp: 2_000, + power: 165, + releaseYear: 2021, + }, + "RTX A4000": { + tflops: 19.17, + memory: [16], + computeCapability: 8.6, + msrp: 1_000, + power: 140, + releaseYear: 2021, + }, + "RTX A4000 Max-Q": { + tflops: 14.28, + memory: [8], + computeCapability: 8.6, + msrp: 1_000, + power: 35, + releaseYear: 2021, + }, + "RTX A4000 Mobile": { + tflops: 17.2, + memory: [8], + computeCapability: 8.6, + msrp: 1_000, + power: 80, + releaseYear: 2021, + }, + "RTX A3000 Mobile": { + tflops: 10.9, + memory: [6, 12], + computeCapability: 8.6, + msrp: 700, + power: 80, + releaseYear: 2021, + }, + "RTX A2000": { + tflops: 7.987, + memory: [6, 12], + computeCapability: 8.6, + msrp: 450, + power: 70, + releaseYear: 2021, + }, + "RTX A2000 Embedded": { + tflops: 6.026, + memory: [4], + computeCapability: 8.6, + msrp: 400, + power: 70, + releaseYear: 2022, + }, + "RTX A2000 Max-Q": { + tflops: 6.1, + memory: [4, 8], + computeCapability: 8.6, + msrp: 450, + power: 35, + releaseYear: 2021, + }, + "RTX A2000 Mobile": { + tflops: 8.4, + memory: [4, 8], + computeCapability: 8.6, + msrp: 450, + power: 95, + releaseYear: 2021, + }, + A800: { + tflops: 77.97, + memory: [40, 80], + computeCapability: 8.0, + msrp: 12_000, + power: 400, + releaseYear: 2022, + }, + A100: { + tflops: 77.97, + memory: [80, 40], + computeCapability: 8.0, + msrp: 15_000, + power: 400, + releaseYear: 2020, + }, + A40: { + tflops: 37.42, + memory: [48], + computeCapability: 8.6, + msrp: 5_500, + power: 300, + releaseYear: 2020, + }, + A30: { + tflops: 10.32, + memory: [24], + computeCapability: 8.0, + msrp: 5_000, + power: 165, + releaseYear: 2021, + }, + A10: { + tflops: 31.24, + memory: [24], + computeCapability: 8.6, + msrp: 3_200, + power: 150, + releaseYear: 2021, + }, + A2: { + tflops: 4.531, + memory: [16], + computeCapability: 8.6, + msrp: 1_000, + power: 60, + releaseYear: 2021, + }, + "RTX 5090": { + tflops: 104.8, + memory: [32], + computeCapability: 12.0, + msrp: 2_000, + power: 575, + releaseYear: 2025, + }, + "RTX 5090 D": { + tflops: 104.8, + memory: [32], + computeCapability: 12.0, + msrp: 2_000, + power: 575, + releaseYear: 2025, + }, + "RTX 5090 Mobile": { + tflops: 31.8, + memory: [24], + computeCapability: 12.0, + msrp: 1_500, + power: 175, + releaseYear: 2025, + }, + "RTX 5080": { + tflops: 56.28, + memory: [16], + computeCapability: 12.0, + msrp: 1_000, + power: 360, + releaseYear: 2025, + }, + "RTX 5080 Mobile": { + tflops: 23.04, + memory: [16], + computeCapability: 12.0, + msrp: 1_000, + power: 175, + releaseYear: 2025, + }, + "RTX 5070": { + tflops: 30.84, + memory: [12], + computeCapability: 12.0, + msrp: 550, + power: 250, + releaseYear: 2025, + }, + "RTX 5070 Mobile": { + tflops: 13.13, + memory: [8], + computeCapability: 12.0, + msrp: 500, + power: 100, + releaseYear: 2025, + }, + "RTX 5070 Ti": { + tflops: 43.94, + memory: [16], + computeCapability: 12.0, + msrp: 750, + power: 300, + releaseYear: 2025, + }, + "RTX 5070 Ti Mobile": { + tflops: 17.04, + memory: [12], + computeCapability: 12.0, + msrp: 700, + power: 140, + releaseYear: 2025, + }, + "RTX 5060 Ti": { + tflops: 23.7, + memory: [16, 8], + computeCapability: 12.0, + msrp: 450, + power: 180, + releaseYear: 2025, + }, + "RTX 5060": { + tflops: 19.18, + memory: [8], + computeCapability: 12.0, + msrp: 300, + power: 150, + releaseYear: 2025, + }, + "RTX 5060 Mobile": { + tflops: 9.684, + memory: [8], + computeCapability: 12.0, + msrp: 300, + power: 100, + releaseYear: 2025, + }, + "RTX 5050": { + tflops: 13.17, + memory: [8], + computeCapability: 12.0, + msrp: 249, + power: 130, + releaseYear: 2025, + }, + "RTX 5050 Mobile": { + tflops: 7.7, + memory: [8], + computeCapability: 12.0, + msrp: 250, + power: 100, + releaseYear: 2025, + }, + "RTX 4090": { + tflops: 82.58, + memory: [24], + computeCapability: 8.9, + msrp: 1_600, + power: 450, + releaseYear: 2022, + }, + "RTX 4090D": { + tflops: 79.49, + memory: [24, 48], + computeCapability: 8.9, + msrp: 1_600, + power: 425, + releaseYear: 2023, + }, + "RTX 4090 Mobile": { + tflops: 32.98, + memory: [16], + computeCapability: 8.9, + msrp: 1_500, + power: 150, + releaseYear: 2023, + }, + "RTX 4080 SUPER": { + tflops: 52.2, + memory: [16], + computeCapability: 8.9, + msrp: 1_000, + power: 320, + releaseYear: 2024, + }, + "RTX 4080": { + tflops: 48.7, + memory: [16], + computeCapability: 8.9, + msrp: 1_200, + power: 320, + releaseYear: 2022, + }, + "RTX 4080 Mobile": { + tflops: 24.72, + memory: [12], + computeCapability: 8.9, + msrp: 1_000, + power: 150, + releaseYear: 2023, + }, + "RTX 4070": { + tflops: 29.15, + memory: [12], + computeCapability: 8.9, + msrp: 600, + power: 200, + releaseYear: 2023, + }, + "RTX 4070 Mobile": { + tflops: 15.62, + memory: [8], + computeCapability: 8.9, + msrp: 500, + power: 115, + releaseYear: 2023, + }, + "RTX 4070 Ti": { + tflops: 40.09, + memory: [12], + computeCapability: 8.9, + msrp: 800, + power: 285, + releaseYear: 2023, + }, + "RTX 4070 Super": { + tflops: 35.48, + memory: [12], + computeCapability: 8.9, + msrp: 600, + power: 220, + releaseYear: 2024, + }, + "RTX 4070 Ti Super": { + tflops: 44.1, + memory: [16], + computeCapability: 8.9, + msrp: 800, + power: 285, + releaseYear: 2024, + }, + "RTX 4060": { + tflops: 15.11, + memory: [8], + computeCapability: 8.9, + msrp: 300, + power: 115, + releaseYear: 2023, + }, + "RTX 4060 Ti": { + tflops: 22.06, + memory: [8, 16], + computeCapability: 8.9, + msrp: 500, + power: 165, + releaseYear: 2023, + }, + "RTX 4090 Laptop": { + tflops: 32.98, + memory: [16], + computeCapability: 8.9, + msrp: 1_500, + power: 150, + releaseYear: 2023, + }, + "RTX 4080 Laptop": { + tflops: 24.72, + memory: [12], + computeCapability: 8.9, + msrp: 1_000, + power: 150, + releaseYear: 2023, + }, + "RTX 4070 Laptop": { + tflops: 15.62, + memory: [8], + computeCapability: 8.9, + msrp: 500, + power: 115, + releaseYear: 2023, + }, + "RTX 4060 Laptop": { + tflops: 11.61, + memory: [8], + computeCapability: 8.9, + msrp: 300, + power: 115, + releaseYear: 2023, + }, + "RTX 4050 Laptop": { + tflops: 8.9, + memory: [6], + computeCapability: 8.9, + msrp: 250, + power: 115, + releaseYear: 2023, + }, + "RTX 3090": { + tflops: 35.58, + memory: [24], + computeCapability: 8.6, + msrp: 1_500, + power: 350, + releaseYear: 2020, + }, + "RTX 3090 Ti": { + tflops: 40, + memory: [24], + computeCapability: 8.6, + msrp: 2_000, + power: 450, + releaseYear: 2022, + }, + "RTX 3080": { + tflops: 30.6, + memory: [12, 10], + computeCapability: 8.6, + msrp: 800, + power: 350, + releaseYear: 2020, + }, + "RTX 3080 Ti": { + tflops: 34.1, + memory: [12], + computeCapability: 8.6, + msrp: 1_200, + power: 350, + releaseYear: 2021, + }, + "RTX 3080 Mobile": { + tflops: 18.98, + memory: [16, 8], + computeCapability: 8.6, + msrp: 800, + power: 150, + releaseYear: 2021, + }, + "RTX 3070": { + tflops: 20.31, + memory: [8], + computeCapability: 8.6, + msrp: 500, + power: 220, + releaseYear: 2020, + }, + "RTX 3070 Ti": { + tflops: 21.75, + memory: [8], + computeCapability: 8.6, + msrp: 600, + power: 290, + releaseYear: 2021, + }, + "RTX 3070 Ti Mobile": { + tflops: 16.6, + memory: [8], + computeCapability: 8.6, + msrp: 700, + power: 125, + releaseYear: 2022, + }, + "RTX 3060 Ti": { + tflops: 16.2, + memory: [8], + computeCapability: 8.6, + msrp: 400, + power: 200, + releaseYear: 2020, + }, + "RTX 3060": { + tflops: 12.74, + memory: [12, 8], + computeCapability: 8.6, + msrp: 350, + power: 170, + releaseYear: 2021, + }, + "RTX 2080 Ti": { + tflops: 26.9, + memory: [11, 22], // 22GB: modded 2080ti + computeCapability: 7.5, + msrp: 1_000, + power: 250, + releaseYear: 2018, + }, + "RTX 2080": { + tflops: 20.14, + memory: [8], + computeCapability: 7.5, + msrp: 700, + power: 215, + releaseYear: 2018, + }, + "RTX 2070": { + tflops: 14.93, + memory: [8], + computeCapability: 7.5, + msrp: 500, + power: 175, + releaseYear: 2018, + }, + "RTX 2070 SUPER Mobile": { + tflops: 14.13, + memory: [8], + computeCapability: 7.5, + msrp: 600, + power: 115, + releaseYear: 2020, + }, + "RTX 2070 SUPER": { + tflops: 18.12, + memory: [8], + computeCapability: 7.5, + msrp: 500, + power: 215, + releaseYear: 2019, + }, + "RTX 3060 Mobile": { + tflops: 10.94, + memory: [6], + computeCapability: 8.6, + msrp: 400, + power: 115, + releaseYear: 2021, + }, + "RTX 3050 Mobile": { + tflops: 7.639, + memory: [4, 6], + computeCapability: 8.6, + msrp: 250, + power: 95, + releaseYear: 2022, + }, + "RTX 2060": { + tflops: 12.9, + memory: [6], + computeCapability: 7.5, + msrp: 350, + power: 160, + releaseYear: 2019, + }, + "RTX 2060 12GB": { + tflops: 14.36, + memory: [12], + computeCapability: 7.5, + msrp: 300, + power: 184, + releaseYear: 2021, + }, + "RTX 2060 Mobile": { + tflops: 9.22, + memory: [6], + computeCapability: 7.5, + msrp: 350, + power: 90, + releaseYear: 2019, + }, + "RTX 2050 Mobile": { + tflops: 10.2, + memory: [4], + computeCapability: 8.6, // Ampere (outlier GPU in the 20xx series) + msrp: 250, + power: 45, + releaseYear: 2021, + }, + "GTX 1080 Ti": { + tflops: 11.34, // float32 (GPU does not support native float16) + memory: [11], + computeCapability: 6.1, + msrp: 700, + power: 250, + releaseYear: 2017, + }, + "GTX 1080": { + tflops: 8.87, // float32 (GPU does not support native float16) + memory: [8], + computeCapability: 6.1, + msrp: 599, + power: 180, + releaseYear: 2016, + }, + "GTX 1070 Ti": { + tflops: 8.2, // float32 (GPU does not support native float16) + memory: [8], + computeCapability: 6.1, + msrp: 450, + power: 180, + releaseYear: 2017, + }, + "GTX 1070": { + tflops: 6.46, // float32 (GPU does not support native float16) + memory: [8], + computeCapability: 6.1, + msrp: 379, + power: 150, + releaseYear: 2016, + }, + "GTX 1060": { + tflops: 3.9, // float32 (GPU does not support native float16) + memory: [3, 6], + computeCapability: 6.1, + msrp: 300, + power: 120, + releaseYear: 2016, + }, + "GTX 1050 Ti": { + tflops: 2.1, // float32 (GPU does not support native float16) + memory: [4], + computeCapability: 6.1, + msrp: 150, + power: 75, + releaseYear: 2016, + }, + "RTX Titan": { + tflops: 32.62, + memory: [24], + computeCapability: 7.5, + msrp: 2_500, + power: 280, + releaseYear: 2018, + }, + "GTX 1660": { + tflops: 10.05, + memory: [6], + computeCapability: 7.5, + msrp: 200, + power: 120, + releaseYear: 2019, + }, + "GTX 1650 Mobile": { + tflops: 6.39, + memory: [4], + computeCapability: 7.5, + msrp: 150, + power: 50, + releaseYear: 2019, + }, + T4: { + tflops: 65.13, + memory: [16], + computeCapability: 7.5, + msrp: 2_000, + power: 70, + releaseYear: 2018, + }, + T10: { + tflops: 20.0, + memory: [16], + computeCapability: 7.5, + msrp: 2_000, + power: 150, + releaseYear: 2020, + }, + V100: { + tflops: 28.26, + memory: [32, 16], + computeCapability: 7.0, + msrp: 10_000, + power: 300, + releaseYear: 2017, + }, + "Quadro P6000": { + tflops: 12.63, // float32 (GPU does not support native float16) + memory: [24], + computeCapability: 6.1, + msrp: 5_000, + power: 250, + releaseYear: 2016, + }, + P40: { + tflops: 11.76, // float32 (GPU does not support native float16) + memory: [24], + computeCapability: 6.1, + msrp: 5_700, + power: 250, + releaseYear: 2016, + }, + P100: { + tflops: 19.05, + memory: [16], + computeCapability: 6.0, + msrp: 7_000, + power: 300, + releaseYear: 2016, + }, + "Jetson AGX Orin 64GB": { + tflops: 10.65, + memory: [64], + computeCapability: 8.7, + msrp: 2_000, + power: 60, + releaseYear: 2022, + }, + "Jetson AGX Orin 32GB": { + tflops: 6.66, + memory: [32], + computeCapability: 8.7, + msrp: 999, + power: 40, + releaseYear: 2022, + }, + "Jetson Orin NX 16GB": { + tflops: 3.76, + memory: [16], + computeCapability: 8.7, + msrp: 600, + power: 25, + releaseYear: 2023, + }, + "Jetson Orin NX 8GB": { + tflops: 3.13, + memory: [8], + computeCapability: 8.7, + msrp: 400, + power: 20, + releaseYear: 2023, + }, + "Jetson Orin Nano 8GB": { + tflops: 2.56, + memory: [8], + computeCapability: 8.7, + msrp: 500, + power: 15, + releaseYear: 2023, + }, + "Jetson Orin Nano 4GB": { + tflops: 1.28, + memory: [4], + computeCapability: 8.7, + msrp: 200, + power: 10, + releaseYear: 2023, + }, + "Jetson AGX Xavier": { + tflops: 2.82, + memory: [32, 64], + computeCapability: 7.2, + msrp: 1_100, + power: 30, + releaseYear: 2018, + }, + "Jetson Xavier NX": { + tflops: 1.69, + memory: [8, 16], + computeCapability: 7.2, + msrp: 400, + power: 20, + releaseYear: 2020, + }, + "Jetson TX2": { + tflops: 1.33, + memory: [4, 8], + computeCapability: 6.2, + msrp: 400, + power: 15, + releaseYear: 2017, + }, + "Jetson Nano": { + tflops: 0.47, + memory: [4], + computeCapability: 5.3, + msrp: 100, + power: 10, + releaseYear: 2019, + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/hardware.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/hardware.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..310b38752315edca143f3fd9f77d9d414c7db2ac --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/hardware.d.ts @@ -0,0 +1,616 @@ +/** + * Biden AI Executive Order (since revoked by President Trump): + * https://web.archive.org/web/20250105222429/https://www.whitehouse.gov/briefing-room/presidential-actions/2023/10/30/executive-order-on-the-safe-secure-and-trustworthy-development-and-use-of-artificial-intelligence/ + */ +export declare const TFLOPS_THRESHOLD_WHITE_HOUSE_MODEL_TRAINING_TOTAL: number; +export declare const TFLOPS_THRESHOLD_WHITE_HOUSE_MODEL_TRAINING_TOTAL_BIOLOGY: number; +export declare const TFLOPS_THRESHOLD_WHITE_HOUSE_CLUSTER: number; +/** + * EU AI Act + * https://ec.europa.eu/commission/presscorner/detail/en/qanda_21_1683 + */ +export declare const TFLOPS_THRESHOLD_EU_AI_ACT_MODEL_TRAINING_TOTAL: number; +export interface HardwareSpec { + /** + * Approximate value, in FP16 whenever possible for GPUs and FP32 for CPUs. + * This is only approximate/theoretical and shouldn't be taken too seriously. + * Currently the CPU values are from cpu-monkey.com + * while the GPU values are from techpowerup.com + * + * Note to reviewers: I got fed up with data entry, + * and HuggingChat running Llama3 with Web search was failing a bit, + * so some of those values might be slightly inaccurate. Forgive me and please feel free to improve. + */ + tflops: number; + /** + * If an array is specified, options of memory size (can be VRAM, unified RAM) + * e.g. an A100 exists in 40 or 80 GB. + */ + memory?: number[]; + /** + * Approximate MSRP in USD at launch. For SKUs with multiple memory variants, + * the price corresponds to the largest memory variant. For datacenter GPUs + * sold via OEMs without a public MSRP (H100, MI300X, ...), this is a + * widely-reported street price. For mobile/laptop GPUs that are not sold + * standalone, this is the approximate module/BOM cost. For Apple Silicon + * SoCs, this is the price of a Mac configured with that chip and the + * largest memory option. For CPU "family" entries (e.g. "Xeon 4th Gen", + * "Ryzen Zen 4 7000 (Ryzen 9)"), this is the tray/box price of a + * representative flagship SKU at launch. + */ + msrp: number; + /** + * Approximate maximum sustained power draw in watts. For GPUs with multiple + * form factors (e.g. H100 SXM vs PCIe), uses the highest variant. For CPUs, + * uses max turbo power (PL2 / MTP for Intel, PPT for AMD), not base TDP. + * For Apple Silicon and Snapdragon SoCs, an estimated package power based + * on benchmarks/teardowns (Apple does not publish TDP). + */ + power: number; + /** + * Year the SKU first became available. For SKUs refreshed later with + * additional memory variants (e.g. A100 40GB → 80GB, RTX 2060 → 12GB), + * this is the original launch year. For CPU "family" entries, this is + * the year the family debuted. + */ + releaseYear: number; +} +export declare const DEFAULT_MEMORY_OPTIONS: number[]; +export declare const SKUS: { + GPU: { + NVIDIA: Record; + AMD: Record; + INTEL: { + "Arc A750": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Arc A770": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Arc B570": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Arc B580": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Arc B50": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Arc B60": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Arc Pro B70": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + }; + QUALCOMM: { + "Snapdragon X Elite X1E-00-1DE": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Snapdragon X Elite X1E-84-100": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Snapdragon X Elite X1E-80-100": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Snapdragon X Elite X1E-78-100": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Snapdragon X Plus X1P-64-100": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + }; + }; + CPU: { + Intel: { + "Xeon 4th Generation (Sapphire Rapids)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Xeon 3th Generation (Ice Lake)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Xeon 2th Generation (Cascade Lake)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Xeon E5v4 (Broadwell)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Xeon E5v3 (Haswell)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Xeon E5v2 (Ivy Bridge)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core Ultra 9 275HX": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core Ultra 7 255HX": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core Ultra 7 265KF": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 14th Generation (i7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 13th Generation (i9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 13th Generation (i7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 13th Generation (i5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 13th Generation (i3)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 12th Generation (i9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 12th Generation (i7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 12th Generation (i5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 12th Generation (i3)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 11th Generation (i9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 11th Generation (i7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 11th Generation (i5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 11th Generation (i3)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 10th Generation (i9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 10th Generation (i7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 10th Generation (i5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 10th Generation (i3)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + }; + AMD: { + "EPYC 5th Generation Zen 5 (Turin)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "EPYC 4th Generation Zen 4 (Genoa)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "EPYC 3th Generation Zen 3 (Milan)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "EPYC 2th Generation Zen 2 (Rome)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "EPYC 1st Generation Zen (Naples)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Threadripper Zen 5 9000 (Shimada Peak)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Threadripper Zen 4 7000 (Storm Peak)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Threadripper Zen 3 5000 (Chagall)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Threadripper Zen 2 3000 (Castle Peak)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Threadripper Zen 1000 (Whitehaven)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen 7 3800X (16)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 5 9000 (Ryzen 9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 5 9000 (Ryzen 7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 5 9000 (Ryzen 5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 4 7000 (Ryzen 9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 4 7000 (Ryzen 7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 4 7000 (Ryzen 5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 3 5000 (Ryzen 9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 3 5000 (Ryzen 7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 3 5000 (Ryzen 5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 2 3000 (Ryzen 9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 2 3000 (Ryzen 7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 2 3000 (Ryzen 5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 2 3000 (Ryzen 3)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen AI 300 (Ryzen AI 9 HX)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen AI 300 (Ryzen AI 9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen AI 300 (Ryzen AI 7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen AI 300 (Ryzen AI 5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + }; + }; + "Apple Silicon": { + "-": { + "Apple MacBook Neo": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M1": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M1 Pro": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M1 Max": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M1 Ultra": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M2": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M2 Pro": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M2 Max": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M2 Ultra": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M3": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M3 Pro": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M3 Max": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M3 Ultra": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M4": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M4 Pro": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M4 Max": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M5": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M5 Pro": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M5 Max": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + }; + }; +}; +export type SkuType = keyof typeof SKUS; +//# sourceMappingURL=hardware.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/hardware.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/hardware.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..898c3b2e810236abf7ac4c266e52f78c05c1452b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/hardware.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"hardware.d.ts","sourceRoot":"","sources":["../../src/hardware.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,eAAO,MAAM,iDAAiD,QAAW,CAAC;AAC1E,eAAO,MAAM,yDAAyD,QAAW,CAAC;AAClF,eAAO,MAAM,oCAAoC,QAAU,CAAC;AAE5D;;;GAGG;AACH,eAAO,MAAM,+CAA+C,QAAW,CAAC;AAExE,MAAM,WAAW,YAAY;IAC5B;;;;;;;;;OASG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB;;;;;;;;;;OAUG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,WAAW,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,sBAAsB,UAElC,CAAC;AAEF,eAAO,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2iBuD,CAAC;AAEzE,MAAM,MAAM,OAAO,GAAG,MAAM,OAAO,IAAI,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/hardware.js b/node_modules/@huggingface/tasks/dist/commonjs/hardware.js new file mode 100644 index 0000000000000000000000000000000000000000..cb65335fd44f6c936f6d0f72d840f6018bd4967a --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/hardware.js @@ -0,0 +1,576 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.SKUS = exports.DEFAULT_MEMORY_OPTIONS = exports.TFLOPS_THRESHOLD_EU_AI_ACT_MODEL_TRAINING_TOTAL = exports.TFLOPS_THRESHOLD_WHITE_HOUSE_CLUSTER = exports.TFLOPS_THRESHOLD_WHITE_HOUSE_MODEL_TRAINING_TOTAL_BIOLOGY = exports.TFLOPS_THRESHOLD_WHITE_HOUSE_MODEL_TRAINING_TOTAL = void 0; +const hardware_amd_js_1 = require("./hardware-amd.js"); +const hardware_nvidia_js_1 = require("./hardware-nvidia.js"); +/** + * Biden AI Executive Order (since revoked by President Trump): + * https://web.archive.org/web/20250105222429/https://www.whitehouse.gov/briefing-room/presidential-actions/2023/10/30/executive-order-on-the-safe-secure-and-trustworthy-development-and-use-of-artificial-intelligence/ + */ +exports.TFLOPS_THRESHOLD_WHITE_HOUSE_MODEL_TRAINING_TOTAL = 10 ** 14; +exports.TFLOPS_THRESHOLD_WHITE_HOUSE_MODEL_TRAINING_TOTAL_BIOLOGY = 10 ** 11; +exports.TFLOPS_THRESHOLD_WHITE_HOUSE_CLUSTER = 10 ** 8; +/** + * EU AI Act + * https://ec.europa.eu/commission/presscorner/detail/en/qanda_21_1683 + */ +exports.TFLOPS_THRESHOLD_EU_AI_ACT_MODEL_TRAINING_TOTAL = 10 ** 13; +exports.DEFAULT_MEMORY_OPTIONS = [ + 8, 16, 24, 32, 40, 48, 64, 80, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, 2048, +]; +exports.SKUS = { + GPU: { + NVIDIA: hardware_nvidia_js_1.NVIDIA_SKUS, + AMD: hardware_amd_js_1.AMD_GPU_SKUS, + INTEL: { + "Arc A750": { + tflops: 34.41, + memory: [8], + msrp: 250, + power: 225, + releaseYear: 2022, + }, + "Arc A770": { + tflops: 39.32, + memory: [8, 16], + msrp: 350, + power: 225, + releaseYear: 2022, + }, + "Arc B570": { + tflops: 23.04, + memory: [10], + msrp: 200, + power: 150, + releaseYear: 2025, + }, + "Arc B580": { + tflops: 27.34, + memory: [12], + msrp: 250, + power: 190, + releaseYear: 2024, + }, + "Arc B50": { + tflops: 21.3, + memory: [16], + msrp: 350, + power: 70, + releaseYear: 2025, + }, + "Arc B60": { + tflops: 24.58, + memory: [24, 48], + msrp: 1_200, + power: 200, + releaseYear: 2025, + }, + "Arc Pro B70": { + tflops: 45.88, + memory: [32], + msrp: 949, + power: 230, + releaseYear: 2026, + }, + }, + QUALCOMM: { + "Snapdragon X Elite X1E-00-1DE": { + tflops: 4.6, + msrp: 900, + power: 80, + releaseYear: 2024, + }, + "Snapdragon X Elite X1E-84-100": { + tflops: 4.6, + msrp: 1_700, + power: 30, + releaseYear: 2024, + }, + "Snapdragon X Elite X1E-80-100": { + tflops: 3.8, + msrp: 1_300, + power: 23, + releaseYear: 2024, + }, + "Snapdragon X Elite X1E-78-100": { + tflops: 3.8, + msrp: 1_200, + power: 23, + releaseYear: 2024, + }, + "Snapdragon X Plus X1P-64-100": { + tflops: 3.8, + msrp: 1_000, + power: 23, + releaseYear: 2024, + }, + }, + }, + CPU: { + Intel: { + "Xeon 4th Generation (Sapphire Rapids)": { + tflops: 1.3, + msrp: 10_500, + power: 350, + releaseYear: 2023, + }, + "Xeon 3th Generation (Ice Lake)": { + tflops: 0.8, + msrp: 8_000, + power: 270, + releaseYear: 2021, + }, + "Xeon 2th Generation (Cascade Lake)": { + tflops: 0.55, + msrp: 10_000, + power: 205, + releaseYear: 2019, + }, + "Xeon E5v4 (Broadwell)": { + tflops: 0.25, + msrp: 4_000, + power: 145, + releaseYear: 2016, + }, + "Xeon E5v3 (Haswell)": { + tflops: 0.2, + msrp: 4_000, + power: 145, + releaseYear: 2014, + }, + "Xeon E5v2 (Ivy Bridge)": { + tflops: 0.15, + msrp: 2_500, + power: 130, + releaseYear: 2013, + }, + "Intel Core Ultra 9 275HX": { + tflops: 1.89, + msrp: 700, + power: 160, + releaseYear: 2025, + }, + "Intel Core Ultra 7 255HX": { + tflops: 1.62, + msrp: 583, + power: 160, + releaseYear: 2025, + }, + "Intel Core Ultra 7 265KF": { + tflops: 1.53, + msrp: 400, + power: 250, + releaseYear: 2024, + }, + "Intel Core 14th Generation (i7)": { + tflops: 0.8, + msrp: 400, + power: 253, + releaseYear: 2023, + }, + "Intel Core 13th Generation (i9)": { + tflops: 0.85, + msrp: 600, + power: 253, + releaseYear: 2022, + }, + "Intel Core 13th Generation (i7)": { + tflops: 0.82, + msrp: 400, + power: 253, + releaseYear: 2022, + }, + "Intel Core 13th Generation (i5)": { + tflops: 0.68, + msrp: 300, + power: 181, + releaseYear: 2022, + }, + "Intel Core 13th Generation (i3)": { + tflops: 0.57, + msrp: 150, + power: 89, + releaseYear: 2023, + }, + "Intel Core 12th Generation (i9)": { + tflops: 0.79, + msrp: 600, + power: 241, + releaseYear: 2021, + }, + "Intel Core 12th Generation (i7)": { + tflops: 0.77, + msrp: 400, + power: 190, + releaseYear: 2021, + }, + "Intel Core 12th Generation (i5)": { + tflops: 0.65, + msrp: 300, + power: 150, + releaseYear: 2021, + }, + "Intel Core 12th Generation (i3)": { + tflops: 0.53, + msrp: 150, + power: 89, + releaseYear: 2022, + }, + "Intel Core 11th Generation (i9)": { + tflops: 0.7, + msrp: 550, + power: 251, + releaseYear: 2021, + }, + "Intel Core 11th Generation (i7)": { + tflops: 0.6, + msrp: 400, + power: 251, + releaseYear: 2021, + }, + "Intel Core 11th Generation (i5)": { + tflops: 0.5, + msrp: 250, + power: 251, + releaseYear: 2021, + }, + "Intel Core 11th Generation (i3)": { + tflops: 0.35, + msrp: 150, + power: 90, + releaseYear: 2021, + }, + "Intel Core 10th Generation (i9)": { + tflops: 0.46, + msrp: 500, + power: 250, + releaseYear: 2020, + }, + "Intel Core 10th Generation (i7)": { + tflops: 0.46, + msrp: 400, + power: 215, + releaseYear: 2020, + }, + "Intel Core 10th Generation (i5)": { + tflops: 0.46, + msrp: 250, + power: 182, + releaseYear: 2020, + }, + "Intel Core 10th Generation (i3)": { + tflops: 0.44, + msrp: 150, + power: 90, + releaseYear: 2020, + }, + }, + AMD: { + "EPYC 5th Generation Zen 5 (Turin)": { + tflops: 13.8, + msrp: 13_000, + power: 500, + releaseYear: 2024, + }, + "EPYC 4th Generation Zen 4 (Genoa)": { + tflops: 5, + msrp: 11_500, + power: 360, + releaseYear: 2022, + }, + "EPYC 3th Generation Zen 3 (Milan)": { + tflops: 2.4, + msrp: 8_000, + power: 280, + releaseYear: 2021, + }, + "EPYC 2th Generation Zen 2 (Rome)": { + tflops: 0.6, + msrp: 7_000, + power: 225, + releaseYear: 2019, + }, + "EPYC 1st Generation Zen (Naples)": { + tflops: 0.6, + msrp: 4_000, + power: 180, + releaseYear: 2017, + }, + "Ryzen Threadripper Zen 5 9000 (Shimada Peak)": { + tflops: 14.0, + msrp: 5_000, + power: 350, + releaseYear: 2025, + }, + "Ryzen Threadripper Zen 4 7000 (Storm Peak)": { + tflops: 10.0, + msrp: 5_000, + power: 350, + releaseYear: 2023, + }, + "Ryzen Threadripper Zen 3 5000 (Chagall)": { + tflops: 4.6, + msrp: 6_500, + power: 280, + releaseYear: 2022, + }, + "Ryzen Threadripper Zen 2 3000 (Castle Peak)": { + tflops: 3.2, + msrp: 4_000, + power: 280, + releaseYear: 2019, + }, + "Ryzen Threadripper Zen 1000 (Whitehaven)": { + tflops: 0.6, + msrp: 1_000, + power: 180, + releaseYear: 2017, + }, + "Ryzen 7 3800X (16)": { + tflops: 1.15, + msrp: 400, + power: 142, + releaseYear: 2019, + }, + "Ryzen Zen 5 9000 (Ryzen 9)": { + tflops: 0.56, + msrp: 650, + power: 230, + releaseYear: 2024, + }, + "Ryzen Zen 5 9000 (Ryzen 7)": { + tflops: 0.56, + msrp: 350, + power: 88, + releaseYear: 2024, + }, + "Ryzen Zen 5 9000 (Ryzen 5)": { + tflops: 0.56, + msrp: 300, + power: 88, + releaseYear: 2024, + }, + "Ryzen Zen 4 7000 (Ryzen 9)": { + tflops: 0.56, + msrp: 700, + power: 230, + releaseYear: 2022, + }, + "Ryzen Zen 4 7000 (Ryzen 7)": { + tflops: 0.56, + msrp: 400, + power: 142, + releaseYear: 2022, + }, + "Ryzen Zen 4 7000 (Ryzen 5)": { + tflops: 0.56, + msrp: 300, + power: 142, + releaseYear: 2022, + }, + "Ryzen Zen 3 5000 (Ryzen 9)": { + tflops: 1.33, + msrp: 800, + power: 142, + releaseYear: 2020, + }, + "Ryzen Zen 3 5000 (Ryzen 7)": { + tflops: 1.33, + msrp: 450, + power: 142, + releaseYear: 2020, + }, + "Ryzen Zen 3 5000 (Ryzen 5)": { + tflops: 0.72, + msrp: 300, + power: 88, + releaseYear: 2020, + }, + "Ryzen Zen 2 3000 (Ryzen 9)": { + tflops: 0.72, + msrp: 750, + power: 142, + releaseYear: 2019, + }, + "Ryzen Zen 2 3000 (Ryzen 7)": { + tflops: 0.72, + msrp: 400, + power: 142, + releaseYear: 2019, + }, + "Ryzen Zen 2 3000 (Ryzen 5)": { + tflops: 0.72, + msrp: 250, + power: 88, + releaseYear: 2019, + }, + "Ryzen Zen 2 3000 (Ryzen 3)": { + tflops: 0.72, + msrp: 150, + power: 88, + releaseYear: 2020, + }, + "Ryzen AI 300 (Ryzen AI 9 HX)": { + tflops: 5.52, + msrp: 500, + power: 54, + releaseYear: 2024, + }, + "Ryzen AI 300 (Ryzen AI 9)": { + tflops: 5.2, + msrp: 450, + power: 54, + releaseYear: 2024, + }, + "Ryzen AI 300 (Ryzen AI 7)": { + tflops: 4.34, + msrp: 350, + power: 54, + releaseYear: 2024, + }, + "Ryzen AI 300 (Ryzen AI 5)": { + tflops: 1.57, + msrp: 250, + power: 28, + releaseYear: 2024, + }, + }, + }, + "Apple Silicon": { + "-": { + "Apple MacBook Neo": { + tflops: 1.9, + memory: [8], + msrp: 700, + power: 10, + releaseYear: 2026, + }, + "Apple M1": { + tflops: 2.6, + memory: [8, 16], + msrp: 1_250, + power: 15, + releaseYear: 2020, + }, + "Apple M1 Pro": { + tflops: 5.2, + memory: [16, 24, 32], + msrp: 2_900, + power: 30, + releaseYear: 2021, + }, + "Apple M1 Max": { + tflops: 10.4, + memory: [16, 24, 32, 64], + msrp: 3_900, + power: 60, + releaseYear: 2021, + }, + "Apple M1 Ultra": { + tflops: 21, + memory: [16, 24, 32, 64, 96, 128], + msrp: 6_200, + power: 120, + releaseYear: 2022, + }, + "Apple M2": { + tflops: 3.6, + memory: [8, 16, 24], + msrp: 1_500, + power: 20, + releaseYear: 2022, + }, + "Apple M2 Pro": { + tflops: 6.8, + memory: [16, 24, 32], + msrp: 2_800, + power: 35, + releaseYear: 2023, + }, + "Apple M2 Max": { + tflops: 13.49, + memory: [32, 64, 96], + msrp: 4_500, + power: 80, + releaseYear: 2023, + }, + "Apple M2 Ultra": { + tflops: 27.2, + memory: [64, 96, 128, 192], + msrp: 7_000, + power: 150, + releaseYear: 2023, + }, + "Apple M3": { + tflops: 4.1, + memory: [8, 16, 24], + msrp: 1_500, + power: 22, + releaseYear: 2023, + }, + "Apple M3 Pro": { + tflops: 7.4, + memory: [18, 36], + msrp: 2_400, + power: 40, + releaseYear: 2023, + }, + "Apple M3 Max": { + tflops: 14.2, + memory: [36, 48, 64, 96, 128], + msrp: 5_000, + power: 90, + releaseYear: 2023, + }, + "Apple M3 Ultra": { + tflops: 28.4, + memory: [96, 256, 512], + msrp: 9_500, + power: 180, + releaseYear: 2025, + }, + "Apple M4": { + tflops: 4.6, + memory: [16, 24, 32], + msrp: 1_600, + power: 22, + releaseYear: 2024, + }, + "Apple M4 Pro": { + tflops: 9.2, + memory: [24, 48, 64], + msrp: 2_600, + power: 45, + releaseYear: 2024, + }, + "Apple M4 Max": { + tflops: 18.4, + memory: [36, 48, 64, 128], + msrp: 5_000, + power: 100, + releaseYear: 2024, + }, + "Apple M5": { + tflops: 5.7, + memory: [16, 24, 32], + msrp: 2_000, + power: 25, + releaseYear: 2025, + }, + "Apple M5 Pro": { + tflops: 11.4, + memory: [24, 36, 48, 64], + msrp: 2_900, + power: 50, + releaseYear: 2026, + }, + "Apple M5 Max": { + tflops: 22.8, + memory: [36, 48, 64, 128], + msrp: 5_000, + power: 110, + releaseYear: 2026, + }, + }, + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/index.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..958e2cd12307a2baa022b82289fb081537d87e35 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/index.d.ts @@ -0,0 +1,28 @@ +export { LIBRARY_TASK_MAPPING } from "./library-to-tasks.js"; +export { MAPPING_DEFAULT_WIDGET } from "./default-widget-inputs.js"; +export type { TaskData, TaskDemo, TaskDemoEntry, ExampleRepo } from "./tasks/index.js"; +export * from "./tasks/index.js"; +export { PIPELINE_DATA, PIPELINE_TYPES, type WidgetType, type PipelineType, type PipelineData, type Modality, MODALITIES, MODALITY_LABELS, SUBTASK_TYPES, PIPELINE_TYPES_SET, } from "./pipelines.js"; +export { ALL_DISPLAY_MODEL_LIBRARY_KEYS, ALL_MODEL_LIBRARY_KEYS, MODEL_LIBRARIES_UI_ELEMENTS, } from "./model-libraries.js"; +export type { LibraryUiElement, ModelLibraryKey } from "./model-libraries.js"; +export type { ModelData, TransformersInfo } from "./model-data.js"; +export type { AddedToken, SpecialTokensMap, TokenizerConfig } from "./tokenizer-data.js"; +export type { WidgetExample, WidgetExampleAttribute, WidgetExampleAssetAndPromptInput, WidgetExampleAssetAndTextInput, WidgetExampleAssetAndZeroShotInput, WidgetExampleAssetInput, WidgetExampleChatInput, WidgetExampleSentenceSimilarityInput, WidgetExampleStructuredDataInput, WidgetExampleTableDataInput, WidgetExampleTextAndContextInput, WidgetExampleTextAndTableInput, WidgetExampleTextInput, WidgetExampleZeroShotTextInput, WidgetExampleOutput, WidgetExampleOutputUrl, WidgetExampleOutputLabels, WidgetExampleOutputAnswerScore, WidgetExampleOutputText, } from "./widget-example.js"; +export { SPECIAL_TOKENS_ATTRIBUTES } from "./tokenizer-data.js"; +export * from "./gguf.js"; +export { type InferenceSnippet, type InferenceSnippetLanguage, type ModelDataMinimal, inferenceSnippetLanguages, stringifyGenerationConfig, stringifyMessages, getModelInputSnippet, } from "./snippets/index.js"; +export { SKUS, DEFAULT_MEMORY_OPTIONS } from "./hardware.js"; +export type { HardwareSpec, SkuType } from "./hardware.js"; +export type { AmdGpuHardwareSpec } from "./hardware-amd.js"; +export type { NvidiaHardwareSpec } from "./hardware-nvidia.js"; +export { LOCAL_APPS } from "./local-apps.js"; +export type { LocalApp, LocalAppKey, LocalAppSnippet } from "./local-apps.js"; +export { DATASET_LIBRARIES_UI_ELEMENTS } from "./dataset-libraries.js"; +export type { DatasetLibraryUiElement, DatasetLibraryKey } from "./dataset-libraries.js"; +export { KERNEL_LIBRARIES_UI_ELEMENTS } from "./kernel-libraries.js"; +export type { KernelLibraryKey, KernelLibraryUiElement } from "./kernel-libraries.js"; +export * from "./inference-providers.js"; +export { EVALUATION_FRAMEWORKS } from "./eval.js"; +export { AGENT_HARNESSES, STANDARD_AGENT_ENV_VARS } from "./agent-harnesses.js"; +export type { AgentHarness, AgentHarnessKey } from "./agent-harnesses.js"; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/index.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..10ee927a0fb7f54095ea82af0e903545d73a987d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACpE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACvF,cAAc,kBAAkB,CAAC;AACjC,OAAO,EACN,aAAa,EACb,cAAc,EACd,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,QAAQ,EACb,UAAU,EACV,eAAe,EACf,aAAa,EACb,kBAAkB,GAClB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACN,8BAA8B,EAC9B,sBAAsB,EACtB,2BAA2B,GAC3B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC9E,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACzF,YAAY,EACX,aAAa,EACb,sBAAsB,EACtB,gCAAgC,EAChC,8BAA8B,EAC9B,kCAAkC,EAClC,uBAAuB,EACvB,sBAAsB,EACtB,oCAAoC,EACpC,gCAAgC,EAChC,2BAA2B,EAC3B,gCAAgC,EAChC,8BAA8B,EAC9B,sBAAsB,EACtB,8BAA8B,EAC9B,mBAAmB,EACnB,sBAAsB,EACtB,yBAAyB,EACzB,8BAA8B,EAC9B,uBAAuB,GACvB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAEhE,cAAc,WAAW,CAAC;AAE1B,OAAO,EACN,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EACrB,yBAAyB,EACzB,yBAAyB,EACzB,iBAAiB,EACjB,oBAAoB,GACpB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,IAAI,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAC7D,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAC3D,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC5D,YAAY,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAE9E,OAAO,EAAE,6BAA6B,EAAE,MAAM,wBAAwB,CAAC;AACvE,YAAY,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAEzF,OAAO,EAAE,4BAA4B,EAAE,MAAM,uBAAuB,CAAC;AACrE,YAAY,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAEtF,cAAc,0BAA0B,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAElD,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAChF,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/index.js b/node_modules/@huggingface/tasks/dist/commonjs/index.js new file mode 100644 index 0000000000000000000000000000000000000000..f699bb4bba003402d6eac41d69e077309d293874 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/index.js @@ -0,0 +1,56 @@ +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __exportStar = (this && this.__exportStar) || function(m, exports) { + for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.STANDARD_AGENT_ENV_VARS = exports.AGENT_HARNESSES = exports.EVALUATION_FRAMEWORKS = exports.KERNEL_LIBRARIES_UI_ELEMENTS = exports.DATASET_LIBRARIES_UI_ELEMENTS = exports.LOCAL_APPS = exports.DEFAULT_MEMORY_OPTIONS = exports.SKUS = exports.getModelInputSnippet = exports.stringifyMessages = exports.stringifyGenerationConfig = exports.inferenceSnippetLanguages = exports.SPECIAL_TOKENS_ATTRIBUTES = exports.MODEL_LIBRARIES_UI_ELEMENTS = exports.ALL_MODEL_LIBRARY_KEYS = exports.ALL_DISPLAY_MODEL_LIBRARY_KEYS = exports.PIPELINE_TYPES_SET = exports.SUBTASK_TYPES = exports.MODALITY_LABELS = exports.MODALITIES = exports.PIPELINE_TYPES = exports.PIPELINE_DATA = exports.MAPPING_DEFAULT_WIDGET = exports.LIBRARY_TASK_MAPPING = void 0; +var library_to_tasks_js_1 = require("./library-to-tasks.js"); +Object.defineProperty(exports, "LIBRARY_TASK_MAPPING", { enumerable: true, get: function () { return library_to_tasks_js_1.LIBRARY_TASK_MAPPING; } }); +var default_widget_inputs_js_1 = require("./default-widget-inputs.js"); +Object.defineProperty(exports, "MAPPING_DEFAULT_WIDGET", { enumerable: true, get: function () { return default_widget_inputs_js_1.MAPPING_DEFAULT_WIDGET; } }); +__exportStar(require("./tasks/index.js"), exports); +var pipelines_js_1 = require("./pipelines.js"); +Object.defineProperty(exports, "PIPELINE_DATA", { enumerable: true, get: function () { return pipelines_js_1.PIPELINE_DATA; } }); +Object.defineProperty(exports, "PIPELINE_TYPES", { enumerable: true, get: function () { return pipelines_js_1.PIPELINE_TYPES; } }); +Object.defineProperty(exports, "MODALITIES", { enumerable: true, get: function () { return pipelines_js_1.MODALITIES; } }); +Object.defineProperty(exports, "MODALITY_LABELS", { enumerable: true, get: function () { return pipelines_js_1.MODALITY_LABELS; } }); +Object.defineProperty(exports, "SUBTASK_TYPES", { enumerable: true, get: function () { return pipelines_js_1.SUBTASK_TYPES; } }); +Object.defineProperty(exports, "PIPELINE_TYPES_SET", { enumerable: true, get: function () { return pipelines_js_1.PIPELINE_TYPES_SET; } }); +var model_libraries_js_1 = require("./model-libraries.js"); +Object.defineProperty(exports, "ALL_DISPLAY_MODEL_LIBRARY_KEYS", { enumerable: true, get: function () { return model_libraries_js_1.ALL_DISPLAY_MODEL_LIBRARY_KEYS; } }); +Object.defineProperty(exports, "ALL_MODEL_LIBRARY_KEYS", { enumerable: true, get: function () { return model_libraries_js_1.ALL_MODEL_LIBRARY_KEYS; } }); +Object.defineProperty(exports, "MODEL_LIBRARIES_UI_ELEMENTS", { enumerable: true, get: function () { return model_libraries_js_1.MODEL_LIBRARIES_UI_ELEMENTS; } }); +var tokenizer_data_js_1 = require("./tokenizer-data.js"); +Object.defineProperty(exports, "SPECIAL_TOKENS_ATTRIBUTES", { enumerable: true, get: function () { return tokenizer_data_js_1.SPECIAL_TOKENS_ATTRIBUTES; } }); +__exportStar(require("./gguf.js"), exports); +var index_js_1 = require("./snippets/index.js"); +Object.defineProperty(exports, "inferenceSnippetLanguages", { enumerable: true, get: function () { return index_js_1.inferenceSnippetLanguages; } }); +Object.defineProperty(exports, "stringifyGenerationConfig", { enumerable: true, get: function () { return index_js_1.stringifyGenerationConfig; } }); +Object.defineProperty(exports, "stringifyMessages", { enumerable: true, get: function () { return index_js_1.stringifyMessages; } }); +Object.defineProperty(exports, "getModelInputSnippet", { enumerable: true, get: function () { return index_js_1.getModelInputSnippet; } }); +var hardware_js_1 = require("./hardware.js"); +Object.defineProperty(exports, "SKUS", { enumerable: true, get: function () { return hardware_js_1.SKUS; } }); +Object.defineProperty(exports, "DEFAULT_MEMORY_OPTIONS", { enumerable: true, get: function () { return hardware_js_1.DEFAULT_MEMORY_OPTIONS; } }); +var local_apps_js_1 = require("./local-apps.js"); +Object.defineProperty(exports, "LOCAL_APPS", { enumerable: true, get: function () { return local_apps_js_1.LOCAL_APPS; } }); +var dataset_libraries_js_1 = require("./dataset-libraries.js"); +Object.defineProperty(exports, "DATASET_LIBRARIES_UI_ELEMENTS", { enumerable: true, get: function () { return dataset_libraries_js_1.DATASET_LIBRARIES_UI_ELEMENTS; } }); +var kernel_libraries_js_1 = require("./kernel-libraries.js"); +Object.defineProperty(exports, "KERNEL_LIBRARIES_UI_ELEMENTS", { enumerable: true, get: function () { return kernel_libraries_js_1.KERNEL_LIBRARIES_UI_ELEMENTS; } }); +__exportStar(require("./inference-providers.js"), exports); +var eval_js_1 = require("./eval.js"); +Object.defineProperty(exports, "EVALUATION_FRAMEWORKS", { enumerable: true, get: function () { return eval_js_1.EVALUATION_FRAMEWORKS; } }); +var agent_harnesses_js_1 = require("./agent-harnesses.js"); +Object.defineProperty(exports, "AGENT_HARNESSES", { enumerable: true, get: function () { return agent_harnesses_js_1.AGENT_HARNESSES; } }); +Object.defineProperty(exports, "STANDARD_AGENT_ENV_VARS", { enumerable: true, get: function () { return agent_harnesses_js_1.STANDARD_AGENT_ENV_VARS; } }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/inference-providers.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/inference-providers.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..08c44ebf82b0fd557e194c83c6117366d4534b0f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/inference-providers.d.ts @@ -0,0 +1,11 @@ +declare const INFERENCE_PROVIDERS: readonly ["cerebras", "cohere", "deepinfra", "fal-ai", "fireworks-ai", "hf-inference", "ovhcloud", "replicate", "together"]; +export type SnippetInferenceProvider = (typeof INFERENCE_PROVIDERS)[number] | string; +export declare const HF_HUB_INFERENCE_PROXY_TEMPLATE = "https://router.huggingface.co/{{PROVIDER}}"; +/** + * URL to set as baseUrl in the OpenAI SDK. + * + * TODO(Expose this from InferenceClient in the future?) + */ +export declare function openAIbaseUrl(provider: SnippetInferenceProvider): string; +export {}; +//# sourceMappingURL=inference-providers.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/inference-providers.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/inference-providers.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c3374c6a738eec476cf3bcb7b5fa2e174994393b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/inference-providers.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference-providers.d.ts","sourceRoot":"","sources":["../../src/inference-providers.ts"],"names":[],"mappings":"AAEA,QAAA,MAAM,mBAAmB,6HAUf,CAAC;AAEX,MAAM,MAAM,wBAAwB,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;AAErF,eAAO,MAAM,+BAA+B,+CAA+C,CAAC;AAE5F;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,wBAAwB,GAAG,MAAM,CAGxE"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/inference-providers.js b/node_modules/@huggingface/tasks/dist/commonjs/inference-providers.js new file mode 100644 index 0000000000000000000000000000000000000000..7ec862c9b53c2c5d865f6d65d867a933787ee717 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/inference-providers.js @@ -0,0 +1,27 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.HF_HUB_INFERENCE_PROXY_TEMPLATE = void 0; +exports.openAIbaseUrl = openAIbaseUrl; +/// This list is for illustration purposes only. +/// in the `tasks` sub-package, we do not need actual strong typing of the inference providers. +const INFERENCE_PROVIDERS = [ + "cerebras", + "cohere", + "deepinfra", + "fal-ai", + "fireworks-ai", + "hf-inference", + "ovhcloud", + "replicate", + "together", +]; +exports.HF_HUB_INFERENCE_PROXY_TEMPLATE = `https://router.huggingface.co/{{PROVIDER}}`; +/** + * URL to set as baseUrl in the OpenAI SDK. + * + * TODO(Expose this from InferenceClient in the future?) + */ +function openAIbaseUrl(provider) { + const url = exports.HF_HUB_INFERENCE_PROXY_TEMPLATE.replace("{{PROVIDER}}", provider); + return provider === "hf-inference" ? `${url}/v1` : url; +} diff --git a/node_modules/@huggingface/tasks/dist/commonjs/kernel-libraries.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/kernel-libraries.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..31eff06c1e4045bc98715ba40ed714716d21eebc --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/kernel-libraries.d.ts @@ -0,0 +1,36 @@ +/** + * Elements configurable by a kernel library. + */ +export interface KernelLibraryUiElement { + /** + * Pretty name of the library. + */ + prettyLabel: string; + /** + * Repo name of the library's (usually on GitHub) code repo + */ + repoName: string; + /** + * URL to library's (usually on GitHub) code repo + */ + repoUrl: string; + /** + * URL to library's docs + */ + docsUrl?: string; + /** + * Code snippet(s) displayed + */ + snippets?: (kernelId: string, version?: number) => string[]; +} +export declare const KERNEL_LIBRARIES_UI_ELEMENTS: { + kernels: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (kernelId: string, version?: number) => string[]; + }; +}; +export type KernelLibraryKey = keyof typeof KERNEL_LIBRARIES_UI_ELEMENTS; +//# sourceMappingURL=kernel-libraries.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/kernel-libraries.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/kernel-libraries.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..685de12f73ff91349ccd786ad085bc97e82e5ff0 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/kernel-libraries.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"kernel-libraries.d.ts","sourceRoot":"","sources":["../../src/kernel-libraries.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CAC5D;AAED,eAAO,MAAM,4BAA4B;;;;;;6BAMlB,MAAM,YAAY,MAAM;;CAQG,CAAC;AAEnD,MAAM,MAAM,gBAAgB,GAAG,MAAM,OAAO,4BAA4B,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/kernel-libraries.js b/node_modules/@huggingface/tasks/dist/commonjs/kernel-libraries.js new file mode 100644 index 0000000000000000000000000000000000000000..9cbaaaa2b175fe1cab0ece9b3777f1c5e2877909 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/kernel-libraries.js @@ -0,0 +1,18 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.KERNEL_LIBRARIES_UI_ELEMENTS = void 0; +exports.KERNEL_LIBRARIES_UI_ELEMENTS = { + kernels: { + prettyLabel: "Kernels", + repoName: "Kernels", + repoUrl: "https://github.com/huggingface/kernels", + docsUrl: "https://huggingface.co/docs/kernels", + snippets: (kernelId, version) => [ + `# !pip install kernels + +from kernels import get_kernel + +kernel = get_kernel("${kernelId}"${version !== undefined ? `, version=${version}` : ""})`, + ], + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/library-to-tasks.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/library-to-tasks.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..61ffd282076871d1b86f92570b95e9a44cded208 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/library-to-tasks.d.ts @@ -0,0 +1,12 @@ +import type { ModelLibraryKey } from "./model-libraries.js"; +import type { PipelineType } from "./pipelines.js"; +/** + * Mapping from library name to its supported tasks. + * HF-Inference API (serverless) should be disabled for all other (library, task) pairs beyond this mapping. + * This mapping is partially generated automatically by "python-api-export-tasks" action in + * huggingface/api-inference-community repo upon merge. For transformers, the mapping is manually + * based on api-inference (hf_types.rs). + */ +export declare const LIBRARY_TASK_MAPPING: Partial>; +export declare const REMOVED_IN_V5_TRANSFORMERS_PIPELINES: PipelineType[]; +//# sourceMappingURL=library-to-tasks.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/library-to-tasks.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/library-to-tasks.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..8b35f76b24f02a9f3c534de64d33fe47930cb7ae --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/library-to-tasks.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"library-to-tasks.d.ts","sourceRoot":"","sources":["../../src/library-to-tasks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD;;;;;;GAMG;AACH,eAAO,MAAM,oBAAoB,EAAE,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,YAAY,EAAE,CAAC,CA6DjF,CAAC;AAGF,eAAO,MAAM,oCAAoC,EAAE,YAAY,EAAsD,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/library-to-tasks.js b/node_modules/@huggingface/tasks/dist/commonjs/library-to-tasks.js new file mode 100644 index 0000000000000000000000000000000000000000..6d6ac94d7f1082c4b2216f1f722ec0808c551204 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/library-to-tasks.js @@ -0,0 +1,74 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.REMOVED_IN_V5_TRANSFORMERS_PIPELINES = exports.LIBRARY_TASK_MAPPING = void 0; +/** + * Mapping from library name to its supported tasks. + * HF-Inference API (serverless) should be disabled for all other (library, task) pairs beyond this mapping. + * This mapping is partially generated automatically by "python-api-export-tasks" action in + * huggingface/api-inference-community repo upon merge. For transformers, the mapping is manually + * based on api-inference (hf_types.rs). + */ +exports.LIBRARY_TASK_MAPPING = { + "adapter-transformers": ["question-answering", "text-classification", "token-classification"], + allennlp: ["question-answering"], + asteroid: [ + // "audio-source-separation", + "audio-to-audio", + ], + bertopic: ["text-classification"], + diffusers: ["image-to-image", "text-to-image"], + doctr: ["object-detection"], + espnet: ["text-to-speech", "automatic-speech-recognition"], + fairseq: ["text-to-speech", "audio-to-audio"], + fastai: ["image-classification"], + fasttext: ["feature-extraction", "text-classification"], + flair: ["token-classification"], + k2: ["automatic-speech-recognition"], + keras: ["image-classification"], + nemo: ["automatic-speech-recognition"], + open_clip: ["zero-shot-classification", "zero-shot-image-classification"], + paddlenlp: ["fill-mask", "summarization", "zero-shot-classification"], + peft: ["text-generation"], + "pyannote-audio": ["automatic-speech-recognition"], + "sentence-transformers": ["feature-extraction", "sentence-similarity"], + setfit: ["text-classification"], + sklearn: ["tabular-classification", "tabular-regression", "text-classification"], + spacy: ["token-classification", "text-classification", "sentence-similarity"], + "span-marker": ["token-classification"], + speechbrain: ["audio-classification", "audio-to-audio", "automatic-speech-recognition", "text-to-speech"], + stanza: ["token-classification"], + timm: ["image-classification", "image-feature-extraction"], + transformers: [ + "audio-classification", + "automatic-speech-recognition", + "depth-estimation", + "document-question-answering", + "feature-extraction", + "fill-mask", + "image-classification", + "image-feature-extraction", + "image-segmentation", + "image-to-image", + "image-to-text", + "image-text-to-text", + "mask-generation", + "object-detection", + "question-answering", + "summarization", + "table-question-answering", + "text-classification", + "text-generation", + "text-to-audio", + "text-to-speech", + "token-classification", + "translation", + "video-classification", + "visual-question-answering", + "zero-shot-classification", + "zero-shot-image-classification", + "zero-shot-object-detection", + ], + mindspore: ["image-classification"], +}; +// Pipeline types that were supported in legacy transformers versions (<5.0.0) +exports.REMOVED_IN_V5_TRANSFORMERS_PIPELINES = ["image-to-text", "summarization", "translation"]; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/local-apps.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/local-apps.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..25a3af87e0fed557b022d70a16db032ec4732e23 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/local-apps.d.ts @@ -0,0 +1,265 @@ +import type { ModelData } from "./model-data.js"; +import type { PipelineType } from "./pipelines.js"; +export interface LocalAppSnippet { + /** + * Title of the snippet + */ + title: string; + /** + * Optional setup guide + */ + setup?: string; + /** + * Content (or command) to be run + */ + content: string | string[]; +} +/** + * Elements configurable by a local app. + */ +export type LocalApp = { + /** + * Name that appears in buttons + */ + prettyLabel: string; + /** + * Link to get more info about a local app (website etc) + */ + docsUrl: string; + /** + * Additional links to display (max 2) + */ + links?: { + label: string; + url: string; + }[] | ((model: ModelData) => { + label: string; + url: string; + }[]); + /** + * main category of app + */ + mainTask: PipelineType; + /** + * Whether to display a pill "macOS-only" + */ + macOSOnly?: boolean; + comingSoon?: boolean; + /** + * IMPORTANT: function to figure out whether to display the button on a model page's main "Use this model" dropdown. + */ + displayOnModelPage: (model: ModelData) => boolean; +} & ({ + /** + * If the app supports deeplink, URL to open. + */ + deeplink: (model: ModelData, filepath?: string) => URL; +} | { + /** + * And if not (mostly llama.cpp), snippet to copy/paste in your terminal + * Support the placeholder {{GGUF_FILE}} that will be replaced by the gguf file path or the list of available files. + * Support the placeholder {{QUANT_TAG}} that will be replaced by the list of available quant tags or will be removed if there are no multiple quant files in a same repo. + */ + snippet: (model: ModelData, filepath?: string) => string | string[] | LocalAppSnippet | LocalAppSnippet[]; +}); +declare function isTgiModel(model: ModelData): boolean; +declare function isLlamaCppGgufModel(model: ModelData): boolean; +declare function isVllmModel(model: ModelData): boolean; +declare function isDockerModelRunnerModel(model: ModelData): boolean; +declare function isUnslothModel(model: ModelData): boolean; +declare function isToolCallingLocalAgentModel(model: ModelData): boolean; +/** + * Add your new local app here. + * + * This is open to new suggestions and awesome upcoming apps. + * + * /!\ IMPORTANT + * + * If possible, you need to support deeplinks and be as cross-platform as possible. + * + * Ping the HF team if we can help with anything! + */ +export declare const LOCAL_APPS: { + "llama.cpp": { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[]; + }; + "node-llama-cpp": { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[]; + }; + vllm: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isVllmModel; + snippet: (model: ModelData) => LocalAppSnippet[]; + }; + sglang: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: (model: ModelData) => boolean; + snippet: (model: ModelData) => LocalAppSnippet[]; + }; + "mlx-lm": { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: (model: ModelData) => boolean; + snippet: (model: ModelData) => LocalAppSnippet[]; + }; + tgi: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isTgiModel; + snippet: (model: ModelData) => LocalAppSnippet[]; + }; + lmstudio: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: (model: ModelData) => boolean; + deeplink: (model: ModelData, filepath: string | undefined) => URL; + }; + localai: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[]; + }; + jan: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + deeplink: (model: ModelData) => URL; + }; + "atomic-chat": { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + deeplink: (model: ModelData) => URL; + }; + backyard: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + deeplink: (model: ModelData) => URL; + }; + sanctum: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + deeplink: (model: ModelData) => URL; + }; + jellybox: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: (model: ModelData) => boolean; + deeplink: (model: ModelData) => URL; + }; + msty: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + deeplink: (model: ModelData) => URL; + }; + recursechat: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + macOSOnly: true; + displayOnModelPage: typeof isLlamaCppGgufModel; + deeplink: (model: ModelData) => URL; + }; + drawthings: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-to-image"; + macOSOnly: true; + displayOnModelPage: (model: ModelData) => boolean; + deeplink: (model: ModelData) => URL; + }; + diffusionbee: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-to-image"; + macOSOnly: true; + displayOnModelPage: (model: ModelData) => boolean; + deeplink: (model: ModelData) => URL; + }; + joyfusion: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-to-image"; + macOSOnly: true; + displayOnModelPage: (model: ModelData) => boolean; + deeplink: (model: ModelData) => URL; + }; + ollama: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + snippet: (model: ModelData, filepath?: string) => string; + }; + unsloth: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isUnslothModel; + snippet: (model: ModelData) => LocalAppSnippet[]; + }; + "docker-model-runner": { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isDockerModelRunnerModel; + snippet: (model: ModelData, filepath?: string) => string; + }; + lemonade: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: (model: ModelData) => boolean; + snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[]; + }; + pi: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isToolCallingLocalAgentModel; + snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[]; + }; + "hermes-agent": { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isToolCallingLocalAgentModel; + snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[]; + }; + openclaw: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isToolCallingLocalAgentModel; + snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[]; + }; +}; +export type LocalAppKey = keyof typeof LOCAL_APPS; +export {}; +//# sourceMappingURL=local-apps.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/local-apps.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/local-apps.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d644c2d36e15262c1a8cb1e7a9e21cfbb3412310 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/local-apps.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"local-apps.d.ts","sourceRoot":"","sources":["../../src/local-apps.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAKnD,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACtB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,SAAS,KAAK;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAAC;IACpG;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IACvB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,kBAAkB,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC;CAClD,GAAG,CACD;IACA;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,CAAC;CACtD,GACD;IACA;;;;OAIG;IACH,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,EAAE,GAAG,eAAe,GAAG,eAAe,EAAE,CAAC;CACzG,CACH,CAAC;AAsBF,iBAAS,UAAU,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE7C;AAED,iBAAS,mBAAmB,CAAC,KAAK,EAAE,SAAS,WAE5C;AAED,iBAAS,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAU9C;AAED,iBAAS,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE3D;AA0BD,iBAAS,cAAc,CAAC,KAAK,EAAE,SAAS,WAEvC;AAED,iBAAS,4BAA4B,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAM/D;AA4dD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,UAAU;;;;;;yBA1dS,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAiDzC,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBA2F3D,SAAS,KAAG,eAAe,EAAE;;;;;;oCAwW3B,SAAS;yBAlTT,SAAS,KAAG,eAAe,EAAE;;;;;;;yBAoF9B,SAAS,KAAG,eAAe,EAAE;;;;;;;yBA7B/B,SAAS,KAAG,eAAe,EAAE;;;;;;;;;;;;;;yBApIzB,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAtDjD,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;yBAIpC,SAAS,KAAG,eAAe,EAAE;;;;;;;yBAgWnB,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;yBAM9C,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAlGtD,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAkCvC,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBA2BnD,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;CAqS5C,CAAC;AAErC,MAAM,MAAM,WAAW,GAAG,MAAM,OAAO,UAAU,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/local-apps.js b/node_modules/@huggingface/tasks/dist/commonjs/local-apps.js new file mode 100644 index 0000000000000000000000000000000000000000..86468daf5bbfb9f7510dc77a886cef9e4dba8103 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/local-apps.js @@ -0,0 +1,716 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.LOCAL_APPS = void 0; +const gguf_js_1 = require("./gguf.js"); +const common_js_1 = require("./snippets/common.js"); +const inputs_js_1 = require("./snippets/inputs.js"); +function isAwqModel(model) { + return model.config?.quantization_config?.quant_method === "awq"; +} +function isGptqModel(model) { + return model.config?.quantization_config?.quant_method === "gptq"; +} +function isAqlmModel(model) { + return model.config?.quantization_config?.quant_method === "aqlm"; +} +function isMarlinModel(model) { + return model.config?.quantization_config?.quant_method === "marlin"; +} +function isTransformersModel(model) { + return model.tags.includes("transformers"); +} +function isTgiModel(model) { + return model.tags.includes("text-generation-inference"); +} +function isLlamaCppGgufModel(model) { + return !!model.gguf?.context_length; +} +function isVllmModel(model) { + return ((isAwqModel(model) || + isGptqModel(model) || + isAqlmModel(model) || + isMarlinModel(model) || + isLlamaCppGgufModel(model) || + isTransformersModel(model)) && + (model.pipeline_tag === "text-generation" || model.pipeline_tag === "image-text-to-text")); +} +function isDockerModelRunnerModel(model) { + return isLlamaCppGgufModel(model) || isVllmModel(model); +} +function isAmdRyzenModel(model) { + return model.tags.includes("ryzenai-hybrid") || model.tags.includes("ryzenai-npu"); +} +function isMlxModel(model) { + return model.tags.includes("mlx"); +} +/** + * Returns the model's chat template string, coalescing across sources: + * GGUF metadata > chat_template_jinja file > tokenizer_config.json + */ +function getChatTemplate(model) { + const ct = model.gguf?.chat_template ?? model.config?.chat_template_jinja ?? model.config?.tokenizer_config?.chat_template; + if (typeof ct === "string") { + return ct; + } + if (Array.isArray(ct)) { + return ct[0]?.template; + } + return undefined; +} +function isUnslothModel(model) { + return model.tags.includes("unsloth") || isLlamaCppGgufModel(model); +} +function isToolCallingLocalAgentModel(model) { + return ((isLlamaCppGgufModel(model) || isMlxModel(model)) && + model.tags.includes("conversational") && + !!getChatTemplate(model)?.includes("tools")); +} +function getQuantTag(filepath) { + const defaultTag = ":{{QUANT_TAG}}"; + if (!filepath) { + return defaultTag; + } + const quantLabel = (0, gguf_js_1.parseGGUFQuantLabel)(filepath); + return quantLabel ? `:${quantLabel}` : defaultTag; +} +const snippetLlamacpp = (model, filepath) => { + const serverCommand = (binary) => { + const snippet = [ + "# Start a local OpenAI-compatible server with a web UI:", + `${binary} -hf ${model.id}${getQuantTag(filepath)}`, + ]; + return snippet.join("\n"); + }; + const cliCommand = (binary) => { + const snippet = ["# Run inference directly in the terminal:", `${binary} -hf ${model.id}${getQuantTag(filepath)}`]; + return snippet.join("\n"); + }; + return [ + { + title: "Install (macOS, Linux)", + setup: "curl -LsSf https://llama.app/install.sh | sh", + content: [serverCommand("llama serve"), cliCommand("llama cli")], + }, + { + title: "Install from WinGet (Windows)", + setup: "winget install llama.cpp", + content: [serverCommand("llama serve"), cliCommand("llama cli")], + }, + { + title: "Use pre-built binary", + setup: [ + // prettier-ignore + "# Download pre-built binary from:", + "# https://github.com/ggerganov/llama.cpp/releases", + ].join("\n"), + content: [serverCommand("./llama-server"), cliCommand("./llama-cli")], + }, + { + title: "Build from source code", + setup: [ + "git clone https://github.com/ggerganov/llama.cpp.git", + "cd llama.cpp", + "cmake -B build", + "cmake --build build -j --target llama-server llama-cli", + ].join("\n"), + content: [serverCommand("./build/bin/llama-server"), cliCommand("./build/bin/llama-cli")], + }, + { + title: "Use Docker", + content: snippetDockerModelRunner(model, filepath), + }, + ]; +}; +const snippetNodeLlamaCppCli = (model, filepath) => { + const tagName = getQuantTag(filepath); + return [ + { + title: "Chat with the model", + content: `npx -y node-llama-cpp chat hf:${model.id}${tagName}`, + }, + { + title: "Estimate the model compatibility with your hardware", + content: `npx -y node-llama-cpp inspect estimate hf:${model.id}${tagName}`, + }, + ]; +}; +const snippetOllama = (model, filepath) => { + return `ollama run hf.co/${model.id}${getQuantTag(filepath)}`; +}; +const snippetUnsloth = (model) => { + const isGguf = isLlamaCppGgufModel(model); + const studio_content = [ + "# Run unsloth studio", + "unsloth studio -H 0.0.0.0 -p 8888", + "# Then open http://localhost:8888 in your browser", + "# Search for " + model.id + " to start chatting", + ].join("\n"); + const studio_instructions = { + title: "Install Unsloth Studio (macOS, Linux, WSL)", + setup: "curl -fsSL https://unsloth.ai/install.sh | sh", + content: studio_content, + }; + const studio_instructions_windows = { + title: "Install Unsloth Studio (Windows)", + setup: "irm https://unsloth.ai/install.ps1 | iex", + content: studio_content, + }; + const hf_spaces_instructions = { + title: "Using HuggingFace Spaces for Unsloth", + setup: "# No setup required", + content: "# Open https://huggingface.co/spaces/unsloth/studio in your browser\n# Search for " + + model.id + + " to start chatting", + }; + const fastmodel_instructions = { + title: "Load model with FastModel", + setup: "pip install unsloth", + content: [ + "from unsloth import FastModel", + "model, tokenizer = FastModel.from_pretrained(", + ' model_name="' + model.id + '",', + " max_seq_length=2048,", + ")", + ].join("\n"), + }; + if (isGguf) { + return [studio_instructions, studio_instructions_windows, hf_spaces_instructions]; + } + else { + return [studio_instructions, studio_instructions_windows, hf_spaces_instructions, fastmodel_instructions]; + } +}; +const snippetLocalAI = (model, filepath) => { + const command = (binary) => ["# Load and run the model:", `${binary} huggingface://${model.id}/${filepath ?? "{{GGUF_FILE}}"}`].join("\n"); + return [ + { + title: "Install from binary", + setup: "curl https://localai.io/install.sh | sh", + content: command("local-ai run"), + }, + { + title: "Use Docker images", + setup: [ + // prettier-ignore + "# Pull the image:", + "docker pull localai/localai:latest-cpu", + ].join("\n"), + content: command("docker run -p 8080:8080 --name localai -v $PWD/models:/build/models localai/localai:latest-cpu"), + }, + ]; +}; +const snippetVllm = (model) => { + const messages = (0, inputs_js_1.getModelInputSnippet)(model); + const isMistral = model.tags.includes("mistral-common"); + const mistralFlags = isMistral + ? " --tokenizer_mode mistral --config_format mistral --load_format mistral --tool-call-parser mistral --enable-auto-tool-choice" + : ""; + const setup = isMistral + ? [ + "# Install vLLM from pip:", + "pip install vllm", + "# Install mistral-common:", + "pip install --upgrade mistral-common", + ].join("\n") + : ["# Install vLLM from pip:", "pip install vllm"].join("\n"); + const serverCommand = `# Start the vLLM server: +vllm serve "${model.id}"${mistralFlags}`; + const runCommandInstruct = `# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/chat/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "${model.id}", + "messages": ${(0, common_js_1.stringifyMessages)(messages, { + indent: "\t\t", + attributeKeyQuotes: true, + customContentEscaper: (str) => str.replace(/'/g, "'\\''"), + })} + }'`; + const runCommandNonInstruct = `# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "${model.id}", + "prompt": "Once upon a time,", + "max_tokens": 512, + "temperature": 0.5 + }'`; + const runCommand = model.tags.includes("conversational") ? runCommandInstruct : runCommandNonInstruct; + return [ + { + title: "Install from pip and serve model", + setup: setup, + content: [serverCommand, runCommand], + }, + { + title: "Use Docker", + content: snippetDockerModelRunner(model), + }, + ]; +}; +const snippetSglang = (model) => { + const messages = (0, inputs_js_1.getModelInputSnippet)(model); + const setup = ["# Install SGLang from pip:", "pip install sglang"].join("\n"); + const serverCommand = `# Start the SGLang server: +python3 -m sglang.launch_server \\ + --model-path "${model.id}" \\ + --host 0.0.0.0 \\ + --port 30000`; + const dockerCommand = `docker run --gpus all \\ + --shm-size 32g \\ + -p 30000:30000 \\ + -v ~/.cache/huggingface:/root/.cache/huggingface \\ + --env "HF_TOKEN=" \\ + --ipc=host \\ + lmsysorg/sglang:latest \\ + python3 -m sglang.launch_server \\ + --model-path "${model.id}" \\ + --host 0.0.0.0 \\ + --port 30000`; + const runCommandInstruct = `# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:30000/v1/chat/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "${model.id}", + "messages": ${(0, common_js_1.stringifyMessages)(messages, { + indent: "\t\t", + attributeKeyQuotes: true, + customContentEscaper: (str) => str.replace(/'/g, "'\\''"), + })} + }'`; + const runCommandNonInstruct = `# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:30000/v1/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "${model.id}", + "prompt": "Once upon a time,", + "max_tokens": 512, + "temperature": 0.5 + }'`; + const runCommand = model.tags.includes("conversational") ? runCommandInstruct : runCommandNonInstruct; + return [ + { + title: "Install from pip and serve model", + setup: setup, + content: [serverCommand, runCommand], + }, + { + title: "Use Docker images", + setup: dockerCommand, + content: [runCommand], + }, + ]; +}; +const snippetTgi = (model) => { + const runCommand = [ + "# Call the server using curl:", + `curl -X POST "http://localhost:8000/v1/chat/completions" \\`, + ` -H "Content-Type: application/json" \\`, + ` --data '{`, + ` "model": "${model.id}",`, + ` "messages": [`, + ` {"role": "user", "content": "What is the capital of France?"}`, + ` ]`, + ` }'`, + ]; + return [ + { + title: "Use Docker images", + setup: [ + "# Deploy with docker on Linux:", + `docker run --gpus all \\`, + ` -v ~/.cache/huggingface:/root/.cache/huggingface \\`, + ` -e HF_TOKEN="" \\`, + ` -p 8000:80 \\`, + ` ghcr.io/huggingface/text-generation-inference:latest \\`, + ` --model-id ${model.id}`, + ].join("\n"), + content: [runCommand.join("\n")], + }, + ]; +}; +const snippetMlxLm = (model) => { + const openaiCurl = [ + "# Calling the OpenAI-compatible server with curl", + `curl -X POST "http://localhost:8000/v1/chat/completions" \\`, + ` -H "Content-Type: application/json" \\`, + ` --data '{`, + ` "model": "${model.id}",`, + ` "messages": [`, + ` {"role": "user", "content": "Hello"}`, + ` ]`, + ` }'`, + ]; + return [ + { + title: "Generate or start a chat session", + setup: ["# Install MLX LM", "uv tool install mlx-lm"].join("\n"), + content: [ + ...(model.tags.includes("conversational") + ? ["# Interactive chat REPL", `mlx_lm.chat --model "${model.id}"`] + : ["# Generate some text", `mlx_lm.generate --model "${model.id}" --prompt "Once upon a time"`]), + ].join("\n"), + }, + ...(model.tags.includes("conversational") + ? [ + { + title: "Run an OpenAI-compatible server", + setup: ["# Install MLX LM", "uv tool install mlx-lm"].join("\n"), + content: ["# Start the server", `mlx_lm.server --model "${model.id}"`, ...openaiCurl].join("\n"), + }, + ] + : []), + ]; +}; +const getLocalServerStep = (model, filepath) => { + return isMlxModel(model) + ? { + title: "Start the MLX server", + setup: "# Install MLX LM:\nuv tool install mlx-lm", + content: `# Start a local OpenAI-compatible server:\nmlx_lm.server --model "${model.id}"`, + } + : { + title: "Start the llama.cpp server", + setup: "# Install llama.cpp:\nbrew install llama.cpp", + content: `# Start a local OpenAI-compatible server:\nllama serve -hf ${model.id}${getQuantTag(filepath)}`, + }; +}; +const snippetPi = (model, filepath) => { + const isMLX = isMlxModel(model); + const modelId = isMLX ? model.id : `${model.id}${getQuantTag(filepath)}`; + const serverStep = getLocalServerStep(model, filepath); + const modelsJson = JSON.stringify({ + providers: { + [isMLX ? "mlx-lm" : "llama-cpp"]: { + baseUrl: "http://localhost:8080/v1", + api: "openai-completions", + apiKey: "none", + models: [{ id: modelId }], + }, + }, + }, null, 2); + return [ + serverStep, + { + title: "Configure the model in Pi", + setup: "# Install Pi:\nnpm install -g @mariozechner/pi-coding-agent", + content: `# Add to ~/.pi/agent/models.json:\n${modelsJson}`, + }, + { + title: "Run Pi", + content: "# Start Pi in your project directory:\npi", + }, + ]; +}; +const snippetHermesAgent = (model, filepath) => { + const modelId = isMlxModel(model) ? model.id : `${model.id}${getQuantTag(filepath)}`; + const serverStep = getLocalServerStep(model, filepath); + return [ + serverStep, + { + title: "Configure Hermes", + setup: [ + "# Install Hermes:", + "curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash", + "hermes setup", + ].join("\n"), + content: [ + "# Point Hermes at the local server:", + "hermes config set model.provider custom", + "hermes config set model.base_url http://127.0.0.1:8080/v1", + `hermes config set model.default ${modelId}`, + ].join("\n"), + }, + { + title: "Run Hermes", + content: "hermes", + }, + ]; +}; +const snippetOpenClaw = (model, filepath) => { + const isMLX = isMlxModel(model); + const providerId = isMLX ? "mlx-lm" : "llama-cpp"; + const modelId = isMLX ? model.id : `${model.id}${getQuantTag(filepath)}`; + const serverStep = getLocalServerStep(model, filepath); + return [ + serverStep, + { + title: "Configure OpenClaw", + setup: "# Install OpenClaw:\nnpm install -g openclaw@latest", + content: [ + "# Register the local server and set it as the default model:", + "openclaw onboard --non-interactive --mode local \\", + " --auth-choice custom-api-key \\", + " --custom-base-url http://127.0.0.1:8080/v1 \\", + ` --custom-model-id "${modelId}" \\`, + ` --custom-provider-id ${providerId} \\`, + " --custom-compatibility openai \\", + " --custom-text-input \\", + " --accept-risk \\", + " --skip-health", + ].join("\n"), + }, + { + title: "Run OpenClaw", + content: `openclaw agent --local --agent main --message "Hello from Hugging Face"`, + }, + ]; +}; +const snippetDockerModelRunner = (model, filepath) => { + // Only add quant tag for GGUF models, not safetensors + const quantTag = isLlamaCppGgufModel(model) ? getQuantTag(filepath) : ""; + return `docker model run hf.co/${model.id}${quantTag}`; +}; +const snippetLemonade = (model, filepath) => { + const modelName = model.id.includes("/") ? model.id.split("/")[1] : model.id; + const isRyzenAI = model.tags.some((tag) => ["ryzenai-npu", "ryzenai-hybrid"].includes(tag)); + // Lemonade auto-registers pulled models as `user.[-]`. + // For GGUF/llamacpp: suggested_name is the repo name and variant is the quant tag. + // For RyzenAI ONNX: there is no per-variant suffix. + let pullArg; + let runName; + let requirements; + if (isRyzenAI) { + pullArg = model.id; + runName = `user.${modelName}`; + requirements = " (requires XDNA 2 NPU)"; + } + else { + const tagName = getQuantTag(filepath); + pullArg = `${model.id}${tagName}`; + runName = `user.${modelName}${tagName.replace(":", "-")}`; + requirements = ""; + } + return [ + { + title: "Pull the model", + setup: "# Download Lemonade from https://lemonade-server.ai/", + content: `lemonade pull ${pullArg}`, + }, + { + title: `Run and chat with the model${requirements}`, + content: `lemonade run ${runName}`, + }, + { + title: "List all available models", + content: "lemonade list", + }, + ]; +}; +/** + * Add your new local app here. + * + * This is open to new suggestions and awesome upcoming apps. + * + * /!\ IMPORTANT + * + * If possible, you need to support deeplinks and be as cross-platform as possible. + * + * Ping the HF team if we can help with anything! + */ +exports.LOCAL_APPS = { + "llama.cpp": { + prettyLabel: "llama.cpp", + docsUrl: "https://github.com/ggerganov/llama.cpp", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + snippet: snippetLlamacpp, + }, + "node-llama-cpp": { + prettyLabel: "node-llama-cpp", + docsUrl: "https://node-llama-cpp.withcat.ai", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + snippet: snippetNodeLlamaCppCli, + }, + vllm: { + prettyLabel: "vLLM", + docsUrl: "https://docs.vllm.ai", + mainTask: "text-generation", + displayOnModelPage: isVllmModel, + snippet: snippetVllm, + }, + sglang: { + prettyLabel: "SGLang", + docsUrl: "https://docs.sglang.io", + mainTask: "text-generation", + displayOnModelPage: (model) => (isAwqModel(model) || + isGptqModel(model) || + isAqlmModel(model) || + isMarlinModel(model) || + isTransformersModel(model)) && + (model.pipeline_tag === "text-generation" || model.pipeline_tag === "image-text-to-text"), + snippet: snippetSglang, + }, + "mlx-lm": { + prettyLabel: "MLX LM", + docsUrl: "https://github.com/ml-explore/mlx-lm", + mainTask: "text-generation", + displayOnModelPage: (model) => model.pipeline_tag === "text-generation" && isMlxModel(model), + snippet: snippetMlxLm, + }, + tgi: { + prettyLabel: "TGI", + docsUrl: "https://huggingface.co/docs/text-generation-inference/", + mainTask: "text-generation", + displayOnModelPage: isTgiModel, + snippet: snippetTgi, + }, + lmstudio: { + prettyLabel: "LM Studio", + docsUrl: "https://lmstudio.ai", + mainTask: "text-generation", + displayOnModelPage: (model) => isLlamaCppGgufModel(model) || isMlxModel(model), + deeplink: (model, filepath) => new URL(`lmstudio://open_from_hf?model=${model.id}${filepath ? `&file=${filepath}` : ""}`), + }, + localai: { + prettyLabel: "LocalAI", + docsUrl: "https://github.com/mudler/LocalAI", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + snippet: snippetLocalAI, + }, + jan: { + prettyLabel: "Jan", + docsUrl: "https://jan.ai", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`jan://models/huggingface/${model.id}`), + }, + "atomic-chat": { + prettyLabel: "Atomic Chat", + docsUrl: "https://atomic.chat", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`atomic-chat://models/huggingface/${model.id}`), + }, + backyard: { + prettyLabel: "Backyard AI", + docsUrl: "https://backyard.ai", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`https://backyard.ai/hf/model/${model.id}`), + }, + sanctum: { + prettyLabel: "Sanctum", + docsUrl: "https://sanctum.ai", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`sanctum://open_from_hf?model=${model.id}`), + }, + jellybox: { + prettyLabel: "Jellybox", + docsUrl: "https://jellybox.com", + mainTask: "text-generation", + displayOnModelPage: (model) => isLlamaCppGgufModel(model) || + (model.library_name === "diffusers" && + model.tags.includes("safetensors") && + (model.pipeline_tag === "text-to-image" || model.tags.includes("lora"))), + deeplink: (model) => { + if (isLlamaCppGgufModel(model)) { + return new URL(`jellybox://llm/models/huggingface/LLM/${model.id}`); + } + else if (model.tags.includes("lora")) { + return new URL(`jellybox://image/models/huggingface/ImageLora/${model.id}`); + } + else { + return new URL(`jellybox://image/models/huggingface/Image/${model.id}`); + } + }, + }, + msty: { + prettyLabel: "Msty", + docsUrl: "https://msty.app", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`msty://models/search/hf/${model.id}`), + }, + recursechat: { + prettyLabel: "RecurseChat", + docsUrl: "https://recurse.chat", + mainTask: "text-generation", + macOSOnly: true, + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`recursechat://new-hf-gguf-model?hf-model-id=${model.id}`), + }, + drawthings: { + prettyLabel: "Draw Things", + docsUrl: "https://drawthings.ai", + mainTask: "text-to-image", + macOSOnly: true, + displayOnModelPage: (model) => model.library_name === "diffusers" && (model.pipeline_tag === "text-to-image" || model.tags.includes("lora")), + deeplink: (model) => { + if (model.tags.includes("lora")) { + return new URL(`https://drawthings.ai/import/diffusers/pipeline.load_lora_weights?repo_id=${model.id}`); + } + else { + return new URL(`https://drawthings.ai/import/diffusers/pipeline.from_pretrained?repo_id=${model.id}`); + } + }, + }, + diffusionbee: { + prettyLabel: "DiffusionBee", + docsUrl: "https://diffusionbee.com", + mainTask: "text-to-image", + macOSOnly: true, + displayOnModelPage: (model) => model.library_name === "diffusers" && model.pipeline_tag === "text-to-image", + deeplink: (model) => new URL(`https://diffusionbee.com/huggingface_import?model_id=${model.id}`), + }, + joyfusion: { + prettyLabel: "JoyFusion", + docsUrl: "https://joyfusion.app", + mainTask: "text-to-image", + macOSOnly: true, + displayOnModelPage: (model) => model.tags.includes("coreml") && model.tags.includes("joyfusion") && model.pipeline_tag === "text-to-image", + deeplink: (model) => new URL(`https://joyfusion.app/import_from_hf?repo_id=${model.id}`), + }, + ollama: { + prettyLabel: "Ollama", + docsUrl: "https://ollama.com", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + snippet: snippetOllama, + }, + unsloth: { + prettyLabel: "Unsloth Studio", + docsUrl: "https://unsloth.ai/docs/new/studio", + mainTask: "text-generation", + displayOnModelPage: isUnslothModel, + snippet: snippetUnsloth, + }, + "docker-model-runner": { + prettyLabel: "Docker Model Runner", + docsUrl: "https://docs.docker.com/ai/model-runner/", + mainTask: "text-generation", + displayOnModelPage: isDockerModelRunnerModel, + snippet: snippetDockerModelRunner, + }, + lemonade: { + prettyLabel: "Lemonade", + docsUrl: "https://lemonade-server.ai", + mainTask: "text-generation", + displayOnModelPage: (model) => isLlamaCppGgufModel(model) || isAmdRyzenModel(model), + snippet: snippetLemonade, + }, + pi: { + prettyLabel: "Pi", + docsUrl: "https://github.com/badlogic/pi-mono", + mainTask: "text-generation", + displayOnModelPage: isToolCallingLocalAgentModel, + snippet: snippetPi, + }, + "hermes-agent": { + prettyLabel: "Hermes Agent", + docsUrl: "https://hermes-agent.nousresearch.com/", + mainTask: "text-generation", + displayOnModelPage: isToolCallingLocalAgentModel, + snippet: snippetHermesAgent, + }, + openclaw: { + prettyLabel: "OpenClaw", + docsUrl: "https://github.com/openclaw/openclaw", + mainTask: "text-generation", + displayOnModelPage: isToolCallingLocalAgentModel, + snippet: snippetOpenClaw, + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/local-apps.spec.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/local-apps.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b894adb772fca8fe09665082d1611092f3acc156 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/local-apps.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=local-apps.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/local-apps.spec.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/local-apps.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..dc8a5d8f78e2829b7e3a97d27fe96149050c6a7b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/local-apps.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"local-apps.spec.d.ts","sourceRoot":"","sources":["../../src/local-apps.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/local-apps.spec.js b/node_modules/@huggingface/tasks/dist/commonjs/local-apps.spec.js new file mode 100644 index 0000000000000000000000000000000000000000..c45dae4e30c465f89dc69663693ac2102b713c04 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/local-apps.spec.js @@ -0,0 +1,319 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const vitest_1 = require("vitest"); +const local_apps_js_1 = require("./local-apps.js"); +(0, vitest_1.describe)("local-apps", () => { + (0, vitest_1.it)("llama.cpp conversational", async () => { + const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["llama.cpp"]; + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + inference: "", + }; + const snippet = snippetFunc(model); + (0, vitest_1.expect)(snippet[0].content).toEqual([ + `# Start a local OpenAI-compatible server with a web UI: +llama serve -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`, + `# Run inference directly in the terminal: +llama cli -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`, + ]); + }); + (0, vitest_1.it)("llama.cpp non-conversational", async () => { + const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["llama.cpp"]; + const model = { + id: "mlabonne/gemma-2b-GGUF", + tags: [], + inference: "", + }; + const snippet = snippetFunc(model); + (0, vitest_1.expect)(snippet[0].content).toEqual([ + `# Start a local OpenAI-compatible server with a web UI: +llama serve -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}}`, + `# Run inference directly in the terminal: +llama cli -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}}`, + ]); + }); + (0, vitest_1.it)("vLLM conversational llm", async () => { + const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["vllm"]; + const model = { + id: "meta-llama/Llama-3.2-3B-Instruct", + pipeline_tag: "text-generation", + tags: ["conversational"], + inference: "", + }; + const snippet = snippetFunc(model); + (0, vitest_1.expect)(snippet[0].content.join("\n")).toEqual(`# Start the vLLM server: +vllm serve "meta-llama/Llama-3.2-3B-Instruct" +# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/chat/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "meta-llama/Llama-3.2-3B-Instruct", + "messages": [ + { + "role": "user", + "content": "What is the capital of France?" + } + ] + }'`); + }); + (0, vitest_1.it)("vLLM non-conversational llm", async () => { + const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["vllm"]; + const model = { + id: "meta-llama/Llama-3.2-3B", + tags: [""], + inference: "", + }; + const snippet = snippetFunc(model); + (0, vitest_1.expect)(snippet[0].content.join("\n")).toEqual(`# Start the vLLM server: +vllm serve "meta-llama/Llama-3.2-3B" +# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "meta-llama/Llama-3.2-3B", + "prompt": "Once upon a time,", + "max_tokens": 512, + "temperature": 0.5 + }'`); + }); + (0, vitest_1.it)("vLLM conversational vlm", async () => { + const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["vllm"]; + const model = { + id: "meta-llama/Llama-3.2-11B-Vision-Instruct", + pipeline_tag: "image-text-to-text", + tags: ["conversational"], + inference: "", + }; + const snippet = snippetFunc(model); + (0, vitest_1.expect)(snippet[0].content.join("\n")).toEqual(`# Start the vLLM server: +vllm serve "meta-llama/Llama-3.2-11B-Vision-Instruct" +# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/chat/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "meta-llama/Llama-3.2-11B-Vision-Instruct", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe this image in one sentence." + }, + { + "type": "image_url", + "image_url": { + "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" + } + } + ] + } + ] + }'`); + }); + (0, vitest_1.it)("pi", async () => { + const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["pi"]; + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096, chat_template: "{% if tools %}" }, + inference: "", + }; + const snippet = snippetFunc(model); + (0, vitest_1.expect)(snippet[0].content).toContain(`llama serve -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); + (0, vitest_1.expect)(snippet[1].setup).toContain("npm install -g @mariozechner/pi-coding-agent"); + (0, vitest_1.expect)(snippet[1].content).toContain(`"id": "bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}"`); + (0, vitest_1.expect)(snippet[2].content).toContain("pi"); + }); + (0, vitest_1.it)("pi - mlx", async () => { + const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["pi"]; + const model = { + id: "mlx-community/Llama-3.2-3B-Instruct-mlx", + tags: ["mlx", "conversational"], + pipeline_tag: "text-generation", + config: { + tokenizer_config: { + chat_template: "{% if tools %}...{% endif %}", + }, + }, + inference: "", + }; + const snippet = snippetFunc(model); + (0, vitest_1.expect)(snippet[0].setup).toContain("uv tool install mlx-lm"); + (0, vitest_1.expect)(snippet[0].content).toContain('mlx_lm.server --model "mlx-community/Llama-3.2-3B-Instruct-mlx"'); + (0, vitest_1.expect)(snippet[1].setup).toContain("npm install -g @mariozechner/pi-coding-agent"); + (0, vitest_1.expect)(snippet[1].content).toContain('"baseUrl": "http://localhost:8080/v1"'); + (0, vitest_1.expect)(snippet[1].content).toContain('"id": "mlx-community/Llama-3.2-3B-Instruct-mlx"'); + (0, vitest_1.expect)(snippet[2].content).toContain("pi"); + }); + (0, vitest_1.it)("hermes-agent", async () => { + const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["hermes-agent"]; + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096, chat_template: "{% if tools %}" }, + inference: "", + }; + const snippet = snippetFunc(model); + (0, vitest_1.expect)(snippet[0].content).toContain(`llama serve -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); + (0, vitest_1.expect)(snippet[1].content).toContain("hermes config set model.provider custom"); + (0, vitest_1.expect)(snippet[1].content).toContain("hermes config set model.base_url http://127.0.0.1:8080/v1"); + (0, vitest_1.expect)(snippet[1].content).toContain("hermes config set model.default bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}"); + (0, vitest_1.expect)(snippet[2].content).toContain("hermes"); + }); + (0, vitest_1.it)("hermes-agent - mlx", async () => { + const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["hermes-agent"]; + const model = { + id: "mlx-community/Llama-3.2-3B-Instruct-mlx", + tags: ["mlx", "conversational"], + pipeline_tag: "text-generation", + config: { + tokenizer_config: { + chat_template: "{% if tools %}...{% endif %}", + }, + }, + inference: "", + }; + const snippet = snippetFunc(model); + (0, vitest_1.expect)(snippet[0].setup).toContain("uv tool install mlx-lm"); + (0, vitest_1.expect)(snippet[1].content).toContain("hermes config set model.provider custom"); + (0, vitest_1.expect)(snippet[1].content).toContain("hermes config set model.default mlx-community/Llama-3.2-3B-Instruct-mlx"); + (0, vitest_1.expect)(snippet[2].content).toContain("hermes"); + }); + (0, vitest_1.it)("openclaw", async () => { + const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS.openclaw; + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096, chat_template: "{% if tools %}" }, + inference: "", + }; + const snippet = snippetFunc(model); + (0, vitest_1.expect)(snippet[0].content).toContain(`llama-server -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); + (0, vitest_1.expect)(snippet[1].setup).toContain("npm install -g openclaw@latest"); + (0, vitest_1.expect)(snippet[1].content).toContain("openclaw onboard --non-interactive --mode local"); + (0, vitest_1.expect)(snippet[1].content).toContain("--auth-choice custom-api-key"); + (0, vitest_1.expect)(snippet[1].content).toContain("--custom-base-url http://127.0.0.1:8080/v1"); + (0, vitest_1.expect)(snippet[1].content).toContain('--custom-model-id "bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}"'); + (0, vitest_1.expect)(snippet[1].content).toContain("--custom-provider-id llama-cpp"); + (0, vitest_1.expect)(snippet[1].content).toContain("--custom-compatibility openai"); + (0, vitest_1.expect)(snippet[1].content).not.toContain("--custom-api-key"); + (0, vitest_1.expect)(snippet[1].content).toContain("--custom-text-input"); + (0, vitest_1.expect)(snippet[1].content).toContain("--accept-risk"); + (0, vitest_1.expect)(snippet[1].content).toContain("--skip-health"); + (0, vitest_1.expect)(snippet[2].content).toContain('openclaw agent --local --agent main --message "Hello from Hugging Face"'); + }); + (0, vitest_1.it)("openclaw - mlx", async () => { + const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS.openclaw; + const model = { + id: "mlx-community/Llama-3.2-3B-Instruct-mlx", + tags: ["mlx", "conversational"], + pipeline_tag: "text-generation", + config: { + tokenizer_config: { + chat_template: "{% if tools %}...{% endif %}", + }, + }, + inference: "", + }; + const snippet = snippetFunc(model); + (0, vitest_1.expect)(snippet[0].setup).toContain("uv tool install mlx-lm"); + (0, vitest_1.expect)(snippet[1].content).toContain("openclaw onboard --non-interactive --mode local"); + (0, vitest_1.expect)(snippet[1].content).toContain('--custom-model-id "mlx-community/Llama-3.2-3B-Instruct-mlx"'); + (0, vitest_1.expect)(snippet[1].content).toContain("--custom-provider-id mlx-lm"); + (0, vitest_1.expect)(snippet[1].content).toContain("--custom-text-input"); + (0, vitest_1.expect)(snippet[2].content).toContain('openclaw agent --local --agent main --message "Hello from Hugging Face"'); + }); + (0, vitest_1.it)("docker model runner", async () => { + const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["docker-model-runner"]; + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096 }, + inference: "", + }; + const snippet = snippetFunc(model); + (0, vitest_1.expect)(snippet).toEqual(`docker model run hf.co/bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); + }); + (0, vitest_1.it)("atomic chat deeplink", async () => { + const { displayOnModelPage, deeplink } = local_apps_js_1.LOCAL_APPS["atomic-chat"]; + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096 }, + inference: "", + }; + (0, vitest_1.expect)(displayOnModelPage(model)).toBe(true); + (0, vitest_1.expect)(deeplink(model).href).toBe("atomic-chat://models/huggingface/bartowski/Llama-3.2-3B-Instruct-GGUF"); + }); + (0, vitest_1.it)("unsloth tagged model", async () => { + const { displayOnModelPage, snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS.unsloth; + const model = { + id: "some-user/my-unsloth-finetune", + tags: ["unsloth", "conversational"], + inference: "", + }; + (0, vitest_1.expect)(displayOnModelPage(model)).toBe(true); + const snippet = snippetFunc(model); + (0, vitest_1.expect)(snippet[0].setup).toBe("curl -fsSL https://unsloth.ai/install.sh | sh"); + (0, vitest_1.expect)(snippet[0].content).toBe("# Run unsloth studio\nunsloth studio -H 0.0.0.0 -p 8888\n# Then open http://localhost:8888 in your browser\n# Search for some-user/my-unsloth-finetune to start chatting"); + (0, vitest_1.expect)(snippet[1].setup).toBe("irm https://unsloth.ai/install.ps1 | iex"); + (0, vitest_1.expect)(snippet[1].content).toBe(snippet[0].content); + (0, vitest_1.expect)(snippet[2].setup).toBe("# No setup required"); + (0, vitest_1.expect)(snippet[2].content).toBe("# Open https://huggingface.co/spaces/unsloth/studio in your browser\n# Search for some-user/my-unsloth-finetune to start chatting"); + (0, vitest_1.expect)(snippet[3].setup).toBe("pip install unsloth"); + (0, vitest_1.expect)(snippet[3].content).toBe('from unsloth import FastModel\nmodel, tokenizer = FastModel.from_pretrained(\n model_name="some-user/my-unsloth-finetune",\n max_seq_length=2048,\n)'); + }); + (0, vitest_1.it)("unsloth namespace gguf model", async () => { + const { displayOnModelPage, snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS.unsloth; + const model = { + id: "unsloth/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096 }, + inference: "", + }; + (0, vitest_1.expect)(displayOnModelPage(model)).toBe(true); + const snippet = snippetFunc(model); + (0, vitest_1.expect)(snippet[0].setup).toBe("curl -fsSL https://unsloth.ai/install.sh | sh"); + (0, vitest_1.expect)(snippet[0].content).toBe("# Run unsloth studio\nunsloth studio -H 0.0.0.0 -p 8888\n# Then open http://localhost:8888 in your browser\n# Search for unsloth/Llama-3.2-3B-Instruct-GGUF to start chatting"); + (0, vitest_1.expect)(snippet[1].setup).toBe("irm https://unsloth.ai/install.ps1 | iex"); + (0, vitest_1.expect)(snippet[1].content).toBe(snippet[0].content); + (0, vitest_1.expect)(snippet[2].setup).toBe("# No setup required"); + (0, vitest_1.expect)(snippet[2].content).toBe("# Open https://huggingface.co/spaces/unsloth/studio in your browser\n# Search for unsloth/Llama-3.2-3B-Instruct-GGUF to start chatting"); + (0, vitest_1.expect)(snippet).toHaveLength(3); // GGUF models only get 3 snippets + }); + (0, vitest_1.it)("non unsloth namespace gguf model", async () => { + const { displayOnModelPage } = local_apps_js_1.LOCAL_APPS.unsloth; + const model = { + id: "dummy/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096 }, + inference: "", + }; + (0, vitest_1.expect)(displayOnModelPage(model)).toBe(true); + }); + (0, vitest_1.it)("unsloth not shown for unrelated model", async () => { + const { displayOnModelPage } = local_apps_js_1.LOCAL_APPS.unsloth; + const model = { + id: "meta-llama/Llama-3.2-3B-Instruct", + tags: ["conversational"], + inference: "", + }; + (0, vitest_1.expect)(displayOnModelPage(model)).toBe(false); + }); + (0, vitest_1.it)("links as a function", async () => { + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + inference: "", + }; + const appWithFnLinks = { + ...local_apps_js_1.LOCAL_APPS["llama.cpp"], + links: (m) => [{ label: "Releases", url: `https://github.com/${m.id}/releases` }], + }; + (0, vitest_1.expect)(appWithFnLinks.links(model)).toEqual([ + { label: "Releases", url: "https://github.com/bartowski/Llama-3.2-3B-Instruct-GGUF/releases" }, + ]); + }); +}); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/model-data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fdb7d166b5308b6102fb8bceef83b41360f1873b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-data.d.ts @@ -0,0 +1,154 @@ +import type { PipelineType } from "./pipelines.js"; +import type { WidgetExample } from "./widget-example.js"; +import type { TokenizerConfig } from "./tokenizer-data.js"; +/** + * Public interface for model metadata + */ +export interface ModelData { + /** + * id of model (e.g. 'user/repo_name') + */ + id: string; + /** + * Whether or not to enable inference widget for this model + * TODO(type it) + */ + inference: string; + /** + * is this model private? + */ + private?: boolean; + /** + * this dictionary has useful information about the model configuration + */ + config?: { + architectures?: string[]; + /** + * Dict of AutoModel or Auto… class name to local import path in the repo + */ + auto_map?: { + /** + * String Property + */ + [x: string]: string; + }; + model_type?: string; + quantization_config?: { + bits?: number; + load_in_4bit?: boolean; + load_in_8bit?: boolean; + /** + * awq, gptq, aqlm, marlin, … Used by vLLM + */ + quant_method?: string; + }; + tokenizer_config?: TokenizerConfig; + processor_config?: { + chat_template?: string; + }; + chat_template_jinja?: string; + adapter_transformers?: { + model_name?: string; + model_class?: string; + }; + diffusers?: { + _class_name?: string; + }; + sklearn?: { + model?: { + file?: string; + }; + model_format?: string; + }; + speechbrain?: { + speechbrain_interface?: string; + vocoder_interface?: string; + vocoder_model_id?: string; + }; + peft?: { + base_model_name_or_path?: string; + task_type?: string; + }; + keras_hub?: { + tasks?: string[]; + }; + }; + /** + * all the model tags + */ + tags: string[]; + /** + * transformers-specific info to display in the code sample. + */ + transformersInfo?: TransformersInfo; + /** + * Pipeline type + */ + pipeline_tag?: PipelineType | undefined; + /** + * for relevant models, get mask token + */ + mask_token?: string | undefined; + /** + * Example data that will be fed into the widget. + * + * can be set in the model card metadata (under `widget`), + * or by default in `DefaultWidget.ts` + */ + widgetData?: WidgetExample[] | undefined; + /** + * Parameters that will be used by the widget when calling Inference API (serverless) + * https://huggingface.co/docs/api-inference/detailed_parameters + * + * can be set in the model card metadata (under `inference/parameters`) + * Example: + * inference: + * parameters: + * key: val + */ + cardData?: { + inference?: boolean | { + parameters?: Record; + }; + base_model?: string | string[]; + instance_prompt?: string | null; + }; + /** + * Library name + * Example: transformers, SpeechBrain, Stanza, etc. + */ + library_name?: string; + safetensors?: { + parameters: Record; + total: number; + sharded: boolean; + }; + gguf?: { + total: number; + architecture?: string; + context_length?: number; + chat_template?: string; + }; +} +/** + * transformers-specific info to display in the code sample. + */ +export interface TransformersInfo { + /** + * e.g. AutoModelForSequenceClassification + */ + auto_model: string; + /** + * if set in config.json's auto_map + */ + custom_class?: string; + /** + * e.g. text-classification + */ + pipeline_tag?: PipelineType; + /** + * e.g. "AutoTokenizer" | "AutoFeatureExtractor" | "AutoProcessor" + */ + processor?: string; +} +//# sourceMappingURL=model-data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/model-data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..ad4e813ed88e562a7293a5858d4485de7e35d6b6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"model-data.d.ts","sourceRoot":"","sources":["../../src/model-data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAE3D;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IACX;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE;QACR,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;QACzB;;WAEG;QACH,QAAQ,CAAC,EAAE;YACV;;eAEG;YACH,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;SACpB,CAAC;QACF,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,mBAAmB,CAAC,EAAE;YACrB,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB;;eAEG;YACH,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,gBAAgB,CAAC,EAAE,eAAe,CAAC;QACnC,gBAAgB,CAAC,EAAE;YAClB,aAAa,CAAC,EAAE,MAAM,CAAC;SACvB,CAAC;QACF,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,oBAAoB,CAAC,EAAE;YACtB,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,SAAS,CAAC,EAAE;YACX,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,OAAO,CAAC,EAAE;YACT,KAAK,CAAC,EAAE;gBACP,IAAI,CAAC,EAAE,MAAM,CAAC;aACd,CAAC;YACF,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,WAAW,CAAC,EAAE;YACb,qBAAqB,CAAC,EAAE,MAAM,CAAC;YAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;YAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;SAC1B,CAAC;QACF,IAAI,CAAC,EAAE;YACN,uBAAuB,CAAC,EAAE,MAAM,CAAC;YACjC,SAAS,CAAC,EAAE,MAAM,CAAC;SACnB,CAAC;QACF,SAAS,CAAC,EAAE;YACX,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC;KACF,CAAC;IACF;;OAEG;IACH,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;;OAEG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACxC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC;;;;;OAKG;IACH,UAAU,CAAC,EAAE,aAAa,EAAE,GAAG,SAAS,CAAC;IACzC;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE;QACV,SAAS,CAAC,EACP,OAAO,GACP;YACA,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACpC,CAAC;QACL,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;QAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAChC,CAAC;IACF;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE;QACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,OAAO,CAAC;KACjB,CAAC;IACF,IAAI,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,aAAa,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-data.js b/node_modules/@huggingface/tasks/dist/commonjs/model-data.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-data.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-downloads.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-downloads.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2068624044fa42a89cd7889633465fa802c9ac71 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-downloads.d.ts @@ -0,0 +1,18 @@ +/** + * This file contains the (simplified) types used + * to represent queries that are made to Elastic + * in order to count number of model downloads + * + * Read this doc about download stats on the Hub: + * + * https://huggingface.co/docs/hub/models-download-stats + * Available fields: + * - path: the complete file path (relative) (e.g: "prefix/file.extension") + * - path_prefix: the prefix of the file path (e.g: "prefix/", empty if no prefix) + * - path_extension: the extension of the file path (e.g: "extension") + * - path_filename: the name of the file path (e.g: "file") + * see also: + * https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html + */ +export type ElasticSearchQuery = string; +//# sourceMappingURL=model-libraries-downloads.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-downloads.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-downloads.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f0a63c2eda1c1650a18fb8060e9cf6fb53c7c272 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-downloads.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"model-libraries-downloads.d.ts","sourceRoot":"","sources":["../../src/model-libraries-downloads.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-downloads.js b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-downloads.js new file mode 100644 index 0000000000000000000000000000000000000000..4302a1ec670e4500a6a71f2dd5dd7f14582d40ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-downloads.js @@ -0,0 +1,18 @@ +"use strict"; +/** + * This file contains the (simplified) types used + * to represent queries that are made to Elastic + * in order to count number of model downloads + * + * Read this doc about download stats on the Hub: + * + * https://huggingface.co/docs/hub/models-download-stats + * Available fields: + * - path: the complete file path (relative) (e.g: "prefix/file.extension") + * - path_prefix: the prefix of the file path (e.g: "prefix/", empty if no prefix) + * - path_extension: the extension of the file path (e.g: "extension") + * - path_filename: the name of the file path (e.g: "file") + * see also: + * https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html + */ +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..cb0d9d16a71eef68e697bbdbe0065d35ac434423 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.d.ts @@ -0,0 +1,116 @@ +import type { ModelData } from "./model-data.js"; +export declare const adapters: (model: ModelData) => string[]; +export declare const allennlp: (model: ModelData) => string[]; +export declare const araclip: (model: ModelData) => string[]; +export declare const asteroid: (model: ModelData) => string[]; +export declare const audioseal: (model: ModelData) => string[]; +export declare const ben2: (model: ModelData) => string[]; +export declare const bertopic: (model: ModelData) => string[]; +export declare const bm25s: (model: ModelData) => string[]; +export declare const chatterbox: () => string[]; +export declare const chronos_forecasting: (model: ModelData) => string[]; +export declare const collectorvision: (model: ModelData) => string[]; +export declare const colipri: (model: ModelData) => string[]; +export declare const sap_rpt_one_oss: () => string[]; +export declare const cxr_foundation: () => string[]; +export declare const depth_anything_v2: (model: ModelData) => string[]; +export declare const depth_pro: (model: ModelData) => string[]; +export declare const derm_foundation: () => string[]; +export declare const dia: (model: ModelData) => string[]; +export declare const dia2: (model: ModelData) => string[]; +export declare const describe_anything: (model: ModelData) => string[]; +export declare const diffusers: (model: ModelData) => string[]; +export declare const diffusionkit: (model: ModelData) => string[]; +export declare const cartesia_pytorch: (model: ModelData) => string[]; +export declare const cartesia_mlx: (model: ModelData) => string[]; +export declare const edsnlp: (model: ModelData) => string[]; +export declare const espnetTTS: (model: ModelData) => string[]; +export declare const espnetASR: (model: ModelData) => string[]; +export declare const espnet: (model: ModelData) => string[]; +export declare const fairseq: (model: ModelData) => string[]; +export declare const flair: (model: ModelData) => string[]; +export declare const gliner: (model: ModelData) => string[]; +export declare const gliner2: (model: ModelData) => string[]; +export declare const indextts: (model: ModelData) => string[]; +export declare const htrflow: (model: ModelData) => string[]; +export declare const keras: (model: ModelData) => string[]; +export declare const keras_hub: (model: ModelData) => string[]; +export declare const kernels: (model: ModelData) => string[]; +export declare const kimi_audio: (model: ModelData) => string[]; +export declare const kittentts: (model: ModelData) => string[]; +export declare const lightning_ir: (model: ModelData) => string[]; +export declare const llama_cpp_python: (model: ModelData) => string[]; +export declare const lerobot: (model: ModelData) => string[]; +export declare const litert_lm: (model: ModelData) => string[]; +export declare const tf_keras: (model: ModelData) => string[]; +export declare const mamba_ssm: (model: ModelData) => string[]; +export declare const mars5_tts: (model: ModelData) => string[]; +export declare const matanyone: (model: ModelData) => string[]; +export declare const mesh_anything: () => string[]; +export declare const multimolecule: (model: ModelData) => string[]; +export declare const open_clip: (model: ModelData) => string[]; +export declare const paddlenlp: (model: ModelData) => string[]; +export declare const paddleocr: (model: ModelData) => string[]; +export declare const perception_encoder: (model: ModelData) => string[]; +export declare const phantom_wan: (model: ModelData) => string[]; +export declare const pocket_tts: (model: ModelData) => string[]; +export declare const pyannote_audio_pipeline: (model: ModelData) => string[]; +export declare const pyannote_audio: (model: ModelData) => string[]; +export declare const relik: (model: ModelData) => string[]; +export declare const renderformer: (model: ModelData) => string[]; +export declare const tensorflowtts: (model: ModelData) => string[]; +export declare const timm: (model: ModelData) => string[]; +export declare const saelens: () => string[]; +export declare const seed_story: () => string[]; +export declare const sklearn: (model: ModelData) => string[]; +export declare const stable_audio_tools: (model: ModelData) => string[]; +export declare const fastai: (model: ModelData) => string[]; +export declare const sam2: (model: ModelData) => string[]; +export declare const sam_3d_objects: (model: ModelData) => string[]; +export declare const sam_3d_body: (model: ModelData) => string[]; +export declare const sampleFactory: (model: ModelData) => string[]; +export declare const sentenceTransformers: (model: ModelData) => string[]; +export declare const setfit: (model: ModelData) => string[]; +export declare const spacy: (model: ModelData) => string[]; +export declare const span_marker: (model: ModelData) => string[]; +export declare const stanza: (model: ModelData) => string[]; +export declare const speechbrain: (model: ModelData) => string[]; +export declare const terratorch: (model: ModelData) => string[]; +export declare const transformers: (model: ModelData) => string[]; +export declare const transformersJS: (model: ModelData) => string[]; +export declare const peft: (model: ModelData) => string[]; +export declare const fasttext: (model: ModelData) => string[]; +export declare const stableBaselines3: (model: ModelData) => string[]; +export declare const mlAgents: (model: ModelData) => string[]; +export declare const sentis: () => string[]; +export declare const sana: (model: ModelData) => string[]; +export declare const vibevoice: (model: ModelData) => string[]; +export declare const videoprism: (model: ModelData) => string[]; +export declare const vfimamba: (model: ModelData) => string[]; +export declare const lvface: (model: ModelData) => string[]; +export declare const voicecraft: (model: ModelData) => string[]; +export declare const voxcpm: (model: ModelData) => string[]; +export declare const vui: () => string[]; +export declare const chattts: () => string[]; +export declare const ultralytics: (model: ModelData) => string[]; +export declare const birefnet: (model: ModelData) => string[]; +export declare const supertonic: () => string[]; +export declare const swarmformer: (model: ModelData) => string[]; +export declare const univa: (model: ModelData) => string[]; +export declare const mlxim: (model: ModelData) => string[]; +export declare const mlx: (model: ModelData) => string[]; +export declare const model2vec: (model: ModelData) => string[]; +export declare const pruna: (model: ModelData) => string[]; +export declare const nemo: (model: ModelData) => string[]; +export declare const outetts: (model: ModelData) => string[]; +export declare const pxia: (model: ModelData) => string[]; +export declare const pythae: (model: ModelData) => string[]; +export declare const qwen3_tts: (model: ModelData) => string[]; +export declare const anemoi: (model: ModelData) => string[]; +export declare const audiocraft: (model: ModelData) => string[]; +export declare const whisperkit: () => string[]; +export declare const threedtopia_xl: (model: ModelData) => string[]; +export declare const hezar: (model: ModelData) => string[]; +export declare const zonos: (model: ModelData) => string[]; +export declare const moshi: (model: ModelData) => string[]; +//# sourceMappingURL=model-libraries-snippets.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..302eca3baf2bca8f5602ca5999586b5eaa1965b4 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"model-libraries-snippets.d.ts","sourceRoot":"","sources":["../../src/model-libraries-snippets.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAkBjD,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAKjD,CAAC;AAkBF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAKjD,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAIhD,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAIjD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAkBlD,CAAC;AAaF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EAY7C,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAIjD,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAI9C,CAAC;AAEF,eAAO,MAAM,UAAU,QAAO,MAAM,EAenC,CAAC;AAEF,eAAO,MAAM,mBAAmB,GAAI,OAAO,SAAS,KAAG,MAAM,EAsB5D,CAAC;AAEF,eAAO,MAAM,eAAe,GAAI,OAAO,SAAS,KAAG,MAAM,EAYxD,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAkBhD,CAAC;AAEF,eAAO,MAAM,eAAe,QAAO,MAAM,EAmDxC,CAAC;AAEF,eAAO,MAAM,cAAc,QAAO,MAAM,EAcvC,CAAC;AAEF,eAAO,MAAM,iBAAiB,GAAI,OAAO,SAAS,KAAG,MAAM,EA6C1D,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAwBlD,CAAC;AAEF,eAAO,MAAM,eAAe,QAAO,MAAM,EAoBxC,CAAC;AAEF,eAAO,MAAM,GAAG,GAAI,OAAO,SAAS,KAAG,MAAM,EAS5C,CAAC;AAEF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EAW7C,CAAC;AAEF,eAAO,MAAM,iBAAiB,GAAI,OAAO,SAAS,KAAG,MAAM,EAY1D,CAAC;AAqLF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAgClD,CAAC;AAEF,eAAO,MAAM,YAAY,GAAI,OAAO,SAAS,KAAG,MAAM,EAwCrD,CAAC;AAEF,eAAO,MAAM,gBAAgB,GAAI,OAAO,SAAS,KAAG,MAAM,EAgBzD,CAAC;AAEF,eAAO,MAAM,YAAY,GAAI,OAAO,SAAS,KAAG,MAAM,EAmBrD,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAgB/C,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAMlD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EASlD,CAAC;AAIF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAO/C,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAMhD,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAI9C,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAI/C,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAUhD,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAgBjD,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAehD,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAS9C,CAAC;AA4EF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAsBlD,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAMhD,CAAC;AAEF,eAAO,MAAM,UAAU,GAAI,OAAO,SAAS,KAAG,MAAM,EA+BnD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EASlD,CAAC;AAEF,eAAO,MAAM,YAAY,GAAI,OAAO,SAAS,KAAG,MAAM,EA+BrD,CAAC;AAEF,eAAO,MAAM,gBAAgB,GAAI,OAAO,SAAS,KAAG,MAAM,EA4BzD,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAuChD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAgBlD,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAOjD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAIlD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAKlD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAQlD,CAAC;AAEF,eAAO,MAAM,aAAa,QAAO,MAAM,EAQtC,CAAC;AAEF,eAAO,MAAM,aAAa,GAAI,OAAO,SAAS,KAAG,MAAM,EAgDtD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAKlD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAsBlD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAuElD,CAAC;AAEF,eAAO,MAAM,kBAAkB,GAAI,OAAO,SAAS,KAAG,MAAM,EAgB3D,CAAC;AACF,eAAO,MAAM,WAAW,GAAI,OAAO,SAAS,KAAG,MAAM,EAUpD,CAAC;AAEF,eAAO,MAAM,UAAU,GAAI,OAAO,SAAS,KAAG,MAAM,EAWnD,CAAC;AAEF,eAAO,MAAM,uBAAuB,GAAI,OAAO,SAAS,KAAG,MAAM,EAehE,CAAC;AAiBF,eAAO,MAAM,cAAc,GAAI,OAAO,SAAS,KAAG,MAAM,EAKvD,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAI9C,CAAC;AAEF,eAAO,MAAM,YAAY,GAAI,OAAO,SAAS,KAAG,MAAM,EAKrD,CAAC;AAyBF,eAAO,MAAM,aAAa,GAAI,OAAO,SAAS,KAAG,MAAM,EAOtD,CAAC;AAEF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EAI7C,CAAC;AAEF,eAAO,MAAM,OAAO,QAA6B,MAAM,EAQtD,CAAC;AAEF,eAAO,MAAM,UAAU,QAAO,MAAM,EAanC,CAAC;AAsCF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAehD,CAAC;AAEF,eAAO,MAAM,kBAAkB,GAAI,OAAO,SAAS,KAAG,MAAM,EAmC3D,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAI/C,CAAC;AAEF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EA2B7C,CAAC;AAEF,eAAO,MAAM,cAAc,GAAI,OAAO,SAAS,KAAG,MAAM,EAWvD,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO,SAAS,KAAG,MAAM,EAMpD,CAAC;AAEF,eAAO,MAAM,aAAa,GAAI,OAAO,SAAS,KAAG,MAAM,EAEtD,CAAC;AASF,eAAO,MAAM,oBAAoB,GAAI,OAAO,SAAS,KAAG,MAAM,EA6D7D,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAI/C,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAU9C,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO,SAAS,KAAG,MAAM,EAIpD,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAK/C,CAAC;AAkBF,eAAO,MAAM,WAAW,GAAI,OAAO,SAAS,KAAG,MAAM,EAkBpD,CAAC;AAEF,eAAO,MAAM,UAAU,GAAI,OAAO,SAAS,KAAG,MAAM,EAInD,CAAC;AAOF,eAAO,MAAM,YAAY,GAAI,OAAO,SAAS,KAAG,MAAM,EAiHrD,CAAC;AAEF,eAAO,MAAM,cAAc,GAAI,OAAO,SAAS,KAAG,MAAM,EAcvD,CAAC;AAiBF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EAiB7C,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAKjD,CAAC;AAEF,eAAO,MAAM,gBAAgB,GAAI,OAAO,SAAS,KAAG,MAAM,EAMzD,CAAC;AAgBF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAEjD,CAAC;AAEF,eAAO,MAAM,MAAM,QAA6B,MAAM,EAMrD,CAAC;AAEF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EAkB7C,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAqBlD,CAAC;AAEF,eAAO,MAAM,UAAU,GAAI,OAAO,SAAS,KAAG,MAAM,EAWnD,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAIjD,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAQ/C,CAAC;AAEF,eAAO,MAAM,UAAU,GAAI,OAAO,SAAS,KAAG,MAAM,EAInD,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAqB/C,CAAC;AAEF,eAAO,MAAM,GAAG,QAAO,MAAM,EAgB5B,CAAC;AAEF,eAAO,MAAM,OAAO,QAAO,MAAM,EAYhC,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO,SAAS,KAAG,MAAM,EAiBpD,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAYjD,CAAC;AAEF,eAAO,MAAM,UAAU,QAAO,MAAM,EAWnC,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO,SAAS,KAAG,MAAM,EAKpD,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAW9C,CAAC;AAmEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAI9C,CAAC;AAEF,eAAO,MAAM,GAAG,GAAI,OAAO,SAAS,KAAG,MAAM,EAY5C,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAIlD,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EA0B9C,CAAC;AAuDF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EAQ7C,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAyBhD,CAAC;AAEF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EAI7C,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAI/C,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAqBlD,CAAC;AA4BF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAO/C,CAAC;AAEF,eAAO,MAAM,UAAU,GAAI,OAAO,SAAS,KAAG,MAAM,EAUnD,CAAC;AAEF,eAAO,MAAM,UAAU,QAAO,MAAM,EAYnC,CAAC;AAEF,eAAO,MAAM,cAAc,GAAI,OAAO,SAAS,KAAG,MAAM,EAKvD,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAI9C,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAiB9C,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAgD9C,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.js b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.js new file mode 100644 index 0000000000000000000000000000000000000000..ee10f9a2bc87eb1a2cdfa7040fe0147e938a50ee --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.js @@ -0,0 +1,2384 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.open_clip = exports.multimolecule = exports.mesh_anything = exports.matanyone = exports.mars5_tts = exports.mamba_ssm = exports.tf_keras = exports.litert_lm = exports.lerobot = exports.llama_cpp_python = exports.lightning_ir = exports.kittentts = exports.kimi_audio = exports.kernels = exports.keras_hub = exports.keras = exports.htrflow = exports.indextts = exports.gliner2 = exports.gliner = exports.flair = exports.fairseq = exports.espnet = exports.espnetASR = exports.espnetTTS = exports.edsnlp = exports.cartesia_mlx = exports.cartesia_pytorch = exports.diffusionkit = exports.diffusers = exports.describe_anything = exports.dia2 = exports.dia = exports.derm_foundation = exports.depth_pro = exports.depth_anything_v2 = exports.cxr_foundation = exports.sap_rpt_one_oss = exports.colipri = exports.collectorvision = exports.chronos_forecasting = exports.chatterbox = exports.bm25s = exports.bertopic = exports.ben2 = exports.audioseal = exports.asteroid = exports.araclip = exports.allennlp = exports.adapters = void 0; +exports.mlx = exports.mlxim = exports.univa = exports.swarmformer = exports.supertonic = exports.birefnet = exports.ultralytics = exports.chattts = exports.vui = exports.voxcpm = exports.voicecraft = exports.lvface = exports.vfimamba = exports.videoprism = exports.vibevoice = exports.sana = exports.sentis = exports.mlAgents = exports.stableBaselines3 = exports.fasttext = exports.peft = exports.transformersJS = exports.transformers = exports.terratorch = exports.speechbrain = exports.stanza = exports.span_marker = exports.spacy = exports.setfit = exports.sentenceTransformers = exports.sampleFactory = exports.sam_3d_body = exports.sam_3d_objects = exports.sam2 = exports.fastai = exports.stable_audio_tools = exports.sklearn = exports.seed_story = exports.saelens = exports.timm = exports.tensorflowtts = exports.renderformer = exports.relik = exports.pyannote_audio = exports.pyannote_audio_pipeline = exports.pocket_tts = exports.phantom_wan = exports.perception_encoder = exports.paddleocr = exports.paddlenlp = void 0; +exports.moshi = exports.zonos = exports.hezar = exports.threedtopia_xl = exports.whisperkit = exports.audiocraft = exports.anemoi = exports.qwen3_tts = exports.pythae = exports.pxia = exports.outetts = exports.nemo = exports.pruna = exports.model2vec = void 0; +const library_to_tasks_js_1 = require("./library-to-tasks.js"); +const inputs_js_1 = require("./snippets/inputs.js"); +const common_js_1 = require("./snippets/common.js"); +const TAG_CUSTOM_CODE = "custom_code"; +function nameWithoutNamespace(modelId) { + const splitted = modelId.split("/"); + return splitted.length === 1 ? splitted[0] : splitted[1]; +} +const escapeStringForJson = (str) => JSON.stringify(str).slice(1, -1); // slice is needed to remove surrounding quotes added by JSON.stringify +//#region snippets +const adapters = (model) => [ + `from adapters import AutoAdapterModel + +model = AutoAdapterModel.from_pretrained("${model.config?.adapter_transformers?.model_name}") +model.load_adapter("${model.id}", set_active=True)`, +]; +exports.adapters = adapters; +const allennlpUnknown = (model) => [ + `import allennlp_models +from allennlp.predictors.predictor import Predictor + +predictor = Predictor.from_path("hf://${model.id}")`, +]; +const allennlpQuestionAnswering = (model) => [ + `import allennlp_models +from allennlp.predictors.predictor import Predictor + +predictor = Predictor.from_path("hf://${model.id}") +predictor_input = {"passage": "My name is Wolfgang and I live in Berlin", "question": "Where do I live?"} +predictions = predictor.predict_json(predictor_input)`, +]; +const allennlp = (model) => { + if (model.tags.includes("question-answering")) { + return allennlpQuestionAnswering(model); + } + return allennlpUnknown(model); +}; +exports.allennlp = allennlp; +const araclip = (model) => [ + `from araclip import AraClip + +model = AraClip.from_pretrained("${model.id}")`, +]; +exports.araclip = araclip; +const asteroid = (model) => [ + `from asteroid.models import BaseModel + +model = BaseModel.from_pretrained("${model.id}")`, +]; +exports.asteroid = asteroid; +const audioseal = (model) => { + const watermarkSnippet = `# Watermark Generator +from audioseal import AudioSeal + +model = AudioSeal.load_generator("${model.id}") +# pass a tensor (tensor_wav) of shape (batch, channels, samples) and a sample rate +wav, sr = tensor_wav, 16000 + +watermark = model.get_watermark(wav, sr) +watermarked_audio = wav + watermark`; + const detectorSnippet = `# Watermark Detector +from audioseal import AudioSeal + +detector = AudioSeal.load_detector("${model.id}") + +result, message = detector.detect_watermark(watermarked_audio, sr)`; + return [watermarkSnippet, detectorSnippet]; +}; +exports.audioseal = audioseal; +function get_base_diffusers_model(model) { + return model.cardData?.base_model?.toString() ?? "fill-in-base-model"; +} +function get_prompt_from_diffusers_model(model) { + const prompt = model.widgetData?.[0]?.text ?? model.cardData?.instance_prompt; + if (prompt) { + return escapeStringForJson(prompt); + } +} +const ben2 = (model) => [ + `import requests +from PIL import Image +from ben2 import AutoModel + +url = "https://huggingface.co/datasets/mishig/sample_images/resolve/main/teapot.jpg" +image = Image.open(requests.get(url, stream=True).raw) + +model = AutoModel.from_pretrained("${model.id}") +model.to("cuda").eval() +foreground = model.inference(image) +`, +]; +exports.ben2 = ben2; +const bertopic = (model) => [ + `from bertopic import BERTopic + +model = BERTopic.load("${model.id}")`, +]; +exports.bertopic = bertopic; +const bm25s = (model) => [ + `from bm25s.hf import BM25HF + +retriever = BM25HF.load_from_hub("${model.id}")`, +]; +exports.bm25s = bm25s; +const chatterbox = () => [ + `# pip install chatterbox-tts +import torchaudio as ta +from chatterbox.tts import ChatterboxTTS + +model = ChatterboxTTS.from_pretrained(device="cuda") + +text = "Ezreal and Jinx teamed up with Ahri, Yasuo, and Teemo to take down the enemy's Nexus in an epic late-game pentakill." +wav = model.generate(text) +ta.save("test-1.wav", wav, model.sr) + +# If you want to synthesize with a different voice, specify the audio prompt +AUDIO_PROMPT_PATH="YOUR_FILE.wav" +wav = model.generate(text, audio_prompt_path=AUDIO_PROMPT_PATH) +ta.save("test-2.wav", wav, model.sr)`, +]; +exports.chatterbox = chatterbox; +const chronos_forecasting = (model) => { + const installSnippet = `pip install chronos-forecasting`; + const exampleSnippet = `import pandas as pd +from chronos import BaseChronosPipeline + +pipeline = BaseChronosPipeline.from_pretrained("${model.id}", device_map="cuda") + +# Load historical data +context_df = pd.read_csv("https://autogluon.s3.us-west-2.amazonaws.com/datasets/timeseries/misc/AirPassengers.csv") + +# Generate predictions +pred_df = pipeline.predict_df( + context_df, + prediction_length=36, # Number of steps to forecast + quantile_levels=[0.1, 0.5, 0.9], # Quantiles for probabilistic forecast + id_column="item_id", # Column identifying different time series + timestamp_column="Month", # Column with datetime information + target="#Passengers", # Column(s) with time series values to predict +)`; + return [installSnippet, exampleSnippet]; +}; +exports.chronos_forecasting = chronos_forecasting; +const collectorvision = (model) => [ + `pip install git+https://github.com/HanClinto/CollectorVision huggingface_hub`, + `from huggingface_hub import hf_hub_download +import collector_vision as cvg + +checkpoint = hf_hub_download(repo_id="${model.id}", filename="model.onnx") + +# Detector models, such as Cornelius: +detector = cvg.NeuralCornerDetector(checkpoint) + +# Embedder models, such as Milo: +embedder = cvg.NeuralEmbedder(checkpoint)`, +]; +exports.collectorvision = collectorvision; +const colipri = (model) => { + const installSnippet = `pip install colipri`; + const exampleSnippet = `from colipri import get_model +from colipri import get_processor +from colipri import load_sample_ct +from colipri import ZeroShotImageClassificationPipeline + +model = get_model().cuda() +processor = get_processor() +pipeline = ZeroShotImageClassificationPipeline("${model.id}", processor) + +image = load_sample_ct() + +pipeline(image, ["No lung nodules", "Lung nodules"]) +`; + return [installSnippet, exampleSnippet]; +}; +exports.colipri = colipri; +const sap_rpt_one_oss = () => { + const installSnippet = `pip install git+https://github.com/SAP-samples/sap-rpt-1-oss`; + const classificationSnippet = `# Run a classification task +from sklearn.datasets import load_breast_cancer +from sklearn.metrics import accuracy_score +from sklearn.model_selection import train_test_split + +from sap_rpt_oss import SAP_RPT_OSS_Classifier + +# Load sample data +X, y = load_breast_cancer(return_X_y=True) +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42) + +# Initialize a classifier, 8k context and 8-fold bagging gives best performance, reduce if running out of memory +clf = SAP_RPT_OSS_Classifier(max_context_size=8192, bagging=8) + +clf.fit(X_train, y_train) + +# Predict probabilities +prediction_probabilities = clf.predict_proba(X_test) +# Predict labels +predictions = clf.predict(X_test) +print("Accuracy", accuracy_score(y_test, predictions))`; + const regressionsSnippet = `# Run a regression task +from sklearn.datasets import fetch_openml +from sklearn.metrics import r2_score +from sklearn.model_selection import train_test_split + +from sap_rpt_oss import SAP_RPT_OSS_Regressor + +# Load sample data +df = fetch_openml(data_id=531, as_frame=True) +X = df.data +y = df.target.astype(float) + +# Train-test split +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42) + +# Initialize the regressor, 8k context and 8-fold bagging gives best performance, reduce if running out of memory +regressor = SAP_RPT_OSS_Regressor(max_context_size=8192, bagging=8) + +regressor.fit(X_train, y_train) + +# Predict on the test set +predictions = regressor.predict(X_test) + +r2 = r2_score(y_test, predictions) +print("R² Score:", r2)`; + return [installSnippet, classificationSnippet, regressionsSnippet]; +}; +exports.sap_rpt_one_oss = sap_rpt_one_oss; +const cxr_foundation = () => [ + `# pip install git+https://github.com/Google-Health/cxr-foundation.git#subdirectory=python + +# Load image as grayscale (Stillwaterising, CC0, via Wikimedia Commons) +import requests +from PIL import Image +from io import BytesIO +image_url = "https://upload.wikimedia.org/wikipedia/commons/c/c8/Chest_Xray_PA_3-8-2010.png" +img = Image.open(requests.get(image_url, headers={'User-Agent': 'Demo'}, stream=True).raw).convert('L') + +# Run inference +from clientside.clients import make_hugging_face_client +cxr_client = make_hugging_face_client('cxr_model') +print(cxr_client.get_image_embeddings_from_images([img]))`, +]; +exports.cxr_foundation = cxr_foundation; +const depth_anything_v2 = (model) => { + let encoder; + let features; + let out_channels; + encoder = ""; + features = ""; + out_channels = ""; + if (model.id === "depth-anything/Depth-Anything-V2-Small") { + encoder = "vits"; + features = "64"; + out_channels = "[48, 96, 192, 384]"; + } + else if (model.id === "depth-anything/Depth-Anything-V2-Base") { + encoder = "vitb"; + features = "128"; + out_channels = "[96, 192, 384, 768]"; + } + else if (model.id === "depth-anything/Depth-Anything-V2-Large") { + encoder = "vitl"; + features = "256"; + out_channels = "[256, 512, 1024, 1024"; + } + return [ + ` +# Install from https://github.com/DepthAnything/Depth-Anything-V2 + +# Load the model and infer depth from an image +import cv2 +import torch + +from depth_anything_v2.dpt import DepthAnythingV2 + +# instantiate the model +model = DepthAnythingV2(encoder="${encoder}", features=${features}, out_channels=${out_channels}) + +# load the weights +filepath = hf_hub_download(repo_id="${model.id}", filename="depth_anything_v2_${encoder}.pth", repo_type="model") +state_dict = torch.load(filepath, map_location="cpu") +model.load_state_dict(state_dict).eval() + +raw_img = cv2.imread("your/image/path") +depth = model.infer_image(raw_img) # HxW raw depth map in numpy + `, + ]; +}; +exports.depth_anything_v2 = depth_anything_v2; +const depth_pro = (model) => { + const installSnippet = `# Download checkpoint +pip install huggingface-hub +huggingface-cli download --local-dir checkpoints ${model.id}`; + const inferenceSnippet = `import depth_pro + +# Load model and preprocessing transform +model, transform = depth_pro.create_model_and_transforms() +model.eval() + +# Load and preprocess an image. +image, _, f_px = depth_pro.load_rgb("example.png") +image = transform(image) + +# Run inference. +prediction = model.infer(image, f_px=f_px) + +# Results: 1. Depth in meters +depth = prediction["depth"] +# Results: 2. Focal length in pixels +focallength_px = prediction["focallength_px"]`; + return [installSnippet, inferenceSnippet]; +}; +exports.depth_pro = depth_pro; +const derm_foundation = () => [ + `from huggingface_hub import from_pretrained_keras +import tensorflow as tf, requests + +# Load and format input +IMAGE_URL = "https://storage.googleapis.com/dx-scin-public-data/dataset/images/3445096909671059178.png" +input_tensor = tf.train.Example( + features=tf.train.Features( + feature={ + "image/encoded": tf.train.Feature( + bytes_list=tf.train.BytesList(value=[requests.get(IMAGE_URL, stream=True).content]) + ) + } + ) +).SerializeToString() + +# Load model and run inference +loaded_model = from_pretrained_keras("google/derm-foundation") +infer = loaded_model.signatures["serving_default"] +print(infer(inputs=tf.constant([input_tensor])))`, +]; +exports.derm_foundation = derm_foundation; +const dia = (model) => [ + `import soundfile as sf +from dia.model import Dia + +model = Dia.from_pretrained("${model.id}") +text = "[S1] Dia is an open weights text to dialogue model. [S2] You get full control over scripts and voices. [S1] Wow. Amazing. (laughs) [S2] Try it now on Git hub or Hugging Face." +output = model.generate(text) + +sf.write("simple.mp3", output, 44100)`, +]; +exports.dia = dia; +const dia2 = (model) => [ + `from dia2 import Dia2, GenerationConfig, SamplingConfig + +dia = Dia2.from_repo("${model.id}", device="cuda", dtype="bfloat16") +config = GenerationConfig( + cfg_scale=2.0, + audio=SamplingConfig(temperature=0.8, top_k=50), + use_cuda_graph=True, +) +result = dia.generate("[S1] Hello Dia2!", config=config, output_wav="hello.wav", verbose=True) +`, +]; +exports.dia2 = dia2; +const describe_anything = (model) => [ + `# pip install git+https://github.com/NVlabs/describe-anything +from huggingface_hub import snapshot_download +from dam import DescribeAnythingModel + +snapshot_download(${model.id}, local_dir="checkpoints") + +dam = DescribeAnythingModel( + model_path="checkpoints", + conv_mode="v1", + prompt_mode="focal_prompt", +)`, +]; +exports.describe_anything = describe_anything; +const diffusers_install = "pip install -U diffusers transformers accelerate"; +const diffusersDefaultPrompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"; +const diffusersImg2ImgDefaultPrompt = "Turn this cat into a dog"; +const diffusersVideoDefaultPrompt = "A man with short gray hair plays a red electric guitar."; +const diffusers_default = (model) => [ + `import torch +from diffusers import DiffusionPipeline + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersDefaultPrompt}" +image = pipe(prompt).images[0]`, +]; +const diffusers_image_to_image = (model) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import load_image + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersImg2ImgDefaultPrompt}" +input_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png") + +image = pipe(image=input_image, prompt=prompt).images[0]`, +]; +const diffusers_image_to_video = (model) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import load_image, export_to_video + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda") +pipe.to("cuda") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersVideoDefaultPrompt}" +image = load_image( + "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/guitar-man.png" +) + +output = pipe(image=image, prompt=prompt).frames[0] +export_to_video(output, "output.mp4")`, +]; +const diffusers_controlnet = (model) => [ + `from diffusers import ControlNetModel, StableDiffusionControlNetPipeline + +controlnet = ControlNetModel.from_pretrained("${model.id}") +pipe = StableDiffusionControlNetPipeline.from_pretrained( + "${get_base_diffusers_model(model)}", controlnet=controlnet +)`, +]; +const diffusers_lora = (model) => [ + `import torch +from diffusers import DiffusionPipeline + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_lora_weights("${model.id}") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersDefaultPrompt}" +image = pipe(prompt).images[0]`, +]; +const diffusers_lora_image_to_image = (model) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import load_image + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_lora_weights("${model.id}") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersImg2ImgDefaultPrompt}" +input_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png") + +image = pipe(image=input_image, prompt=prompt).images[0]`, +]; +const diffusers_lora_text_to_video = (model) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import export_to_video + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_lora_weights("${model.id}") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersVideoDefaultPrompt}" + +output = pipe(prompt=prompt).frames[0] +export_to_video(output, "output.mp4")`, +]; +const diffusers_lora_image_to_video = (model) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import load_image, export_to_video + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_lora_weights("${model.id}") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersVideoDefaultPrompt}" +input_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/guitar-man.png") + +image = pipe(image=input_image, prompt=prompt).frames[0] +export_to_video(output, "output.mp4")`, +]; +const diffusers_textual_inversion = (model) => [ + `import torch +from diffusers import DiffusionPipeline + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_textual_inversion("${model.id}")`, +]; +const diffusers_flux_fill = (model) => [ + `import torch +from diffusers import FluxFillPipeline +from diffusers.utils import load_image + +image = load_image("https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup.png") +mask = load_image("https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup_mask.png") + +# switch to "mps" for apple devices +pipe = FluxFillPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda") +image = pipe( + prompt="a white paper cup", + image=image, + mask_image=mask, + height=1632, + width=1232, + guidance_scale=30, + num_inference_steps=50, + max_sequence_length=512, + generator=torch.Generator("cpu").manual_seed(0) +).images[0] +image.save(f"flux-fill-dev.png")`, +]; +const diffusers_inpainting = (model) => [ + `import torch +from diffusers import AutoPipelineForInpainting +from diffusers.utils import load_image + +# switch to "mps" for apple devices +pipe = AutoPipelineForInpainting.from_pretrained("${model.id}", dtype=torch.float16, variant="fp16", device_map="cuda") + +img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png" +mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png" + +image = load_image(img_url).resize((1024, 1024)) +mask_image = load_image(mask_url).resize((1024, 1024)) + +prompt = "a tiger sitting on a park bench" +generator = torch.Generator(device="cuda").manual_seed(0) + +image = pipe( + prompt=prompt, + image=image, + mask_image=mask_image, + guidance_scale=8.0, + num_inference_steps=20, # steps between 15 and 30 work well for us + strength=0.99, # make sure to use \`strength\` below 1.0 + generator=generator, +).images[0]`, +]; +const diffusers = (model) => { + let codeSnippets; + if (model.tags.includes("StableDiffusionInpaintPipeline") || + model.tags.includes("StableDiffusionXLInpaintPipeline")) { + codeSnippets = diffusers_inpainting(model); + } + else if (model.tags.includes("controlnet")) { + codeSnippets = diffusers_controlnet(model); + } + else if (model.tags.includes("lora")) { + if (model.pipeline_tag === "image-to-image") { + codeSnippets = diffusers_lora_image_to_image(model); + } + else if (model.pipeline_tag === "image-to-video") { + codeSnippets = diffusers_lora_image_to_video(model); + } + else if (model.pipeline_tag === "text-to-video") { + codeSnippets = diffusers_lora_text_to_video(model); + } + else { + codeSnippets = diffusers_lora(model); + } + } + else if (model.tags.includes("textual_inversion")) { + codeSnippets = diffusers_textual_inversion(model); + } + else if (model.tags.includes("FluxFillPipeline")) { + codeSnippets = diffusers_flux_fill(model); + } + else if (model.pipeline_tag === "image-to-video") { + codeSnippets = diffusers_image_to_video(model); + } + else if (model.pipeline_tag === "image-to-image") { + codeSnippets = diffusers_image_to_image(model); + } + else { + codeSnippets = diffusers_default(model); + } + return [diffusers_install, ...codeSnippets]; +}; +exports.diffusers = diffusers; +const diffusionkit = (model) => { + const sd3Snippet = `# Pipeline for Stable Diffusion 3 +from diffusionkit.mlx import DiffusionPipeline + +pipeline = DiffusionPipeline( + shift=3.0, + use_t5=False, + model_version=${model.id}, + low_memory_mode=True, + a16=True, + w16=True, +)`; + const fluxSnippet = `# Pipeline for Flux +from diffusionkit.mlx import FluxPipeline + +pipeline = FluxPipeline( + shift=1.0, + model_version=${model.id}, + low_memory_mode=True, + a16=True, + w16=True, +)`; + const generateSnippet = `# Image Generation +HEIGHT = 512 +WIDTH = 512 +NUM_STEPS = ${model.tags.includes("flux") ? 4 : 50} +CFG_WEIGHT = ${model.tags.includes("flux") ? 0 : 5} + +image, _ = pipeline.generate_image( + "a photo of a cat", + cfg_weight=CFG_WEIGHT, + num_steps=NUM_STEPS, + latent_size=(HEIGHT // 8, WIDTH // 8), +)`; + const pipelineSnippet = model.tags.includes("flux") ? fluxSnippet : sd3Snippet; + return [pipelineSnippet, generateSnippet]; +}; +exports.diffusionkit = diffusionkit; +const cartesia_pytorch = (model) => [ + `# pip install --no-binary :all: cartesia-pytorch +from cartesia_pytorch import ReneLMHeadModel +from transformers import AutoTokenizer + +model = ReneLMHeadModel.from_pretrained("${model.id}") +tokenizer = AutoTokenizer.from_pretrained("allenai/OLMo-1B-hf") + +in_message = ["Rene Descartes was"] +inputs = tokenizer(in_message, return_tensors="pt") + +outputs = model.generate(inputs.input_ids, max_length=50, top_k=100, top_p=0.99) +out_message = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0] + +print(out_message) +)`, +]; +exports.cartesia_pytorch = cartesia_pytorch; +const cartesia_mlx = (model) => [ + `import mlx.core as mx +import cartesia_mlx as cmx + +model = cmx.from_pretrained("${model.id}") +model.set_dtype(mx.float32) + +prompt = "Rene Descartes was" + +for text in model.generate( + prompt, + max_tokens=500, + eval_every_n=5, + verbose=True, + top_p=0.99, + temperature=0.85, +): + print(text, end="", flush=True) +`, +]; +exports.cartesia_mlx = cartesia_mlx; +const edsnlp = (model) => { + const packageName = nameWithoutNamespace(model.id).replaceAll("-", "_"); + return [ + `# Load it from the Hub directly +import edsnlp +nlp = edsnlp.load("${model.id}") +`, + `# Or install it as a package +!pip install git+https://huggingface.co/${model.id} + +# and import it as a module +import ${packageName} + +nlp = ${packageName}.load() # or edsnlp.load("${packageName}") +`, + ]; +}; +exports.edsnlp = edsnlp; +const espnetTTS = (model) => [ + `from espnet2.bin.tts_inference import Text2Speech + +model = Text2Speech.from_pretrained("${model.id}") + +speech, *_ = model("text to generate speech from")`, +]; +exports.espnetTTS = espnetTTS; +const espnetASR = (model) => [ + `from espnet2.bin.asr_inference import Speech2Text + +model = Speech2Text.from_pretrained( + "${model.id}" +) + +speech, rate = soundfile.read("speech.wav") +text, *_ = model(speech)[0]`, +]; +exports.espnetASR = espnetASR; +const espnetUnknown = () => [`unknown model type (must be text-to-speech or automatic-speech-recognition)`]; +const espnet = (model) => { + if (model.tags.includes("text-to-speech")) { + return (0, exports.espnetTTS)(model); + } + else if (model.tags.includes("automatic-speech-recognition")) { + return (0, exports.espnetASR)(model); + } + return espnetUnknown(); +}; +exports.espnet = espnet; +const fairseq = (model) => [ + `from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub + +models, cfg, task = load_model_ensemble_and_task_from_hf_hub( + "${model.id}" +)`, +]; +exports.fairseq = fairseq; +const flair = (model) => [ + `from flair.models import SequenceTagger + +tagger = SequenceTagger.load("${model.id}")`, +]; +exports.flair = flair; +const gliner = (model) => [ + `from gliner import GLiNER + +model = GLiNER.from_pretrained("${model.id}")`, +]; +exports.gliner = gliner; +const gliner2 = (model) => [ + `from gliner2 import GLiNER2 + +model = GLiNER2.from_pretrained("${model.id}") + +# Extract entities +text = "Apple CEO Tim Cook announced iPhone 15 in Cupertino yesterday." +result = extractor.extract_entities(text, ["company", "person", "product", "location"]) + +print(result)`, +]; +exports.gliner2 = gliner2; +const indextts = (model) => [ + `# Download model +from huggingface_hub import snapshot_download + +snapshot_download(${model.id}, local_dir="checkpoints") + +from indextts.infer import IndexTTS + +# Ensure config.yaml is present in the checkpoints directory +tts = IndexTTS(model_dir="checkpoints", cfg_path="checkpoints/config.yaml") + +voice = "path/to/your/reference_voice.wav" # Path to the voice reference audio file +text = "Hello, how are you?" +output_path = "output_index.wav" + +tts.infer(voice, text, output_path)`, +]; +exports.indextts = indextts; +const htrflow = (model) => [ + `# CLI usage +# see docs: https://ai-riksarkivet.github.io/htrflow/latest/getting_started/quick_start.html +htrflow pipeline `, + `# Python usage +from htrflow.pipeline.pipeline import Pipeline +from htrflow.pipeline.steps import Task +from htrflow.models.framework.model import ModelClass + +pipeline = Pipeline( + [ + Task( + ModelClass, {"model": "${model.id}"}, {} + ), + ])`, +]; +exports.htrflow = htrflow; +const keras = (model) => [ + `# Available backend options are: "jax", "torch", "tensorflow". +import os +os.environ["KERAS_BACKEND"] = "jax" + +import keras + +model = keras.saving.load_model("hf://${model.id}") +`, +]; +exports.keras = keras; +const _keras_hub_causal_lm = (modelId) => ` +import keras_hub + +# Load CausalLM model (optional: use half precision for inference) +causal_lm = keras_hub.models.CausalLM.from_preset("hf://${modelId}", dtype="bfloat16") +causal_lm.compile(sampler="greedy") # (optional) specify a sampler + +# Generate text +causal_lm.generate("Keras: deep learning for", max_length=64) +`; +const _keras_hub_text_to_image = (modelId) => ` +import keras_hub + +# Load TextToImage model (optional: use half precision for inference) +text_to_image = keras_hub.models.TextToImage.from_preset("hf://${modelId}", dtype="bfloat16") + +# Generate images with a TextToImage model. +text_to_image.generate("Astronaut in a jungle") +`; +const _keras_hub_text_classifier = (modelId) => ` +import keras_hub + +# Load TextClassifier model +text_classifier = keras_hub.models.TextClassifier.from_preset( + "hf://${modelId}", + num_classes=2, +) +# Fine-tune +text_classifier.fit(x=["Thilling adventure!", "Total snoozefest."], y=[1, 0]) +# Classify text +text_classifier.predict(["Not my cup of tea."]) +`; +const _keras_hub_image_classifier = (modelId) => ` +import keras_hub +import keras + +# Load ImageClassifier model +image_classifier = keras_hub.models.ImageClassifier.from_preset( + "hf://${modelId}", + num_classes=2, +) +# Fine-tune +image_classifier.fit( + x=keras.random.randint((32, 64, 64, 3), 0, 256), + y=keras.random.randint((32, 1), 0, 2), +) +# Classify image +image_classifier.predict(keras.random.randint((1, 64, 64, 3), 0, 256)) +`; +const _keras_hub_tasks_with_example = { + CausalLM: _keras_hub_causal_lm, + TextToImage: _keras_hub_text_to_image, + TextClassifier: _keras_hub_text_classifier, + ImageClassifier: _keras_hub_image_classifier, +}; +const _keras_hub_task_without_example = (task, modelId) => ` +import keras_hub + +# Create a ${task} model +task = keras_hub.models.${task}.from_preset("hf://${modelId}") +`; +const _keras_hub_generic_backbone = (modelId) => ` +import keras_hub + +# Create a Backbone model unspecialized for any task +backbone = keras_hub.models.Backbone.from_preset("hf://${modelId}") +`; +const keras_hub = (model) => { + const modelId = model.id; + const tasks = model.config?.keras_hub?.tasks ?? []; + const snippets = []; + // First, generate tasks with examples + for (const [task, snippet] of Object.entries(_keras_hub_tasks_with_example)) { + if (tasks.includes(task)) { + snippets.push(snippet(modelId)); + } + } + // Then, add remaining tasks + for (const task of tasks) { + if (!Object.keys(_keras_hub_tasks_with_example).includes(task)) { + snippets.push(_keras_hub_task_without_example(task, modelId)); + } + } + // Finally, add generic backbone snippet + snippets.push(_keras_hub_generic_backbone(modelId)); + return snippets; +}; +exports.keras_hub = keras_hub; +const kernels = (model) => [ + `# !pip install kernels + +from kernels import get_kernel + +kernel = get_kernel("${model.id}")`, +]; +exports.kernels = kernels; +const kimi_audio = (model) => [ + `# Example usage for KimiAudio +# pip install git+https://github.com/MoonshotAI/Kimi-Audio.git + +from kimia_infer.api.kimia import KimiAudio + +model = KimiAudio(model_path="${model.id}", load_detokenizer=True) + +sampling_params = { + "audio_temperature": 0.8, + "audio_top_k": 10, + "text_temperature": 0.0, + "text_top_k": 5, +} + +# For ASR +asr_audio = "asr_example.wav" +messages_asr = [ + {"role": "user", "message_type": "text", "content": "Please transcribe the following audio:"}, + {"role": "user", "message_type": "audio", "content": asr_audio} +] +_, text = model.generate(messages_asr, **sampling_params, output_type="text") +print(text) + +# For Q&A +qa_audio = "qa_example.wav" +messages_conv = [{"role": "user", "message_type": "audio", "content": qa_audio}] +wav, text = model.generate(messages_conv, **sampling_params, output_type="both") +sf.write("output_audio.wav", wav.cpu().view(-1).numpy(), 24000) +print(text) +`, +]; +exports.kimi_audio = kimi_audio; +const kittentts = (model) => [ + `from kittentts import KittenTTS +m = KittenTTS("${model.id}") + +audio = m.generate("This high quality TTS model works without a GPU") + +# Save the audio +import soundfile as sf +sf.write('output.wav', audio, 24000)`, +]; +exports.kittentts = kittentts; +const lightning_ir = (model) => { + if (model.tags.includes("bi-encoder")) { + return [ + `#install from https://github.com/webis-de/lightning-ir + +from lightning_ir import BiEncoderModule +model = BiEncoderModule("${model.id}") + +model.score("query", ["doc1", "doc2", "doc3"])`, + ]; + } + else if (model.tags.includes("cross-encoder")) { + return [ + `#install from https://github.com/webis-de/lightning-ir + +from lightning_ir import CrossEncoderModule +model = CrossEncoderModule("${model.id}") + +model.score("query", ["doc1", "doc2", "doc3"])`, + ]; + } + return [ + `#install from https://github.com/webis-de/lightning-ir + +from lightning_ir import BiEncoderModule, CrossEncoderModule + +# depending on the model type, use either BiEncoderModule or CrossEncoderModule +model = BiEncoderModule("${model.id}") +# model = CrossEncoderModule("${model.id}") + +model.score("query", ["doc1", "doc2", "doc3"])`, + ]; +}; +exports.lightning_ir = lightning_ir; +const llama_cpp_python = (model) => { + const snippets = [ + `# !pip install llama-cpp-python + +from llama_cpp import Llama + +llm = Llama.from_pretrained( + repo_id="${model.id}", + filename="{{GGUF_FILE}}", +) +`, + ]; + if (model.tags.includes("conversational")) { + const messages = (0, inputs_js_1.getModelInputSnippet)(model); + snippets.push(`llm.create_chat_completion( + messages = ${(0, common_js_1.stringifyMessages)(messages, { attributeKeyQuotes: true, indent: "\t" })} +)`); + } + else { + snippets.push(`output = llm( + "Once upon a time,", + max_tokens=512, + echo=True +) +print(output)`); + } + return snippets; +}; +exports.llama_cpp_python = llama_cpp_python; +const lerobot = (model) => { + if (model.tags.includes("smolvla")) { + const smolvlaSnippets = [ + // Installation snippet + `# See https://github.com/huggingface/lerobot?tab=readme-ov-file#installation for more details +git clone https://github.com/huggingface/lerobot.git +cd lerobot +pip install -e .[smolvla]`, + // Finetune snippet + `# Launch finetuning on your dataset +python lerobot/scripts/train.py \\ +--policy.path=${model.id} \\ +--dataset.repo_id=lerobot/svla_so101_pickplace \\ +--batch_size=64 \\ +--steps=20000 \\ +--output_dir=outputs/train/my_smolvla \\ +--job_name=my_smolvla_training \\ +--policy.device=cuda \\ +--wandb.enable=true`, + ]; + if (model.id !== "lerobot/smolvla_base") { + // Inference snippet (only if not base model) + smolvlaSnippets.push(`# Run the policy using the record function +python -m lerobot.record \\ + --robot.type=so101_follower \\ + --robot.port=/dev/ttyACM0 \\ # <- Use your port + --robot.id=my_blue_follower_arm \\ # <- Use your robot id + --robot.cameras="{ front: {type: opencv, index_or_path: 8, width: 640, height: 480, fps: 30}}" \\ # <- Use your cameras + --dataset.single_task="Grasp a lego block and put it in the bin." \\ # <- Use the same task description you used in your dataset recording + --dataset.repo_id=HF_USER/dataset_name \\ # <- This will be the dataset name on HF Hub + --dataset.episode_time_s=50 \\ + --dataset.num_episodes=10 \\ + --policy.path=${model.id}`); + } + return smolvlaSnippets; + } + return []; +}; +exports.lerobot = lerobot; +const litert_lm = (model) => [ + `# LiteRT-LM runs on various platforms (Android, iOS, Windows, Linux, macOS, IoT, Web/WASM) +# and supports many APIs (C++, Python, Kotlin, Swift, JavaScript, Flutter). +# For platform-specific integration guides, please refer to the official developer website: +# https://ai.google.dev/edge/litert-lm + +# To try LiteRT-LM, the easiest way is to use our CLI tool. +# 1. Install the LiteRT-LM CLI tool: +pip install litert-lm + +# 2. Download and run this model locally: +# See: https://ai.google.dev/edge/litert-lm/cli +litert-lm run \\ + --from-huggingface-repo=${model.id} \\ + model.litertlm \\ + --prompt="Write me a poem"`, +]; +exports.litert_lm = litert_lm; +const tf_keras = (model) => [ + `# Note: 'keras<3.x' or 'tf_keras' must be installed (legacy) +# See https://github.com/keras-team/tf-keras for more details. +from huggingface_hub import from_pretrained_keras + +model = from_pretrained_keras("${model.id}") +`, +]; +exports.tf_keras = tf_keras; +const mamba_ssm = (model) => [ + `from mamba_ssm import MambaLMHeadModel + +model = MambaLMHeadModel.from_pretrained("${model.id}")`, +]; +exports.mamba_ssm = mamba_ssm; +const mars5_tts = (model) => [ + `# Install from https://github.com/Camb-ai/MARS5-TTS + +from inference import Mars5TTS +mars5 = Mars5TTS.from_pretrained("${model.id}")`, +]; +exports.mars5_tts = mars5_tts; +const matanyone = (model) => [ + `# Install from https://github.com/pq-yang/MatAnyone.git + +from matanyone.model.matanyone import MatAnyone +model = MatAnyone.from_pretrained("${model.id}")`, + ` +from matanyone import InferenceCore +processor = InferenceCore("${model.id}")`, +]; +exports.matanyone = matanyone; +const mesh_anything = () => [ + `# Install from https://github.com/buaacyw/MeshAnything.git + +from MeshAnything.models.meshanything import MeshAnything + +# refer to https://github.com/buaacyw/MeshAnything/blob/main/main.py#L91 on how to define args +# and https://github.com/buaacyw/MeshAnything/blob/main/app.py regarding usage +model = MeshAnything(args)`, +]; +exports.mesh_anything = mesh_anything; +const multimolecule = (model) => { + const widgetExample = model.widgetData?.[0]; + const exampleText = widgetExample?.text; + const maskToken = model.mask_token ?? ""; + const sequence = exampleText?.replace(maskToken, "A"); + const snippets = [`pip install multimolecule`]; + if (sequence) { + snippets.push(`from multimolecule import AutoModel, AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained("${model.id}") +model = AutoModel.from_pretrained("${model.id}") + +inputs = tokenizer("${sequence}", return_tensors="pt") +outputs = model(**inputs) +embeddings = outputs.last_hidden_state`); + } + else { + snippets.push(`from multimolecule import AutoModel, AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained("${model.id}") +model = AutoModel.from_pretrained("${model.id}")`); + } + if (model.tags.includes("rna-secondary-structure") && exampleText) { + snippets.push(`import multimolecule +from transformers import pipeline + +predictor = pipeline("rna-secondary-structure", model="${model.id}") +output = predictor("${exampleText}") +print(output["secondary_structure"])`); + } + else if (model.pipeline_tag === "fill-mask" && exampleText) { + snippets.push(`import multimolecule +from transformers import pipeline + +predictor = pipeline("fill-mask", model="${model.id}") +output = predictor("${exampleText}")`); + } + return snippets; +}; +exports.multimolecule = multimolecule; +const open_clip = (model) => [ + `import open_clip + +model, preprocess_train, preprocess_val = open_clip.create_model_and_transforms('hf-hub:${model.id}') +tokenizer = open_clip.get_tokenizer('hf-hub:${model.id}')`, +]; +exports.open_clip = open_clip; +const paddlenlp = (model) => { + if (model.config?.architectures?.[0]) { + const architecture = model.config.architectures[0]; + return [ + [ + `from paddlenlp.transformers import AutoTokenizer, ${architecture}`, + "", + `tokenizer = AutoTokenizer.from_pretrained("${model.id}", from_hf_hub=True)`, + `model = ${architecture}.from_pretrained("${model.id}", from_hf_hub=True)`, + ].join("\n"), + ]; + } + else { + return [ + [ + `# ⚠️ Type of model unknown`, + `from paddlenlp.transformers import AutoTokenizer, AutoModel`, + "", + `tokenizer = AutoTokenizer.from_pretrained("${model.id}", from_hf_hub=True)`, + `model = AutoModel.from_pretrained("${model.id}", from_hf_hub=True)`, + ].join("\n"), + ]; + } +}; +exports.paddlenlp = paddlenlp; +const paddleocr = (model) => { + const mapping = { + textline_detection: { className: "TextDetection" }, + textline_recognition: { className: "TextRecognition" }, + seal_text_detection: { className: "SealTextDetection" }, + doc_img_unwarping: { className: "TextImageUnwarping" }, + doc_img_orientation_classification: { className: "DocImgOrientationClassification" }, + textline_orientation_classification: { className: "TextLineOrientationClassification" }, + chart_parsing: { className: "ChartParsing" }, + formula_recognition: { className: "FormulaRecognition" }, + layout_detection: { className: "LayoutDetection" }, + table_cells_detection: { className: "TableCellsDetection" }, + wired_table_classification: { className: "TableClassification" }, + table_structure_recognition: { className: "TableStructureRecognition" }, + }; + if (model.tags.includes("doc_vlm")) { + return [ + `# 1. See https://www.paddlepaddle.org.cn/en/install to install paddlepaddle +# 2. pip install paddleocr + +from paddleocr import DocVLM +model = DocVLM(model_name="${nameWithoutNamespace(model.id)}") +output = model.predict( + input={"image": "path/to/image.png", "query": "Parsing this image and output the content in Markdown format."}, + batch_size=1 +) +for res in output: + res.print() + res.save_to_json(save_path="./output/res.json")`, + ]; + } + if (model.tags.includes("document-parse")) { + const rawVersion = model.id.replace("PaddlePaddle/PaddleOCR-VL-", "v"); + const version = rawVersion === "PaddlePaddle/PaddleOCR-VL" ? "v1" : rawVersion; + return [ + `# See https://www.paddleocr.ai/latest/version3.x/pipeline_usage/PaddleOCR-VL.html to installation + +from paddleocr import PaddleOCRVL +pipeline = PaddleOCRVL(pipeline_version="${version}") +output = pipeline.predict("path/to/document_image.png") +for res in output: + res.print() + res.save_to_json(save_path="output") + res.save_to_markdown(save_path="output")`, + ]; + } + for (const tag of model.tags) { + if (tag in mapping) { + const { className } = mapping[tag]; + return [ + `# 1. See https://www.paddlepaddle.org.cn/en/install to install paddlepaddle +# 2. pip install paddleocr + +from paddleocr import ${className} +model = ${className}(model_name="${nameWithoutNamespace(model.id)}") +output = model.predict(input="path/to/image.png", batch_size=1) +for res in output: + res.print() + res.save_to_img(save_path="./output/") + res.save_to_json(save_path="./output/res.json")`, + ]; + } + } + return [ + `# Please refer to the document for information on how to use the model. +# https://paddlepaddle.github.io/PaddleOCR/latest/en/version3.x/module_usage/module_overview.html`, + ]; +}; +exports.paddleocr = paddleocr; +const perception_encoder = (model) => { + const clip_model = `# Use PE-Core models as CLIP models +import core.vision_encoder.pe as pe + +model = pe.CLIP.from_config("${model.id}", pretrained=True)`; + const vision_encoder = `# Use any PE model as a vision encoder +import core.vision_encoder.pe as pe + +model = pe.VisionTransformer.from_config("${model.id}", pretrained=True)`; + if (model.id.includes("Core")) { + return [clip_model, vision_encoder]; + } + else { + return [vision_encoder]; + } +}; +exports.perception_encoder = perception_encoder; +const phantom_wan = (model) => [ + `from huggingface_hub import snapshot_download +from phantom_wan import WANI2V, configs + +checkpoint_dir = snapshot_download("${model.id}") +wan_i2v = WanI2V( + config=configs.WAN_CONFIGS['i2v-14B'], + checkpoint_dir=checkpoint_dir, + ) + video = wan_i2v.generate(text_prompt, image_prompt)`, +]; +exports.phantom_wan = phantom_wan; +const pocket_tts = (model) => [ + `from pocket_tts import TTSModel +import scipy.io.wavfile + +tts_model = TTSModel.load_model("${model.id}") +voice_state = tts_model.get_state_for_audio_prompt( + "hf://kyutai/tts-voices/alba-mackenna/casual.wav" +) +audio = tts_model.generate_audio(voice_state, "Hello world, this is a test.") +# Audio is a 1D torch tensor containing PCM data. +scipy.io.wavfile.write("output.wav", tts_model.sample_rate, audio.numpy())`, +]; +exports.pocket_tts = pocket_tts; +const pyannote_audio_pipeline = (model) => [ + `from pyannote.audio import Pipeline + +pipeline = Pipeline.from_pretrained("${model.id}") + +# inference on the whole file +pipeline("file.wav") + +# inference on an excerpt +from pyannote.core import Segment +excerpt = Segment(start=2.0, end=5.0) + +from pyannote.audio import Audio +waveform, sample_rate = Audio().crop("file.wav", excerpt) +pipeline({"waveform": waveform, "sample_rate": sample_rate})`, +]; +exports.pyannote_audio_pipeline = pyannote_audio_pipeline; +const pyannote_audio_model = (model) => [ + `from pyannote.audio import Model, Inference + +model = Model.from_pretrained("${model.id}") +inference = Inference(model) + +# inference on the whole file +inference("file.wav") + +# inference on an excerpt +from pyannote.core import Segment +excerpt = Segment(start=2.0, end=5.0) +inference.crop("file.wav", excerpt)`, +]; +const pyannote_audio = (model) => { + if (model.tags.includes("pyannote-audio-pipeline")) { + return (0, exports.pyannote_audio_pipeline)(model); + } + return pyannote_audio_model(model); +}; +exports.pyannote_audio = pyannote_audio; +const relik = (model) => [ + `from relik import Relik + +relik = Relik.from_pretrained("${model.id}")`, +]; +exports.relik = relik; +const renderformer = (model) => [ + `# Install from https://github.com/microsoft/renderformer + +from renderformer import RenderFormerRenderingPipeline +pipeline = RenderFormerRenderingPipeline.from_pretrained("${model.id}")`, +]; +exports.renderformer = renderformer; +const tensorflowttsTextToMel = (model) => [ + `from tensorflow_tts.inference import AutoProcessor, TFAutoModel + +processor = AutoProcessor.from_pretrained("${model.id}") +model = TFAutoModel.from_pretrained("${model.id}") +`, +]; +const tensorflowttsMelToWav = (model) => [ + `from tensorflow_tts.inference import TFAutoModel + +model = TFAutoModel.from_pretrained("${model.id}") +audios = model.inference(mels) +`, +]; +const tensorflowttsUnknown = (model) => [ + `from tensorflow_tts.inference import TFAutoModel + +model = TFAutoModel.from_pretrained("${model.id}") +`, +]; +const tensorflowtts = (model) => { + if (model.tags.includes("text-to-mel")) { + return tensorflowttsTextToMel(model); + } + else if (model.tags.includes("mel-to-wav")) { + return tensorflowttsMelToWav(model); + } + return tensorflowttsUnknown(model); +}; +exports.tensorflowtts = tensorflowtts; +const timm = (model) => [ + `import timm + +model = timm.create_model("hf_hub:${model.id}", pretrained=True)`, +]; +exports.timm = timm; +const saelens = ( /* model: ModelData */) => [ + `# pip install sae-lens +from sae_lens import SAE + +sae, cfg_dict, sparsity = SAE.from_pretrained( + release = "RELEASE_ID", # e.g., "gpt2-small-res-jb". See other options in https://github.com/jbloomAus/SAELens/blob/main/sae_lens/pretrained_saes.yaml + sae_id = "SAE_ID", # e.g., "blocks.8.hook_resid_pre". Won't always be a hook point +)`, +]; +exports.saelens = saelens; +const seed_story = () => [ + `# seed_story_cfg_path refers to 'https://github.com/TencentARC/SEED-Story/blob/master/configs/clm_models/agent_7b_sft.yaml' +# llm_cfg_path refers to 'https://github.com/TencentARC/SEED-Story/blob/master/configs/clm_models/llama2chat7b_lora.yaml' +from omegaconf import OmegaConf +import hydra + +# load Llama2 +llm_cfg = OmegaConf.load(llm_cfg_path) +llm = hydra.utils.instantiate(llm_cfg, torch_dtype="fp16") + +# initialize seed_story +seed_story_cfg = OmegaConf.load(seed_story_cfg_path) +seed_story = hydra.utils.instantiate(seed_story_cfg, llm=llm) `, +]; +exports.seed_story = seed_story; +const skopsPickle = (model, modelFile) => { + return [ + `import joblib +from skops.hub_utils import download +download("${model.id}", "path_to_folder") +model = joblib.load( + "${modelFile}" +) +# only load pickle files from sources you trust +# read more about it here https://skops.readthedocs.io/en/stable/persistence.html`, + ]; +}; +const skopsFormat = (model, modelFile) => { + return [ + `from skops.hub_utils import download +from skops.io import load +download("${model.id}", "path_to_folder") +# make sure model file is in skops format +# if model is a pickle file, make sure it's from a source you trust +model = load("path_to_folder/${modelFile}")`, + ]; +}; +const skopsJobLib = (model) => { + return [ + `from huggingface_hub import hf_hub_download +import joblib +model = joblib.load( + hf_hub_download("${model.id}", "sklearn_model.joblib") +) +# only load pickle files from sources you trust +# read more about it here https://skops.readthedocs.io/en/stable/persistence.html`, + ]; +}; +const sklearn = (model) => { + if (model.tags.includes("skops")) { + const skopsmodelFile = model.config?.sklearn?.model?.file; + const skopssaveFormat = model.config?.sklearn?.model_format; + if (!skopsmodelFile) { + return [`# ⚠️ Model filename not specified in config.json`]; + } + if (skopssaveFormat === "pickle") { + return skopsPickle(model, skopsmodelFile); + } + else { + return skopsFormat(model, skopsmodelFile); + } + } + else { + return skopsJobLib(model); + } +}; +exports.sklearn = sklearn; +const stable_audio_tools = (model) => [ + `import torch +import torchaudio +from einops import rearrange +from stable_audio_tools import get_pretrained_model +from stable_audio_tools.inference.generation import generate_diffusion_cond + +device = "cuda" if torch.cuda.is_available() else "cpu" + +# Download model +model, model_config = get_pretrained_model("${model.id}") +sample_rate = model_config["sample_rate"] +sample_size = model_config["sample_size"] + +model = model.to(device) + +# Set up text and timing conditioning +conditioning = [{ + "prompt": "128 BPM tech house drum loop", +}] + +# Generate stereo audio +output = generate_diffusion_cond( + model, + conditioning=conditioning, + sample_size=sample_size, + device=device +) + +# Rearrange audio batch to a single sequence +output = rearrange(output, "b d n -> d (b n)") + +# Peak normalize, clip, convert to int16, and save to file +output = output.to(torch.float32).div(torch.max(torch.abs(output))).clamp(-1, 1).mul(32767).to(torch.int16).cpu() +torchaudio.save("output.wav", output, sample_rate)`, +]; +exports.stable_audio_tools = stable_audio_tools; +const fastai = (model) => [ + `from huggingface_hub import from_pretrained_fastai + +learn = from_pretrained_fastai("${model.id}")`, +]; +exports.fastai = fastai; +const sam2 = (model) => { + const image_predictor = `# Use SAM2 with images +import torch +from sam2.sam2_image_predictor import SAM2ImagePredictor + +predictor = SAM2ImagePredictor.from_pretrained(${model.id}) + +with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16): + predictor.set_image() + masks, _, _ = predictor.predict()`; + const video_predictor = `# Use SAM2 with videos +import torch +from sam2.sam2_video_predictor import SAM2VideoPredictor + +predictor = SAM2VideoPredictor.from_pretrained(${model.id}) + +with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16): + state = predictor.init_state() + + # add new prompts and instantly get the output on the same frame + frame_idx, object_ids, masks = predictor.add_new_points(state, ): + + # propagate the prompts to get masklets throughout the video + for frame_idx, object_ids, masks in predictor.propagate_in_video(state): + ...`; + return [image_predictor, video_predictor]; +}; +exports.sam2 = sam2; +const sam_3d_objects = (model) => [ + `from inference import Inference, load_image, load_single_mask +from huggingface_hub import hf_hub_download + +path = hf_hub_download("${model.id}", "pipeline.yaml") +inference = Inference(path, compile=False) + +image = load_image("path_to_image.png") +mask = load_single_mask("path_to_mask.png", index=14) + +output = inference(image, mask)`, +]; +exports.sam_3d_objects = sam_3d_objects; +const sam_3d_body = (model) => [ + `from notebook.utils import setup_sam_3d_body + +estimator = setup_sam_3d_body(${model.id}) +outputs = estimator.process_one_image(image) +rend_img = visualize_sample_together(image, outputs, estimator.faces)`, +]; +exports.sam_3d_body = sam_3d_body; +const sampleFactory = (model) => [ + `python -m sample_factory.huggingface.load_from_hub -r ${model.id} -d ./train_dir`, +]; +exports.sampleFactory = sampleFactory; +function get_widget_examples_from_st_model(model) { + const widgetExample = model.widgetData?.[0]; + if (widgetExample?.source_sentence && widgetExample?.sentences?.length) { + return [widgetExample.source_sentence, ...widgetExample.sentences]; + } +} +const sentenceTransformers = (model) => { + const remote_code_snippet = model.tags.includes(TAG_CUSTOM_CODE) ? ", trust_remote_code=True" : ""; + if (model.tags.includes("PyLate")) { + return [ + `from pylate import models + +queries = [ + "Which planet is known as the Red Planet?", + "What is the largest planet in our solar system?", +] + +documents = [ + ["Mars is the Red Planet.", "Venus is Earth's twin."], + ["Jupiter is the largest planet.", "Saturn has rings."], +] + +model = models.ColBERT(model_name_or_path="${model.id}") + +queries_emb = model.encode(queries, is_query=True) +docs_emb = model.encode(documents, is_query=False)`, + ]; + } + if (model.tags.includes("cross-encoder") || model.pipeline_tag == "text-ranking") { + return [ + `from sentence_transformers import CrossEncoder + +model = CrossEncoder("${model.id}"${remote_code_snippet}) + +query = "Which planet is known as the Red Planet?" +passages = [ + "Venus is often called Earth's twin because of its similar size and proximity.", + "Mars, known for its reddish appearance, is often referred to as the Red Planet.", + "Jupiter, the largest planet in our solar system, has a prominent red spot.", + "Saturn, famous for its rings, is sometimes mistaken for the Red Planet." +] + +scores = model.predict([(query, passage) for passage in passages]) +print(scores)`, + ]; + } + const exampleSentences = get_widget_examples_from_st_model(model) ?? [ + "The weather is lovely today.", + "It's so sunny outside!", + "He drove to the stadium.", + ]; + return [ + `from sentence_transformers import SentenceTransformer + +model = SentenceTransformer("${model.id}"${remote_code_snippet}) + +sentences = ${JSON.stringify(exampleSentences, null, 4)} +embeddings = model.encode(sentences) + +similarities = model.similarity(embeddings, embeddings) +print(similarities.shape) +# [${exampleSentences.length}, ${exampleSentences.length}]`, + ]; +}; +exports.sentenceTransformers = sentenceTransformers; +const setfit = (model) => [ + `from setfit import SetFitModel + +model = SetFitModel.from_pretrained("${model.id}")`, +]; +exports.setfit = setfit; +const spacy = (model) => [ + `!pip install https://huggingface.co/${model.id}/resolve/main/${nameWithoutNamespace(model.id)}-any-py3-none-any.whl + +# Using spacy.load(). +import spacy +nlp = spacy.load("${nameWithoutNamespace(model.id)}") + +# Importing as module. +import ${nameWithoutNamespace(model.id)} +nlp = ${nameWithoutNamespace(model.id)}.load()`, +]; +exports.spacy = spacy; +const span_marker = (model) => [ + `from span_marker import SpanMarkerModel + +model = SpanMarkerModel.from_pretrained("${model.id}")`, +]; +exports.span_marker = span_marker; +const stanza = (model) => [ + `import stanza + +stanza.download("${nameWithoutNamespace(model.id).replace("stanza-", "")}") +nlp = stanza.Pipeline("${nameWithoutNamespace(model.id).replace("stanza-", "")}")`, +]; +exports.stanza = stanza; +const speechBrainMethod = (speechbrainInterface) => { + switch (speechbrainInterface) { + case "EncoderClassifier": + return "classify_file"; + case "EncoderDecoderASR": + case "EncoderASR": + return "transcribe_file"; + case "SpectralMaskEnhancement": + return "enhance_file"; + case "SepformerSeparation": + return "separate_file"; + default: + return undefined; + } +}; +const speechbrain = (model) => { + const speechbrainInterface = model.config?.speechbrain?.speechbrain_interface; + if (speechbrainInterface === undefined) { + return [`# interface not specified in config.json`]; + } + const speechbrainMethod = speechBrainMethod(speechbrainInterface); + if (speechbrainMethod === undefined) { + return [`# interface in config.json invalid`]; + } + return [ + `from speechbrain.pretrained import ${speechbrainInterface} +model = ${speechbrainInterface}.from_hparams( + "${model.id}" +) +model.${speechbrainMethod}("file.wav")`, + ]; +}; +exports.speechbrain = speechbrain; +const terratorch = (model) => [ + `from terratorch.registry import BACKBONE_REGISTRY + +model = BACKBONE_REGISTRY.build("${model.id}")`, +]; +exports.terratorch = terratorch; +const hasChatTemplate = (model) => model.config?.tokenizer_config?.chat_template !== undefined || + model.config?.processor_config?.chat_template !== undefined || + model.config?.chat_template_jinja !== undefined; +const transformers = (model) => { + const info = model.transformersInfo; + if (!info) { + return [`# ⚠️ Type of model unknown`]; + } + const remote_code_snippet = model.tags.includes(TAG_CUSTOM_CODE) ? ", trust_remote_code=True" : ""; + const autoSnippet = []; + if (info.processor) { + const processorVarName = info.processor === "AutoTokenizer" + ? "tokenizer" + : info.processor === "AutoFeatureExtractor" + ? "extractor" + : "processor"; + autoSnippet.push("# Load model directly", `from transformers import ${info.processor}, ${info.auto_model}`, "", `${processorVarName} = ${info.processor}.from_pretrained("${model.id}"` + remote_code_snippet + ")", `model = ${info.auto_model}.from_pretrained("${model.id}"` + remote_code_snippet + ")"); + if (model.tags.includes("conversational") && hasChatTemplate(model)) { + if (model.tags.includes("image-text-to-text")) { + autoSnippet.push("messages = [", [ + " {", + ' "role": "user",', + ' "content": [', + ' {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"},', + ' {"type": "text", "text": "What animal is on the candy?"}', + " ]", + " },", + ].join("\n"), "]"); + } + else { + autoSnippet.push("messages = [", ' {"role": "user", "content": "Who are you?"},', "]"); + } + autoSnippet.push(`inputs = ${processorVarName}.apply_chat_template(`, " messages,", " add_generation_prompt=True,", " tokenize=True,", " return_dict=True,", ' return_tensors="pt",', ").to(model.device)", "", "outputs = model.generate(**inputs, max_new_tokens=40)", `print(${processorVarName}.decode(outputs[0][inputs["input_ids"].shape[-1]:]))`); + } + } + else { + autoSnippet.push("# Load model directly", `from transformers import ${info.auto_model}`, `model = ${info.auto_model}.from_pretrained("${model.id}"` + remote_code_snippet + ', dtype="auto")'); + } + if (model.pipeline_tag && library_to_tasks_js_1.LIBRARY_TASK_MAPPING.transformers?.includes(model.pipeline_tag)) { + const pipelineSnippet = ["# Use a pipeline as a high-level helper"]; + if (library_to_tasks_js_1.REMOVED_IN_V5_TRANSFORMERS_PIPELINES.includes(model.pipeline_tag)) { + pipelineSnippet.push(`# Warning: Pipeline type "${model.pipeline_tag}" is no longer supported in transformers v5.`, `# You must load the model directly (see below) or downgrade to v4.x with:`, `# 'pip install "transformers<5.0.0'`); + } + pipelineSnippet.push("from transformers import pipeline", "", `pipe = pipeline("${model.pipeline_tag}", model="${model.id}"` + remote_code_snippet + ")"); + if (model.tags.includes("conversational")) { + if (model.tags.includes("image-text-to-text")) { + pipelineSnippet.push("messages = [", [ + " {", + ' "role": "user",', + ' "content": [', + ' {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"},', + ' {"type": "text", "text": "What animal is on the candy?"}', + " ]", + " },", + ].join("\n"), "]"); + pipelineSnippet.push("pipe(text=messages)"); + } + else { + pipelineSnippet.push("messages = [", ' {"role": "user", "content": "Who are you?"},', "]"); + pipelineSnippet.push("pipe(messages)"); + } + } + else if (model.pipeline_tag === "zero-shot-image-classification") { + pipelineSnippet.push("pipe(", ' "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/parrots.png",', ' candidate_labels=["animals", "humans", "landscape"],', ")"); + } + else if (model.pipeline_tag === "image-classification") { + pipelineSnippet.push('pipe("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/parrots.png")'); + } + return [pipelineSnippet.join("\n"), autoSnippet.join("\n")]; + } + return [autoSnippet.join("\n")]; +}; +exports.transformers = transformers; +const transformersJS = (model) => { + if (!model.pipeline_tag) { + return [`// ⚠️ Unknown pipeline tag`]; + } + const libName = "@huggingface/transformers"; + return [ + `// npm i ${libName} +import { pipeline } from '${libName}'; + +// Allocate pipeline +const pipe = await pipeline('${model.pipeline_tag}', '${model.id}');`, + ]; +}; +exports.transformersJS = transformersJS; +const peftTask = (peftTaskType) => { + switch (peftTaskType) { + case "CAUSAL_LM": + return "CausalLM"; + case "SEQ_2_SEQ_LM": + return "Seq2SeqLM"; + case "TOKEN_CLS": + return "TokenClassification"; + case "SEQ_CLS": + return "SequenceClassification"; + default: + return undefined; + } +}; +const peft = (model) => { + const { base_model_name_or_path: peftBaseModel, task_type: peftTaskType } = model.config?.peft ?? {}; + const pefttask = peftTask(peftTaskType); + if (!pefttask) { + return [`Task type is invalid.`]; + } + if (!peftBaseModel) { + return [`Base model is not found.`]; + } + return [ + `from peft import PeftModel +from transformers import AutoModelFor${pefttask} + +base_model = AutoModelFor${pefttask}.from_pretrained("${peftBaseModel}") +model = PeftModel.from_pretrained(base_model, "${model.id}")`, + ]; +}; +exports.peft = peft; +const fasttext = (model) => [ + `from huggingface_hub import hf_hub_download +import fasttext + +model = fasttext.load_model(hf_hub_download("${model.id}", "model.bin"))`, +]; +exports.fasttext = fasttext; +const stableBaselines3 = (model) => [ + `from huggingface_sb3 import load_from_hub +checkpoint = load_from_hub( + repo_id="${model.id}", + filename="{MODEL FILENAME}.zip", +)`, +]; +exports.stableBaselines3 = stableBaselines3; +const nemoDomainResolver = (domain, model) => { + switch (domain) { + case "ASR": + return [ + `import nemo.collections.asr as nemo_asr +asr_model = nemo_asr.models.ASRModel.from_pretrained("${model.id}") + +transcriptions = asr_model.transcribe(["file.wav"])`, + ]; + default: + return undefined; + } +}; +const mlAgents = (model) => [ + `mlagents-load-from-hf --repo-id="${model.id}" --local-dir="./download: string[]s"`, +]; +exports.mlAgents = mlAgents; +const sentis = ( /* model: ModelData */) => [ + `string modelName = "[Your model name here].sentis"; +Model model = ModelLoader.Load(Application.streamingAssetsPath + "/" + modelName); +IWorker engine = WorkerFactory.CreateWorker(BackendType.GPUCompute, model); +// Please see provided C# file for more details +`, +]; +exports.sentis = sentis; +const sana = (model) => [ + ` +# Load the model and infer image from text +import torch +from app.sana_pipeline import SanaPipeline +from torchvision.utils import save_image + +sana = SanaPipeline("configs/sana_config/1024ms/Sana_1600M_img1024.yaml") +sana.from_pretrained("hf://${model.id}") + +image = sana( + prompt='a cyberpunk cat with a neon sign that says "Sana"', + height=1024, + width=1024, + guidance_scale=5.0, + pag_guidance_scale=2.0, + num_inference_steps=18, +) `, +]; +exports.sana = sana; +const vibevoice = (model) => [ + `import torch, soundfile as sf, librosa, numpy as np +from vibevoice.processor.vibevoice_processor import VibeVoiceProcessor +from vibevoice.modular.modeling_vibevoice_inference import VibeVoiceForConditionalGenerationInference + +# Load voice sample (should be 24kHz mono) +voice, sr = sf.read("path/to/voice_sample.wav") +if voice.ndim > 1: voice = voice.mean(axis=1) +if sr != 24000: voice = librosa.resample(voice, sr, 24000) + +processor = VibeVoiceProcessor.from_pretrained("${model.id}") +model = VibeVoiceForConditionalGenerationInference.from_pretrained( + "${model.id}", torch_dtype=torch.bfloat16 +).to("cuda").eval() +model.set_ddpm_inference_steps(5) + +inputs = processor(text=["Speaker 0: Hello!\\nSpeaker 1: Hi there!"], + voice_samples=[[voice]], return_tensors="pt") +audio = model.generate(**inputs, cfg_scale=1.3, + tokenizer=processor.tokenizer).speech_outputs[0] +sf.write("output.wav", audio.cpu().numpy().squeeze(), 24000)`, +]; +exports.vibevoice = vibevoice; +const videoprism = (model) => [ + `# Install from https://github.com/google-deepmind/videoprism +import jax +from videoprism import models as vp + +flax_model = vp.get_model("${model.id}") +loaded_state = vp.load_pretrained_weights("${model.id}") + +@jax.jit +def forward_fn(inputs, train=False): + return flax_model.apply(loaded_state, inputs, train=train)`, +]; +exports.videoprism = videoprism; +const vfimamba = (model) => [ + `from Trainer_finetune import Model + +model = Model.from_pretrained("${model.id}")`, +]; +exports.vfimamba = vfimamba; +const lvface = (model) => [ + `from huggingface_hub import hf_hub_download + from inference_onnx import LVFaceONNXInferencer + +model_path = hf_hub_download("${model.id}", "LVFace-L_Glint360K/LVFace-L_Glint360K.onnx") +inferencer = LVFaceONNXInferencer(model_path, use_gpu=True, timeout=300) +img_path = 'path/to/image1.jpg' +embedding = inferencer.infer_from_image(img_path)`, +]; +exports.lvface = lvface; +const voicecraft = (model) => [ + `from voicecraft import VoiceCraft + +model = VoiceCraft.from_pretrained("${model.id}")`, +]; +exports.voicecraft = voicecraft; +const voxcpm = (model) => [ + `import soundfile as sf +from voxcpm import VoxCPM + +model = VoxCPM.from_pretrained("${model.id}") + +wav = model.generate( + text="VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly expressive speech.", + prompt_wav_path=None, # optional: path to a prompt speech for voice cloning + prompt_text=None, # optional: reference text + cfg_value=2.0, # LM guidance on LocDiT, higher for better adherence to the prompt, but maybe worse + inference_timesteps=10, # LocDiT inference timesteps, higher for better result, lower for fast speed + normalize=True, # enable external TN tool + denoise=True, # enable external Denoise tool + retry_badcase=True, # enable retrying mode for some bad cases (unstoppable) + retry_badcase_max_times=3, # maximum retrying times + retry_badcase_ratio_threshold=6.0, # maximum length restriction for bad case detection (simple but effective), it could be adjusted for slow pace speech +) + +sf.write("output.wav", wav, 16000) +print("saved: output.wav")`, +]; +exports.voxcpm = voxcpm; +const vui = () => [ + `# !pip install git+https://github.com/fluxions-ai/vui + +import torchaudio + +from vui.inference import render +from vui.model import Vui, + +model = Vui.from_pretrained().cuda() +waveform = render( + model, + "Hey, here is some random stuff, usually something quite long as the shorter the text the less likely the model can cope!", +) +print(waveform.shape) +torchaudio.save("out.opus", waveform[0], 22050) +`, +]; +exports.vui = vui; +const chattts = () => [ + `import ChatTTS +import torchaudio + +chat = ChatTTS.Chat() +chat.load_models(compile=False) # Set to True for better performance + +texts = ["PUT YOUR TEXT HERE",] + +wavs = chat.infer(texts, ) + +torchaudio.save("output1.wav", torch.from_numpy(wavs[0]), 24000)`, +]; +exports.chattts = chattts; +const ultralytics = (model) => { + // ultralytics models must have a version tag (e.g. `yolov8`) + const versionTag = model.tags.find((tag) => tag.match(/^yolov\d+$/)); + const className = versionTag ? `YOLOv${versionTag.slice(4)}` : "YOLOvXX"; + const prefix = versionTag + ? "" + : `# Couldn't find a valid YOLO version tag.\n# Replace XX with the correct version.\n`; + return [ + prefix + + `from ultralytics import ${className} + +model = ${className}.from_pretrained("${model.id}") +source = 'http://images.cocodataset.org/val2017/000000039769.jpg' +model.predict(source=source, save=True)`, + ]; +}; +exports.ultralytics = ultralytics; +const birefnet = (model) => [ + `# Option 1: use with transformers + +from transformers import AutoModelForImageSegmentation +birefnet = AutoModelForImageSegmentation.from_pretrained("${model.id}", trust_remote_code=True) +`, + `# Option 2: use with BiRefNet + +# Install from https://github.com/ZhengPeng7/BiRefNet + +from models.birefnet import BiRefNet +model = BiRefNet.from_pretrained("${model.id}")`, +]; +exports.birefnet = birefnet; +const supertonic = () => [ + `from supertonic import TTS + +tts = TTS(auto_download=True) + +style = tts.get_voice_style(voice_name="M1") + +text = "The train delay was announced at 4:45 PM on Wed, Apr 3, 2024 due to track maintenance." +wav, duration = tts.synthesize(text, voice_style=style) + +tts.save_audio(wav, "output.wav")`, +]; +exports.supertonic = supertonic; +const swarmformer = (model) => [ + `from swarmformer import SwarmFormerModel + +model = SwarmFormerModel.from_pretrained("${model.id}") +`, +]; +exports.swarmformer = swarmformer; +const univa = (model) => [ + `# Follow installation instructions at https://github.com/PKU-YuanGroup/UniWorld-V1 + +from univa.models.qwen2p5vl.modeling_univa_qwen2p5vl import UnivaQwen2p5VLForConditionalGeneration + model = UnivaQwen2p5VLForConditionalGeneration.from_pretrained( + "${model.id}", + torch_dtype=torch.bfloat16, + attn_implementation="flash_attention_2", + ).to("cuda") + processor = AutoProcessor.from_pretrained("${model.id}") +`, +]; +exports.univa = univa; +const mlx_unknown = (model) => [ + `# Download the model from the Hub +pip install huggingface_hub[hf_xet] + +huggingface-cli download --local-dir ${nameWithoutNamespace(model.id)} ${model.id}`, +]; +const mlxlm = (model) => [ + `# Make sure mlx-lm is installed +# pip install --upgrade mlx-lm +# if on a CUDA device, also pip install mlx[cuda] + +# Generate text with mlx-lm +from mlx_lm import load, generate + +model, tokenizer = load("${model.id}") + +prompt = "Once upon a time in" +text = generate(model, tokenizer, prompt=prompt, verbose=True)`, +]; +const mlxchat = (model) => [ + `# Make sure mlx-lm is installed +# pip install --upgrade mlx-lm + +# Generate text with mlx-lm +from mlx_lm import load, generate + +model, tokenizer = load("${model.id}") + +prompt = "Write a story about Einstein" +messages = [{"role": "user", "content": prompt}] +prompt = tokenizer.apply_chat_template( + messages, add_generation_prompt=True +) + +text = generate(model, tokenizer, prompt=prompt, verbose=True)`, +]; +const mlxvlm = (model) => [ + `# Make sure mlx-vlm is installed +# pip install --upgrade mlx-vlm + +from mlx_vlm import load, generate +from mlx_vlm.prompt_utils import apply_chat_template +from mlx_vlm.utils import load_config + +# Load the model +model, processor = load("${model.id}") +config = load_config("${model.id}") + +# Prepare input +image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] +prompt = "Describe this image." + +# Apply chat template +formatted_prompt = apply_chat_template( + processor, config, prompt, num_images=1 +) + +# Generate output +output = generate(model, processor, formatted_prompt, image) +print(output)`, +]; +const mlxim = (model) => [ + `from mlxim.model import create_model + +model = create_model(${model.id})`, +]; +exports.mlxim = mlxim; +const mlx = (model) => { + if (model.pipeline_tag === "image-text-to-text") { + return mlxvlm(model); + } + if (model.pipeline_tag === "text-generation") { + if (model.tags.includes("conversational")) { + return mlxchat(model); + } + else { + return mlxlm(model); + } + } + return mlx_unknown(model); +}; +exports.mlx = mlx; +const model2vec = (model) => [ + `from model2vec import StaticModel + +model = StaticModel.from_pretrained("${model.id}")`, +]; +exports.model2vec = model2vec; +const pruna = (model) => { + let snippets; + if (model.tags.includes("diffusers")) { + snippets = pruna_diffusers(model); + } + else if (model.tags.includes("transformers")) { + snippets = pruna_transformers(model); + } + else { + snippets = pruna_default(model); + } + const ensurePrunaModelImport = (snippet) => { + if (!/^from pruna import PrunaModel/m.test(snippet)) { + return `from pruna import PrunaModel\n${snippet}`; + } + return snippet; + }; + snippets = snippets.map(ensurePrunaModelImport); + if (model.tags.includes("pruna_pro-ai")) { + return snippets.map((snippet) => snippet.replace(/\bpruna\b/g, "pruna_pro").replace(/\bPrunaModel\b/g, "PrunaProModel")); + } + return snippets; +}; +exports.pruna = pruna; +const pruna_diffusers = (model) => { + const diffusersSnippets = (0, exports.diffusers)(model); + return diffusersSnippets.map((snippet) => snippet + // Replace pipeline classes with PrunaModel + .replace(/\b\w*Pipeline\w*\b/g, "PrunaModel") + // Clean up diffusers imports containing PrunaModel + .replace(/from diffusers import ([^,\n]*PrunaModel[^,\n]*)/g, "") + .replace(/from diffusers import ([^,\n]+),?\s*([^,\n]*PrunaModel[^,\n]*)/g, "from diffusers import $1") + .replace(/from diffusers import\s*(\n|$)/g, "") + // Fix PrunaModel imports + .replace(/from diffusers import PrunaModel/g, "from pruna import PrunaModel") + .replace(/from diffusers import ([^,\n]+), PrunaModel/g, "from diffusers import $1") + .replace(/from diffusers import PrunaModel, ([^,\n]+)/g, "from diffusers import $1") + // Clean up whitespace + .replace(/\n\n+/g, "\n") + .trim()); +}; +const pruna_transformers = (model) => { + const info = model.transformersInfo; + const transformersSnippets = (0, exports.transformers)(model); + // Replace pipeline with PrunaModel + let processedSnippets = transformersSnippets.map((snippet) => snippet + .replace(/from transformers import pipeline/g, "from pruna import PrunaModel") + .replace(/pipeline\([^)]*\)/g, `PrunaModel.from_pretrained("${model.id}")`)); + // Additional cleanup if auto_model info is available + if (info?.auto_model) { + processedSnippets = processedSnippets.map((snippet) => snippet + .replace(new RegExp(`from transformers import ${info.auto_model}\n?`, "g"), "") + .replace(new RegExp(`${info.auto_model}.from_pretrained`, "g"), "PrunaModel.from_pretrained") + .replace(new RegExp(`^.*from.*import.*(, *${info.auto_model})+.*$`, "gm"), (line) => line.replace(new RegExp(`, *${info.auto_model}`, "g"), ""))); + } + return processedSnippets; +}; +const pruna_default = (model) => [ + `from pruna import PrunaModel +model = PrunaModel.from_pretrained("${model.id}") +`, +]; +const nemo = (model) => { + let command = undefined; + // Resolve the tag to a nemo domain/sub-domain + if (model.tags.includes("automatic-speech-recognition")) { + command = nemoDomainResolver("ASR", model); + } + return command ?? [`# tag did not correspond to a valid NeMo domain.`]; +}; +exports.nemo = nemo; +const outetts = (model) => { + // Don’t show this block on GGUF / ONNX mirrors + const t = model.tags ?? []; + if (t.includes("gguf") || t.includes("onnx")) { + return []; + } + // v1.0 HF → minimal runnable snippet + return [ + ` + import outetts + + enum = outetts.Models("${model.id}".split("/", 1)[1]) # VERSION_1_0_SIZE_1B + cfg = outetts.ModelConfig.auto_config(enum, outetts.Backend.HF) + tts = outetts.Interface(cfg) + + speaker = tts.load_default_speaker("EN-FEMALE-1-NEUTRAL") + tts.generate( + outetts.GenerationConfig( + text="Hello there, how are you doing?", + speaker=speaker, + ) + ).save("output.wav") + `, + ]; +}; +exports.outetts = outetts; +const pxia = (model) => [ + `from pxia import AutoModel + +model = AutoModel.from_pretrained("${model.id}")`, +]; +exports.pxia = pxia; +const pythae = (model) => [ + `from pythae.models import AutoModel + +model = AutoModel.load_from_hf_hub("${model.id}")`, +]; +exports.pythae = pythae; +const qwen3_tts = (model) => [ + `# pip install qwen-tts +import torch +import soundfile as sf +from qwen_tts import Qwen3TTSModel + +model = Qwen3TTSModel.from_pretrained( + "${model.id}", + device_map="cuda:0", + dtype=torch.bfloat16, + attn_implementation="flash_attention_2", +) + +wavs, sr = model.generate_custom_voice( + text="Your text here.", + language="English", + speaker="Ryan", + instruct="Speak in a natural tone.", +) + +sf.write("output.wav", wavs[0], sr)`, +]; +exports.qwen3_tts = qwen3_tts; +const musicgen = (model) => [ + `from audiocraft.models import MusicGen + +model = MusicGen.get_pretrained("${model.id}") + +descriptions = ['happy rock', 'energetic EDM', 'sad jazz'] +wav = model.generate(descriptions) # generates 3 samples.`, +]; +const magnet = (model) => [ + `from audiocraft.models import MAGNeT + +model = MAGNeT.get_pretrained("${model.id}") + +descriptions = ['disco beat', 'energetic EDM', 'funky groove'] +wav = model.generate(descriptions) # generates 3 samples.`, +]; +const audiogen = (model) => [ + `from audiocraft.models import AudioGen + +model = AudioGen.get_pretrained("${model.id}") +model.set_generation_params(duration=5) # generate 5 seconds. +descriptions = ['dog barking', 'sirene of an emergency vehicle', 'footsteps in a corridor'] +wav = model.generate(descriptions) # generates 3 samples.`, +]; +const anemoi = (model) => [ + `from anemoi.inference.runners.default import DefaultRunner +from anemoi.inference.config.run import RunConfiguration +# Create Configuration +config = RunConfiguration(checkpoint = {"huggingface":"${model.id}"}) +# Load Runner +runner = DefaultRunner(config)`, +]; +exports.anemoi = anemoi; +const audiocraft = (model) => { + if (model.tags.includes("musicgen")) { + return musicgen(model); + } + else if (model.tags.includes("audiogen")) { + return audiogen(model); + } + else if (model.tags.includes("magnet")) { + return magnet(model); + } + else { + return [`# Type of model unknown.`]; + } +}; +exports.audiocraft = audiocraft; +const whisperkit = () => [ + `# Install CLI with Homebrew on macOS device +brew install whisperkit-cli + +# View all available inference options +whisperkit-cli transcribe --help + +# Download and run inference using whisper base model +whisperkit-cli transcribe --audio-path /path/to/audio.mp3 + +# Or use your preferred model variant +whisperkit-cli transcribe --model "large-v3" --model-prefix "distil" --audio-path /path/to/audio.mp3 --verbose`, +]; +exports.whisperkit = whisperkit; +const threedtopia_xl = (model) => [ + `from threedtopia_xl.models import threedtopia_xl + +model = threedtopia_xl.from_pretrained("${model.id}") +model.generate(cond="path/to/image.png")`, +]; +exports.threedtopia_xl = threedtopia_xl; +const hezar = (model) => [ + `from hezar import Model + +model = Model.load("${model.id}")`, +]; +exports.hezar = hezar; +const zonos = (model) => [ + `# pip install git+https://github.com/Zyphra/Zonos.git +import torchaudio +from zonos.model import Zonos +from zonos.conditioning import make_cond_dict + +model = Zonos.from_pretrained("${model.id}", device="cuda") + +wav, sr = torchaudio.load("speaker.wav") # 5-10s reference clip +speaker = model.make_speaker_embedding(wav, sr) + +cond = make_cond_dict(text="Hello, world!", speaker=speaker, language="en-us") +codes = model.generate(model.prepare_conditioning(cond)) + +audio = model.autoencoder.decode(codes)[0].cpu() +torchaudio.save("sample.wav", audio, model.autoencoder.sampling_rate) +`, +]; +exports.zonos = zonos; +const moshi = (model) => { + // Detect backend from model name (no distinguishing tags available) + if (model.id.includes("-mlx")) { + // MLX backend (macOS Apple Silicon) + // -q flag only accepts 4 or 8, bf16 models don't use it + const quantFlag = model.id.includes("-q4") ? " -q 4" : model.id.includes("-q8") ? " -q 8" : ""; + return [ + `# pip install moshi_mlx +# Run local inference (macOS Apple Silicon) +python -m moshi_mlx.local${quantFlag} --hf-repo "${model.id}" + +# Or run with web UI +python -m moshi_mlx.local_web${quantFlag} --hf-repo "${model.id}"`, + ]; + } + if (model.id.includes("-candle")) { + // Rust/Candle backend + return [ + `# pip install rustymimi +# Candle backend - see https://github.com/kyutai-labs/moshi +# for Rust installation instructions`, + ]; + } + // PyTorch backend (default) + return [ + `# pip install moshi +# Run the interactive web server +python -m moshi.server --hf-repo "${model.id}" +# Then open https://localhost:8998 in your browser`, + `# pip install moshi +import torch +from moshi.models import loaders + +# Load checkpoint info from HuggingFace +checkpoint = loaders.CheckpointInfo.from_hf_repo("${model.id}") + +# Load the Mimi audio codec +mimi = checkpoint.get_mimi(device="cuda") +mimi.set_num_codebooks(8) + +# Encode audio (24kHz, mono) +wav = torch.randn(1, 1, 24000 * 10) # [batch, channels, samples] +with torch.no_grad(): + codes = mimi.encode(wav.cuda()) + decoded = mimi.decode(codes)`, + ]; +}; +exports.moshi = moshi; +//#endregion diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.spec.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..f4992762e83f6647ee2d130b8b7dc9a66c0d93af --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=model-libraries-snippets.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.spec.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c78932806acc5f8931c62472c22ccb33afa974da --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"model-libraries-snippets.spec.d.ts","sourceRoot":"","sources":["../../src/model-libraries-snippets.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.spec.js b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.spec.js new file mode 100644 index 0000000000000000000000000000000000000000..8c6a4b3b4d7aedcbaf53e6ae141cf29fec599370 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries-snippets.spec.js @@ -0,0 +1,55 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const vitest_1 = require("vitest"); +const model_libraries_snippets_js_1 = require("./model-libraries-snippets.js"); +(0, vitest_1.describe)("model-libraries-snippets", () => { + (0, vitest_1.it)("llama_cpp_python conversational", async () => { + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + pipeline_tag: "text-generation", + tags: ["conversational"], + inference: "", + }; + const snippet = (0, model_libraries_snippets_js_1.llama_cpp_python)(model); + (0, vitest_1.expect)(snippet.join("\n")).toEqual(`# !pip install llama-cpp-python + +from llama_cpp import Llama + +llm = Llama.from_pretrained( + repo_id="bartowski/Llama-3.2-3B-Instruct-GGUF", + filename="{{GGUF_FILE}}", +) + +llm.create_chat_completion( + messages = [ + { + "role": "user", + "content": "What is the capital of France?" + } + ] +)`); + }); + (0, vitest_1.it)("llama_cpp_python non-conversational", async () => { + const model = { + id: "mlabonne/gemma-2b-GGUF", + tags: [""], + inference: "", + }; + const snippet = (0, model_libraries_snippets_js_1.llama_cpp_python)(model); + (0, vitest_1.expect)(snippet.join("\n")).toEqual(`# !pip install llama-cpp-python + +from llama_cpp import Llama + +llm = Llama.from_pretrained( + repo_id="mlabonne/gemma-2b-GGUF", + filename="{{GGUF_FILE}}", +) + +output = llm( + "Once upon a time,", + max_tokens=512, + echo=True +) +print(output)`); + }); +}); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-libraries.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..320f1da3af988daf046b9ba44473b8b0c11c32a1 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries.d.ts @@ -0,0 +1,1705 @@ +import type { ModelData } from "./model-data.js"; +import type { ElasticSearchQuery } from "./model-libraries-downloads.js"; +/** + * Elements configurable by a model library. + */ +export interface LibraryUiElement { + /** + * Pretty name of the library. + * displayed in tags, and on the main + * call-to-action button on the model page. + */ + prettyLabel: string; + /** + * Repo name of the library's (usually on GitHub) code repo + */ + repoName: string; + /** + * URL to library's (usually on GitHub) code repo + */ + repoUrl: string; + /** + * URL to library's docs + */ + docsUrl?: string; + /** + * Code snippet(s) displayed on model page + */ + snippets?: (model: ModelData) => string[]; + /** + * Elastic query used to count this library's model downloads + * + * By default, those files are counted: + * "config.json", "config.yaml", "hyperparams.yaml", "params.json", "meta.yaml" + */ + countDownloads?: ElasticSearchQuery; + /** + * should we display this library in hf.co/models filter + * (only for popular libraries with > 100 models) + */ + filter?: boolean; +} +/** + * Add your new library here. + * + * This is for modeling (= architectures) libraries, not for file formats (like ONNX, etc). + * (unlike libraries, file formats live in an enum inside the internal codebase.) + * + * Doc on how to add a library to the Hub: + * + * https://huggingface.co/docs/hub/models-adding-libraries + * + * /!\ IMPORTANT + * + * The key you choose is the tag your models have in their library_name on the Hub. + */ +export declare const MODEL_LIBRARIES_UI_ELEMENTS: { + acestep: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "adapter-transformers": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + allennlp: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + anemoi: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + araclip: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + }; + "aviation-ner": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + countDownloads: string; + filter: false; + }; + asteroid: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + audiocraft: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + audioseal: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + "bagel-mot": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + bboxmaskpose: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + ben2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + bertopic: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + big_vision: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + bionemo: { + prettyLabel: string; + repoName: string; + filter: false; + repoUrl: string; + countDownloads: string; + }; + birder: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + birefnet: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + bm25s: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + boltzgen: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + cancertathomev2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + cartesia_pytorch: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + cartesia_mlx: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + champ: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + chatterbox: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: () => string[]; + countDownloads: string; + filter: false; + }; + chaossim: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + filter: false; + }; + chat_tts: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: () => string[]; + filter: false; + countDownloads: string; + }; + chexmix: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "chronos-forecasting": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + clara: { + prettyLabel: string; + repoName: string; + filter: false; + repoUrl: string; + countDownloads: string; + }; + clipscope: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "cloud-agents": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + collectorvision: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + colipri: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + cosyvoice: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + cotracker: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + colpali: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + comet: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + cosmos: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "cxr-foundation": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: () => string[]; + filter: false; + countDownloads: string; + }; + deepforest: { + prettyLabel: string; + repoName: string; + docsUrl: string; + repoUrl: string; + }; + "depth-anything-v2": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + "depth-pro": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "derm-foundation": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: () => string[]; + filter: false; + countDownloads: string; + }; + "describe-anything": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "dia-tts": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + dia2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "diff-interpretation-tuning": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + diffree: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + diffusers: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + diffusionkit: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + "docking-at-home": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + doctr: { + prettyLabel: string; + repoName: string; + repoUrl: string; + }; + edsnlp: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + countDownloads: string; + }; + elm: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + encoderfile: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + espnet: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + eupe: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + fairseq: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + fastai: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + fastprint: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + fasttext: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + fixer: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + flair: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + fme: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + }; + "gemma.cpp": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "geometry-crafter": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + gliner: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + gliner2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "glm-tts": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "glyph-byt5": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "granite-library": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + grok: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "habibi-tts": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + hallo: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + hermes: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + holomotion: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + hezar: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + countDownloads: string; + }; + htrflow: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + }; + "hunyuan-dit": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "hunyuan3d-2": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "hunyuanworld-voyager": { + prettyLabel: string; + repoName: string; + repoUrl: string; + }; + "hy-worldplay": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "hy-world-2": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "image-matching-models": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + imstoucan: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "index-tts": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + infinitetalk: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "infinite-you": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + intellifold: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "ising-decoding": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + keras: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "tf-keras": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + countDownloads: string; + }; + "keras-hub": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + kernels: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + countDownloads: string; + }; + "kimi-audio": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + kittentts: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + kronos: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + k2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + }; + "lyra-2.0": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + lagernvs: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "lightning-ir": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + litert: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "litert-lm": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + lerobot: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + }; + lightglue: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + liveportrait: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "longcat-video-avatar-1.5": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + }; + "llama-cpp-python": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + "mini-omni2": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + mindspore: { + prettyLabel: string; + repoName: string; + repoUrl: string; + }; + "magi-1": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "magenta-realtime": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "magenta-realtime-2": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "mamba-ssm": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + }; + "manas-1": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "mars5-tts": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + matanyone: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "mesh-anything": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + snippets: () => string[]; + }; + merlin: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + medvae: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + mitie: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "ml-agents": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "ml-sharp": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + mlx: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + "mlx-image": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + "mlc-llm": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + }; + model2vec: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + moshi: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + mtvcraft: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + multimolecule: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + nemo: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "nv-medtech": { + prettyLabel: string; + repoName: string; + filter: false; + repoUrl: string; + countDownloads: string; + }; + "open-oasis": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + open_clip: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + openpeerllm: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + countDownloads: string; + filter: false; + }; + "open-sora": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + outetts: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + paddlenlp: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + PaddleOCR: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + peft: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "perception-encoder": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + countDownloads: string; + }; + "phantom-wan": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + "pocket-tts": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + "pruna-ai": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + docsUrl: string; + }; + pxia: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "pyannote-audio": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + "py-feat": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + }; + pythae: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + quantumpeer: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + qwen3_tts: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + recurrentgemma: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + relik: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + refiners: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + }; + renderformer: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + reverb: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + }; + rkllm: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "robo-orchard-lab": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + rwkv: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + }; + saelens: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: () => string[]; + filter: false; + }; + "scail-2": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + sam2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + countDownloads: string; + }; + "sam-3d-body": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + countDownloads: string; + }; + "sam-3d-objects": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + countDownloads: string; + }; + same: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "sample-factory": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "sap-rpt-1-oss": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + snippets: () => string[]; + }; + sapiens: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + sapiens2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + seedvr: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "self-forcing": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "sentence-transformers": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + setfit: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + sklearn: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + spacy: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "span-marker": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + speechbrain: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "ssr-speech": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "stable-audio-3": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "stable-audio-tools": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + monkeyocr: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "diffusion-single-file": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "seed-story": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + snippets: () => string[]; + }; + skala: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + soloaudio: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + songbloom: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "stable-baselines3": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + stanza: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + supertonic: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: () => string[]; + filter: false; + }; + swarmformer: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "synthefy-migas": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "f5-tts": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + genmo: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "tencent-song-generation": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + tensorflowtts: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + tensorrt: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + tabpfn: { + prettyLabel: string; + repoName: string; + repoUrl: string; + }; + terratorch: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + "tic-clip": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + timesfm: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + timm: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + tirex: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + torchgeo: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + }; + transformers: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + "transformers.js": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + trellis: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + trellis2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + tunejury: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + ultralytics: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + univa: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "uni-3dar": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + countDownloads: string; + }; + "unity-sentis": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: () => string[]; + filter: true; + countDownloads: string; + }; + sana: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + videoprism: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + "vfi-mamba": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + vismatch: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + lvface: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + voicecraft: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + }; + voxcpm: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + vui: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + snippets: () => string[]; + }; + vibevoice: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + videox_fun: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "wan2.2": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + wham: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + countDownloads: string; + }; + whisperkit: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: () => string[]; + countDownloads: string; + }; + yolov10: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + yolov26: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + countDownloads: string; + }; + zonos: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "3dtopia-xl": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; +}; +export type ModelLibraryKey = keyof typeof MODEL_LIBRARIES_UI_ELEMENTS; +export declare const ALL_MODEL_LIBRARY_KEYS: ModelLibraryKey[]; +export declare const ALL_DISPLAY_MODEL_LIBRARY_KEYS: ("acestep" | "adapter-transformers" | "allennlp" | "anemoi" | "araclip" | "aviation-ner" | "asteroid" | "audiocraft" | "audioseal" | "bagel-mot" | "bboxmaskpose" | "ben2" | "bertopic" | "big_vision" | "bionemo" | "birder" | "birefnet" | "bm25s" | "boltzgen" | "cancertathomev2" | "cartesia_pytorch" | "cartesia_mlx" | "champ" | "chatterbox" | "chaossim" | "chat_tts" | "chexmix" | "chronos-forecasting" | "clara" | "clipscope" | "cloud-agents" | "collectorvision" | "colipri" | "cosyvoice" | "cotracker" | "colpali" | "comet" | "cosmos" | "cxr-foundation" | "deepforest" | "depth-anything-v2" | "depth-pro" | "derm-foundation" | "describe-anything" | "dia-tts" | "dia2" | "diff-interpretation-tuning" | "diffree" | "diffusers" | "diffusionkit" | "docking-at-home" | "doctr" | "edsnlp" | "elm" | "encoderfile" | "espnet" | "eupe" | "fairseq" | "fastai" | "fastprint" | "fasttext" | "fixer" | "flair" | "fme" | "gemma.cpp" | "geometry-crafter" | "gliner" | "gliner2" | "glm-tts" | "glyph-byt5" | "granite-library" | "grok" | "habibi-tts" | "hallo" | "hermes" | "holomotion" | "hezar" | "htrflow" | "hunyuan-dit" | "hunyuan3d-2" | "hunyuanworld-voyager" | "hy-worldplay" | "hy-world-2" | "image-matching-models" | "imstoucan" | "index-tts" | "infinitetalk" | "infinite-you" | "intellifold" | "ising-decoding" | "keras" | "tf-keras" | "keras-hub" | "kernels" | "kimi-audio" | "kittentts" | "kronos" | "k2" | "lyra-2.0" | "lagernvs" | "lightning-ir" | "litert" | "litert-lm" | "lerobot" | "lightglue" | "liveportrait" | "longcat-video-avatar-1.5" | "llama-cpp-python" | "mini-omni2" | "mindspore" | "magi-1" | "magenta-realtime" | "magenta-realtime-2" | "mamba-ssm" | "manas-1" | "mars5-tts" | "matanyone" | "mesh-anything" | "merlin" | "medvae" | "mitie" | "ml-agents" | "ml-sharp" | "mlx" | "mlx-image" | "mlc-llm" | "model2vec" | "moshi" | "mtvcraft" | "multimolecule" | "nemo" | "nv-medtech" | "open-oasis" | "open_clip" | "openpeerllm" | "open-sora" | "outetts" | "paddlenlp" | "PaddleOCR" | "peft" | "perception-encoder" | "phantom-wan" | "pocket-tts" | "pruna-ai" | "pxia" | "pyannote-audio" | "py-feat" | "pythae" | "quantumpeer" | "qwen3_tts" | "recurrentgemma" | "relik" | "refiners" | "renderformer" | "reverb" | "rkllm" | "robo-orchard-lab" | "rwkv" | "saelens" | "scail-2" | "sam2" | "sam-3d-body" | "sam-3d-objects" | "same" | "sample-factory" | "sap-rpt-1-oss" | "sapiens" | "sapiens2" | "seedvr" | "self-forcing" | "sentence-transformers" | "setfit" | "sklearn" | "spacy" | "span-marker" | "speechbrain" | "ssr-speech" | "stable-audio-3" | "stable-audio-tools" | "monkeyocr" | "diffusion-single-file" | "seed-story" | "skala" | "soloaudio" | "songbloom" | "stable-baselines3" | "stanza" | "supertonic" | "swarmformer" | "synthefy-migas" | "f5-tts" | "genmo" | "tencent-song-generation" | "tensorflowtts" | "tensorrt" | "tabpfn" | "terratorch" | "tic-clip" | "timesfm" | "timm" | "tirex" | "torchgeo" | "transformers" | "transformers.js" | "trellis" | "trellis2" | "tunejury" | "ultralytics" | "univa" | "uni-3dar" | "unity-sentis" | "sana" | "videoprism" | "vfi-mamba" | "vismatch" | "lvface" | "voicecraft" | "voxcpm" | "vui" | "vibevoice" | "videox_fun" | "wan2.2" | "wham" | "whisperkit" | "yolov10" | "yolov26" | "zonos" | "3dtopia-xl")[]; +//# sourceMappingURL=model-libraries.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-libraries.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f6ce894237e76719a0d6d8e17d9ab95c16bc841a --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"model-libraries.d.ts","sourceRoot":"","sources":["../../src/model-libraries.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAEzE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;;;OAIG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,MAAM,EAAE,CAAC;IAC1C;;;;;OAKG;IACH,cAAc,CAAC,EAAE,kBAAkB,CAAC;IACpC;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;;;;;;GAaG;AAEH,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAonDI,CAAC;AAE7C,MAAM,MAAM,eAAe,GAAG,MAAM,OAAO,2BAA2B,CAAC;AAEvE,eAAO,MAAM,sBAAsB,EAA+C,eAAe,EAAE,CAAC;AAEpG,eAAO,MAAM,8BAA8B,urGAQ1B,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/model-libraries.js b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries.js new file mode 100644 index 0000000000000000000000000000000000000000..068dbbc1b09fe45e7a38dc76c4896d6eb88c57e1 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/model-libraries.js @@ -0,0 +1,1708 @@ +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || (function () { + var ownKeys = function(o) { + ownKeys = Object.getOwnPropertyNames || function (o) { + var ar = []; + for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k; + return ar; + }; + return ownKeys(o); + }; + return function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]); + __setModuleDefault(result, mod); + return result; + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.ALL_DISPLAY_MODEL_LIBRARY_KEYS = exports.ALL_MODEL_LIBRARY_KEYS = exports.MODEL_LIBRARIES_UI_ELEMENTS = void 0; +const snippets = __importStar(require("./model-libraries-snippets.js")); +/** + * Add your new library here. + * + * This is for modeling (= architectures) libraries, not for file formats (like ONNX, etc). + * (unlike libraries, file formats live in an enum inside the internal codebase.) + * + * Doc on how to add a library to the Hub: + * + * https://huggingface.co/docs/hub/models-adding-libraries + * + * /!\ IMPORTANT + * + * The key you choose is the tag your models have in their library_name on the Hub. + */ +exports.MODEL_LIBRARIES_UI_ELEMENTS = { + acestep: { + prettyLabel: "ACE-Step", + repoName: "ACE-Step", + repoUrl: "https://github.com/ace-step/ACE-Step", + filter: false, + countDownloads: `path:"ace_step_transformer/config.json"`, + }, + "adapter-transformers": { + prettyLabel: "Adapters", + repoName: "adapters", + repoUrl: "https://github.com/Adapter-Hub/adapters", + docsUrl: "https://huggingface.co/docs/hub/adapters", + snippets: snippets.adapters, + filter: true, + countDownloads: `path:"adapter_config.json"`, + }, + allennlp: { + prettyLabel: "AllenNLP", + repoName: "AllenNLP", + repoUrl: "https://github.com/allenai/allennlp", + docsUrl: "https://huggingface.co/docs/hub/allennlp", + snippets: snippets.allennlp, + filter: true, + }, + anemoi: { + prettyLabel: "AnemoI", + repoName: "AnemoI", + repoUrl: "https://github.com/ecmwf/anemoi-inference", + docsUrl: "https://anemoi.readthedocs.io/en/latest/", + filter: false, + countDownloads: `path_extension:"ckpt"`, + snippets: snippets.anemoi, + }, + araclip: { + prettyLabel: "AraClip", + repoName: "AraClip", + repoUrl: "https://huggingface.co/Arabic-Clip/araclip", + filter: false, + snippets: snippets.araclip, + }, + "aviation-ner": { + prettyLabel: "Aviation NER", + repoName: "Aviation NER", + repoUrl: "https://github.com/Boeing/aviation_ner_sdr", + docsUrl: "https://github.com/Boeing/aviation_ner_sdr", + countDownloads: `path:"gliner_config.json"`, + filter: false, + }, + asteroid: { + prettyLabel: "Asteroid", + repoName: "Asteroid", + repoUrl: "https://github.com/asteroid-team/asteroid", + docsUrl: "https://huggingface.co/docs/hub/asteroid", + snippets: snippets.asteroid, + filter: true, + countDownloads: `path:"pytorch_model.bin"`, + }, + audiocraft: { + prettyLabel: "Audiocraft", + repoName: "audiocraft", + repoUrl: "https://github.com/facebookresearch/audiocraft", + snippets: snippets.audiocraft, + filter: false, + countDownloads: `path:"state_dict.bin"`, + }, + audioseal: { + prettyLabel: "AudioSeal", + repoName: "audioseal", + repoUrl: "https://github.com/facebookresearch/audioseal", + filter: false, + countDownloads: `path_extension:"pth"`, + snippets: snippets.audioseal, + }, + "bagel-mot": { + prettyLabel: "Bagel", + repoName: "Bagel", + repoUrl: "https://github.com/ByteDance-Seed/Bagel/", + filter: false, + countDownloads: `path:"llm_config.json"`, + }, + bboxmaskpose: { + prettyLabel: "BBoxMaskPose", + repoName: "BBoxMaskPose", + repoUrl: "https://github.com/MiraPurkrabek/BBoxMaskPose", + filter: false, + countDownloads: `path_extension:"pth"`, + }, + ben2: { + prettyLabel: "BEN2", + repoName: "BEN2", + repoUrl: "https://github.com/PramaLLC/BEN2", + snippets: snippets.ben2, + filter: false, + }, + bertopic: { + prettyLabel: "BERTopic", + repoName: "BERTopic", + repoUrl: "https://github.com/MaartenGr/BERTopic", + snippets: snippets.bertopic, + filter: true, + }, + big_vision: { + prettyLabel: "Big Vision", + repoName: "big_vision", + repoUrl: "https://github.com/google-research/big_vision", + filter: false, + countDownloads: `path_extension:"npz"`, + }, + bionemo: { + prettyLabel: "BioNeMo", + repoName: "BioNeMo", + filter: false, + repoUrl: "https://github.com/nvidia/BioNeMo", + countDownloads: `path_extension:"ckpt" OR path:"config.json"`, + }, + birder: { + prettyLabel: "Birder", + repoName: "Birder", + repoUrl: "https://gitlab.com/birder/birder", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + birefnet: { + prettyLabel: "BiRefNet", + repoName: "BiRefNet", + repoUrl: "https://github.com/ZhengPeng7/BiRefNet", + snippets: snippets.birefnet, + filter: false, + }, + bm25s: { + prettyLabel: "BM25S", + repoName: "bm25s", + repoUrl: "https://github.com/xhluca/bm25s", + snippets: snippets.bm25s, + filter: false, + countDownloads: `path:"params.index.json"`, + }, + boltzgen: { + prettyLabel: "BoltzGen", + repoName: "BoltzGen", + repoUrl: "https://github.com/HannesStark/boltzgen", + filter: false, + countDownloads: `path:"boltzgen1_diverse.ckpt"`, + }, + cancertathomev2: { + prettyLabel: "Cancer@HomeV2", + repoName: "Cancer@HomeV2", + repoUrl: "https://huggingface.co/OpenPeerAI/CancerAtHomeV2", + filter: false, + countDownloads: `path:"run.py"`, + }, + cartesia_pytorch: { + prettyLabel: "Cartesia Pytorch", + repoName: "Cartesia Pytorch", + repoUrl: "https://github.com/cartesia-ai/cartesia_pytorch", + snippets: snippets.cartesia_pytorch, + }, + cartesia_mlx: { + prettyLabel: "Cartesia MLX", + repoName: "Cartesia MLX", + repoUrl: "https://github.com/cartesia-ai/cartesia_mlx", + snippets: snippets.cartesia_mlx, + }, + champ: { + prettyLabel: "Champ", + repoName: "Champ", + repoUrl: "https://github.com/fudan-generative-vision/champ", + countDownloads: `path:"champ/motion_module.pth"`, + }, + chatterbox: { + prettyLabel: "Chatterbox", + repoName: "Chatterbox", + repoUrl: "https://github.com/resemble-ai/chatterbox", + snippets: snippets.chatterbox, + countDownloads: `path:"tokenizer.json"`, + filter: false, + }, + chaossim: { + prettyLabel: "ChaosSIM", + repoName: "ChaosSIM", + repoUrl: "https://huggingface.co/OpenPeerAI/ChaosSIM/", + countDownloads: `path:"ChaosSim.nb"`, + filter: false, + }, + chat_tts: { + prettyLabel: "ChatTTS", + repoName: "ChatTTS", + repoUrl: "https://github.com/2noise/ChatTTS.git", + snippets: snippets.chattts, + filter: false, + countDownloads: `path:"asset/GPT.pt"`, + }, + chexmix: { + prettyLabel: "CheXmix", + repoName: "CheXmix", + repoUrl: "https://github.com/StanfordMIMI/CheXmix", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + "chronos-forecasting": { + prettyLabel: "Chronos", + repoName: "Chronos", + repoUrl: "https://github.com/amazon-science/chronos-forecasting", + snippets: snippets.chronos_forecasting, + }, + clara: { + prettyLabel: "Clara", + repoName: "Clara", + filter: false, + repoUrl: "https://github.com/nvidia/clara", + countDownloads: `path_extension:"ckpt" OR path:"config.json"`, + }, + clipscope: { + prettyLabel: "clipscope", + repoName: "clipscope", + repoUrl: "https://github.com/Lewington-pitsos/clipscope", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + "cloud-agents": { + prettyLabel: "Cloud Agents", + repoName: "Cloud Agents", + repoUrl: "https://huggingface.co/OpenPeerAI/Cloud-Agents", + filter: false, + countDownloads: `path:"setup.py"`, + }, + collectorvision: { + prettyLabel: "CollectorVision", + repoName: "CollectorVision", + repoUrl: "https://github.com/HanClinto/CollectorVision", + snippets: snippets.collectorvision, + filter: false, + countDownloads: `path_extension:"onnx"`, + }, + colipri: { + prettyLabel: "COLIPRI", + repoName: "COLIPRI", + repoUrl: "https://huggingface.co/microsoft/colipri", + snippets: snippets.colipri, + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + cosyvoice: { + prettyLabel: "CosyVoice", + repoName: "CosyVoice", + repoUrl: "https://github.com/FunAudioLLM/CosyVoice", + filter: false, + countDownloads: `path_extension:"onnx" OR path_extension:"pt"`, + }, + cotracker: { + prettyLabel: "CoTracker", + repoName: "CoTracker", + repoUrl: "https://github.com/facebookresearch/co-tracker", + filter: false, + countDownloads: `path_extension:"pth"`, + }, + colpali: { + prettyLabel: "ColPali", + repoName: "ColPali", + repoUrl: "https://github.com/ManuelFay/colpali", + filter: false, + countDownloads: `path:"adapter_config.json"`, + }, + comet: { + prettyLabel: "COMET", + repoName: "COMET", + repoUrl: "https://github.com/Unbabel/COMET/", + countDownloads: `path:"hparams.yaml"`, + }, + cosmos: { + prettyLabel: "Cosmos", + repoName: "Cosmos", + repoUrl: "https://github.com/NVIDIA/Cosmos", + countDownloads: `path:"config.json" OR path_extension:"pt"`, + }, + "cxr-foundation": { + prettyLabel: "CXR Foundation", + repoName: "cxr-foundation", + repoUrl: "https://github.com/google-health/cxr-foundation", + snippets: snippets.cxr_foundation, + filter: false, + countDownloads: `path:"precomputed_embeddings/embeddings.npz" OR path:"pax-elixr-b-text/saved_model.pb"`, + }, + deepforest: { + prettyLabel: "DeepForest", + repoName: "deepforest", + docsUrl: "https://deepforest.readthedocs.io/en/latest/", + repoUrl: "https://github.com/weecology/DeepForest", + }, + "depth-anything-v2": { + prettyLabel: "DepthAnythingV2", + repoName: "Depth Anything V2", + repoUrl: "https://github.com/DepthAnything/Depth-Anything-V2", + snippets: snippets.depth_anything_v2, + filter: false, + countDownloads: `path_extension:"pth"`, + }, + "depth-pro": { + prettyLabel: "Depth Pro", + repoName: "Depth Pro", + repoUrl: "https://github.com/apple/ml-depth-pro", + countDownloads: `path_extension:"pt"`, + snippets: snippets.depth_pro, + filter: false, + }, + "derm-foundation": { + prettyLabel: "Derm Foundation", + repoName: "derm-foundation", + repoUrl: "https://github.com/google-health/derm-foundation", + snippets: snippets.derm_foundation, + filter: false, + countDownloads: `path:"scin_dataset_precomputed_embeddings.npz" OR path:"saved_model.pb"`, + }, + "describe-anything": { + prettyLabel: "Describe Anything", + repoName: "Describe Anything", + repoUrl: "https://github.com/NVlabs/describe-anything", + snippets: snippets.describe_anything, + filter: false, + }, + "dia-tts": { + prettyLabel: "Dia", + repoName: "Dia", + repoUrl: "https://github.com/nari-labs/dia", + snippets: snippets.dia, + filter: false, + }, + dia2: { + prettyLabel: "Dia2", + repoName: "Dia2", + repoUrl: "https://github.com/nari-labs/dia2", + snippets: snippets.dia2, + filter: false, + }, + "diff-interpretation-tuning": { + prettyLabel: "Diff Interpretation Tuning", + repoName: "Diff Interpretation Tuning", + repoUrl: "https://github.com/Aviously/diff-interpretation-tuning", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + diffree: { + prettyLabel: "Diffree", + repoName: "Diffree", + repoUrl: "https://github.com/OpenGVLab/Diffree", + filter: false, + countDownloads: `path:"diffree-step=000010999.ckpt"`, + }, + diffusers: { + prettyLabel: "Diffusers", + repoName: "🤗/diffusers", + repoUrl: "https://github.com/huggingface/diffusers", + docsUrl: "https://huggingface.co/docs/hub/diffusers", + snippets: snippets.diffusers, + filter: true, + /// diffusers has its own more complex "countDownloads" query + }, + diffusionkit: { + prettyLabel: "DiffusionKit", + repoName: "DiffusionKit", + repoUrl: "https://github.com/argmaxinc/DiffusionKit", + snippets: snippets.diffusionkit, + }, + "docking-at-home": { + prettyLabel: "Docking@Home", + repoName: "Docking@Home", + repoUrl: "https://huggingface.co/OpenPeerAI/DockingAtHOME", + filter: false, + countDownloads: `path:"setup.py"`, + }, + doctr: { + prettyLabel: "docTR", + repoName: "doctr", + repoUrl: "https://github.com/mindee/doctr", + }, + edsnlp: { + prettyLabel: "EDS-NLP", + repoName: "edsnlp", + repoUrl: "https://github.com/aphp/edsnlp", + docsUrl: "https://aphp.github.io/edsnlp/latest/", + filter: false, + snippets: snippets.edsnlp, + countDownloads: `path_filename:"config" AND path_extension:"cfg"`, + }, + elm: { + prettyLabel: "ELM", + repoName: "elm", + repoUrl: "https://github.com/slicex-ai/elm", + filter: false, + countDownloads: `path_filename:"slicex_elm_config" AND path_extension:"json"`, + }, + encoderfile: { + prettyLabel: "encoderfile", + repoName: "encoderfile", + repoUrl: "https://github.com/mozilla-ai/encoderfile", + filter: false, + countDownloads: `path_extension:"encoderfile"`, + }, + espnet: { + prettyLabel: "ESPnet", + repoName: "ESPnet", + repoUrl: "https://github.com/espnet/espnet", + docsUrl: "https://huggingface.co/docs/hub/espnet", + snippets: snippets.espnet, + filter: true, + }, + eupe: { + prettyLabel: "EUPE", + repoName: "EUPE", + repoUrl: "https://github.com/facebookresearch/EUPE", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + fairseq: { + prettyLabel: "Fairseq", + repoName: "fairseq", + repoUrl: "https://github.com/pytorch/fairseq", + snippets: snippets.fairseq, + filter: true, + }, + fastai: { + prettyLabel: "fastai", + repoName: "fastai", + repoUrl: "https://github.com/fastai/fastai", + docsUrl: "https://huggingface.co/docs/hub/fastai", + snippets: snippets.fastai, + filter: true, + }, + fastprint: { + prettyLabel: "Fast Print", + repoName: "Fast Print", + repoUrl: "https://huggingface.co/OpenPeerAI/FastPrint", + countDownloads: `path_extension:"cs"`, + }, + fasttext: { + prettyLabel: "fastText", + repoName: "fastText", + repoUrl: "https://fasttext.cc/", + snippets: snippets.fasttext, + filter: true, + countDownloads: `path_extension:"bin"`, + }, + fixer: { + prettyLabel: "Fixer", + repoName: "Fixer", + repoUrl: "https://github.com/nv-tlabs/Fixer", + filter: false, + countDownloads: `path:"pretrained/pretrained_fixer.pkl"`, + }, + flair: { + prettyLabel: "Flair", + repoName: "Flair", + repoUrl: "https://github.com/flairNLP/flair", + docsUrl: "https://huggingface.co/docs/hub/flair", + snippets: snippets.flair, + filter: true, + countDownloads: `path:"pytorch_model.bin"`, + }, + fme: { + prettyLabel: "Full Model Emulation", + repoName: "Full Model Emulation", + repoUrl: "https://github.com/ai2cm/ace", + docsUrl: "https://ai2-climate-emulator.readthedocs.io/en/latest/", + filter: false, + countDownloads: `path_extension:"tar"`, + }, + "gemma.cpp": { + prettyLabel: "gemma.cpp", + repoName: "gemma.cpp", + repoUrl: "https://github.com/google/gemma.cpp", + filter: false, + countDownloads: `path_extension:"sbs"`, + }, + "geometry-crafter": { + prettyLabel: "GeometryCrafter", + repoName: "GeometryCrafter", + repoUrl: "https://github.com/TencentARC/GeometryCrafter", + countDownloads: `path:"point_map_vae/diffusion_pytorch_model.safetensors"`, + }, + gliner: { + prettyLabel: "GLiNER", + repoName: "GLiNER", + repoUrl: "https://github.com/urchade/GLiNER", + snippets: snippets.gliner, + filter: false, + countDownloads: `path:"gliner_config.json"`, + }, + gliner2: { + prettyLabel: "GLiNER2", + repoName: "GLiNER2", + repoUrl: "https://github.com/fastino-ai/GLiNER2", + snippets: snippets.gliner2, + filter: false, + }, + "glm-tts": { + prettyLabel: "GLM-TTS", + repoName: "GLM-TTS", + repoUrl: "https://github.com/zai-org/GLM-TTS", + filter: false, + countDownloads: `path:"flow/flow.pt"`, + }, + "glyph-byt5": { + prettyLabel: "Glyph-ByT5", + repoName: "Glyph-ByT5", + repoUrl: "https://github.com/AIGText/Glyph-ByT5", + filter: false, + countDownloads: `path:"checkpoints/byt5_model.pt"`, + }, + "granite-library": { + prettyLabel: "Granite Library", + repoName: "mellea", + repoUrl: "https://github.com/generative-computing/mellea", + filter: false, + countDownloads: `path_filename:"adapter_config" AND path_extension:"json"`, + }, + grok: { + prettyLabel: "Grok", + repoName: "Grok", + repoUrl: "https://github.com/xai-org/grok-1", + filter: false, + countDownloads: `path:"ckpt/tensor00000_000" OR path:"ckpt-0/tensor00000_000"`, + }, + "habibi-tts": { + prettyLabel: "Habibi-TTS", + repoName: "Habibi-TTS", + repoUrl: "https://github.com/SWivid/Habibi-TTS", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + hallo: { + prettyLabel: "Hallo", + repoName: "Hallo", + repoUrl: "https://github.com/fudan-generative-vision/hallo", + countDownloads: `path:"hallo/net.pth"`, + }, + hermes: { + prettyLabel: "HERMES", + repoName: "HERMES", + repoUrl: "https://github.com/LMD0311/HERMES", + filter: false, + countDownloads: `path:"ckpt/hermes_final.pth"`, + }, + holomotion: { + prettyLabel: "HoloMotion", + repoName: "HoloMotion", + repoUrl: "https://github.com/HorizonRobotics/HoloMotion", + filter: false, + countDownloads: `path_extension:"onnx"`, + }, + hezar: { + prettyLabel: "Hezar", + repoName: "Hezar", + repoUrl: "https://github.com/hezarai/hezar", + docsUrl: "https://hezarai.github.io/hezar", + countDownloads: `path:"model_config.yaml" OR path:"embedding/embedding_config.yaml"`, + }, + htrflow: { + prettyLabel: "HTRflow", + repoName: "HTRflow", + repoUrl: "https://github.com/AI-Riksarkivet/htrflow", + docsUrl: "https://ai-riksarkivet.github.io/htrflow", + snippets: snippets.htrflow, + }, + "hunyuan-dit": { + prettyLabel: "HunyuanDiT", + repoName: "HunyuanDiT", + repoUrl: "https://github.com/Tencent/HunyuanDiT", + countDownloads: `path:"pytorch_model_ema.pt" OR path:"pytorch_model_distill.pt"`, + }, + "hunyuan3d-2": { + prettyLabel: "Hunyuan3D-2", + repoName: "Hunyuan3D-2", + repoUrl: "https://github.com/Tencent/Hunyuan3D-2", + countDownloads: `path_filename:"model_index" OR path_filename:"config"`, + }, + "hunyuanworld-voyager": { + prettyLabel: "HunyuanWorld-voyager", + repoName: "HunyuanWorld-voyager", + repoUrl: "https://github.com/Tencent-Hunyuan/HunyuanWorld-Voyager", + }, + "hy-worldplay": { + prettyLabel: "HY-WorldPlay", + repoName: "HY-WorldPlay", + repoUrl: "https://github.com/Tencent-Hunyuan/HY-WorldPlay", + filter: false, + countDownloads: `path_extension:"json"`, + }, + "hy-world-2": { + prettyLabel: "HY-World-2.0", + repoName: "HY-World-2.0", + repoUrl: "https://github.com/Tencent-Hunyuan/HY-World-2.0", + filter: false, + countDownloads: `path_extension:"json"`, + }, + "image-matching-models": { + prettyLabel: "Image Matching Models", + repoName: "Image Matching Models", + repoUrl: "https://github.com/alexstoken/image-matching-models", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + imstoucan: { + prettyLabel: "IMS Toucan", + repoName: "IMS-Toucan", + repoUrl: "https://github.com/DigitalPhonetics/IMS-Toucan", + countDownloads: `path:"embedding_gan.pt" OR path:"Vocoder.pt" OR path:"ToucanTTS.pt"`, + }, + "index-tts": { + prettyLabel: "IndexTTS", + repoName: "IndexTTS", + repoUrl: "https://github.com/index-tts/index-tts", + snippets: snippets.indextts, + filter: false, + }, + infinitetalk: { + prettyLabel: "InfiniteTalk", + repoName: "InfiniteTalk", + repoUrl: "https://github.com/MeiGen-AI/InfiniteTalk", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + "infinite-you": { + prettyLabel: "InfiniteYou", + repoName: "InfiniteYou", + repoUrl: "https://github.com/bytedance/InfiniteYou", + filter: false, + countDownloads: `path:"infu_flux_v1.0/sim_stage1/image_proj_model.bin" OR path:"infu_flux_v1.0/aes_stage2/image_proj_model.bin"`, + }, + intellifold: { + prettyLabel: "IntelliFold", + repoName: "IntelliFold", + repoUrl: "https://github.com/IntelliGen-AI/IntelliFold", + filter: false, + countDownloads: `path_extension:"pt" OR path_extension:"zst"`, + }, + "ising-decoding": { + prettyLabel: "Ising Decoding", + repoName: "Ising-Decoding", + repoUrl: "https://github.com/NVIDIA/Ising-Decoding", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + keras: { + prettyLabel: "Keras", + repoName: "Keras", + repoUrl: "https://github.com/keras-team/keras", + docsUrl: "https://huggingface.co/docs/hub/keras", + snippets: snippets.keras, + filter: true, + countDownloads: `path:"config.json" OR path_extension:"keras"`, + }, + "tf-keras": { + // Legacy "Keras 2" library (tensorflow-only) + prettyLabel: "TF-Keras", + repoName: "TF-Keras", + repoUrl: "https://github.com/keras-team/tf-keras", + docsUrl: "https://huggingface.co/docs/hub/tf-keras", + snippets: snippets.tf_keras, + countDownloads: `path:"saved_model.pb"`, + }, + "keras-hub": { + prettyLabel: "KerasHub", + repoName: "KerasHub", + repoUrl: "https://github.com/keras-team/keras-hub", + docsUrl: "https://keras.io/keras_hub/", + snippets: snippets.keras_hub, + filter: true, + }, + kernels: { + prettyLabel: "Kernels", + repoName: "Kernels", + repoUrl: "https://github.com/huggingface/kernels", + docsUrl: "https://huggingface.co/docs/kernels", + snippets: snippets.kernels, + countDownloads: `path_filename:"_ops" AND path_extension:"py"`, + }, + "kimi-audio": { + prettyLabel: "KimiAudio", + repoName: "KimiAudio", + repoUrl: "https://github.com/MoonshotAI/Kimi-Audio", + snippets: snippets.kimi_audio, + filter: false, + }, + kittentts: { + prettyLabel: "KittenTTS", + repoName: "KittenTTS", + repoUrl: "https://github.com/KittenML/KittenTTS", + snippets: snippets.kittentts, + }, + kronos: { + prettyLabel: "KRONOS", + repoName: "KRONOS", + repoUrl: "https://github.com/mahmoodlab/KRONOS", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + k2: { + prettyLabel: "K2", + repoName: "k2", + repoUrl: "https://github.com/k2-fsa/k2", + }, + "lyra-2.0": { + prettyLabel: "Lyra-2.0", + repoName: "Lyra-2.0", + repoUrl: "https://github.com/nv-tlabs/lyra", + filter: false, + countDownloads: `path:"checkpoints/image_encoder/model.pth"`, + }, + lagernvs: { + prettyLabel: "LagerNVS", + repoName: "LagerNVS", + repoUrl: "https://github.com/facebookresearch/lagernvs", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + "lightning-ir": { + prettyLabel: "Lightning IR", + repoName: "Lightning IR", + repoUrl: "https://github.com/webis-de/lightning-ir", + snippets: snippets.lightning_ir, + }, + litert: { + prettyLabel: "LiteRT", + repoName: "LiteRT", + repoUrl: "https://github.com/google-ai-edge/LiteRT", + filter: false, + countDownloads: `path_extension:"tflite"`, + }, + "litert-lm": { + prettyLabel: "LiteRT-LM", + repoName: "LiteRT-LM", + repoUrl: "https://github.com/google-ai-edge/LiteRT-LM", + snippets: snippets.litert_lm, + filter: false, + countDownloads: `path_extension:"litertlm" OR path_extension:"task"`, + }, + lerobot: { + prettyLabel: "LeRobot", + repoName: "LeRobot", + repoUrl: "https://github.com/huggingface/lerobot", + docsUrl: "https://huggingface.co/docs/lerobot", + filter: false, + snippets: snippets.lerobot, + }, + lightglue: { + prettyLabel: "LightGlue", + repoName: "LightGlue", + repoUrl: "https://github.com/cvg/LightGlue", + filter: false, + countDownloads: `path_extension:"pth" OR path:"config.json"`, + }, + liveportrait: { + prettyLabel: "LivePortrait", + repoName: "LivePortrait", + repoUrl: "https://github.com/KwaiVGI/LivePortrait", + filter: false, + countDownloads: `path:"liveportrait/landmark.onnx"`, + }, + "longcat-video-avatar-1.5": { + prettyLabel: "LongCat-Video-Avatar 1.5", + repoName: "LongCat-Video-Avatar 1.5", + repoUrl: "https://github.com/meituan-longcat/LongCat-Video", + filter: false, + }, + "llama-cpp-python": { + prettyLabel: "llama-cpp-python", + repoName: "llama-cpp-python", + repoUrl: "https://github.com/abetlen/llama-cpp-python", + snippets: snippets.llama_cpp_python, + }, + "mini-omni2": { + prettyLabel: "Mini-Omni2", + repoName: "Mini-Omni2", + repoUrl: "https://github.com/gpt-omni/mini-omni2", + countDownloads: `path:"model_config.yaml"`, + }, + mindspore: { + prettyLabel: "MindSpore", + repoName: "mindspore", + repoUrl: "https://github.com/mindspore-ai/mindspore", + }, + "magi-1": { + prettyLabel: "MAGI-1", + repoName: "MAGI-1", + repoUrl: "https://github.com/SandAI-org/MAGI-1", + countDownloads: `path:"ckpt/vae/config.json"`, + }, + "magenta-realtime": { + prettyLabel: "Magenta RT", + repoName: "Magenta RT", + repoUrl: "https://github.com/magenta/magenta-realtime", + countDownloads: `path:"checkpoints/llm_base_x4286_c1860k.tar" OR path:"checkpoints/llm_large_x3047_c1860k.tar" OR path:"checkpoints/llm_large_x3047_c1860k/checkpoint"`, + }, + "magenta-realtime-2": { + prettyLabel: "Magenta RT 2", + repoName: "Magenta RT 2", + repoUrl: "https://github.com/magenta/magenta-realtime", + countDownloads: `path:"models/mrt2_base/mrt2_base.mlxfn" OR path:"models/mrt2_small/mrt2_small.mlxfn" OR path:"checkpoints/mrt2_base.safetensors" OR path:"checkpoints/mrt2_small.safetensors"`, + }, + "mamba-ssm": { + prettyLabel: "MambaSSM", + repoName: "MambaSSM", + repoUrl: "https://github.com/state-spaces/mamba", + filter: false, + snippets: snippets.mamba_ssm, + }, + "manas-1": { + prettyLabel: "MANAS-1", + repoName: "MANAS-1", + repoUrl: "https://github.com/NeurodxAI/manas-1", + countDownloads: `path_extension:"pt"`, + }, + "mars5-tts": { + prettyLabel: "MARS5-TTS", + repoName: "MARS5-TTS", + repoUrl: "https://github.com/Camb-ai/MARS5-TTS", + filter: false, + countDownloads: `path:"mars5_ar.safetensors"`, + snippets: snippets.mars5_tts, + }, + matanyone: { + prettyLabel: "MatAnyone", + repoName: "MatAnyone", + repoUrl: "https://github.com/pq-yang/MatAnyone", + snippets: snippets.matanyone, + filter: false, + }, + "mesh-anything": { + prettyLabel: "MeshAnything", + repoName: "MeshAnything", + repoUrl: "https://github.com/buaacyw/MeshAnything", + filter: false, + countDownloads: `path:"MeshAnything_350m.pth"`, + snippets: snippets.mesh_anything, + }, + merlin: { + prettyLabel: "Merlin", + repoName: "Merlin", + repoUrl: "https://github.com/StanfordMIMI/Merlin", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + medvae: { + prettyLabel: "MedVAE", + repoName: "MedVAE", + repoUrl: "https://github.com/StanfordMIMI/MedVAE", + filter: false, + countDownloads: `path_extension:"ckpt"`, + }, + mitie: { + prettyLabel: "MITIE", + repoName: "MITIE", + repoUrl: "https://github.com/mit-nlp/MITIE", + countDownloads: `path_filename:"total_word_feature_extractor"`, + }, + "ml-agents": { + prettyLabel: "ml-agents", + repoName: "ml-agents", + repoUrl: "https://github.com/Unity-Technologies/ml-agents", + docsUrl: "https://huggingface.co/docs/hub/ml-agents", + snippets: snippets.mlAgents, + filter: true, + countDownloads: `path_extension:"onnx"`, + }, + "ml-sharp": { + prettyLabel: "Sharp", + repoName: "Sharp", + repoUrl: "https://github.com/apple/ml-sharp", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + mlx: { + prettyLabel: "MLX", + repoName: "MLX", + repoUrl: "https://github.com/ml-explore/mlx-examples/tree/main", + snippets: snippets.mlx, + filter: true, + }, + "mlx-image": { + prettyLabel: "mlx-image", + repoName: "mlx-image", + repoUrl: "https://github.com/riccardomusmeci/mlx-image", + docsUrl: "https://huggingface.co/docs/hub/mlx-image", + snippets: snippets.mlxim, + filter: false, + countDownloads: `path:"model.safetensors"`, + }, + "mlc-llm": { + prettyLabel: "MLC-LLM", + repoName: "MLC-LLM", + repoUrl: "https://github.com/mlc-ai/mlc-llm", + docsUrl: "https://llm.mlc.ai/docs/", + filter: false, + countDownloads: `path:"mlc-chat-config.json"`, + }, + model2vec: { + prettyLabel: "Model2Vec", + repoName: "model2vec", + repoUrl: "https://github.com/MinishLab/model2vec", + snippets: snippets.model2vec, + filter: false, + }, + moshi: { + prettyLabel: "Moshi", + repoName: "Moshi", + repoUrl: "https://github.com/kyutai-labs/moshi", + snippets: snippets.moshi, + filter: false, + countDownloads: `path:"tokenizer-e351c8d8-checkpoint125.safetensors"`, + }, + mtvcraft: { + prettyLabel: "MTVCraft", + repoName: "MTVCraft", + repoUrl: "https://github.com/baaivision/MTVCraft", + filter: false, + countDownloads: `path:"vae/3d-vae.pt"`, + }, + multimolecule: { + prettyLabel: "MultiMolecule", + repoName: "MultiMolecule", + repoUrl: "https://github.com/MultiMolecule/multimolecule", + docsUrl: "https://multimolecule.danling.org", + snippets: snippets.multimolecule, + filter: false, + }, + nemo: { + prettyLabel: "NeMo", + repoName: "NeMo", + repoUrl: "https://github.com/NVIDIA/NeMo", + snippets: snippets.nemo, + filter: true, + countDownloads: `path_extension:"nemo" OR path:"model_config.yaml" OR path_extension:"json"`, + }, + "nv-medtech": { + prettyLabel: "NV-MedTech", + repoName: "NV-MedTech", + filter: false, + repoUrl: "https://github.com/nvidia-medtech", + countDownloads: `path_extension:"pt" OR path_extension:"safetensors" OR path:"config.json"`, + }, + "open-oasis": { + prettyLabel: "open-oasis", + repoName: "open-oasis", + repoUrl: "https://github.com/etched-ai/open-oasis", + countDownloads: `path:"oasis500m.safetensors"`, + }, + open_clip: { + prettyLabel: "OpenCLIP", + repoName: "OpenCLIP", + repoUrl: "https://github.com/mlfoundations/open_clip", + snippets: snippets.open_clip, + filter: true, + countDownloads: `path:"open_clip_model.safetensors" + OR path:"model.safetensors" + OR path:"open_clip_pytorch_model.bin" + OR path:"pytorch_model.bin"`, + }, + openpeerllm: { + prettyLabel: "OpenPeerLLM", + repoName: "OpenPeerLLM", + repoUrl: "https://huggingface.co/openpeerai/openpeerllm", + docsUrl: "https://huggingface.co/OpenPeerAI/OpenPeerLLM/blob/main/README.md", + countDownloads: `path:".meta-huggingface.json"`, + filter: false, + }, + "open-sora": { + prettyLabel: "Open-Sora", + repoName: "Open-Sora", + repoUrl: "https://github.com/hpcaitech/Open-Sora", + filter: false, + countDownloads: `path:"Open_Sora_v2.safetensors"`, + }, + outetts: { + prettyLabel: "OuteTTS", + repoName: "OuteTTS", + repoUrl: "https://github.com/edwko/OuteTTS", + snippets: snippets.outetts, + filter: false, + }, + paddlenlp: { + prettyLabel: "paddlenlp", + repoName: "PaddleNLP", + repoUrl: "https://github.com/PaddlePaddle/PaddleNLP", + docsUrl: "https://huggingface.co/docs/hub/paddlenlp", + snippets: snippets.paddlenlp, + filter: true, + countDownloads: `path:"model_config.json"`, + }, + PaddleOCR: { + prettyLabel: "PaddleOCR", + repoName: "PaddleOCR", + repoUrl: "https://github.com/PaddlePaddle/PaddleOCR", + docsUrl: "https://www.paddleocr.ai/", + snippets: snippets.paddleocr, + filter: true, + countDownloads: `path_extension:"safetensors" OR path:"inference.pdiparams" OR path:"inference.onnx"`, + }, + peft: { + prettyLabel: "PEFT", + repoName: "PEFT", + repoUrl: "https://github.com/huggingface/peft", + snippets: snippets.peft, + filter: true, + countDownloads: `path:"adapter_config.json"`, + }, + "perception-encoder": { + prettyLabel: "PerceptionEncoder", + repoName: "PerceptionModels", + repoUrl: "https://github.com/facebookresearch/perception_models", + filter: false, + snippets: snippets.perception_encoder, + countDownloads: `path_extension:"pt"`, + }, + "phantom-wan": { + prettyLabel: "Phantom", + repoName: "Phantom", + repoUrl: "https://github.com/Phantom-video/Phantom", + snippets: snippets.phantom_wan, + filter: false, + countDownloads: `path_extension:"pth"`, + }, + "pocket-tts": { + prettyLabel: "Pocket-TTS", + repoName: "PocketTTS", + repoUrl: "https://github.com/kyutai-labs/pocket-tts", + snippets: snippets.pocket_tts, + filter: false, + countDownloads: `path:"tts_b6369a24.safetensors"`, + }, + "pruna-ai": { + prettyLabel: "Pruna AI", + repoName: "Pruna AI", + repoUrl: "https://github.com/PrunaAI/pruna", + snippets: snippets.pruna, + docsUrl: "https://docs.pruna.ai", + }, + pxia: { + prettyLabel: "pxia", + repoName: "pxia", + repoUrl: "https://github.com/not-lain/pxia", + snippets: snippets.pxia, + filter: false, + }, + "pyannote-audio": { + prettyLabel: "pyannote.audio", + repoName: "pyannote-audio", + repoUrl: "https://github.com/pyannote/pyannote-audio", + snippets: snippets.pyannote_audio, + filter: true, + }, + "py-feat": { + prettyLabel: "Py-Feat", + repoName: "Py-Feat", + repoUrl: "https://github.com/cosanlab/py-feat", + docsUrl: "https://py-feat.org/", + filter: false, + }, + pythae: { + prettyLabel: "pythae", + repoName: "pythae", + repoUrl: "https://github.com/clementchadebec/benchmark_VAE", + snippets: snippets.pythae, + filter: false, + }, + quantumpeer: { + prettyLabel: "QuantumPeer", + repoName: "QuantumPeer", + repoUrl: "https://github.com/OpenPeer-AI/QuantumPeer", + filter: false, + countDownloads: `path_extension:"setup.py"`, + }, + qwen3_tts: { + prettyLabel: "Qwen3-TTS", + repoName: "Qwen3-TTS", + repoUrl: "https://github.com/QwenLM/Qwen3-TTS", + snippets: snippets.qwen3_tts, + filter: false, + }, + recurrentgemma: { + prettyLabel: "RecurrentGemma", + repoName: "recurrentgemma", + repoUrl: "https://github.com/google-deepmind/recurrentgemma", + filter: false, + countDownloads: `path:"tokenizer.model"`, + }, + relik: { + prettyLabel: "Relik", + repoName: "Relik", + repoUrl: "https://github.com/SapienzaNLP/relik", + snippets: snippets.relik, + filter: false, + }, + refiners: { + prettyLabel: "Refiners", + repoName: "Refiners", + repoUrl: "https://github.com/finegrain-ai/refiners", + docsUrl: "https://refine.rs/", + filter: false, + countDownloads: `path:"model.safetensors"`, + }, + renderformer: { + prettyLabel: "RenderFormer", + repoName: "RenderFormer", + repoUrl: "https://github.com/microsoft/renderformer", + snippets: snippets.renderformer, + filter: false, + }, + reverb: { + prettyLabel: "Reverb", + repoName: "Reverb", + repoUrl: "https://github.com/revdotcom/reverb", + filter: false, + }, + rkllm: { + prettyLabel: "RKLLM", + repoName: "RKLLM", + repoUrl: "https://github.com/airockchip/rknn-llm", + countDownloads: `path_extension:"rkllm"`, + }, + "robo-orchard-lab": { + prettyLabel: "RoboOrchardLab", + repoName: "RoboOrchardLab", + repoUrl: "https://github.com/HorizonRobotics/RoboOrchardLab", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + rwkv: { + prettyLabel: "RWKV", + repoName: "RWKV-LM", + repoUrl: "https://github.com/BlinkDL/RWKV-LM", + docsUrl: "https://rwkv.com/", + filter: false, + countDownloads: `path_extension:"pth"`, + }, + saelens: { + prettyLabel: "SAELens", + repoName: "SAELens", + repoUrl: "https://github.com/jbloomAus/SAELens", + snippets: snippets.saelens, + filter: false, + }, + "scail-2": { + prettyLabel: "SCAIL-2", + repoName: "SCAIL-2", + repoUrl: "https://github.com/zai-org/SCAIL-2", + filter: false, + countDownloads: `path:"model/1/fsdp2_rank_0000_checkpoint.pt"`, + }, + sam2: { + prettyLabel: "sam2", + repoName: "sam2", + repoUrl: "https://github.com/facebookresearch/segment-anything-2", + filter: false, + snippets: snippets.sam2, + countDownloads: `path_extension:"pt"`, + }, + "sam-3d-body": { + prettyLabel: "SAM 3D Body", + repoName: "SAM 3D Body", + repoUrl: "https://github.com/facebookresearch/sam-3d-body", + filter: false, + snippets: snippets.sam_3d_body, + countDownloads: `path:"model_config.yaml"`, + }, + "sam-3d-objects": { + prettyLabel: "SAM 3D Objects", + repoName: "SAM 3D Objects", + repoUrl: "https://github.com/facebookresearch/sam-3d-objects", + filter: false, + snippets: snippets.sam_3d_objects, + countDownloads: `path:"checkpoints/pipeline.yaml"`, + }, + same: { + prettyLabel: "SAME", + repoName: "SAME", + repoUrl: "https://github.com/GengzeZhou/SAME", + filter: false, + countDownloads: `path:"ckpt/SAME.pt" OR path:"pretrain/Attnq_pretrained_ckpt.pt"`, + }, + "sample-factory": { + prettyLabel: "sample-factory", + repoName: "sample-factory", + repoUrl: "https://github.com/alex-petrenko/sample-factory", + docsUrl: "https://huggingface.co/docs/hub/sample-factory", + snippets: snippets.sampleFactory, + filter: true, + countDownloads: `path:"cfg.json"`, + }, + "sap-rpt-1-oss": { + prettyLabel: "sap-rpt-1-oss", + repoName: "sap-rpt-1-oss", + repoUrl: "https://github.com/SAP-samples/sap-rpt-1-oss", + countDownloads: `path_extension:"pt"`, + snippets: snippets.sap_rpt_one_oss, + }, + sapiens: { + prettyLabel: "sapiens", + repoName: "sapiens", + repoUrl: "https://github.com/facebookresearch/sapiens", + filter: false, + countDownloads: `path_extension:"pt2" OR path_extension:"pth" OR path_extension:"onnx"`, + }, + sapiens2: { + prettyLabel: "sapiens2", + repoName: "sapiens2", + repoUrl: "https://github.com/facebookresearch/sapiens2", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + seedvr: { + prettyLabel: "SeedVR", + repoName: "SeedVR", + repoUrl: "https://github.com/ByteDance-Seed/SeedVR", + filter: false, + countDownloads: `path_extension:"pth"`, + }, + "self-forcing": { + prettyLabel: "SelfForcing", + repoName: "SelfForcing", + repoUrl: "https://github.com/guandeh17/Self-Forcing", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + "sentence-transformers": { + prettyLabel: "sentence-transformers", + repoName: "sentence-transformers", + repoUrl: "https://github.com/UKPLab/sentence-transformers", + docsUrl: "https://huggingface.co/docs/hub/sentence-transformers", + snippets: snippets.sentenceTransformers, + filter: true, + }, + setfit: { + prettyLabel: "setfit", + repoName: "setfit", + repoUrl: "https://github.com/huggingface/setfit", + docsUrl: "https://huggingface.co/docs/hub/setfit", + snippets: snippets.setfit, + filter: true, + }, + sklearn: { + prettyLabel: "Scikit-learn", + repoName: "Scikit-learn", + repoUrl: "https://github.com/scikit-learn/scikit-learn", + snippets: snippets.sklearn, + filter: true, + countDownloads: `path:"sklearn_model.joblib"`, + }, + spacy: { + prettyLabel: "spaCy", + repoName: "spaCy", + repoUrl: "https://github.com/explosion/spaCy", + docsUrl: "https://huggingface.co/docs/hub/spacy", + snippets: snippets.spacy, + filter: true, + countDownloads: `path_extension:"whl"`, + }, + "span-marker": { + prettyLabel: "SpanMarker", + repoName: "SpanMarkerNER", + repoUrl: "https://github.com/tomaarsen/SpanMarkerNER", + docsUrl: "https://huggingface.co/docs/hub/span_marker", + snippets: snippets.span_marker, + filter: true, + }, + speechbrain: { + prettyLabel: "speechbrain", + repoName: "speechbrain", + repoUrl: "https://github.com/speechbrain/speechbrain", + docsUrl: "https://huggingface.co/docs/hub/speechbrain", + snippets: snippets.speechbrain, + filter: true, + countDownloads: `path:"hyperparams.yaml"`, + }, + "ssr-speech": { + prettyLabel: "SSR-Speech", + repoName: "SSR-Speech", + repoUrl: "https://github.com/WangHelin1997/SSR-Speech", + filter: false, + countDownloads: `path_extension:".pth"`, + }, + "stable-audio-3": { + prettyLabel: "Stable Audio 3", + repoName: "stable-audio-3", + repoUrl: "https://github.com/Stability-AI/stable-audio-3", + filter: false, + countDownloads: `path:"model_config.json"`, + }, + "stable-audio-tools": { + prettyLabel: "Stable Audio Tools", + repoName: "stable-audio-tools", + repoUrl: "https://github.com/Stability-AI/stable-audio-tools.git", + filter: false, + countDownloads: `path:"model.safetensors"`, + snippets: snippets.stable_audio_tools, + }, + monkeyocr: { + prettyLabel: "MonkeyOCR", + repoName: "monkeyocr", + repoUrl: "https://github.com/Yuliang-Liu/MonkeyOCR", + filter: false, + countDownloads: `path:"Recognition/config.json"`, + }, + "diffusion-single-file": { + prettyLabel: "Diffusion Single File", + repoName: "diffusion-single-file", + repoUrl: "https://github.com/comfyanonymous/ComfyUI", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + "seed-story": { + prettyLabel: "SEED-Story", + repoName: "SEED-Story", + repoUrl: "https://github.com/TencentARC/SEED-Story", + filter: false, + countDownloads: `path:"cvlm_llama2_tokenizer/tokenizer.model"`, + snippets: snippets.seed_story, + }, + skala: { + prettyLabel: "Skala", + repoName: "Skala", + repoUrl: "https://github.com/microsoft/skala", + filter: false, + countDownloads: `path_extension:"fun"`, + }, + soloaudio: { + prettyLabel: "SoloAudio", + repoName: "SoloAudio", + repoUrl: "https://github.com/WangHelin1997/SoloAudio", + filter: false, + countDownloads: `path:"soloaudio_v2.pt"`, + }, + songbloom: { + prettyLabel: "SongBloom", + repoName: "SongBloom", + repoUrl: "https://github.com/Cypress-Yang/SongBloom", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + "stable-baselines3": { + prettyLabel: "stable-baselines3", + repoName: "stable-baselines3", + repoUrl: "https://github.com/huggingface/huggingface_sb3", + docsUrl: "https://huggingface.co/docs/hub/stable-baselines3", + snippets: snippets.stableBaselines3, + filter: true, + countDownloads: `path_extension:"zip"`, + }, + stanza: { + prettyLabel: "Stanza", + repoName: "stanza", + repoUrl: "https://github.com/stanfordnlp/stanza", + docsUrl: "https://huggingface.co/docs/hub/stanza", + snippets: snippets.stanza, + filter: true, + countDownloads: `path:"models/default.zip"`, + }, + supertonic: { + prettyLabel: "Supertonic", + repoName: "Supertonic", + repoUrl: "https://github.com/supertone-inc/supertonic", + snippets: snippets.supertonic, + filter: false, + }, + swarmformer: { + prettyLabel: "SwarmFormer", + repoName: "SwarmFormer", + repoUrl: "https://github.com/takara-ai/SwarmFormer", + snippets: snippets.swarmformer, + filter: false, + }, + "synthefy-migas": { + prettyLabel: "Migas", + repoName: "Migas", + repoUrl: "https://github.com/Synthefy/synthefy-migas", + filter: false, + countDownloads: `path:"model.pt"`, + }, + "f5-tts": { + prettyLabel: "F5-TTS", + repoName: "F5-TTS", + repoUrl: "https://github.com/SWivid/F5-TTS", + filter: false, + countDownloads: `path_extension:"safetensors" OR path_extension:"pt"`, + }, + genmo: { + prettyLabel: "Genmo", + repoName: "Genmo", + repoUrl: "https://github.com/genmoai/models", + filter: false, + countDownloads: `path:"vae_stats.json"`, + }, + "tencent-song-generation": { + prettyLabel: "SongGeneration", + repoName: "SongGeneration", + repoUrl: "https://github.com/tencent-ailab/songgeneration", + filter: false, + countDownloads: `path:"ckpt/songgeneration_base/model.pt"`, + }, + tensorflowtts: { + prettyLabel: "TensorFlowTTS", + repoName: "TensorFlowTTS", + repoUrl: "https://github.com/TensorSpeech/TensorFlowTTS", + snippets: snippets.tensorflowtts, + }, + tensorrt: { + prettyLabel: "TensorRT", + repoName: "TensorRT", + repoUrl: "https://github.com/NVIDIA/TensorRT", + countDownloads: `path_extension:"onnx"`, + }, + tabpfn: { + prettyLabel: "TabPFN", + repoName: "TabPFN", + repoUrl: "https://github.com/PriorLabs/TabPFN", + }, + terratorch: { + prettyLabel: "TerraTorch", + repoName: "TerraTorch", + repoUrl: "https://github.com/IBM/terratorch", + docsUrl: "https://ibm.github.io/terratorch/", + filter: false, + countDownloads: `path_extension:"pt" OR path_extension:"ckpt"`, + snippets: snippets.terratorch, + }, + "tic-clip": { + prettyLabel: "TiC-CLIP", + repoName: "TiC-CLIP", + repoUrl: "https://github.com/apple/ml-tic-clip", + filter: false, + countDownloads: `path_extension:"pt" AND path_prefix:"checkpoints/"`, + }, + timesfm: { + prettyLabel: "TimesFM", + repoName: "timesfm", + repoUrl: "https://github.com/google-research/timesfm", + filter: false, + countDownloads: `path:"checkpoints/checkpoint_1100000/state/checkpoint" OR path:"checkpoints/checkpoint_2150000/state/checkpoint" OR path_extension:"ckpt"`, + }, + timm: { + prettyLabel: "timm", + repoName: "pytorch-image-models", + repoUrl: "https://github.com/rwightman/pytorch-image-models", + docsUrl: "https://huggingface.co/docs/hub/timm", + snippets: snippets.timm, + filter: true, + countDownloads: `path:"pytorch_model.bin" OR path:"model.safetensors"`, + }, + tirex: { + prettyLabel: "TiRex", + repoName: "TiRex", + repoUrl: "https://github.com/NX-AI/tirex", + countDownloads: `path_extension:"ckpt"`, + }, + torchgeo: { + prettyLabel: "TorchGeo", + repoName: "TorchGeo", + repoUrl: "https://github.com/microsoft/torchgeo", + docsUrl: "https://torchgeo.readthedocs.io/", + filter: false, + countDownloads: `path_extension:"pt" OR path_extension:"pth"`, + }, + transformers: { + prettyLabel: "Transformers", + repoName: "🤗/transformers", + repoUrl: "https://github.com/huggingface/transformers", + docsUrl: "https://huggingface.co/docs/hub/transformers", + snippets: snippets.transformers, + filter: true, + }, + "transformers.js": { + prettyLabel: "Transformers.js", + repoName: "transformers.js", + repoUrl: "https://github.com/huggingface/transformers.js", + docsUrl: "https://huggingface.co/docs/hub/transformers-js", + snippets: snippets.transformersJS, + filter: true, + }, + trellis: { + prettyLabel: "Trellis", + repoName: "Trellis", + repoUrl: "https://github.com/microsoft/TRELLIS", + countDownloads: `path_extension:"safetensors"`, + }, + trellis2: { + prettyLabel: "TRELLIS.2", + repoName: "TRELLIS.2", + repoUrl: "https://github.com/microsoft/TRELLIS.2", + countDownloads: `path_extension:"safetensors"`, + }, + tunejury: { + prettyLabel: "TuneJury", + repoName: "TuneJury", + repoUrl: "https://github.com/yonghyunk1m/TuneJury", + countDownloads: `path_extension:"pt"`, + }, + ultralytics: { + prettyLabel: "ultralytics", + repoName: "ultralytics", + repoUrl: "https://github.com/ultralytics/ultralytics", + docsUrl: "https://github.com/ultralytics/ultralytics", + filter: false, + countDownloads: `path_extension:"pt"`, + snippets: snippets.ultralytics, + }, + univa: { + prettyLabel: "univa", + repoName: "univa", + repoUrl: "https://github.com/PKU-YuanGroup/UniWorld-V1", + snippets: snippets.univa, + filter: true, + countDownloads: `path:"config.json"`, + }, + "uni-3dar": { + prettyLabel: "Uni-3DAR", + repoName: "Uni-3DAR", + repoUrl: "https://github.com/dptech-corp/Uni-3DAR", + docsUrl: "https://github.com/dptech-corp/Uni-3DAR", + countDownloads: `path_extension:"pt"`, + }, + "unity-sentis": { + prettyLabel: "unity-sentis", + repoName: "unity-sentis", + repoUrl: "https://github.com/Unity-Technologies/sentis-samples", + snippets: snippets.sentis, + filter: true, + countDownloads: `path_extension:"sentis"`, + }, + sana: { + prettyLabel: "Sana", + repoName: "Sana", + repoUrl: "https://github.com/NVlabs/Sana", + countDownloads: `path_extension:"pth"`, + snippets: snippets.sana, + }, + videoprism: { + prettyLabel: "VideoPrism", + repoName: "VideoPrism", + repoUrl: "https://github.com/google-deepmind/videoprism", + countDownloads: `path_extension:"npz"`, + snippets: snippets.videoprism, + }, + "vfi-mamba": { + prettyLabel: "VFIMamba", + repoName: "VFIMamba", + repoUrl: "https://github.com/MCG-NJU/VFIMamba", + countDownloads: `path_extension:"pkl"`, + snippets: snippets.vfimamba, + }, + vismatch: { + prettyLabel: "VisMatch", + repoName: "VisMatch", + repoUrl: "https://github.com/gmberton/vismatch", + filter: false, + countDownloads: `path:"vismatch.yaml"`, + }, + lvface: { + prettyLabel: "LVFace", + repoName: "LVFace", + repoUrl: "https://github.com/bytedance/LVFace", + countDownloads: `path_extension:"pt" OR path_extension:"onnx"`, + snippets: snippets.lvface, + }, + voicecraft: { + prettyLabel: "VoiceCraft", + repoName: "VoiceCraft", + repoUrl: "https://github.com/jasonppy/VoiceCraft", + docsUrl: "https://github.com/jasonppy/VoiceCraft", + snippets: snippets.voicecraft, + }, + voxcpm: { + prettyLabel: "VoxCPM", + repoName: "VoxCPM", + repoUrl: "https://github.com/OpenBMB/VoxCPM", + snippets: snippets.voxcpm, + filter: false, + }, + vui: { + prettyLabel: "Vui", + repoName: "Vui", + repoUrl: "https://github.com/vui-ai/vui", + countDownloads: `path_extension:"pt"`, + snippets: snippets.vui, + }, + vibevoice: { + prettyLabel: "VibeVoice", + repoName: "VibeVoice", + repoUrl: "https://github.com/microsoft/VibeVoice", + snippets: snippets.vibevoice, + filter: false, + }, + videox_fun: { + prettyLabel: "VideoX Fun", + repoName: "VideoX Fun", + repoUrl: "https://github.com/aigc-apps/VideoX-Fun", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + "wan2.2": { + prettyLabel: "Wan2.2", + repoName: "Wan2.2", + repoUrl: "https://github.com/Wan-Video/Wan2.2", + countDownloads: `path_filename:"config" AND path_extension:"json"`, + }, + wham: { + prettyLabel: "WHAM", + repoName: "wham", + repoUrl: "https://huggingface.co/microsoft/wham", + docsUrl: "https://huggingface.co/microsoft/wham/blob/main/README.md", + countDownloads: `path_extension:"ckpt"`, + }, + whisperkit: { + prettyLabel: "WhisperKit", + repoName: "WhisperKit", + repoUrl: "https://github.com/argmaxinc/WhisperKit", + docsUrl: "https://github.com/argmaxinc/WhisperKit?tab=readme-ov-file#homebrew", + snippets: snippets.whisperkit, + countDownloads: `path_filename:"model" AND path_extension:"mil" AND _exists_:"path_prefix"`, + }, + yolov10: { + // YOLOv10 is a fork of ultraLytics. Code snippets and download count are the same but the repo is different. + prettyLabel: "YOLOv10", + repoName: "YOLOv10", + repoUrl: "https://github.com/THU-MIG/yolov10", + docsUrl: "https://github.com/THU-MIG/yolov10", + countDownloads: `path_extension:"pt" OR path_extension:"safetensors"`, + snippets: snippets.ultralytics, + }, + yolov26: { + prettyLabel: "YOLOv26", + repoName: "YOLOv26", + repoUrl: "https://github.com/ultralytics/ultralytics", + docsUrl: "https://docs.ultralytics.com/models/yolo26/", + countDownloads: `path_extension:"pt" OR path_extension:"safetensors"`, + }, + zonos: { + prettyLabel: "Zonos", + repoName: "Zonos", + repoUrl: "https://github.com/Zyphra/Zonos", + docsUrl: "https://github.com/Zyphra/Zonos", + snippets: snippets.zonos, + filter: false, + }, + "3dtopia-xl": { + prettyLabel: "3DTopia-XL", + repoName: "3DTopia-XL", + repoUrl: "https://github.com/3DTopia/3DTopia-XL", + filter: false, + countDownloads: `path:"model_vae_fp16.pt"`, + snippets: snippets.threedtopia_xl, + }, +}; +exports.ALL_MODEL_LIBRARY_KEYS = Object.keys(exports.MODEL_LIBRARIES_UI_ELEMENTS); +exports.ALL_DISPLAY_MODEL_LIBRARY_KEYS = Object.entries(exports.MODEL_LIBRARIES_UI_ELEMENTS) + // eslint-disable-next-line @typescript-eslint/no-unused-vars + .filter(([_, v]) => v.filter) + .map(([k]) => k); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/package.json b/node_modules/@huggingface/tasks/dist/commonjs/package.json new file mode 100644 index 0000000000000000000000000000000000000000..5bbefffbabee392d1855491b84dc0a716b6a3bf2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/package.json @@ -0,0 +1,3 @@ +{ + "type": "commonjs" +} diff --git a/node_modules/@huggingface/tasks/dist/commonjs/pipelines.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/pipelines.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..48b5bfceac989cb636f78400358077608a9f9af3 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/pipelines.d.ts @@ -0,0 +1,385 @@ +export declare const MODALITIES: readonly ["multimodal", "nlp", "cv", "audio", "tabular", "rl", "other"]; +export type Modality = (typeof MODALITIES)[number]; +export declare const MODALITY_LABELS: { + multimodal: string; + nlp: string; + audio: string; + cv: string; + rl: string; + tabular: string; + other: string; +}; +/** + * Public interface for a sub task. + * + * This can be used in a model card's `model-index` metadata. + * and is more granular classification that can grow significantly + * over time as new tasks are added. + */ +export interface SubTask { + /** + * type of the task (e.g. audio-source-separation) + */ + type: string; + /** + * displayed name of the task (e.g. Audio Source Separation) + */ + name: string; +} +/** + * Public interface for a PipelineData. + * + * This information corresponds to a pipeline type (aka task) + * in the Hub. + */ +export interface PipelineData { + /** + * displayed name of the task (e.g. Text Classification) + */ + name: string; + subtasks?: SubTask[]; + modality: Modality; + /** + * whether to hide in /models filters + */ + hideInModels?: boolean; + /** + * whether to hide in /datasets filters + */ + hideInDatasets?: boolean; +} +export declare const PIPELINE_DATA: { + "text-classification": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + }; + "token-classification": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + }; + "table-question-answering": { + name: string; + modality: "nlp"; + }; + "question-answering": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + }; + "zero-shot-classification": { + name: string; + modality: "nlp"; + }; + translation: { + name: string; + modality: "nlp"; + }; + summarization: { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + }; + "feature-extraction": { + name: string; + modality: "nlp"; + }; + "text-generation": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + }; + "fill-mask": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + }; + "sentence-similarity": { + name: string; + modality: "nlp"; + }; + "text-to-speech": { + name: string; + modality: "audio"; + }; + "text-to-audio": { + name: string; + modality: "audio"; + }; + "automatic-speech-recognition": { + name: string; + modality: "audio"; + }; + "audio-to-audio": { + name: string; + modality: "audio"; + }; + "audio-classification": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "audio"; + }; + "audio-text-to-text": { + name: string; + modality: "multimodal"; + hideInDatasets: true; + }; + "voice-activity-detection": { + name: string; + modality: "audio"; + }; + "depth-estimation": { + name: string; + modality: "cv"; + }; + "image-classification": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "cv"; + }; + "object-detection": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "cv"; + }; + "image-segmentation": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "cv"; + }; + "text-to-image": { + name: string; + modality: "cv"; + }; + "image-to-text": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "cv"; + }; + "image-to-image": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "cv"; + }; + "image-to-video": { + name: string; + modality: "cv"; + }; + "unconditional-image-generation": { + name: string; + modality: "cv"; + }; + "video-classification": { + name: string; + modality: "cv"; + }; + "reinforcement-learning": { + name: string; + modality: "rl"; + }; + robotics: { + name: string; + modality: "rl"; + subtasks: { + type: string; + name: string; + }[]; + }; + "tabular-classification": { + name: string; + modality: "tabular"; + subtasks: { + type: string; + name: string; + }[]; + }; + "tabular-regression": { + name: string; + modality: "tabular"; + subtasks: { + type: string; + name: string; + }[]; + }; + "tabular-to-text": { + name: string; + modality: "tabular"; + subtasks: { + type: string; + name: string; + }[]; + hideInModels: true; + }; + "table-to-text": { + name: string; + modality: "nlp"; + hideInModels: true; + }; + "multiple-choice": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + hideInModels: true; + }; + "text-ranking": { + name: string; + modality: "nlp"; + }; + "text-retrieval": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + hideInModels: true; + }; + "time-series-forecasting": { + name: string; + modality: "tabular"; + subtasks: { + type: string; + name: string; + }[]; + }; + "text-to-video": { + name: string; + modality: "cv"; + }; + "image-text-to-text": { + name: string; + modality: "multimodal"; + }; + "image-text-to-image": { + name: string; + modality: "multimodal"; + }; + "image-text-to-video": { + name: string; + modality: "multimodal"; + }; + "visual-question-answering": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "multimodal"; + }; + "document-question-answering": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "multimodal"; + hideInDatasets: true; + }; + "zero-shot-image-classification": { + name: string; + modality: "cv"; + }; + "graph-ml": { + name: string; + modality: "other"; + }; + "mask-generation": { + name: string; + modality: "cv"; + }; + "zero-shot-object-detection": { + name: string; + modality: "cv"; + }; + "text-to-3d": { + name: string; + modality: "cv"; + }; + "image-to-3d": { + name: string; + modality: "cv"; + }; + "image-feature-extraction": { + name: string; + modality: "cv"; + }; + "video-text-to-text": { + name: string; + modality: "multimodal"; + hideInDatasets: false; + }; + "keypoint-detection": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "cv"; + hideInDatasets: true; + }; + "visual-document-retrieval": { + name: string; + modality: "multimodal"; + }; + "any-to-any": { + name: string; + modality: "multimodal"; + }; + "video-to-video": { + name: string; + modality: "cv"; + hideInDatasets: true; + }; + other: { + name: string; + modality: "other"; + hideInModels: true; + hideInDatasets: true; + }; +}; +export type PipelineType = keyof typeof PIPELINE_DATA; +export type WidgetType = PipelineType | "conversational"; +export declare const PIPELINE_TYPES: PipelineType[]; +export declare const SUBTASK_TYPES: string[]; +export declare const PIPELINE_TYPES_SET: Set<"other" | "text-classification" | "token-classification" | "table-question-answering" | "question-answering" | "zero-shot-classification" | "translation" | "summarization" | "feature-extraction" | "text-generation" | "fill-mask" | "sentence-similarity" | "text-to-speech" | "text-to-audio" | "automatic-speech-recognition" | "audio-to-audio" | "audio-classification" | "audio-text-to-text" | "voice-activity-detection" | "depth-estimation" | "image-classification" | "object-detection" | "image-segmentation" | "text-to-image" | "image-to-text" | "image-to-image" | "image-to-video" | "unconditional-image-generation" | "video-classification" | "reinforcement-learning" | "robotics" | "tabular-classification" | "tabular-regression" | "tabular-to-text" | "table-to-text" | "multiple-choice" | "text-ranking" | "text-retrieval" | "time-series-forecasting" | "text-to-video" | "image-text-to-text" | "image-text-to-image" | "image-text-to-video" | "visual-question-answering" | "document-question-answering" | "zero-shot-image-classification" | "graph-ml" | "mask-generation" | "zero-shot-object-detection" | "text-to-3d" | "image-to-3d" | "image-feature-extraction" | "video-text-to-text" | "keypoint-detection" | "visual-document-retrieval" | "any-to-any" | "video-to-video">; +//# sourceMappingURL=pipelines.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/pipelines.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/pipelines.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e44b58ed1d6363fdc5bd08b53ab435961d17e281 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/pipelines.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"pipelines.d.ts","sourceRoot":"","sources":["../../src/pipelines.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,UAAU,yEAA0E,CAAC;AAElG,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD,eAAO,MAAM,eAAe;;;;;;;;CAQQ,CAAC;AAErC;;;;;;GAMG;AACH,MAAM,WAAW,OAAO;IACvB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;CACb;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IACrB,QAAQ,EAAE,QAAQ,CAAC;IACnB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CACzB;AAcD,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwkBc,CAAC;AAEzC,MAAM,MAAM,YAAY,GAAG,MAAM,OAAO,aAAa,CAAC;AAEtD,MAAM,MAAM,UAAU,GAAG,YAAY,GAAG,gBAAgB,CAAC;AAEzD,eAAO,MAAM,cAAc,EAAiC,YAAY,EAAE,CAAC;AAE3E,eAAO,MAAM,aAAa,UAEN,CAAC;AAErB,eAAO,MAAM,kBAAkB,iwCAA0B,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/pipelines.js b/node_modules/@huggingface/tasks/dist/commonjs/pipelines.js new file mode 100644 index 0000000000000000000000000000000000000000..6986116337d99a0939b71fbfbba39fab75aa16d9 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/pipelines.js @@ -0,0 +1,615 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PIPELINE_TYPES_SET = exports.SUBTASK_TYPES = exports.PIPELINE_TYPES = exports.PIPELINE_DATA = exports.MODALITY_LABELS = exports.MODALITIES = void 0; +exports.MODALITIES = ["multimodal", "nlp", "cv", "audio", "tabular", "rl", "other"]; +exports.MODALITY_LABELS = { + multimodal: "Multimodal", + nlp: "Natural Language Processing", + audio: "Audio", + cv: "Computer Vision", + rl: "Reinforcement Learning", + tabular: "Tabular", + other: "Other", +}; +/// Coarse-grained taxonomy of tasks +/// +/// This type is used in multiple places in the Hugging Face +/// ecosystem: +/// - To determine which widget to show. +/// - To determine which endpoint of Inference Endpoints to use. +/// - As filters at the left of models and datasets page. +/// +/// Note that this is sensitive to order. +/// For each domain, the order should be of decreasing specificity. +/// This will impact the default pipeline tag of a model when not +/// specified. +exports.PIPELINE_DATA = { + "text-classification": { + name: "Text Classification", + subtasks: [ + { + type: "acceptability-classification", + name: "Acceptability Classification", + }, + { + type: "entity-linking-classification", + name: "Entity Linking Classification", + }, + { + type: "fact-checking", + name: "Fact Checking", + }, + { + type: "intent-classification", + name: "Intent Classification", + }, + { + type: "language-identification", + name: "Language Identification", + }, + { + type: "multi-class-classification", + name: "Multi Class Classification", + }, + { + type: "multi-label-classification", + name: "Multi Label Classification", + }, + { + type: "multi-input-text-classification", + name: "Multi-input Text Classification", + }, + { + type: "natural-language-inference", + name: "Natural Language Inference", + }, + { + type: "semantic-similarity-classification", + name: "Semantic Similarity Classification", + }, + { + type: "sentiment-classification", + name: "Sentiment Classification", + }, + { + type: "topic-classification", + name: "Topic Classification", + }, + { + type: "semantic-similarity-scoring", + name: "Semantic Similarity Scoring", + }, + { + type: "sentiment-scoring", + name: "Sentiment Scoring", + }, + { + type: "sentiment-analysis", + name: "Sentiment Analysis", + }, + { + type: "hate-speech-detection", + name: "Hate Speech Detection", + }, + { + type: "text-scoring", + name: "Text Scoring", + }, + ], + modality: "nlp", + }, + "token-classification": { + name: "Token Classification", + subtasks: [ + { + type: "named-entity-recognition", + name: "Named Entity Recognition", + }, + { + type: "part-of-speech", + name: "Part of Speech", + }, + { + type: "parsing", + name: "Parsing", + }, + { + type: "lemmatization", + name: "Lemmatization", + }, + { + type: "word-sense-disambiguation", + name: "Word Sense Disambiguation", + }, + { + type: "coreference-resolution", + name: "Coreference-resolution", + }, + ], + modality: "nlp", + }, + "table-question-answering": { + name: "Table Question Answering", + modality: "nlp", + }, + "question-answering": { + name: "Question Answering", + subtasks: [ + { + type: "extractive-qa", + name: "Extractive QA", + }, + { + type: "open-domain-qa", + name: "Open Domain QA", + }, + { + type: "closed-domain-qa", + name: "Closed Domain QA", + }, + ], + modality: "nlp", + }, + "zero-shot-classification": { + name: "Zero-Shot Classification", + modality: "nlp", + }, + translation: { + name: "Translation", + modality: "nlp", + }, + summarization: { + name: "Summarization", + subtasks: [ + { + type: "news-articles-summarization", + name: "News Articles Summarization", + }, + { + type: "news-articles-headline-generation", + name: "News Articles Headline Generation", + }, + ], + modality: "nlp", + }, + "feature-extraction": { + name: "Feature Extraction", + modality: "nlp", + }, + "text-generation": { + name: "Text Generation", + subtasks: [ + { + type: "dialogue-modeling", + name: "Dialogue Modeling", + }, + { + type: "dialogue-generation", + name: "Dialogue Generation", + }, + { + type: "conversational", + name: "Conversational", + }, + { + type: "language-modeling", + name: "Language Modeling", + }, + { + type: "text-simplification", + name: "Text simplification", + }, + { + type: "explanation-generation", + name: "Explanation Generation", + }, + { + type: "abstractive-qa", + name: "Abstractive QA", + }, + { + type: "open-domain-abstractive-qa", + name: "Open Domain Abstractive QA", + }, + { + type: "closed-domain-qa", + name: "Closed Domain QA", + }, + { + type: "open-book-qa", + name: "Open Book QA", + }, + { + type: "closed-book-qa", + name: "Closed Book QA", + }, + { + type: "text2text-generation", + name: "Text2Text Generation", + }, + ], + modality: "nlp", + }, + "fill-mask": { + name: "Fill-Mask", + subtasks: [ + { + type: "slot-filling", + name: "Slot Filling", + }, + { + type: "masked-language-modeling", + name: "Masked Language Modeling", + }, + ], + modality: "nlp", + }, + "sentence-similarity": { + name: "Sentence Similarity", + modality: "nlp", + }, + "text-to-speech": { + name: "Text-to-Speech", + modality: "audio", + }, + "text-to-audio": { + name: "Text-to-Audio", + modality: "audio", + }, + "automatic-speech-recognition": { + name: "Automatic Speech Recognition", + modality: "audio", + }, + "audio-to-audio": { + name: "Audio-to-Audio", + modality: "audio", + }, + "audio-classification": { + name: "Audio Classification", + subtasks: [ + { + type: "keyword-spotting", + name: "Keyword Spotting", + }, + { + type: "speaker-identification", + name: "Speaker Identification", + }, + { + type: "audio-intent-classification", + name: "Audio Intent Classification", + }, + { + type: "audio-emotion-recognition", + name: "Audio Emotion Recognition", + }, + { + type: "audio-language-identification", + name: "Audio Language Identification", + }, + ], + modality: "audio", + }, + "audio-text-to-text": { + name: "Audio-Text-to-Text", + modality: "multimodal", + hideInDatasets: true, + }, + "voice-activity-detection": { + name: "Voice Activity Detection", + modality: "audio", + }, + "depth-estimation": { + name: "Depth Estimation", + modality: "cv", + }, + "image-classification": { + name: "Image Classification", + subtasks: [ + { + type: "multi-label-image-classification", + name: "Multi Label Image Classification", + }, + { + type: "multi-class-image-classification", + name: "Multi Class Image Classification", + }, + ], + modality: "cv", + }, + "object-detection": { + name: "Object Detection", + subtasks: [ + { + type: "face-detection", + name: "Face Detection", + }, + { + type: "vehicle-detection", + name: "Vehicle Detection", + }, + ], + modality: "cv", + }, + "image-segmentation": { + name: "Image Segmentation", + subtasks: [ + { + type: "instance-segmentation", + name: "Instance Segmentation", + }, + { + type: "semantic-segmentation", + name: "Semantic Segmentation", + }, + { + type: "panoptic-segmentation", + name: "Panoptic Segmentation", + }, + ], + modality: "cv", + }, + "text-to-image": { + name: "Text-to-Image", + modality: "cv", + }, + "image-to-text": { + name: "Image-to-Text", + subtasks: [ + { + type: "image-captioning", + name: "Image Captioning", + }, + ], + modality: "cv", + }, + "image-to-image": { + name: "Image-to-Image", + subtasks: [ + { + type: "image-inpainting", + name: "Image Inpainting", + }, + { + type: "image-colorization", + name: "Image Colorization", + }, + { + type: "super-resolution", + name: "Super Resolution", + }, + ], + modality: "cv", + }, + "image-to-video": { + name: "Image-to-Video", + modality: "cv", + }, + "unconditional-image-generation": { + name: "Unconditional Image Generation", + modality: "cv", + }, + "video-classification": { + name: "Video Classification", + modality: "cv", + }, + "reinforcement-learning": { + name: "Reinforcement Learning", + modality: "rl", + }, + robotics: { + name: "Robotics", + modality: "rl", + subtasks: [ + { + type: "grasping", + name: "Grasping", + }, + { + type: "task-planning", + name: "Task Planning", + }, + ], + }, + "tabular-classification": { + name: "Tabular Classification", + modality: "tabular", + subtasks: [ + { + type: "tabular-multi-class-classification", + name: "Tabular Multi Class Classification", + }, + { + type: "tabular-multi-label-classification", + name: "Tabular Multi Label Classification", + }, + ], + }, + "tabular-regression": { + name: "Tabular Regression", + modality: "tabular", + subtasks: [ + { + type: "tabular-single-column-regression", + name: "Tabular Single Column Regression", + }, + ], + }, + "tabular-to-text": { + name: "Tabular to Text", + modality: "tabular", + subtasks: [ + { + type: "rdf-to-text", + name: "RDF to text", + }, + ], + hideInModels: true, + }, + "table-to-text": { + name: "Table to Text", + modality: "nlp", + hideInModels: true, + }, + "multiple-choice": { + name: "Multiple Choice", + subtasks: [ + { + type: "multiple-choice-qa", + name: "Multiple Choice QA", + }, + { + type: "multiple-choice-coreference-resolution", + name: "Multiple Choice Coreference Resolution", + }, + ], + modality: "nlp", + hideInModels: true, + }, + "text-ranking": { + name: "Text Ranking", + modality: "nlp", + }, + "text-retrieval": { + name: "Text Retrieval", + subtasks: [ + { + type: "document-retrieval", + name: "Document Retrieval", + }, + { + type: "utterance-retrieval", + name: "Utterance Retrieval", + }, + { + type: "entity-linking-retrieval", + name: "Entity Linking Retrieval", + }, + { + type: "fact-checking-retrieval", + name: "Fact Checking Retrieval", + }, + ], + modality: "nlp", + hideInModels: true, + }, + "time-series-forecasting": { + name: "Time Series Forecasting", + modality: "tabular", + subtasks: [ + { + type: "univariate-time-series-forecasting", + name: "Univariate Time Series Forecasting", + }, + { + type: "multivariate-time-series-forecasting", + name: "Multivariate Time Series Forecasting", + }, + ], + }, + "text-to-video": { + name: "Text-to-Video", + modality: "cv", + }, + "image-text-to-text": { + name: "Image-Text-to-Text", + modality: "multimodal", + }, + "image-text-to-image": { + name: "Image-Text-to-Image", + modality: "multimodal", + }, + "image-text-to-video": { + name: "Image-Text-to-Video", + modality: "multimodal", + }, + "visual-question-answering": { + name: "Visual Question Answering", + subtasks: [ + { + type: "visual-question-answering", + name: "Visual Question Answering", + }, + ], + modality: "multimodal", + }, + "document-question-answering": { + name: "Document Question Answering", + subtasks: [ + { + type: "document-question-answering", + name: "Document Question Answering", + }, + ], + modality: "multimodal", + hideInDatasets: true, + }, + "zero-shot-image-classification": { + name: "Zero-Shot Image Classification", + modality: "cv", + }, + "graph-ml": { + name: "Graph Machine Learning", + modality: "other", + }, + "mask-generation": { + name: "Mask Generation", + modality: "cv", + }, + "zero-shot-object-detection": { + name: "Zero-Shot Object Detection", + modality: "cv", + }, + "text-to-3d": { + name: "Text-to-3D", + modality: "cv", + }, + "image-to-3d": { + name: "Image-to-3D", + modality: "cv", + }, + "image-feature-extraction": { + name: "Image Feature Extraction", + modality: "cv", + }, + "video-text-to-text": { + name: "Video-Text-to-Text", + modality: "multimodal", + hideInDatasets: false, + }, + "keypoint-detection": { + name: "Keypoint Detection", + subtasks: [ + { + type: "pose-estimation", + name: "Pose Estimation", + }, + ], + modality: "cv", + hideInDatasets: true, + }, + "visual-document-retrieval": { + name: "Visual Document Retrieval", + modality: "multimodal", + }, + "any-to-any": { + name: "Any-to-Any", + modality: "multimodal", + }, + "video-to-video": { + name: "Video-to-Video", + modality: "cv", + hideInDatasets: true, + }, + other: { + name: "Other", + modality: "other", + hideInModels: true, + hideInDatasets: true, + }, +}; +exports.PIPELINE_TYPES = Object.keys(exports.PIPELINE_DATA); +exports.SUBTASK_TYPES = Object.values(exports.PIPELINE_DATA) + .flatMap((data) => ("subtasks" in data ? data.subtasks : [])) + .map((s) => s.type); +exports.PIPELINE_TYPES_SET = new Set(exports.PIPELINE_TYPES); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/snippets/common.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/snippets/common.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..bf5e9f386dcfeaef440383931a5892d91c9f01f8 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/snippets/common.d.ts @@ -0,0 +1,14 @@ +import type { ChatCompletionInputMessage, GenerationParameters } from "../tasks/index.js"; +export declare function stringifyMessages(messages: ChatCompletionInputMessage[], opts?: { + indent?: string; + attributeKeyQuotes?: boolean; + customContentEscaper?: (str: string) => string; +}): string; +type PartialGenerationParameters = Partial>; +export declare function stringifyGenerationConfig(config: PartialGenerationParameters, opts: { + indent: string; + attributeValueConnector: string; + attributeKeyQuotes?: boolean; +}): string; +export {}; +//# sourceMappingURL=common.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/snippets/common.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/snippets/common.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..22fc37561512c88d91a21b3ff56a7c29cdd680d6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/snippets/common.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../../src/snippets/common.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAE1F,wBAAgB,iBAAiB,CAChC,QAAQ,EAAE,0BAA0B,EAAE,EACtC,IAAI,CAAC,EAAE;IACN,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,oBAAoB,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;CAC/C,GACC,MAAM,CAYR;AAED,KAAK,2BAA2B,GAAG,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,aAAa,GAAG,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC;AAE/G,wBAAgB,yBAAyB,CACxC,MAAM,EAAE,2BAA2B,EACnC,IAAI,EAAE;IACL,MAAM,EAAE,MAAM,CAAC;IACf,uBAAuB,EAAE,MAAM,CAAC;IAChC,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC7B,GACC,MAAM,CAMR"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/snippets/common.js b/node_modules/@huggingface/tasks/dist/commonjs/snippets/common.js new file mode 100644 index 0000000000000000000000000000000000000000..bd6d395ed86cd9da6976465ae84228d23dd3d89c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/snippets/common.js @@ -0,0 +1,23 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.stringifyMessages = stringifyMessages; +exports.stringifyGenerationConfig = stringifyGenerationConfig; +function stringifyMessages(messages, opts) { + let messagesStr = JSON.stringify(messages, null, "\t"); + if (opts?.indent) { + messagesStr = messagesStr.replaceAll("\n", `\n${opts.indent}`); + } + if (!opts?.attributeKeyQuotes) { + messagesStr = messagesStr.replace(/"([^"]+)":/g, "$1:"); + } + if (opts?.customContentEscaper) { + messagesStr = opts.customContentEscaper(messagesStr); + } + return messagesStr; +} +function stringifyGenerationConfig(config, opts) { + const quote = opts.attributeKeyQuotes ? `"` : ""; + return Object.entries(config) + .map(([key, val]) => `${quote}${key}${quote}${opts.attributeValueConnector}${val},`) + .join(`${opts.indent}`); +} diff --git a/node_modules/@huggingface/tasks/dist/commonjs/snippets/index.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/snippets/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..991b953e5f6ee7ddd737c3ad8c2a674f6197c98d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/snippets/index.d.ts @@ -0,0 +1,4 @@ +export * from "./common.js"; +export * from "./inputs.js"; +export * from "./types.js"; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/snippets/index.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/snippets/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..548a10edf879f1c90bf3b4e790a3ee666694d74f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/snippets/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/snippets/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/snippets/index.js b/node_modules/@huggingface/tasks/dist/commonjs/snippets/index.js new file mode 100644 index 0000000000000000000000000000000000000000..1cc09552069bf9c84078aa41270d83b37d5e3353 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/snippets/index.js @@ -0,0 +1,19 @@ +"use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + var desc = Object.getOwnPropertyDescriptor(m, k); + if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { + desc = { enumerable: true, get: function() { return m[k]; } }; + } + Object.defineProperty(o, k2, desc); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __exportStar = (this && this.__exportStar) || function(m, exports) { + for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +__exportStar(require("./common.js"), exports); +__exportStar(require("./inputs.js"), exports); +__exportStar(require("./types.js"), exports); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/snippets/inputs.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/snippets/inputs.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b7195b500182bf3159236cc9a51dfd441b5c4743 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/snippets/inputs.d.ts @@ -0,0 +1,4 @@ +import type { ChatCompletionInputMessage } from "../tasks/index.js"; +import type { ModelDataMinimal } from "./types.js"; +export declare function getModelInputSnippet(model: ModelDataMinimal, noWrap?: boolean, noQuotes?: boolean): string | ChatCompletionInputMessage[]; +//# sourceMappingURL=inputs.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/snippets/inputs.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/snippets/inputs.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e78561c1f373ff55e20a0e5a281a40ddfcb99836 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/snippets/inputs.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inputs.d.ts","sourceRoot":"","sources":["../../../src/snippets/inputs.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAqKnD,wBAAgB,oBAAoB,CACnC,KAAK,EAAE,gBAAgB,EACvB,MAAM,UAAQ,EACd,QAAQ,UAAQ,GACd,MAAM,GAAG,0BAA0B,EAAE,CAmBvC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/snippets/inputs.js b/node_modules/@huggingface/tasks/dist/commonjs/snippets/inputs.js new file mode 100644 index 0000000000000000000000000000000000000000..4a7badfae5da53c5e405245a8e5947a358442950 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/snippets/inputs.js @@ -0,0 +1,147 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.getModelInputSnippet = getModelInputSnippet; +const inputsZeroShotClassification = () => `"Hi, I recently bought a device from your company but it is not working as advertised and I would like to get reimbursed!"`; +const inputsTranslation = () => `"Меня зовут Вольфганг и я живу в Берлине"`; +const inputsSummarization = () => `"The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct."`; +const inputsTableQuestionAnswering = () => `{ + "query": "How many stars does the transformers repository have?", + "table": { + "Repository": ["Transformers", "Datasets", "Tokenizers"], + "Stars": ["36542", "4512", "3934"], + "Contributors": ["651", "77", "34"], + "Programming language": [ + "Python", + "Python", + "Rust, Python and NodeJS" + ] + } +}`; +const inputsVisualQuestionAnswering = () => `{ + "image": "cat.png", + "question": "What is in this image?" + }`; +const inputsQuestionAnswering = () => `{ + "question": "What is my name?", + "context": "My name is Clara and I live in Berkeley." +}`; +const inputsTextClassification = () => `"I like you. I love you"`; +const inputsTokenClassification = () => `"My name is Sarah Jessica Parker but you can call me Jessica"`; +const inputsTextGeneration = (model) => { + if (model.tags.includes("conversational")) { + return model.pipeline_tag === "text-generation" + ? [{ role: "user", content: "What is the capital of France?" }] + : [ + { + role: "user", + content: [ + { + type: "text", + text: "Describe this image in one sentence.", + }, + { + type: "image_url", + image_url: { + url: "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg", + }, + }, + ], + }, + ]; + } + return `"Can you please let us know more details about your "`; +}; +const inputsFillMask = (model) => `"The answer to the universe is ${model.mask_token}."`; +const inputsSentenceSimilarity = () => `{ + "source_sentence": "That is a happy person", + "sentences": [ + "That is a happy dog", + "That is a very happy person", + "Today is a sunny day" + ] +}`; +const inputsFeatureExtraction = () => `"Today is a sunny day and I will get some ice cream."`; +const inputsImageClassification = () => `"cats.jpg"`; +const inputsImageToText = () => `"cats.jpg"`; +const inputsImageToImage = () => `{ + "image": "cat.png", + "prompt": "Turn the cat into a tiger." +}`; +const inputsImageToVideo = () => `{ + "image": "cat.png", + "prompt": "The cat starts to dance" +}`; +const inputsImageTextToImage = () => `{ + "image": "cat.png", + "prompt": "Turn the cat into a tiger." +}`; +const inputsImageTextToVideo = () => `{ + "image": "cat.png", + "prompt": "The cat starts to dance" +}`; +const inputsImageSegmentation = () => `"cats.jpg"`; +const inputsObjectDetection = () => `"cats.jpg"`; +const inputsAudioToAudio = () => `"sample1.flac"`; +const inputsAudioClassification = () => `"sample1.flac"`; +const inputsTextToImage = () => `"Astronaut riding a horse"`; +const inputsTextToVideo = () => `"A young man walking on the street"`; +const inputsTextToSpeech = () => `"The answer to the universe is 42"`; +const inputsTextToAudio = () => `"liquid drum and bass, atmospheric synths, airy sounds"`; +const inputsAutomaticSpeechRecognition = () => `"sample1.flac"`; +const inputsTabularPrediction = () => `'{"Height":[11.52,12.48],"Length1":[23.2,24.0],"Length2":[25.4,26.3],"Species": ["Bream","Bream"]}'`; +const inputsZeroShotImageClassification = () => `"cats.jpg"`; +const modelInputSnippets = { + "audio-to-audio": inputsAudioToAudio, + "audio-classification": inputsAudioClassification, + "automatic-speech-recognition": inputsAutomaticSpeechRecognition, + "document-question-answering": inputsVisualQuestionAnswering, + "feature-extraction": inputsFeatureExtraction, + "fill-mask": inputsFillMask, + "image-classification": inputsImageClassification, + "image-to-text": inputsImageToText, + "image-to-image": inputsImageToImage, + "image-to-video": inputsImageToVideo, + "image-text-to-image": inputsImageTextToImage, + "image-text-to-video": inputsImageTextToVideo, + "image-segmentation": inputsImageSegmentation, + "object-detection": inputsObjectDetection, + "question-answering": inputsQuestionAnswering, + "sentence-similarity": inputsSentenceSimilarity, + summarization: inputsSummarization, + "table-question-answering": inputsTableQuestionAnswering, + "tabular-regression": inputsTabularPrediction, + "tabular-classification": inputsTabularPrediction, + "text-classification": inputsTextClassification, + "text-generation": inputsTextGeneration, + "image-text-to-text": inputsTextGeneration, + "text-to-image": inputsTextToImage, + "text-to-video": inputsTextToVideo, + "text-to-speech": inputsTextToSpeech, + "text-to-audio": inputsTextToAudio, + "token-classification": inputsTokenClassification, + translation: inputsTranslation, + "zero-shot-classification": inputsZeroShotClassification, + "zero-shot-image-classification": inputsZeroShotImageClassification, +}; +// Use noWrap to put the whole snippet on a single line (removing new lines and tabulations) +// Use noQuotes to strip quotes from start & end (example: "abc" -> abc) +function getModelInputSnippet(model, noWrap = false, noQuotes = false) { + if (model.pipeline_tag) { + const inputs = modelInputSnippets[model.pipeline_tag]; + if (inputs) { + let result = inputs(model); + if (typeof result === "string") { + if (noWrap) { + result = result.replace(/(?:(?:\r?\n|\r)\t*)|\t+/g, " "); + } + if (noQuotes) { + const REGEX_QUOTES = /^"(.+)"$/s; + const match = result.match(REGEX_QUOTES); + result = match ? match[1] : result; + } + } + return result; + } + } + return "No input example has been defined for this model task."; +} diff --git a/node_modules/@huggingface/tasks/dist/commonjs/snippets/types.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/snippets/types.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..684489ef8f7272cde9a9f5039481a92122373f5f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/snippets/types.d.ts @@ -0,0 +1,15 @@ +import type { ModelData } from "../model-data.js"; +/** + * Minimal model data required for snippets. + * + * Add more fields as needed. + */ +export type ModelDataMinimal = Pick; +export declare const inferenceSnippetLanguages: readonly ["python", "js", "sh"]; +export type InferenceSnippetLanguage = (typeof inferenceSnippetLanguages)[number]; +export interface InferenceSnippet { + language: InferenceSnippetLanguage; + client: string; + content: string; +} +//# sourceMappingURL=types.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/snippets/types.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/snippets/types.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..3e04ceef9eea3456ea042301bd21850e9794f79b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/snippets/types.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/snippets/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAElD;;;;GAIG;AACH,MAAM,MAAM,gBAAgB,GAAG,IAAI,CAClC,SAAS,EACT,IAAI,GAAG,cAAc,GAAG,YAAY,GAAG,cAAc,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CACvF,CAAC;AAGF,eAAO,MAAM,yBAAyB,iCAAkC,CAAC;AACzE,MAAM,MAAM,wBAAwB,GAAG,CAAC,OAAO,yBAAyB,CAAC,CAAC,MAAM,CAAC,CAAC;AAElF,MAAM,WAAW,gBAAgB;IAChC,QAAQ,EAAE,wBAAwB,CAAC;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CAChB"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/snippets/types.js b/node_modules/@huggingface/tasks/dist/commonjs/snippets/types.js new file mode 100644 index 0000000000000000000000000000000000000000..2df04a6055fd934776dfd0cd70d7288ebbad3674 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/snippets/types.js @@ -0,0 +1,5 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.inferenceSnippetLanguages = void 0; +// Order of the elements in InferenceModal.svelte is determined by this const +exports.inferenceSnippetLanguages = ["python", "js", "sh"]; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/any-to-any/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/any-to-any/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/any-to-any/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/any-to-any/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/any-to-any/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..316d468d5a30916ed526b156e185469fadbf2567 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/any-to-any/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/any-to-any/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA4Df,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/any-to-any/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/any-to-any/data.js new file mode 100644 index 0000000000000000000000000000000000000000..d4e0ca260bf1c1bd6b29caefbba8dff501232bb8 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/any-to-any/data.js @@ -0,0 +1,63 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A dataset with multiple modality input and output pairs.", + id: "PKU-Alignment/align-anything", + }, + ], + demo: { + inputs: [ + { + filename: "any-to-any-input.jpg", + type: "img", + }, + { + label: "Text Prompt", + content: "What is the significance of this place?", + type: "text", + }, + ], + outputs: [ + { + label: "Generated Text", + content: "The place in the picture is Osaka Castle, located in Osaka, Japan. Osaka Castle is a historic castle that was originally built in the 16th century by Toyotomi Hideyoshi, a powerful warlord of the time. It is one of the most famous landmarks in Osaka and is known for its distinctive white walls and black roof tiles. The castle has been rebuilt several times over the centuries and is now a popular tourist attraction, offering visitors a glimpse into Japan's rich history and culture.", + type: "text", + }, + { + filename: "any-to-any-output.wav", + type: "audio", + }, + ], + }, + metrics: [], + models: [ + { + description: "Strong model that can take in video, audio, image, text and output text and natural speech.", + id: "Qwen/Qwen2.5-Omni-7B", + }, + { + description: "Robust model that can take in image and text and generate image and text.", + id: "OmniGen2/OmniGen2", + }, + { + description: "Any-to-any model with speech, video, audio, image and text understanding capabilities.", + id: "openbmb/MiniCPM-o-2_6", + }, + { + description: "A model that can understand image and text and generate image and text.", + id: "ByteDance-Seed/BAGEL-7B-MoT", + }, + ], + spaces: [ + { + description: "An application to chat with an any-to-any (image & text) model.", + id: "OmniGen2/OmniGen2", + }, + ], + summary: "Any-to-any models can understand two or more modalities and output two or more modalities.", + widgetModels: [], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..bf29381612f3a120b322349e0cbf0b3d83167dc5 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/audio-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA4Ef,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..72d39597b4d0a9b2533be264235b23dc80420b8b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/data.js @@ -0,0 +1,79 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A benchmark of 10 different audio tasks.", + id: "s3prl/superb", + }, + { + description: "A dataset of YouTube clips and their sound categories.", + id: "agkphysics/AudioSet", + }, + ], + demo: { + inputs: [ + { + filename: "audio.wav", + type: "audio", + }, + ], + outputs: [ + { + data: [ + { + label: "Up", + score: 0.2, + }, + { + label: "Down", + score: 0.8, + }, + ], + type: "chart", + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + description: "An easy-to-use model for command recognition.", + id: "speechbrain/google_speech_command_xvector", + }, + { + description: "An emotion recognition model.", + id: "ehcalabres/wav2vec2-lg-xlsr-en-speech-emotion-recognition", + }, + { + description: "A language identification model.", + id: "facebook/mms-lid-126", + }, + ], + spaces: [ + { + description: "An application that can classify music into different genre.", + id: "kurianbenoy/audioclassification", + }, + ], + summary: "Audio classification is the task of assigning a label or class to a given audio. It can be used for recognizing which command a user is giving or the emotion of a statement, as well as identifying a speaker.", + widgetModels: ["MIT/ast-finetuned-audioset-10-10-0.4593"], + youtubeId: "KWwzcmG98Ds", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ddc1cffad496de5105c0ecf687f7302737086f82 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/inference.d.ts @@ -0,0 +1,54 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Audio Classification inference + */ +export interface AudioClassificationInput { + /** + * The input audio data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the audio data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Audio Classification + */ + parameters?: AudioClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Audio Classification + */ +export interface AudioClassificationParameters { + /** + * The function to apply to the model outputs in order to retrieve the scores. + */ + function_to_apply?: ClassificationOutputTransform; + /** + * When specified, limits the output to the top K most probable classes. + */ + top_k?: number; + [property: string]: unknown; +} +/** + * The function to apply to the model outputs in order to retrieve the scores. + */ +export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none"; +export type AudioClassificationOutput = AudioClassificationOutputElement[]; +/** + * Outputs for Audio Classification inference + */ +export interface AudioClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..36b8340d1109765339d12eda90f6e0d6c8be3e04 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/audio-classification/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACxC;;;OAGG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,6BAA6B,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC7C;;OAEG;IACH,iBAAiB,CAAC,EAAE,6BAA6B,CAAC;IAClD;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAC3E,MAAM,MAAM,yBAAyB,GAAG,gCAAgC,EAAE,CAAC;AAC3E;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-classification/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-text-to-text/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-text-to-text/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-text-to-text/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-text-to-text/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-text-to-text/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5a31cfccf308085ac90260ae5454e2c10df544b2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-text-to-text/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/audio-text-to-text/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAiEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-text-to-text/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-text-to-text/data.js new file mode 100644 index 0000000000000000000000000000000000000000..77222904e70ad7ea1a1b89d5b249ef951dbba495 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-text-to-text/data.js @@ -0,0 +1,67 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A dataset containing audio conversations with question–answer pairs.", + id: "nvidia/AF-Think", + }, + { + description: "A more advanced and comprehensive dataset that contains characteristics of the audio as well", + id: "tsinghua-ee/QualiSpeech", + }, + ], + demo: { + inputs: [ + { + filename: "audio.wav", + type: "audio", + }, + { + label: "Text Prompt", + content: "What is the gender of the speaker?", + type: "text", + }, + ], + outputs: [ + { + label: "Generated Text", + content: "The gender of the speaker is female.", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "A lightweight model that has capabilities of taking both audio and text as inputs and generating responses.", + id: "fixie-ai/ultravox-v0_5-llama-3_2-1b", + }, + { + description: "A multimodal model that supports voice chat and audio analysis.", + id: "Qwen/Qwen2-Audio-7B-Instruct", + }, + { + description: "A model for audio understanding, speech translation, and transcription.", + id: "mistralai/Voxtral-Small-24B-2507", + }, + { + description: "A new model capable of audio question answering and reasoning.", + id: "nvidia/audio-flamingo-3", + }, + ], + spaces: [ + { + description: "A space that takes input as both audio and text and generates answers.", + id: "iamomtiwari/ATTT", + }, + { + description: "A web application that demonstrates chatting with the Qwen2Audio Model.", + id: "freddyaboulton/talk-to-qwen-webrtc", + }, + ], + summary: "Audio-text-to-text models take both an audio clip and a text prompt as input, and generate natural language text as output. These models can answer questions about spoken content, summarize meetings, analyze music, or interpret speech beyond simple transcription. They are useful for applications that combine speech understanding with reasoning or conversation.", + widgetModels: [], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-to-audio/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-to-audio/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-to-audio/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-to-audio/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-to-audio/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e20ea8823de86f46b9aa9c8658cf637fb2e9d198 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-to-audio/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/audio-to-audio/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA6Df,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-to-audio/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-to-audio/data.js new file mode 100644 index 0000000000000000000000000000000000000000..8c9c834c8f5b7cd4b5db7642e43eab303be238ba --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/audio-to-audio/data.js @@ -0,0 +1,62 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "512-element X-vector embeddings of speakers from CMU ARCTIC dataset.", + id: "Matthijs/cmu-arctic-xvectors", + }, + ], + demo: { + inputs: [ + { + filename: "input.wav", + type: "audio", + }, + ], + outputs: [ + { + filename: "label-0.wav", + type: "audio", + }, + { + filename: "label-1.wav", + type: "audio", + }, + ], + }, + metrics: [ + { + description: "The Signal-to-Noise ratio is the relationship between the target signal level and the background noise level. It is calculated as the logarithm of the target signal divided by the background noise, in decibels.", + id: "snri", + }, + { + description: "The Signal-to-Distortion ratio is the relationship between the target signal and the sum of noise, interference, and artifact errors", + id: "sdri", + }, + ], + models: [ + { + description: "A speech enhancement model.", + id: "ResembleAI/resemble-enhance", + }, + { + description: "A model that can change the voice in a speech recording.", + id: "microsoft/speecht5_vc", + }, + ], + spaces: [ + { + description: "An application for speech separation.", + id: "younver/speechbrain-speech-separation", + }, + { + description: "An application for audio style transfer.", + id: "nakas/audio-diffusion_style_transfer", + }, + ], + summary: "Audio-to-Audio is a family of tasks in which the input is an audio and the output is one or multiple generated audios. Some example tasks are speech enhancement and source separation.", + widgetModels: ["speechbrain/sepformer-wham"], + youtubeId: "iohj7nCCYoM", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..987e72b47dbd3da5b501f1b9a5a875a736b3af59 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/automatic-speech-recognition/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAyFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/data.js new file mode 100644 index 0000000000000000000000000000000000000000..601f58d8adf6819a3614e47526baf20859daaeb2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/data.js @@ -0,0 +1,92 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "31,175 hours of multilingual audio-text dataset in 108 languages.", + id: "mozilla-foundation/common_voice_17_0", + }, + { + description: "Multilingual and diverse audio dataset with 101k hours of audio.", + id: "amphion/Emilia-Dataset", + }, + { + description: "A dataset with 44.6k hours of English speaker data and 6k hours of other language speakers.", + id: "parler-tts/mls_eng", + }, + { + description: "A multilingual audio dataset with 370K hours of audio.", + id: "espnet/yodas", + }, + ], + demo: { + inputs: [ + { + filename: "input.flac", + type: "audio", + }, + ], + outputs: [ + { + /// GOING ALONG SLUSHY COUNTRY ROADS AND SPEAKING TO DAMP AUDIENCES I + label: "Transcript", + content: "Going along slushy country roads and speaking to damp audiences in...", + type: "text", + }, + ], + }, + metrics: [ + { + description: "", + id: "wer", + }, + { + description: "", + id: "cer", + }, + ], + models: [ + { + description: "A powerful ASR model by OpenAI.", + id: "openai/whisper-large-v3", + }, + { + description: "A good generic speech model by MetaAI for fine-tuning.", + id: "facebook/w2v-bert-2.0", + }, + { + description: "An end-to-end model that performs ASR and Speech Translation by MetaAI.", + id: "facebook/seamless-m4t-v2-large", + }, + { + description: "A powerful multilingual ASR and Speech Translation model by Nvidia.", + id: "nvidia/canary-1b", + }, + { + description: "Powerful speaker diarization model.", + id: "pyannote/speaker-diarization-3.1", + }, + ], + spaces: [ + { + description: "A powerful general-purpose speech recognition application.", + id: "hf-audio/whisper-large-v3", + }, + { + description: "Latest ASR model from Useful Sensors.", + id: "mrfakename/Moonshinex", + }, + { + description: "A high quality speech and text translation model by Meta.", + id: "facebook/seamless_m4t", + }, + { + description: "A powerful multilingual ASR and Speech Translation model by Nvidia", + id: "nvidia/canary-1b", + }, + ], + summary: "Automatic Speech Recognition (ASR), also known as Speech to Text (STT), is the task of transcribing a given audio to text. It has many applications, such as voice user interfaces.", + widgetModels: ["openai/whisper-large-v3"], + youtubeId: "TksaY_FDgnk", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..66b335f310af19f6c5741e82287897ecad32f647 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/inference.d.ts @@ -0,0 +1,151 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Automatic Speech Recognition inference + */ +export interface AutomaticSpeechRecognitionInput { + /** + * The input audio data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the audio data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Automatic Speech Recognition + */ + parameters?: AutomaticSpeechRecognitionParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Automatic Speech Recognition + */ +export interface AutomaticSpeechRecognitionParameters { + /** + * Parametrization of the text generation process + */ + generation_parameters?: GenerationParameters; + /** + * Whether to output corresponding timestamps with the generated text + */ + return_timestamps?: boolean; + [property: string]: unknown; +} +/** + * Parametrization of the text generation process + */ +export interface GenerationParameters { + /** + * Whether to use sampling instead of greedy decoding when generating new tokens. + */ + do_sample?: boolean; + /** + * Controls the stopping condition for beam-based methods. + */ + early_stopping?: EarlyStoppingUnion; + /** + * If set to float strictly between 0 and 1, only tokens with a conditional probability + * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + * Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + */ + epsilon_cutoff?: number; + /** + * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + * float strictly between 0 and 1, a token is only considered if it is greater than either + * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + * for more details. + */ + eta_cutoff?: number; + /** + * The maximum length (in tokens) of the generated text, including the input. + */ + max_length?: number; + /** + * The maximum number of tokens to generate. Takes precedence over max_length. + */ + max_new_tokens?: number; + /** + * The minimum length (in tokens) of the generated text, including the input. + */ + min_length?: number; + /** + * The minimum number of tokens to generate. Takes precedence over min_length. + */ + min_new_tokens?: number; + /** + * Number of groups to divide num_beams into in order to ensure diversity among different + * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + */ + num_beam_groups?: number; + /** + * Number of beams to use for beam search. + */ + num_beams?: number; + /** + * The value balances the model confidence and the degeneration penalty in contrastive + * search decoding. + */ + penalty_alpha?: number; + /** + * The value used to modulate the next token probabilities. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * If set to float < 1, only the smallest set of most probable tokens with probabilities + * that add up to top_p or higher are kept for generation. + */ + top_p?: number; + /** + * Local typicality measures how similar the conditional probability of predicting a target + * token next is to the expected conditional probability of predicting a random token next, + * given the partial text already generated. If set to float < 1, the smallest set of the + * most locally typical tokens with probabilities that add up to typical_p or higher are + * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + */ + typical_p?: number; + /** + * Whether the model should use the past last key/values attentions to speed up decoding + */ + use_cache?: boolean; + [property: string]: unknown; +} +/** + * Controls the stopping condition for beam-based methods. + */ +export type EarlyStoppingUnion = boolean | "never"; +/** + * Outputs of inference for the Automatic Speech Recognition task + */ +export interface AutomaticSpeechRecognitionOutput { + /** + * When returnTimestamps is enabled, chunks contains a list of audio chunks identified by + * the model. + */ + chunks?: AutomaticSpeechRecognitionOutputChunk[]; + /** + * The recognized text. + */ + text: string; + [property: string]: unknown; +} +export interface AutomaticSpeechRecognitionOutputChunk { + /** + * A chunk of text identified by the model + */ + text: string; + /** + * The start and end timestamps corresponding with the text + */ + timestamp: number[]; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..dece49d1b3a428996138b9dce1065a1c4b72d803 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/automatic-speech-recognition/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,+BAA+B;IAC/C;;;OAGG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,oCAAoC,CAAC;IAClD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,oCAAoC;IACpD;;OAEG;IACH,qBAAqB,CAAC,EAAE,oBAAoB,CAAC;IAC7C;;OAEG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,kBAAkB,CAAC;IACpC;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,OAAO,GAAG,OAAO,CAAC;AACnD;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;;OAGG;IACH,MAAM,CAAC,EAAE,qCAAqC,EAAE,CAAC;IACjD;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,qCAAqC;IACrD;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/automatic-speech-recognition/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/chat-completion/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/chat-completion/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..3f6f888f2a3d197dd648129c1d2dde804f6c248f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/chat-completion/inference.d.ts @@ -0,0 +1,334 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Chat Completion Input. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface ChatCompletionInput { + /** + * Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing + * frequency in the text so far, + * decreasing the model's likelihood to repeat the same line verbatim. + */ + frequency_penalty?: number; + /** + * UNUSED + * Modify the likelihood of specified tokens appearing in the completion. Accepts a JSON + * object that maps tokens + * (specified by their token ID in the tokenizer) to an associated bias value from -100 to + * 100. Mathematically, + * the bias is added to the logits generated by the model prior to sampling. The exact + * effect will vary per model, + * but values between -1 and 1 should decrease or increase likelihood of selection; values + * like -100 or 100 should + * result in a ban or exclusive selection of the relevant token. + */ + logit_bias?: number[]; + /** + * Whether to return log probabilities of the output tokens or not. If true, returns the log + * probabilities of each + * output token returned in the content of message. + */ + logprobs?: boolean; + /** + * The maximum number of tokens that can be generated in the chat completion. + */ + max_tokens?: number; + /** + * A list of messages comprising the conversation so far. + */ + messages: ChatCompletionInputMessage[]; + /** + * [UNUSED] ID of the model to use. See the model endpoint compatibility table for details + * on which models work with the Chat API. + */ + model?: string; + /** + * UNUSED + * How many chat completion choices to generate for each input message. Note that you will + * be charged based on the + * number of generated tokens across all of the choices. Keep n as 1 to minimize costs. + */ + n?: number; + /** + * Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they + * appear in the text so far, + * increasing the model's likelihood to talk about new topics + */ + presence_penalty?: number; + /** + * Optional. Constrains effort on reasoning for reasoning models. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning. Common values: none, minimal, low, medium, high, xhigh. Support and defaults are provider and model-dependent. + */ + reasoning_effort?: string; + response_format?: ChatCompletionInputGrammarType; + seed?: number; + /** + * Up to 4 sequences where the API will stop generating further tokens. + */ + stop?: string[]; + stream?: boolean; + stream_options?: ChatCompletionInputStreamOptions; + /** + * What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the + * output more random, while + * lower values like 0.2 will make it more focused and deterministic. + * + * We generally recommend altering this or `top_p` but not both. + */ + temperature?: number; + tool_choice?: ChatCompletionInputToolChoice; + /** + * A prompt to be appended before the tools + */ + tool_prompt?: string; + /** + * A list of tools the model may call. Currently, only functions are supported as a tool. + * Use this to provide a list of + * functions the model may generate JSON inputs for. + */ + tools?: ChatCompletionInputTool[]; + /** + * An integer between 0 and 5 specifying the number of most likely tokens to return at each + * token position, each with + * an associated log probability. logprobs must be set to true if this parameter is used. + */ + top_logprobs?: number; + /** + * An alternative to sampling with temperature, called nucleus sampling, where the model + * considers the results of the + * tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% + * probability mass are considered. + */ + top_p?: number; + [property: string]: unknown; +} +export interface ChatCompletionInputMessage { + content?: ChatCompletionInputMessageContent; + name?: string; + role: string; + tool_calls?: ChatCompletionInputToolCall[]; + [property: string]: unknown; +} +export type ChatCompletionInputMessageContent = ChatCompletionInputMessageChunk[] | string; +export interface ChatCompletionInputMessageChunk { + image_url?: ChatCompletionInputURL; + text?: string; + type: ChatCompletionInputMessageChunkType; + [property: string]: unknown; +} +export interface ChatCompletionInputURL { + url: string; + [property: string]: unknown; +} +export type ChatCompletionInputMessageChunkType = "text" | "image_url"; +export interface ChatCompletionInputToolCall { + function: ChatCompletionInputFunctionDefinition; + id: string; + type: string; + [property: string]: unknown; +} +export interface ChatCompletionInputFunctionDefinition { + description?: string; + name: string; + parameters?: unknown; + [property: string]: unknown; +} +export interface ChatCompletionInputGrammarType { + json_schema?: ChatCompletionInputJSONSchemaConfig; + type: ChatCompletionInputGrammarTypeType; + [property: string]: unknown; +} +export interface ChatCompletionInputJSONSchemaConfig { + /** + * A description of what the response format is for, used by the model to determine how to + * respond in the format. + */ + description?: string; + /** + * The name of the response format. + */ + name: string; + /** + * The schema for the response format, described as a JSON Schema object. Learn how to build + * JSON schemas [here](https://json-schema.org/). + */ + schema?: { + [key: string]: unknown; + }; + /** + * Whether to enable strict schema adherence when generating the output. If set to true, the + * model will always follow the exact schema defined in the `schema` field. + */ + strict?: boolean; + [property: string]: unknown; +} +export type ChatCompletionInputGrammarTypeType = "text" | "json_schema" | "json_object"; +export interface ChatCompletionInputStreamOptions { + /** + * If set, an additional chunk will be streamed before the data: [DONE] message. The usage + * field on this chunk shows the token usage statistics for the entire request, and the + * choices field will always be an empty array. All other chunks will also include a usage + * field, but with a null value. + */ + include_usage?: boolean; + [property: string]: unknown; +} +/** + * + * + */ +export type ChatCompletionInputToolChoice = ChatCompletionInputToolChoiceEnum | ChatCompletionInputToolChoiceObject; +/** + * Means the model can pick between generating a message or calling one or more tools. + * + * Means the model will not call any tool and instead generates a message. + * + * Means the model must call one or more tools. + */ +export type ChatCompletionInputToolChoiceEnum = "auto" | "none" | "required"; +export interface ChatCompletionInputToolChoiceObject { + function: ChatCompletionInputFunctionName; + [property: string]: unknown; +} +export interface ChatCompletionInputFunctionName { + name: string; + [property: string]: unknown; +} +export interface ChatCompletionInputTool { + function: ChatCompletionInputFunctionDefinition; + type: string; + [property: string]: unknown; +} +/** + * Chat Completion Output. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface ChatCompletionOutput { + choices: ChatCompletionOutputComplete[]; + created: number; + id: string; + model: string; + system_fingerprint: string; + usage: ChatCompletionOutputUsage; + [property: string]: unknown; +} +export interface ChatCompletionOutputComplete { + finish_reason: string; + index: number; + logprobs?: ChatCompletionOutputLogprobs; + message: ChatCompletionOutputMessage; + [property: string]: unknown; +} +export interface ChatCompletionOutputLogprobs { + content: ChatCompletionOutputLogprob[]; + [property: string]: unknown; +} +export interface ChatCompletionOutputLogprob { + logprob: number; + token: string; + top_logprobs: ChatCompletionOutputTopLogprob[]; + [property: string]: unknown; +} +export interface ChatCompletionOutputTopLogprob { + logprob: number; + token: string; + [property: string]: unknown; +} +export interface ChatCompletionOutputMessage { + content?: string; + role: string; + tool_call_id?: string; + tool_calls?: ChatCompletionOutputToolCall[]; + [property: string]: unknown; +} +export interface ChatCompletionOutputToolCall { + function: ChatCompletionOutputFunctionDefinition; + id: string; + type: string; + [property: string]: unknown; +} +export interface ChatCompletionOutputFunctionDefinition { + arguments: string; + description?: string; + name: string; + [property: string]: unknown; +} +export interface ChatCompletionOutputUsage { + completion_tokens: number; + prompt_tokens: number; + total_tokens: number; + [property: string]: unknown; +} +/** + * Chat Completion Stream Output. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface ChatCompletionStreamOutput { + choices: ChatCompletionStreamOutputChoice[]; + created: number; + id: string; + model: string; + system_fingerprint: string; + usage?: ChatCompletionStreamOutputUsage; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputChoice { + delta: ChatCompletionStreamOutputDelta; + finish_reason?: string; + index: number; + logprobs?: ChatCompletionStreamOutputLogprobs; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputDelta { + content?: string; + role: string; + tool_call_id?: string; + tool_calls?: ChatCompletionStreamOutputDeltaToolCall[]; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputDeltaToolCall { + function: ChatCompletionStreamOutputFunction; + id: string; + index: number; + type: string; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputFunction { + arguments: string; + name?: string; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputLogprobs { + content: ChatCompletionStreamOutputLogprob[]; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputLogprob { + logprob: number; + token: string; + top_logprobs: ChatCompletionStreamOutputTopLogprob[]; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputTopLogprob { + logprob: number; + token: string; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputUsage { + completion_tokens: number; + prompt_tokens: number; + total_tokens: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/chat-completion/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/chat-completion/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c38ba2fd0d997d34b0bb5ecdeae8cda391c784c7 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/chat-completion/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/chat-completion/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IACnC;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,QAAQ,EAAE,0BAA0B,EAAE,CAAC;IACvC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,CAAC,CAAC,EAAE,MAAM,CAAC;IACX;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,8BAA8B,CAAC;IACjD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,gCAAgC,CAAC;IAClD;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,6BAA6B,CAAC;IAC5C;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,KAAK,CAAC,EAAE,uBAAuB,EAAE,CAAC;IAClC;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,0BAA0B;IAC1C,OAAO,CAAC,EAAE,iCAAiC,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,2BAA2B,EAAE,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,iCAAiC,GAAG,+BAA+B,EAAE,GAAG,MAAM,CAAC;AAC3F,MAAM,WAAW,+BAA+B;IAC/C,SAAS,CAAC,EAAE,sBAAsB,CAAC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,mCAAmC,CAAC;IAC1C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,sBAAsB;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,mCAAmC,GAAG,MAAM,GAAG,WAAW,CAAC;AACvE,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,EAAE,qCAAqC,CAAC;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,qCAAqC;IACrD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,8BAA8B;IAC9C,WAAW,CAAC,EAAE,mCAAmC,CAAC;IAClD,IAAI,EAAE,kCAAkC,CAAC;IACzC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,mCAAmC;IACnD;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,MAAM,CAAC,EAAE;QACR,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,kCAAkC,GAAG,MAAM,GAAG,aAAa,GAAG,aAAa,CAAC;AACxF,MAAM,WAAW,gCAAgC;IAChD;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,MAAM,6BAA6B,GAAG,iCAAiC,GAAG,mCAAmC,CAAC;AACpH;;;;;;GAMG;AACH,MAAM,MAAM,iCAAiC,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;AAC7E,MAAM,WAAW,mCAAmC;IACnD,QAAQ,EAAE,+BAA+B,CAAC;IAC1C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,qCAAqC,CAAC;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACpC,OAAO,EAAE,4BAA4B,EAAE,CAAC;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,yBAAyB,CAAC;IACjC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,4BAA4B,CAAC;IACxC,OAAO,EAAE,2BAA2B,CAAC;IACrC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,OAAO,EAAE,2BAA2B,EAAE,CAAC;IACvC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,8BAA8B,EAAE,CAAC;IAC/C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,8BAA8B;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,4BAA4B,EAAE,CAAC;IAC5C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,QAAQ,EAAE,sCAAsC,CAAC;IACjD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,sCAAsC;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,yBAAyB;IACzC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,0BAA0B;IAC1C,OAAO,EAAE,gCAAgC,EAAE,CAAC;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,CAAC,EAAE,+BAA+B,CAAC;IACxC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,gCAAgC;IAChD,KAAK,EAAE,+BAA+B,CAAC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,kCAAkC,CAAC;IAC9C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,uCAAuC,EAAE,CAAC;IACvD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,uCAAuC;IACvD,QAAQ,EAAE,kCAAkC,CAAC;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,kCAAkC;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,kCAAkC;IAClD,OAAO,EAAE,iCAAiC,EAAE,CAAC;IAC7C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,iCAAiC;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,oCAAoC,EAAE,CAAC;IACrD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,oCAAoC;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/chat-completion/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/chat-completion/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/chat-completion/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..cb232576320cf1d06f658b28334cf5d6afa01a01 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/depth-estimation/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAiEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/data.js new file mode 100644 index 0000000000000000000000000000000000000000..3a2618113845d1620d13fae1f45b52f33492986f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/data.js @@ -0,0 +1,69 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "NYU Depth V2 Dataset: Video dataset containing both RGB and depth sensor data.", + id: "sayakpaul/nyu_depth_v2", + }, + { + description: "Monocular depth estimation benchmark based without noise and errors.", + id: "depth-anything/DA-2K", + }, + ], + demo: { + inputs: [ + { + filename: "depth-estimation-input.jpg", + type: "img", + }, + ], + outputs: [ + { + filename: "depth-estimation-output.png", + type: "img", + }, + ], + }, + metrics: [], + models: [ + { + description: "Cutting-edge depth estimation model.", + id: "depth-anything/Depth-Anything-V2-Large", + }, + { + description: "A strong monocular depth estimation model.", + id: "jingheya/lotus-depth-g-v1-0", + }, + { + description: "A depth estimation model that predicts depth in videos.", + id: "tencent/DepthCrafter", + }, + { + description: "A robust depth estimation model.", + id: "apple/DepthPro-hf", + }, + ], + spaces: [ + { + description: "An application that predicts the depth of an image and then reconstruct the 3D model as voxels.", + id: "radames/dpt-depth-estimation-3d-voxels", + }, + { + description: "An application for bleeding-edge depth estimation.", + id: "akhaliq/depth-pro", + }, + { + description: "An application on cutting-edge depth estimation in videos.", + id: "tencent/DepthCrafter", + }, + { + description: "A human-centric depth estimation application.", + id: "facebook/sapiens-depth", + }, + ], + summary: "Depth estimation is the task of predicting depth of the objects present in an image.", + widgetModels: [""], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..7585f1ac33bb2d5ea520295d56da1ca4face71fb --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/inference.d.ts @@ -0,0 +1,36 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Depth Estimation inference + */ +export interface DepthEstimationInput { + /** + * The input image data + */ + inputs: unknown; + /** + * Additional inference parameters for Depth Estimation + */ + parameters?: { + [key: string]: unknown; + }; + [property: string]: unknown; +} +/** + * Outputs of inference for the Depth Estimation task + */ +export interface DepthEstimationOutput { + /** + * The predicted depth as an image + */ + depth?: unknown; + /** + * The predicted depth as a tensor + */ + predicted_depth?: unknown; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..033a915d9d5a928ebb48b36b6eda9f5e7cd5e4fb --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/depth-estimation/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC;IAChB;;OAEG;IACH,UAAU,CAAC,EAAE;QACZ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/depth-estimation/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..9c729fbee845c5ce760e5b2345f751973770add5 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/document-question-answering/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAgFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/data.js new file mode 100644 index 0000000000000000000000000000000000000000..f5207d13cb6ba039eb808a892595f0e18ce381cc --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/data.js @@ -0,0 +1,80 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "Largest document understanding dataset.", + id: "HuggingFaceM4/Docmatix", + }, + { + description: "Dataset from the 2020 DocVQA challenge. The documents are taken from the UCSF Industry Documents Library.", + id: "eliolio/docvqa", + }, + ], + demo: { + inputs: [ + { + label: "Question", + content: "What is the idea behind the consumer relations efficiency team?", + type: "text", + }, + { + filename: "document-question-answering-input.png", + type: "img", + }, + ], + outputs: [ + { + label: "Answer", + content: "Balance cost efficiency with quality customer service", + type: "text", + }, + ], + }, + metrics: [ + { + description: "The evaluation metric for the DocVQA challenge is the Average Normalized Levenshtein Similarity (ANLS). This metric is flexible to character regognition errors and compares the predicted answer with the ground truth answer.", + id: "anls", + }, + { + description: "Exact Match is a metric based on the strict character match of the predicted answer and the right answer. For answers predicted correctly, the Exact Match will be 1. Even if only one character is different, Exact Match will be 0", + id: "exact-match", + }, + ], + models: [ + { + description: "A robust document question answering model.", + id: "impira/layoutlm-document-qa", + }, + { + description: "A document question answering model specialized in invoices.", + id: "impira/layoutlm-invoices", + }, + { + description: "A special model for OCR-free document question answering.", + id: "microsoft/udop-large", + }, + { + description: "A powerful model for document question answering.", + id: "google/pix2struct-docvqa-large", + }, + ], + spaces: [ + { + description: "A robust document question answering application.", + id: "impira/docquery", + }, + { + description: "An application that can answer questions from invoices.", + id: "impira/invoices", + }, + { + description: "An application to compare different document question answering models.", + id: "merve/compare_docvqa_models", + }, + ], + summary: "Document Question Answering (also known as Document Visual Question Answering) is the task of answering questions on document images. Document question answering models take a (document, question) pair as input and return an answer in natural language. Models usually rely on multi-modal features, combining text, position of words (bounding-boxes) and image.", + widgetModels: ["impira/layoutlm-invoices"], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d0201e7778a3a2003841e54a7e944a2b59a46653 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/inference.d.ts @@ -0,0 +1,105 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Document Question Answering inference + */ +export interface DocumentQuestionAnsweringInput { + /** + * One (document, question) pair to answer + */ + inputs: DocumentQuestionAnsweringInputData; + /** + * Additional inference parameters for Document Question Answering + */ + parameters?: DocumentQuestionAnsweringParameters; + [property: string]: unknown; +} +/** + * One (document, question) pair to answer + */ +export interface DocumentQuestionAnsweringInputData { + /** + * The image on which the question is asked + */ + image: unknown; + /** + * A question to ask of the document + */ + question: string; + [property: string]: unknown; +} +/** + * Additional inference parameters for Document Question Answering + */ +export interface DocumentQuestionAnsweringParameters { + /** + * If the words in the document are too long to fit with the question for the model, it will + * be split in several chunks with some overlap. This argument controls the size of that + * overlap. + */ + doc_stride?: number; + /** + * Whether to accept impossible as an answer + */ + handle_impossible_answer?: boolean; + /** + * Language to use while running OCR. Defaults to english. + */ + lang?: string; + /** + * The maximum length of predicted answers (e.g., only answers with a shorter length are + * considered). + */ + max_answer_len?: number; + /** + * The maximum length of the question after tokenization. It will be truncated if needed. + */ + max_question_len?: number; + /** + * The maximum length of the total sentence (context + question) in tokens of each chunk + * passed to the model. The context will be split in several chunks (using doc_stride as + * overlap) if needed. + */ + max_seq_len?: number; + /** + * The number of answers to return (will be chosen by order of likelihood). Can return less + * than top_k answers if there are not enough options available within the context. + */ + top_k?: number; + /** + * A list of words and bounding boxes (normalized 0->1000). If provided, the inference will + * skip the OCR step and use the provided bounding boxes instead. + */ + word_boxes?: WordBox[]; + [property: string]: unknown; +} +export type WordBox = number[] | string; +export type DocumentQuestionAnsweringOutput = DocumentQuestionAnsweringOutputElement[]; +/** + * Outputs of inference for the Document Question Answering task + */ +export interface DocumentQuestionAnsweringOutputElement { + /** + * The answer to the question. + */ + answer: string; + /** + * The end word index of the answer (in the OCR’d version of the input or provided word + * boxes). + */ + end: number; + /** + * The probability associated to the answer. + */ + score: number; + /** + * The start word index of the answer (in the OCR’d version of the input or provided word + * boxes). + */ + start: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f86c5635adab67721cb8843206eb5fc48b4c7d0e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/document-question-answering/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC9C;;OAEG;IACH,MAAM,EAAE,kCAAkC,CAAC;IAC3C;;OAEG;IACH,UAAU,CAAC,EAAE,mCAAmC,CAAC;IACjD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,kCAAkC;IAClD;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,mCAAmC;IACnD;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,EAAE,CAAC;IACvB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,OAAO,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC;AACxC,MAAM,MAAM,+BAA+B,GAAG,sCAAsC,EAAE,CAAC;AACvF;;GAEG;AACH,MAAM,WAAW,sCAAsC;IACtD;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/document-question-answering/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..1b7c67323ed90db6335e841bb8cdb74d2f429d4c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/feature-extraction/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAoDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/data.js new file mode 100644 index 0000000000000000000000000000000000000000..c8782958720d7b6a26216cf5a03dd171fe43d879 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/data.js @@ -0,0 +1,55 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "Wikipedia dataset containing cleaned articles of all languages. Can be used to train `feature-extraction` models.", + id: "wikipedia", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "India, officially the Republic of India, is a country in South Asia.", + type: "text", + }, + ], + outputs: [ + { + table: [ + ["Dimension 1", "Dimension 2", "Dimension 3"], + ["2.583383083343506", "2.757075071334839", "0.9023529887199402"], + ["8.29393482208252", "1.1071064472198486", "2.03399395942688"], + ["-0.7754912972450256", "-1.647324562072754", "-0.6113331913948059"], + ["0.07087723910808563", "1.5942802429199219", "1.4610432386398315"], + ], + type: "tabular", + }, + ], + }, + metrics: [], + models: [ + { + description: "A powerful feature extraction model for natural language processing tasks.", + id: "thenlper/gte-large", + }, + { + description: "A strong feature extraction model for retrieval.", + id: "Alibaba-NLP/gte-Qwen1.5-7B-instruct", + }, + ], + spaces: [ + { + description: "A leaderboard to rank text feature extraction models based on a benchmark.", + id: "mteb/leaderboard", + }, + { + description: "A leaderboard to rank best feature extraction models based on human feedback.", + id: "mteb/arena", + }, + ], + summary: "Feature extraction is the task of extracting features learnt in a model.", + widgetModels: ["facebook/bart-base"], +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..688961c27e3dc737f10f5c7751b906eb109b4252 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/inference.d.ts @@ -0,0 +1,42 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +export type FeatureExtractionOutput = Array; +/** + * Feature Extraction Input. + * + * Auto-generated from TEI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tei-import.ts. + */ +export interface FeatureExtractionInput { + /** + * The text or list of texts to embed. + */ + inputs: FeatureExtractionInputs; + normalize?: boolean; + /** + * The name of the prompt that should be used by for encoding. If not set, no prompt + * will be applied. + * + * Must be a key in the `sentence-transformers` configuration `prompts` dictionary. + * + * For example if ``prompt_name`` is "query" and the ``prompts`` is {"query": "query: ", + * ...}, + * then the sentence "What is the capital of France?" will be encoded as + * "query: What is the capital of France?" because the prompt text will be prepended before + * any text to encode. + */ + prompt_name?: string; + truncate?: boolean; + truncation_direction?: FeatureExtractionInputTruncationDirection; + [property: string]: unknown; +} +/** + * The text or list of texts to embed. + */ +export type FeatureExtractionInputs = string[] | string; +export type FeatureExtractionInputTruncationDirection = "left" | "right"; +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6c2bd1f9aef9f4084c503ce5754da17e463ea4e7 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/feature-extraction/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,MAAM,uBAAuB,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;AACtD;;;;;;GAMG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,MAAM,EAAE,uBAAuB,CAAC;IAChC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;;;;;;;;OAWG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,oBAAoB,CAAC,EAAE,yCAAyC,CAAC;IACjE,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,uBAAuB,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC;AACxD,MAAM,MAAM,yCAAyC,GAAG,MAAM,GAAG,OAAO,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/feature-extraction/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d8447494880d737be45833c566928770eced4b7f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/fill-mask/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA0Ef,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/data.js new file mode 100644 index 0000000000000000000000000000000000000000..cd7233797cc4dd2b914affced4579ea67544838d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/data.js @@ -0,0 +1,75 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A common dataset that is used to train models for many languages.", + id: "wikipedia", + }, + { + description: "A large English dataset with text crawled from the web.", + id: "c4", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "The barked at me", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "wolf", + score: 0.487, + }, + { + label: "dog", + score: 0.061, + }, + { + label: "cat", + score: 0.058, + }, + { + label: "fox", + score: 0.047, + }, + { + label: "squirrel", + score: 0.025, + }, + ], + }, + ], + }, + metrics: [ + { + description: "Cross Entropy is a metric that calculates the difference between two probability distributions. Each probability distribution is the distribution of predicted words", + id: "cross_entropy", + }, + { + description: "Perplexity is the exponential of the cross-entropy loss. It evaluates the probabilities assigned to the next word by the model. Lower perplexity indicates better performance", + id: "perplexity", + }, + ], + models: [ + { + description: "State-of-the-art masked language model.", + id: "answerdotai/ModernBERT-large", + }, + { + description: "A multilingual model trained on 100 languages.", + id: "FacebookAI/xlm-roberta-base", + }, + ], + spaces: [], + summary: "Masked language modeling is the task of masking some of the words in a sentence and predicting which words should replace those masks. These models are useful when we want to get a statistical understanding of the language in which the model is trained in.", + widgetModels: ["distilroberta-base"], + youtubeId: "mqElG5QJWUg", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2d695741a7232bad148db87e1e55cf9825d20dcd --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/inference.d.ts @@ -0,0 +1,61 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Fill Mask inference + */ +export interface FillMaskInput { + /** + * The text with masked tokens + */ + inputs: string; + /** + * Additional inference parameters for Fill Mask + */ + parameters?: FillMaskParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Fill Mask + */ +export interface FillMaskParameters { + /** + * When passed, the model will limit the scores to the passed targets instead of looking up + * in the whole vocabulary. If the provided targets are not in the model vocab, they will be + * tokenized and the first resulting token will be used (with a warning, and that might be + * slower). + */ + targets?: string[]; + /** + * When passed, overrides the number of predictions to return. + */ + top_k?: number; + [property: string]: unknown; +} +export type FillMaskOutput = FillMaskOutputElement[]; +/** + * Outputs of inference for the Fill Mask task + */ +export interface FillMaskOutputElement { + /** + * The corresponding probability + */ + score: number; + /** + * The corresponding input with the mask token prediction. + */ + sequence: string; + /** + * The predicted token id (to replace the masked one). + */ + token: number; + tokenStr: unknown; + /** + * The predicted token (to replace the masked one). + */ + token_str?: string; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5d530f0a2d16b0ef869050b3c6edf02eb4922ebf --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/fill-mask/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,kBAAkB,CAAC;IAChC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,cAAc,GAAG,qBAAqB,EAAE,CAAC;AACrD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/fill-mask/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f157fe0bf42e631cd4810f5d9d3b88a1bd8cd488 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAkFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..92e626f89b0725db39a8ff4e336f961e9ca48e6d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/data.js @@ -0,0 +1,85 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + // TODO write proper description + description: "Benchmark dataset used for image classification with images that belong to 100 classes.", + id: "cifar100", + }, + { + // TODO write proper description + description: "Dataset consisting of images of garments.", + id: "fashion_mnist", + }, + ], + demo: { + inputs: [ + { + filename: "image-classification-input.jpeg", + type: "img", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Egyptian cat", + score: 0.514, + }, + { + label: "Tabby cat", + score: 0.193, + }, + { + label: "Tiger cat", + score: 0.068, + }, + ], + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + description: "A strong image classification model.", + id: "google/vit-base-patch16-224", + }, + { + description: "A robust image classification model.", + id: "facebook/deit-base-distilled-patch16-224", + }, + { + description: "A strong image classification model.", + id: "facebook/convnext-large-224", + }, + ], + spaces: [ + { + description: "A leaderboard to evaluate different image classification models.", + id: "timm/leaderboard", + }, + ], + summary: "Image classification is the task of assigning a label or class to an entire image. Images are expected to have only one class for each image. Image classification models take an image as input and return a prediction about which class the image belongs to.", + widgetModels: ["google/vit-base-patch16-224"], + youtubeId: "tjAIM7BOYhw", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..cea2bf9efe0f9dfa738b46d40fb17dcaba022793 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/inference.d.ts @@ -0,0 +1,54 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image Classification inference + */ +export interface ImageClassificationInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Image Classification + */ + parameters?: ImageClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image Classification + */ +export interface ImageClassificationParameters { + /** + * The function to apply to the model outputs in order to retrieve the scores. + */ + function_to_apply?: ClassificationOutputTransform; + /** + * When specified, limits the output to the top K most probable classes. + */ + top_k?: number; + [property: string]: unknown; +} +/** + * The function to apply to the model outputs in order to retrieve the scores. + */ +export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none"; +export type ImageClassificationOutput = ImageClassificationOutputElement[]; +/** + * Outputs of inference for the Image Classification task + */ +export interface ImageClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..037c9227770f76072b14980b75b858c399c7a54a --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-classification/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACxC;;;OAGG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,6BAA6B,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC7C;;OAEG;IACH,iBAAiB,CAAC,EAAE,6BAA6B,CAAC;IAClD;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAC3E,MAAM,MAAM,yBAAyB,GAAG,gCAAgC,EAAE,CAAC;AAC3E;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-classification/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-feature-extraction/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-feature-extraction/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-feature-extraction/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-feature-extraction/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-feature-extraction/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..90bc944282b5d5de9e390cc55fa1f326f5f1b6bf --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-feature-extraction/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-feature-extraction/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA2Df,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-feature-extraction/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-feature-extraction/data.js new file mode 100644 index 0000000000000000000000000000000000000000..a5a34f85c4b4f5d7b48deb69f0ce050fe6b50498 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-feature-extraction/data.js @@ -0,0 +1,62 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "ImageNet-1K is a image classification dataset in which images are used to train image-feature-extraction models.", + id: "imagenet-1k", + }, + ], + demo: { + inputs: [ + { + filename: "mask-generation-input.png", + type: "img", + }, + ], + outputs: [ + { + table: [ + ["Dimension 1", "Dimension 2", "Dimension 3"], + ["0.21236686408519745", "1.0919708013534546", "0.8512550592422485"], + ["0.809657871723175", "-0.18544459342956543", "-0.7851548194885254"], + ["1.3103108406066895", "-0.2479034662246704", "-0.9107287526130676"], + ["1.8536205291748047", "-0.36419737339019775", "0.09717650711536407"], + ], + type: "tabular", + }, + ], + }, + metrics: [], + models: [ + { + description: "A powerful image feature extraction model.", + id: "timm/vit_large_patch14_dinov2.lvd142m", + }, + { + description: "A strong image feature extraction model.", + id: "nvidia/MambaVision-T-1K", + }, + { + description: "A robust image feature extraction model.", + id: "facebook/dino-vitb16", + }, + { + description: "Cutting-edge image feature extraction model.", + id: "apple/aimv2-large-patch14-336-distilled", + }, + { + description: "Strong image feature extraction model that can be used on images and documents.", + id: "OpenGVLab/InternViT-6B-448px-V1-2", + }, + ], + spaces: [ + { + description: "A leaderboard to evaluate different image-feature-extraction models on classification performances", + id: "timm/leaderboard", + }, + ], + summary: "Image feature extraction is the task of extracting features learnt in a computer vision model.", + widgetModels: [], +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..23ff6576f4a0f6e8cee58c7500193af6426f7738 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-segmentation/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA8Ff,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/data.js new file mode 100644 index 0000000000000000000000000000000000000000..d4bb17fa6913a34a37c13fcd4d07c519d320d42c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/data.js @@ -0,0 +1,95 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "Scene segmentation dataset.", + id: "scene_parse_150", + }, + ], + demo: { + inputs: [ + { + filename: "image-segmentation-input.jpeg", + type: "img", + }, + ], + outputs: [ + { + filename: "image-segmentation-output.png", + type: "img", + }, + ], + }, + metrics: [ + { + description: "Average Precision (AP) is the Area Under the PR Curve (AUC-PR). It is calculated for each semantic class separately", + id: "Average Precision", + }, + { + description: "Mean Average Precision (mAP) is the overall average of the AP values", + id: "Mean Average Precision", + }, + { + description: "Intersection over Union (IoU) is the overlap of segmentation masks. Mean IoU is the average of the IoU of all semantic classes", + id: "Mean Intersection over Union", + }, + { + description: "APα is the Average Precision at the IoU threshold of a α value, for example, AP50 and AP75", + id: "APα", + }, + ], + models: [ + { + // TO DO: write description + description: "Solid panoptic segmentation model trained on COCO.", + id: "tue-mps/coco_panoptic_eomt_large_640", + }, + { + description: "Background removal model.", + id: "briaai/RMBG-1.4", + }, + { + description: "A multipurpose image segmentation model for high resolution images.", + id: "ZhengPeng7/BiRefNet", + }, + { + description: "Powerful human-centric image segmentation model.", + id: "facebook/sapiens-seg-1b", + }, + { + description: "Panoptic segmentation model trained on the COCO (common objects) dataset.", + id: "facebook/mask2former-swin-large-coco-panoptic", + }, + ], + spaces: [ + { + description: "A semantic segmentation application that can predict unseen instances out of the box.", + id: "facebook/ov-seg", + }, + { + description: "One of the strongest segmentation applications.", + id: "jbrinkma/segment-anything", + }, + { + description: "A human-centric segmentation model.", + id: "facebook/sapiens-pose", + }, + { + description: "An instance segmentation application to predict neuronal cell types from microscopy images.", + id: "rashmi/sartorius-cell-instance-segmentation", + }, + { + description: "An application that segments videos.", + id: "ArtGAN/Segment-Anything-Video", + }, + { + description: "An panoptic segmentation application built for outdoor environments.", + id: "segments/panoptic-segment-anything", + }, + ], + summary: "Image Segmentation divides an image into segments where each pixel in the image is mapped to an object. This task has multiple variants such as instance segmentation, panoptic segmentation and semantic segmentation.", + widgetModels: ["nvidia/segformer-b0-finetuned-ade-512-512"], + youtubeId: "dKE8SIt9C-w", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..62893d683d831473f37662e89e8742a0582f6d4f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/inference.d.ts @@ -0,0 +1,68 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image Segmentation inference + */ +export interface ImageSegmentationInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Image Segmentation + */ + parameters?: ImageSegmentationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image Segmentation + */ +export interface ImageSegmentationParameters { + /** + * Threshold to use when turning the predicted masks into binary values. + */ + mask_threshold?: number; + /** + * Mask overlap threshold to eliminate small, disconnected segments. + */ + overlap_mask_area_threshold?: number; + /** + * Segmentation task to be performed, depending on model capabilities. + */ + subtask?: ImageSegmentationSubtask; + /** + * Probability threshold to filter out predicted masks. + */ + threshold?: number; + [property: string]: unknown; +} +/** + * Segmentation task to be performed, depending on model capabilities. + */ +export type ImageSegmentationSubtask = "instance" | "panoptic" | "semantic"; +export type ImageSegmentationOutput = ImageSegmentationOutputElement[]; +/** + * Outputs of inference for the Image Segmentation task + * + * A predicted mask / segment + */ +export interface ImageSegmentationOutputElement { + /** + * The label of the predicted segment. + */ + label: string; + /** + * The corresponding mask as a black-and-white image (base64-encoded). + */ + mask: string; + /** + * The score or confidence degree the model has. + */ + score?: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..002f53684e81edb0b5ac7dd0ed180a404781f3e2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-segmentation/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;;OAGG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,2BAA2B,CAAC;IACzC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC3C;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC;;OAEG;IACH,OAAO,CAAC,EAAE,wBAAwB,CAAC;IACnC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAC5E,MAAM,MAAM,uBAAuB,GAAG,8BAA8B,EAAE,CAAC;AACvE;;;;GAIG;AACH,MAAM,WAAW,8BAA8B;IAC9C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-segmentation/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..53a0eeeace77cd5956a7331e9ab7cb5c8f3fd1fb --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-text-to-image/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAiDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/data.js new file mode 100644 index 0000000000000000000000000000000000000000..2af0abfa134fa1695cc0f5ed0a5852188e6b8a24 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/data.js @@ -0,0 +1,50 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [], + demo: { + inputs: [ + { + filename: "image-text-to-image-input.jpeg", + type: "img", + }, + { + label: "Input", + content: "A city above clouds, pastel colors, Victorian style", + type: "text", + }, + ], + outputs: [ + { + filename: "image-text-to-image-output.png", + type: "img", + }, + ], + }, + metrics: [ + { + description: "The Fréchet Inception Distance (FID) calculates the distance between distributions between synthetic and real samples. A lower FID score indicates better similarity between the distributions of real and generated images.", + id: "FID", + }, + { + description: "CLIP Score measures the similarity between the generated image and the text prompt using CLIP embeddings. A higher score indicates better alignment with the text prompt.", + id: "CLIP", + }, + ], + models: [ + { + description: "A powerful model for image-text-to-image generation.", + id: "black-forest-labs/FLUX.2-dev", + }, + ], + spaces: [ + { + description: "An application for image-text-to-image generation.", + id: "black-forest-labs/FLUX.2-dev", + }, + ], + summary: "Image-text-to-image models take an image and a text prompt as input and generate a new image based on the reference image and text instructions. These models are useful for image editing, style transfer, image variations, and guided image generation tasks.", + widgetModels: ["black-forest-labs/FLUX.2-dev"], + youtubeId: undefined, +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d6d2dab3a0b73c4600be87602613629b9d288570 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/inference.d.ts @@ -0,0 +1,76 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image Text To Image inference. Either inputs (image) or prompt (in parameters) + * must be provided, or both. + */ +export interface ImageTextToImageInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. Either this or prompt must be + * provided. + */ + inputs?: Blob; + /** + * Additional inference parameters for Image Text To Image + */ + parameters?: ImageTextToImageParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image Text To Image + */ +export interface ImageTextToImageParameters { + /** + * For diffusion models. A higher guidance scale value encourages the model to generate + * images closely linked to the text prompt at the expense of lower image quality. + */ + guidance_scale?: number; + /** + * One prompt to guide what NOT to include in image generation. + */ + negative_prompt?: string; + /** + * For diffusion models. The number of denoising steps. More denoising steps usually lead to + * a higher quality image at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * The text prompt to guide the image generation. Either this or inputs (image) must be + * provided. + */ + prompt?: string; + /** + * Seed for the random number generator. + */ + seed?: number; + /** + * The size in pixels of the output image. This parameter is only supported by some + * providers and for specific models. It will be ignored when unsupported. + */ + target_size?: TargetSize; + [property: string]: unknown; +} +/** + * The size in pixels of the output image. This parameter is only supported by some + * providers and for specific models. It will be ignored when unsupported. + */ +export interface TargetSize { + height: number; + width: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Image Text To Image task + */ +export interface ImageTextToImageOutput { + /** + * The generated image returned as raw bytes in the payload. + */ + image: unknown; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e0caa1a268aacfd85428f46c8615abf68d4f0606 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-text-to-image/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACrC;;;;OAIG;IACH,MAAM,CAAC,EAAE,IAAI,CAAC;IACd;;OAEG;IACH,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,0BAA0B;IAC1C;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,WAAW,UAAU;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-image/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-text/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-text/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-text/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-text/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-text/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..b600eb9ac66eb184e991671ecca7a6e8d4b6e8ab --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-text/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-text-to-text/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAiFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-text/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-text/data.js new file mode 100644 index 0000000000000000000000000000000000000000..eeffaade20f09824f163de8e64f293c449e5d7b9 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-text/data.js @@ -0,0 +1,83 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "Instructions composed of image and text.", + id: "liuhaotian/LLaVA-Instruct-150K", + }, + { + description: "Collection of image-text pairs on scientific topics.", + id: "DAMO-NLP-SG/multimodal_textbook", + }, + { + description: "A collection of datasets made for model fine-tuning.", + id: "HuggingFaceM4/the_cauldron", + }, + { + description: "Screenshots of websites with their HTML/CSS codes.", + id: "HuggingFaceM4/WebSight", + }, + ], + demo: { + inputs: [ + { + filename: "image-text-to-text-input.png", + type: "img", + }, + { + label: "Text Prompt", + content: "Describe the position of the bee in detail.", + type: "text", + }, + ], + outputs: [ + { + label: "Answer", + content: "The bee is sitting on a pink flower, surrounded by other flowers. The bee is positioned in the center of the flower, with its head and front legs sticking out.", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "Small and efficient yet powerful vision language model.", + id: "HuggingFaceTB/SmolVLM-Instruct", + }, + { + description: "Cutting-edge reasoning vision language model.", + id: "zai-org/GLM-4.5V", + }, + { + description: "Cutting-edge small vision language model to convert documents to text.", + id: "rednote-hilab/dots.ocr", + }, + { + description: "Small yet powerful model.", + id: "Qwen/Qwen2.5-VL-3B-Instruct", + }, + { + description: "Image-text-to-text model with agentic capabilities.", + id: "microsoft/Magma-8B", + }, + ], + spaces: [ + { + description: "Leaderboard to evaluate vision language models.", + id: "opencompass/open_vlm_leaderboard", + }, + { + description: "An application that compares object detection capabilities of different vision language models.", + id: "sergiopaniego/vlm_object_understanding", + }, + { + description: "An application to compare different OCR models.", + id: "prithivMLmods/Multimodal-OCR", + }, + ], + summary: "Image-text-to-text models take in an image and text prompt and output text. These models are also called vision-language models, or VLMs. The difference from image-to-text models is that these models take an additional text input, not restricting the model to certain use cases like image captioning, and may also be trained to accept a conversation as input.", + widgetModels: ["zai-org/GLM-4.5V"], + youtubeId: "IoGaGfU1CIg", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..4b9e273281951252ef914c15fc3f2df7c63fdfbd --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-text-to-video/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAiDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/data.js new file mode 100644 index 0000000000000000000000000000000000000000..b9d89e46a3ae6c7c5f9ad18ac3570bb1daa3b4bf --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/data.js @@ -0,0 +1,50 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [], + demo: { + inputs: [ + { + filename: "image-text-to-video-input.jpg", + type: "img", + }, + { + label: "Input", + content: "Darth Vader is surfing on the waves.", + type: "text", + }, + ], + outputs: [ + { + filename: "image-text-to-video-output.gif", + type: "img", + }, + ], + }, + metrics: [ + { + description: "Frechet Video Distance uses a model that captures coherence for changes in frames and the quality of each frame. A smaller score indicates better video generation.", + id: "fvd", + }, + { + description: "CLIPSIM measures similarity between video frames and text using an image-text similarity model. A higher score indicates better video generation.", + id: "clipsim", + }, + ], + models: [ + { + description: "A powerful model for image-text-to-video generation.", + id: "Lightricks/LTX-Video", + }, + ], + spaces: [ + { + description: "An application for image-text-to-video generation.", + id: "Lightricks/ltx-video-distilled", + }, + ], + summary: "Image-text-to-video models take an reference image and a text instructions as and generate a video based on them. These models are useful for animating still images, creating dynamic content from static references, and generating videos with specific motion or transformation guidance.", + widgetModels: ["Lightricks/LTX-Video"], + youtubeId: undefined, +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..069db05bcf626591087431e5284525e1b189cc41 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/inference.d.ts @@ -0,0 +1,78 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image Text To Video inference. Either inputs (image) or prompt (in parameters) + * must be provided, or both. + */ +export interface ImageTextToVideoInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. Either this or prompt must be + * provided. + */ + inputs?: Blob; + /** + * Additional inference parameters for Image Text To Video + */ + parameters?: ImageTextToVideoParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image Text To Video + */ +export interface ImageTextToVideoParameters { + /** + * For diffusion models. A higher guidance scale value encourages the model to generate + * videos closely linked to the text prompt at the expense of lower image quality. + */ + guidance_scale?: number; + /** + * One prompt to guide what NOT to include in video generation. + */ + negative_prompt?: string; + /** + * The num_frames parameter determines how many video frames are generated. + */ + num_frames?: number; + /** + * The number of denoising steps. More denoising steps usually lead to a higher quality + * video at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * The text prompt to guide the video generation. Either this or inputs (image) must be + * provided. + */ + prompt?: string; + /** + * Seed for the random number generator. + */ + seed?: number; + /** + * The size in pixel of the output video frames. + */ + target_size?: TargetSize; + [property: string]: unknown; +} +/** + * The size in pixel of the output video frames. + */ +export interface TargetSize { + height: number; + width: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Image Text To Video task + */ +export interface ImageTextToVideoOutput { + /** + * The generated video returned as raw bytes in the payload. + */ + video: unknown; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..dddebe7ae1d8e967527b72e26ba1a9b5f1544c2c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-text-to-video/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACrC;;;;OAIG;IACH,MAAM,CAAC,EAAE,IAAI,CAAC;IACd;;OAEG;IACH,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,0BAA0B;IAC1C;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-text-to-video/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-3d/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-3d/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-3d/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-3d/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-3d/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..8c8b2f7cc89572fcc9c798d136a3bc242d3d98bd --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-3d/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-to-3d/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAsEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-3d/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-3d/data.js new file mode 100644 index 0000000000000000000000000000000000000000..c43723a73cfabf77214bf04b2900a416e81e704d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-3d/data.js @@ -0,0 +1,74 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A large dataset of over 10 million 3D objects.", + id: "allenai/objaverse-xl", + }, + { + description: "A dataset of isolated object images for evaluating image-to-3D models.", + id: "dylanebert/iso3d", + }, + ], + demo: { + inputs: [ + { + filename: "image-to-3d-image-input.png", + type: "img", + }, + ], + outputs: [ + { + label: "Result", + content: "image-to-3d-3d-output-filename.glb", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "Fast image-to-3D mesh model by Tencent.", + id: "TencentARC/InstantMesh", + }, + { + description: "3D world generation model.", + id: "tencent/HunyuanWorld-1", + }, + { + description: "A scaled up image-to-3D mesh model derived from TripoSR.", + id: "hwjiang/Real3D", + }, + { + description: "Consistent image-to-3d generation model.", + id: "stabilityai/stable-point-aware-3d", + }, + ], + spaces: [ + { + description: "Leaderboard to evaluate image-to-3D models.", + id: "dylanebert/3d-arena", + }, + { + description: "Image-to-3D demo with mesh outputs.", + id: "TencentARC/InstantMesh", + }, + { + description: "Image-to-3D demo.", + id: "stabilityai/stable-point-aware-3d", + }, + { + description: "Image-to-3D demo with mesh outputs.", + id: "hwjiang/Real3D", + }, + { + description: "Image-to-3D demo with splat outputs.", + id: "dylanebert/LGM-mini", + }, + ], + summary: "Image-to-3D models take in image input and produce 3D output.", + widgetModels: [], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..df6672e2ac2e6e69cdfdc88c3e5300bb1a441eb8 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-to-image/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA2Ff,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/data.js new file mode 100644 index 0000000000000000000000000000000000000000..f65b45a7ba7a9dab6a019297e50368d03e13ca2b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/data.js @@ -0,0 +1,91 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "Synthetic dataset, for image relighting", + id: "VIDIT", + }, + { + description: "Multiple images of celebrities, used for facial expression translation", + id: "huggan/CelebA-faces", + }, + { + description: "12M image-caption pairs.", + id: "Spawning/PD12M", + }, + ], + demo: { + inputs: [ + { + filename: "image-to-image-input.jpeg", + type: "img", + }, + ], + outputs: [ + { + filename: "image-to-image-output.png", + type: "img", + }, + ], + }, + isPlaceholder: false, + metrics: [ + { + description: "Peak Signal to Noise Ratio (PSNR) is an approximation of the human perception, considering the ratio of the absolute intensity with respect to the variations. Measured in dB, a high value indicates a high fidelity.", + id: "PSNR", + }, + { + description: "Structural Similarity Index (SSIM) is a perceptual metric which compares the luminance, contrast and structure of two images. The values of SSIM range between -1 and 1, and higher values indicate closer resemblance to the original image.", + id: "SSIM", + }, + { + description: "Inception Score (IS) is an analysis of the labels predicted by an image classification model when presented with a sample of the generated images.", + id: "IS", + }, + ], + models: [ + { + description: "An image-to-image model to improve image resolution.", + id: "fal/AuraSR-v2", + }, + { + description: "Powerful image editing model.", + id: "black-forest-labs/FLUX.1-Kontext-dev", + }, + { + description: "Virtual try-on model.", + id: "yisol/IDM-VTON", + }, + { + description: "Image re-lighting model.", + id: "kontext-community/relighting-kontext-dev-lora-v3", + }, + { + description: "Strong model for inpainting and outpainting.", + id: "black-forest-labs/FLUX.1-Fill-dev", + }, + { + description: "Strong model for image editing using depth maps.", + id: "black-forest-labs/FLUX.1-Depth-dev-lora", + }, + ], + spaces: [ + { + description: "Image editing application.", + id: "black-forest-labs/FLUX.1-Kontext-Dev", + }, + { + description: "Image relighting application.", + id: "lllyasviel/iclight-v2-vary", + }, + { + description: "An application for image upscaling.", + id: "jasperai/Flux.1-dev-Controlnet-Upscaler", + }, + ], + summary: "Image-to-image is the task of transforming an input image through a variety of possible manipulations and enhancements, such as super-resolution, image inpainting, colorization, and more.", + widgetModels: ["Qwen/Qwen-Image"], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..c5c7a1dc660bc32cac43d7d9298c4710c82fc186 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/inference.d.ts @@ -0,0 +1,69 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image To Image inference + */ +export interface ImageToImageInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Image To Image + */ + parameters?: ImageToImageParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image To Image + */ +export interface ImageToImageParameters { + /** + * For diffusion models. A higher guidance scale value encourages the model to generate + * images closely linked to the text prompt at the expense of lower image quality. + */ + guidance_scale?: number; + /** + * One prompt to guide what NOT to include in image generation. + */ + negative_prompt?: string; + /** + * For diffusion models. The number of denoising steps. More denoising steps usually lead to + * a higher quality image at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * The text prompt to guide the image generation. + */ + prompt?: string; + /** + * The size in pixels of the output image. This parameter is only supported by some + * providers and for specific models. It will be ignored when unsupported. + */ + target_size?: TargetSize; + [property: string]: unknown; +} +/** + * The size in pixels of the output image. This parameter is only supported by some + * providers and for specific models. It will be ignored when unsupported. + */ +export interface TargetSize { + height: number; + width: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Image To Image task + */ +export interface ImageToImageOutput { + /** + * The output image returned as raw bytes in the payload. + */ + image?: unknown; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6a40c1d8e247354fc5934c1b438879b3888011e6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-to-image/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;;OAGG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,WAAW,UAAU;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-image/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..aa4d932b0466467f4b50fda065faf7fd02b50f87 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-to-text/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAyDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/data.js new file mode 100644 index 0000000000000000000000000000000000000000..a56605b1f65bcb7364ecbe7b1fd6ea1fba5eae27 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/data.js @@ -0,0 +1,60 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + // TODO write proper description + description: "Dataset from 12M image-text of Reddit", + id: "red_caps", + }, + { + // TODO write proper description + description: "Dataset from 3.3M images of Google", + id: "datasets/conceptual_captions", + }, + ], + demo: { + inputs: [ + { + filename: "savanna.jpg", + type: "img", + }, + ], + outputs: [ + { + label: "Detailed description", + content: "a herd of giraffes and zebras grazing in a field", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "Strong OCR model.", + id: "allenai/olmOCR-7B-0725", + }, + { + description: "Powerful image captioning model.", + id: "fancyfeast/llama-joycaption-beta-one-hf-llava", + }, + ], + spaces: [ + { + description: "SVG generator app from images.", + id: "multimodalart/OmniSVG-3B", + }, + { + description: "An application that converts documents to markdown.", + id: "numind/NuMarkdown-8B-Thinking", + }, + { + description: "An application that can caption images.", + id: "fancyfeast/joy-caption-beta-one", + }, + ], + summary: "Image to text models output a text from a given image. Image captioning or optical character recognition can be considered as the most common applications of image to text.", + widgetModels: ["Salesforce/blip-image-captioning-large"], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..9944a906d4fa9529af0e38ef2473969de421da4c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/inference.d.ts @@ -0,0 +1,135 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image To Text inference + */ +export interface ImageToTextInput { + /** + * The input image data + */ + inputs: Blob; + /** + * Additional inference parameters for Image To Text + */ + parameters?: ImageToTextParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image To Text + */ +export interface ImageToTextParameters { + /** + * Parametrization of the text generation process + */ + generation_parameters?: GenerationParameters; + /** + * The amount of maximum tokens to generate. + */ + max_new_tokens?: number; + [property: string]: unknown; +} +/** + * Parametrization of the text generation process + */ +export interface GenerationParameters { + /** + * Whether to use sampling instead of greedy decoding when generating new tokens. + */ + do_sample?: boolean; + /** + * Controls the stopping condition for beam-based methods. + */ + early_stopping?: EarlyStoppingUnion; + /** + * If set to float strictly between 0 and 1, only tokens with a conditional probability + * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + * Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + */ + epsilon_cutoff?: number; + /** + * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + * float strictly between 0 and 1, a token is only considered if it is greater than either + * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + * for more details. + */ + eta_cutoff?: number; + /** + * The maximum length (in tokens) of the generated text, including the input. + */ + max_length?: number; + /** + * The maximum number of tokens to generate. Takes precedence over max_length. + */ + max_new_tokens?: number; + /** + * The minimum length (in tokens) of the generated text, including the input. + */ + min_length?: number; + /** + * The minimum number of tokens to generate. Takes precedence over min_length. + */ + min_new_tokens?: number; + /** + * Number of groups to divide num_beams into in order to ensure diversity among different + * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + */ + num_beam_groups?: number; + /** + * Number of beams to use for beam search. + */ + num_beams?: number; + /** + * The value balances the model confidence and the degeneration penalty in contrastive + * search decoding. + */ + penalty_alpha?: number; + /** + * The value used to modulate the next token probabilities. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * If set to float < 1, only the smallest set of most probable tokens with probabilities + * that add up to top_p or higher are kept for generation. + */ + top_p?: number; + /** + * Local typicality measures how similar the conditional probability of predicting a target + * token next is to the expected conditional probability of predicting a random token next, + * given the partial text already generated. If set to float < 1, the smallest set of the + * most locally typical tokens with probabilities that add up to typical_p or higher are + * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + */ + typical_p?: number; + /** + * Whether the model should use the past last key/values attentions to speed up decoding + */ + use_cache?: boolean; + [property: string]: unknown; +} +/** + * Controls the stopping condition for beam-based methods. + */ +export type EarlyStoppingUnion = boolean | "never"; +/** + * Outputs of inference for the Image To Text task + */ +export interface ImageToTextOutput { + generatedText: unknown; + /** + * The generated text. + */ + generated_text?: string; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..b4d812f2f06619307ca7ebf5c53526474c51ab2d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-to-text/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,qBAAqB,CAAC,EAAE,oBAAoB,CAAC;IAC7C;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,kBAAkB,CAAC;IACpC;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,OAAO,GAAG,OAAO,CAAC;AACnD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,aAAa,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-text/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e5d0273d48ff15ed08a8d8bcd0d85efb1ba1a091 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-to-video/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAyHf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/data.js new file mode 100644 index 0000000000000000000000000000000000000000..c413e3be08aec203ae4a94efdd6588f32084b8b7 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/data.js @@ -0,0 +1,119 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A benchmark dataset for reference image controlled video generation.", + id: "ali-vilab/VACE-Benchmark", + }, + { + description: "A dataset of video generation style preferences.", + id: "Rapidata/sora-video-generation-style-likert-scoring", + }, + { + description: "A dataset with videos and captions throughout the videos.", + id: "BestWishYsh/ChronoMagic", + }, + ], + demo: { + inputs: [ + { + filename: "image-to-video-input.jpg", + type: "img", + }, + { + label: "Optional Text Prompt", + content: "This penguin is dancing", + type: "text", + }, + ], + outputs: [ + { + filename: "image-to-video-output.gif", + type: "img", + }, + ], + }, + metrics: [ + { + description: "Fréchet Video Distance (FVD) measures the perceptual similarity between the distributions of generated videos and a set of real videos, assessing overall visual quality and temporal coherence of the video generated from an input image.", + id: "fvd", + }, + { + description: "CLIP Score measures the semantic similarity between a textual prompt (if provided alongside the input image) and the generated video frames. It evaluates how well the video's generated content and motion align with the textual description, conditioned on the initial image.", + id: "clip_score", + }, + { + description: "First Frame Fidelity, often measured using LPIPS (Learned Perceptual Image Patch Similarity), PSNR, or SSIM, quantifies how closely the first frame of the generated video matches the input conditioning image.", + id: "lpips", + }, + { + description: "Identity Preservation Score measures the consistency of identity (e.g., a person's face or a specific object's characteristics) between the input image and throughout the generated video frames, often calculated using features from specialized models like face recognition (e.g., ArcFace) or re-identification models.", + id: "identity_preservation", + }, + { + description: "Motion Score evaluates the quality, realism, and temporal consistency of motion in the video generated from a static image. This can be based on optical flow analysis (e.g., smoothness, magnitude), consistency of object trajectories, or specific motion plausibility assessments.", + id: "motion_score", + }, + ], + models: [ + { + description: "LTX-Video, a 13B parameter model for high quality video generation", + id: "Lightricks/LTX-Video-0.9.7-dev", + }, + { + description: "A 14B parameter model for reference image controlled video generation", + id: "Wan-AI/Wan2.1-VACE-14B", + }, + { + description: "An image-to-video generation model using FramePack F1 methodology with Hunyuan-DiT architecture", + id: "lllyasviel/FramePack_F1_I2V_HY_20250503", + }, + { + description: "A distilled version of the LTX-Video-0.9.7-dev model for faster inference", + id: "Lightricks/LTX-Video-0.9.7-distilled", + }, + { + description: "An image-to-video generation model by Skywork AI, 14B parameters, producing 720p videos.", + id: "Skywork/SkyReels-V2-I2V-14B-720P", + }, + { + description: "Image-to-video variant of Tencent's HunyuanVideo.", + id: "tencent/HunyuanVideo-I2V", + }, + { + description: "A 14B parameter model for 720p image-to-video generation by Wan-AI.", + id: "Wan-AI/Wan2.1-I2V-14B-720P", + }, + { + description: "A Diffusers version of the Wan2.1-I2V-14B-720P model for 720p image-to-video generation.", + id: "Wan-AI/Wan2.1-I2V-14B-720P-Diffusers", + }, + ], + spaces: [ + { + description: "An application to generate videos fast.", + id: "Lightricks/ltx-video-distilled", + }, + { + description: "Generate videos with the FramePack-F1", + id: "linoyts/FramePack-F1", + }, + { + description: "Generate videos with the FramePack", + id: "lisonallen/framepack-i2v", + }, + { + description: "Wan2.1 with CausVid LoRA", + id: "multimodalart/wan2-1-fast", + }, + { + description: "A demo for Stable Video Diffusion", + id: "multimodalart/stable-video-diffusion", + }, + ], + summary: "Image-to-video models take a still image as input and generate a video. These models can be guided by text prompts to influence the content and style of the output video.", + widgetModels: [], + youtubeId: undefined, +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..9d6c62757ec0115df3abf86575056526232d0a7b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/inference.d.ts @@ -0,0 +1,75 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image To Video inference + */ +export interface ImageToVideoInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Image To Video + */ + parameters?: ImageToVideoParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image To Video + */ +export interface ImageToVideoParameters { + /** + * For diffusion models. A higher guidance scale value encourages the model to generate + * videos closely linked to the text prompt at the expense of lower image quality. + */ + guidance_scale?: number; + /** + * One prompt to guide what NOT to include in video generation. + */ + negative_prompt?: string; + /** + * The num_frames parameter determines how many video frames are generated. + */ + num_frames?: number; + /** + * The number of denoising steps. More denoising steps usually lead to a higher quality + * video at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * The text prompt to guide the video generation. + */ + prompt?: string; + /** + * Seed for the random number generator. + */ + seed?: number; + /** + * The size in pixel of the output video frames. + */ + target_size?: TargetSize; + [property: string]: unknown; +} +/** + * The size in pixel of the output video frames. + */ +export interface TargetSize { + height: number; + width: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Image To Video task + */ +export interface ImageToVideoOutput { + /** + * The generated video returned as raw bytes in the payload. + */ + video: unknown; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..81dcff95e010b3e686370e299598f3966b398664 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-to-video/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;;OAGG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/image-to-video/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/index.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..a29860d78e4b4dd742787d45f069abfc39f3ade2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/index.d.ts @@ -0,0 +1,92 @@ +import type { PipelineType } from "../pipelines.js"; +export type * from "./audio-classification/inference.js"; +export type * from "./automatic-speech-recognition/inference.js"; +export type { ChatCompletionInput, ChatCompletionInputMessage, ChatCompletionInputMessageChunkType, ChatCompletionOutput, ChatCompletionOutputComplete, ChatCompletionOutputMessage, ChatCompletionStreamOutput, ChatCompletionStreamOutputChoice, ChatCompletionStreamOutputDelta, } from "./chat-completion/inference.js"; +export type * from "./document-question-answering/inference.js"; +export type * from "./feature-extraction/inference.js"; +export type * from "./fill-mask/inference.js"; +export type { ImageClassificationInput, ImageClassificationOutput, ImageClassificationOutputElement, ImageClassificationParameters, } from "./image-classification/inference.js"; +export type * from "./image-to-image/inference.js"; +export type { ImageToTextInput, ImageToTextOutput, ImageToTextParameters } from "./image-to-text/inference.js"; +export type * from "./image-segmentation/inference.js"; +export type { ImageToVideoInput, ImageToVideoOutput, ImageToVideoParameters } from "./image-to-video/inference.js"; +export type { ImageTextToImageInput, ImageTextToImageOutput, ImageTextToImageParameters, } from "./image-text-to-image/inference.js"; +export type { ImageTextToVideoInput, ImageTextToVideoOutput, ImageTextToVideoParameters, } from "./image-text-to-video/inference.js"; +export type * from "./object-detection/inference.js"; +export type * from "./depth-estimation/inference.js"; +export type * from "./question-answering/inference.js"; +export type * from "./sentence-similarity/inference.js"; +export type * from "./summarization/inference.js"; +export type * from "./table-question-answering/inference.js"; +export type { TextToImageInput, TextToImageOutput, TextToImageParameters } from "./text-to-image/inference.js"; +export type { TextToVideoParameters, TextToVideoOutput, TextToVideoInput } from "./text-to-video/inference.js"; +export type { TextToSpeechParameters, TextToSpeechInput, TextToSpeechOutput } from "./text-to-speech/inference.js"; +export type { TextToAudioInput, TextToAudioOutput, TextToAudioParameters } from "./text-to-audio/inference.js"; +export type * from "./token-classification/inference.js"; +export type { TranslationInput, TranslationOutput } from "./translation/inference.js"; +export type { ClassificationOutputTransform, TextClassificationInput, TextClassificationOutput, TextClassificationOutputElement, TextClassificationParameters, } from "./text-classification/inference.js"; +export type { TextGenerationOutputFinishReason, TextGenerationOutputPrefillToken, TextGenerationInput, TextGenerationOutput, TextGenerationOutputDetails, TextGenerationInputGenerateParameters, TextGenerationOutputBestOfSequence, TextGenerationOutputToken, TextGenerationStreamOutputStreamDetails, TextGenerationStreamOutput, } from "./text-generation/inference.js"; +export type * from "./video-classification/inference.js"; +export type * from "./visual-question-answering/inference.js"; +export type * from "./zero-shot-classification/inference.js"; +export type * from "./zero-shot-image-classification/inference.js"; +export type { BoundingBox, ZeroShotObjectDetectionInput, ZeroShotObjectDetectionOutput, ZeroShotObjectDetectionOutputElement, } from "./zero-shot-object-detection/inference.js"; +import type { ModelLibraryKey } from "../model-libraries.js"; +/** + * Model libraries compatible with each ML task + */ +export declare const TASKS_MODEL_LIBRARIES: Record; +export declare const TASKS_DATA: Record; +export interface ExampleRepo { + description: string; + id: string; +} +export type TaskDemoEntry = { + filename: string; + type: "audio"; +} | { + data: Array<{ + label: string; + score: number; + }>; + type: "chart"; +} | { + filename: string; + type: "img"; +} | { + table: string[][]; + type: "tabular"; +} | { + content: string; + label: string; + type: "text"; +} | { + text: string; + tokens: Array<{ + end: number; + start: number; + type: string; + }>; + type: "text-with-tokens"; +}; +export interface TaskDemo { + inputs: TaskDemoEntry[]; + outputs: TaskDemoEntry[]; +} +export interface TaskData { + datasets: ExampleRepo[]; + demo: TaskDemo; + id: PipelineType; + canonicalId?: PipelineType; + isPlaceholder?: boolean; + label: string; + libraries: ModelLibraryKey[]; + metrics: ExampleRepo[]; + models: ExampleRepo[]; + spaces: ExampleRepo[]; + summary: string; + widgetModels: string[]; + youtubeId?: string; +} +export type TaskDataCustom = Omit; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/index.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5f05d142917b923971ede20c87bb836a755daaad --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tasks/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAoDpD,mBAAmB,qCAAqC,CAAC;AACzD,mBAAmB,6CAA6C,CAAC;AACjE,YAAY,EACX,mBAAmB,EACnB,0BAA0B,EAC1B,mCAAmC,EACnC,oBAAoB,EACpB,4BAA4B,EAC5B,2BAA2B,EAC3B,0BAA0B,EAC1B,gCAAgC,EAChC,+BAA+B,GAC/B,MAAM,gCAAgC,CAAC;AACxC,mBAAmB,4CAA4C,CAAC;AAChE,mBAAmB,mCAAmC,CAAC;AACvD,mBAAmB,0BAA0B,CAAC;AAC9C,YAAY,EACX,wBAAwB,EACxB,yBAAyB,EACzB,gCAAgC,EAChC,6BAA6B,GAC7B,MAAM,qCAAqC,CAAC;AAC7C,mBAAmB,+BAA+B,CAAC;AACnD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAC/G,mBAAmB,mCAAmC,CAAC;AACvD,YAAY,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACnH,YAAY,EACX,qBAAqB,EACrB,sBAAsB,EACtB,0BAA0B,GAC1B,MAAM,oCAAoC,CAAC;AAC5C,YAAY,EACX,qBAAqB,EACrB,sBAAsB,EACtB,0BAA0B,GAC1B,MAAM,oCAAoC,CAAC;AAC5C,mBAAmB,iCAAiC,CAAC;AACrD,mBAAmB,iCAAiC,CAAC;AACrD,mBAAmB,mCAAmC,CAAC;AACvD,mBAAmB,oCAAoC,CAAC;AACxD,mBAAmB,8BAA8B,CAAC;AAClD,mBAAmB,yCAAyC,CAAC;AAC7D,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAC/G,YAAY,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAC/G,YAAY,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnH,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAC/G,mBAAmB,qCAAqC,CAAC;AACzD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AACtF,YAAY,EACX,6BAA6B,EAC7B,uBAAuB,EACvB,wBAAwB,EACxB,+BAA+B,EAC/B,4BAA4B,GAC5B,MAAM,oCAAoC,CAAC;AAC5C,YAAY,EACX,gCAAgC,EAChC,gCAAgC,EAChC,mBAAmB,EACnB,oBAAoB,EACpB,2BAA2B,EAC3B,qCAAqC,EACrC,kCAAkC,EAClC,yBAAyB,EACzB,uCAAuC,EACvC,0BAA0B,GAC1B,MAAM,gCAAgC,CAAC;AACxC,mBAAmB,qCAAqC,CAAC;AACzD,mBAAmB,0CAA0C,CAAC;AAC9D,mBAAmB,yCAAyC,CAAC;AAC7D,mBAAmB,+CAA+C,CAAC;AACnE,YAAY,EACX,WAAW,EACX,4BAA4B,EAC5B,6BAA6B,EAC7B,oCAAoC,GACpC,MAAM,2CAA2C,CAAC;AAEnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,YAAY,EAAE,eAAe,EAAE,CAkEzE,CAAC;AAoBF,eAAO,MAAM,UAAU,EAAE,MAAM,CAAC,YAAY,EAAE,QAAQ,GAAG,SAAS,CA0DxD,CAAC;AAEX,MAAM,WAAW,WAAW;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,EAAE,EAAE,MAAM,CAAC;CACX;AAED,MAAM,MAAM,aAAa,GACtB;IACA,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;CACb,GACD;IACA,IAAI,EAAE,KAAK,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACd,CAAC,CAAC;IACH,IAAI,EAAE,OAAO,CAAC;CACb,GACD;IACA,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,KAAK,CAAC;CACX,GACD;IACA,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC;IAClB,IAAI,EAAE,SAAS,CAAC;CACf,GACD;IACA,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACZ,GACD;IACA,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,KAAK,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;KACb,CAAC,CAAC;IACH,IAAI,EAAE,kBAAkB,CAAC;CACxB,CAAC;AAEL,MAAM,WAAW,QAAQ;IACxB,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,OAAO,EAAE,aAAa,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACxB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,IAAI,EAAE,QAAQ,CAAC;IACf,EAAE,EAAE,YAAY,CAAC;IACjB,WAAW,CAAC,EAAE,YAAY,CAAC;IAC3B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,eAAe,EAAE,CAAC;IAC7B,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,GAAG,WAAW,CAAC,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/index.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/index.js new file mode 100644 index 0000000000000000000000000000000000000000..ea77963d93d8c81459d9b80b5de16b9031b1ad28 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/index.js @@ -0,0 +1,201 @@ +"use strict"; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.TASKS_DATA = exports.TASKS_MODEL_LIBRARIES = void 0; +const pipelines_js_1 = require("../pipelines.js"); +const data_js_1 = __importDefault(require("./any-to-any/data.js")); +const data_js_2 = __importDefault(require("./audio-classification/data.js")); +const data_js_3 = __importDefault(require("./audio-text-to-text/data.js")); +const data_js_4 = __importDefault(require("./audio-to-audio/data.js")); +const data_js_5 = __importDefault(require("./automatic-speech-recognition/data.js")); +const data_js_6 = __importDefault(require("./document-question-answering/data.js")); +const data_js_7 = __importDefault(require("./feature-extraction/data.js")); +const data_js_8 = __importDefault(require("./fill-mask/data.js")); +const data_js_9 = __importDefault(require("./image-classification/data.js")); +const data_js_10 = __importDefault(require("./image-feature-extraction/data.js")); +const data_js_11 = __importDefault(require("./image-to-image/data.js")); +const data_js_12 = __importDefault(require("./image-to-text/data.js")); +const data_js_13 = __importDefault(require("./image-text-to-text/data.js")); +const data_js_14 = __importDefault(require("./image-text-to-image/data.js")); +const data_js_15 = __importDefault(require("./image-text-to-video/data.js")); +const data_js_16 = __importDefault(require("./image-segmentation/data.js")); +const data_js_17 = __importDefault(require("./image-to-video/data.js")); +const data_js_18 = __importDefault(require("./mask-generation/data.js")); +const data_js_19 = __importDefault(require("./object-detection/data.js")); +const data_js_20 = __importDefault(require("./depth-estimation/data.js")); +const data_js_21 = __importDefault(require("./placeholder/data.js")); +const data_js_22 = __importDefault(require("./reinforcement-learning/data.js")); +const data_js_23 = __importDefault(require("./question-answering/data.js")); +const data_js_24 = __importDefault(require("./sentence-similarity/data.js")); +const data_js_25 = __importDefault(require("./summarization/data.js")); +const data_js_26 = __importDefault(require("./table-question-answering/data.js")); +const data_js_27 = __importDefault(require("./tabular-classification/data.js")); +const data_js_28 = __importDefault(require("./tabular-regression/data.js")); +const data_js_29 = __importDefault(require("./text-to-image/data.js")); +const data_js_30 = __importDefault(require("./text-to-speech/data.js")); +const data_js_31 = __importDefault(require("./token-classification/data.js")); +const data_js_32 = __importDefault(require("./translation/data.js")); +const data_js_33 = __importDefault(require("./text-classification/data.js")); +const data_js_34 = __importDefault(require("./text-generation/data.js")); +const data_js_35 = __importDefault(require("./text-ranking/data.js")); +const data_js_36 = __importDefault(require("./text-to-video/data.js")); +const data_js_37 = __importDefault(require("./unconditional-image-generation/data.js")); +const data_js_38 = __importDefault(require("./video-classification/data.js")); +const data_js_39 = __importDefault(require("./visual-document-retrieval/data.js")); +const data_js_40 = __importDefault(require("./visual-question-answering/data.js")); +const data_js_41 = __importDefault(require("./zero-shot-classification/data.js")); +const data_js_42 = __importDefault(require("./zero-shot-image-classification/data.js")); +const data_js_43 = __importDefault(require("./zero-shot-object-detection/data.js")); +const data_js_44 = __importDefault(require("./image-to-3d/data.js")); +const data_js_45 = __importDefault(require("./text-to-3d/data.js")); +const data_js_46 = __importDefault(require("./keypoint-detection/data.js")); +const data_js_47 = __importDefault(require("./video-text-to-text/data.js")); +const data_js_48 = __importDefault(require("./video-to-video/data.js")); +/** + * Model libraries compatible with each ML task + */ +exports.TASKS_MODEL_LIBRARIES = { + "audio-classification": ["speechbrain", "transformers", "transformers.js"], + "audio-to-audio": ["asteroid", "fairseq", "speechbrain"], + "automatic-speech-recognition": ["espnet", "nemo", "speechbrain", "transformers", "transformers.js"], + "audio-text-to-text": ["transformers"], + "depth-estimation": ["transformers", "transformers.js"], + "document-question-answering": ["transformers", "transformers.js"], + "feature-extraction": ["sentence-transformers", "transformers", "transformers.js"], + "fill-mask": ["transformers", "transformers.js"], + "graph-ml": ["transformers"], + "image-classification": ["keras", "timm", "transformers", "transformers.js"], + "image-feature-extraction": ["timm", "transformers"], + "image-segmentation": ["transformers", "transformers.js"], + "image-text-to-text": ["transformers"], + "image-text-to-image": ["diffusers"], + "image-text-to-video": ["diffusers"], + "image-to-image": ["diffusers", "transformers", "transformers.js"], + "image-to-text": ["transformers", "transformers.js"], + "image-to-video": ["diffusers"], + "keypoint-detection": ["transformers"], + "video-classification": ["transformers"], + "mask-generation": ["transformers"], + "multiple-choice": ["transformers"], + "object-detection": ["transformers", "transformers.js", "ultralytics"], + other: [], + "question-answering": ["adapter-transformers", "allennlp", "transformers", "transformers.js"], + robotics: [], + "reinforcement-learning": ["transformers", "stable-baselines3", "ml-agents", "sample-factory"], + "sentence-similarity": ["sentence-transformers", "spacy", "transformers.js"], + summarization: ["transformers", "transformers.js"], + "table-question-answering": ["transformers"], + "table-to-text": ["transformers"], + "tabular-classification": ["sklearn"], + "tabular-regression": ["sklearn"], + "tabular-to-text": ["transformers"], + "text-classification": ["adapter-transformers", "setfit", "spacy", "transformers", "transformers.js"], + "text-generation": ["transformers", "transformers.js"], + "text-ranking": ["sentence-transformers", "transformers"], + "text-retrieval": [], + "text-to-image": ["diffusers"], + "text-to-speech": ["espnet", "tensorflowtts", "transformers", "transformers.js"], + "text-to-audio": ["transformers", "transformers.js"], + "text-to-video": ["diffusers"], + "time-series-forecasting": [], + "token-classification": [ + "adapter-transformers", + "flair", + "spacy", + "span-marker", + "stanza", + "transformers", + "transformers.js", + ], + translation: ["transformers", "transformers.js"], + "unconditional-image-generation": ["diffusers"], + "video-text-to-text": ["transformers"], + "visual-question-answering": ["transformers", "transformers.js"], + "voice-activity-detection": [], + "zero-shot-classification": ["transformers", "transformers.js"], + "zero-shot-image-classification": ["transformers", "transformers.js"], + "zero-shot-object-detection": ["transformers", "transformers.js"], + "text-to-3d": ["diffusers"], + "image-to-3d": ["diffusers"], + "any-to-any": ["transformers"], + "visual-document-retrieval": ["transformers"], + "video-to-video": ["diffusers"], +}; +/** + * Return the whole TaskData object for a certain task. + * If the partialTaskData argument is left undefined, + * the default placeholder data will be used. + */ +function getData(type, partialTaskData = data_js_21.default) { + return { + ...partialTaskData, + id: type, + label: pipelines_js_1.PIPELINE_DATA[type].name, + libraries: exports.TASKS_MODEL_LIBRARIES[type], + }; +} +// To make comparisons easier, task order is the same as in const.ts +// Tasks set to undefined won't have an associated task page. +// Tasks that call getData() without the second argument will +// have a "placeholder" page. +exports.TASKS_DATA = { + "any-to-any": getData("any-to-any", data_js_1.default), + "audio-classification": getData("audio-classification", data_js_2.default), + "audio-to-audio": getData("audio-to-audio", data_js_4.default), + "audio-text-to-text": getData("audio-text-to-text", data_js_3.default), + "automatic-speech-recognition": getData("automatic-speech-recognition", data_js_5.default), + "depth-estimation": getData("depth-estimation", data_js_20.default), + "document-question-answering": getData("document-question-answering", data_js_6.default), + "visual-document-retrieval": getData("visual-document-retrieval", data_js_39.default), + "feature-extraction": getData("feature-extraction", data_js_7.default), + "fill-mask": getData("fill-mask", data_js_8.default), + "graph-ml": undefined, + "image-classification": getData("image-classification", data_js_9.default), + "image-feature-extraction": getData("image-feature-extraction", data_js_10.default), + "image-segmentation": getData("image-segmentation", data_js_16.default), + "image-to-image": getData("image-to-image", data_js_11.default), + "image-text-to-text": getData("image-text-to-text", data_js_13.default), + "image-text-to-image": getData("image-text-to-image", data_js_14.default), + "image-text-to-video": getData("image-text-to-video", data_js_15.default), + "image-to-text": getData("image-to-text", data_js_12.default), + "image-to-video": getData("image-to-video", data_js_17.default), + "keypoint-detection": getData("keypoint-detection", data_js_46.default), + "mask-generation": getData("mask-generation", data_js_18.default), + "multiple-choice": undefined, + "object-detection": getData("object-detection", data_js_19.default), + "video-classification": getData("video-classification", data_js_38.default), + other: undefined, + "question-answering": getData("question-answering", data_js_23.default), + "reinforcement-learning": getData("reinforcement-learning", data_js_22.default), + robotics: undefined, + "sentence-similarity": getData("sentence-similarity", data_js_24.default), + summarization: getData("summarization", data_js_25.default), + "table-question-answering": getData("table-question-answering", data_js_26.default), + "table-to-text": undefined, + "tabular-classification": getData("tabular-classification", data_js_27.default), + "tabular-regression": getData("tabular-regression", data_js_28.default), + "tabular-to-text": undefined, + "text-classification": getData("text-classification", data_js_33.default), + "text-generation": getData("text-generation", data_js_34.default), + "text-ranking": getData("text-ranking", data_js_35.default), + "text-retrieval": undefined, + "text-to-image": getData("text-to-image", data_js_29.default), + "text-to-speech": getData("text-to-speech", data_js_30.default), + "text-to-audio": undefined, + "text-to-video": getData("text-to-video", data_js_36.default), + "time-series-forecasting": undefined, + "token-classification": getData("token-classification", data_js_31.default), + translation: getData("translation", data_js_32.default), + "unconditional-image-generation": getData("unconditional-image-generation", data_js_37.default), + "video-text-to-text": getData("video-text-to-text", data_js_47.default), + "video-to-video": getData("video-to-video", data_js_48.default), + "visual-question-answering": getData("visual-question-answering", data_js_40.default), + "voice-activity-detection": undefined, + "zero-shot-classification": getData("zero-shot-classification", data_js_41.default), + "zero-shot-image-classification": getData("zero-shot-image-classification", data_js_42.default), + "zero-shot-object-detection": getData("zero-shot-object-detection", data_js_43.default), + "text-to-3d": getData("text-to-3d", data_js_45.default), + "image-to-3d": getData("image-to-3d", data_js_44.default), +}; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/keypoint-detection/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/keypoint-detection/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/keypoint-detection/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/keypoint-detection/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/keypoint-detection/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..eb823f21e8304e3e89965c2360ab9594e5d59139 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/keypoint-detection/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/keypoint-detection/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAqDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/keypoint-detection/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/keypoint-detection/data.js new file mode 100644 index 0000000000000000000000000000000000000000..94880999e406d783ecaff09d3e041f547c9c6723 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/keypoint-detection/data.js @@ -0,0 +1,57 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A dataset of hand keypoints of over 500k examples.", + id: "Vincent-luo/hagrid-mediapipe-hands", + }, + ], + demo: { + inputs: [ + { + filename: "keypoint-detection-input.png", + type: "img", + }, + ], + outputs: [ + { + filename: "keypoint-detection-output.png", + type: "img", + }, + ], + }, + metrics: [], + models: [ + { + description: "A robust keypoint detection model.", + id: "magic-leap-community/superpoint", + }, + { + description: "A robust keypoint matching model.", + id: "magic-leap-community/superglue_outdoor", + }, + { + description: "Strong keypoint detection model used to detect human pose.", + id: "qualcomm/RTMPose-Body2d", + }, + { + description: "Powerful keypoint matching model.", + id: "ETH-CVG/lightglue_disk", + }, + ], + spaces: [ + { + description: "An application that detects hand keypoints in real-time.", + id: "datasciencedojo/Hand-Keypoint-Detection-Realtime", + }, + { + description: "An application for keypoint detection and matching.", + id: "ETH-CVG/LightGlue", + }, + ], + summary: "Keypoint detection is the task of identifying meaningful distinctive points or features in an image.", + widgetModels: [], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/mask-generation/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/mask-generation/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/mask-generation/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/mask-generation/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/mask-generation/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..4c15a98fdabb31e019d38076d691da7391cefa47 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/mask-generation/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/mask-generation/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAgEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/mask-generation/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/mask-generation/data.js new file mode 100644 index 0000000000000000000000000000000000000000..20d3efc52f5252944f4701981559907eb3f04486 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/mask-generation/data.js @@ -0,0 +1,66 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "Widely used benchmark dataset for multiple Vision tasks.", + id: "merve/coco2017", + }, + { + description: "Medical Imaging dataset of the Human Brain for segmentation and mask generating tasks", + id: "rocky93/BraTS_segmentation", + }, + ], + demo: { + inputs: [ + { + filename: "mask-generation-input.png", + type: "img", + }, + ], + outputs: [ + { + filename: "mask-generation-output.png", + type: "img", + }, + ], + }, + metrics: [ + { + description: "IoU is used to measure the overlap between predicted mask and the ground truth mask.", + id: "Intersection over Union (IoU)", + }, + ], + models: [ + { + description: "Small yet powerful mask generation model.", + id: "Zigeng/SlimSAM-uniform-50", + }, + { + description: "Very strong mask generation model.", + id: "facebook/sam2-hiera-large", + }, + ], + spaces: [ + { + description: "An application that combines a mask generation model with a zero-shot object detection model for text-guided image segmentation.", + id: "merve/OWLSAM2", + }, + { + description: "An application that compares the performance of a large and a small mask generation model.", + id: "merve/slimsam", + }, + { + description: "An application based on an improved mask generation model.", + id: "SkalskiP/segment-anything-model-2", + }, + { + description: "An application to remove objects from videos using mask generation models.", + id: "SkalskiP/SAM_and_ProPainter", + }, + ], + summary: "Mask generation is the task of generating masks that identify a specific object or region of interest in a given image. Masks are often used in segmentation tasks, where they provide a precise way to isolate the object of interest for further processing or analysis.", + widgetModels: [], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..628b2c78e9a1f523dcab289b02218d52718855ac --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/object-detection/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAqFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/data.js new file mode 100644 index 0000000000000000000000000000000000000000..497e960bb08cf88dfa3d1b309f4420d9ee987b62 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/data.js @@ -0,0 +1,86 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "Widely used benchmark dataset for multiple vision tasks.", + id: "merve/coco2017", + }, + { + description: "Multi-task computer vision benchmark.", + id: "merve/pascal-voc", + }, + ], + demo: { + inputs: [ + { + filename: "object-detection-input.jpg", + type: "img", + }, + ], + outputs: [ + { + filename: "object-detection-output.jpg", + type: "img", + }, + ], + }, + metrics: [ + { + description: "The Average Precision (AP) metric is the Area Under the PR Curve (AUC-PR). It is calculated for each class separately", + id: "Average Precision", + }, + { + description: "The Mean Average Precision (mAP) metric is the overall average of the AP values", + id: "Mean Average Precision", + }, + { + description: "The APα metric is the Average Precision at the IoU threshold of a α value, for example, AP50 and AP75", + id: "APα", + }, + ], + models: [ + { + description: "Solid object detection model pre-trained on the COCO 2017 dataset.", + id: "facebook/detr-resnet-50", + }, + { + description: "Accurate object detection model.", + id: "IDEA-Research/dab-detr-resnet-50", + }, + { + description: "Fast and accurate object detection model.", + id: "PekingU/rtdetr_v2_r50vd", + }, + { + description: "Object detection model for low-lying objects.", + id: "StephanST/WALDO30", + }, + ], + spaces: [ + { + description: "Real-time object detection demo.", + id: "Roboflow/RF-DETR", + }, + { + description: "An application that contains various object detection models to try from.", + id: "Gradio-Blocks/Object-Detection-With-DETR-and-YOLOS", + }, + { + description: "A cutting-edge object detection application.", + id: "sunsmarterjieleaf/yolov12", + }, + { + description: "An object tracking, segmentation and inpainting application.", + id: "VIPLab/Track-Anything", + }, + { + description: "Very fast object tracking application based on object detection.", + id: "merve/RT-DETR-tracking-coco", + }, + ], + summary: "Object Detection models allow users to identify objects of certain defined classes. Object detection models receive an image as input and output the images with bounding boxes and labels on detected objects.", + widgetModels: ["facebook/detr-resnet-50"], + youtubeId: "WdAeKSOpxhw", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..14661180189671f77efac91846a902f1f85d91a9 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/inference.d.ts @@ -0,0 +1,74 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Object Detection inference + */ +export interface ObjectDetectionInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Object Detection + */ + parameters?: ObjectDetectionParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Object Detection + */ +export interface ObjectDetectionParameters { + /** + * The probability necessary to make a prediction. + */ + threshold?: number; + [property: string]: unknown; +} +/** + * The predicted bounding box. Coordinates are relative to the top left corner of the input + * image. + */ +export interface BoundingBox { + /** + * The x-coordinate of the bottom-right corner of the bounding box. + */ + xmax: number; + /** + * The x-coordinate of the top-left corner of the bounding box. + */ + xmin: number; + /** + * The y-coordinate of the bottom-right corner of the bounding box. + */ + ymax: number; + /** + * The y-coordinate of the top-left corner of the bounding box. + */ + ymin: number; + [property: string]: unknown; +} +export type ObjectDetectionOutput = ObjectDetectionOutputElement[]; +/** + * Outputs of inference for the Object Detection task + */ +export interface ObjectDetectionOutputElement { + /** + * The predicted bounding box. Coordinates are relative to the top left corner of the input + * image. + */ + box: BoundingBox; + /** + * The predicted label for the bounding box. + */ + label: string; + /** + * The associated score / probability. + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..64ac1c743fb4d62b1c12cceb58c692b409d268e1 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/object-detection/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;;OAGG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,yBAAyB,CAAC;IACvC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACzC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC3B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,qBAAqB,GAAG,4BAA4B,EAAE,CAAC;AACnE;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC5C;;;OAGG;IACH,GAAG,EAAE,WAAW,CAAC;IACjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/object-detection/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/placeholder/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/placeholder/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/placeholder/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/placeholder/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/placeholder/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5db21ad2a46706da5b2235bfb24ee2a5a9cc0757 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/placeholder/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/placeholder/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAgBf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/placeholder/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/placeholder/data.js new file mode 100644 index 0000000000000000000000000000000000000000..491ecdc94848cdd3de410eaba62a51c9343e9352 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/placeholder/data.js @@ -0,0 +1,20 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [], + demo: { + inputs: [], + outputs: [], + }, + isPlaceholder: true, + metrics: [], + models: [], + spaces: [], + summary: "", + widgetModels: [], + youtubeId: undefined, + /// If this is a subtask, link to the most general task ID + /// (eg, text-generation is the canonical ID of text-simplification) + canonicalId: undefined, +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..796067455ff80c17a422ec70958bc4311e8124e3 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/question-answering/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAsEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/data.js new file mode 100644 index 0000000000000000000000000000000000000000..305489e7689da030d4ea07bb2aff0ae8b60d0979 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/data.js @@ -0,0 +1,71 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + // TODO write proper description + description: "A famous question answering dataset based on English articles from Wikipedia.", + id: "squad_v2", + }, + { + // TODO write proper description + description: "A dataset of aggregated anonymized actual queries issued to the Google search engine.", + id: "natural_questions", + }, + ], + demo: { + inputs: [ + { + label: "Question", + content: "Which name is also used to describe the Amazon rainforest in English?", + type: "text", + }, + { + label: "Context", + content: "The Amazon rainforest, also known in English as Amazonia or the Amazon Jungle", + type: "text", + }, + ], + outputs: [ + { + label: "Answer", + content: "Amazonia", + type: "text", + }, + ], + }, + metrics: [ + { + description: "Exact Match is a metric based on the strict character match of the predicted answer and the right answer. For answers predicted correctly, the Exact Match will be 1. Even if only one character is different, Exact Match will be 0", + id: "exact-match", + }, + { + description: " The F1-Score metric is useful if we value both false positives and false negatives equally. The F1-Score is calculated on each word in the predicted sequence against the correct answer", + id: "f1", + }, + ], + models: [ + { + description: "A robust baseline model for most question answering domains.", + id: "deepset/roberta-base-squad2", + }, + { + description: "Small yet robust model that can answer questions.", + id: "distilbert/distilbert-base-cased-distilled-squad", + }, + { + description: "A special model that can answer questions from tables.", + id: "google/tapas-base-finetuned-wtq", + }, + ], + spaces: [ + { + description: "An application that can answer a long question from Wikipedia.", + id: "deepset/wikipedia-assistant", + }, + ], + summary: "Question Answering models can retrieve the answer to a question from a given text, which is useful for searching for an answer in a document. Some question answering models can generate answers without context!", + widgetModels: ["deepset/roberta-base-squad2"], + youtubeId: "ajPx5LwJD-I", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..0956db6c47a1bb07ee6c2bd1fd5603b4d66b406f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/inference.d.ts @@ -0,0 +1,98 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Question Answering inference + */ +export interface QuestionAnsweringInput { + /** + * One (context, question) pair to answer + */ + inputs: QuestionAnsweringInputData; + /** + * Additional inference parameters for Question Answering + */ + parameters?: QuestionAnsweringParameters; + [property: string]: unknown; +} +/** + * One (context, question) pair to answer + */ +export interface QuestionAnsweringInputData { + /** + * The context to be used for answering the question + */ + context: string; + /** + * The question to be answered + */ + question: string; + [property: string]: unknown; +} +/** + * Additional inference parameters for Question Answering + */ +export interface QuestionAnsweringParameters { + /** + * Attempts to align the answer to real words. Improves quality on space separated + * languages. Might hurt on non-space-separated languages (like Japanese or Chinese) + */ + align_to_words?: boolean; + /** + * If the context is too long to fit with the question for the model, it will be split in + * several chunks with some overlap. This argument controls the size of that overlap. + */ + doc_stride?: number; + /** + * Whether to accept impossible as an answer. + */ + handle_impossible_answer?: boolean; + /** + * The maximum length of predicted answers (e.g., only answers with a shorter length are + * considered). + */ + max_answer_len?: number; + /** + * The maximum length of the question after tokenization. It will be truncated if needed. + */ + max_question_len?: number; + /** + * The maximum length of the total sentence (context + question) in tokens of each chunk + * passed to the model. The context will be split in several chunks (using docStride as + * overlap) if needed. + */ + max_seq_len?: number; + /** + * The number of answers to return (will be chosen by order of likelihood). Note that we + * return less than topk answers if there are not enough options available within the + * context. + */ + top_k?: number; + [property: string]: unknown; +} +export type QuestionAnsweringOutput = QuestionAnsweringOutputElement[]; +/** + * Outputs of inference for the Question Answering task + */ +export interface QuestionAnsweringOutputElement { + /** + * The answer to the question. + */ + answer: string; + /** + * The character position in the input where the answer ends. + */ + end: number; + /** + * The probability associated to the answer. + */ + score: number; + /** + * The character position in the input where the answer begins. + */ + start: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..bf237c1fb8f8f2de5f02091faf25bed254771a75 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/question-answering/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,MAAM,EAAE,0BAA0B,CAAC;IACnC;;OAEG;IACH,UAAU,CAAC,EAAE,2BAA2B,CAAC;IACzC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,0BAA0B;IAC1C;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC3C;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,uBAAuB,GAAG,8BAA8B,EAAE,CAAC;AACvE;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC9C;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/question-answering/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/reinforcement-learning/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/reinforcement-learning/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/reinforcement-learning/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/reinforcement-learning/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/reinforcement-learning/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..da6859fc13463382ff95b67b4ee618679d06845c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/reinforcement-learning/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/reinforcement-learning/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAsEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/reinforcement-learning/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/reinforcement-learning/data.js new file mode 100644 index 0000000000000000000000000000000000000000..65aa55e5b63c964cfb3f93bfae99aeca042118ae --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/reinforcement-learning/data.js @@ -0,0 +1,69 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A curation of widely used datasets for Data Driven Deep Reinforcement Learning (D4RL)", + id: "edbeeching/decision_transformer_gym_replay", + }, + ], + demo: { + inputs: [ + { + label: "State", + content: "Red traffic light, pedestrians are about to pass.", + type: "text", + }, + ], + outputs: [ + { + label: "Action", + content: "Stop the car.", + type: "text", + }, + { + label: "Next State", + content: "Yellow light, pedestrians have crossed.", + type: "text", + }, + ], + }, + metrics: [ + { + description: "Accumulated reward across all time steps discounted by a factor that ranges between 0 and 1 and determines how much the agent optimizes for future relative to immediate rewards. Measures how good is the policy ultimately found by a given algorithm considering uncertainty over the future.", + id: "Discounted Total Reward", + }, + { + description: "Average return obtained after running the policy for a certain number of evaluation episodes. As opposed to total reward, mean reward considers how much reward a given algorithm receives while learning.", + id: "Mean Reward", + }, + { + description: "Measures how good a given algorithm is after a predefined time. Some algorithms may be guaranteed to converge to optimal behavior across many time steps. However, an agent that reaches an acceptable level of optimality after a given time horizon may be preferable to one that ultimately reaches optimality but takes a long time.", + id: "Level of Performance After Some Time", + }, + ], + models: [ + { + description: "A Reinforcement Learning model trained on expert data from the Gym Hopper environment", + id: "edbeeching/decision-transformer-gym-hopper-expert", + }, + { + description: "A PPO agent playing seals/CartPole-v0 using the stable-baselines3 library and the RL Zoo.", + id: "HumanCompatibleAI/ppo-seals-CartPole-v0", + }, + ], + spaces: [ + { + description: "An application for a cute puppy agent learning to catch a stick.", + id: "ThomasSimonini/Huggy", + }, + { + description: "An application to play Snowball Fight with a reinforcement learning agent.", + id: "ThomasSimonini/SnowballFight", + }, + ], + summary: "Reinforcement learning is the computational approach of learning from action by interacting with an environment through trial and error and receiving rewards (negative or positive) as feedback", + widgetModels: [], + youtubeId: "q0BiUn5LiBc", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d457bc40bbb8d21b76a24e583381c7ea694b063e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/sentence-similarity/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAoGf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/data.js new file mode 100644 index 0000000000000000000000000000000000000000..2f01e72b547817bb5ad879043af8d2ab7890c5de --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/data.js @@ -0,0 +1,99 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "Bing queries with relevant passages from various web sources.", + id: "microsoft/ms_marco", + }, + ], + demo: { + inputs: [ + { + label: "Source sentence", + content: "Machine learning is so easy.", + type: "text", + }, + { + label: "Sentences to compare to", + content: "Deep learning is so straightforward.", + type: "text", + }, + { + label: "", + content: "This is so difficult, like rocket science.", + type: "text", + }, + { + label: "", + content: "I can't believe how much I struggled with this.", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Deep learning is so straightforward.", + score: 0.623, + }, + { + label: "This is so difficult, like rocket science.", + score: 0.413, + }, + { + label: "I can't believe how much I struggled with this.", + score: 0.256, + }, + ], + }, + ], + }, + metrics: [ + { + description: "Reciprocal Rank is a measure used to rank the relevancy of documents given a set of documents. Reciprocal Rank is the reciprocal of the rank of the document retrieved, meaning, if the rank is 3, the Reciprocal Rank is 0.33. If the rank is 1, the Reciprocal Rank is 1", + id: "Mean Reciprocal Rank", + }, + { + description: "The similarity of the embeddings is evaluated mainly on cosine similarity. It is calculated as the cosine of the angle between two vectors. It is particularly useful when your texts are not the same length", + id: "Cosine Similarity", + }, + ], + models: [ + { + description: "This model works well for sentences and paragraphs and can be used for clustering/grouping and semantic searches.", + id: "sentence-transformers/all-mpnet-base-v2", + }, + { + description: "A multilingual robust sentence similarity model.", + id: "BAAI/bge-m3", + }, + { + description: "A robust sentence similarity model.", + id: "HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v1.5", + }, + ], + spaces: [ + { + description: "An application that leverages sentence similarity to answer questions from YouTube videos.", + id: "Gradio-Blocks/Ask_Questions_To_YouTube_Videos", + }, + { + description: "An application that retrieves relevant PubMed abstracts for a given online article which can be used as further references.", + id: "Gradio-Blocks/pubmed-abstract-retriever", + }, + { + description: "An application that leverages sentence similarity to summarize text.", + id: "nickmuchi/article-text-summarizer", + }, + { + description: "A guide that explains how Sentence Transformers can be used for semantic search.", + id: "sentence-transformers/Sentence_Transformers_for_semantic_search", + }, + ], + summary: "Sentence Similarity is the task of determining how similar two texts are. Sentence similarity models convert input texts into vectors (embeddings) that capture semantic information and calculate how close (similar) they are between them. This task is particularly useful for information retrieval and clustering/grouping.", + widgetModels: ["sentence-transformers/all-MiniLM-L6-v2"], + youtubeId: "VCZq5AkbNEU", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..7836ed35d431c28daa542326aca7ea75563f5b9c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/inference.d.ts @@ -0,0 +1,32 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +export type SentenceSimilarityOutput = number[]; +/** + * Inputs for Sentence similarity inference + */ +export interface SentenceSimilarityInput { + inputs: SentenceSimilarityInputData; + /** + * Additional inference parameters for Sentence Similarity + */ + parameters?: { + [key: string]: unknown; + }; + [property: string]: unknown; +} +export interface SentenceSimilarityInputData { + /** + * A list of strings which will be compared against the source_sentence. + */ + sentences: string[]; + /** + * The string that you wish to compare the other strings with. This can be a phrase, + * sentence, or longer passage, depending on the model being used. + */ + source_sentence: string; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..1a31ca5b3e6cbd66e566a52e86cba1a024684a97 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/sentence-similarity/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,EAAE,CAAC;AAChD;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC,MAAM,EAAE,2BAA2B,CAAC;IACpC;;OAEG;IACH,UAAU,CAAC,EAAE;QACZ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C;;OAEG;IACH,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB;;;OAGG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/sentence-similarity/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7f112b24d58cd35588a17df77cc61bad268f5b37 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/summarization/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAuEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/data.js new file mode 100644 index 0000000000000000000000000000000000000000..a1018747c3cb91ac2d5be946b806d31a72ac8f9f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/data.js @@ -0,0 +1,69 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + canonicalId: "text-generation", + datasets: [ + { + description: "News articles in five different languages along with their summaries. Widely used for benchmarking multilingual summarization models.", + id: "mlsum", + }, + { + description: "English conversations and their summaries. Useful for benchmarking conversational agents.", + id: "samsum", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. It was the first structure to reach a height of 300 metres. Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct.", + type: "text", + }, + ], + outputs: [ + { + label: "Output", + content: "The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building. It was the first structure to reach a height of 300 metres.", + type: "text", + }, + ], + }, + metrics: [ + { + description: "The generated sequence is compared against its summary, and the overlap of tokens are counted. ROUGE-N refers to overlap of N subsequent tokens, ROUGE-1 refers to overlap of single tokens and ROUGE-2 is the overlap of two subsequent tokens.", + id: "rouge", + }, + ], + models: [ + { + description: "A strong summarization model trained on English news articles. Excels at generating factual summaries.", + id: "facebook/bart-large-cnn", + }, + { + description: "A summarization model trained on medical articles.", + id: "Falconsai/medical_summarization", + }, + ], + spaces: [ + { + description: "An application that can summarize long paragraphs.", + id: "pszemraj/summarize-long-text", + }, + { + description: "A much needed summarization application for terms and conditions.", + id: "ml6team/distilbart-tos-summarizer-tosdr", + }, + { + description: "An application that summarizes long documents.", + id: "pszemraj/document-summarization", + }, + { + description: "An application that can detect errors in abstractive summarization.", + id: "ml6team/post-processing-summarization", + }, + ], + summary: "Summarization is the task of producing a shorter version of a document while preserving its important information. Some models can extract text from the original input, while other models can generate entirely new text.", + widgetModels: ["facebook/bart-large-cnn"], + youtubeId: "yHnr5Dk2zCI", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b3153fbb4fdb5eeb040516b68b90dfdda2057798 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/inference.d.ts @@ -0,0 +1,54 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Summarization inference + */ +export interface SummarizationInput { + /** + * The input text to summarize. + */ + inputs: string; + /** + * Additional inference parameters for summarization. + */ + parameters?: SummarizationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for summarization. + */ +export interface SummarizationParameters { + /** + * Whether to clean up the potential extra spaces in the text output. + */ + clean_up_tokenization_spaces?: boolean; + /** + * Additional parametrization of the text generation algorithm. + */ + generate_parameters?: { + [key: string]: unknown; + }; + /** + * The truncation strategy to use. + */ + truncation?: SummarizationTruncationStrategy; + [property: string]: unknown; +} +/** + * The truncation strategy to use. + */ +export type SummarizationTruncationStrategy = "do_not_truncate" | "longest_first" | "only_first" | "only_second"; +/** + * Outputs of inference for the Summarization task + */ +export interface SummarizationOutput { + /** + * The summarized text. + */ + summary_text: string; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e2834ebbaa672e6227241c2f2c7fad3e67a586ba --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/summarization/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,uBAAuB,CAAC;IACrC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC;;OAEG;IACH,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC;;OAEG;IACH,mBAAmB,CAAC,EAAE;QACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF;;OAEG;IACH,UAAU,CAAC,EAAE,+BAA+B,CAAC;IAC7C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,+BAA+B,GAAG,iBAAiB,GAAG,eAAe,GAAG,YAAY,GAAG,aAAa,CAAC;AACjH;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/summarization/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7df5547173450d1e1d1218adc4b5e3c9a2d90075 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/table-question-answering/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAsDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/data.js new file mode 100644 index 0000000000000000000000000000000000000000..9e79709f75feb6ffca8337d2b7280d4537930f87 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/data.js @@ -0,0 +1,54 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "The WikiTableQuestions dataset is a large-scale dataset for the task of question answering on semi-structured tables.", + id: "wikitablequestions", + }, + { + description: "WikiSQL is a dataset of 80654 hand-annotated examples of questions and SQL queries distributed across 24241 tables from Wikipedia.", + id: "wikisql", + }, + ], + demo: { + inputs: [ + { + table: [ + ["Rank", "Name", "No.of reigns", "Combined days"], + ["1", "lou Thesz", "3", "3749"], + ["2", "Ric Flair", "8", "3103"], + ["3", "Harley Race", "7", "1799"], + ], + type: "tabular", + }, + { label: "Question", content: "What is the number of reigns for Harley Race?", type: "text" }, + ], + outputs: [{ label: "Result", content: "7", type: "text" }], + }, + metrics: [ + { + description: "Checks whether the predicted answer(s) is the same as the ground-truth answer(s).", + id: "Denotation Accuracy", + }, + ], + models: [ + { + description: "A table question answering model that is capable of neural SQL execution, i.e., employ TAPEX to execute a SQL query on a given table.", + id: "microsoft/tapex-base", + }, + { + description: "A robust table question answering model.", + id: "google/tapas-base-finetuned-wtq", + }, + ], + spaces: [ + { + description: "An application that answers questions based on table CSV files.", + id: "katanaml/table-query", + }, + ], + summary: "Table Question Answering (Table QA) is the answering a question about an information on a given table.", + widgetModels: ["google/tapas-base-finetuned-wtq"], +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..bfe5048481141e19695ee9459efa19ceda69949d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/inference.d.ts @@ -0,0 +1,84 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Table Question Answering inference + */ +export interface TableQuestionAnsweringInput { + /** + * One (table, question) pair to answer + */ + inputs: TableQuestionAnsweringInputData; + /** + * Additional inference parameters for Table Question Answering + */ + parameters?: TableQuestionAnsweringParameters; + [property: string]: unknown; +} +/** + * One (table, question) pair to answer + */ +export interface TableQuestionAnsweringInputData { + /** + * The question to be answered about the table + */ + question: string; + /** + * The table to serve as context for the questions + */ + table: { + [key: string]: string[]; + }; + [property: string]: unknown; +} +/** + * Additional inference parameters for Table Question Answering + */ +export interface TableQuestionAnsweringParameters { + /** + * Activates and controls padding. + */ + padding?: Padding; + /** + * Whether to do inference sequentially or as a batch. Batching is faster, but models like + * SQA require the inference to be done sequentially to extract relations within sequences, + * given their conversational nature. + */ + sequential?: boolean; + /** + * Activates and controls truncation. + */ + truncation?: boolean; + [property: string]: unknown; +} +/** + * Activates and controls padding. + */ +export type Padding = "do_not_pad" | "longest" | "max_length"; +export type TableQuestionAnsweringOutput = TableQuestionAnsweringOutputElement[]; +/** + * Outputs of inference for the Table Question Answering task + */ +export interface TableQuestionAnsweringOutputElement { + /** + * If the model has an aggregator, this returns the aggregator. + */ + aggregator?: string; + /** + * The answer of the question given the table. If there is an aggregator, the answer will be + * preceded by `AGGREGATOR >`. + */ + answer: string; + /** + * List of strings made up of the answer cell values. + */ + cells: string[]; + /** + * Coordinates of the cells of the answers. + */ + coordinates: Array; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..662dc6b5c7390184441f95d326a449f226b02dbb --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/table-question-answering/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC3C;;OAEG;IACH,MAAM,EAAE,+BAA+B,CAAC;IACxC;;OAEG;IACH,UAAU,CAAC,EAAE,gCAAgC,CAAC;IAC9C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,+BAA+B;IAC/C;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,KAAK,EAAE;QACN,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;KACxB,CAAC;IACF,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,OAAO,GAAG,YAAY,GAAG,SAAS,GAAG,YAAY,CAAC;AAC9D,MAAM,MAAM,4BAA4B,GAAG,mCAAmC,EAAE,CAAC;AACjF;;GAEG;AACH,MAAM,WAAW,mCAAmC;IACnD;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB;;OAEG;IACH,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/table-question-answering/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5a327b8a6c01a349e8a3eb677fa7d63a604fd2a6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/tabular-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA+Df,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-classification/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..b84cc249aae2889eae65ee642dd10d08aa565667 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-classification/data.js @@ -0,0 +1,67 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A comprehensive curation of datasets covering all benchmarks.", + id: "inria-soda/tabular-benchmark", + }, + ], + demo: { + inputs: [ + { + table: [ + ["Glucose", "Blood Pressure ", "Skin Thickness", "Insulin", "BMI"], + ["148", "72", "35", "0", "33.6"], + ["150", "50", "30", "0", "35.1"], + ["141", "60", "29", "1", "39.2"], + ], + type: "tabular", + }, + ], + outputs: [ + { + table: [["Diabetes"], ["1"], ["1"], ["0"]], + type: "tabular", + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + description: "Breast cancer prediction model based on decision trees.", + id: "scikit-learn/cancer-prediction-trees", + }, + ], + spaces: [ + { + description: "An application that can predict defective products on a production line.", + id: "scikit-learn/tabular-playground", + }, + { + description: "An application that compares various tabular classification techniques on different datasets.", + id: "scikit-learn/classification", + }, + ], + summary: "Tabular classification is the task of classifying a target category (a group) based on set of attributes.", + widgetModels: ["scikit-learn/tabular-playground"], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-regression/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-regression/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-regression/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-regression/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-regression/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..8603a1fb917be27322c628610908f41d7228a969 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-regression/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/tabular-regression/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAoDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-regression/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-regression/data.js new file mode 100644 index 0000000000000000000000000000000000000000..12c17cbcea4da96e8059afe3a58338c069986912 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/tabular-regression/data.js @@ -0,0 +1,55 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A comprehensive curation of datasets covering all benchmarks.", + id: "inria-soda/tabular-benchmark", + }, + ], + demo: { + inputs: [ + { + table: [ + ["Car Name", "Horsepower", "Weight"], + ["ford torino", "140", "3,449"], + ["amc hornet", "97", "2,774"], + ["toyota corolla", "65", "1,773"], + ], + type: "tabular", + }, + ], + outputs: [ + { + table: [["MPG (miles per gallon)"], ["17"], ["18"], ["31"]], + type: "tabular", + }, + ], + }, + metrics: [ + { + description: "", + id: "mse", + }, + { + description: "Coefficient of determination (or R-squared) is a measure of how well the model fits the data. Higher R-squared is considered a better fit.", + id: "r-squared", + }, + ], + models: [ + { + description: "Fish weight prediction based on length measurements and species.", + id: "scikit-learn/Fish-Weight", + }, + ], + spaces: [ + { + description: "An application that can predict weight of a fish based on set of attributes.", + id: "scikit-learn/fish-weight-prediction", + }, + ], + summary: "Tabular regression is the task of predicting a numerical value given a set of attributes.", + widgetModels: ["scikit-learn/Fish-Weight"], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..90bc031f2e0d672008b73b37088e7779fba975a2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAkGf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..caf9f204e61ec8e6fcd11d4b244378aa5188f1b2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/data.js @@ -0,0 +1,100 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A widely used dataset used to benchmark multiple variants of text classification.", + id: "nyu-mll/glue", + }, + { + description: "A text classification dataset used to benchmark natural language inference models", + id: "stanfordnlp/snli", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "I love Hugging Face!", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "POSITIVE", + score: 0.9, + }, + { + label: "NEUTRAL", + score: 0.1, + }, + { + label: "NEGATIVE", + score: 0.0, + }, + ], + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "The F1 metric is the harmonic mean of the precision and recall. It can be calculated as: F1 = 2 * (precision * recall) / (precision + recall)", + id: "f1", + }, + ], + models: [ + { + description: "A robust model trained for sentiment analysis.", + id: "distilbert/distilbert-base-uncased-finetuned-sst-2-english", + }, + { + description: "A sentiment analysis model specialized in financial sentiment.", + id: "ProsusAI/finbert", + }, + { + description: "A sentiment analysis model specialized in analyzing tweets.", + id: "cardiffnlp/twitter-roberta-base-sentiment-latest", + }, + { + description: "A model that can classify languages.", + id: "papluca/xlm-roberta-base-language-detection", + }, + { + description: "A model that can classify text generation attacks.", + id: "meta-llama/Prompt-Guard-86M", + }, + ], + spaces: [ + { + description: "An application that can classify financial sentiment.", + id: "IoannisTr/Tech_Stocks_Trading_Assistant", + }, + { + description: "A dashboard that contains various text classification tasks.", + id: "miesnerjacob/Multi-task-NLP", + }, + { + description: "An application that analyzes user reviews in healthcare.", + id: "spacy/healthsea-demo", + }, + ], + summary: "Text Classification is the task of assigning a label or class to a given text. Some use cases are sentiment analysis, natural language inference, and assessing grammatical correctness.", + widgetModels: ["distilbert/distilbert-base-uncased-finetuned-sst-2-english"], + youtubeId: "leNG9fN9FQU", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ff29b09e9a909bb94b21f96d8a939dadcd7deb3b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/inference.d.ts @@ -0,0 +1,53 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text Classification inference + */ +export interface TextClassificationInput { + /** + * The text to classify + */ + inputs: string; + /** + * Additional inference parameters for Text Classification + */ + parameters?: TextClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text Classification + */ +export interface TextClassificationParameters { + /** + * The function to apply to the model outputs in order to retrieve the scores. + */ + function_to_apply?: ClassificationOutputTransform; + /** + * When specified, limits the output to the top K most probable classes. + */ + top_k?: number; + [property: string]: unknown; +} +/** + * The function to apply to the model outputs in order to retrieve the scores. + */ +export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none"; +export type TextClassificationOutput = TextClassificationOutputElement[]; +/** + * Outputs of inference for the Text Classification task + */ +export interface TextClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..328ab1be6f6979d09ce733cb3208285bba6b5fba --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-classification/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,4BAA4B,CAAC;IAC1C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC5C;;OAEG;IACH,iBAAiB,CAAC,EAAE,6BAA6B,CAAC;IAClD;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAC3E,MAAM,MAAM,wBAAwB,GAAG,+BAA+B,EAAE,CAAC;AACzE;;GAEG;AACH,MAAM,WAAW,+BAA+B;IAC/C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-classification/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..1cb375d4130e99f5926dd0eacd8fe90366df7b10 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-generation/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA6Hf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/data.js new file mode 100644 index 0000000000000000000000000000000000000000..b7872517a9a01a24053f1782d39087b0213d144d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/data.js @@ -0,0 +1,125 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "Multilingual dataset used to evaluate text generation models.", + id: "CohereForAI/Global-MMLU", + }, + { + description: "High quality multilingual data used to train text-generation models.", + id: "HuggingFaceFW/fineweb-2", + }, + { + description: "Truly open-source, curated and cleaned dialogue dataset.", + id: "HuggingFaceH4/ultrachat_200k", + }, + { + description: "A reasoning dataset.", + id: "open-r1/OpenThoughts-114k-math", + }, + { + description: "A multilingual instruction dataset with preference ratings on responses.", + id: "allenai/tulu-3-sft-mixture", + }, + { + description: "A large synthetic dataset for alignment of text generation models.", + id: "HuggingFaceTB/smoltalk", + }, + { + description: "A dataset made for training text generation models solving math questions.", + id: "HuggingFaceTB/finemath", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "Once upon a time,", + type: "text", + }, + ], + outputs: [ + { + label: "Output", + content: "Once upon a time, we knew that our ancestors were on the verge of extinction. The great explorers and poets of the Old World, from Alexander the Great to Chaucer, are dead and gone. A good many of our ancient explorers and poets have", + type: "text", + }, + ], + }, + metrics: [ + { + description: "Cross Entropy is a metric that calculates the difference between two probability distributions. Each probability distribution is the distribution of predicted words", + id: "Cross Entropy", + }, + { + description: "The Perplexity metric is the exponential of the cross-entropy loss. It evaluates the probabilities assigned to the next word by the model. Lower perplexity indicates better performance", + id: "Perplexity", + }, + ], + models: [ + { description: "A text-generation model trained to follow instructions.", id: "google/gemma-2-2b-it" }, + { + description: "Powerful text generation model for coding.", + id: "Qwen/Qwen3-Coder-480B-A35B-Instruct", + }, + { + description: "Great text generation model with top-notch tool calling capabilities.", + id: "openai/gpt-oss-120b", + }, + { + description: "Powerful text generation model.", + id: "zai-org/GLM-4.5", + }, + { + description: "A powerful small model with reasoning capabilities.", + id: "Qwen/Qwen3-4B-Thinking-2507", + }, + { + description: "Strong conversational model that supports very long instructions.", + id: "Qwen/Qwen2.5-7B-Instruct-1M", + }, + { + description: "Text generation model used to write code.", + id: "Qwen/Qwen2.5-Coder-32B-Instruct", + }, + { + description: "Powerful reasoning based open large language model.", + id: "deepseek-ai/DeepSeek-R1", + }, + ], + spaces: [ + { + description: "An application that writes and executes code from text instructions and supports many models.", + id: "akhaliq/anycoder", + }, + { + description: "An application that builds websites from natural language prompts.", + id: "enzostvs/deepsite", + }, + { + description: "A leaderboard for comparing chain-of-thought performance of models.", + id: "logikon/open_cot_leaderboard", + }, + { + description: "An text generation based application based on a very powerful LLaMA2 model.", + id: "ysharma/Explore_llamav2_with_TGI", + }, + { + description: "An text generation based application to converse with Zephyr model.", + id: "HuggingFaceH4/zephyr-chat", + }, + { + description: "A leaderboard that ranks text generation models based on blind votes from people.", + id: "lmsys/chatbot-arena-leaderboard", + }, + { + description: "An chatbot to converse with a very powerful text generation model.", + id: "mlabonne/phixtral-chat", + }, + ], + summary: "Generating text is the task of generating new text given another text. These models can, for example, fill in incomplete text or paraphrase.", + widgetModels: ["mistralai/Mistral-Nemo-Instruct-2407"], + youtubeId: "e9gNEAlsOvU", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..0495ad64fc55455d8b2a2933ae99b37d86775f1f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/inference.d.ts @@ -0,0 +1,188 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Text Generation Input. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface TextGenerationInput { + inputs: string; + parameters?: TextGenerationInputGenerateParameters; + stream?: boolean; + [property: string]: unknown; +} +export interface TextGenerationInputGenerateParameters { + /** + * Lora adapter id + */ + adapter_id?: string; + /** + * Generate best_of sequences and return the one if the highest token logprobs. + */ + best_of?: number; + /** + * Whether to return decoder input token logprobs and ids. + */ + decoder_input_details?: boolean; + /** + * Whether to return generation details. + */ + details?: boolean; + /** + * Activate logits sampling. + */ + do_sample?: boolean; + /** + * The parameter for frequency penalty. 1.0 means no penalty + * Penalize new tokens based on their existing frequency in the text so far, + * decreasing the model's likelihood to repeat the same line verbatim. + */ + frequency_penalty?: number; + grammar?: TextGenerationInputGrammarType; + /** + * Maximum number of tokens to generate. + */ + max_new_tokens?: number; + /** + * The parameter for repetition penalty. 1.0 means no penalty. + * See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. + */ + repetition_penalty?: number; + /** + * Whether to prepend the prompt to the generated text + */ + return_full_text?: boolean; + /** + * Random sampling seed. + */ + seed?: number; + /** + * Stop generating tokens if a member of `stop` is generated. + */ + stop?: string[]; + /** + * The value used to module the logits distribution. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-n-filtering. + */ + top_n_tokens?: number; + /** + * Top-p value for nucleus sampling. + */ + top_p?: number; + /** + * Truncate inputs tokens to the given size. + */ + truncate?: number; + /** + * Typical Decoding mass + * See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) + * for more information. + */ + typical_p?: number; + /** + * Watermarking with [A Watermark for Large Language + * Models](https://arxiv.org/abs/2301.10226). + */ + watermark?: boolean; + [property: string]: unknown; +} +export interface TextGenerationInputGrammarType { + type: Type; + /** + * A string that represents a [JSON Schema](https://json-schema.org/). + * + * JSON Schema is a declarative language that allows to annotate JSON documents + * with types and descriptions. + */ + value: unknown; + [property: string]: unknown; +} +export type Type = "json" | "regex" | "json_schema"; +/** + * Text Generation Output. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface TextGenerationOutput { + details?: TextGenerationOutputDetails; + generated_text: string; + [property: string]: unknown; +} +export interface TextGenerationOutputDetails { + best_of_sequences?: TextGenerationOutputBestOfSequence[]; + finish_reason: TextGenerationOutputFinishReason; + generated_tokens: number; + prefill: TextGenerationOutputPrefillToken[]; + seed?: number; + tokens: TextGenerationOutputToken[]; + top_tokens?: Array; + [property: string]: unknown; +} +export interface TextGenerationOutputBestOfSequence { + finish_reason: TextGenerationOutputFinishReason; + generated_text: string; + generated_tokens: number; + prefill: TextGenerationOutputPrefillToken[]; + seed?: number; + tokens: TextGenerationOutputToken[]; + top_tokens?: Array; + [property: string]: unknown; +} +export type TextGenerationOutputFinishReason = "length" | "eos_token" | "stop_sequence"; +export interface TextGenerationOutputPrefillToken { + id: number; + logprob: number; + text: string; + [property: string]: unknown; +} +export interface TextGenerationOutputToken { + id: number; + logprob: number; + special: boolean; + text: string; + [property: string]: unknown; +} +/** + * Text Generation Stream Output. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface TextGenerationStreamOutput { + details?: TextGenerationStreamOutputStreamDetails; + generated_text?: string; + index: number; + token: TextGenerationStreamOutputToken; + top_tokens?: TextGenerationStreamOutputToken[]; + [property: string]: unknown; +} +export interface TextGenerationStreamOutputStreamDetails { + finish_reason: TextGenerationOutputFinishReason; + generated_tokens: number; + input_length: number; + seed?: number; + [property: string]: unknown; +} +export interface TextGenerationStreamOutputToken { + id: number; + logprob: number; + special: boolean; + text: string; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..44c5f235fb539f2e65e39fae7af67582b0bf0a04 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-generation/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,qCAAqC,CAAC;IACnD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,qCAAqC;IACrD;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,8BAA8B,CAAC;IACzC;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;OAEG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,8BAA8B;IAC9C,IAAI,EAAE,IAAI,CAAC;IACX;;;;;OAKG;IACH,KAAK,EAAE,OAAO,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,IAAI,GAAG,MAAM,GAAG,OAAO,GAAG,aAAa,CAAC;AACpD;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACpC,OAAO,CAAC,EAAE,2BAA2B,CAAC;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C,iBAAiB,CAAC,EAAE,kCAAkC,EAAE,CAAC;IACzD,aAAa,EAAE,gCAAgC,CAAC;IAChD,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,gCAAgC,EAAE,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,yBAAyB,EAAE,CAAC;IACpC,UAAU,CAAC,EAAE,KAAK,CAAC,yBAAyB,EAAE,CAAC,CAAC;IAChD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,kCAAkC;IAClD,aAAa,EAAE,gCAAgC,CAAC;IAChD,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,gCAAgC,EAAE,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,yBAAyB,EAAE,CAAC;IACpC,UAAU,CAAC,EAAE,KAAK,CAAC,yBAAyB,EAAE,CAAC,CAAC;IAChD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,gCAAgC,GAAG,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;AACxF,MAAM,WAAW,gCAAgC;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,yBAAyB;IACzC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,0BAA0B;IAC1C,OAAO,CAAC,EAAE,uCAAuC,CAAC;IAClD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,+BAA+B,CAAC;IACvC,UAAU,CAAC,EAAE,+BAA+B,EAAE,CAAC;IAC/C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,uCAAuC;IACvD,aAAa,EAAE,gCAAgC,CAAC;IAChD,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-generation/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-ranking/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-ranking/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-ranking/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-ranking/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-ranking/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..4e751a3bbd1ecc3171184413375e276af5c186d3 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-ranking/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-ranking/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAsFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-ranking/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-ranking/data.js new file mode 100644 index 0000000000000000000000000000000000000000..15d031afcb84419a02e3b591704c238fe9d7aa2d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-ranking/data.js @@ -0,0 +1,86 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "Bing queries with relevant passages from various web sources.", + id: "microsoft/ms_marco", + }, + ], + demo: { + inputs: [ + { + label: "Source sentence", + content: "Machine learning is so easy.", + type: "text", + }, + { + label: "Sentences to compare to", + content: "Deep learning is so straightforward.", + type: "text", + }, + { + label: "", + content: "This is so difficult, like rocket science.", + type: "text", + }, + { + label: "", + content: "I can't believe how much I struggled with this.", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Deep learning is so straightforward.", + score: 2.2006407, + }, + { + label: "This is so difficult, like rocket science.", + score: -6.2634873, + }, + { + label: "I can't believe how much I struggled with this.", + score: -10.251488, + }, + ], + }, + ], + }, + metrics: [ + { + description: "Discounted Cumulative Gain (DCG) measures the gain, or usefulness, of search results discounted by their position. The normalization is done by dividing the DCG by the ideal DCG, which is the DCG of the perfect ranking.", + id: "Normalized Discounted Cumulative Gain", + }, + { + description: "Reciprocal Rank is a measure used to rank the relevancy of documents given a set of documents. Reciprocal Rank is the reciprocal of the rank of the document retrieved, meaning, if the rank is 3, the Reciprocal Rank is 0.33. If the rank is 1, the Reciprocal Rank is 1", + id: "Mean Reciprocal Rank", + }, + { + description: "Mean Average Precision (mAP) is the overall average of the Average Precision (AP) values, where AP is the Area Under the PR Curve (AUC-PR)", + id: "Mean Average Precision", + }, + ], + models: [ + { + description: "An extremely efficient text ranking model trained on a web search dataset.", + id: "cross-encoder/ms-marco-MiniLM-L6-v2", + }, + { + description: "A strong multilingual text reranker model.", + id: "Alibaba-NLP/gte-multilingual-reranker-base", + }, + { + description: "An efficient text ranking model that punches above its weight.", + id: "Alibaba-NLP/gte-reranker-modernbert-base", + }, + ], + spaces: [], + summary: "Text Ranking is the task of ranking a set of texts based on their relevance to a query. Text ranking models are trained on large datasets of queries and relevant documents to learn how to rank documents based on their relevance to the query. This task is particularly useful for search engines and information retrieval systems.", + widgetModels: ["cross-encoder/ms-marco-MiniLM-L6-v2"], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-3d/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-3d/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-3d/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-3d/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-3d/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a56f91f9a5ea203fe8a085e3c5d555f5306b119e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-3d/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-3d/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAmDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-3d/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-3d/data.js new file mode 100644 index 0000000000000000000000000000000000000000..9ea2c46e6616a81b1fdd8bfd2e4a027ac71b5b3f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-3d/data.js @@ -0,0 +1,55 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A large dataset of over 10 million 3D objects.", + id: "allenai/objaverse-xl", + }, + { + description: "Descriptive captions for 3D objects in Objaverse.", + id: "tiange/Cap3D", + }, + ], + demo: { + inputs: [ + { + label: "Prompt", + content: "a cat statue", + type: "text", + }, + ], + outputs: [ + { + label: "Result", + content: "text-to-3d-3d-output-filename.glb", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "Text-to-3D mesh model by OpenAI", + id: "openai/shap-e", + }, + { + description: "Generative 3D gaussian splatting model.", + id: "ashawkey/LGM", + }, + ], + spaces: [ + { + description: "Text-to-3D demo with mesh outputs.", + id: "hysts/Shap-E", + }, + { + description: "Text/image-to-3D demo with splat outputs.", + id: "ashawkey/LGM", + }, + ], + summary: "Text-to-3D models take in text input and produce 3D output.", + widgetModels: [], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-audio/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-audio/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..811850173233fed08a447e510bede7a576e09822 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-audio/inference.d.ts @@ -0,0 +1,134 @@ +/** + * Outputs of inference for the Text To Audio task + */ +export interface TextToAudioOutput { + /** + * The generated audio waveform. + */ + audio: Blob; + /** + * The sampling rate of the generated audio waveform. + */ + sampling_rate: number; + [property: string]: unknown; +} +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text To Audio inference + */ +export interface TextToAudioInput { + /** + * The input text data + */ + inputs: string; + /** + * Additional inference parameters for Text To Audio + */ + parameters?: TextToAudioParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text To Audio + */ +export interface TextToAudioParameters { + /** + * Parametrization of the text generation process + */ + generation_parameters?: GenerationParameters; + [property: string]: unknown; +} +/** + * Parametrization of the text generation process + */ +export interface GenerationParameters { + /** + * Whether to use sampling instead of greedy decoding when generating new tokens. + */ + do_sample?: boolean; + /** + * Controls the stopping condition for beam-based methods. + */ + early_stopping?: EarlyStoppingUnion; + /** + * If set to float strictly between 0 and 1, only tokens with a conditional probability + * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + * Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + */ + epsilon_cutoff?: number; + /** + * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + * float strictly between 0 and 1, a token is only considered if it is greater than either + * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + * for more details. + */ + eta_cutoff?: number; + /** + * The maximum length (in tokens) of the generated text, including the input. + */ + max_length?: number; + /** + * The maximum number of tokens to generate. Takes precedence over max_length. + */ + max_new_tokens?: number; + /** + * The minimum length (in tokens) of the generated text, including the input. + */ + min_length?: number; + /** + * The minimum number of tokens to generate. Takes precedence over min_length. + */ + min_new_tokens?: number; + /** + * Number of groups to divide num_beams into in order to ensure diversity among different + * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + */ + num_beam_groups?: number; + /** + * Number of beams to use for beam search. + */ + num_beams?: number; + /** + * The value balances the model confidence and the degeneration penalty in contrastive + * search decoding. + */ + penalty_alpha?: number; + /** + * The value used to modulate the next token probabilities. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * If set to float < 1, only the smallest set of most probable tokens with probabilities + * that add up to top_p or higher are kept for generation. + */ + top_p?: number; + /** + * Local typicality measures how similar the conditional probability of predicting a target + * token next is to the expected conditional probability of predicting a random token next, + * given the partial text already generated. If set to float < 1, the smallest set of the + * most locally typical tokens with probabilities that add up to typical_p or higher are + * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + */ + typical_p?: number; + /** + * Whether the model should use the past last key/values attentions to speed up decoding + */ + use_cache?: boolean; + [property: string]: unknown; +} +/** + * Controls the stopping condition for beam-based methods. + */ +export type EarlyStoppingUnion = boolean | "never"; +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-audio/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-audio/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..eec3725373942525186fac3145889c21d7eab2d7 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-audio/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-audio/inference.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,KAAK,EAAE,IAAI,CAAC;IACZ;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,qBAAqB,CAAC,EAAE,oBAAoB,CAAC;IAC7C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,kBAAkB,CAAC;IACpC;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,OAAO,GAAG,OAAO,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-audio/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-audio/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-audio/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..9f3b6ad845035578c7d5be1c8af56971c67a1f3f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-image/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAmGf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/data.js new file mode 100644 index 0000000000000000000000000000000000000000..6951db8903fa9e515362d64453764bc6e704c958 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/data.js @@ -0,0 +1,99 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "RedCaps is a large-scale dataset of 12M image-text pairs collected from Reddit.", + id: "red_caps", + }, + { + description: "Conceptual Captions is a dataset consisting of ~3.3M images annotated with captions.", + id: "conceptual_captions", + }, + { + description: "12M image-caption pairs.", + id: "Spawning/PD12M", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "A city above clouds, pastel colors, Victorian style", + type: "text", + }, + ], + outputs: [ + { + filename: "image.jpeg", + type: "img", + }, + ], + }, + metrics: [ + { + description: "The Inception Score (IS) measure assesses diversity and meaningfulness. It uses a generated image sample to predict its label. A higher score signifies more diverse and meaningful images.", + id: "IS", + }, + { + description: "The Fréchet Inception Distance (FID) calculates the distance between distributions between synthetic and real samples. A lower FID score indicates better similarity between the distributions of real and generated images.", + id: "FID", + }, + { + description: "R-precision assesses how the generated image aligns with the provided text description. It uses the generated images as queries to retrieve relevant text descriptions. The top 'r' relevant descriptions are selected and used to calculate R-precision as r/R, where 'R' is the number of ground truth descriptions associated with the generated images. A higher R-precision value indicates a better model.", + id: "R-Precision", + }, + ], + models: [ + { + description: "One of the most powerful image generation models that can generate realistic outputs.", + id: "black-forest-labs/FLUX.1-Krea-dev", + }, + { + description: "A powerful image generation model.", + id: "Qwen/Qwen-Image", + }, + { + description: "Powerful and fast image generation model.", + id: "ByteDance/SDXL-Lightning", + }, + { + description: "A powerful text-to-image model.", + id: "ByteDance/Hyper-SD", + }, + ], + spaces: [ + { + description: "A powerful text-to-image application.", + id: "stabilityai/stable-diffusion-3-medium", + }, + { + description: "A text-to-image application to generate comics.", + id: "jbilcke-hf/ai-comic-factory", + }, + { + description: "An application to match multiple custom image generation models.", + id: "multimodalart/flux-lora-lab", + }, + { + description: "A powerful yet very fast image generation application.", + id: "latent-consistency/lcm-lora-for-sdxl", + }, + { + description: "A gallery to explore various text-to-image models.", + id: "multimodalart/LoraTheExplorer", + }, + { + description: "An application for `text-to-image`, `image-to-image` and image inpainting.", + id: "ArtGAN/Stable-Diffusion-ControlNet-WebUI", + }, + { + description: "An application to generate realistic images given photos of a person and a prompt.", + id: "InstantX/InstantID", + }, + ], + summary: "Text-to-image is the task of generating images from input text. These pipelines can also be used to modify and edit images based on text prompts.", + widgetModels: ["black-forest-labs/FLUX.1-dev"], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..af711fe85ee463c28cdd3f514466a6344a3a2cdb --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/inference.d.ts @@ -0,0 +1,66 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text To Image inference + */ +export interface TextToImageInput { + /** + * The input text data (sometimes called "prompt") + */ + inputs: string; + /** + * Additional inference parameters for Text To Image + */ + parameters?: TextToImageParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text To Image + */ +export interface TextToImageParameters { + /** + * A higher guidance scale value encourages the model to generate images closely linked to + * the text prompt, but values too high may cause saturation and other artifacts. + */ + guidance_scale?: number; + /** + * The height in pixels of the output image + */ + height?: number; + /** + * One prompt to guide what NOT to include in image generation. + */ + negative_prompt?: string; + /** + * The number of denoising steps. More denoising steps usually lead to a higher quality + * image at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * Override the scheduler with a compatible one. + */ + scheduler?: string; + /** + * Seed for the random number generator. + */ + seed?: number; + /** + * The width in pixels of the output image + */ + width?: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Text To Image task + */ +export interface TextToImageOutput { + /** + * The generated image returned as raw bytes in the payload. + */ + image: unknown; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..03467437eb89c69ccf8ef265aa92d9e6eae427e5 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-image/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-image/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..ed4c56895bf73bf3252bfe24ad8c688f910332fc --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-speech/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAiFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/data.js new file mode 100644 index 0000000000000000000000000000000000000000..1683400355cc14648439e38a8e4ace2860b62bb1 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/data.js @@ -0,0 +1,84 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + canonicalId: "text-to-audio", + datasets: [ + { + description: "10K hours of multi-speaker English dataset.", + id: "parler-tts/mls_eng_10k", + }, + { + description: "Multi-speaker English dataset.", + id: "mythicinfinity/libritts_r", + }, + { + description: "Multi-lingual dataset.", + id: "facebook/multilingual_librispeech", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "I love audio models on the Hub!", + type: "text", + }, + ], + outputs: [ + { + filename: "audio.wav", + type: "audio", + }, + ], + }, + metrics: [ + { + description: "The Mel Cepstral Distortion (MCD) metric is used to calculate the quality of generated speech.", + id: "mel cepstral distortion", + }, + ], + models: [ + { + description: "Small yet powerful TTS model.", + id: "KittenML/kitten-tts-nano-0.1", + }, + { + description: "Bleeding edge TTS model.", + id: "ResembleAI/chatterbox", + }, + { + description: "A massively multi-lingual TTS model.", + id: "fishaudio/fish-speech-1.5", + }, + { + description: "A text-to-dialogue model.", + id: "nari-labs/Dia-1.6B-0626", + }, + ], + spaces: [ + { + description: "An application for generate high quality speech in different languages.", + id: "hexgrad/Kokoro-TTS", + }, + { + description: "A multilingual text-to-speech application.", + id: "fishaudio/fish-speech-1", + }, + { + description: "Performant TTS application.", + id: "ResembleAI/Chatterbox", + }, + { + description: "An application to compare different TTS models.", + id: "TTS-AGI/TTS-Arena-V2", + }, + { + description: "An application that generates podcast episodes.", + id: "ngxson/kokoro-podcast-generator", + }, + ], + summary: "Text-to-Speech (TTS) is the task of generating natural sounding speech given text input. TTS models can be extended to have a single model that generates speech for multiple speakers and multiple languages.", + widgetModels: ["suno/bark"], + youtubeId: "NW62DpzJ274", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..6cc131914ae1bf76593f4a2958c5cda23baff842 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/inference.d.ts @@ -0,0 +1,134 @@ +/** + * Outputs of inference for the Text To Speech task + */ +export interface TextToSpeechOutput { + /** + * The generated audio + */ + audio: Blob; + /** + * The sampling rate of the generated audio waveform. + */ + sampling_rate?: number; + [property: string]: unknown; +} +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text To Speech inference + */ +export interface TextToSpeechInput { + /** + * The input text data + */ + inputs: string; + /** + * Additional inference parameters for Text To Speech + */ + parameters?: TextToSpeechParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text To Speech + */ +export interface TextToSpeechParameters { + /** + * Parametrization of the text generation process + */ + generation_parameters?: GenerationParameters; + [property: string]: unknown; +} +/** + * Parametrization of the text generation process + */ +export interface GenerationParameters { + /** + * Whether to use sampling instead of greedy decoding when generating new tokens. + */ + do_sample?: boolean; + /** + * Controls the stopping condition for beam-based methods. + */ + early_stopping?: EarlyStoppingUnion; + /** + * If set to float strictly between 0 and 1, only tokens with a conditional probability + * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + * Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + */ + epsilon_cutoff?: number; + /** + * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + * float strictly between 0 and 1, a token is only considered if it is greater than either + * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + * for more details. + */ + eta_cutoff?: number; + /** + * The maximum length (in tokens) of the generated text, including the input. + */ + max_length?: number; + /** + * The maximum number of tokens to generate. Takes precedence over max_length. + */ + max_new_tokens?: number; + /** + * The minimum length (in tokens) of the generated text, including the input. + */ + min_length?: number; + /** + * The minimum number of tokens to generate. Takes precedence over min_length. + */ + min_new_tokens?: number; + /** + * Number of groups to divide num_beams into in order to ensure diversity among different + * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + */ + num_beam_groups?: number; + /** + * Number of beams to use for beam search. + */ + num_beams?: number; + /** + * The value balances the model confidence and the degeneration penalty in contrastive + * search decoding. + */ + penalty_alpha?: number; + /** + * The value used to modulate the next token probabilities. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * If set to float < 1, only the smallest set of most probable tokens with probabilities + * that add up to top_p or higher are kept for generation. + */ + top_p?: number; + /** + * Local typicality measures how similar the conditional probability of predicting a target + * token next is to the expected conditional probability of predicting a random token next, + * given the partial text already generated. If set to float < 1, the smallest set of the + * most locally typical tokens with probabilities that add up to typical_p or higher are + * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + */ + typical_p?: number; + /** + * Whether the model should use the past last key/values attentions to speed up decoding + */ + use_cache?: boolean; + [property: string]: unknown; +} +/** + * Controls the stopping condition for beam-based methods. + */ +export type EarlyStoppingUnion = boolean | "never"; +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..47fb15af47a221be88e3d57de5fdb4b950d5af25 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-speech/inference.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,KAAK,EAAE,IAAI,CAAC;IACZ;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,qBAAqB,CAAC,EAAE,oBAAoB,CAAC;IAC7C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,kBAAkB,CAAC;IACpC;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,OAAO,GAAG,OAAO,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-speech/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..15f63d971713a969dbf522ff03184c46e982cf0e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-video/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAqGf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/data.js new file mode 100644 index 0000000000000000000000000000000000000000..b15983167ef23a79decf6f6a296f795f39eaea9c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/data.js @@ -0,0 +1,99 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "Microsoft Research Video to Text is a large-scale dataset for open domain video captioning", + id: "iejMac/CLIP-MSR-VTT", + }, + { + description: "UCF101 Human Actions dataset consists of 13,320 video clips from YouTube, with 101 classes.", + id: "quchenyuan/UCF101-ZIP", + }, + { + description: "A high-quality dataset for human action recognition in YouTube videos.", + id: "nateraw/kinetics", + }, + { + description: "A dataset of video clips of humans performing pre-defined basic actions with everyday objects.", + id: "HuggingFaceM4/something_something_v2", + }, + { + description: "This dataset consists of text-video pairs and contains noisy samples with irrelevant video descriptions", + id: "HuggingFaceM4/webvid", + }, + { + description: "A dataset of short Flickr videos for the temporal localization of events with descriptions.", + id: "iejMac/CLIP-DiDeMo", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "Darth Vader is surfing on the waves.", + type: "text", + }, + ], + outputs: [ + { + filename: "text-to-video-output.gif", + type: "img", + }, + ], + }, + metrics: [ + { + description: "Inception Score uses an image classification model that predicts class labels and evaluates how distinct and diverse the images are. A higher score indicates better video generation.", + id: "is", + }, + { + description: "Frechet Inception Distance uses an image classification model to obtain image embeddings. The metric compares mean and standard deviation of the embeddings of real and generated images. A smaller score indicates better video generation.", + id: "fid", + }, + { + description: "Frechet Video Distance uses a model that captures coherence for changes in frames and the quality of each frame. A smaller score indicates better video generation.", + id: "fvd", + }, + { + description: "CLIPSIM measures similarity between video frames and text using an image-text similarity model. A higher score indicates better video generation.", + id: "clipsim", + }, + ], + models: [ + { + description: "A strong model for consistent video generation.", + id: "tencent/HunyuanVideo", + }, + { + description: "A text-to-video model with high fidelity motion and strong prompt adherence.", + id: "Lightricks/LTX-Video", + }, + { + description: "A text-to-video model focusing on physics-aware applications like robotics.", + id: "nvidia/Cosmos-1.0-Diffusion-7B-Text2World", + }, + { + description: "Very fast model for video generation.", + id: "Lightricks/LTX-Video-0.9.8-13B-distilled", + }, + ], + spaces: [ + { + description: "An application that generates video from text.", + id: "VideoCrafter/VideoCrafter", + }, + { + description: "Consistent video generation application.", + id: "Wan-AI/Wan2.1", + }, + { + description: "A cutting edge video generation application.", + id: "Pyramid-Flow/pyramid-flow", + }, + ], + summary: "Text-to-video models can be used in any application that requires generating consistent sequence of images from text. ", + widgetModels: ["Wan-AI/Wan2.2-TI2V-5B"], + youtubeId: undefined, +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4c3791350c974ef6ec40fc61866deb614344b143 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/inference.d.ts @@ -0,0 +1,58 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text To Video inference + */ +export interface TextToVideoInput { + /** + * The input text data (sometimes called "prompt") + */ + inputs: string; + /** + * Additional inference parameters for Text To Video + */ + parameters?: TextToVideoParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text To Video + */ +export interface TextToVideoParameters { + /** + * A higher guidance scale value encourages the model to generate videos closely linked to + * the text prompt, but values too high may cause saturation and other artifacts. + */ + guidance_scale?: number; + /** + * One or several prompt to guide what NOT to include in video generation. + */ + negative_prompt?: string[]; + /** + * The num_frames parameter determines how many video frames are generated. + */ + num_frames?: number; + /** + * The number of denoising steps. More denoising steps usually lead to a higher quality + * video at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * Seed for the random number generator. + */ + seed?: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Text To Video task + */ +export interface TextToVideoOutput { + /** + * The generated video returned as raw bytes in the payload. + */ + video: unknown; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..420f50f10ede66c1dfc384c98c82e226bb0a2f14 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-video/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/text-to-video/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..be4523e103c5a787df5558ad8ae91a69cda08bb5 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/token-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAuFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..b7d37491e17b757d97e6aedef4241e42ea7d3614 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/data.js @@ -0,0 +1,87 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A widely used dataset useful to benchmark named entity recognition models.", + id: "eriktks/conll2003", + }, + { + description: "A multilingual dataset of Wikipedia articles annotated for named entity recognition in over 150 different languages.", + id: "unimelb-nlp/wikiann", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "My name is Omar and I live in Zürich.", + type: "text", + }, + ], + outputs: [ + { + text: "My name is Omar and I live in Zürich.", + tokens: [ + { + type: "PERSON", + start: 11, + end: 15, + }, + { + type: "GPE", + start: 30, + end: 36, + }, + ], + type: "text-with-tokens", + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + description: "A robust performance model to identify people, locations, organizations and names of miscellaneous entities.", + id: "dslim/bert-base-NER", + }, + { + description: "A strong model to identify people, locations, organizations and names in multiple languages.", + id: "FacebookAI/xlm-roberta-large-finetuned-conll03-english", + }, + { + description: "A token classification model specialized on medical entity recognition.", + id: "blaze999/Medical-NER", + }, + { + description: "Flair models are typically the state of the art in named entity recognition tasks.", + id: "flair/ner-english", + }, + ], + spaces: [ + { + description: "An application that can recognizes entities, extracts noun chunks and recognizes various linguistic features of each token.", + id: "spacy/gradio_pipeline_visualizer", + }, + ], + summary: "Token classification is a natural language understanding task in which a label is assigned to some tokens in a text. Some popular token classification subtasks are Named Entity Recognition (NER) and Part-of-Speech (PoS) tagging. NER models could be trained to identify specific entities in a text, such as dates, individuals and places; and PoS tagging would identify, for example, which words in a text are verbs, nouns, and punctuation marks.", + widgetModels: ["FacebookAI/xlm-roberta-large-finetuned-conll03-english"], + youtubeId: "wVHdVlPScxA", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..eac5ea3f3006f8f8909e67692888a5290d2c766f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/inference.d.ts @@ -0,0 +1,84 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Token Classification inference + */ +export interface TokenClassificationInput { + /** + * The input text data + */ + inputs: string; + /** + * Additional inference parameters for Token Classification + */ + parameters?: TokenClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Token Classification + */ +export interface TokenClassificationParameters { + /** + * The strategy used to fuse tokens based on model predictions + */ + aggregation_strategy?: TokenClassificationAggregationStrategy; + /** + * A list of labels to ignore + */ + ignore_labels?: string[]; + /** + * The number of overlapping tokens between chunks when splitting the input text. + */ + stride?: number; + [property: string]: unknown; +} +/** + * Do not aggregate tokens + * + * Group consecutive tokens with the same label in a single entity. + * + * Similar to "simple", also preserves word integrity (use the label predicted for the first + * token in a word). + * + * Similar to "simple", also preserves word integrity (uses the label with the highest + * score, averaged across the word's tokens). + * + * Similar to "simple", also preserves word integrity (uses the label with the highest score + * across the word's tokens). + */ +export type TokenClassificationAggregationStrategy = "none" | "simple" | "first" | "average" | "max"; +export type TokenClassificationOutput = TokenClassificationOutputElement[]; +/** + * Outputs of inference for the Token Classification task + */ +export interface TokenClassificationOutputElement { + /** + * The character position in the input where this group ends. + */ + end: number; + /** + * The predicted label for a single token + */ + entity?: string; + /** + * The predicted label for a group of one or more tokens + */ + entity_group?: string; + /** + * The associated score / probability + */ + score: number; + /** + * The character position in the input where this group begins. + */ + start: number; + /** + * The corresponding text + */ + word: string; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c8afeae4c523c21a681a7a3453d1df2ec382d7a3 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/token-classification/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACxC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,6BAA6B,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC7C;;OAEG;IACH,oBAAoB,CAAC,EAAE,sCAAsC,CAAC;IAC9D;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,sCAAsC,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,GAAG,KAAK,CAAC;AACrG,MAAM,MAAM,yBAAyB,GAAG,gCAAgC,EAAE,CAAC;AAC3E;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/token-classification/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..2db6b74c8d7f7cb9ecc51018f83b40c1749d641f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/translation/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAiEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/data.js new file mode 100644 index 0000000000000000000000000000000000000000..d5a05554ba0ea7b73627e0eddbf3515718fe329c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/data.js @@ -0,0 +1,65 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + canonicalId: "text-generation", + datasets: [ + { + description: "A dataset of copyright-free books translated into 16 different languages.", + id: "Helsinki-NLP/opus_books", + }, + { + description: "An example of translation between programming languages. This dataset consists of functions in Java and C#.", + id: "google/code_x_glue_cc_code_to_code_trans", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "My name is Omar and I live in Zürich.", + type: "text", + }, + ], + outputs: [ + { + label: "Output", + content: "Mein Name ist Omar und ich wohne in Zürich.", + type: "text", + }, + ], + }, + metrics: [ + { + description: "BLEU score is calculated by counting the number of shared single or subsequent tokens between the generated sequence and the reference. Subsequent n tokens are called “n-grams”. Unigram refers to a single token while bi-gram refers to token pairs and n-grams refer to n subsequent tokens. The score ranges from 0 to 1, where 1 means the translation perfectly matched and 0 did not match at all", + id: "bleu", + }, + { + description: "", + id: "sacrebleu", + }, + ], + models: [ + { + description: "Very powerful model that can translate many languages between each other, especially low-resource languages.", + id: "facebook/nllb-200-1.3B", + }, + { + description: "A general-purpose Transformer that can be used to translate from English to German, French, or Romanian.", + id: "google-t5/t5-base", + }, + ], + spaces: [ + { + description: "An application that can translate between 100 languages.", + id: "Iker/Translate-100-languages", + }, + { + description: "An application that can translate between many languages.", + id: "Geonmo/nllb-translation-demo", + }, + ], + summary: "Translation is the task of converting text from one language to another.", + widgetModels: ["facebook/mbart-large-50-many-to-many-mmt"], + youtubeId: "1JvfrvZgi6c", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4df641a2cb758ae4ef24e0313e9dec7f9a4d8059 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/inference.d.ts @@ -0,0 +1,64 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Translation inference + */ +export interface TranslationInput { + /** + * The text to translate. + */ + inputs: string; + /** + * Additional inference parameters for Translation + */ + parameters?: TranslationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Translation + */ +export interface TranslationParameters { + /** + * Whether to clean up the potential extra spaces in the text output. + */ + clean_up_tokenization_spaces?: boolean; + /** + * Additional parametrization of the text generation algorithm. + */ + generate_parameters?: { + [key: string]: unknown; + }; + /** + * The source language of the text. Required for models that can translate from multiple + * languages. + */ + src_lang?: string; + /** + * Target language to translate to. Required for models that can translate to multiple + * languages. + */ + tgt_lang?: string; + /** + * The truncation strategy to use. + */ + truncation?: TranslationTruncationStrategy; + [property: string]: unknown; +} +/** + * The truncation strategy to use. + */ +export type TranslationTruncationStrategy = "do_not_truncate" | "longest_first" | "only_first" | "only_second"; +/** + * Outputs of inference for the Translation task + */ +export interface TranslationOutput { + /** + * The translated text. + */ + translation_text: string; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..03d7567511144732e069f733012ac65822b2e539 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/translation/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC;;OAEG;IACH,mBAAmB,CAAC,EAAE;QACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,UAAU,CAAC,EAAE,6BAA6B,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG,iBAAiB,GAAG,eAAe,GAAG,YAAY,GAAG,aAAa,CAAC;AAC/G;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/translation/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/unconditional-image-generation/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/unconditional-image-generation/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/unconditional-image-generation/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/unconditional-image-generation/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/unconditional-image-generation/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7ae1ca65807d46bdc5161ad0d750274be15306cb --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/unconditional-image-generation/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/unconditional-image-generation/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAmEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/unconditional-image-generation/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/unconditional-image-generation/data.js new file mode 100644 index 0000000000000000000000000000000000000000..b361750c37c6cc44afa88bbceec3e55aeaefc74b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/unconditional-image-generation/data.js @@ -0,0 +1,65 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "The CIFAR-100 dataset consists of 60000 32x32 colour images in 100 classes, with 600 images per class.", + id: "cifar100", + }, + { + description: "Multiple images of celebrities, used for facial expression translation.", + id: "CelebA", + }, + ], + demo: { + inputs: [ + { + label: "Seed", + content: "42", + type: "text", + }, + { + label: "Number of images to generate:", + content: "4", + type: "text", + }, + ], + outputs: [ + { + filename: "unconditional-image-generation-output.jpeg", + type: "img", + }, + ], + }, + metrics: [ + { + description: "The inception score (IS) evaluates the quality of generated images. It measures the diversity of the generated images (the model predictions are evenly distributed across all possible labels) and their 'distinction' or 'sharpness' (the model confidently predicts a single label for each image).", + id: "Inception score (IS)", + }, + { + description: "The Fréchet Inception Distance (FID) evaluates the quality of images created by a generative model by calculating the distance between feature vectors for real and generated images.", + id: "Frećhet Inception Distance (FID)", + }, + ], + models: [ + { + description: "High-quality image generation model trained on the CIFAR-10 dataset. It synthesizes images of the ten classes presented in the dataset using diffusion probabilistic models, a class of latent variable models inspired by considerations from nonequilibrium thermodynamics.", + id: "google/ddpm-cifar10-32", + }, + { + description: "High-quality image generation model trained on the 256x256 CelebA-HQ dataset. It synthesizes images of faces using diffusion probabilistic models, a class of latent variable models inspired by considerations from nonequilibrium thermodynamics.", + id: "google/ddpm-celebahq-256", + }, + ], + spaces: [ + { + description: "An application that can generate realistic faces.", + id: "CompVis/celeba-latent-diffusion", + }, + ], + summary: "Unconditional image generation is the task of generating images with no condition in any context (like a prompt text or another image). Once trained, the model will create images that resemble its training data distribution.", + widgetModels: [""], + // TODO: Add related video + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7ddd46629dd3f6fdc2e7bebd42b3cbe2cb701d6e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/video-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA+Ef,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..9a83d475f76f64c7da35a8000180a1c24d639da5 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/data.js @@ -0,0 +1,82 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + // TODO write proper description + description: "Benchmark dataset used for video classification with videos that belong to 400 classes.", + id: "kinetics400", + }, + ], + demo: { + inputs: [ + { + filename: "video-classification-input.gif", + type: "img", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Playing Guitar", + score: 0.514, + }, + { + label: "Playing Tennis", + score: 0.193, + }, + { + label: "Cooking", + score: 0.068, + }, + ], + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + // TO DO: write description + description: "Strong Video Classification model trained on the Kinetics 400 dataset.", + id: "google/vivit-b-16x2-kinetics400", + }, + { + // TO DO: write description + description: "Strong Video Classification model trained on the Kinetics 400 dataset.", + id: "microsoft/xclip-base-patch32", + }, + ], + spaces: [ + { + description: "An application that classifies video at different timestamps.", + id: "nateraw/lavila", + }, + { + description: "An application that classifies video.", + id: "fcakyon/video-classification", + }, + ], + summary: "Video classification is the task of assigning a label or class to an entire video. Videos are expected to have only one class for each video. Video classification models take a video as input and return a prediction about which class the video belongs to.", + widgetModels: [], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..97499d022d9a2406ecf3ae55c64b7855f16fcb62 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/inference.d.ts @@ -0,0 +1,61 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Video Classification inference + */ +export interface VideoClassificationInput { + /** + * The input video data + */ + inputs: unknown; + /** + * Additional inference parameters for Video Classification + */ + parameters?: VideoClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Video Classification + */ +export interface VideoClassificationParameters { + /** + * The sampling rate used to select frames from the video. + */ + frame_sampling_rate?: number; + /** + * The function to apply to the model outputs in order to retrieve the scores. + */ + function_to_apply?: ClassificationOutputTransform; + /** + * The number of sampled frames to consider for classification. + */ + num_frames?: number; + /** + * When specified, limits the output to the top K most probable classes. + */ + top_k?: number; + [property: string]: unknown; +} +/** + * The function to apply to the model outputs in order to retrieve the scores. + */ +export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none"; +export type VideoClassificationOutput = VideoClassificationOutputElement[]; +/** + * Outputs of inference for the Video Classification task + */ +export interface VideoClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..becc24194891238147e7e67309e2d194a029bce1 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/video-classification/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACxC;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC;IAChB;;OAEG;IACH,UAAU,CAAC,EAAE,6BAA6B,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC7C;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;OAEG;IACH,iBAAiB,CAAC,EAAE,6BAA6B,CAAC;IAClD;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAC3E,MAAM,MAAM,yBAAyB,GAAG,gCAAgC,EAAE,CAAC;AAC3E;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-classification/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-text-to-text/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-text-to-text/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-text-to-text/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-text-to-text/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-text-to-text/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..9fd072183cf99733f8fa8c8d9917610b5984eca7 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-text-to-text/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/video-text-to-text/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAqEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-text-to-text/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-text-to-text/data.js new file mode 100644 index 0000000000000000000000000000000000000000..d6c54bf7634edd019e6054baa5934bc3366dc44c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-text-to-text/data.js @@ -0,0 +1,71 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "Multiple-choice questions and answers about videos.", + id: "lmms-lab/Video-MME", + }, + { + description: "A dataset of instructions and question-answer pairs about videos.", + id: "lmms-lab/VideoChatGPT", + }, + { + description: "Large video understanding dataset.", + id: "HuggingFaceFV/finevideo", + }, + ], + demo: { + inputs: [ + { + filename: "video-text-to-text-input.gif", + type: "img", + }, + { + label: "Text Prompt", + content: "What is happening in this video?", + type: "text", + }, + ], + outputs: [ + { + label: "Answer", + content: "The video shows a series of images showing a fountain with water jets and a variety of colorful flowers and butterflies in the background.", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "A robust video-text-to-text model.", + id: "Vision-CAIR/LongVU_Qwen2_7B", + }, + { + description: "Strong video-text-to-text model with reasoning capabilities.", + id: "GoodiesHere/Apollo-LMMs-Apollo-7B-t32", + }, + { + description: "Strong video-text-to-text model.", + id: "HuggingFaceTB/SmolVLM2-2.2B-Instruct", + }, + ], + spaces: [ + { + description: "An application to chat with a video-text-to-text model.", + id: "llava-hf/video-llava", + }, + { + description: "A leaderboard for various video-text-to-text models.", + id: "opencompass/openvlm_video_leaderboard", + }, + { + description: "An application to generate highlights from a video.", + id: "HuggingFaceTB/SmolVLM2-HighlightGenerator", + }, + ], + summary: "Video-text-to-text models take in a video and a text prompt and output text. These models are also called video-language models.", + widgetModels: [""], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-to-video/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-to-video/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-to-video/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-to-video/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-to-video/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..69a3a40fc74e02b3eae44a8bc24cd05d4de41b61 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-to-video/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/video-to-video/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA8Df,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-to-video/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-to-video/data.js new file mode 100644 index 0000000000000000000000000000000000000000..960ea9b641cc1249851a82b20ae96b9e310d1603 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/video-to-video/data.js @@ -0,0 +1,65 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "Dataset with detailed annotations for training and benchmarking video instance editing.", + id: "suimu/VIRESET", + }, + { + description: "Dataset to evaluate models on long video generation and understanding.", + id: "zhangsh2001/LongV-EVAL", + }, + { + description: "Collection of 104 demo videos from the SeedVR/SeedVR2 series showcasing model outputs.", + id: "Iceclear/SeedVR_VideoDemos", + }, + ], + demo: { + inputs: [ + { + filename: "input.gif", + type: "img", + }, + ], + outputs: [ + { + filename: "output.gif", + type: "img", + }, + ], + }, + metrics: [], + models: [ + { + description: "Model for editing outfits, character, and scenery in videos.", + id: "decart-ai/Lucy-Edit-Dev", + }, + { + description: "Framework that uses 3D mesh proxies for precise, consistent video editing.", + id: "LeoLau/Shape-for-Motion", + }, + { + description: "Model for generating physics-aware videos from input videos and control conditions.", + id: "nvidia/Cosmos-Transfer2.5-2B", + }, + { + description: "A model to upscale videos at input, designed for seamless use with ComfyUI.", + id: "numz/SeedVR2_comfyUI", + }, + ], + spaces: [ + { + description: "Interactive demo space for Lucy-Edit-Dev video editing.", + id: "decart-ai/lucy-edit-dev", + }, + { + description: "Demo space for SeedVR2-3B showcasing video upscaling and restoration.", + id: "ByteDance-Seed/SeedVR2-3B", + }, + ], + summary: "Video-to-video models take one or more videos as input and generate new videos as output. They can enhance quality, interpolate frames, modify styles, or create new motion dynamics, enabling creative applications, video production, and research.", + widgetModels: [], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-document-retrieval/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-document-retrieval/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-document-retrieval/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-document-retrieval/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-document-retrieval/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..186ad37def7b38501f210bec15304d1c534ebdd4 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-document-retrieval/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/visual-document-retrieval/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAuEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-document-retrieval/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-document-retrieval/data.js new file mode 100644 index 0000000000000000000000000000000000000000..48875b5643fb7efe17a7c1455671c87817e284e9 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-document-retrieval/data.js @@ -0,0 +1,73 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A large dataset used to train visual document retrieval models.", + id: "vidore/colpali_train_set", + }, + ], + demo: { + inputs: [ + { + filename: "input.png", + type: "img", + }, + { + label: "Question", + content: "Is the model in this paper the fastest for inference?", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Page 10", + score: 0.7, + }, + { + label: "Page 11", + score: 0.06, + }, + { + label: "Page 9", + score: 0.003, + }, + ], + }, + ], + }, + isPlaceholder: false, + metrics: [ + { + description: "NDCG@k scores ranked recommendation lists for top-k results. 0 is the worst, 1 is the best.", + id: "Normalized Discounted Cumulative Gain at K", + }, + ], + models: [ + { + description: "Very accurate visual document retrieval model for multilingual queries and documents.", + id: "vidore/colqwen2-v1.0", + }, + { + description: "Very fast and efficient visual document retrieval model that can also take in other modalities like audio.", + id: "Tevatron/OmniEmbed-v0.1", + }, + ], + spaces: [ + { + description: "A leaderboard of visual document retrieval models.", + id: "vidore/vidore-leaderboard", + }, + { + description: "Visual retrieval augmented generation demo based on ColQwen2 model.", + id: "vidore/visual-rag-tool", + }, + ], + summary: "Visual document retrieval is the task of searching for relevant image-based documents, such as PDFs. These models take a text query and multiple documents as input and return the top-most relevant documents and relevancy scores as output.", + widgetModels: [""], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a5bb4ed58442694fd0d8a32ae92012c6083a496c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/visual-question-answering/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA4Ff,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/data.js new file mode 100644 index 0000000000000000000000000000000000000000..17a288284a6c506980bf0659a28c92970e45aae8 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/data.js @@ -0,0 +1,93 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A widely used dataset containing questions (with answers) about images.", + id: "Graphcore/vqa", + }, + { + description: "A dataset to benchmark visual reasoning based on text in images.", + id: "facebook/textvqa", + }, + ], + demo: { + inputs: [ + { + filename: "elephant.jpeg", + type: "img", + }, + { + label: "Question", + content: "What is in this image?", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "elephant", + score: 0.97, + }, + { + label: "elephants", + score: 0.06, + }, + { + label: "animal", + score: 0.003, + }, + ], + }, + ], + }, + isPlaceholder: false, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "Measures how much a predicted answer differs from the ground truth based on the difference in their semantic meaning.", + id: "wu-palmer similarity", + }, + ], + models: [ + { + description: "A visual question answering model trained to convert charts and plots to text.", + id: "google/deplot", + }, + { + description: "A visual question answering model trained for mathematical reasoning and chart derendering from images.", + id: "google/matcha-base", + }, + { + description: "A strong visual question answering that answers questions from book covers.", + id: "google/pix2struct-ocrvqa-large", + }, + ], + spaces: [ + { + description: "An application that compares visual question answering models across different tasks.", + id: "merve/pix2struct", + }, + { + description: "An application that can answer questions based on images.", + id: "nielsr/vilt-vqa", + }, + { + description: "An application that can caption images and answer questions about a given image. ", + id: "Salesforce/BLIP", + }, + { + description: "An application that can caption images and answer questions about a given image. ", + id: "vumichien/Img2Prompt", + }, + ], + summary: "Visual Question Answering is the task of answering open-ended questions based on an image. They output natural language responses to natural language questions.", + widgetModels: ["dandelin/vilt-b32-finetuned-vqa"], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4736a2d592969c569d706924af56da6583ee0891 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/inference.d.ts @@ -0,0 +1,61 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Visual Question Answering inference + */ +export interface VisualQuestionAnsweringInput { + /** + * One (image, question) pair to answer + */ + inputs: VisualQuestionAnsweringInputData; + /** + * Additional inference parameters for Visual Question Answering + */ + parameters?: VisualQuestionAnsweringParameters; + [property: string]: unknown; +} +/** + * One (image, question) pair to answer + */ +export interface VisualQuestionAnsweringInputData { + /** + * The image. + */ + image: unknown; + /** + * The question to answer based on the image. + */ + question: string; + [property: string]: unknown; +} +/** + * Additional inference parameters for Visual Question Answering + */ +export interface VisualQuestionAnsweringParameters { + /** + * The number of answers to return (will be chosen by order of likelihood). Note that we + * return less than topk answers if there are not enough options available within the + * context. + */ + top_k?: number; + [property: string]: unknown; +} +export type VisualQuestionAnsweringOutput = VisualQuestionAnsweringOutputElement[]; +/** + * Outputs of inference for the Visual Question Answering task + */ +export interface VisualQuestionAnsweringOutputElement { + /** + * The answer to the question + */ + answer?: string; + /** + * The associated score / probability + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a0f2508331d6046894c38d3b358c1ff70aef4bb5 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/visual-question-answering/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC5C;;OAEG;IACH,MAAM,EAAE,gCAAgC,CAAC;IACzC;;OAEG;IACH,UAAU,CAAC,EAAE,iCAAiC,CAAC;IAC/C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,iCAAiC;IACjD;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,6BAA6B,GAAG,oCAAoC,EAAE,CAAC;AACnF;;GAEG;AACH,MAAM,WAAW,oCAAoC;IACpD;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/visual-question-answering/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..8ae7284ee6e360d3a874ce894594cd62659f2cf9 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/zero-shot-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAqEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..712f9c8b3a813168a3bff3d8dc68482627156e65 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/data.js @@ -0,0 +1,70 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + description: "A widely used dataset used to benchmark multiple variants of text classification.", + id: "nyu-mll/glue", + }, + { + description: "The Multi-Genre Natural Language Inference (MultiNLI) corpus is a crowd-sourced collection of 433k sentence pairs annotated with textual entailment information.", + id: "nyu-mll/multi_nli", + }, + { + description: "FEVER is a publicly available dataset for fact extraction and verification against textual sources.", + id: "fever/fever", + }, + ], + demo: { + inputs: [ + { + label: "Text Input", + content: "Dune is the best movie ever.", + type: "text", + }, + { + label: "Candidate Labels", + content: "CINEMA, ART, MUSIC", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "CINEMA", + score: 0.9, + }, + { + label: "ART", + score: 0.1, + }, + { + label: "MUSIC", + score: 0.0, + }, + ], + }, + ], + }, + metrics: [], + models: [ + { + description: "Powerful zero-shot text classification model.", + id: "facebook/bart-large-mnli", + }, + { + description: "Cutting-edge zero-shot multilingual text classification model.", + id: "MoritzLaurer/ModernBERT-large-zeroshot-v2.0", + }, + { + description: "Zero-shot text classification model that can be used for topic and sentiment classification.", + id: "knowledgator/gliclass-modern-base-v2.0-init", + }, + ], + spaces: [], + summary: "Zero-shot text classification is a task in natural language processing where a model is trained on a set of labeled examples but is then able to classify new examples from previously unseen classes.", + widgetModels: ["facebook/bart-large-mnli"], +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2d9281733c9a213395c423d02a1e1e11b3ebf8d2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/inference.d.ts @@ -0,0 +1,56 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Zero Shot Classification inference + */ +export interface ZeroShotClassificationInput { + /** + * The text to classify + */ + inputs: string; + /** + * Additional inference parameters for Zero Shot Classification + */ + parameters: ZeroShotClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Zero Shot Classification + */ +export interface ZeroShotClassificationParameters { + /** + * The set of possible class labels to classify the text into. + */ + candidate_labels: string[]; + /** + * The sentence used in conjunction with `candidate_labels` to attempt the text + * classification by replacing the placeholder with the candidate labels. + */ + hypothesis_template?: string; + /** + * Whether multiple candidate labels can be true. If false, the scores are normalized such + * that the sum of the label likelihoods for each sequence is 1. If true, the labels are + * considered independent and probabilities are normalized for each candidate. + */ + multi_label?: boolean; + [property: string]: unknown; +} +export type ZeroShotClassificationOutput = ZeroShotClassificationOutputElement[]; +/** + * Outputs of inference for the Zero Shot Classification task + */ +export interface ZeroShotClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..1a132031034157cd5ba0a7b20a870ef33c337044 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/zero-shot-classification/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC3C;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,EAAE,gCAAgC,CAAC;IAC7C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,4BAA4B,GAAG,mCAAmC,EAAE,CAAC;AACjF;;GAEG;AACH,MAAM,WAAW,mCAAmC;IACnD;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-classification/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..18ffbbd656c7012e0d48ef08fb8065a700213622 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/zero-shot-image-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAmFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..06b622c3ce2d5f838491e8a177098940eac14c43 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/data.js @@ -0,0 +1,85 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [ + { + // TODO write proper description + description: "", + id: "", + }, + ], + demo: { + inputs: [ + { + filename: "image-classification-input.jpeg", + type: "img", + }, + { + label: "Classes", + content: "cat, dog, bird", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Cat", + score: 0.664, + }, + { + label: "Dog", + score: 0.329, + }, + { + label: "Bird", + score: 0.008, + }, + ], + }, + ], + }, + metrics: [ + { + description: "Computes the number of times the correct label appears in top K labels predicted", + id: "top-K accuracy", + }, + ], + models: [ + { + description: "Multilingual image classification model for 80 languages.", + id: "visheratin/mexma-siglip", + }, + { + description: "Strong zero-shot image classification model.", + id: "google/siglip2-base-patch16-224", + }, + { + description: "Robust zero-shot image classification model.", + id: "intfloat/mmE5-mllama-11b-instruct", + }, + { + description: "Powerful zero-shot image classification model supporting 94 languages.", + id: "jinaai/jina-clip-v2", + }, + { + description: "Strong image classification model for biomedical domain.", + id: "microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224", + }, + ], + spaces: [ + { + description: "An application that leverages zero-shot image classification to find best captions to generate an image. ", + id: "pharma/CLIP-Interrogator", + }, + { + description: "An application to compare different zero-shot image classification models. ", + id: "merve/compare_clip_siglip", + }, + ], + summary: "Zero-shot image classification is the task of classifying previously unseen classes during training of a model.", + widgetModels: ["google/siglip-so400m-patch14-224"], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..8e80503ecf772e75dbaa641b0533c1f7d61ed22c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/inference.d.ts @@ -0,0 +1,50 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Zero Shot Image Classification inference + */ +export interface ZeroShotImageClassificationInput { + /** + * The input image data to classify as a base64-encoded string. + */ + inputs: Blob; + /** + * Additional inference parameters for Zero Shot Image Classification + */ + parameters: ZeroShotImageClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Zero Shot Image Classification + */ +export interface ZeroShotImageClassificationParameters { + /** + * The candidate labels for this image + */ + candidate_labels: string[]; + /** + * The sentence used in conjunction with `candidate_labels` to attempt the image + * classification by replacing the placeholder with the candidate labels. + */ + hypothesis_template?: string; + [property: string]: unknown; +} +export type ZeroShotImageClassificationOutput = ZeroShotImageClassificationOutputElement[]; +/** + * Outputs of inference for the Zero Shot Image Classification task + */ +export interface ZeroShotImageClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6c9de1ceb345e5457fd3755db377e73e3ac9b836 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/zero-shot-image-classification/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,EAAE,qCAAqC,CAAC;IAClD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,qCAAqC;IACrD;;OAEG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,iCAAiC,GAAG,wCAAwC,EAAE,CAAC;AAC3F;;GAEG;AACH,MAAM,WAAW,wCAAwC;IACxD;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-image-classification/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d3836f53f66b8db2c829bfe01d8fe3505d975832 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/zero-shot-object-detection/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA8Df,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/data.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/data.js new file mode 100644 index 0000000000000000000000000000000000000000..ee2e41f5567406e414158c44028de62443ce409d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/data.js @@ -0,0 +1,62 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +const taskData = { + datasets: [], + demo: { + inputs: [ + { + filename: "zero-shot-object-detection-input.jpg", + type: "img", + }, + { + label: "Classes", + content: "cat, dog, bird", + type: "text", + }, + ], + outputs: [ + { + filename: "zero-shot-object-detection-output.jpg", + type: "img", + }, + ], + }, + metrics: [ + { + description: "The Average Precision (AP) metric is the Area Under the PR Curve (AUC-PR). It is calculated for each class separately", + id: "Average Precision", + }, + { + description: "The Mean Average Precision (mAP) metric is the overall average of the AP values", + id: "Mean Average Precision", + }, + { + description: "The APα metric is the Average Precision at the IoU threshold of a α value, for example, AP50 and AP75", + id: "APα", + }, + ], + models: [ + { + description: "Solid zero-shot object detection model.", + id: "openmmlab-community/mm_grounding_dino_large_all", + }, + { + description: "Cutting-edge zero-shot object detection model.", + id: "fushh7/LLMDet", + }, + ], + spaces: [ + { + description: "A demo to compare different zero-shot object detection models per output and latency.", + id: "ariG23498/zero-shot-od", + }, + { + description: "A demo that combines a zero-shot object detection and mask generation model for zero-shot segmentation.", + id: "merve/OWLSAM", + }, + ], + summary: "Zero-shot object detection is a computer vision task to detect objects and their classes in images, without any prior training or knowledge of the classes. Zero-shot object detection models receive an image as input, as well as a list of candidate classes, and output the bounding boxes and labels where the objects have been detected.", + widgetModels: [], + youtubeId: "", +}; +exports.default = taskData; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/inference.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..00bfad1dfb69b256e202c541e3c00d200be46ca6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/inference.d.ts @@ -0,0 +1,61 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Zero Shot Object Detection inference + */ +export interface ZeroShotObjectDetectionInput { + /** + * The input image data as a base64-encoded string. + */ + inputs: Blob; + /** + * Additional inference parameters for Zero Shot Object Detection + */ + parameters: ZeroShotObjectDetectionParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Zero Shot Object Detection + */ +export interface ZeroShotObjectDetectionParameters { + /** + * The candidate labels for this image + */ + candidate_labels: string[]; + [property: string]: unknown; +} +/** + * The predicted bounding box. Coordinates are relative to the top left corner of the input + * image. + */ +export interface BoundingBox { + xmax: number; + xmin: number; + ymax: number; + ymin: number; + [property: string]: unknown; +} +export type ZeroShotObjectDetectionOutput = ZeroShotObjectDetectionOutputElement[]; +/** + * Outputs of inference for the Zero Shot Object Detection task + */ +export interface ZeroShotObjectDetectionOutputElement { + /** + * The predicted bounding box. Coordinates are relative to the top left corner of the input + * image. + */ + box: BoundingBox; + /** + * A candidate label + */ + label: string; + /** + * The associated score / probability + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d31807a4e74fc4e9f381f911e9838ca83df9e351 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/zero-shot-object-detection/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC5C;;OAEG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,EAAE,iCAAiC,CAAC;IAC9C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,iCAAiC;IACjD;;OAEG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,6BAA6B,GAAG,oCAAoC,EAAE,CAAC;AACnF;;GAEG;AACH,MAAM,WAAW,oCAAoC;IACpD;;;OAGG;IACH,GAAG,EAAE,WAAW,CAAC;IACjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/inference.js b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..c8ad2e549bdc6801e0d1c80b0308d4b9bd4985ce --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tasks/zero-shot-object-detection/inference.js @@ -0,0 +1,2 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tokenizer-data.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/tokenizer-data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..5863744080832186bd5f7482bd3f55c446872e84 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tokenizer-data.d.ts @@ -0,0 +1,26 @@ +export declare const SPECIAL_TOKENS_ATTRIBUTES: readonly ["bos_token", "eos_token", "unk_token", "sep_token", "pad_token", "cls_token", "mask_token"]; +/** + * Public interface for a tokenizer's special tokens mapping + */ +export interface AddedToken { + __type: "AddedToken"; + content?: string; + lstrip?: boolean; + normalized?: boolean; + rstrip?: boolean; + single_word?: boolean; +} +export type SpecialTokensMap = { + [key in (typeof SPECIAL_TOKENS_ATTRIBUTES)[number]]?: string | AddedToken | null; +}; +/** + * Public interface for tokenizer config + */ +export interface TokenizerConfig extends SpecialTokensMap { + use_default_system_prompt?: boolean; + chat_template?: string | Array<{ + name: string; + template: string; + }>; +} +//# sourceMappingURL=tokenizer-data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tokenizer-data.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/tokenizer-data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..8f5b0a57b829b2f6c0060ac3f36d00e093d8195c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tokenizer-data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"tokenizer-data.d.ts","sourceRoot":"","sources":["../../src/tokenizer-data.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,yBAAyB,uGAS5B,CAAC;AAEX;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;CACtB;AACD,MAAM,MAAM,gBAAgB,GAAG;KAC7B,GAAG,IAAI,CAAC,OAAO,yBAAyB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI;CAChF,CAAC;AACF;;GAEG;AACH,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACxD,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,aAAa,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACnE"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/tokenizer-data.js b/node_modules/@huggingface/tasks/dist/commonjs/tokenizer-data.js new file mode 100644 index 0000000000000000000000000000000000000000..bc254c993e5c903169ea1477bb0527e743d03cab --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/tokenizer-data.js @@ -0,0 +1,13 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.SPECIAL_TOKENS_ATTRIBUTES = void 0; +exports.SPECIAL_TOKENS_ATTRIBUTES = [ + "bos_token", + "eos_token", + "unk_token", + "sep_token", + "pad_token", + "cls_token", + "mask_token", + // additional_special_tokens (TODO) +]; diff --git a/node_modules/@huggingface/tasks/dist/commonjs/widget-example.d.ts b/node_modules/@huggingface/tasks/dist/commonjs/widget-example.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..3e99fa12e050a01ce6c7e167346e18184dde2f96 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/widget-example.d.ts @@ -0,0 +1,83 @@ +/** + * See default-widget-inputs.ts for the default widget inputs, this files only contains the types + */ +import type { ChatCompletionInputMessage } from "./tasks/index.js"; +type TableData = Record; +export type WidgetExampleOutputLabels = Array<{ + label: string; + score: number; +}>; +export interface WidgetExampleOutputAnswerScore { + answer: string; + score: number; +} +export interface WidgetExampleOutputText { + text: string; +} +export interface WidgetExampleOutputUrl { + url: string; +} +export type WidgetExampleOutput = WidgetExampleOutputLabels | WidgetExampleOutputAnswerScore | WidgetExampleOutputText | WidgetExampleOutputUrl; +export interface WidgetExampleBase { + example_title?: string; + group?: string; + /** + * Potential overrides to API parameters for this specific example + * (takes precedences over the model card metadata's inference.parameters) + */ + parameters?: { + aggregation_strategy?: string; + top_k?: number; + top_p?: number; + temperature?: number; + max_new_tokens?: number; + do_sample?: boolean; + negative_prompt?: string; + guidance_scale?: number; + num_inference_steps?: number; + }; + /** + * Optional output + */ + output?: TOutput; +} +export interface WidgetExampleChatInput extends WidgetExampleBase { + messages: ChatCompletionInputMessage[]; +} +export interface WidgetExampleTextInput extends WidgetExampleBase { + text: string; +} +export interface WidgetExampleTextAndContextInput extends WidgetExampleTextInput { + context: string; +} +export interface WidgetExampleTextAndTableInput extends WidgetExampleTextInput { + table: TableData; +} +export interface WidgetExampleAssetInput extends WidgetExampleBase { + src: string; +} +export interface WidgetExampleAssetAndPromptInput extends WidgetExampleAssetInput { + prompt: string; +} +export type WidgetExampleAssetAndTextInput = WidgetExampleAssetInput & WidgetExampleTextInput; +export type WidgetExampleAssetAndZeroShotInput = WidgetExampleAssetInput & WidgetExampleZeroShotTextInput; +export interface WidgetExampleStructuredDataInput extends WidgetExampleBase { + structured_data: TableData; +} +export interface WidgetExampleTableDataInput extends WidgetExampleBase { + table: TableData; +} +export interface WidgetExampleZeroShotTextInput extends WidgetExampleTextInput { + text: string; + candidate_labels: string; + multi_class: boolean; +} +export interface WidgetExampleSentenceSimilarityInput extends WidgetExampleBase { + source_sentence: string; + sentences: string[]; +} +export type WidgetExample = WidgetExampleChatInput | WidgetExampleTextInput | WidgetExampleTextAndContextInput | WidgetExampleTextAndTableInput | WidgetExampleAssetInput | WidgetExampleAssetAndPromptInput | WidgetExampleAssetAndTextInput | WidgetExampleAssetAndZeroShotInput | WidgetExampleStructuredDataInput | WidgetExampleTableDataInput | WidgetExampleZeroShotTextInput | WidgetExampleSentenceSimilarityInput; +type KeysOfUnion = T extends unknown ? keyof T : never; +export type WidgetExampleAttribute = KeysOfUnion; +export {}; +//# sourceMappingURL=widget-example.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/widget-example.d.ts.map b/node_modules/@huggingface/tasks/dist/commonjs/widget-example.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..2a7075f6b0bbf8789a174607be1ffbb685c86987 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/widget-example.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"widget-example.d.ts","sourceRoot":"","sources":["../../src/widget-example.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,kBAAkB,CAAC;AAEnE,KAAK,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;AAGrD,MAAM,MAAM,yBAAyB,GAAG,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAChF,MAAM,WAAW,8BAA8B;IAC9C,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACd;AACD,MAAM,WAAW,uBAAuB;IACvC,IAAI,EAAE,MAAM,CAAC;CACb;AACD,MAAM,WAAW,sBAAsB;IACtC,GAAG,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,MAAM,mBAAmB,GAC5B,yBAAyB,GACzB,8BAA8B,GAC9B,uBAAuB,GACvB,sBAAsB,CAAC;AAG1B,MAAM,WAAW,iBAAiB,CAAC,OAAO;IACzC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,UAAU,CAAC,EAAE;QAEZ,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAE9B,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,SAAS,CAAC,EAAE,OAAO,CAAC;QAEpB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,mBAAmB,CAAC,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,sBAAsB,CAAC,OAAO,GAAG,mBAAmB,CAAE,SAAQ,iBAAiB,CAAC,OAAO,CAAC;IACxG,QAAQ,EAAE,0BAA0B,EAAE,CAAC;CACvC;AAED,MAAM,WAAW,sBAAsB,CAAC,OAAO,GAAG,mBAAmB,CAAE,SAAQ,iBAAiB,CAAC,OAAO,CAAC;IACxG,IAAI,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,gCAAgC,CAChD,OAAO,GAAG,mBAAmB,CAC5B,SAAQ,sBAAsB,CAAC,OAAO,CAAC;IACxC,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,8BAA8B,CAAC,OAAO,GAAG,mBAAmB,CAAE,SAAQ,sBAAsB,CAAC,OAAO,CAAC;IACrH,KAAK,EAAE,SAAS,CAAC;CACjB;AAED,MAAM,WAAW,uBAAuB,CAAC,OAAO,GAAG,mBAAmB,CAAE,SAAQ,iBAAiB,CAAC,OAAO,CAAC;IACzG,GAAG,EAAE,MAAM,CAAC;CACZ;AACD,MAAM,WAAW,gCAAgC,CAChD,OAAO,GAAG,mBAAmB,CAC5B,SAAQ,uBAAuB,CAAC,OAAO,CAAC;IACzC,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,8BAA8B,CAAC,OAAO,GAAG,mBAAmB,IAAI,uBAAuB,CAAC,OAAO,CAAC,GAC3G,sBAAsB,CAAC,OAAO,CAAC,CAAC;AAEjC,MAAM,MAAM,kCAAkC,CAAC,OAAO,GAAG,mBAAmB,IAAI,uBAAuB,CAAC,OAAO,CAAC,GAC/G,8BAA8B,CAAC,OAAO,CAAC,CAAC;AAEzC,MAAM,WAAW,gCAAgC,CAAC,OAAO,GAAG,mBAAmB,CAAE,SAAQ,iBAAiB,CAAC,OAAO,CAAC;IAClH,eAAe,EAAE,SAAS,CAAC;CAC3B;AAED,MAAM,WAAW,2BAA2B,CAAC,OAAO,GAAG,mBAAmB,CAAE,SAAQ,iBAAiB,CAAC,OAAO,CAAC;IAC7G,KAAK,EAAE,SAAS,CAAC;CACjB;AAED,MAAM,WAAW,8BAA8B,CAAC,OAAO,GAAG,mBAAmB,CAAE,SAAQ,sBAAsB,CAAC,OAAO,CAAC;IACrH,IAAI,EAAE,MAAM,CAAC;IACb,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,oCAAoC,CACpD,OAAO,GAAG,mBAAmB,CAC5B,SAAQ,iBAAiB,CAAC,OAAO,CAAC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,EAAE,CAAC;CACpB;AAID,MAAM,MAAM,aAAa,CAAC,OAAO,GAAG,mBAAmB,IACpD,sBAAsB,CAAC,OAAO,CAAC,GAC/B,sBAAsB,CAAC,OAAO,CAAC,GAC/B,gCAAgC,CAAC,OAAO,CAAC,GACzC,8BAA8B,CAAC,OAAO,CAAC,GACvC,uBAAuB,CAAC,OAAO,CAAC,GAChC,gCAAgC,CAAC,OAAO,CAAC,GACzC,8BAA8B,CAAC,OAAO,CAAC,GACvC,kCAAkC,CAAC,OAAO,CAAC,GAC3C,gCAAgC,CAAC,OAAO,CAAC,GACzC,2BAA2B,CAAC,OAAO,CAAC,GACpC,8BAA8B,CAAC,OAAO,CAAC,GACvC,oCAAoC,CAAC,OAAO,CAAC,CAAC;AAEjD,KAAK,WAAW,CAAC,CAAC,IAAI,CAAC,SAAS,OAAO,GAAG,MAAM,CAAC,GAAG,KAAK,CAAC;AAE1D,MAAM,MAAM,sBAAsB,GAAG,WAAW,CAAC,aAAa,CAAC,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/commonjs/widget-example.js b/node_modules/@huggingface/tasks/dist/commonjs/widget-example.js new file mode 100644 index 0000000000000000000000000000000000000000..bcf2f800db81ac96d242fbed337068401d8b4fb5 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/commonjs/widget-example.js @@ -0,0 +1,5 @@ +"use strict"; +/** + * See default-widget-inputs.ts for the default widget inputs, this files only contains the types + */ +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/node_modules/@huggingface/tasks/dist/esm/agent-harnesses.d.ts b/node_modules/@huggingface/tasks/dist/esm/agent-harnesses.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..0438a2882598de35dc1868d839130942dfbc5acf --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/agent-harnesses.d.ts @@ -0,0 +1,264 @@ +/** + * Registry of AI coding agents / harnesses known to use the Hugging Face Hub. + * + * To add your harness, append an entry below keyed by its `id` (the name used + * when reporting Hub activity), and list the environment variable(s) that + * identify it. + */ +export interface AgentHarness { + /** + * Human-readable name of the harness, e.g. displayed in a leaderboard. + */ + prettyLabel: string; + /** + * URL to the harness's code repository (usually on GitHub). + */ + repoUrl?: string; + /** + * URL to the harness's documentation or website. + */ + docsUrl?: string; + /** + * Short description of the harness. + */ + description?: string; + /** + * Environment variable(s) that identify this harness, mapped to the value + * pattern they must match. Detection matches if ANY entry matches. + * + * The value pattern is one of: + * - `"*"`: the variable is set to any (non-empty) value + * - `""`: the variable equals this exact value + * - `"*"`: the variable value starts with `` (fuzzy match, resolved client-side) + * + * If not provided, the harness is detected through the standard AI_AGENT / AGENT variables only. + */ + envVars?: Record; +} +/** + * Standard environment variables that any tool can set to identify itself. + * When one of these is set, its value is used directly as the agent `id` + * (matched against the keys of `AGENT_HARNESSES`); unrecognized values are + * reported as `"unknown"`. + */ +export declare const STANDARD_AGENT_ENV_VARS: readonly ["AI_AGENT", "AGENT"]; +/** + * Add your new agent harness here. + * + * /!\ IMPORTANT + * + * Insertion order matters for detection priority: harnesses are checked from + * top to bottom and the first match wins. In particular, `cowork` must stay + * before `claude-code` so the more specific signal takes priority when both + * `CLAUDE_CODE` and `CLAUDE_CODE_IS_COWORK` are set. + */ +export declare const AGENT_HARNESSES: { + antigravity: { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + ANTIGRAVITY_AGENT: string; + }; + }; + "augment-cli": { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + AUGMENT_AGENT: string; + }; + }; + cline: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + CLINE_ACTIVE: string; + }; + }; + cowork: { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + CLAUDE_CODE_IS_COWORK: string; + }; + }; + "claude-code": { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + CLAUDECODE: string; + CLAUDE_CODE: string; + }; + }; + codex: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + CODEX_SANDBOX: string; + CODEX_CI: string; + CODEX_THREAD_ID: string; + }; + }; + crush: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + CRUSH: string; + }; + }; + "gemini-cli": { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + GEMINI_CLI: string; + }; + }; + "github-copilot": { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + COPILOT_MODEL: string; + COPILOT_ALLOW_ALL: string; + COPILOT_GITHUB_TOKEN: string; + }; + }; + goose: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + GOOSE_TERMINAL: string; + }; + }; + "hermes-agent": { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + HERMES_SESSION_ID: string; + }; + }; + hi: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + }; + "kilo-code": { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + KILOCODE_FEATURE: string; + }; + }; + kiro: { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + AGENT_CONTEXT_OUT: string; + }; + }; + openclaw: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + OPENCLAW_SHELL: string; + }; + }; + opencode: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + OPENCODE_CLIENT: string; + }; + }; + pi: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + PI_CODING_AGENT: string; + }; + }; + replit: { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + REPL_ID: string; + }; + }; + trae: { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + TRAE_AI_SHELL_ID: string; + }; + }; + warp: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + TERM_PROGRAM: string; + }; + }; + zed: { + prettyLabel: string; + repoUrl: string; + docsUrl: string; + description: string; + envVars: { + ZED_TERM: string; + }; + }; + "cursor-cli": { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + CURSOR_AGENT: string; + }; + }; + cursor: { + prettyLabel: string; + docsUrl: string; + description: string; + envVars: { + CURSOR_TRACE_ID: string; + }; + }; + devin: { + prettyLabel: string; + docsUrl: string; + description: string; + }; +}; +export type AgentHarnessKey = keyof typeof AGENT_HARNESSES; +//# sourceMappingURL=agent-harnesses.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/agent-harnesses.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/agent-harnesses.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..291dd4c88d5c3f53d749702b4313a0ccf1859a78 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/agent-harnesses.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"agent-harnesses.d.ts","sourceRoot":"","sources":["../../src/agent-harnesses.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IAC5B;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,gCAAiC,CAAC;AAEtE;;;;;;;;;GASG;AACH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkKY,CAAC;AAGzC,MAAM,MAAM,eAAe,GAAG,MAAM,OAAO,eAAe,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/agent-harnesses.js b/node_modules/@huggingface/tasks/dist/esm/agent-harnesses.js new file mode 100644 index 0000000000000000000000000000000000000000..e80310db0557faee53a225af14eb12635403cd2b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/agent-harnesses.js @@ -0,0 +1,180 @@ +/** + * Standard environment variables that any tool can set to identify itself. + * When one of these is set, its value is used directly as the agent `id` + * (matched against the keys of `AGENT_HARNESSES`); unrecognized values are + * reported as `"unknown"`. + */ +export const STANDARD_AGENT_ENV_VARS = ["AI_AGENT", "AGENT"]; +/** + * Add your new agent harness here. + * + * /!\ IMPORTANT + * + * Insertion order matters for detection priority: harnesses are checked from + * top to bottom and the first match wins. In particular, `cowork` must stay + * before `claude-code` so the more specific signal takes priority when both + * `CLAUDE_CODE` and `CLAUDE_CODE_IS_COWORK` are set. + */ +export const AGENT_HARNESSES = { + antigravity: { + prettyLabel: "Antigravity", + docsUrl: "https://antigravity.google", + description: "Agentic development platform from Google built around Gemini.", + envVars: { ANTIGRAVITY_AGENT: "*" }, + }, + "augment-cli": { + prettyLabel: "Augment CLI", + repoUrl: "https://github.com/augmentcode/auggie", + docsUrl: "https://www.augmentcode.com", + description: "Auggie, the command-line coding agent from Augment Code.", + envVars: { AUGMENT_AGENT: "*" }, + }, + cline: { + prettyLabel: "Cline", + repoUrl: "https://github.com/cline/cline", + docsUrl: "https://cline.bot", + description: "Open-source autonomous coding agent for VS Code.", + envVars: { CLINE_ACTIVE: "*" }, + }, + cowork: { + // must stay before `claude-code` so the more specific signal takes priority when both `CLAUDE_CODE` and `CLAUDE_CODE_IS_COWORK` are set. + prettyLabel: "Cowork", + docsUrl: "https://claude.com/product/cowork", + description: "Anthropic's agent for autonomous knowledge work, built on top of Claude Code.", + envVars: { CLAUDE_CODE_IS_COWORK: "*" }, + }, + "claude-code": { + prettyLabel: "Claude Code", + repoUrl: "https://github.com/anthropics/claude-code", + docsUrl: "https://code.claude.com/docs", + description: "Anthropic's agentic coding tool that lives in your terminal.", + envVars: { CLAUDECODE: "*", CLAUDE_CODE: "*" }, + }, + codex: { + prettyLabel: "Codex", + repoUrl: "https://github.com/openai/codex", + docsUrl: "https://developers.openai.com/codex", + description: "OpenAI's lightweight coding agent that runs in your terminal.", + envVars: { CODEX_SANDBOX: "*", CODEX_CI: "*", CODEX_THREAD_ID: "*" }, + }, + crush: { + prettyLabel: "Crush", + repoUrl: "https://github.com/charmbracelet/crush", + docsUrl: "https://github.com/charmbracelet/crush", + description: "Charm's open-source AI coding agent for the terminal.", + envVars: { CRUSH: "*" }, + }, + "gemini-cli": { + prettyLabel: "Gemini CLI", + repoUrl: "https://github.com/google-gemini/gemini-cli", + docsUrl: "https://geminicli.com", + description: "Google's open-source terminal AI coding agent powered by Gemini models.", + envVars: { GEMINI_CLI: "*" }, + }, + "github-copilot": { + prettyLabel: "GitHub Copilot", + docsUrl: "https://docs.github.com/copilot", + description: "GitHub's AI coding assistant.", + envVars: { COPILOT_MODEL: "*", COPILOT_ALLOW_ALL: "*", COPILOT_GITHUB_TOKEN: "*" }, + }, + goose: { + prettyLabel: "Goose", + repoUrl: "https://github.com/aaif-goose/goose", + docsUrl: "https://goose-docs.ai/", + description: "Open-source, extensible AI agent, originally from Block and now part of the Agentic AI Foundation.", + envVars: { GOOSE_TERMINAL: "*" }, + }, + "hermes-agent": { + prettyLabel: "Hermes Agent", + repoUrl: "https://github.com/NousResearch/hermes-agent", + docsUrl: "https://hermes-agent.nousresearch.com/docs", + description: "Nous Research's self-improving, multi-provider terminal AI agent.", + envVars: { HERMES_SESSION_ID: "*" }, + }, + hi: { + prettyLabel: "hi", + repoUrl: "https://github.com/PipeNetwork/hi", + docsUrl: "https://github.com/PipeNetwork/hi#readme", + description: "Rust terminal coding agent with verification-in-the-loop.", + }, + "kilo-code": { + prettyLabel: "Kilo Code", + repoUrl: "https://github.com/Kilo-Org/kilocode", + docsUrl: "https://kilocode.ai/docs", + description: "Open-source agentic coding agent for VS Code, JetBrains, and the terminal.", + envVars: { KILOCODE_FEATURE: "*" }, + }, + kiro: { + prettyLabel: "Kiro", + docsUrl: "https://kiro.dev", + description: "AWS's agentic IDE for spec-driven AI software development.", + envVars: { AGENT_CONTEXT_OUT: "*" }, + }, + openclaw: { + prettyLabel: "OpenClaw", + repoUrl: "https://github.com/openclaw/openclaw", + docsUrl: "https://openclaw.ai", + description: "Open-source, self-hosted personal AI assistant that runs on your own devices.", + envVars: { OPENCLAW_SHELL: "*" }, + }, + opencode: { + prettyLabel: "opencode", + repoUrl: "https://github.com/anomalyco/opencode", + docsUrl: "https://opencode.ai", + description: "Open-source AI coding agent built for the terminal.", + envVars: { OPENCODE_CLIENT: "*" }, + }, + pi: { + prettyLabel: "Pi", + repoUrl: "https://github.com/earendil-works/pi", + docsUrl: "https://pi.dev", + description: "Minimal, self-extensible terminal coding agent with a unified multi-provider LLM API.", + envVars: { PI_CODING_AGENT: "*" }, + }, + replit: { + prettyLabel: "Replit", + docsUrl: "https://replit.com", + description: "Cloud development environment with an AI coding agent.", + envVars: { REPL_ID: "*" }, + }, + trae: { + prettyLabel: "Trae", + docsUrl: "https://trae.ai", + description: "AI-powered IDE from ByteDance.", + envVars: { TRAE_AI_SHELL_ID: "*" }, + }, + warp: { + prettyLabel: "Warp", + repoUrl: "https://github.com/warpdotdev/Warp", + docsUrl: "https://docs.warp.dev", + description: "AI-powered terminal with an agentic Agent Mode.", + envVars: { TERM_PROGRAM: "WarpTerminal" }, + }, + zed: { + prettyLabel: "Zed", + repoUrl: "https://github.com/zed-industries/zed", + docsUrl: "https://zed.dev", + description: "High-performance code editor with an integrated AI agent panel and terminal.", + envVars: { ZED_TERM: "*" }, + }, + "cursor-cli": { + // Kept near the bottom (and before `cursor`): when another agent runs inside the Cursor editor's terminal, + // its child processes inherit `CURSOR_TRACE_ID`, so `cursor` must stay a low-priority fallback and lose to + // the agent's own marker. `cursor-cli` is the more specific Cursor signal (`CURSOR_AGENT`), so it comes first. + prettyLabel: "Cursor CLI", + docsUrl: "https://cursor.com/docs/cli/overview", + description: "Cursor's coding agent for the command line.", + envVars: { CURSOR_AGENT: "*" }, + }, + cursor: { + prettyLabel: "Cursor", + docsUrl: "https://cursor.com", + description: "AI-powered code editor.", + envVars: { CURSOR_TRACE_ID: "*" }, + }, + devin: { + prettyLabel: "Devin", + docsUrl: "https://devin.ai", + description: "Autonomous AI software engineer from Cognition.", + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/esm/dataset-libraries.d.ts b/node_modules/@huggingface/tasks/dist/esm/dataset-libraries.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..7dc46b97a7ba271fef3a38989e1f0dcd6a5e6af6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/dataset-libraries.d.ts @@ -0,0 +1,99 @@ +/** + * Elements configurable by a dataset library. + */ +export interface DatasetLibraryUiElement { + /** + * Pretty name of the library. + * displayed (in tags?, and) on the main + * call-to-action button on the dataset page. + */ + prettyLabel: string; + /** + * Repo name of the library's (usually on GitHub) code repo + */ + repoName: string; + /** + * URL to library's (usually on GitHub) code repo + */ + repoUrl: string; + /** + * URL to library's docs + */ + docsUrl?: string; +} +export declare const DATASET_LIBRARIES_UI_ELEMENTS: { + mlcroissant: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + webdataset: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + datasets: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + pandas: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + dask: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + distilabel: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + fiftyone: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + lance: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + argilla: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + polars: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + duckdb: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; + datadesigner: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + }; +}; +export type DatasetLibraryKey = keyof typeof DATASET_LIBRARIES_UI_ELEMENTS; +//# sourceMappingURL=dataset-libraries.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/dataset-libraries.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/dataset-libraries.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..8c7553eddf91259a20de8bdc39ec70c1733e4dac --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/dataset-libraries.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"dataset-libraries.d.ts","sourceRoot":"","sources":["../../src/dataset-libraries.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC;;;;OAIG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyES,CAAC;AAGpD,MAAM,MAAM,iBAAiB,GAAG,MAAM,OAAO,6BAA6B,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/dataset-libraries.js b/node_modules/@huggingface/tasks/dist/esm/dataset-libraries.js new file mode 100644 index 0000000000000000000000000000000000000000..668161b73ab811b1c56a6dfa9b2f86b33436df9c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/dataset-libraries.js @@ -0,0 +1,74 @@ +export const DATASET_LIBRARIES_UI_ELEMENTS = { + mlcroissant: { + prettyLabel: "Croissant", + repoName: "croissant", + repoUrl: "https://github.com/mlcommons/croissant/tree/main/python/mlcroissant", + docsUrl: "https://huggingface.co/docs/dataset-viewer/mlcroissant", + }, + webdataset: { + prettyLabel: "WebDataset", + repoName: "webdataset", + repoUrl: "https://github.com/webdataset/webdataset", + docsUrl: "https://huggingface.co/docs/hub/datasets-webdataset", + }, + datasets: { + prettyLabel: "Datasets", + repoName: "datasets", + repoUrl: "https://github.com/huggingface/datasets", + docsUrl: "https://huggingface.co/docs/hub/datasets-usage", + }, + pandas: { + prettyLabel: "pandas", + repoName: "pandas", + repoUrl: "https://github.com/pandas-dev/pandas", + docsUrl: "https://huggingface.co/docs/hub/datasets-pandas", + }, + dask: { + prettyLabel: "Dask", + repoName: "dask", + repoUrl: "https://github.com/dask/dask", + docsUrl: "https://huggingface.co/docs/hub/datasets-dask", + }, + distilabel: { + prettyLabel: "Distilabel", + repoName: "distilabel", + repoUrl: "https://github.com/argilla-io/distilabel", + docsUrl: "https://huggingface.co/docs/hub/datasets-distilabel", + }, + fiftyone: { + prettyLabel: "FiftyOne", + repoName: "fiftyone", + repoUrl: "https://github.com/voxel51/fiftyone", + docsUrl: "https://huggingface.co/docs/hub/datasets-fiftyone", + }, + lance: { + prettyLabel: "Lance", + repoName: "lance", + repoUrl: "https://github.com/lance-format/lance", + docsUrl: "https://huggingface.co/docs/hub/datasets-lance", + }, + argilla: { + prettyLabel: "Argilla", + repoName: "argilla", + repoUrl: "https://github.com/argilla-io/argilla", + docsUrl: "https://huggingface.co/docs/hub/datasets-argilla", + }, + polars: { + prettyLabel: "Polars", + repoName: "polars", + repoUrl: "https://github.com/pola-rs/polars", + docsUrl: "https://huggingface.co/docs/hub/datasets-polars", + }, + duckdb: { + prettyLabel: "DuckDB", + repoName: "duckdb", + repoUrl: "https://github.com/duckdb/duckdb", + docsUrl: "https://huggingface.co/docs/hub/datasets-duckdb", + }, + datadesigner: { + prettyLabel: "NeMo Data Designer", + repoName: "datadesigner", + repoUrl: "https://github.com/NVIDIA-NeMo/DataDesigner", + docsUrl: "https://nvidia-nemo.github.io/DataDesigner/", + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/esm/default-widget-inputs.d.ts b/node_modules/@huggingface/tasks/dist/esm/default-widget-inputs.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b9bd77d2be8a451d248b0f8776226127e315a039 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/default-widget-inputs.d.ts @@ -0,0 +1,6 @@ +import type { WidgetExample } from "./widget-example.js"; +import type { WidgetType } from "./pipelines.js"; +type PerLanguageMapping = Map; +export declare const MAPPING_DEFAULT_WIDGET: Map; +export {}; +//# sourceMappingURL=default-widget-inputs.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/default-widget-inputs.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/default-widget-inputs.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e652e43f9eb09082b9409d0ec2c35eca1d52a541 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/default-widget-inputs.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"default-widget-inputs.d.ts","sourceRoot":"","sources":["../../src/default-widget-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAIjD,KAAK,kBAAkB,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,aAAa,EAAE,CAAC,CAAC;AAoqBtE,eAAO,MAAM,sBAAsB,iCAejC,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/default-widget-inputs.js b/node_modules/@huggingface/tasks/dist/esm/default-widget-inputs.js new file mode 100644 index 0000000000000000000000000000000000000000..cc853d066e146ed6103158014676d63e36f01283 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/default-widget-inputs.js @@ -0,0 +1,674 @@ +/// NOTE TO CONTRIBUTORS: +/// +/// When adding sample inputs for a new language, you don't +/// necessarily have to translate the inputs from existing languages. +/// (which were quite random to begin with) +/// +/// i.e. Feel free to be creative and provide better samples. +// +/// The placeholder will be replaced by the correct mask token +/// in the following examples, depending on the model type +/// +/// see [INTERNAL] github.com/huggingface/moon-landing/blob/c5c3d45fe0ab27347b3ab27bdad646ef20732351/server/lib/App.ts#L254 +// +const MAPPING_EN = new Map([ + ["text-classification", [`I like you. I love you`]], + [ + "token-classification", + [ + `My name is Wolfgang and I live in Berlin`, + `My name is Sarah and I live in London`, + `My name is Clara and I live in Berkeley, California.`, + ], + ], + [ + "table-question-answering", + [ + { + text: `How many stars does the transformers repository have?`, + table: { + Repository: ["Transformers", "Datasets", "Tokenizers"], + Stars: [36542, 4512, 3934], + Contributors: [651, 77, 34], + "Programming language": ["Python", "Python", "Rust, Python and NodeJS"], + }, + }, + ], + ], + [ + "question-answering", + [ + { + text: `Where do I live?`, + context: `My name is Wolfgang and I live in Berlin`, + }, + { + text: `Where do I live?`, + context: `My name is Sarah and I live in London`, + }, + { + text: `What's my name?`, + context: `My name is Clara and I live in Berkeley.`, + }, + { + text: `Which name is also used to describe the Amazon rainforest in English?`, + context: `The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain "Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species.`, + }, + ], + ], + [ + "zero-shot-classification", + [ + { + text: "I have a problem with my iphone that needs to be resolved asap!", + candidate_labels: "urgent, not urgent, phone, tablet, computer", + multi_class: true, + }, + { + text: "Last week I upgraded my iOS version and ever since then my phone has been overheating whenever I use your app.", + candidate_labels: "mobile, website, billing, account access", + multi_class: false, + }, + { + text: "A new model offers an explanation for how the Galilean satellites formed around the solar system’s largest world. Konstantin Batygin did not set out to solve one of the solar system’s most puzzling mysteries when he went for a run up a hill in Nice, France. Dr. Batygin, a Caltech researcher, best known for his contributions to the search for the solar system’s missing “Planet Nine,” spotted a beer bottle. At a steep, 20 degree grade, he wondered why it wasn’t rolling down the hill. He realized there was a breeze at his back holding the bottle in place. Then he had a thought that would only pop into the mind of a theoretical astrophysicist: “Oh! This is how Europa formed.” Europa is one of Jupiter’s four large Galilean moons. And in a paper published Monday in the Astrophysical Journal, Dr. Batygin and a co-author, Alessandro Morbidelli, a planetary scientist at the Côte d’Azur Observatory in France, present a theory explaining how some moons form around gas giants like Jupiter and Saturn, suggesting that millimeter-sized grains of hail produced during the solar system’s formation became trapped around these massive worlds, taking shape one at a time into the potentially habitable moons we know today.", + candidate_labels: "space & cosmos, scientific discovery, microbiology, robots, archeology", + multi_class: true, + }, + ], + ], + ["translation", [`My name is Wolfgang and I live in Berlin`, `My name is Sarah and I live in London`]], + [ + "summarization", + [ + `The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct.`, + ], + ], + [ + "conversational", + [ + `Hi, what can you help me with?`, + `What is 84 * 3 / 2?`, + `Tell me an interesting fact about the universe!`, + `Explain quantum computing in simple terms.`, + ], + ], + [ + "text-generation", + [ + `My name is Julien and I like to`, + `I like traveling by train because`, + `Paris is an amazing place to visit,`, + `Once upon a time,`, + ], + ], + ["fill-mask", [`Paris is the of France.`, `The goal of life is .`]], + [ + "sentence-similarity", + [ + { + source_sentence: "That is a happy person", + sentences: ["That is a happy dog", "That is a very happy person", "Today is a sunny day"], + }, + ], + ], +]); +const MAPPING_ZH = new Map([ + ["text-classification", [`我喜欢你。 我爱你`]], + ["token-classification", [`我叫沃尔夫冈，我住在柏林。`, `我叫萨拉，我住在伦敦。`, `我叫克拉拉，我住在加州伯克利。`]], + [ + "question-answering", + [ + { + text: `我住在哪里？`, + context: `我叫沃尔夫冈，我住在柏林。`, + }, + { + text: `我住在哪里？`, + context: `我叫萨拉，我住在伦敦。`, + }, + { + text: `我的名字是什么？`, + context: `我叫克拉拉，我住在伯克利。`, + }, + ], + ], + ["translation", [`我叫沃尔夫冈，我住在柏林。`, `我叫萨拉，我住在伦敦。`]], + [ + "zero-shot-classification", + [ + { + text: "房间干净明亮，非常不错", + candidate_labels: "这是一条差评, 这是一条好评", + }, + ], + ], + [ + "summarization", + [ + `该塔高324米（1063英尺），与一幢81层的建筑物一样高，是巴黎最高的建筑物。 它的底座是方形的，每边长125米（410英尺）。 在建造过程中，艾菲尔铁塔超过了华盛顿纪念碑，成为世界上最高的人造结构，它保持了41年的头衔，直到1930年纽约市的克莱斯勒大楼竣工。这是第一个到达300米高度的结构。 由于1957年在塔顶增加了广播天线，因此它现在比克莱斯勒大厦高5.2米（17英尺）。 除发射器外，艾菲尔铁塔是法国第二高的独立式建筑，仅次于米劳高架桥。`, + ], + ], + [ + "text-generation", + [`我叫朱利安，我喜欢`, `我叫托马斯，我的主要`, `我叫玛丽亚，我最喜欢的`, `我叫克拉拉，我是`, `从前，`], + ], + ["fill-mask", [`巴黎是国的首都。`, `生活的真谛是。`]], + [ + "sentence-similarity", + [ + { + source_sentence: "那是 個快樂的人", + sentences: ["那是 條快樂的狗", "那是 個非常幸福的人", "今天是晴天"], + }, + ], + ], +]); +const MAPPING_FR = new Map([ + ["text-classification", [`Je t'apprécie beaucoup. Je t'aime.`]], + ["token-classification", [`Mon nom est Wolfgang et je vis à Berlin`]], + [ + "question-answering", + [ + { + text: `Où est-ce que je vis?`, + context: `Mon nom est Wolfgang et je vis à Berlin`, + }, + ], + ], + ["translation", [`Mon nom est Wolfgang et je vis à Berlin`]], + [ + "summarization", + [ + `La tour fait 324 mètres (1,063 pieds) de haut, environ la même hauteur qu'un immeuble de 81 étages, et est la plus haute structure de Paris. Sa base est carrée, mesurant 125 mètres (410 pieds) sur chaque côté. Durant sa construction, la tour Eiffel surpassa le Washington Monument pour devenir la plus haute structure construite par l'homme dans le monde, un titre qu'elle conserva pendant 41 ans jusqu'à l'achèvement du Chrysler Building à New-York City en 1930. Ce fut la première structure à atteindre une hauteur de 300 mètres. Avec l'ajout d'une antenne de radiodiffusion au sommet de la tour Eiffel en 1957, celle-ci redevint plus haute que le Chrysler Building de 5,2 mètres (17 pieds). En excluant les transmetteurs, elle est la seconde plus haute structure autoportante de France après le viaduc de Millau.`, + ], + ], + ["text-generation", [`Mon nom est Julien et j'aime`, `Mon nom est Thomas et mon principal`, `Il était une fois`]], + ["fill-mask", [`Paris est la de la France.`]], + [ + "sentence-similarity", + [ + { + source_sentence: "C'est une personne heureuse", + sentences: [ + "C'est un chien heureux", + "C'est une personne très heureuse", + "Aujourd'hui est une journée ensoleillée", + ], + }, + ], + ], +]); +const MAPPING_ES = new Map([ + ["text-classification", [`Te quiero. Te amo.`]], + ["token-classification", [`Me llamo Wolfgang y vivo en Berlin`]], + [ + "question-answering", + [ + { + text: `¿Dónde vivo?`, + context: `Me llamo Wolfgang y vivo en Berlin`, + }, + { + text: `¿Quién inventó el submarino?`, + context: `Isaac Peral fue un murciano que inventó el submarino`, + }, + { + text: `¿Cuántas personas hablan español?`, + context: `El español es el segundo idioma más hablado del mundo con más de 442 millones de hablantes`, + }, + ], + ], + [ + "translation", + [ + `Me llamo Wolfgang y vivo en Berlin`, + `Los ingredientes de una tortilla de patatas son: huevos, patatas y cebolla`, + ], + ], + [ + "summarization", + [ + `La torre tiene 324 metros (1.063 pies) de altura, aproximadamente la misma altura que un edificio de 81 pisos y la estructura más alta de París. Su base es cuadrada, mide 125 metros (410 pies) a cada lado. Durante su construcción, la Torre Eiffel superó al Washington Monument para convertirse en la estructura artificial más alta del mundo, un título que mantuvo durante 41 años hasta que el Chrysler Building en la ciudad de Nueva York se terminó en 1930. Fue la primera estructura en llegar Una altura de 300 metros. Debido a la adición de una antena de transmisión en la parte superior de la torre en 1957, ahora es más alta que el Chrysler Building en 5,2 metros (17 pies). Excluyendo los transmisores, la Torre Eiffel es la segunda estructura independiente más alta de Francia después del Viaducto de Millau.`, + ], + ], + [ + "text-generation", + [ + `Me llamo Julien y me gusta`, + `Me llamo Thomas y mi principal`, + `Me llamo Manuel y trabajo en`, + `Érase una vez,`, + `Si tú me dices ven, `, + ], + ], + ["fill-mask", [`Mi nombre es y vivo en Nueva York.`, `El español es un idioma muy en el mundo.`]], + [ + "sentence-similarity", + [ + { + source_sentence: "Esa es una persona feliz", + sentences: ["Ese es un perro feliz", "Esa es una persona muy feliz", "Hoy es un día soleado"], + }, + ], + ], +]); +const MAPPING_RU = new Map([ + ["text-classification", [`Ты мне нравишься. Я тебя люблю`]], + ["token-classification", [`Меня зовут Вольфганг и я живу в Берлине`]], + [ + "question-answering", + [ + { + text: `Где живу?`, + context: `Меня зовут Вольфганг и я живу в Берлине`, + }, + ], + ], + ["translation", [`Меня зовут Вольфганг и я живу в Берлине`]], + [ + "summarization", + [ + `Высота башни составляет 324 метра (1063 фута), примерно такая же высота, как у 81-этажного здания, и самое высокое сооружение в Париже. Его основание квадратно, размером 125 метров (410 футов) с любой стороны. Во время строительства Эйфелева башня превзошла монумент Вашингтона, став самым высоким искусственным сооружением в мире, и этот титул она удерживала в течение 41 года до завершения строительство здания Крайслер в Нью-Йорке в 1930 году. Это первое сооружение которое достигло высоты 300 метров. Из-за добавления вещательной антенны на вершине башни в 1957 году она сейчас выше здания Крайслер на 5,2 метра (17 футов). За исключением передатчиков, Эйфелева башня является второй самой высокой отдельно стоящей структурой во Франции после виадука Мийо.`, + ], + ], + ["text-generation", [`Меня зовут Жюльен и`, `Меня зовут Томас и мой основной`, `Однажды`]], + ["fill-mask", [`Меня зовут и я инженер живущий в Нью-Йорке.`]], + [ + "sentence-similarity", + [ + { + source_sentence: "Это счастливый человек", + sentences: ["Это счастливая собака", "Это очень счастливый человек", "Сегодня солнечный день"], + }, + ], + ], +]); +const MAPPING_UK = new Map([ + ["translation", [`Мене звати Вольфґанґ і я живу в Берліні.`]], + ["fill-mask", [`Мене звати .`]], +]); +const MAPPING_IT = new Map([ + ["text-classification", [`Mi piaci. Ti amo`]], + [ + "token-classification", + [ + `Mi chiamo Wolfgang e vivo a Berlino`, + `Mi chiamo Sarah e vivo a Londra`, + `Mi chiamo Clara e vivo a Berkeley in California.`, + ], + ], + [ + "question-answering", + [ + { + text: `Dove vivo?`, + context: `Mi chiamo Wolfgang e vivo a Berlino`, + }, + { + text: `Dove vivo?`, + context: `Mi chiamo Sarah e vivo a Londra`, + }, + { + text: `Come mio chiamo?`, + context: `Mi chiamo Clara e vivo a Berkeley.`, + }, + ], + ], + ["translation", [`Mi chiamo Wolfgang e vivo a Berlino`, `Mi chiamo Sarah e vivo a Londra`]], + [ + "summarization", + [ + `La torre degli Asinelli è una delle cosiddette due torri di Bologna, simbolo della città, situate in piazza di porta Ravegnana, all'incrocio tra le antiche strade San Donato (ora via Zamboni), San Vitale, Maggiore e Castiglione. Eretta, secondo la tradizione, fra il 1109 e il 1119 dal nobile Gherardo Asinelli, la torre è alta 97,20 metri, pende verso ovest per 2,23 metri e presenta all'interno una scalinata composta da 498 gradini. Ancora non si può dire con certezza quando e da chi fu costruita la torre degli Asinelli. Si presume che la torre debba il proprio nome a Gherardo Asinelli, il nobile cavaliere di fazione ghibellina al quale se ne attribuisce la costruzione, iniziata secondo una consolidata tradizione l'11 ottobre 1109 e terminata dieci anni dopo, nel 1119.`, + ], + ], + [ + "text-generation", + [ + `Mi chiamo Loreto e mi piace`, + `Mi chiamo Thomas e il mio principale`, + `Mi chiamo Marianna, la mia cosa preferita`, + `Mi chiamo Clara e sono`, + `C'era una volta`, + ], + ], + ["fill-mask", [`Roma è la d'Italia.`, `Lo scopo della vita è .`]], + [ + "sentence-similarity", + [ + { + source_sentence: "Questa è una persona felice", + sentences: ["Questo è un cane felice", "Questa è una persona molto felice", "Oggi è una giornata di sole"], + }, + ], + ], +]); +const MAPPING_FA = new Map([ + [ + "text-classification", + [`پروژه به موقع تحویل شد و همه چیز خوب بود.`, `سیب‌زمینی بی‌کیفیت بود.`, `قیمت و کیفیت عالی`, `خوب نبود اصلا`], + ], + [ + "token-classification", + [ + `این سریال به صورت رسمی در تاریخ دهم می ۲۰۱۱ توسط شبکه فاکس برای پخش رزرو شد.`, + `دفتر مرکزی شرکت پارس‌مینو در شهر اراک در استان مرکزی قرار دارد.`, + `وی در سال ۲۰۱۳ درگذشت و مسئول خاکسپاری و اقوامش برای او مراسم یادبود گرفتند.`, + ], + ], + [ + "question-answering", + [ + { + text: `من کجا زندگی میکنم؟`, + context: `نام من پژمان است و در گرگان زندگی میکنم.`, + }, + { + text: `نامم چیست و کجا زندگی می‌کنم؟`, + context: `اسمم سارا است و در آفریقای جنوبی زندگی میکنم.`, + }, + { + text: `نام من چیست؟`, + context: `من مریم هستم و در تبریز زندگی می‌کنم.`, + }, + { + text: `بیشترین مساحت جنگل آمازون در کدام کشور است؟`, + context: [ + "آمازون نام بزرگ‌ترین جنگل بارانی جهان است که در شمال آمریکای جنوبی قرار گرفته و بیشتر آن در خاک برزیل و پرو", + "جای دارد. بیش از نیمی از همه جنگل‌های بارانی باقی‌مانده در جهان در آمازون قرار دارد.", + "مساحت جنگل‌های آمازون ۵٫۵ میلیون کیلومتر مربع است که بین ۹ کشور تقسیم شده‌است.", + ].join("\n"), + }, + ], + ], + [ + "translation", + [ + "بیشتر مساحت جنگل‌های آمازون در حوضه آبریز رود آمازون و ۱۱۰۰ شاخه آن واقع شده‌است.", + "مردمان نَبَطی از هزاره‌های یکم و دوم پیش از میلاد در این منطقه زندگی می‌کردند.", + ], + ], + [ + "summarization", + [ + [ + "شاهنامه اثر حکیم ابوالقاسم فردوسی توسی، حماسه‌ای منظوم، بر حسب دست نوشته‌های ", + "موجود دربرگیرنده نزدیک به ۵۰٬۰۰۰ بیت تا نزدیک به ۶۱٬۰۰۰ بیت و یکی از ", + "بزرگ‌ترین و برجسته‌ترین سروده‌های حماسی جهان است که سرایش آن دست‌آوردِ ", + "دست‌کم سی سال کارِ پیوستهٔ این سخن‌سرای نامدار ایرانی است. موضوع این شاهکار ادبی،", + " افسانه‌ها و تاریخ ایران از آغاز تا حملهٔ عرب‌ها به ایران در سدهٔ هفتم میلادی است", + " (شاهنامه از سه بخش اسطوره، پهلوانی و تاریخی تشکیل شده‌است) که در چهار", + " دودمان پادشاهیِ پیشدادیان، کیانیان، اشکانیان و ساسانیان گنجانده می‌شود.", + " شاهنامه بر وزن «فَعولُن فعولن فعولن فَعَلْ»، در بحرِ مُتَقارِبِ مثمَّنِ محذوف نگاشته شده‌است.", + "هنگامی که زبان دانش و ادبیات در ایران زبان عربی بود، فردوسی، با سرودن شاهنامه", + " با ویژگی‌های هدف‌مندی که داشت، زبان پارسی را زنده و پایدار کرد. یکی از ", + " بن‌مایه‌های مهمی که فردوسی برای سرودن شاهنامه از آن استفاده کرد،", + " شاهنامهٔ ابومنصوری بود. شاهنامه نفوذ بسیاری در جهت‌گیری ", + " فرهنگ فارسی و نیز بازتاب‌های شکوه‌مندی در ادبیات جهان داشته‌است و شاعران ", + " بزرگی مانند گوته و ویکتور هوگو از آن به نیکی یاد کرده‌اند.", + ].join("\n"), + ], + ], + ["text-generation", ["اسم من نازنین است و من", "روزی روزگاری"]], + [ + "fill-mask", + [ + `زندگی یک سوال است و این که چگونه کنیم پاسخ این سوال!`, + `زندگی از مرگ پرسید: چرا همه من را دارند اما از تو متنفرند؟`, + ], + ], +]); +const MAPPING_AR = new Map([ + ["text-classification", [`أحبك. أهواك`]], + [ + "token-classification", + [`إسمي محمد وأسكن في برلين`, `إسمي ساره وأسكن في لندن`, `إسمي سامي وأسكن في القدس في فلسطين.`], + ], + [ + "question-answering", + [ + { + text: `أين أسكن؟`, + context: `إسمي محمد وأسكن في بيروت`, + }, + { + text: `أين أسكن؟`, + context: `إسمي ساره وأسكن في لندن`, + }, + { + text: `ما اسمي؟`, + context: `اسمي سعيد وأسكن في حيفا.`, + }, + { + text: `ما لقب خالد بن الوليد بالعربية؟`, + context: `خالد بن الوليد من أبطال وقادة الفتح الإسلامي وقد تحدثت عنه اللغات الإنجليزية والفرنسية والإسبانية ولقب بسيف الله المسلول.`, + }, + ], + ], + ["translation", [`إسمي محمد وأسكن في برلين`, `إسمي ساره وأسكن في لندن`]], + [ + "summarization", + [ + `تقع الأهرامات في الجيزة قرب القاهرة في مصر وقد بنيت منذ عدة قرون، وقيل إنها كانت قبورا للفراعنة وتم بناؤها بعملية هندسية رائعة واستقدمت حجارتها من جبل المقطم وتم نقلها بالسفن أو على الرمل، وما تزال شامخة ويقصدها السياح من كافة أرجاء المعمورة.`, + ], + ], + [ + "text-generation", + [ + `إسمي محمد وأحب أن`, + `دع المكارم لا ترحل لبغيتها - واقعد فإنك أنت الطاعم الكاسي.`, + `لماذا نحن هنا؟`, + `القدس مدينة تاريخية، بناها الكنعانيون في`, + `كان يا ما كان في قديم الزمان`, + ], + ], + ["fill-mask", [`باريس فرنسا.`, `فلسفة الحياة هي .`]], + [ + "sentence-similarity", + [ + { + source_sentence: "هذا شخص سعيد", + sentences: ["هذا كلب سعيد", "هذا شخص سعيد جدا", "اليوم هو يوم مشمس"], + }, + ], + ], +]); +const MAPPING_BN = new Map([ + ["text-classification", [`বাঙালির ঘরে ঘরে আজ নবান্ন উৎসব।`]], + [ + "token-classification", + [`আমার নাম জাহিদ এবং আমি ঢাকায় বাস করি।`, `তিনি গুগলে চাকরী করেন।`, `আমার নাম সুস্মিতা এবং আমি কলকাতায় বাস করি।`], + ], + ["translation", [`আমার নাম জাহিদ, আমি রংপুরে বাস করি।`, `আপনি কী আজকে বাসায় আসবেন?`]], + [ + "summarization", + [ + `‘ইকোনমিস্ট’ লিখেছে, অ্যান্টিবডির চার মাস স্থায়ী হওয়ার খবরটি দুই কারণে আনন্দের। অ্যান্টিবডি যত দিন পর্যন্ত শরীরে টিকবে, তত দিন সংক্রমণ থেকে সুরক্ষিত থাকা সম্ভব। অর্থাৎ, এমন এক টিকার প্রয়োজন হবে, যা অ্যান্টিবডির উত্পাদনকে প্ররোচিত করতে পারে এবং দীর্ঘস্থায়ী সুরক্ষা দিতে পারে। এগুলো খুঁজে বের করাও সহজ। এটি আভাস দেয়, ব্যাপক হারে অ্যান্টিবডি শনাক্তকরণ ফলাফল মোটামুটি নির্ভুল হওয়া উচিত। দ্বিতীয় আরেকটি গবেষণার নেতৃত্ব দিয়েছেন যুক্তরাজ্যের মেডিকেল রিসার্চ কাউন্সিলের (এমআরসি) ইমিউনোলজিস্ট তাও দং। তিনি টি-সেল শনাক্তকরণে কাজ করেছেন। টি-সেল শনাক্তকরণের প্রক্রিয়া অবশ্য অ্যান্টিবডির মতো এত আলোচিত নয়। তবে সংক্রমণের বিরুদ্ধে লড়াই এবং দীর্ঘমেয়াদি সুরক্ষায় সমান গুরুত্বপূর্ণ ভূমিকা পালন করে। গবেষণাসংক্রান্ত নিবন্ধ প্রকাশিত হয়েছে ‘নেচার ইমিউনোলজি’ সাময়িকীতে। তাঁরা বলছেন, গবেষণার ক্ষেত্রে কোভিড-১৯ মৃদু সংক্রমণের শিকার ২৮ ব্যক্তির রক্তের নমুনা, ১৪ জন গুরুতর অসুস্থ ও ১৬ জন সুস্থ ব্যক্তির রক্তের নমুনা পরীক্ষা করেছেন। গবেষণা নিবন্ধে বলা হয়, সংক্রমিত ব্যক্তিদের ক্ষেত্রে টি-সেলের তীব্র প্রতিক্রিয়া তাঁরা দেখেছেন। এ ক্ষেত্রে মৃদু ও গুরুতর অসুস্থ ব্যক্তিদের ক্ষেত্রে প্রতিক্রিয়ার ভিন্নতা পাওয়া গেছে।`, + ], + ], + ["text-generation", [`আমি রতন এবং আমি`, `তুমি যদি চাও তবে`, `মিথিলা আজকে বড্ড`]], + ["fill-mask", [`আমি বাংলায় গাই।`, `আমি খুব ভালোবাসি। `]], + [ + "question-answering", + [ + { + text: `প্রথম এশিয়া কাপ ক্রিকেট টুর্নামেন্ট কোথায় অনুষ্ঠিত হয় ?`, + context: `প্রথম টুর্নামেন্ট অনুষ্ঠিত হয় ১৯৮৪ সালে সংযুক্ত আরব আমিরাত এর শারজাহ তে যেখানে কাউন্সিলের মূল অফিস ছিল (১৯৯৫ পর্যন্ত)। ভারত শ্রীলঙ্কার সাথে আন্তরিকতাহীন ক্রিকেট সম্পর্কের কারণে ১৯৮৬ সালের টুর্নামেন্ট বর্জন করে। ১৯৯৩ সালে ভারত ও পাকিস্তান এর মধ্যে রাজনৈতিক অস্থিরতার কারণে এটি বাতিল হয়ে যায়। শ্রীলঙ্কা এশিয়া কাপ শুরু থেকে অংশ গ্রহণ করে আসছে। আন্তর্জাতিক ক্রিকেট কাউন্সিল নিয়ম করে দিয়েছে যে এশিয়া কাপের সকল খেলা অনুষ্ঠিত হবে অফিসিয়াল একদিনের আন্তর্জাতিক ক্রিকেট হিসেবে। এসিসি ঘোষনা অনুযায়ী প্রতি দুই বছর পর পর টুর্নামেন্ট অনুষ্ঠিত হয় ২০০৮ সাল থেকে।`, + }, + { + text: `ভারতীয় বাঙালি কথাসাহিত্যিক মহাশ্বেতা দেবীর মৃত্যু কবে হয় ?`, + context: `২০১৬ সালের ২৩ জুলাই হৃদরোগে আক্রান্ত হয়ে মহাশ্বেতা দেবী কলকাতার বেল ভিউ ক্লিনিকে ভর্তি হন। সেই বছরই ২৮ জুলাই একাধিক অঙ্গ বিকল হয়ে তাঁর মৃত্যু ঘটে। তিনি মধুমেহ, সেপ্টিসেমিয়া ও মূত্র সংক্রমণ রোগেও ভুগছিলেন।`, + }, + { + text: `মাস্টারদা সূর্যকুমার সেনের বাবার নাম কী ছিল ?`, + context: `সূর্য সেন ১৮৯৪ সালের ২২ মার্চ চট্টগ্রামের রাউজান থানার নোয়াপাড়ায় অর্থনৈতিক ভাবে অস্বচ্ছল পরিবারে জন্মগ্রহণ করেন। তাঁর পিতার নাম রাজমনি সেন এবং মাতার নাম শশী বালা সেন। রাজমনি সেনের দুই ছেলে আর চার মেয়ে। সূর্য সেন তাঁদের পরিবারের চতুর্থ সন্তান। দুই ছেলের নাম সূর্য ও কমল। চার মেয়ের নাম বরদাসুন্দরী, সাবিত্রী, ভানুমতী ও প্রমিলা। শৈশবে পিতা মাতাকে হারানো সূর্য সেন কাকা গৌরমনি সেনের কাছে মানুষ হয়েছেন। সূর্য সেন ছেলেবেলা থেকেই খুব মনোযোগী ভাল ছাত্র ছিলেন এবং ধর্মভাবাপন্ন গম্ভীর প্রকৃতির ছিলেন।`, + }, + ], + ], + [ + "sentence-similarity", + [ + { + source_sentence: "সে একজন সুখী ব্যক্তি", + sentences: ["সে হ্যাপি কুকুর", "সে খুব সুখী মানুষ", "আজ একটি রৌদ্রোজ্জ্বল দিন"], + }, + ], + ], +]); +const MAPPING_MN = new Map([ + ["text-classification", [`Би чамд хайртай`]], + [ + "token-classification", + [ + `Намайг Дорж гэдэг. Би Улаанбаатарт амьдардаг.`, + `Намайг Ганбат гэдэг. Би Увс аймагт төрсөн.`, + `Манай улс таван хошуу малтай.`, + ], + ], + [ + "question-answering", + [ + { + text: `Та хаана амьдардаг вэ?`, + context: `Намайг Дорж гэдэг. Би Улаанбаатарт амьдардаг.`, + }, + { + text: `Таныг хэн гэдэг вэ?`, + context: `Намайг Дорж гэдэг. Би Улаанбаатарт амьдардаг.`, + }, + { + text: `Миний нэрийг хэн гэдэг вэ?`, + context: `Намайг Ганбат гэдэг. Би Увс аймагт төрсөн.`, + }, + ], + ], + ["translation", [`Намайг Дорж гэдэг. Би Улаанбаатарт амьдардаг.`, `Намайг Ганбат гэдэг. Би Увс аймагт төрсөн.`]], + [ + "summarization", + [ + `Монгол Улс (1992 оноос хойш) — дорно болон төв Азид оршдог бүрэн эрхт улс. Хойд талаараа Орос, бусад талаараа Хятад улстай хиллэдэг далайд гарцгүй орон. Нийслэл — Улаанбаатар хот. Алтайн нуруунаас Хянган, Соёноос Говь хүрсэн 1 сая 566 мянган км2 уудам нутагтай, дэлхийд нутаг дэвсгэрийн хэмжээгээр 19-рт жагсдаг. 2015 оны эхэнд Монгол Улсын хүн ам 3 сая хүрсэн (135-р олон). Үндсэндээ монгол үндэстэн (95 хувь), мөн хасаг, тува хүн байна. 16-р зуунаас хойш буддын шашин, 20-р зуунаас шашингүй байдал дэлгэрсэн ба албан хэрэгт монгол хэлээр харилцана.`, + ], + ], + [ + "text-generation", + [`Намайг Дорж гэдэг. Би`, `Хамгийн сайн дуучин бол`, `Миний дуртай хамтлаг бол`, `Эрт урьдын цагт`], + ], + ["fill-mask", [`Монгол улсын Улаанбаатар хотоос ярьж байна.`, `Миний амьдралын зорилго бол .`]], + [ + "automatic-speech-recognition", + [ + { + label: `Common Voice Train Example`, + src: `https://cdn-media.huggingface.co/common_voice/train/common_voice_mn_18577472.wav`, + }, + { + label: `Common Voice Test Example`, + src: `https://cdn-media.huggingface.co/common_voice/test/common_voice_mn_18577346.wav`, + }, + ], + ], + [ + "text-to-speech", + [ + `Би Монгол улсын иргэн.`, + `Энэхүү жишээ нь цаанаа ямар ч утга агуулаагүй болно`, + `Сар шинэдээ сайхан шинэлэж байна уу?`, + ], + ], + [ + "sentence-similarity", + [ + { + source_sentence: "Энэ бол аз жаргалтай хүн юм", + sentences: ["Энэ бол аз жаргалтай нохой юм", "Энэ бол маш их аз жаргалтай хүн юм", "Өнөөдөр нарлаг өдөр байна"], + }, + ], + ], +]); +const MAPPING_SI = new Map([ + ["translation", [`සිංහල ඉතා අලංකාර භාෂාවකි.`, `මෙම තාක්ෂණය භාවිතා කරන ඔබට ස්තූතියි.`]], + ["fill-mask", [`මම ගෙදර .`, ` ඉගෙනීමට ගියාය.`]], +]); +const MAPPING_DE = new Map([ + [ + "question-answering", + [ + { + text: `Wo wohne ich?`, + context: `Mein Name ist Wolfgang und ich lebe in Berlin`, + }, + { + text: `Welcher Name wird auch verwendet, um den Amazonas-Regenwald auf Englisch zu beschreiben?`, + context: `Der Amazonas-Regenwald, auf Englisch auch als Amazonien oder Amazonas-Dschungel bekannt, ist ein feuchter Laubwald, der den größten Teil des Amazonas-Beckens Südamerikas bedeckt. Dieses Becken umfasst 7.000.000 Quadratkilometer (2.700.000 Quadratmeilen), von denen 5.500.000 Quadratkilometer (2.100.000 Quadratmeilen) vom Regenwald bedeckt sind. Diese Region umfasst Gebiete von neun Nationen. Der größte Teil des Waldes befindet sich in Brasilien mit 60% des Regenwaldes, gefolgt von Peru mit 13%, Kolumbien mit 10% und geringen Mengen in Venezuela, Ecuador, Bolivien, Guyana, Suriname und Französisch-Guayana. Staaten oder Abteilungen in vier Nationen enthalten "Amazonas" in ihren Namen. Der Amazonas repräsentiert mehr als die Hälfte der verbleibenden Regenwälder des Planeten und umfasst den größten und artenreichsten tropischen Regenwald der Welt mit geschätzten 390 Milliarden Einzelbäumen, die in 16.000 Arten unterteilt sind.`, + }, + ], + ], + [ + "sentence-similarity", + [ + { + source_sentence: "Das ist eine glückliche Person", + sentences: [ + "Das ist ein glücklicher Hund", + "Das ist eine sehr glückliche Person", + "Heute ist ein sonniger Tag", + ], + }, + ], + ], +]); +const MAPPING_DV = new Map([ + ["text-classification", [`އަހަރެން ގަޔާވޭ. އަހަރެން ލޯބިވޭ`]], + [ + "token-classification", + [`އަހަރެންގެ ނަމަކީ އަހުމަދު އަދި އަހަރެން ދިރިއުޅެނީ މާލޭގަ`, `އަހަރެންގެ ނަމަކީ ސާރާ އަދި އަހަރެން ދިރިއުޅެނީ އުތީމުގަ`, `އަހަރެންގެ ނަމަކީ އައިޝާ އަދި އަހަރެން ދިރިއުޅެނީ ފޭދޫ، އައްޑޫގަ`], + ], + [ + "question-answering", + [ + { + text: `އަހަރެން ދިރިއުޅެނީ ކޮންތާކު؟`, + context: `އަހަރެންގެ ނަމަކީ އަހުމަދު އަދި އަހަރެން ދިރިއުޅެނީ މާލޭގަ`, + }, + { + text: `އަހަރެން ދިރިއުޅެނީ ކޮންތާކު؟`, + context: `އަހަރެންގެ ނަމަކީ ސާރާ އަދި އަހަރެން ދިރިއުޅެނީ އުތީމުގަ`, + }, + { + text: `އަހަރެންގެ ނަމަކީ ކޮބާ؟`, + context: `އަހަރެންގެ ނަމަކީ އައިޝާ އަދި އަހަރެން ދިރިއުޅެނީ ފޭދޫގަ`, + }, + { + text: `އެމޭޒަން ރެއިންފޮރެސްޓް ސިފަކޮށްދިނުމަށް އިނގިރޭސި ބަހުން ބޭނުންކުރާނީ ކޮންނަމެއް؟`, + context: `އެމޭޒަން ރެއިންފޮރެސްޓް (ޕޯޗުޖީޒް: ފްލޮރެސްޓާ އެމަސޮނިކާ ނުވަތަ އެމަސޮނިއާ؛ ސްޕެނިޝް: ސެލްވާ އެމަސޮނިކާ, އެމަސޮނިއާ ނޫނީ އާންމުކޮށް އެމަޒޯނިއާ؛ ފްރެންޗް: ފޮރޭ އެމެޒޮނިއެން؛ ޑަޗް: އެމެޒޯންރޭގެވައުޑް)، އިގިރޭސި ބަހުން ބުނާ އެމެޒޯނިއާ ނުވަތަ ދަ އެމޭޒަން ޖަންގަލް އަކީ, ސައުތު އެމެރިކާގެ އެމޭޒަން ބޭސިން ސަރަހައްދުގެ ބޮޑުބައެއްގައި ހިމެނޭ މޮއިސްޓް ބޮރޯޑްލީފް ފޮރެސްޓެއެކެވެ. އެމޭޒަން ބޭސިން ސަރަހައްދުގެ ބޮޑު މިނަކީ 7 މިލިއަން އަކަ ކިލޯމީޓަރ (2.7 މިލިއަން އަކަ މައިލް(. މީގެ ތެރެއިން 5.5 މިލިއަން އަކަ ކިލޯމީޓަރ (2.1 މިލިއަން އަކަ މައިލް) އަކީ މި ފޮރެސްޓެވެ. މި ސަރަހައްދުގައި 9 ގައުމަކަށް ނިސްބަތްވާ ޓެރިޓަރީ ހިމެނެއެވެ. 60% އާއިއެކެ އެންމެ ބޮޑު ބައެއް ނިސްބަތްވަނީ ބްރެޒިލްއަށެވެ. އޭގެ ފަހުތުން 13% އާއެކު ޕެރޫ އާއި 10% އާއެކު ކޮލަމްބިއާ އަދި ކުޑަ ބައެއް ހިމެނޭ ގޮތުން ވެނެޒުއެލާ, އެކްއަޑޯ, ބޮލިވިއާ, ގުޔާނާ, ސުރިނާމް އަދި ފްރެންޗް ގްއާނާ އަށް ވެސް ނިސްބަތްވެއެވެ. މީގެ ތެރެއިން 4 ގައުމެއްގައި "އެމެޒޮނާސް" ހިމަނައިގެން ސްޓޭޓް ނުވަތަ ޑިޕާޓްމަންޓް އަކަށް ނަންދީފައިވެއެވެ. މުޅި ދުނިޔޭގައި ބާކީ ހުރި ރެއިންފޮރެސްޓްގެ ތެރެއިން ދެބައިކުޅަ އެއްބަޔަށްވުރެބޮޑުވަރެއް އެމޭޒޮން ރެއިންފޮރެސްޓް ހިއްސާކުރެއެވެ. މިއީ މުޅި ދުނިޔެއިން އެންމޮ ބޮޑު އަދި އެންމެ ބައޮޑައިވަރސް ރެއިންފޮރެސްޓް ޓްރެކްޓެވެ. ލަފާކުރެވޭ ގޮތުން 16 ހާސް ސްޕީޝީސްއަށް ބެހިގެންވާ 390 މިލިއަން ވައްތަރުގެ ގަސް މިތާގައި ހިމެނެއެވެ`, + }, + ], + ], + ["translation", [`އަހަރެންގެ ނަމަކީ އަހުމަދު އަދި އަހަރެން ދިރިއުޅެނީ މާލޭގަ`, `އަހަރެންގެ ނަމަކީ ސާރާ އަދި އަހަރެން ދިރިއުޅެނީ އުތީމުގަ`]], + [ + "summarization", + [ + `ޓަވަރުގެ އުސްމިނަކީ 324 މީޓަރު، އެއީ ގާތްގަނޑަކަށް 81 ބުރީގެ އިމާރާތަކާއި އެއްވަރެވެ. އެއީ ޕެރިސްގައި ހުރި އެންމެ އުސް އިމާރާތެވެ. އޭގެ ހަތަރެސްކަނަށް ހުރި ބުޑުގެ ދިގުމިނަކީ ކޮންމެ ފަރާތަކުން 125 މީޓަރެވެ. (410 ފޫޓު) އައިފިލް ޓަވަރު ބިނާކުރި އިރު، ވޮޝިންގްޓަން މޮނިއުމެންޓްގެ އުސްމިން ފަހަނައަޅާ ގޮސް، ދުނިޔޭގައި މީހުން އުފެއްދި ތަންތަނުގެ ތެރެއިން އެންމެ އުސް ތަނުގެ ލަގަބު ލިބުނެވެ. އަދި 1930 ގައި ނިއު ޔޯކްގެ ކްރައިސްލަރ ބިލްޑިންގް ބިނާކުރުމާއި ހަމައަށް 41 އަހަރު ވަންދެން މިލަގަބު ހިފެހެއްޓިއެވެ. މިއީ 300 މީޓަރަށް ވުރެ އުސްކޮށް އިމާރާތްކުރެވުނު ފުރަތަމަ ތަނެވެ. 1957 ގައި ޓަވަރުގެ އެންމެ މަތީގައި ހަރުކުރެވުނު ބްރޯޑްކާސްޓިންގ އޭރިއަލްގެ ސަބަބުން މިހާރު މި ޓަވަރު ކްރައިސްލަރ ބިލްޑިންގއަށް ވުރެ 5.2 މީޓަރ (17 ފޫޓު) އުހެވެ. މި ޓްރާންސްމިޓަރު ނުލާ، އައިފިލް ޓަވަރަކީ، މިލާއު ވިއާޑަކްޓަށް ފަހު ފްރާންސްގައި ހުރި 2 ވަނައަށް އެންމެ އުސް ފްރީސްޓޭންޑިންގ އިމާރާތެވެ`, + ], + ], + [ + "text-generation", + [`އަހަރެންގެ ނަމަކީ ޔޫސުފް އަދި އަހަރެންގެ މައިގަނޑު`, `އަހަރެންގެ ނަމަކީ މަރިއަމް، އަހަރެން އެންމެ ގަޔާވާ`, `އަހަރެންގެ ނަމަކީ ފާތުމަތު އަދި އަހަރެން`, `،އެއް ޒަމާނެއްގައި`], + ], + ["fill-mask", [`. މާލެ އަކީ ދިވެހިރާއްޖޭގެ`, `ގަރުދިޔައަކީ ދިވެހިންގެ މެދުގައި ކެއުމެއް.`]], +]); +export const MAPPING_DEFAULT_WIDGET = new Map([ + ["en", MAPPING_EN], + ["zh", MAPPING_ZH], + ["fr", MAPPING_FR], + ["es", MAPPING_ES], + ["ru", MAPPING_RU], + ["uk", MAPPING_UK], + ["it", MAPPING_IT], + ["fa", MAPPING_FA], + ["ar", MAPPING_AR], + ["bn", MAPPING_BN], + ["mn", MAPPING_MN], + ["si", MAPPING_SI], + ["de", MAPPING_DE], + ["dv", MAPPING_DV], +]); diff --git a/node_modules/@huggingface/tasks/dist/esm/eval.d.ts b/node_modules/@huggingface/tasks/dist/esm/eval.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..1fb6f2217d4d13662bcede9660978a954883346c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/eval.d.ts @@ -0,0 +1,146 @@ +/** + * List of supported Evaluation Frameworks supported in the `eval.yaml` file in benchmarks datasets. + */ +export declare const EVALUATION_FRAMEWORKS: { + readonly exgentic: { + readonly name: "exgentic"; + readonly description: "Exgentic is an open evaluation framework for general-purpose AI agents across diverse domains and benchmarks."; + readonly url: "https://github.com/Exgentic/exgentic"; + }; + readonly "inspect-ai": { + readonly name: "inspect-ai"; + readonly description: "Inspect AI is an open-source framework for large language model evaluations."; + readonly url: "https://inspect.aisi.org.uk/"; + }; + readonly "math-arena": { + readonly name: "math-arena"; + readonly description: "MathArena is a platform for evaluation of LLMs on latest math competitions and olympiads."; + readonly url: "https://github.com/eth-sri/matharena"; + }; + readonly mteb: { + readonly name: "mteb"; + readonly description: "Multimodal toolbox for evaluating embeddings and retrieval systems."; + readonly url: "https://github.com/embeddings-benchmark/mteb"; + }; + readonly "olmocr-bench": { + readonly name: "olmocr-bench"; + readonly description: "olmOCR-Bench is a framework for evaluating document-level OCR of various tools."; + readonly url: "https://github.com/allenai/olmocr/tree/main/olmocr/bench"; + }; + readonly harbor: { + readonly name: "harbor"; + readonly description: "Harbor is a framework for evaluating and optimizing agents and language models."; + readonly url: "https://github.com/laude-institute/harbor"; + }; + readonly ifstruct: { + readonly name: "ifstruct"; + readonly description: "IFStruct is a benchmark for structured-output compliance: whether a model produces valid JSON/YAML that follows a requested schema, scored without constrained decoding."; + readonly url: "https://github.com/Liquid4All/ifstruct"; + }; + readonly pier: { + readonly name: "pier"; + readonly description: "Pier is a Harbor fork built for DeepSWE, with stronger support for CLI agents in no-internet tasks and more faithful, consistent agent trajectories."; + readonly url: "https://github.com/datacurve-ai/pier"; + }; + readonly "redline-bench": { + readonly name: "redline-bench"; + readonly description: "RedlineBench measures multi-turn contract redlining: agents produce tracked-change .docx edits that are graded against attorney-authored weighted rubrics by an LLM judge panel across five dimensions. Report: https://intelligence.crosby.ai/benchmark/"; + readonly url: "https://github.com/crosbylegal/redline-bench"; + }; + readonly archipelago: { + readonly name: "archipelago"; + readonly description: "Archipelago is a system for running and evaluating AI agents against MCP applications."; + readonly url: "https://github.com/Mercor-Intelligence/archipelago"; + }; + readonly benchflow: { + readonly name: "benchflow"; + readonly description: "BenchFlow is an evaluation framework for AI agents on professional, skill-aware workflows. It powers SkillsBench and runs containerized agent trials with paired with-skills / without-skills configurations."; + readonly url: "https://github.com/benchflow-ai/benchflow"; + }; + readonly "apex-evals": { + readonly name: "apex-evals"; + readonly description: "APEX Evals is a benchmark suite and evaluation harness for evaluating large language models."; + readonly url: "https://github.com/Mercor-Intelligence/apex-evals"; + }; + readonly "screenspot-pro": { + readonly name: "screenspot-pro"; + readonly description: "ScreenSpot-Pro is a GUI grounding benchmark designed to evaluate how well AI agents can locate and identify UI elements across professional software applications in high-resolution screenshots, covering 1,585 annotated images from 26 professional tools."; + readonly url: "https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding"; + }; + readonly "swe-bench": { + readonly name: "swe-bench"; + readonly description: "SWE Bench is a framework for evaluating the performance of LLMs on software engineering tasks."; + readonly url: "https://github.com/swe-bench/swe-bench"; + }; + readonly "swe-bench-pro": { + readonly name: "swe-bench-pro"; + readonly description: "SWE-Bench Pro is a challenging benchmark evaluating LLMs/Agents on long-horizon software engineering tasks."; + readonly url: "https://github.com/scaleapi/SWE-bench_Pro-os"; + }; + readonly "nemo-evaluator": { + readonly name: "nemo-evaluator"; + readonly description: "NeMo Evaluator is an open-source platform for robust, reproducible, and scalable evaluation of Large Language Models across 100+ benchmarks."; + readonly url: "https://github.com/NVIDIA-NeMo/Evaluator"; + }; + readonly "yc-bench": { + readonly name: "yc-bench"; + readonly description: "YC Bench is a long-horizon deterministic benchmark for LLM agents. The agent plays CEO of an AI startup over a simulated 1–3 year run."; + readonly url: "https://github.com/collinear-ai/yc-bench"; + }; + readonly "open-asr-leaderboard": { + readonly name: "open-asr-leaderboard"; + readonly description: "The Open ASR Leaderboard ranks and evaluates speech recognition models."; + readonly url: "https://github.com/huggingface/open_asr_leaderboard"; + }; + readonly mdpbench: { + readonly name: "mdpbench"; + readonly description: "MDPBench is a benchmark for evaluating multilingual document parsing across digital, photographed, Latin, and non-Latin document subsets."; + readonly url: "https://github.com/Yuliang-Liu/MultimodalOCR"; + }; + readonly parsebench: { + readonly name: "parsebench"; + readonly description: "ParseBench is a benchmark for evaluating document parsing systems on real-world enterprise documents across tables, charts, content faithfulness, semantic formatting, and visual grounding."; + readonly url: "https://github.com/run-llama/ParseBench"; + }; + readonly "video-mme-v2": { + readonly name: "video-mme-v2"; + readonly description: "Video-MME-v2 is a benchmark for evaluating the next stage of video understanding capabilities of multimodal large language models."; + readonly url: "https://github.com/MME-Benchmarks/Video-MME-v2"; + }; + readonly "claw-eval": { + readonly name: "claw-eval"; + readonly description: "CLAW-Eval is an evaluation framework for assessing LLMs as autonomous agents across 300 human-verified tasks covering communication, finance, and productivity domains."; + readonly url: "https://github.com/claw-eval/claw-eval"; + }; + readonly researchclawbench: { + readonly name: "researchclawbench"; + readonly description: "ResearchClawBench is a benchmark for evaluating AI agents on end-to-end scientific research tasks, from reading data and related work to producing code, figures, and publication-style reports."; + readonly url: "https://github.com/InternScience/ResearchClawBench"; + }; + readonly pbench: { + readonly name: "pbench"; + readonly description: "PBench is a multi-level referring expression segmentation benchmark for evaluating vision-language perception across a structured hierarchy of skills."; + readonly url: "https://github.com/tiiuae/Falcon-Perception"; + }; + readonly wildclawbench: { + readonly name: "wildclawbench"; + readonly description: "WildClawBench is an in-the-wild benchmark for evaluating AI agents in the OpenClaw environment across 60 hand-built, end-to-end tasks spanning productivity, code intelligence, social interaction, search, creative synthesis, and safety domains."; + readonly url: "https://github.com/InternLM/WildClawBench"; + }; + readonly wbench: { + readonly name: "wbench"; + readonly description: "WBench is a comprehensive multi-turn benchmark for interactive video world model evaluation, assessing models across 5 dimensions (video quality, setting adherence, interaction adherence, consistency, physics compliance) and 22 metrics over 289 multi-turn interaction cases."; + readonly url: "https://github.com/meituan-longcat/WBench"; + }; + readonly nanofold: { + readonly name: "nanofold"; + readonly description: "nanoFold is a data-efficiency benchmark for protein structure prediction. Its goal is to evaluate models on scenarios with scarce data."; + readonly url: "https://github.com/ChrisHayduk/nanoFold-Competition"; + }; + readonly mmmu: { + readonly name: "mmmu"; + readonly description: "MMMU is a new benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning."; + readonly url: "https://mmmu-benchmark.github.io/"; + }; +}; +//# sourceMappingURL=eval.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/eval.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/eval.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d58ccab984e6b65388afae7024098283d1bbce64 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/eval.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/eval.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgKxB,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/eval.js b/node_modules/@huggingface/tasks/dist/esm/eval.js new file mode 100644 index 0000000000000000000000000000000000000000..7ec3e40cb9bd4a9b600e4ae1f2807f67695163a1 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/eval.js @@ -0,0 +1,145 @@ +/** + * List of supported Evaluation Frameworks supported in the `eval.yaml` file in benchmarks datasets. + */ +export const EVALUATION_FRAMEWORKS = { + exgentic: { + name: "exgentic", + description: "Exgentic is an open evaluation framework for general-purpose AI agents across diverse domains and benchmarks.", + url: "https://github.com/Exgentic/exgentic", + }, + "inspect-ai": { + name: "inspect-ai", + description: "Inspect AI is an open-source framework for large language model evaluations.", + url: "https://inspect.aisi.org.uk/", + }, + "math-arena": { + name: "math-arena", + description: "MathArena is a platform for evaluation of LLMs on latest math competitions and olympiads.", + url: "https://github.com/eth-sri/matharena", + }, + mteb: { + name: "mteb", + description: "Multimodal toolbox for evaluating embeddings and retrieval systems.", + url: "https://github.com/embeddings-benchmark/mteb", + }, + "olmocr-bench": { + name: "olmocr-bench", + description: "olmOCR-Bench is a framework for evaluating document-level OCR of various tools.", + url: "https://github.com/allenai/olmocr/tree/main/olmocr/bench", + }, + harbor: { + name: "harbor", + description: "Harbor is a framework for evaluating and optimizing agents and language models.", + url: "https://github.com/laude-institute/harbor", + }, + ifstruct: { + name: "ifstruct", + description: "IFStruct is a benchmark for structured-output compliance: whether a model produces valid JSON/YAML that follows a requested schema, scored without constrained decoding.", + url: "https://github.com/Liquid4All/ifstruct", + }, + pier: { + name: "pier", + description: "Pier is a Harbor fork built for DeepSWE, with stronger support for CLI agents in no-internet tasks and more faithful, consistent agent trajectories.", + url: "https://github.com/datacurve-ai/pier", + }, + "redline-bench": { + name: "redline-bench", + description: "RedlineBench measures multi-turn contract redlining: agents produce tracked-change .docx edits that are graded against attorney-authored weighted rubrics by an LLM judge panel across five dimensions. Report: https://intelligence.crosby.ai/benchmark/", + url: "https://github.com/crosbylegal/redline-bench", + }, + archipelago: { + name: "archipelago", + description: "Archipelago is a system for running and evaluating AI agents against MCP applications.", + url: "https://github.com/Mercor-Intelligence/archipelago", + }, + benchflow: { + name: "benchflow", + description: "BenchFlow is an evaluation framework for AI agents on professional, skill-aware workflows. It powers SkillsBench and runs containerized agent trials with paired with-skills / without-skills configurations.", + url: "https://github.com/benchflow-ai/benchflow", + }, + "apex-evals": { + name: "apex-evals", + description: "APEX Evals is a benchmark suite and evaluation harness for evaluating large language models.", + url: "https://github.com/Mercor-Intelligence/apex-evals", + }, + "screenspot-pro": { + name: "screenspot-pro", + description: "ScreenSpot-Pro is a GUI grounding benchmark designed to evaluate how well AI agents can locate and identify UI elements across professional software applications in high-resolution screenshots, covering 1,585 annotated images from 26 professional tools.", + url: "https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding", + }, + "swe-bench": { + name: "swe-bench", + description: "SWE Bench is a framework for evaluating the performance of LLMs on software engineering tasks.", + url: "https://github.com/swe-bench/swe-bench", + }, + "swe-bench-pro": { + name: "swe-bench-pro", + description: "SWE-Bench Pro is a challenging benchmark evaluating LLMs/Agents on long-horizon software engineering tasks.", + url: "https://github.com/scaleapi/SWE-bench_Pro-os", + }, + "nemo-evaluator": { + name: "nemo-evaluator", + description: "NeMo Evaluator is an open-source platform for robust, reproducible, and scalable evaluation of Large Language Models across 100+ benchmarks.", + url: "https://github.com/NVIDIA-NeMo/Evaluator", + }, + "yc-bench": { + name: "yc-bench", + description: "YC Bench is a long-horizon deterministic benchmark for LLM agents. The agent plays CEO of an AI startup over a simulated 1–3 year run.", + url: "https://github.com/collinear-ai/yc-bench", + }, + "open-asr-leaderboard": { + name: "open-asr-leaderboard", + description: "The Open ASR Leaderboard ranks and evaluates speech recognition models.", + url: "https://github.com/huggingface/open_asr_leaderboard", + }, + mdpbench: { + name: "mdpbench", + description: "MDPBench is a benchmark for evaluating multilingual document parsing across digital, photographed, Latin, and non-Latin document subsets.", + url: "https://github.com/Yuliang-Liu/MultimodalOCR", + }, + parsebench: { + name: "parsebench", + description: "ParseBench is a benchmark for evaluating document parsing systems on real-world enterprise documents across tables, charts, content faithfulness, semantic formatting, and visual grounding.", + url: "https://github.com/run-llama/ParseBench", + }, + "video-mme-v2": { + name: "video-mme-v2", + description: "Video-MME-v2 is a benchmark for evaluating the next stage of video understanding capabilities of multimodal large language models.", + url: "https://github.com/MME-Benchmarks/Video-MME-v2", + }, + "claw-eval": { + name: "claw-eval", + description: "CLAW-Eval is an evaluation framework for assessing LLMs as autonomous agents across 300 human-verified tasks covering communication, finance, and productivity domains.", + url: "https://github.com/claw-eval/claw-eval", + }, + researchclawbench: { + name: "researchclawbench", + description: "ResearchClawBench is a benchmark for evaluating AI agents on end-to-end scientific research tasks, from reading data and related work to producing code, figures, and publication-style reports.", + url: "https://github.com/InternScience/ResearchClawBench", + }, + pbench: { + name: "pbench", + description: "PBench is a multi-level referring expression segmentation benchmark for evaluating vision-language perception across a structured hierarchy of skills.", + url: "https://github.com/tiiuae/Falcon-Perception", + }, + wildclawbench: { + name: "wildclawbench", + description: "WildClawBench is an in-the-wild benchmark for evaluating AI agents in the OpenClaw environment across 60 hand-built, end-to-end tasks spanning productivity, code intelligence, social interaction, search, creative synthesis, and safety domains.", + url: "https://github.com/InternLM/WildClawBench", + }, + wbench: { + name: "wbench", + description: "WBench is a comprehensive multi-turn benchmark for interactive video world model evaluation, assessing models across 5 dimensions (video quality, setting adherence, interaction adherence, consistency, physics compliance) and 22 metrics over 289 multi-turn interaction cases.", + url: "https://github.com/meituan-longcat/WBench", + }, + nanofold: { + name: "nanofold", + description: "nanoFold is a data-efficiency benchmark for protein structure prediction. Its goal is to evaluate models on scenarios with scarce data.", + url: "https://github.com/ChrisHayduk/nanoFold-Competition", + }, + mmmu: { + name: "mmmu", + description: "MMMU is a new benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning.", + url: "https://mmmu-benchmark.github.io/", + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/esm/gguf.d.ts b/node_modules/@huggingface/tasks/dist/esm/gguf.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ddf4be14d547aabeae88ee0b398e2e950b020038 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/gguf.d.ts @@ -0,0 +1,91 @@ +export declare enum GGMLFileQuantizationType { + F32 = 0, + F16 = 1, + Q4_0 = 2, + Q4_1 = 3, + Q4_1_SOME_F16 = 4, + Q4_2 = 5, + Q4_3 = 6, + Q8_0 = 7, + Q5_0 = 8, + Q5_1 = 9, + Q2_K = 10, + Q3_K_S = 11, + Q3_K_M = 12, + Q3_K_L = 13, + Q4_K_S = 14, + Q4_K_M = 15, + Q5_K_S = 16, + Q5_K_M = 17, + Q6_K = 18, + IQ2_XXS = 19, + IQ2_XS = 20, + Q2_K_S = 21, + IQ3_XS = 22, + IQ3_XXS = 23, + IQ1_S = 24, + IQ4_NL = 25, + IQ3_S = 26, + IQ3_M = 27, + IQ2_S = 28, + IQ2_M = 29, + IQ4_XS = 30, + IQ1_M = 31, + BF16 = 32, + Q4_0_4_4 = 33, + Q4_0_4_8 = 34, + Q4_0_8_8 = 35, + TQ1_0 = 36, + TQ2_0 = 37, + MXFP4_MOE = 38, + NVFP4 = 39, + Q1_0 = 40, + Q2_K_XL = 1000, + Q3_K_XL = 1001, + Q4_K_XL = 1002, + Q5_K_XL = 1003, + Q6_K_XL = 1004, + Q8_K_XL = 1005 +} +export declare const GGUF_QUANT_RE: RegExp; +export declare const GGUF_QUANT_RE_GLOBAL: RegExp; +export declare function parseGGUFQuantLabel(fname: string): string | undefined; +export declare const GGUF_QUANT_ORDER: GGMLFileQuantizationType[]; +export declare function findNearestQuantType(quant: GGMLFileQuantizationType, availableQuants: GGMLFileQuantizationType[]): GGMLFileQuantizationType | undefined; +export declare enum GGMLQuantizationType { + F32 = 0, + F16 = 1, + Q4_0 = 2, + Q4_1 = 3, + Q5_0 = 6, + Q5_1 = 7, + Q8_0 = 8, + Q8_1 = 9, + Q2_K = 10, + Q3_K = 11, + Q4_K = 12, + Q5_K = 13, + Q6_K = 14, + Q8_K = 15, + IQ2_XXS = 16, + IQ2_XS = 17, + IQ3_XXS = 18, + IQ1_S = 19, + IQ4_NL = 20, + IQ3_S = 21, + IQ2_S = 22, + IQ4_XS = 23, + I8 = 24, + I16 = 25, + I32 = 26, + I64 = 27, + F64 = 28, + IQ1_M = 29, + BF16 = 30, + TQ1_0 = 34, + TQ2_0 = 35, + MXFP4 = 39, + NVFP4 = 40, + Q1_0 = 41 +} +//# sourceMappingURL=gguf.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/gguf.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/gguf.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..76c02d0778cbcc9723a71a01fcad76e6b5944465 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/gguf.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"gguf.d.ts","sourceRoot":"","sources":["../../src/gguf.ts"],"names":[],"mappings":"AAGA,oBAAY,wBAAwB;IACnC,GAAG,IAAI;IACP,GAAG,IAAI;IACP,IAAI,IAAI;IACR,IAAI,IAAI;IACR,aAAa,IAAI;IACjB,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,KAAK;IACT,MAAM,KAAK;IACX,MAAM,KAAK;IACX,MAAM,KAAK;IACX,MAAM,KAAK;IACX,MAAM,KAAK;IACX,MAAM,KAAK;IACX,MAAM,KAAK;IACX,IAAI,KAAK;IACT,OAAO,KAAK;IACZ,MAAM,KAAK;IACX,MAAM,KAAK;IACX,MAAM,KAAK;IACX,OAAO,KAAK;IACZ,KAAK,KAAK;IACV,MAAM,KAAK;IACX,KAAK,KAAK;IACV,KAAK,KAAK;IACV,KAAK,KAAK;IACV,KAAK,KAAK;IACV,MAAM,KAAK;IACX,KAAK,KAAK;IACV,IAAI,KAAK;IACT,QAAQ,KAAK;IACb,QAAQ,KAAK;IACb,QAAQ,KAAK;IACb,KAAK,KAAK;IACV,KAAK,KAAK;IACV,SAAS,KAAK;IACd,KAAK,KAAK;IACV,IAAI,KAAK;IAIT,OAAO,OAAO;IACd,OAAO,OAAO;IACd,OAAO,OAAO;IACd,OAAO,OAAO;IACd,OAAO,OAAO;IACd,OAAO,OAAO;CACd;AAGD,eAAO,MAAM,aAAa,QAEzB,CAAC;AACF,eAAO,MAAM,oBAAoB,QAAiC,CAAC;AAEnE,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAGrE;AAKD,eAAO,MAAM,gBAAgB,EAAE,wBAAwB,EA4DtD,CAAC;AAIF,wBAAgB,oBAAoB,CACnC,KAAK,EAAE,wBAAwB,EAC/B,eAAe,EAAE,wBAAwB,EAAE,GACzC,wBAAwB,GAAG,SAAS,CAmCtC;AAGD,oBAAY,oBAAoB;IAC/B,GAAG,IAAI;IACP,GAAG,IAAI;IACP,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,IAAI;IACR,IAAI,KAAK;IACT,IAAI,KAAK;IACT,IAAI,KAAK;IACT,IAAI,KAAK;IACT,IAAI,KAAK;IACT,IAAI,KAAK;IACT,OAAO,KAAK;IACZ,MAAM,KAAK;IACX,OAAO,KAAK;IACZ,KAAK,KAAK;IACV,MAAM,KAAK;IACX,KAAK,KAAK;IACV,KAAK,KAAK;IACV,MAAM,KAAK;IACX,EAAE,KAAK;IACP,GAAG,KAAK;IACR,GAAG,KAAK;IACR,GAAG,KAAK;IACR,GAAG,KAAK;IACR,KAAK,KAAK;IACV,IAAI,KAAK;IACT,KAAK,KAAK;IACV,KAAK,KAAK;IACV,KAAK,KAAK;IACV,KAAK,KAAK;IACV,IAAI,KAAK;CACT"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/gguf.js b/node_modules/@huggingface/tasks/dist/esm/gguf.js new file mode 100644 index 0000000000000000000000000000000000000000..3de558071df7ba5761307857d67b1e5c0835c8e0 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/gguf.js @@ -0,0 +1,191 @@ +// This list is copied from gguf/types.ts, but will all types available (for backward compatibility) +// NOT to be confused with GGMLQuantizationType, a FileQuantization can contain multiple GGMLQuantizationType +// For example, Q4_K_M model can contains Q4_K and Q6_K tensors +export var GGMLFileQuantizationType; +(function (GGMLFileQuantizationType) { + GGMLFileQuantizationType[GGMLFileQuantizationType["F32"] = 0] = "F32"; + GGMLFileQuantizationType[GGMLFileQuantizationType["F16"] = 1] = "F16"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_0"] = 2] = "Q4_0"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_1"] = 3] = "Q4_1"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_1_SOME_F16"] = 4] = "Q4_1_SOME_F16"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_2"] = 5] = "Q4_2"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_3"] = 6] = "Q4_3"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q8_0"] = 7] = "Q8_0"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q5_0"] = 8] = "Q5_0"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q5_1"] = 9] = "Q5_1"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q2_K"] = 10] = "Q2_K"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q3_K_S"] = 11] = "Q3_K_S"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q3_K_M"] = 12] = "Q3_K_M"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q3_K_L"] = 13] = "Q3_K_L"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_K_S"] = 14] = "Q4_K_S"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_K_M"] = 15] = "Q4_K_M"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q5_K_S"] = 16] = "Q5_K_S"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q5_K_M"] = 17] = "Q5_K_M"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q6_K"] = 18] = "Q6_K"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ2_XXS"] = 19] = "IQ2_XXS"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ2_XS"] = 20] = "IQ2_XS"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q2_K_S"] = 21] = "Q2_K_S"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ3_XS"] = 22] = "IQ3_XS"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ3_XXS"] = 23] = "IQ3_XXS"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ1_S"] = 24] = "IQ1_S"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ4_NL"] = 25] = "IQ4_NL"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ3_S"] = 26] = "IQ3_S"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ3_M"] = 27] = "IQ3_M"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ2_S"] = 28] = "IQ2_S"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ2_M"] = 29] = "IQ2_M"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ4_XS"] = 30] = "IQ4_XS"; + GGMLFileQuantizationType[GGMLFileQuantizationType["IQ1_M"] = 31] = "IQ1_M"; + GGMLFileQuantizationType[GGMLFileQuantizationType["BF16"] = 32] = "BF16"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_0_4_4"] = 33] = "Q4_0_4_4"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_0_4_8"] = 34] = "Q4_0_4_8"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_0_8_8"] = 35] = "Q4_0_8_8"; + GGMLFileQuantizationType[GGMLFileQuantizationType["TQ1_0"] = 36] = "TQ1_0"; + GGMLFileQuantizationType[GGMLFileQuantizationType["TQ2_0"] = 37] = "TQ2_0"; + GGMLFileQuantizationType[GGMLFileQuantizationType["MXFP4_MOE"] = 38] = "MXFP4_MOE"; + GGMLFileQuantizationType[GGMLFileQuantizationType["NVFP4"] = 39] = "NVFP4"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q1_0"] = 40] = "Q1_0"; + // custom quants used by unsloth + // they are not officially a scheme enum value in GGUF, but only here for naming + GGMLFileQuantizationType[GGMLFileQuantizationType["Q2_K_XL"] = 1000] = "Q2_K_XL"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q3_K_XL"] = 1001] = "Q3_K_XL"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q4_K_XL"] = 1002] = "Q4_K_XL"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q5_K_XL"] = 1003] = "Q5_K_XL"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q6_K_XL"] = 1004] = "Q6_K_XL"; + GGMLFileQuantizationType[GGMLFileQuantizationType["Q8_K_XL"] = 1005] = "Q8_K_XL"; +})(GGMLFileQuantizationType || (GGMLFileQuantizationType = {})); +const ggufQuants = Object.values(GGMLFileQuantizationType).filter((v) => typeof v === "string"); +export const GGUF_QUANT_RE = new RegExp("(?UD-)?" + `(?${ggufQuants.join("|")})` + "(_(?[A-Z]+))?"); +export const GGUF_QUANT_RE_GLOBAL = new RegExp(GGUF_QUANT_RE, "g"); +export function parseGGUFQuantLabel(fname) { + const quantLabel = fname.toUpperCase().match(GGUF_QUANT_RE_GLOBAL)?.at(-1); // if there is multiple quant substrings in a name, we prefer the last one + return quantLabel; +} +// order of quantization, from biggest to smallest +// this list must be in sync with the order in GGMLFileQuantizationType +// the gguf.spec.ts tests are using verify if the order is correct +export const GGUF_QUANT_ORDER = [ + GGMLFileQuantizationType.F32, + GGMLFileQuantizationType.BF16, + GGMLFileQuantizationType.F16, + GGMLFileQuantizationType.Q8_K_XL, + GGMLFileQuantizationType.Q8_0, + // 6-bit quantizations + GGMLFileQuantizationType.Q6_K_XL, + GGMLFileQuantizationType.Q6_K, + // 5-bit quantizations + GGMLFileQuantizationType.Q5_K_XL, + GGMLFileQuantizationType.Q5_K_M, + GGMLFileQuantizationType.Q5_K_S, + GGMLFileQuantizationType.Q5_0, + GGMLFileQuantizationType.Q5_1, + // 4-bit quantizations + GGMLFileQuantizationType.Q4_K_XL, + GGMLFileQuantizationType.Q4_K_M, + GGMLFileQuantizationType.Q4_K_S, + GGMLFileQuantizationType.IQ4_NL, + GGMLFileQuantizationType.IQ4_XS, + GGMLFileQuantizationType.Q4_0_4_4, + GGMLFileQuantizationType.Q4_0_4_8, + GGMLFileQuantizationType.Q4_0_8_8, + GGMLFileQuantizationType.Q4_1_SOME_F16, + GGMLFileQuantizationType.Q4_0, + GGMLFileQuantizationType.Q4_1, + GGMLFileQuantizationType.Q4_2, + GGMLFileQuantizationType.Q4_3, + GGMLFileQuantizationType.MXFP4_MOE, + GGMLFileQuantizationType.NVFP4, + // 3-bit quantizations + GGMLFileQuantizationType.Q3_K_XL, + GGMLFileQuantizationType.Q3_K_L, + GGMLFileQuantizationType.Q3_K_M, + GGMLFileQuantizationType.Q3_K_S, + GGMLFileQuantizationType.IQ3_M, + GGMLFileQuantizationType.IQ3_S, + GGMLFileQuantizationType.IQ3_XS, + GGMLFileQuantizationType.IQ3_XXS, + // 2-bit quantizations + GGMLFileQuantizationType.Q2_K_XL, + GGMLFileQuantizationType.Q2_K, + GGMLFileQuantizationType.Q2_K_S, + GGMLFileQuantizationType.IQ2_M, + GGMLFileQuantizationType.IQ2_S, + GGMLFileQuantizationType.IQ2_XS, + GGMLFileQuantizationType.IQ2_XXS, + // 1-bit quantizations + GGMLFileQuantizationType.IQ1_S, + GGMLFileQuantizationType.IQ1_M, + GGMLFileQuantizationType.TQ1_0, + GGMLFileQuantizationType.TQ2_0, + GGMLFileQuantizationType.Q1_0, +]; +// This function finds the nearest quantization type that is less than or equal to the given quantization type. +// It returns undefined if no such quantization type is found. +export function findNearestQuantType(quant, availableQuants) { + // Create a map for quick index lookup from the defined order + const orderMap = new Map(); + GGUF_QUANT_ORDER.forEach((q, index) => { + orderMap.set(q, index); + }); + const targetIndex = orderMap.get(quant) ?? 0; // the 0 case should never happen + // Filter the available quantizations to include only those defined in the order map, + // then sort them according to the GGUF_QUANT_ORDER (from largest/index 0 to smallest/highest index). + const sortedAvailable = availableQuants + .filter((q) => orderMap.has(q)) + .sort((a, b) => (orderMap.get(a) ?? Infinity) - (orderMap.get(b) ?? Infinity)); + // If no valid quantizations are available after filtering + if (sortedAvailable.length === 0) { + return undefined; + } + // Iterate through the sorted available quantizations (largest to smallest). + // Find the first one whose order index is >= the target index. + // This means finding the largest quantization that is smaller than or equal to the target. + for (const availableQuant of sortedAvailable) { + // We know the key exists due to the filter above. + const availableIndex = orderMap.get(availableQuant) ?? 0; + if (availableIndex >= targetIndex) { + return availableQuant; + } + } + // If the loop completes, it means all available quantizations are larger (have a smaller index) + // than the target quantization. In this case, return the "smallest" available quantization, + // which is the last element in the sorted list (highest index among available). + return sortedAvailable[sortedAvailable.length - 1]; +} +// This list is only used to calculate the size of the model, NOT to be confused with the quantization FILE type +export var GGMLQuantizationType; +(function (GGMLQuantizationType) { + GGMLQuantizationType[GGMLQuantizationType["F32"] = 0] = "F32"; + GGMLQuantizationType[GGMLQuantizationType["F16"] = 1] = "F16"; + GGMLQuantizationType[GGMLQuantizationType["Q4_0"] = 2] = "Q4_0"; + GGMLQuantizationType[GGMLQuantizationType["Q4_1"] = 3] = "Q4_1"; + GGMLQuantizationType[GGMLQuantizationType["Q5_0"] = 6] = "Q5_0"; + GGMLQuantizationType[GGMLQuantizationType["Q5_1"] = 7] = "Q5_1"; + GGMLQuantizationType[GGMLQuantizationType["Q8_0"] = 8] = "Q8_0"; + GGMLQuantizationType[GGMLQuantizationType["Q8_1"] = 9] = "Q8_1"; + GGMLQuantizationType[GGMLQuantizationType["Q2_K"] = 10] = "Q2_K"; + GGMLQuantizationType[GGMLQuantizationType["Q3_K"] = 11] = "Q3_K"; + GGMLQuantizationType[GGMLQuantizationType["Q4_K"] = 12] = "Q4_K"; + GGMLQuantizationType[GGMLQuantizationType["Q5_K"] = 13] = "Q5_K"; + GGMLQuantizationType[GGMLQuantizationType["Q6_K"] = 14] = "Q6_K"; + GGMLQuantizationType[GGMLQuantizationType["Q8_K"] = 15] = "Q8_K"; + GGMLQuantizationType[GGMLQuantizationType["IQ2_XXS"] = 16] = "IQ2_XXS"; + GGMLQuantizationType[GGMLQuantizationType["IQ2_XS"] = 17] = "IQ2_XS"; + GGMLQuantizationType[GGMLQuantizationType["IQ3_XXS"] = 18] = "IQ3_XXS"; + GGMLQuantizationType[GGMLQuantizationType["IQ1_S"] = 19] = "IQ1_S"; + GGMLQuantizationType[GGMLQuantizationType["IQ4_NL"] = 20] = "IQ4_NL"; + GGMLQuantizationType[GGMLQuantizationType["IQ3_S"] = 21] = "IQ3_S"; + GGMLQuantizationType[GGMLQuantizationType["IQ2_S"] = 22] = "IQ2_S"; + GGMLQuantizationType[GGMLQuantizationType["IQ4_XS"] = 23] = "IQ4_XS"; + GGMLQuantizationType[GGMLQuantizationType["I8"] = 24] = "I8"; + GGMLQuantizationType[GGMLQuantizationType["I16"] = 25] = "I16"; + GGMLQuantizationType[GGMLQuantizationType["I32"] = 26] = "I32"; + GGMLQuantizationType[GGMLQuantizationType["I64"] = 27] = "I64"; + GGMLQuantizationType[GGMLQuantizationType["F64"] = 28] = "F64"; + GGMLQuantizationType[GGMLQuantizationType["IQ1_M"] = 29] = "IQ1_M"; + GGMLQuantizationType[GGMLQuantizationType["BF16"] = 30] = "BF16"; + GGMLQuantizationType[GGMLQuantizationType["TQ1_0"] = 34] = "TQ1_0"; + GGMLQuantizationType[GGMLQuantizationType["TQ2_0"] = 35] = "TQ2_0"; + GGMLQuantizationType[GGMLQuantizationType["MXFP4"] = 39] = "MXFP4"; + GGMLQuantizationType[GGMLQuantizationType["NVFP4"] = 40] = "NVFP4"; + GGMLQuantizationType[GGMLQuantizationType["Q1_0"] = 41] = "Q1_0"; +})(GGMLQuantizationType || (GGMLQuantizationType = {})); diff --git a/node_modules/@huggingface/tasks/dist/esm/hardware-amd.d.ts b/node_modules/@huggingface/tasks/dist/esm/hardware-amd.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b134a5211c2b52c3db3a66f9b2d706e3886fb2c1 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/hardware-amd.d.ts @@ -0,0 +1,11 @@ +import type { HardwareSpec } from "./hardware.js"; +export interface AmdGpuHardwareSpec extends HardwareSpec { + /** + * GFX version / LLVM ISA target (AMD GPUs only), e.g. "gfx1100" + * + * potential source https://llvm.org/docs/AMDGPUUsage.html#processors + */ + gfxVersion: string; +} +export declare const AMD_GPU_SKUS: Record; +//# sourceMappingURL=hardware-amd.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/hardware-amd.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/hardware-amd.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..589d94268bc87de1e2aa593fe04f0f64bc29f10e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/hardware-amd.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"hardware-amd.d.ts","sourceRoot":"","sources":["../../src/hardware-amd.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElD,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACvD;;;;OAIG;IACH,UAAU,EAAE,MAAM,CAAC;CACnB;AAID,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAiU3D,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/hardware-amd.js b/node_modules/@huggingface/tasks/dist/esm/hardware-amd.js new file mode 100644 index 0000000000000000000000000000000000000000..625b9122fdb92cfc28f802356eae2bb1777e8e75 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/hardware-amd.js @@ -0,0 +1,323 @@ +const AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS = [16, 24, 32, 48, 64, 96]; +export const AMD_GPU_SKUS = { + MI300: { + tflops: 383.0, + memory: [192], + gfxVersion: "gfx942", + msrp: 15_000, + power: 750, + releaseYear: 2023, + }, + MI250: { + tflops: 362.1, + memory: [128], + gfxVersion: "gfx90a", + msrp: 10_000, + power: 560, + releaseYear: 2021, + }, + MI210: { + tflops: 181.0, + memory: [64], + gfxVersion: "gfx90a", + msrp: 8_000, + power: 300, + releaseYear: 2022, + }, + MI100: { + tflops: 184.6, + memory: [32], + gfxVersion: "gfx908", + msrp: 6_400, + power: 300, + releaseYear: 2020, + }, + MI60: { + tflops: 29.5, + memory: [32], + gfxVersion: "gfx906", + msrp: 3_000, + power: 300, + releaseYear: 2018, + }, + MI50: { + tflops: 26.5, + memory: [16, 32], + gfxVersion: "gfx906", + msrp: 1_800, + power: 300, + releaseYear: 2018, + }, + "R9700 PRO": { + tflops: 95.7, + memory: [32], + gfxVersion: "gfx1201", + msrp: 1_250, + power: 300, + releaseYear: 2025, + }, + "RX 9070 XT": { + tflops: 97.32, + memory: [16], + gfxVersion: "gfx1201", + msrp: 600, + power: 304, + releaseYear: 2025, + }, + "RX 9070": { + tflops: 72.25, + memory: [16], + gfxVersion: "gfx1201", + msrp: 550, + power: 220, + releaseYear: 2025, + }, + "RX 9060 XT": { + tflops: 51.28, + memory: [8, 16], + gfxVersion: "gfx1200", + msrp: 350, + power: 160, + releaseYear: 2025, + }, + "PRO W7900": { + tflops: 122.6, + memory: [48], + gfxVersion: "gfx1100", + msrp: 4_000, + power: 295, + releaseYear: 2023, + }, + "PRO W7800": { + tflops: 90.5, + memory: [32, 48], + gfxVersion: "gfx1100", + msrp: 2_500, + power: 260, + releaseYear: 2023, + }, + "RX 7900 XTX": { + tflops: 122.8, + memory: [24], + gfxVersion: "gfx1100", + msrp: 1_000, + power: 355, + releaseYear: 2022, + }, + "RX 7900 XT": { + tflops: 103.0, + memory: [20], + gfxVersion: "gfx1100", + msrp: 900, + power: 315, + releaseYear: 2022, + }, + "RX 7900 GRE": { + tflops: 91.96, + memory: [16], + gfxVersion: "gfx1100", + msrp: 550, + power: 260, + releaseYear: 2023, + }, + "RX 7800 XT": { + tflops: 74.65, + memory: [16], + gfxVersion: "gfx1101", + msrp: 500, + power: 263, + releaseYear: 2023, + }, + "RX 7700 XT": { + tflops: 70.34, + memory: [12], + gfxVersion: "gfx1101", + msrp: 450, + power: 245, + releaseYear: 2023, + }, + "RX 7600 XT": { + tflops: 45.14, + memory: [16, 8], + gfxVersion: "gfx1102", + msrp: 350, + power: 190, + releaseYear: 2024, + }, + "RX 6950 XT": { + tflops: 47.31, + memory: [16], + gfxVersion: "gfx1030", + msrp: 1_100, + power: 335, + releaseYear: 2022, + }, + "RX 6800": { + tflops: 32.33, + memory: [16], + gfxVersion: "gfx1030", + msrp: 600, + power: 250, + releaseYear: 2020, + }, + "RX 6700 XT": { + tflops: 26.43, + memory: [12], + gfxVersion: "gfx1031", + msrp: 500, + power: 230, + releaseYear: 2021, + }, + "RX 6700": { + tflops: 22.58, + memory: [10], + gfxVersion: "gfx1031", + msrp: 329, + power: 175, + releaseYear: 2022, + }, + "RX 6650 XT": { + tflops: 21.59, + memory: [8], + gfxVersion: "gfx1032", + msrp: 400, + power: 180, + releaseYear: 2022, + }, + "RX 6600 XT": { + tflops: 21.21, + memory: [8], + gfxVersion: "gfx1032", + msrp: 400, + power: 160, + releaseYear: 2021, + }, + "RX 6600": { + tflops: 17.86, + memory: [8], + gfxVersion: "gfx1032", + msrp: 350, + power: 132, + releaseYear: 2021, + }, + "RX 5700 XT": { + tflops: 19.51, + memory: [8], + gfxVersion: "gfx1010", + msrp: 399, + power: 225, + releaseYear: 2019, + }, + "RX 5700": { + tflops: 15.9, + memory: [8], + gfxVersion: "gfx1010", + msrp: 349, + power: 180, + releaseYear: 2019, + }, + "RX 5500 XT": { + tflops: 10.39, + memory: [4, 8], + gfxVersion: "gfx1012", + msrp: 200, + power: 130, + releaseYear: 2019, + }, + "Radeon Pro V620": { + tflops: 40.55, + memory: [32], + gfxVersion: "gfx1030", + msrp: 3_000, + power: 300, + releaseYear: 2021, + }, + "Radeon Pro VII": { + tflops: 26.11, + memory: [16, 32], + gfxVersion: "gfx906", + msrp: 1_900, + power: 250, + releaseYear: 2020, + }, + "Radeon 610M": { + tflops: 0.97, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1037", + msrp: 300, + power: 15, + releaseYear: 2022, + }, + "Radeon 740M": { + tflops: 5.12, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1103", + msrp: 179, + power: 15, + releaseYear: 2023, + }, + "Radeon 760M": { + tflops: 10.65, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1103", + msrp: 229, + power: 15, + releaseYear: 2023, + }, + "Radeon 780M": { + tflops: 16.59, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1103", + msrp: 329, + power: 15, + releaseYear: 2023, + }, + "Radeon 820M": { + tflops: 1.434, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1152", + msrp: 200, + power: 15, + releaseYear: 2025, + }, + "Radeon 840M": { + tflops: 2.97, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1152", + msrp: 250, + power: 15, + releaseYear: 2025, + }, + "Radeon 860M": { + tflops: 6.14, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1152", + msrp: 300, + power: 15, + releaseYear: 2025, + }, + "Radeon 880M": { + tflops: 8.91, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1150", + msrp: 400, + power: 15, + releaseYear: 2024, + }, + "Radeon 890M": { + tflops: 11.88, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1150", + msrp: 450, + power: 15, + releaseYear: 2024, + }, + "Ryzen AI Max+ 395": { + tflops: 29.7, + memory: [64, 96, 128], + gfxVersion: "gfx1151", + msrp: 1_500, + power: 120, + releaseYear: 2025, + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/esm/hardware-nvidia.d.ts b/node_modules/@huggingface/tasks/dist/esm/hardware-nvidia.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..390f423e6b8fb5703a98de527976f81d1f0a5108 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/hardware-nvidia.d.ts @@ -0,0 +1,28 @@ +import type { HardwareSpec } from "./hardware.js"; +export interface NvidiaHardwareSpec extends HardwareSpec { + /** + * CUDA Compute Capability (NVIDIA GPUs only) + * + * potential source https://developer.nvidia.com/cuda/gpus + */ + computeCapability: number; +} +export declare enum NvidiaComputeCapabilities { + BLACKWELL_ULTRA = 12.1, + BLACKWELL_RTX = 12, + BLACKWELL = 10, + HOPPER = 9, + ADA_LOVELACE = 8.9, + ORIN = 8.7, + AMPERE_RTX = 8.6, + AMPERE = 8, + TURING = 7.5, + XAVIER = 7.2, + VOLTA = 7, + PASCAL_TEGRA = 6.2, + PASCAL = 6.1, + PASCAL_DATACENTER = 6, + MAXWELL = 5.3 +} +export declare const NVIDIA_SKUS: Record; +//# sourceMappingURL=hardware-nvidia.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/hardware-nvidia.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/hardware-nvidia.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..0df030e7b56b0cbd874df04b301ad6c6c79d51ab --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/hardware-nvidia.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"hardware-nvidia.d.ts","sourceRoot":"","sources":["../../src/hardware-nvidia.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElD,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACvD;;;;OAIG;IACH,iBAAiB,EAAE,MAAM,CAAC;CAC1B;AAED,oBAAY,yBAAyB;IACpC,eAAe,OAAO;IACtB,aAAa,KAAO;IACpB,SAAS,KAAO;IAChB,MAAM,IAAM;IACZ,YAAY,MAAM;IAClB,IAAI,MAAM;IACV,UAAU,MAAM;IAChB,MAAM,IAAM;IACZ,MAAM,MAAM;IACZ,MAAM,MAAM;IACZ,KAAK,IAAM;IACX,YAAY,MAAM;IAClB,MAAM,MAAM;IACZ,iBAAiB,IAAM;IACvB,OAAO,MAAM;CACb;AAED,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAi9B1D,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/hardware-nvidia.js b/node_modules/@huggingface/tasks/dist/esm/hardware-nvidia.js new file mode 100644 index 0000000000000000000000000000000000000000..689e4cac9aeadaa35025df77380da7e50943420c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/hardware-nvidia.js @@ -0,0 +1,996 @@ +export var NvidiaComputeCapabilities; +(function (NvidiaComputeCapabilities) { + NvidiaComputeCapabilities[NvidiaComputeCapabilities["BLACKWELL_ULTRA"] = 12.1] = "BLACKWELL_ULTRA"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["BLACKWELL_RTX"] = 12] = "BLACKWELL_RTX"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["BLACKWELL"] = 10] = "BLACKWELL"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["HOPPER"] = 9] = "HOPPER"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["ADA_LOVELACE"] = 8.9] = "ADA_LOVELACE"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["ORIN"] = 8.7] = "ORIN"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["AMPERE_RTX"] = 8.6] = "AMPERE_RTX"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["AMPERE"] = 8] = "AMPERE"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["TURING"] = 7.5] = "TURING"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["XAVIER"] = 7.2] = "XAVIER"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["VOLTA"] = 7] = "VOLTA"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["PASCAL_TEGRA"] = 6.2] = "PASCAL_TEGRA"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["PASCAL"] = 6.1] = "PASCAL"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["PASCAL_DATACENTER"] = 6] = "PASCAL_DATACENTER"; + NvidiaComputeCapabilities[NvidiaComputeCapabilities["MAXWELL"] = 5.3] = "MAXWELL"; +})(NvidiaComputeCapabilities || (NvidiaComputeCapabilities = {})); +export const NVIDIA_SKUS = { + B300: { + tflops: 1232, + memory: [288], + computeCapability: 10.0, + msrp: 45_000, + power: 1400, + releaseYear: 2026, + }, + B200: { + tflops: 496.6, + memory: [192], + computeCapability: 10.0, + msrp: 40_000, + power: 1000, + releaseYear: 2024, + }, + H200: { + tflops: 241.3, + memory: [141], + computeCapability: 9.0, + msrp: 32_000, + power: 700, + releaseYear: 2024, + }, + H100: { + tflops: 267.6, + memory: [80], + computeCapability: 9.0, + msrp: 30_000, + power: 700, + releaseYear: 2022, + }, + H800: { + tflops: 237.2, + memory: [80], + computeCapability: 9.0, + msrp: 30_000, + power: 700, + releaseYear: 2023, + }, + H20: { + tflops: 148, + memory: [96], + computeCapability: 9.0, + msrp: 13_500, + power: 400, + releaseYear: 2024, + }, + L40s: { + tflops: 91.61, + memory: [48], + computeCapability: 8.9, + msrp: 8_500, + power: 350, + releaseYear: 2023, + }, + L40: { + tflops: 90.52, + memory: [48], + computeCapability: 8.9, + msrp: 7_500, + power: 300, + releaseYear: 2022, + }, + L20: { + tflops: 59.35, + memory: [48], + computeCapability: 8.9, + msrp: 5_000, + power: 275, + releaseYear: 2023, + }, + L4: { + tflops: 30.29, + memory: [24], + computeCapability: 8.9, + msrp: 2_500, + power: 72, + releaseYear: 2023, + }, + GB10: { + tflops: 29.71, + memory: [128], + computeCapability: 12.1, + msrp: 3_999, + power: 140, + releaseYear: 2025, + }, + "RTX PRO 6000 WS": { + tflops: 126, + memory: [96], + computeCapability: 12.0, + msrp: 8_600, + power: 600, + releaseYear: 2025, + }, + "RTX PRO 6000 Max-Q": { + tflops: 116, + memory: [96], + computeCapability: 12.0, + msrp: 8_600, + power: 300, + releaseYear: 2025, + }, + "RTX PRO 5000": { + tflops: 66.94, + memory: [48, 72], + computeCapability: 12.0, + msrp: 4_500, + power: 300, + releaseYear: 2025, + }, + "RTX PRO 4500 WS": { + tflops: 50.53, + memory: [32], + computeCapability: 12.0, + msrp: 2_800, + power: 200, + releaseYear: 2025, + }, + "RTX PRO 4000": { + tflops: 36.83, + memory: [24], + computeCapability: 12.0, + msrp: 1_500, + power: 140, + releaseYear: 2025, + }, + "RTX PRO 4000 SFF": { + tflops: 24.05, + memory: [24], + computeCapability: 12.0, + msrp: 1_500, + power: 70, + releaseYear: 2025, + }, + "RTX PRO 2000": { + tflops: 17.03, + memory: [16], + computeCapability: 12.0, + msrp: 700, + power: 70, + releaseYear: 2025, + }, + "RTX 6000 Ada": { + tflops: 91.1, + memory: [48], + computeCapability: 8.9, + msrp: 6_800, + power: 300, + releaseYear: 2022, + }, + "RTX 5880 Ada": { + tflops: 69.3, + memory: [48], + computeCapability: 8.9, + msrp: 6_000, + power: 285, + releaseYear: 2024, + }, + "RTX 5000 Ada": { + tflops: 65.3, + memory: [32], + computeCapability: 8.9, + msrp: 4_000, + power: 250, + releaseYear: 2023, + }, + "RTX 4500 Ada": { + tflops: 39.6, + memory: [24], + computeCapability: 8.9, + msrp: 2_250, + power: 210, + releaseYear: 2023, + }, + "RTX 4000 Ada": { + tflops: 26.7, + memory: [20], + computeCapability: 8.9, + msrp: 1_250, + power: 130, + releaseYear: 2023, + }, + "RTX 4000 SFF Ada": { + tflops: 19.2, + memory: [20], + computeCapability: 8.9, + msrp: 1_250, + power: 70, + releaseYear: 2023, + }, + "RTX 3500 Ada Mobile": { + tflops: 15.8, + memory: [12], + computeCapability: 8.9, + msrp: 1_500, + power: 150, + releaseYear: 2023, + }, + "RTX 2000 Ada": { + tflops: 12.0, + memory: [16], + computeCapability: 8.9, + msrp: 650, + power: 70, + releaseYear: 2024, + }, + "RTX A6000": { + tflops: 38.7, + memory: [48], + computeCapability: 8.6, + msrp: 4_650, + power: 300, + releaseYear: 2020, + }, + "RTX A5000": { + tflops: 27.77, + memory: [8, 12, 24], + computeCapability: 8.6, + msrp: 2_250, + power: 230, + releaseYear: 2021, + }, + "RTX A5000 Max-Q": { + tflops: 16.59, + memory: [16], + computeCapability: 8.6, + msrp: 2_000, + power: 80, + releaseYear: 2021, + }, + "RTX A5000 Mobile": { + tflops: 19.35, + memory: [16], + computeCapability: 8.6, + msrp: 2_000, + power: 165, + releaseYear: 2021, + }, + "RTX A4000": { + tflops: 19.17, + memory: [16], + computeCapability: 8.6, + msrp: 1_000, + power: 140, + releaseYear: 2021, + }, + "RTX A4000 Max-Q": { + tflops: 14.28, + memory: [8], + computeCapability: 8.6, + msrp: 1_000, + power: 35, + releaseYear: 2021, + }, + "RTX A4000 Mobile": { + tflops: 17.2, + memory: [8], + computeCapability: 8.6, + msrp: 1_000, + power: 80, + releaseYear: 2021, + }, + "RTX A3000 Mobile": { + tflops: 10.9, + memory: [6, 12], + computeCapability: 8.6, + msrp: 700, + power: 80, + releaseYear: 2021, + }, + "RTX A2000": { + tflops: 7.987, + memory: [6, 12], + computeCapability: 8.6, + msrp: 450, + power: 70, + releaseYear: 2021, + }, + "RTX A2000 Embedded": { + tflops: 6.026, + memory: [4], + computeCapability: 8.6, + msrp: 400, + power: 70, + releaseYear: 2022, + }, + "RTX A2000 Max-Q": { + tflops: 6.1, + memory: [4, 8], + computeCapability: 8.6, + msrp: 450, + power: 35, + releaseYear: 2021, + }, + "RTX A2000 Mobile": { + tflops: 8.4, + memory: [4, 8], + computeCapability: 8.6, + msrp: 450, + power: 95, + releaseYear: 2021, + }, + A800: { + tflops: 77.97, + memory: [40, 80], + computeCapability: 8.0, + msrp: 12_000, + power: 400, + releaseYear: 2022, + }, + A100: { + tflops: 77.97, + memory: [80, 40], + computeCapability: 8.0, + msrp: 15_000, + power: 400, + releaseYear: 2020, + }, + A40: { + tflops: 37.42, + memory: [48], + computeCapability: 8.6, + msrp: 5_500, + power: 300, + releaseYear: 2020, + }, + A30: { + tflops: 10.32, + memory: [24], + computeCapability: 8.0, + msrp: 5_000, + power: 165, + releaseYear: 2021, + }, + A10: { + tflops: 31.24, + memory: [24], + computeCapability: 8.6, + msrp: 3_200, + power: 150, + releaseYear: 2021, + }, + A2: { + tflops: 4.531, + memory: [16], + computeCapability: 8.6, + msrp: 1_000, + power: 60, + releaseYear: 2021, + }, + "RTX 5090": { + tflops: 104.8, + memory: [32], + computeCapability: 12.0, + msrp: 2_000, + power: 575, + releaseYear: 2025, + }, + "RTX 5090 D": { + tflops: 104.8, + memory: [32], + computeCapability: 12.0, + msrp: 2_000, + power: 575, + releaseYear: 2025, + }, + "RTX 5090 Mobile": { + tflops: 31.8, + memory: [24], + computeCapability: 12.0, + msrp: 1_500, + power: 175, + releaseYear: 2025, + }, + "RTX 5080": { + tflops: 56.28, + memory: [16], + computeCapability: 12.0, + msrp: 1_000, + power: 360, + releaseYear: 2025, + }, + "RTX 5080 Mobile": { + tflops: 23.04, + memory: [16], + computeCapability: 12.0, + msrp: 1_000, + power: 175, + releaseYear: 2025, + }, + "RTX 5070": { + tflops: 30.84, + memory: [12], + computeCapability: 12.0, + msrp: 550, + power: 250, + releaseYear: 2025, + }, + "RTX 5070 Mobile": { + tflops: 13.13, + memory: [8], + computeCapability: 12.0, + msrp: 500, + power: 100, + releaseYear: 2025, + }, + "RTX 5070 Ti": { + tflops: 43.94, + memory: [16], + computeCapability: 12.0, + msrp: 750, + power: 300, + releaseYear: 2025, + }, + "RTX 5070 Ti Mobile": { + tflops: 17.04, + memory: [12], + computeCapability: 12.0, + msrp: 700, + power: 140, + releaseYear: 2025, + }, + "RTX 5060 Ti": { + tflops: 23.7, + memory: [16, 8], + computeCapability: 12.0, + msrp: 450, + power: 180, + releaseYear: 2025, + }, + "RTX 5060": { + tflops: 19.18, + memory: [8], + computeCapability: 12.0, + msrp: 300, + power: 150, + releaseYear: 2025, + }, + "RTX 5060 Mobile": { + tflops: 9.684, + memory: [8], + computeCapability: 12.0, + msrp: 300, + power: 100, + releaseYear: 2025, + }, + "RTX 5050": { + tflops: 13.17, + memory: [8], + computeCapability: 12.0, + msrp: 249, + power: 130, + releaseYear: 2025, + }, + "RTX 5050 Mobile": { + tflops: 7.7, + memory: [8], + computeCapability: 12.0, + msrp: 250, + power: 100, + releaseYear: 2025, + }, + "RTX 4090": { + tflops: 82.58, + memory: [24], + computeCapability: 8.9, + msrp: 1_600, + power: 450, + releaseYear: 2022, + }, + "RTX 4090D": { + tflops: 79.49, + memory: [24, 48], + computeCapability: 8.9, + msrp: 1_600, + power: 425, + releaseYear: 2023, + }, + "RTX 4090 Mobile": { + tflops: 32.98, + memory: [16], + computeCapability: 8.9, + msrp: 1_500, + power: 150, + releaseYear: 2023, + }, + "RTX 4080 SUPER": { + tflops: 52.2, + memory: [16], + computeCapability: 8.9, + msrp: 1_000, + power: 320, + releaseYear: 2024, + }, + "RTX 4080": { + tflops: 48.7, + memory: [16], + computeCapability: 8.9, + msrp: 1_200, + power: 320, + releaseYear: 2022, + }, + "RTX 4080 Mobile": { + tflops: 24.72, + memory: [12], + computeCapability: 8.9, + msrp: 1_000, + power: 150, + releaseYear: 2023, + }, + "RTX 4070": { + tflops: 29.15, + memory: [12], + computeCapability: 8.9, + msrp: 600, + power: 200, + releaseYear: 2023, + }, + "RTX 4070 Mobile": { + tflops: 15.62, + memory: [8], + computeCapability: 8.9, + msrp: 500, + power: 115, + releaseYear: 2023, + }, + "RTX 4070 Ti": { + tflops: 40.09, + memory: [12], + computeCapability: 8.9, + msrp: 800, + power: 285, + releaseYear: 2023, + }, + "RTX 4070 Super": { + tflops: 35.48, + memory: [12], + computeCapability: 8.9, + msrp: 600, + power: 220, + releaseYear: 2024, + }, + "RTX 4070 Ti Super": { + tflops: 44.1, + memory: [16], + computeCapability: 8.9, + msrp: 800, + power: 285, + releaseYear: 2024, + }, + "RTX 4060": { + tflops: 15.11, + memory: [8], + computeCapability: 8.9, + msrp: 300, + power: 115, + releaseYear: 2023, + }, + "RTX 4060 Ti": { + tflops: 22.06, + memory: [8, 16], + computeCapability: 8.9, + msrp: 500, + power: 165, + releaseYear: 2023, + }, + "RTX 4090 Laptop": { + tflops: 32.98, + memory: [16], + computeCapability: 8.9, + msrp: 1_500, + power: 150, + releaseYear: 2023, + }, + "RTX 4080 Laptop": { + tflops: 24.72, + memory: [12], + computeCapability: 8.9, + msrp: 1_000, + power: 150, + releaseYear: 2023, + }, + "RTX 4070 Laptop": { + tflops: 15.62, + memory: [8], + computeCapability: 8.9, + msrp: 500, + power: 115, + releaseYear: 2023, + }, + "RTX 4060 Laptop": { + tflops: 11.61, + memory: [8], + computeCapability: 8.9, + msrp: 300, + power: 115, + releaseYear: 2023, + }, + "RTX 4050 Laptop": { + tflops: 8.9, + memory: [6], + computeCapability: 8.9, + msrp: 250, + power: 115, + releaseYear: 2023, + }, + "RTX 3090": { + tflops: 35.58, + memory: [24], + computeCapability: 8.6, + msrp: 1_500, + power: 350, + releaseYear: 2020, + }, + "RTX 3090 Ti": { + tflops: 40, + memory: [24], + computeCapability: 8.6, + msrp: 2_000, + power: 450, + releaseYear: 2022, + }, + "RTX 3080": { + tflops: 30.6, + memory: [12, 10], + computeCapability: 8.6, + msrp: 800, + power: 350, + releaseYear: 2020, + }, + "RTX 3080 Ti": { + tflops: 34.1, + memory: [12], + computeCapability: 8.6, + msrp: 1_200, + power: 350, + releaseYear: 2021, + }, + "RTX 3080 Mobile": { + tflops: 18.98, + memory: [16, 8], + computeCapability: 8.6, + msrp: 800, + power: 150, + releaseYear: 2021, + }, + "RTX 3070": { + tflops: 20.31, + memory: [8], + computeCapability: 8.6, + msrp: 500, + power: 220, + releaseYear: 2020, + }, + "RTX 3070 Ti": { + tflops: 21.75, + memory: [8], + computeCapability: 8.6, + msrp: 600, + power: 290, + releaseYear: 2021, + }, + "RTX 3070 Ti Mobile": { + tflops: 16.6, + memory: [8], + computeCapability: 8.6, + msrp: 700, + power: 125, + releaseYear: 2022, + }, + "RTX 3060 Ti": { + tflops: 16.2, + memory: [8], + computeCapability: 8.6, + msrp: 400, + power: 200, + releaseYear: 2020, + }, + "RTX 3060": { + tflops: 12.74, + memory: [12, 8], + computeCapability: 8.6, + msrp: 350, + power: 170, + releaseYear: 2021, + }, + "RTX 2080 Ti": { + tflops: 26.9, + memory: [11, 22], // 22GB: modded 2080ti + computeCapability: 7.5, + msrp: 1_000, + power: 250, + releaseYear: 2018, + }, + "RTX 2080": { + tflops: 20.14, + memory: [8], + computeCapability: 7.5, + msrp: 700, + power: 215, + releaseYear: 2018, + }, + "RTX 2070": { + tflops: 14.93, + memory: [8], + computeCapability: 7.5, + msrp: 500, + power: 175, + releaseYear: 2018, + }, + "RTX 2070 SUPER Mobile": { + tflops: 14.13, + memory: [8], + computeCapability: 7.5, + msrp: 600, + power: 115, + releaseYear: 2020, + }, + "RTX 2070 SUPER": { + tflops: 18.12, + memory: [8], + computeCapability: 7.5, + msrp: 500, + power: 215, + releaseYear: 2019, + }, + "RTX 3060 Mobile": { + tflops: 10.94, + memory: [6], + computeCapability: 8.6, + msrp: 400, + power: 115, + releaseYear: 2021, + }, + "RTX 3050 Mobile": { + tflops: 7.639, + memory: [4, 6], + computeCapability: 8.6, + msrp: 250, + power: 95, + releaseYear: 2022, + }, + "RTX 2060": { + tflops: 12.9, + memory: [6], + computeCapability: 7.5, + msrp: 350, + power: 160, + releaseYear: 2019, + }, + "RTX 2060 12GB": { + tflops: 14.36, + memory: [12], + computeCapability: 7.5, + msrp: 300, + power: 184, + releaseYear: 2021, + }, + "RTX 2060 Mobile": { + tflops: 9.22, + memory: [6], + computeCapability: 7.5, + msrp: 350, + power: 90, + releaseYear: 2019, + }, + "RTX 2050 Mobile": { + tflops: 10.2, + memory: [4], + computeCapability: 8.6, // Ampere (outlier GPU in the 20xx series) + msrp: 250, + power: 45, + releaseYear: 2021, + }, + "GTX 1080 Ti": { + tflops: 11.34, // float32 (GPU does not support native float16) + memory: [11], + computeCapability: 6.1, + msrp: 700, + power: 250, + releaseYear: 2017, + }, + "GTX 1080": { + tflops: 8.87, // float32 (GPU does not support native float16) + memory: [8], + computeCapability: 6.1, + msrp: 599, + power: 180, + releaseYear: 2016, + }, + "GTX 1070 Ti": { + tflops: 8.2, // float32 (GPU does not support native float16) + memory: [8], + computeCapability: 6.1, + msrp: 450, + power: 180, + releaseYear: 2017, + }, + "GTX 1070": { + tflops: 6.46, // float32 (GPU does not support native float16) + memory: [8], + computeCapability: 6.1, + msrp: 379, + power: 150, + releaseYear: 2016, + }, + "GTX 1060": { + tflops: 3.9, // float32 (GPU does not support native float16) + memory: [3, 6], + computeCapability: 6.1, + msrp: 300, + power: 120, + releaseYear: 2016, + }, + "GTX 1050 Ti": { + tflops: 2.1, // float32 (GPU does not support native float16) + memory: [4], + computeCapability: 6.1, + msrp: 150, + power: 75, + releaseYear: 2016, + }, + "RTX Titan": { + tflops: 32.62, + memory: [24], + computeCapability: 7.5, + msrp: 2_500, + power: 280, + releaseYear: 2018, + }, + "GTX 1660": { + tflops: 10.05, + memory: [6], + computeCapability: 7.5, + msrp: 200, + power: 120, + releaseYear: 2019, + }, + "GTX 1650 Mobile": { + tflops: 6.39, + memory: [4], + computeCapability: 7.5, + msrp: 150, + power: 50, + releaseYear: 2019, + }, + T4: { + tflops: 65.13, + memory: [16], + computeCapability: 7.5, + msrp: 2_000, + power: 70, + releaseYear: 2018, + }, + T10: { + tflops: 20.0, + memory: [16], + computeCapability: 7.5, + msrp: 2_000, + power: 150, + releaseYear: 2020, + }, + V100: { + tflops: 28.26, + memory: [32, 16], + computeCapability: 7.0, + msrp: 10_000, + power: 300, + releaseYear: 2017, + }, + "Quadro P6000": { + tflops: 12.63, // float32 (GPU does not support native float16) + memory: [24], + computeCapability: 6.1, + msrp: 5_000, + power: 250, + releaseYear: 2016, + }, + P40: { + tflops: 11.76, // float32 (GPU does not support native float16) + memory: [24], + computeCapability: 6.1, + msrp: 5_700, + power: 250, + releaseYear: 2016, + }, + P100: { + tflops: 19.05, + memory: [16], + computeCapability: 6.0, + msrp: 7_000, + power: 300, + releaseYear: 2016, + }, + "Jetson AGX Orin 64GB": { + tflops: 10.65, + memory: [64], + computeCapability: 8.7, + msrp: 2_000, + power: 60, + releaseYear: 2022, + }, + "Jetson AGX Orin 32GB": { + tflops: 6.66, + memory: [32], + computeCapability: 8.7, + msrp: 999, + power: 40, + releaseYear: 2022, + }, + "Jetson Orin NX 16GB": { + tflops: 3.76, + memory: [16], + computeCapability: 8.7, + msrp: 600, + power: 25, + releaseYear: 2023, + }, + "Jetson Orin NX 8GB": { + tflops: 3.13, + memory: [8], + computeCapability: 8.7, + msrp: 400, + power: 20, + releaseYear: 2023, + }, + "Jetson Orin Nano 8GB": { + tflops: 2.56, + memory: [8], + computeCapability: 8.7, + msrp: 500, + power: 15, + releaseYear: 2023, + }, + "Jetson Orin Nano 4GB": { + tflops: 1.28, + memory: [4], + computeCapability: 8.7, + msrp: 200, + power: 10, + releaseYear: 2023, + }, + "Jetson AGX Xavier": { + tflops: 2.82, + memory: [32, 64], + computeCapability: 7.2, + msrp: 1_100, + power: 30, + releaseYear: 2018, + }, + "Jetson Xavier NX": { + tflops: 1.69, + memory: [8, 16], + computeCapability: 7.2, + msrp: 400, + power: 20, + releaseYear: 2020, + }, + "Jetson TX2": { + tflops: 1.33, + memory: [4, 8], + computeCapability: 6.2, + msrp: 400, + power: 15, + releaseYear: 2017, + }, + "Jetson Nano": { + tflops: 0.47, + memory: [4], + computeCapability: 5.3, + msrp: 100, + power: 10, + releaseYear: 2019, + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/esm/hardware.d.ts b/node_modules/@huggingface/tasks/dist/esm/hardware.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..310b38752315edca143f3fd9f77d9d414c7db2ac --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/hardware.d.ts @@ -0,0 +1,616 @@ +/** + * Biden AI Executive Order (since revoked by President Trump): + * https://web.archive.org/web/20250105222429/https://www.whitehouse.gov/briefing-room/presidential-actions/2023/10/30/executive-order-on-the-safe-secure-and-trustworthy-development-and-use-of-artificial-intelligence/ + */ +export declare const TFLOPS_THRESHOLD_WHITE_HOUSE_MODEL_TRAINING_TOTAL: number; +export declare const TFLOPS_THRESHOLD_WHITE_HOUSE_MODEL_TRAINING_TOTAL_BIOLOGY: number; +export declare const TFLOPS_THRESHOLD_WHITE_HOUSE_CLUSTER: number; +/** + * EU AI Act + * https://ec.europa.eu/commission/presscorner/detail/en/qanda_21_1683 + */ +export declare const TFLOPS_THRESHOLD_EU_AI_ACT_MODEL_TRAINING_TOTAL: number; +export interface HardwareSpec { + /** + * Approximate value, in FP16 whenever possible for GPUs and FP32 for CPUs. + * This is only approximate/theoretical and shouldn't be taken too seriously. + * Currently the CPU values are from cpu-monkey.com + * while the GPU values are from techpowerup.com + * + * Note to reviewers: I got fed up with data entry, + * and HuggingChat running Llama3 with Web search was failing a bit, + * so some of those values might be slightly inaccurate. Forgive me and please feel free to improve. + */ + tflops: number; + /** + * If an array is specified, options of memory size (can be VRAM, unified RAM) + * e.g. an A100 exists in 40 or 80 GB. + */ + memory?: number[]; + /** + * Approximate MSRP in USD at launch. For SKUs with multiple memory variants, + * the price corresponds to the largest memory variant. For datacenter GPUs + * sold via OEMs without a public MSRP (H100, MI300X, ...), this is a + * widely-reported street price. For mobile/laptop GPUs that are not sold + * standalone, this is the approximate module/BOM cost. For Apple Silicon + * SoCs, this is the price of a Mac configured with that chip and the + * largest memory option. For CPU "family" entries (e.g. "Xeon 4th Gen", + * "Ryzen Zen 4 7000 (Ryzen 9)"), this is the tray/box price of a + * representative flagship SKU at launch. + */ + msrp: number; + /** + * Approximate maximum sustained power draw in watts. For GPUs with multiple + * form factors (e.g. H100 SXM vs PCIe), uses the highest variant. For CPUs, + * uses max turbo power (PL2 / MTP for Intel, PPT for AMD), not base TDP. + * For Apple Silicon and Snapdragon SoCs, an estimated package power based + * on benchmarks/teardowns (Apple does not publish TDP). + */ + power: number; + /** + * Year the SKU first became available. For SKUs refreshed later with + * additional memory variants (e.g. A100 40GB → 80GB, RTX 2060 → 12GB), + * this is the original launch year. For CPU "family" entries, this is + * the year the family debuted. + */ + releaseYear: number; +} +export declare const DEFAULT_MEMORY_OPTIONS: number[]; +export declare const SKUS: { + GPU: { + NVIDIA: Record; + AMD: Record; + INTEL: { + "Arc A750": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Arc A770": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Arc B570": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Arc B580": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Arc B50": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Arc B60": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Arc Pro B70": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + }; + QUALCOMM: { + "Snapdragon X Elite X1E-00-1DE": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Snapdragon X Elite X1E-84-100": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Snapdragon X Elite X1E-80-100": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Snapdragon X Elite X1E-78-100": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Snapdragon X Plus X1P-64-100": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + }; + }; + CPU: { + Intel: { + "Xeon 4th Generation (Sapphire Rapids)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Xeon 3th Generation (Ice Lake)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Xeon 2th Generation (Cascade Lake)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Xeon E5v4 (Broadwell)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Xeon E5v3 (Haswell)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Xeon E5v2 (Ivy Bridge)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core Ultra 9 275HX": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core Ultra 7 255HX": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core Ultra 7 265KF": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 14th Generation (i7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 13th Generation (i9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 13th Generation (i7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 13th Generation (i5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 13th Generation (i3)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 12th Generation (i9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 12th Generation (i7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 12th Generation (i5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 12th Generation (i3)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 11th Generation (i9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 11th Generation (i7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 11th Generation (i5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 11th Generation (i3)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 10th Generation (i9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 10th Generation (i7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 10th Generation (i5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Intel Core 10th Generation (i3)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + }; + AMD: { + "EPYC 5th Generation Zen 5 (Turin)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "EPYC 4th Generation Zen 4 (Genoa)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "EPYC 3th Generation Zen 3 (Milan)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "EPYC 2th Generation Zen 2 (Rome)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "EPYC 1st Generation Zen (Naples)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Threadripper Zen 5 9000 (Shimada Peak)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Threadripper Zen 4 7000 (Storm Peak)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Threadripper Zen 3 5000 (Chagall)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Threadripper Zen 2 3000 (Castle Peak)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Threadripper Zen 1000 (Whitehaven)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen 7 3800X (16)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 5 9000 (Ryzen 9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 5 9000 (Ryzen 7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 5 9000 (Ryzen 5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 4 7000 (Ryzen 9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 4 7000 (Ryzen 7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 4 7000 (Ryzen 5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 3 5000 (Ryzen 9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 3 5000 (Ryzen 7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 3 5000 (Ryzen 5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 2 3000 (Ryzen 9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 2 3000 (Ryzen 7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 2 3000 (Ryzen 5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen Zen 2 3000 (Ryzen 3)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen AI 300 (Ryzen AI 9 HX)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen AI 300 (Ryzen AI 9)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen AI 300 (Ryzen AI 7)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + "Ryzen AI 300 (Ryzen AI 5)": { + tflops: number; + msrp: number; + power: number; + releaseYear: number; + }; + }; + }; + "Apple Silicon": { + "-": { + "Apple MacBook Neo": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M1": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M1 Pro": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M1 Max": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M1 Ultra": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M2": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M2 Pro": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M2 Max": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M2 Ultra": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M3": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M3 Pro": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M3 Max": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M3 Ultra": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M4": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M4 Pro": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M4 Max": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M5": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M5 Pro": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + "Apple M5 Max": { + tflops: number; + memory: number[]; + msrp: number; + power: number; + releaseYear: number; + }; + }; + }; +}; +export type SkuType = keyof typeof SKUS; +//# sourceMappingURL=hardware.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/hardware.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/hardware.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..898c3b2e810236abf7ac4c266e52f78c05c1452b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/hardware.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"hardware.d.ts","sourceRoot":"","sources":["../../src/hardware.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,eAAO,MAAM,iDAAiD,QAAW,CAAC;AAC1E,eAAO,MAAM,yDAAyD,QAAW,CAAC;AAClF,eAAO,MAAM,oCAAoC,QAAU,CAAC;AAE5D;;;GAGG;AACH,eAAO,MAAM,+CAA+C,QAAW,CAAC;AAExE,MAAM,WAAW,YAAY;IAC5B;;;;;;;;;OASG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB;;;;;;;;;;OAUG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,WAAW,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,sBAAsB,UAElC,CAAC;AAEF,eAAO,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2iBuD,CAAC;AAEzE,MAAM,MAAM,OAAO,GAAG,MAAM,OAAO,IAAI,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/hardware.js b/node_modules/@huggingface/tasks/dist/esm/hardware.js new file mode 100644 index 0000000000000000000000000000000000000000..9e1c07f6a705362fc2ee00d249a6ec8f81f582be --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/hardware.js @@ -0,0 +1,573 @@ +import { AMD_GPU_SKUS } from "./hardware-amd.js"; +import { NVIDIA_SKUS } from "./hardware-nvidia.js"; +/** + * Biden AI Executive Order (since revoked by President Trump): + * https://web.archive.org/web/20250105222429/https://www.whitehouse.gov/briefing-room/presidential-actions/2023/10/30/executive-order-on-the-safe-secure-and-trustworthy-development-and-use-of-artificial-intelligence/ + */ +export const TFLOPS_THRESHOLD_WHITE_HOUSE_MODEL_TRAINING_TOTAL = 10 ** 14; +export const TFLOPS_THRESHOLD_WHITE_HOUSE_MODEL_TRAINING_TOTAL_BIOLOGY = 10 ** 11; +export const TFLOPS_THRESHOLD_WHITE_HOUSE_CLUSTER = 10 ** 8; +/** + * EU AI Act + * https://ec.europa.eu/commission/presscorner/detail/en/qanda_21_1683 + */ +export const TFLOPS_THRESHOLD_EU_AI_ACT_MODEL_TRAINING_TOTAL = 10 ** 13; +export const DEFAULT_MEMORY_OPTIONS = [ + 8, 16, 24, 32, 40, 48, 64, 80, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, 2048, +]; +export const SKUS = { + GPU: { + NVIDIA: NVIDIA_SKUS, + AMD: AMD_GPU_SKUS, + INTEL: { + "Arc A750": { + tflops: 34.41, + memory: [8], + msrp: 250, + power: 225, + releaseYear: 2022, + }, + "Arc A770": { + tflops: 39.32, + memory: [8, 16], + msrp: 350, + power: 225, + releaseYear: 2022, + }, + "Arc B570": { + tflops: 23.04, + memory: [10], + msrp: 200, + power: 150, + releaseYear: 2025, + }, + "Arc B580": { + tflops: 27.34, + memory: [12], + msrp: 250, + power: 190, + releaseYear: 2024, + }, + "Arc B50": { + tflops: 21.3, + memory: [16], + msrp: 350, + power: 70, + releaseYear: 2025, + }, + "Arc B60": { + tflops: 24.58, + memory: [24, 48], + msrp: 1_200, + power: 200, + releaseYear: 2025, + }, + "Arc Pro B70": { + tflops: 45.88, + memory: [32], + msrp: 949, + power: 230, + releaseYear: 2026, + }, + }, + QUALCOMM: { + "Snapdragon X Elite X1E-00-1DE": { + tflops: 4.6, + msrp: 900, + power: 80, + releaseYear: 2024, + }, + "Snapdragon X Elite X1E-84-100": { + tflops: 4.6, + msrp: 1_700, + power: 30, + releaseYear: 2024, + }, + "Snapdragon X Elite X1E-80-100": { + tflops: 3.8, + msrp: 1_300, + power: 23, + releaseYear: 2024, + }, + "Snapdragon X Elite X1E-78-100": { + tflops: 3.8, + msrp: 1_200, + power: 23, + releaseYear: 2024, + }, + "Snapdragon X Plus X1P-64-100": { + tflops: 3.8, + msrp: 1_000, + power: 23, + releaseYear: 2024, + }, + }, + }, + CPU: { + Intel: { + "Xeon 4th Generation (Sapphire Rapids)": { + tflops: 1.3, + msrp: 10_500, + power: 350, + releaseYear: 2023, + }, + "Xeon 3th Generation (Ice Lake)": { + tflops: 0.8, + msrp: 8_000, + power: 270, + releaseYear: 2021, + }, + "Xeon 2th Generation (Cascade Lake)": { + tflops: 0.55, + msrp: 10_000, + power: 205, + releaseYear: 2019, + }, + "Xeon E5v4 (Broadwell)": { + tflops: 0.25, + msrp: 4_000, + power: 145, + releaseYear: 2016, + }, + "Xeon E5v3 (Haswell)": { + tflops: 0.2, + msrp: 4_000, + power: 145, + releaseYear: 2014, + }, + "Xeon E5v2 (Ivy Bridge)": { + tflops: 0.15, + msrp: 2_500, + power: 130, + releaseYear: 2013, + }, + "Intel Core Ultra 9 275HX": { + tflops: 1.89, + msrp: 700, + power: 160, + releaseYear: 2025, + }, + "Intel Core Ultra 7 255HX": { + tflops: 1.62, + msrp: 583, + power: 160, + releaseYear: 2025, + }, + "Intel Core Ultra 7 265KF": { + tflops: 1.53, + msrp: 400, + power: 250, + releaseYear: 2024, + }, + "Intel Core 14th Generation (i7)": { + tflops: 0.8, + msrp: 400, + power: 253, + releaseYear: 2023, + }, + "Intel Core 13th Generation (i9)": { + tflops: 0.85, + msrp: 600, + power: 253, + releaseYear: 2022, + }, + "Intel Core 13th Generation (i7)": { + tflops: 0.82, + msrp: 400, + power: 253, + releaseYear: 2022, + }, + "Intel Core 13th Generation (i5)": { + tflops: 0.68, + msrp: 300, + power: 181, + releaseYear: 2022, + }, + "Intel Core 13th Generation (i3)": { + tflops: 0.57, + msrp: 150, + power: 89, + releaseYear: 2023, + }, + "Intel Core 12th Generation (i9)": { + tflops: 0.79, + msrp: 600, + power: 241, + releaseYear: 2021, + }, + "Intel Core 12th Generation (i7)": { + tflops: 0.77, + msrp: 400, + power: 190, + releaseYear: 2021, + }, + "Intel Core 12th Generation (i5)": { + tflops: 0.65, + msrp: 300, + power: 150, + releaseYear: 2021, + }, + "Intel Core 12th Generation (i3)": { + tflops: 0.53, + msrp: 150, + power: 89, + releaseYear: 2022, + }, + "Intel Core 11th Generation (i9)": { + tflops: 0.7, + msrp: 550, + power: 251, + releaseYear: 2021, + }, + "Intel Core 11th Generation (i7)": { + tflops: 0.6, + msrp: 400, + power: 251, + releaseYear: 2021, + }, + "Intel Core 11th Generation (i5)": { + tflops: 0.5, + msrp: 250, + power: 251, + releaseYear: 2021, + }, + "Intel Core 11th Generation (i3)": { + tflops: 0.35, + msrp: 150, + power: 90, + releaseYear: 2021, + }, + "Intel Core 10th Generation (i9)": { + tflops: 0.46, + msrp: 500, + power: 250, + releaseYear: 2020, + }, + "Intel Core 10th Generation (i7)": { + tflops: 0.46, + msrp: 400, + power: 215, + releaseYear: 2020, + }, + "Intel Core 10th Generation (i5)": { + tflops: 0.46, + msrp: 250, + power: 182, + releaseYear: 2020, + }, + "Intel Core 10th Generation (i3)": { + tflops: 0.44, + msrp: 150, + power: 90, + releaseYear: 2020, + }, + }, + AMD: { + "EPYC 5th Generation Zen 5 (Turin)": { + tflops: 13.8, + msrp: 13_000, + power: 500, + releaseYear: 2024, + }, + "EPYC 4th Generation Zen 4 (Genoa)": { + tflops: 5, + msrp: 11_500, + power: 360, + releaseYear: 2022, + }, + "EPYC 3th Generation Zen 3 (Milan)": { + tflops: 2.4, + msrp: 8_000, + power: 280, + releaseYear: 2021, + }, + "EPYC 2th Generation Zen 2 (Rome)": { + tflops: 0.6, + msrp: 7_000, + power: 225, + releaseYear: 2019, + }, + "EPYC 1st Generation Zen (Naples)": { + tflops: 0.6, + msrp: 4_000, + power: 180, + releaseYear: 2017, + }, + "Ryzen Threadripper Zen 5 9000 (Shimada Peak)": { + tflops: 14.0, + msrp: 5_000, + power: 350, + releaseYear: 2025, + }, + "Ryzen Threadripper Zen 4 7000 (Storm Peak)": { + tflops: 10.0, + msrp: 5_000, + power: 350, + releaseYear: 2023, + }, + "Ryzen Threadripper Zen 3 5000 (Chagall)": { + tflops: 4.6, + msrp: 6_500, + power: 280, + releaseYear: 2022, + }, + "Ryzen Threadripper Zen 2 3000 (Castle Peak)": { + tflops: 3.2, + msrp: 4_000, + power: 280, + releaseYear: 2019, + }, + "Ryzen Threadripper Zen 1000 (Whitehaven)": { + tflops: 0.6, + msrp: 1_000, + power: 180, + releaseYear: 2017, + }, + "Ryzen 7 3800X (16)": { + tflops: 1.15, + msrp: 400, + power: 142, + releaseYear: 2019, + }, + "Ryzen Zen 5 9000 (Ryzen 9)": { + tflops: 0.56, + msrp: 650, + power: 230, + releaseYear: 2024, + }, + "Ryzen Zen 5 9000 (Ryzen 7)": { + tflops: 0.56, + msrp: 350, + power: 88, + releaseYear: 2024, + }, + "Ryzen Zen 5 9000 (Ryzen 5)": { + tflops: 0.56, + msrp: 300, + power: 88, + releaseYear: 2024, + }, + "Ryzen Zen 4 7000 (Ryzen 9)": { + tflops: 0.56, + msrp: 700, + power: 230, + releaseYear: 2022, + }, + "Ryzen Zen 4 7000 (Ryzen 7)": { + tflops: 0.56, + msrp: 400, + power: 142, + releaseYear: 2022, + }, + "Ryzen Zen 4 7000 (Ryzen 5)": { + tflops: 0.56, + msrp: 300, + power: 142, + releaseYear: 2022, + }, + "Ryzen Zen 3 5000 (Ryzen 9)": { + tflops: 1.33, + msrp: 800, + power: 142, + releaseYear: 2020, + }, + "Ryzen Zen 3 5000 (Ryzen 7)": { + tflops: 1.33, + msrp: 450, + power: 142, + releaseYear: 2020, + }, + "Ryzen Zen 3 5000 (Ryzen 5)": { + tflops: 0.72, + msrp: 300, + power: 88, + releaseYear: 2020, + }, + "Ryzen Zen 2 3000 (Ryzen 9)": { + tflops: 0.72, + msrp: 750, + power: 142, + releaseYear: 2019, + }, + "Ryzen Zen 2 3000 (Ryzen 7)": { + tflops: 0.72, + msrp: 400, + power: 142, + releaseYear: 2019, + }, + "Ryzen Zen 2 3000 (Ryzen 5)": { + tflops: 0.72, + msrp: 250, + power: 88, + releaseYear: 2019, + }, + "Ryzen Zen 2 3000 (Ryzen 3)": { + tflops: 0.72, + msrp: 150, + power: 88, + releaseYear: 2020, + }, + "Ryzen AI 300 (Ryzen AI 9 HX)": { + tflops: 5.52, + msrp: 500, + power: 54, + releaseYear: 2024, + }, + "Ryzen AI 300 (Ryzen AI 9)": { + tflops: 5.2, + msrp: 450, + power: 54, + releaseYear: 2024, + }, + "Ryzen AI 300 (Ryzen AI 7)": { + tflops: 4.34, + msrp: 350, + power: 54, + releaseYear: 2024, + }, + "Ryzen AI 300 (Ryzen AI 5)": { + tflops: 1.57, + msrp: 250, + power: 28, + releaseYear: 2024, + }, + }, + }, + "Apple Silicon": { + "-": { + "Apple MacBook Neo": { + tflops: 1.9, + memory: [8], + msrp: 700, + power: 10, + releaseYear: 2026, + }, + "Apple M1": { + tflops: 2.6, + memory: [8, 16], + msrp: 1_250, + power: 15, + releaseYear: 2020, + }, + "Apple M1 Pro": { + tflops: 5.2, + memory: [16, 24, 32], + msrp: 2_900, + power: 30, + releaseYear: 2021, + }, + "Apple M1 Max": { + tflops: 10.4, + memory: [16, 24, 32, 64], + msrp: 3_900, + power: 60, + releaseYear: 2021, + }, + "Apple M1 Ultra": { + tflops: 21, + memory: [16, 24, 32, 64, 96, 128], + msrp: 6_200, + power: 120, + releaseYear: 2022, + }, + "Apple M2": { + tflops: 3.6, + memory: [8, 16, 24], + msrp: 1_500, + power: 20, + releaseYear: 2022, + }, + "Apple M2 Pro": { + tflops: 6.8, + memory: [16, 24, 32], + msrp: 2_800, + power: 35, + releaseYear: 2023, + }, + "Apple M2 Max": { + tflops: 13.49, + memory: [32, 64, 96], + msrp: 4_500, + power: 80, + releaseYear: 2023, + }, + "Apple M2 Ultra": { + tflops: 27.2, + memory: [64, 96, 128, 192], + msrp: 7_000, + power: 150, + releaseYear: 2023, + }, + "Apple M3": { + tflops: 4.1, + memory: [8, 16, 24], + msrp: 1_500, + power: 22, + releaseYear: 2023, + }, + "Apple M3 Pro": { + tflops: 7.4, + memory: [18, 36], + msrp: 2_400, + power: 40, + releaseYear: 2023, + }, + "Apple M3 Max": { + tflops: 14.2, + memory: [36, 48, 64, 96, 128], + msrp: 5_000, + power: 90, + releaseYear: 2023, + }, + "Apple M3 Ultra": { + tflops: 28.4, + memory: [96, 256, 512], + msrp: 9_500, + power: 180, + releaseYear: 2025, + }, + "Apple M4": { + tflops: 4.6, + memory: [16, 24, 32], + msrp: 1_600, + power: 22, + releaseYear: 2024, + }, + "Apple M4 Pro": { + tflops: 9.2, + memory: [24, 48, 64], + msrp: 2_600, + power: 45, + releaseYear: 2024, + }, + "Apple M4 Max": { + tflops: 18.4, + memory: [36, 48, 64, 128], + msrp: 5_000, + power: 100, + releaseYear: 2024, + }, + "Apple M5": { + tflops: 5.7, + memory: [16, 24, 32], + msrp: 2_000, + power: 25, + releaseYear: 2025, + }, + "Apple M5 Pro": { + tflops: 11.4, + memory: [24, 36, 48, 64], + msrp: 2_900, + power: 50, + releaseYear: 2026, + }, + "Apple M5 Max": { + tflops: 22.8, + memory: [36, 48, 64, 128], + msrp: 5_000, + power: 110, + releaseYear: 2026, + }, + }, + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/esm/index.d.ts b/node_modules/@huggingface/tasks/dist/esm/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..958e2cd12307a2baa022b82289fb081537d87e35 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/index.d.ts @@ -0,0 +1,28 @@ +export { LIBRARY_TASK_MAPPING } from "./library-to-tasks.js"; +export { MAPPING_DEFAULT_WIDGET } from "./default-widget-inputs.js"; +export type { TaskData, TaskDemo, TaskDemoEntry, ExampleRepo } from "./tasks/index.js"; +export * from "./tasks/index.js"; +export { PIPELINE_DATA, PIPELINE_TYPES, type WidgetType, type PipelineType, type PipelineData, type Modality, MODALITIES, MODALITY_LABELS, SUBTASK_TYPES, PIPELINE_TYPES_SET, } from "./pipelines.js"; +export { ALL_DISPLAY_MODEL_LIBRARY_KEYS, ALL_MODEL_LIBRARY_KEYS, MODEL_LIBRARIES_UI_ELEMENTS, } from "./model-libraries.js"; +export type { LibraryUiElement, ModelLibraryKey } from "./model-libraries.js"; +export type { ModelData, TransformersInfo } from "./model-data.js"; +export type { AddedToken, SpecialTokensMap, TokenizerConfig } from "./tokenizer-data.js"; +export type { WidgetExample, WidgetExampleAttribute, WidgetExampleAssetAndPromptInput, WidgetExampleAssetAndTextInput, WidgetExampleAssetAndZeroShotInput, WidgetExampleAssetInput, WidgetExampleChatInput, WidgetExampleSentenceSimilarityInput, WidgetExampleStructuredDataInput, WidgetExampleTableDataInput, WidgetExampleTextAndContextInput, WidgetExampleTextAndTableInput, WidgetExampleTextInput, WidgetExampleZeroShotTextInput, WidgetExampleOutput, WidgetExampleOutputUrl, WidgetExampleOutputLabels, WidgetExampleOutputAnswerScore, WidgetExampleOutputText, } from "./widget-example.js"; +export { SPECIAL_TOKENS_ATTRIBUTES } from "./tokenizer-data.js"; +export * from "./gguf.js"; +export { type InferenceSnippet, type InferenceSnippetLanguage, type ModelDataMinimal, inferenceSnippetLanguages, stringifyGenerationConfig, stringifyMessages, getModelInputSnippet, } from "./snippets/index.js"; +export { SKUS, DEFAULT_MEMORY_OPTIONS } from "./hardware.js"; +export type { HardwareSpec, SkuType } from "./hardware.js"; +export type { AmdGpuHardwareSpec } from "./hardware-amd.js"; +export type { NvidiaHardwareSpec } from "./hardware-nvidia.js"; +export { LOCAL_APPS } from "./local-apps.js"; +export type { LocalApp, LocalAppKey, LocalAppSnippet } from "./local-apps.js"; +export { DATASET_LIBRARIES_UI_ELEMENTS } from "./dataset-libraries.js"; +export type { DatasetLibraryUiElement, DatasetLibraryKey } from "./dataset-libraries.js"; +export { KERNEL_LIBRARIES_UI_ELEMENTS } from "./kernel-libraries.js"; +export type { KernelLibraryKey, KernelLibraryUiElement } from "./kernel-libraries.js"; +export * from "./inference-providers.js"; +export { EVALUATION_FRAMEWORKS } from "./eval.js"; +export { AGENT_HARNESSES, STANDARD_AGENT_ENV_VARS } from "./agent-harnesses.js"; +export type { AgentHarness, AgentHarnessKey } from "./agent-harnesses.js"; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/index.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..10ee927a0fb7f54095ea82af0e903545d73a987d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACpE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACvF,cAAc,kBAAkB,CAAC;AACjC,OAAO,EACN,aAAa,EACb,cAAc,EACd,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,QAAQ,EACb,UAAU,EACV,eAAe,EACf,aAAa,EACb,kBAAkB,GAClB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACN,8BAA8B,EAC9B,sBAAsB,EACtB,2BAA2B,GAC3B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC9E,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACzF,YAAY,EACX,aAAa,EACb,sBAAsB,EACtB,gCAAgC,EAChC,8BAA8B,EAC9B,kCAAkC,EAClC,uBAAuB,EACvB,sBAAsB,EACtB,oCAAoC,EACpC,gCAAgC,EAChC,2BAA2B,EAC3B,gCAAgC,EAChC,8BAA8B,EAC9B,sBAAsB,EACtB,8BAA8B,EAC9B,mBAAmB,EACnB,sBAAsB,EACtB,yBAAyB,EACzB,8BAA8B,EAC9B,uBAAuB,GACvB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAEhE,cAAc,WAAW,CAAC;AAE1B,OAAO,EACN,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EACrB,yBAAyB,EACzB,yBAAyB,EACzB,iBAAiB,EACjB,oBAAoB,GACpB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,IAAI,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAC7D,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAC3D,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC5D,YAAY,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAE9E,OAAO,EAAE,6BAA6B,EAAE,MAAM,wBAAwB,CAAC;AACvE,YAAY,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAEzF,OAAO,EAAE,4BAA4B,EAAE,MAAM,uBAAuB,CAAC;AACrE,YAAY,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAEtF,cAAc,0BAA0B,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAElD,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAChF,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/index.js b/node_modules/@huggingface/tasks/dist/esm/index.js new file mode 100644 index 0000000000000000000000000000000000000000..263b9d324e04e607d65990e839548dc3256a8a54 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/index.js @@ -0,0 +1,15 @@ +export { LIBRARY_TASK_MAPPING } from "./library-to-tasks.js"; +export { MAPPING_DEFAULT_WIDGET } from "./default-widget-inputs.js"; +export * from "./tasks/index.js"; +export { PIPELINE_DATA, PIPELINE_TYPES, MODALITIES, MODALITY_LABELS, SUBTASK_TYPES, PIPELINE_TYPES_SET, } from "./pipelines.js"; +export { ALL_DISPLAY_MODEL_LIBRARY_KEYS, ALL_MODEL_LIBRARY_KEYS, MODEL_LIBRARIES_UI_ELEMENTS, } from "./model-libraries.js"; +export { SPECIAL_TOKENS_ATTRIBUTES } from "./tokenizer-data.js"; +export * from "./gguf.js"; +export { inferenceSnippetLanguages, stringifyGenerationConfig, stringifyMessages, getModelInputSnippet, } from "./snippets/index.js"; +export { SKUS, DEFAULT_MEMORY_OPTIONS } from "./hardware.js"; +export { LOCAL_APPS } from "./local-apps.js"; +export { DATASET_LIBRARIES_UI_ELEMENTS } from "./dataset-libraries.js"; +export { KERNEL_LIBRARIES_UI_ELEMENTS } from "./kernel-libraries.js"; +export * from "./inference-providers.js"; +export { EVALUATION_FRAMEWORKS } from "./eval.js"; +export { AGENT_HARNESSES, STANDARD_AGENT_ENV_VARS } from "./agent-harnesses.js"; diff --git a/node_modules/@huggingface/tasks/dist/esm/inference-providers.d.ts b/node_modules/@huggingface/tasks/dist/esm/inference-providers.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..08c44ebf82b0fd557e194c83c6117366d4534b0f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/inference-providers.d.ts @@ -0,0 +1,11 @@ +declare const INFERENCE_PROVIDERS: readonly ["cerebras", "cohere", "deepinfra", "fal-ai", "fireworks-ai", "hf-inference", "ovhcloud", "replicate", "together"]; +export type SnippetInferenceProvider = (typeof INFERENCE_PROVIDERS)[number] | string; +export declare const HF_HUB_INFERENCE_PROXY_TEMPLATE = "https://router.huggingface.co/{{PROVIDER}}"; +/** + * URL to set as baseUrl in the OpenAI SDK. + * + * TODO(Expose this from InferenceClient in the future?) + */ +export declare function openAIbaseUrl(provider: SnippetInferenceProvider): string; +export {}; +//# sourceMappingURL=inference-providers.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/inference-providers.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/inference-providers.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c3374c6a738eec476cf3bcb7b5fa2e174994393b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/inference-providers.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference-providers.d.ts","sourceRoot":"","sources":["../../src/inference-providers.ts"],"names":[],"mappings":"AAEA,QAAA,MAAM,mBAAmB,6HAUf,CAAC;AAEX,MAAM,MAAM,wBAAwB,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;AAErF,eAAO,MAAM,+BAA+B,+CAA+C,CAAC;AAE5F;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,wBAAwB,GAAG,MAAM,CAGxE"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/inference-providers.js b/node_modules/@huggingface/tasks/dist/esm/inference-providers.js new file mode 100644 index 0000000000000000000000000000000000000000..b26af174cfac3f62078a26ff2a43b08b40046444 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/inference-providers.js @@ -0,0 +1,23 @@ +/// This list is for illustration purposes only. +/// in the `tasks` sub-package, we do not need actual strong typing of the inference providers. +const INFERENCE_PROVIDERS = [ + "cerebras", + "cohere", + "deepinfra", + "fal-ai", + "fireworks-ai", + "hf-inference", + "ovhcloud", + "replicate", + "together", +]; +export const HF_HUB_INFERENCE_PROXY_TEMPLATE = `https://router.huggingface.co/{{PROVIDER}}`; +/** + * URL to set as baseUrl in the OpenAI SDK. + * + * TODO(Expose this from InferenceClient in the future?) + */ +export function openAIbaseUrl(provider) { + const url = HF_HUB_INFERENCE_PROXY_TEMPLATE.replace("{{PROVIDER}}", provider); + return provider === "hf-inference" ? `${url}/v1` : url; +} diff --git a/node_modules/@huggingface/tasks/dist/esm/kernel-libraries.d.ts b/node_modules/@huggingface/tasks/dist/esm/kernel-libraries.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..31eff06c1e4045bc98715ba40ed714716d21eebc --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/kernel-libraries.d.ts @@ -0,0 +1,36 @@ +/** + * Elements configurable by a kernel library. + */ +export interface KernelLibraryUiElement { + /** + * Pretty name of the library. + */ + prettyLabel: string; + /** + * Repo name of the library's (usually on GitHub) code repo + */ + repoName: string; + /** + * URL to library's (usually on GitHub) code repo + */ + repoUrl: string; + /** + * URL to library's docs + */ + docsUrl?: string; + /** + * Code snippet(s) displayed + */ + snippets?: (kernelId: string, version?: number) => string[]; +} +export declare const KERNEL_LIBRARIES_UI_ELEMENTS: { + kernels: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (kernelId: string, version?: number) => string[]; + }; +}; +export type KernelLibraryKey = keyof typeof KERNEL_LIBRARIES_UI_ELEMENTS; +//# sourceMappingURL=kernel-libraries.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/kernel-libraries.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/kernel-libraries.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..685de12f73ff91349ccd786ad085bc97e82e5ff0 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/kernel-libraries.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"kernel-libraries.d.ts","sourceRoot":"","sources":["../../src/kernel-libraries.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CAC5D;AAED,eAAO,MAAM,4BAA4B;;;;;;6BAMlB,MAAM,YAAY,MAAM;;CAQG,CAAC;AAEnD,MAAM,MAAM,gBAAgB,GAAG,MAAM,OAAO,4BAA4B,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/kernel-libraries.js b/node_modules/@huggingface/tasks/dist/esm/kernel-libraries.js new file mode 100644 index 0000000000000000000000000000000000000000..fceffa689e3b82eef137d47cd386cc4c38d7dc5e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/kernel-libraries.js @@ -0,0 +1,15 @@ +export const KERNEL_LIBRARIES_UI_ELEMENTS = { + kernels: { + prettyLabel: "Kernels", + repoName: "Kernels", + repoUrl: "https://github.com/huggingface/kernels", + docsUrl: "https://huggingface.co/docs/kernels", + snippets: (kernelId, version) => [ + `# !pip install kernels + +from kernels import get_kernel + +kernel = get_kernel("${kernelId}"${version !== undefined ? `, version=${version}` : ""})`, + ], + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/esm/library-to-tasks.d.ts b/node_modules/@huggingface/tasks/dist/esm/library-to-tasks.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..61ffd282076871d1b86f92570b95e9a44cded208 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/library-to-tasks.d.ts @@ -0,0 +1,12 @@ +import type { ModelLibraryKey } from "./model-libraries.js"; +import type { PipelineType } from "./pipelines.js"; +/** + * Mapping from library name to its supported tasks. + * HF-Inference API (serverless) should be disabled for all other (library, task) pairs beyond this mapping. + * This mapping is partially generated automatically by "python-api-export-tasks" action in + * huggingface/api-inference-community repo upon merge. For transformers, the mapping is manually + * based on api-inference (hf_types.rs). + */ +export declare const LIBRARY_TASK_MAPPING: Partial>; +export declare const REMOVED_IN_V5_TRANSFORMERS_PIPELINES: PipelineType[]; +//# sourceMappingURL=library-to-tasks.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/library-to-tasks.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/library-to-tasks.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..8b35f76b24f02a9f3c534de64d33fe47930cb7ae --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/library-to-tasks.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"library-to-tasks.d.ts","sourceRoot":"","sources":["../../src/library-to-tasks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD;;;;;;GAMG;AACH,eAAO,MAAM,oBAAoB,EAAE,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,YAAY,EAAE,CAAC,CA6DjF,CAAC;AAGF,eAAO,MAAM,oCAAoC,EAAE,YAAY,EAAsD,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/library-to-tasks.js b/node_modules/@huggingface/tasks/dist/esm/library-to-tasks.js new file mode 100644 index 0000000000000000000000000000000000000000..852ae31a9983a69e5887371eda57e2cf7a0f5097 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/library-to-tasks.js @@ -0,0 +1,71 @@ +/** + * Mapping from library name to its supported tasks. + * HF-Inference API (serverless) should be disabled for all other (library, task) pairs beyond this mapping. + * This mapping is partially generated automatically by "python-api-export-tasks" action in + * huggingface/api-inference-community repo upon merge. For transformers, the mapping is manually + * based on api-inference (hf_types.rs). + */ +export const LIBRARY_TASK_MAPPING = { + "adapter-transformers": ["question-answering", "text-classification", "token-classification"], + allennlp: ["question-answering"], + asteroid: [ + // "audio-source-separation", + "audio-to-audio", + ], + bertopic: ["text-classification"], + diffusers: ["image-to-image", "text-to-image"], + doctr: ["object-detection"], + espnet: ["text-to-speech", "automatic-speech-recognition"], + fairseq: ["text-to-speech", "audio-to-audio"], + fastai: ["image-classification"], + fasttext: ["feature-extraction", "text-classification"], + flair: ["token-classification"], + k2: ["automatic-speech-recognition"], + keras: ["image-classification"], + nemo: ["automatic-speech-recognition"], + open_clip: ["zero-shot-classification", "zero-shot-image-classification"], + paddlenlp: ["fill-mask", "summarization", "zero-shot-classification"], + peft: ["text-generation"], + "pyannote-audio": ["automatic-speech-recognition"], + "sentence-transformers": ["feature-extraction", "sentence-similarity"], + setfit: ["text-classification"], + sklearn: ["tabular-classification", "tabular-regression", "text-classification"], + spacy: ["token-classification", "text-classification", "sentence-similarity"], + "span-marker": ["token-classification"], + speechbrain: ["audio-classification", "audio-to-audio", "automatic-speech-recognition", "text-to-speech"], + stanza: ["token-classification"], + timm: ["image-classification", "image-feature-extraction"], + transformers: [ + "audio-classification", + "automatic-speech-recognition", + "depth-estimation", + "document-question-answering", + "feature-extraction", + "fill-mask", + "image-classification", + "image-feature-extraction", + "image-segmentation", + "image-to-image", + "image-to-text", + "image-text-to-text", + "mask-generation", + "object-detection", + "question-answering", + "summarization", + "table-question-answering", + "text-classification", + "text-generation", + "text-to-audio", + "text-to-speech", + "token-classification", + "translation", + "video-classification", + "visual-question-answering", + "zero-shot-classification", + "zero-shot-image-classification", + "zero-shot-object-detection", + ], + mindspore: ["image-classification"], +}; +// Pipeline types that were supported in legacy transformers versions (<5.0.0) +export const REMOVED_IN_V5_TRANSFORMERS_PIPELINES = ["image-to-text", "summarization", "translation"]; diff --git a/node_modules/@huggingface/tasks/dist/esm/local-apps.d.ts b/node_modules/@huggingface/tasks/dist/esm/local-apps.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..25a3af87e0fed557b022d70a16db032ec4732e23 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/local-apps.d.ts @@ -0,0 +1,265 @@ +import type { ModelData } from "./model-data.js"; +import type { PipelineType } from "./pipelines.js"; +export interface LocalAppSnippet { + /** + * Title of the snippet + */ + title: string; + /** + * Optional setup guide + */ + setup?: string; + /** + * Content (or command) to be run + */ + content: string | string[]; +} +/** + * Elements configurable by a local app. + */ +export type LocalApp = { + /** + * Name that appears in buttons + */ + prettyLabel: string; + /** + * Link to get more info about a local app (website etc) + */ + docsUrl: string; + /** + * Additional links to display (max 2) + */ + links?: { + label: string; + url: string; + }[] | ((model: ModelData) => { + label: string; + url: string; + }[]); + /** + * main category of app + */ + mainTask: PipelineType; + /** + * Whether to display a pill "macOS-only" + */ + macOSOnly?: boolean; + comingSoon?: boolean; + /** + * IMPORTANT: function to figure out whether to display the button on a model page's main "Use this model" dropdown. + */ + displayOnModelPage: (model: ModelData) => boolean; +} & ({ + /** + * If the app supports deeplink, URL to open. + */ + deeplink: (model: ModelData, filepath?: string) => URL; +} | { + /** + * And if not (mostly llama.cpp), snippet to copy/paste in your terminal + * Support the placeholder {{GGUF_FILE}} that will be replaced by the gguf file path or the list of available files. + * Support the placeholder {{QUANT_TAG}} that will be replaced by the list of available quant tags or will be removed if there are no multiple quant files in a same repo. + */ + snippet: (model: ModelData, filepath?: string) => string | string[] | LocalAppSnippet | LocalAppSnippet[]; +}); +declare function isTgiModel(model: ModelData): boolean; +declare function isLlamaCppGgufModel(model: ModelData): boolean; +declare function isVllmModel(model: ModelData): boolean; +declare function isDockerModelRunnerModel(model: ModelData): boolean; +declare function isUnslothModel(model: ModelData): boolean; +declare function isToolCallingLocalAgentModel(model: ModelData): boolean; +/** + * Add your new local app here. + * + * This is open to new suggestions and awesome upcoming apps. + * + * /!\ IMPORTANT + * + * If possible, you need to support deeplinks and be as cross-platform as possible. + * + * Ping the HF team if we can help with anything! + */ +export declare const LOCAL_APPS: { + "llama.cpp": { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[]; + }; + "node-llama-cpp": { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[]; + }; + vllm: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isVllmModel; + snippet: (model: ModelData) => LocalAppSnippet[]; + }; + sglang: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: (model: ModelData) => boolean; + snippet: (model: ModelData) => LocalAppSnippet[]; + }; + "mlx-lm": { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: (model: ModelData) => boolean; + snippet: (model: ModelData) => LocalAppSnippet[]; + }; + tgi: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isTgiModel; + snippet: (model: ModelData) => LocalAppSnippet[]; + }; + lmstudio: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: (model: ModelData) => boolean; + deeplink: (model: ModelData, filepath: string | undefined) => URL; + }; + localai: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[]; + }; + jan: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + deeplink: (model: ModelData) => URL; + }; + "atomic-chat": { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + deeplink: (model: ModelData) => URL; + }; + backyard: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + deeplink: (model: ModelData) => URL; + }; + sanctum: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + deeplink: (model: ModelData) => URL; + }; + jellybox: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: (model: ModelData) => boolean; + deeplink: (model: ModelData) => URL; + }; + msty: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + deeplink: (model: ModelData) => URL; + }; + recursechat: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + macOSOnly: true; + displayOnModelPage: typeof isLlamaCppGgufModel; + deeplink: (model: ModelData) => URL; + }; + drawthings: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-to-image"; + macOSOnly: true; + displayOnModelPage: (model: ModelData) => boolean; + deeplink: (model: ModelData) => URL; + }; + diffusionbee: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-to-image"; + macOSOnly: true; + displayOnModelPage: (model: ModelData) => boolean; + deeplink: (model: ModelData) => URL; + }; + joyfusion: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-to-image"; + macOSOnly: true; + displayOnModelPage: (model: ModelData) => boolean; + deeplink: (model: ModelData) => URL; + }; + ollama: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isLlamaCppGgufModel; + snippet: (model: ModelData, filepath?: string) => string; + }; + unsloth: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isUnslothModel; + snippet: (model: ModelData) => LocalAppSnippet[]; + }; + "docker-model-runner": { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isDockerModelRunnerModel; + snippet: (model: ModelData, filepath?: string) => string; + }; + lemonade: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: (model: ModelData) => boolean; + snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[]; + }; + pi: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isToolCallingLocalAgentModel; + snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[]; + }; + "hermes-agent": { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isToolCallingLocalAgentModel; + snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[]; + }; + openclaw: { + prettyLabel: string; + docsUrl: string; + mainTask: "text-generation"; + displayOnModelPage: typeof isToolCallingLocalAgentModel; + snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[]; + }; +}; +export type LocalAppKey = keyof typeof LOCAL_APPS; +export {}; +//# sourceMappingURL=local-apps.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/local-apps.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/local-apps.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d644c2d36e15262c1a8cb1e7a9e21cfbb3412310 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/local-apps.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"local-apps.d.ts","sourceRoot":"","sources":["../../src/local-apps.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAKnD,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACtB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,SAAS,KAAK;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAAC;IACpG;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IACvB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,kBAAkB,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC;CAClD,GAAG,CACD;IACA;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,CAAC;CACtD,GACD;IACA;;;;OAIG;IACH,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,EAAE,GAAG,eAAe,GAAG,eAAe,EAAE,CAAC;CACzG,CACH,CAAC;AAsBF,iBAAS,UAAU,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE7C;AAED,iBAAS,mBAAmB,CAAC,KAAK,EAAE,SAAS,WAE5C;AAED,iBAAS,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAU9C;AAED,iBAAS,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE3D;AA0BD,iBAAS,cAAc,CAAC,KAAK,EAAE,SAAS,WAEvC;AAED,iBAAS,4BAA4B,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAM/D;AA4dD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,UAAU;;;;;;yBA1dS,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAiDzC,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBA2F3D,SAAS,KAAG,eAAe,EAAE;;;;;;oCAwW3B,SAAS;yBAlTT,SAAS,KAAG,eAAe,EAAE;;;;;;;yBAoF9B,SAAS,KAAG,eAAe,EAAE;;;;;;;yBA7B/B,SAAS,KAAG,eAAe,EAAE;;;;;;;;;;;;;;yBApIzB,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAtDjD,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;yBAIpC,SAAS,KAAG,eAAe,EAAE;;;;;;;yBAgWnB,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;yBAM9C,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAlGtD,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAkCvC,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBA2BnD,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;CAqS5C,CAAC;AAErC,MAAM,MAAM,WAAW,GAAG,MAAM,OAAO,UAAU,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/local-apps.js b/node_modules/@huggingface/tasks/dist/esm/local-apps.js new file mode 100644 index 0000000000000000000000000000000000000000..3353ab3f89a0319c40a78eab1daf1b02da2a269d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/local-apps.js @@ -0,0 +1,713 @@ +import { parseGGUFQuantLabel } from "./gguf.js"; +import { stringifyMessages } from "./snippets/common.js"; +import { getModelInputSnippet } from "./snippets/inputs.js"; +function isAwqModel(model) { + return model.config?.quantization_config?.quant_method === "awq"; +} +function isGptqModel(model) { + return model.config?.quantization_config?.quant_method === "gptq"; +} +function isAqlmModel(model) { + return model.config?.quantization_config?.quant_method === "aqlm"; +} +function isMarlinModel(model) { + return model.config?.quantization_config?.quant_method === "marlin"; +} +function isTransformersModel(model) { + return model.tags.includes("transformers"); +} +function isTgiModel(model) { + return model.tags.includes("text-generation-inference"); +} +function isLlamaCppGgufModel(model) { + return !!model.gguf?.context_length; +} +function isVllmModel(model) { + return ((isAwqModel(model) || + isGptqModel(model) || + isAqlmModel(model) || + isMarlinModel(model) || + isLlamaCppGgufModel(model) || + isTransformersModel(model)) && + (model.pipeline_tag === "text-generation" || model.pipeline_tag === "image-text-to-text")); +} +function isDockerModelRunnerModel(model) { + return isLlamaCppGgufModel(model) || isVllmModel(model); +} +function isAmdRyzenModel(model) { + return model.tags.includes("ryzenai-hybrid") || model.tags.includes("ryzenai-npu"); +} +function isMlxModel(model) { + return model.tags.includes("mlx"); +} +/** + * Returns the model's chat template string, coalescing across sources: + * GGUF metadata > chat_template_jinja file > tokenizer_config.json + */ +function getChatTemplate(model) { + const ct = model.gguf?.chat_template ?? model.config?.chat_template_jinja ?? model.config?.tokenizer_config?.chat_template; + if (typeof ct === "string") { + return ct; + } + if (Array.isArray(ct)) { + return ct[0]?.template; + } + return undefined; +} +function isUnslothModel(model) { + return model.tags.includes("unsloth") || isLlamaCppGgufModel(model); +} +function isToolCallingLocalAgentModel(model) { + return ((isLlamaCppGgufModel(model) || isMlxModel(model)) && + model.tags.includes("conversational") && + !!getChatTemplate(model)?.includes("tools")); +} +function getQuantTag(filepath) { + const defaultTag = ":{{QUANT_TAG}}"; + if (!filepath) { + return defaultTag; + } + const quantLabel = parseGGUFQuantLabel(filepath); + return quantLabel ? `:${quantLabel}` : defaultTag; +} +const snippetLlamacpp = (model, filepath) => { + const serverCommand = (binary) => { + const snippet = [ + "# Start a local OpenAI-compatible server with a web UI:", + `${binary} -hf ${model.id}${getQuantTag(filepath)}`, + ]; + return snippet.join("\n"); + }; + const cliCommand = (binary) => { + const snippet = ["# Run inference directly in the terminal:", `${binary} -hf ${model.id}${getQuantTag(filepath)}`]; + return snippet.join("\n"); + }; + return [ + { + title: "Install (macOS, Linux)", + setup: "curl -LsSf https://llama.app/install.sh | sh", + content: [serverCommand("llama serve"), cliCommand("llama cli")], + }, + { + title: "Install from WinGet (Windows)", + setup: "winget install llama.cpp", + content: [serverCommand("llama serve"), cliCommand("llama cli")], + }, + { + title: "Use pre-built binary", + setup: [ + // prettier-ignore + "# Download pre-built binary from:", + "# https://github.com/ggerganov/llama.cpp/releases", + ].join("\n"), + content: [serverCommand("./llama-server"), cliCommand("./llama-cli")], + }, + { + title: "Build from source code", + setup: [ + "git clone https://github.com/ggerganov/llama.cpp.git", + "cd llama.cpp", + "cmake -B build", + "cmake --build build -j --target llama-server llama-cli", + ].join("\n"), + content: [serverCommand("./build/bin/llama-server"), cliCommand("./build/bin/llama-cli")], + }, + { + title: "Use Docker", + content: snippetDockerModelRunner(model, filepath), + }, + ]; +}; +const snippetNodeLlamaCppCli = (model, filepath) => { + const tagName = getQuantTag(filepath); + return [ + { + title: "Chat with the model", + content: `npx -y node-llama-cpp chat hf:${model.id}${tagName}`, + }, + { + title: "Estimate the model compatibility with your hardware", + content: `npx -y node-llama-cpp inspect estimate hf:${model.id}${tagName}`, + }, + ]; +}; +const snippetOllama = (model, filepath) => { + return `ollama run hf.co/${model.id}${getQuantTag(filepath)}`; +}; +const snippetUnsloth = (model) => { + const isGguf = isLlamaCppGgufModel(model); + const studio_content = [ + "# Run unsloth studio", + "unsloth studio -H 0.0.0.0 -p 8888", + "# Then open http://localhost:8888 in your browser", + "# Search for " + model.id + " to start chatting", + ].join("\n"); + const studio_instructions = { + title: "Install Unsloth Studio (macOS, Linux, WSL)", + setup: "curl -fsSL https://unsloth.ai/install.sh | sh", + content: studio_content, + }; + const studio_instructions_windows = { + title: "Install Unsloth Studio (Windows)", + setup: "irm https://unsloth.ai/install.ps1 | iex", + content: studio_content, + }; + const hf_spaces_instructions = { + title: "Using HuggingFace Spaces for Unsloth", + setup: "# No setup required", + content: "# Open https://huggingface.co/spaces/unsloth/studio in your browser\n# Search for " + + model.id + + " to start chatting", + }; + const fastmodel_instructions = { + title: "Load model with FastModel", + setup: "pip install unsloth", + content: [ + "from unsloth import FastModel", + "model, tokenizer = FastModel.from_pretrained(", + ' model_name="' + model.id + '",', + " max_seq_length=2048,", + ")", + ].join("\n"), + }; + if (isGguf) { + return [studio_instructions, studio_instructions_windows, hf_spaces_instructions]; + } + else { + return [studio_instructions, studio_instructions_windows, hf_spaces_instructions, fastmodel_instructions]; + } +}; +const snippetLocalAI = (model, filepath) => { + const command = (binary) => ["# Load and run the model:", `${binary} huggingface://${model.id}/${filepath ?? "{{GGUF_FILE}}"}`].join("\n"); + return [ + { + title: "Install from binary", + setup: "curl https://localai.io/install.sh | sh", + content: command("local-ai run"), + }, + { + title: "Use Docker images", + setup: [ + // prettier-ignore + "# Pull the image:", + "docker pull localai/localai:latest-cpu", + ].join("\n"), + content: command("docker run -p 8080:8080 --name localai -v $PWD/models:/build/models localai/localai:latest-cpu"), + }, + ]; +}; +const snippetVllm = (model) => { + const messages = getModelInputSnippet(model); + const isMistral = model.tags.includes("mistral-common"); + const mistralFlags = isMistral + ? " --tokenizer_mode mistral --config_format mistral --load_format mistral --tool-call-parser mistral --enable-auto-tool-choice" + : ""; + const setup = isMistral + ? [ + "# Install vLLM from pip:", + "pip install vllm", + "# Install mistral-common:", + "pip install --upgrade mistral-common", + ].join("\n") + : ["# Install vLLM from pip:", "pip install vllm"].join("\n"); + const serverCommand = `# Start the vLLM server: +vllm serve "${model.id}"${mistralFlags}`; + const runCommandInstruct = `# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/chat/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "${model.id}", + "messages": ${stringifyMessages(messages, { + indent: "\t\t", + attributeKeyQuotes: true, + customContentEscaper: (str) => str.replace(/'/g, "'\\''"), + })} + }'`; + const runCommandNonInstruct = `# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "${model.id}", + "prompt": "Once upon a time,", + "max_tokens": 512, + "temperature": 0.5 + }'`; + const runCommand = model.tags.includes("conversational") ? runCommandInstruct : runCommandNonInstruct; + return [ + { + title: "Install from pip and serve model", + setup: setup, + content: [serverCommand, runCommand], + }, + { + title: "Use Docker", + content: snippetDockerModelRunner(model), + }, + ]; +}; +const snippetSglang = (model) => { + const messages = getModelInputSnippet(model); + const setup = ["# Install SGLang from pip:", "pip install sglang"].join("\n"); + const serverCommand = `# Start the SGLang server: +python3 -m sglang.launch_server \\ + --model-path "${model.id}" \\ + --host 0.0.0.0 \\ + --port 30000`; + const dockerCommand = `docker run --gpus all \\ + --shm-size 32g \\ + -p 30000:30000 \\ + -v ~/.cache/huggingface:/root/.cache/huggingface \\ + --env "HF_TOKEN=" \\ + --ipc=host \\ + lmsysorg/sglang:latest \\ + python3 -m sglang.launch_server \\ + --model-path "${model.id}" \\ + --host 0.0.0.0 \\ + --port 30000`; + const runCommandInstruct = `# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:30000/v1/chat/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "${model.id}", + "messages": ${stringifyMessages(messages, { + indent: "\t\t", + attributeKeyQuotes: true, + customContentEscaper: (str) => str.replace(/'/g, "'\\''"), + })} + }'`; + const runCommandNonInstruct = `# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:30000/v1/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "${model.id}", + "prompt": "Once upon a time,", + "max_tokens": 512, + "temperature": 0.5 + }'`; + const runCommand = model.tags.includes("conversational") ? runCommandInstruct : runCommandNonInstruct; + return [ + { + title: "Install from pip and serve model", + setup: setup, + content: [serverCommand, runCommand], + }, + { + title: "Use Docker images", + setup: dockerCommand, + content: [runCommand], + }, + ]; +}; +const snippetTgi = (model) => { + const runCommand = [ + "# Call the server using curl:", + `curl -X POST "http://localhost:8000/v1/chat/completions" \\`, + ` -H "Content-Type: application/json" \\`, + ` --data '{`, + ` "model": "${model.id}",`, + ` "messages": [`, + ` {"role": "user", "content": "What is the capital of France?"}`, + ` ]`, + ` }'`, + ]; + return [ + { + title: "Use Docker images", + setup: [ + "# Deploy with docker on Linux:", + `docker run --gpus all \\`, + ` -v ~/.cache/huggingface:/root/.cache/huggingface \\`, + ` -e HF_TOKEN="" \\`, + ` -p 8000:80 \\`, + ` ghcr.io/huggingface/text-generation-inference:latest \\`, + ` --model-id ${model.id}`, + ].join("\n"), + content: [runCommand.join("\n")], + }, + ]; +}; +const snippetMlxLm = (model) => { + const openaiCurl = [ + "# Calling the OpenAI-compatible server with curl", + `curl -X POST "http://localhost:8000/v1/chat/completions" \\`, + ` -H "Content-Type: application/json" \\`, + ` --data '{`, + ` "model": "${model.id}",`, + ` "messages": [`, + ` {"role": "user", "content": "Hello"}`, + ` ]`, + ` }'`, + ]; + return [ + { + title: "Generate or start a chat session", + setup: ["# Install MLX LM", "uv tool install mlx-lm"].join("\n"), + content: [ + ...(model.tags.includes("conversational") + ? ["# Interactive chat REPL", `mlx_lm.chat --model "${model.id}"`] + : ["# Generate some text", `mlx_lm.generate --model "${model.id}" --prompt "Once upon a time"`]), + ].join("\n"), + }, + ...(model.tags.includes("conversational") + ? [ + { + title: "Run an OpenAI-compatible server", + setup: ["# Install MLX LM", "uv tool install mlx-lm"].join("\n"), + content: ["# Start the server", `mlx_lm.server --model "${model.id}"`, ...openaiCurl].join("\n"), + }, + ] + : []), + ]; +}; +const getLocalServerStep = (model, filepath) => { + return isMlxModel(model) + ? { + title: "Start the MLX server", + setup: "# Install MLX LM:\nuv tool install mlx-lm", + content: `# Start a local OpenAI-compatible server:\nmlx_lm.server --model "${model.id}"`, + } + : { + title: "Start the llama.cpp server", + setup: "# Install llama.cpp:\nbrew install llama.cpp", + content: `# Start a local OpenAI-compatible server:\nllama serve -hf ${model.id}${getQuantTag(filepath)}`, + }; +}; +const snippetPi = (model, filepath) => { + const isMLX = isMlxModel(model); + const modelId = isMLX ? model.id : `${model.id}${getQuantTag(filepath)}`; + const serverStep = getLocalServerStep(model, filepath); + const modelsJson = JSON.stringify({ + providers: { + [isMLX ? "mlx-lm" : "llama-cpp"]: { + baseUrl: "http://localhost:8080/v1", + api: "openai-completions", + apiKey: "none", + models: [{ id: modelId }], + }, + }, + }, null, 2); + return [ + serverStep, + { + title: "Configure the model in Pi", + setup: "# Install Pi:\nnpm install -g @mariozechner/pi-coding-agent", + content: `# Add to ~/.pi/agent/models.json:\n${modelsJson}`, + }, + { + title: "Run Pi", + content: "# Start Pi in your project directory:\npi", + }, + ]; +}; +const snippetHermesAgent = (model, filepath) => { + const modelId = isMlxModel(model) ? model.id : `${model.id}${getQuantTag(filepath)}`; + const serverStep = getLocalServerStep(model, filepath); + return [ + serverStep, + { + title: "Configure Hermes", + setup: [ + "# Install Hermes:", + "curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash", + "hermes setup", + ].join("\n"), + content: [ + "# Point Hermes at the local server:", + "hermes config set model.provider custom", + "hermes config set model.base_url http://127.0.0.1:8080/v1", + `hermes config set model.default ${modelId}`, + ].join("\n"), + }, + { + title: "Run Hermes", + content: "hermes", + }, + ]; +}; +const snippetOpenClaw = (model, filepath) => { + const isMLX = isMlxModel(model); + const providerId = isMLX ? "mlx-lm" : "llama-cpp"; + const modelId = isMLX ? model.id : `${model.id}${getQuantTag(filepath)}`; + const serverStep = getLocalServerStep(model, filepath); + return [ + serverStep, + { + title: "Configure OpenClaw", + setup: "# Install OpenClaw:\nnpm install -g openclaw@latest", + content: [ + "# Register the local server and set it as the default model:", + "openclaw onboard --non-interactive --mode local \\", + " --auth-choice custom-api-key \\", + " --custom-base-url http://127.0.0.1:8080/v1 \\", + ` --custom-model-id "${modelId}" \\`, + ` --custom-provider-id ${providerId} \\`, + " --custom-compatibility openai \\", + " --custom-text-input \\", + " --accept-risk \\", + " --skip-health", + ].join("\n"), + }, + { + title: "Run OpenClaw", + content: `openclaw agent --local --agent main --message "Hello from Hugging Face"`, + }, + ]; +}; +const snippetDockerModelRunner = (model, filepath) => { + // Only add quant tag for GGUF models, not safetensors + const quantTag = isLlamaCppGgufModel(model) ? getQuantTag(filepath) : ""; + return `docker model run hf.co/${model.id}${quantTag}`; +}; +const snippetLemonade = (model, filepath) => { + const modelName = model.id.includes("/") ? model.id.split("/")[1] : model.id; + const isRyzenAI = model.tags.some((tag) => ["ryzenai-npu", "ryzenai-hybrid"].includes(tag)); + // Lemonade auto-registers pulled models as `user.[-]`. + // For GGUF/llamacpp: suggested_name is the repo name and variant is the quant tag. + // For RyzenAI ONNX: there is no per-variant suffix. + let pullArg; + let runName; + let requirements; + if (isRyzenAI) { + pullArg = model.id; + runName = `user.${modelName}`; + requirements = " (requires XDNA 2 NPU)"; + } + else { + const tagName = getQuantTag(filepath); + pullArg = `${model.id}${tagName}`; + runName = `user.${modelName}${tagName.replace(":", "-")}`; + requirements = ""; + } + return [ + { + title: "Pull the model", + setup: "# Download Lemonade from https://lemonade-server.ai/", + content: `lemonade pull ${pullArg}`, + }, + { + title: `Run and chat with the model${requirements}`, + content: `lemonade run ${runName}`, + }, + { + title: "List all available models", + content: "lemonade list", + }, + ]; +}; +/** + * Add your new local app here. + * + * This is open to new suggestions and awesome upcoming apps. + * + * /!\ IMPORTANT + * + * If possible, you need to support deeplinks and be as cross-platform as possible. + * + * Ping the HF team if we can help with anything! + */ +export const LOCAL_APPS = { + "llama.cpp": { + prettyLabel: "llama.cpp", + docsUrl: "https://github.com/ggerganov/llama.cpp", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + snippet: snippetLlamacpp, + }, + "node-llama-cpp": { + prettyLabel: "node-llama-cpp", + docsUrl: "https://node-llama-cpp.withcat.ai", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + snippet: snippetNodeLlamaCppCli, + }, + vllm: { + prettyLabel: "vLLM", + docsUrl: "https://docs.vllm.ai", + mainTask: "text-generation", + displayOnModelPage: isVllmModel, + snippet: snippetVllm, + }, + sglang: { + prettyLabel: "SGLang", + docsUrl: "https://docs.sglang.io", + mainTask: "text-generation", + displayOnModelPage: (model) => (isAwqModel(model) || + isGptqModel(model) || + isAqlmModel(model) || + isMarlinModel(model) || + isTransformersModel(model)) && + (model.pipeline_tag === "text-generation" || model.pipeline_tag === "image-text-to-text"), + snippet: snippetSglang, + }, + "mlx-lm": { + prettyLabel: "MLX LM", + docsUrl: "https://github.com/ml-explore/mlx-lm", + mainTask: "text-generation", + displayOnModelPage: (model) => model.pipeline_tag === "text-generation" && isMlxModel(model), + snippet: snippetMlxLm, + }, + tgi: { + prettyLabel: "TGI", + docsUrl: "https://huggingface.co/docs/text-generation-inference/", + mainTask: "text-generation", + displayOnModelPage: isTgiModel, + snippet: snippetTgi, + }, + lmstudio: { + prettyLabel: "LM Studio", + docsUrl: "https://lmstudio.ai", + mainTask: "text-generation", + displayOnModelPage: (model) => isLlamaCppGgufModel(model) || isMlxModel(model), + deeplink: (model, filepath) => new URL(`lmstudio://open_from_hf?model=${model.id}${filepath ? `&file=${filepath}` : ""}`), + }, + localai: { + prettyLabel: "LocalAI", + docsUrl: "https://github.com/mudler/LocalAI", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + snippet: snippetLocalAI, + }, + jan: { + prettyLabel: "Jan", + docsUrl: "https://jan.ai", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`jan://models/huggingface/${model.id}`), + }, + "atomic-chat": { + prettyLabel: "Atomic Chat", + docsUrl: "https://atomic.chat", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`atomic-chat://models/huggingface/${model.id}`), + }, + backyard: { + prettyLabel: "Backyard AI", + docsUrl: "https://backyard.ai", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`https://backyard.ai/hf/model/${model.id}`), + }, + sanctum: { + prettyLabel: "Sanctum", + docsUrl: "https://sanctum.ai", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`sanctum://open_from_hf?model=${model.id}`), + }, + jellybox: { + prettyLabel: "Jellybox", + docsUrl: "https://jellybox.com", + mainTask: "text-generation", + displayOnModelPage: (model) => isLlamaCppGgufModel(model) || + (model.library_name === "diffusers" && + model.tags.includes("safetensors") && + (model.pipeline_tag === "text-to-image" || model.tags.includes("lora"))), + deeplink: (model) => { + if (isLlamaCppGgufModel(model)) { + return new URL(`jellybox://llm/models/huggingface/LLM/${model.id}`); + } + else if (model.tags.includes("lora")) { + return new URL(`jellybox://image/models/huggingface/ImageLora/${model.id}`); + } + else { + return new URL(`jellybox://image/models/huggingface/Image/${model.id}`); + } + }, + }, + msty: { + prettyLabel: "Msty", + docsUrl: "https://msty.app", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`msty://models/search/hf/${model.id}`), + }, + recursechat: { + prettyLabel: "RecurseChat", + docsUrl: "https://recurse.chat", + mainTask: "text-generation", + macOSOnly: true, + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`recursechat://new-hf-gguf-model?hf-model-id=${model.id}`), + }, + drawthings: { + prettyLabel: "Draw Things", + docsUrl: "https://drawthings.ai", + mainTask: "text-to-image", + macOSOnly: true, + displayOnModelPage: (model) => model.library_name === "diffusers" && (model.pipeline_tag === "text-to-image" || model.tags.includes("lora")), + deeplink: (model) => { + if (model.tags.includes("lora")) { + return new URL(`https://drawthings.ai/import/diffusers/pipeline.load_lora_weights?repo_id=${model.id}`); + } + else { + return new URL(`https://drawthings.ai/import/diffusers/pipeline.from_pretrained?repo_id=${model.id}`); + } + }, + }, + diffusionbee: { + prettyLabel: "DiffusionBee", + docsUrl: "https://diffusionbee.com", + mainTask: "text-to-image", + macOSOnly: true, + displayOnModelPage: (model) => model.library_name === "diffusers" && model.pipeline_tag === "text-to-image", + deeplink: (model) => new URL(`https://diffusionbee.com/huggingface_import?model_id=${model.id}`), + }, + joyfusion: { + prettyLabel: "JoyFusion", + docsUrl: "https://joyfusion.app", + mainTask: "text-to-image", + macOSOnly: true, + displayOnModelPage: (model) => model.tags.includes("coreml") && model.tags.includes("joyfusion") && model.pipeline_tag === "text-to-image", + deeplink: (model) => new URL(`https://joyfusion.app/import_from_hf?repo_id=${model.id}`), + }, + ollama: { + prettyLabel: "Ollama", + docsUrl: "https://ollama.com", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + snippet: snippetOllama, + }, + unsloth: { + prettyLabel: "Unsloth Studio", + docsUrl: "https://unsloth.ai/docs/new/studio", + mainTask: "text-generation", + displayOnModelPage: isUnslothModel, + snippet: snippetUnsloth, + }, + "docker-model-runner": { + prettyLabel: "Docker Model Runner", + docsUrl: "https://docs.docker.com/ai/model-runner/", + mainTask: "text-generation", + displayOnModelPage: isDockerModelRunnerModel, + snippet: snippetDockerModelRunner, + }, + lemonade: { + prettyLabel: "Lemonade", + docsUrl: "https://lemonade-server.ai", + mainTask: "text-generation", + displayOnModelPage: (model) => isLlamaCppGgufModel(model) || isAmdRyzenModel(model), + snippet: snippetLemonade, + }, + pi: { + prettyLabel: "Pi", + docsUrl: "https://github.com/badlogic/pi-mono", + mainTask: "text-generation", + displayOnModelPage: isToolCallingLocalAgentModel, + snippet: snippetPi, + }, + "hermes-agent": { + prettyLabel: "Hermes Agent", + docsUrl: "https://hermes-agent.nousresearch.com/", + mainTask: "text-generation", + displayOnModelPage: isToolCallingLocalAgentModel, + snippet: snippetHermesAgent, + }, + openclaw: { + prettyLabel: "OpenClaw", + docsUrl: "https://github.com/openclaw/openclaw", + mainTask: "text-generation", + displayOnModelPage: isToolCallingLocalAgentModel, + snippet: snippetOpenClaw, + }, +}; diff --git a/node_modules/@huggingface/tasks/dist/esm/local-apps.spec.d.ts b/node_modules/@huggingface/tasks/dist/esm/local-apps.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b894adb772fca8fe09665082d1611092f3acc156 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/local-apps.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=local-apps.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/local-apps.spec.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/local-apps.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..dc8a5d8f78e2829b7e3a97d27fe96149050c6a7b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/local-apps.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"local-apps.spec.d.ts","sourceRoot":"","sources":["../../src/local-apps.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/local-apps.spec.js b/node_modules/@huggingface/tasks/dist/esm/local-apps.spec.js new file mode 100644 index 0000000000000000000000000000000000000000..221d627289a785550b3ed12561c3eb22e601fb85 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/local-apps.spec.js @@ -0,0 +1,317 @@ +import { describe, expect, it } from "vitest"; +import { LOCAL_APPS } from "./local-apps.js"; +describe("local-apps", () => { + it("llama.cpp conversational", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["llama.cpp"]; + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + inference: "", + }; + const snippet = snippetFunc(model); + expect(snippet[0].content).toEqual([ + `# Start a local OpenAI-compatible server with a web UI: +llama serve -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`, + `# Run inference directly in the terminal: +llama cli -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`, + ]); + }); + it("llama.cpp non-conversational", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["llama.cpp"]; + const model = { + id: "mlabonne/gemma-2b-GGUF", + tags: [], + inference: "", + }; + const snippet = snippetFunc(model); + expect(snippet[0].content).toEqual([ + `# Start a local OpenAI-compatible server with a web UI: +llama serve -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}}`, + `# Run inference directly in the terminal: +llama cli -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}}`, + ]); + }); + it("vLLM conversational llm", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["vllm"]; + const model = { + id: "meta-llama/Llama-3.2-3B-Instruct", + pipeline_tag: "text-generation", + tags: ["conversational"], + inference: "", + }; + const snippet = snippetFunc(model); + expect(snippet[0].content.join("\n")).toEqual(`# Start the vLLM server: +vllm serve "meta-llama/Llama-3.2-3B-Instruct" +# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/chat/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "meta-llama/Llama-3.2-3B-Instruct", + "messages": [ + { + "role": "user", + "content": "What is the capital of France?" + } + ] + }'`); + }); + it("vLLM non-conversational llm", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["vllm"]; + const model = { + id: "meta-llama/Llama-3.2-3B", + tags: [""], + inference: "", + }; + const snippet = snippetFunc(model); + expect(snippet[0].content.join("\n")).toEqual(`# Start the vLLM server: +vllm serve "meta-llama/Llama-3.2-3B" +# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "meta-llama/Llama-3.2-3B", + "prompt": "Once upon a time,", + "max_tokens": 512, + "temperature": 0.5 + }'`); + }); + it("vLLM conversational vlm", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["vllm"]; + const model = { + id: "meta-llama/Llama-3.2-11B-Vision-Instruct", + pipeline_tag: "image-text-to-text", + tags: ["conversational"], + inference: "", + }; + const snippet = snippetFunc(model); + expect(snippet[0].content.join("\n")).toEqual(`# Start the vLLM server: +vllm serve "meta-llama/Llama-3.2-11B-Vision-Instruct" +# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/chat/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "meta-llama/Llama-3.2-11B-Vision-Instruct", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe this image in one sentence." + }, + { + "type": "image_url", + "image_url": { + "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" + } + } + ] + } + ] + }'`); + }); + it("pi", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["pi"]; + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096, chat_template: "{% if tools %}" }, + inference: "", + }; + const snippet = snippetFunc(model); + expect(snippet[0].content).toContain(`llama serve -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); + expect(snippet[1].setup).toContain("npm install -g @mariozechner/pi-coding-agent"); + expect(snippet[1].content).toContain(`"id": "bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}"`); + expect(snippet[2].content).toContain("pi"); + }); + it("pi - mlx", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["pi"]; + const model = { + id: "mlx-community/Llama-3.2-3B-Instruct-mlx", + tags: ["mlx", "conversational"], + pipeline_tag: "text-generation", + config: { + tokenizer_config: { + chat_template: "{% if tools %}...{% endif %}", + }, + }, + inference: "", + }; + const snippet = snippetFunc(model); + expect(snippet[0].setup).toContain("uv tool install mlx-lm"); + expect(snippet[0].content).toContain('mlx_lm.server --model "mlx-community/Llama-3.2-3B-Instruct-mlx"'); + expect(snippet[1].setup).toContain("npm install -g @mariozechner/pi-coding-agent"); + expect(snippet[1].content).toContain('"baseUrl": "http://localhost:8080/v1"'); + expect(snippet[1].content).toContain('"id": "mlx-community/Llama-3.2-3B-Instruct-mlx"'); + expect(snippet[2].content).toContain("pi"); + }); + it("hermes-agent", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["hermes-agent"]; + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096, chat_template: "{% if tools %}" }, + inference: "", + }; + const snippet = snippetFunc(model); + expect(snippet[0].content).toContain(`llama serve -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); + expect(snippet[1].content).toContain("hermes config set model.provider custom"); + expect(snippet[1].content).toContain("hermes config set model.base_url http://127.0.0.1:8080/v1"); + expect(snippet[1].content).toContain("hermes config set model.default bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}"); + expect(snippet[2].content).toContain("hermes"); + }); + it("hermes-agent - mlx", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["hermes-agent"]; + const model = { + id: "mlx-community/Llama-3.2-3B-Instruct-mlx", + tags: ["mlx", "conversational"], + pipeline_tag: "text-generation", + config: { + tokenizer_config: { + chat_template: "{% if tools %}...{% endif %}", + }, + }, + inference: "", + }; + const snippet = snippetFunc(model); + expect(snippet[0].setup).toContain("uv tool install mlx-lm"); + expect(snippet[1].content).toContain("hermes config set model.provider custom"); + expect(snippet[1].content).toContain("hermes config set model.default mlx-community/Llama-3.2-3B-Instruct-mlx"); + expect(snippet[2].content).toContain("hermes"); + }); + it("openclaw", async () => { + const { snippet: snippetFunc } = LOCAL_APPS.openclaw; + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096, chat_template: "{% if tools %}" }, + inference: "", + }; + const snippet = snippetFunc(model); + expect(snippet[0].content).toContain(`llama-server -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); + expect(snippet[1].setup).toContain("npm install -g openclaw@latest"); + expect(snippet[1].content).toContain("openclaw onboard --non-interactive --mode local"); + expect(snippet[1].content).toContain("--auth-choice custom-api-key"); + expect(snippet[1].content).toContain("--custom-base-url http://127.0.0.1:8080/v1"); + expect(snippet[1].content).toContain('--custom-model-id "bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}"'); + expect(snippet[1].content).toContain("--custom-provider-id llama-cpp"); + expect(snippet[1].content).toContain("--custom-compatibility openai"); + expect(snippet[1].content).not.toContain("--custom-api-key"); + expect(snippet[1].content).toContain("--custom-text-input"); + expect(snippet[1].content).toContain("--accept-risk"); + expect(snippet[1].content).toContain("--skip-health"); + expect(snippet[2].content).toContain('openclaw agent --local --agent main --message "Hello from Hugging Face"'); + }); + it("openclaw - mlx", async () => { + const { snippet: snippetFunc } = LOCAL_APPS.openclaw; + const model = { + id: "mlx-community/Llama-3.2-3B-Instruct-mlx", + tags: ["mlx", "conversational"], + pipeline_tag: "text-generation", + config: { + tokenizer_config: { + chat_template: "{% if tools %}...{% endif %}", + }, + }, + inference: "", + }; + const snippet = snippetFunc(model); + expect(snippet[0].setup).toContain("uv tool install mlx-lm"); + expect(snippet[1].content).toContain("openclaw onboard --non-interactive --mode local"); + expect(snippet[1].content).toContain('--custom-model-id "mlx-community/Llama-3.2-3B-Instruct-mlx"'); + expect(snippet[1].content).toContain("--custom-provider-id mlx-lm"); + expect(snippet[1].content).toContain("--custom-text-input"); + expect(snippet[2].content).toContain('openclaw agent --local --agent main --message "Hello from Hugging Face"'); + }); + it("docker model runner", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["docker-model-runner"]; + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096 }, + inference: "", + }; + const snippet = snippetFunc(model); + expect(snippet).toEqual(`docker model run hf.co/bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); + }); + it("atomic chat deeplink", async () => { + const { displayOnModelPage, deeplink } = LOCAL_APPS["atomic-chat"]; + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096 }, + inference: "", + }; + expect(displayOnModelPage(model)).toBe(true); + expect(deeplink(model).href).toBe("atomic-chat://models/huggingface/bartowski/Llama-3.2-3B-Instruct-GGUF"); + }); + it("unsloth tagged model", async () => { + const { displayOnModelPage, snippet: snippetFunc } = LOCAL_APPS.unsloth; + const model = { + id: "some-user/my-unsloth-finetune", + tags: ["unsloth", "conversational"], + inference: "", + }; + expect(displayOnModelPage(model)).toBe(true); + const snippet = snippetFunc(model); + expect(snippet[0].setup).toBe("curl -fsSL https://unsloth.ai/install.sh | sh"); + expect(snippet[0].content).toBe("# Run unsloth studio\nunsloth studio -H 0.0.0.0 -p 8888\n# Then open http://localhost:8888 in your browser\n# Search for some-user/my-unsloth-finetune to start chatting"); + expect(snippet[1].setup).toBe("irm https://unsloth.ai/install.ps1 | iex"); + expect(snippet[1].content).toBe(snippet[0].content); + expect(snippet[2].setup).toBe("# No setup required"); + expect(snippet[2].content).toBe("# Open https://huggingface.co/spaces/unsloth/studio in your browser\n# Search for some-user/my-unsloth-finetune to start chatting"); + expect(snippet[3].setup).toBe("pip install unsloth"); + expect(snippet[3].content).toBe('from unsloth import FastModel\nmodel, tokenizer = FastModel.from_pretrained(\n model_name="some-user/my-unsloth-finetune",\n max_seq_length=2048,\n)'); + }); + it("unsloth namespace gguf model", async () => { + const { displayOnModelPage, snippet: snippetFunc } = LOCAL_APPS.unsloth; + const model = { + id: "unsloth/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096 }, + inference: "", + }; + expect(displayOnModelPage(model)).toBe(true); + const snippet = snippetFunc(model); + expect(snippet[0].setup).toBe("curl -fsSL https://unsloth.ai/install.sh | sh"); + expect(snippet[0].content).toBe("# Run unsloth studio\nunsloth studio -H 0.0.0.0 -p 8888\n# Then open http://localhost:8888 in your browser\n# Search for unsloth/Llama-3.2-3B-Instruct-GGUF to start chatting"); + expect(snippet[1].setup).toBe("irm https://unsloth.ai/install.ps1 | iex"); + expect(snippet[1].content).toBe(snippet[0].content); + expect(snippet[2].setup).toBe("# No setup required"); + expect(snippet[2].content).toBe("# Open https://huggingface.co/spaces/unsloth/studio in your browser\n# Search for unsloth/Llama-3.2-3B-Instruct-GGUF to start chatting"); + expect(snippet).toHaveLength(3); // GGUF models only get 3 snippets + }); + it("non unsloth namespace gguf model", async () => { + const { displayOnModelPage } = LOCAL_APPS.unsloth; + const model = { + id: "dummy/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096 }, + inference: "", + }; + expect(displayOnModelPage(model)).toBe(true); + }); + it("unsloth not shown for unrelated model", async () => { + const { displayOnModelPage } = LOCAL_APPS.unsloth; + const model = { + id: "meta-llama/Llama-3.2-3B-Instruct", + tags: ["conversational"], + inference: "", + }; + expect(displayOnModelPage(model)).toBe(false); + }); + it("links as a function", async () => { + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + inference: "", + }; + const appWithFnLinks = { + ...LOCAL_APPS["llama.cpp"], + links: (m) => [{ label: "Releases", url: `https://github.com/${m.id}/releases` }], + }; + expect(appWithFnLinks.links(model)).toEqual([ + { label: "Releases", url: "https://github.com/bartowski/Llama-3.2-3B-Instruct-GGUF/releases" }, + ]); + }); +}); diff --git a/node_modules/@huggingface/tasks/dist/esm/model-data.d.ts b/node_modules/@huggingface/tasks/dist/esm/model-data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fdb7d166b5308b6102fb8bceef83b41360f1873b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-data.d.ts @@ -0,0 +1,154 @@ +import type { PipelineType } from "./pipelines.js"; +import type { WidgetExample } from "./widget-example.js"; +import type { TokenizerConfig } from "./tokenizer-data.js"; +/** + * Public interface for model metadata + */ +export interface ModelData { + /** + * id of model (e.g. 'user/repo_name') + */ + id: string; + /** + * Whether or not to enable inference widget for this model + * TODO(type it) + */ + inference: string; + /** + * is this model private? + */ + private?: boolean; + /** + * this dictionary has useful information about the model configuration + */ + config?: { + architectures?: string[]; + /** + * Dict of AutoModel or Auto… class name to local import path in the repo + */ + auto_map?: { + /** + * String Property + */ + [x: string]: string; + }; + model_type?: string; + quantization_config?: { + bits?: number; + load_in_4bit?: boolean; + load_in_8bit?: boolean; + /** + * awq, gptq, aqlm, marlin, … Used by vLLM + */ + quant_method?: string; + }; + tokenizer_config?: TokenizerConfig; + processor_config?: { + chat_template?: string; + }; + chat_template_jinja?: string; + adapter_transformers?: { + model_name?: string; + model_class?: string; + }; + diffusers?: { + _class_name?: string; + }; + sklearn?: { + model?: { + file?: string; + }; + model_format?: string; + }; + speechbrain?: { + speechbrain_interface?: string; + vocoder_interface?: string; + vocoder_model_id?: string; + }; + peft?: { + base_model_name_or_path?: string; + task_type?: string; + }; + keras_hub?: { + tasks?: string[]; + }; + }; + /** + * all the model tags + */ + tags: string[]; + /** + * transformers-specific info to display in the code sample. + */ + transformersInfo?: TransformersInfo; + /** + * Pipeline type + */ + pipeline_tag?: PipelineType | undefined; + /** + * for relevant models, get mask token + */ + mask_token?: string | undefined; + /** + * Example data that will be fed into the widget. + * + * can be set in the model card metadata (under `widget`), + * or by default in `DefaultWidget.ts` + */ + widgetData?: WidgetExample[] | undefined; + /** + * Parameters that will be used by the widget when calling Inference API (serverless) + * https://huggingface.co/docs/api-inference/detailed_parameters + * + * can be set in the model card metadata (under `inference/parameters`) + * Example: + * inference: + * parameters: + * key: val + */ + cardData?: { + inference?: boolean | { + parameters?: Record; + }; + base_model?: string | string[]; + instance_prompt?: string | null; + }; + /** + * Library name + * Example: transformers, SpeechBrain, Stanza, etc. + */ + library_name?: string; + safetensors?: { + parameters: Record; + total: number; + sharded: boolean; + }; + gguf?: { + total: number; + architecture?: string; + context_length?: number; + chat_template?: string; + }; +} +/** + * transformers-specific info to display in the code sample. + */ +export interface TransformersInfo { + /** + * e.g. AutoModelForSequenceClassification + */ + auto_model: string; + /** + * if set in config.json's auto_map + */ + custom_class?: string; + /** + * e.g. text-classification + */ + pipeline_tag?: PipelineType; + /** + * e.g. "AutoTokenizer" | "AutoFeatureExtractor" | "AutoProcessor" + */ + processor?: string; +} +//# sourceMappingURL=model-data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/model-data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/model-data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..ad4e813ed88e562a7293a5858d4485de7e35d6b6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"model-data.d.ts","sourceRoot":"","sources":["../../src/model-data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAE3D;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IACX;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE;QACR,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;QACzB;;WAEG;QACH,QAAQ,CAAC,EAAE;YACV;;eAEG;YACH,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;SACpB,CAAC;QACF,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,mBAAmB,CAAC,EAAE;YACrB,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB;;eAEG;YACH,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,gBAAgB,CAAC,EAAE,eAAe,CAAC;QACnC,gBAAgB,CAAC,EAAE;YAClB,aAAa,CAAC,EAAE,MAAM,CAAC;SACvB,CAAC;QACF,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,oBAAoB,CAAC,EAAE;YACtB,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,SAAS,CAAC,EAAE;YACX,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,OAAO,CAAC,EAAE;YACT,KAAK,CAAC,EAAE;gBACP,IAAI,CAAC,EAAE,MAAM,CAAC;aACd,CAAC;YACF,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,WAAW,CAAC,EAAE;YACb,qBAAqB,CAAC,EAAE,MAAM,CAAC;YAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;YAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;SAC1B,CAAC;QACF,IAAI,CAAC,EAAE;YACN,uBAAuB,CAAC,EAAE,MAAM,CAAC;YACjC,SAAS,CAAC,EAAE,MAAM,CAAC;SACnB,CAAC;QACF,SAAS,CAAC,EAAE;YACX,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC;KACF,CAAC;IACF;;OAEG;IACH,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;;OAEG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACxC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC;;;;;OAKG;IACH,UAAU,CAAC,EAAE,aAAa,EAAE,GAAG,SAAS,CAAC;IACzC;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE;QACV,SAAS,CAAC,EACP,OAAO,GACP;YACA,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACpC,CAAC;QACL,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;QAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAChC,CAAC;IACF;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE;QACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,OAAO,CAAC;KACjB,CAAC;IACF,IAAI,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,aAAa,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/model-data.js b/node_modules/@huggingface/tasks/dist/esm/model-data.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-data.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/model-libraries-downloads.d.ts b/node_modules/@huggingface/tasks/dist/esm/model-libraries-downloads.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2068624044fa42a89cd7889633465fa802c9ac71 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-libraries-downloads.d.ts @@ -0,0 +1,18 @@ +/** + * This file contains the (simplified) types used + * to represent queries that are made to Elastic + * in order to count number of model downloads + * + * Read this doc about download stats on the Hub: + * + * https://huggingface.co/docs/hub/models-download-stats + * Available fields: + * - path: the complete file path (relative) (e.g: "prefix/file.extension") + * - path_prefix: the prefix of the file path (e.g: "prefix/", empty if no prefix) + * - path_extension: the extension of the file path (e.g: "extension") + * - path_filename: the name of the file path (e.g: "file") + * see also: + * https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html + */ +export type ElasticSearchQuery = string; +//# sourceMappingURL=model-libraries-downloads.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/model-libraries-downloads.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/model-libraries-downloads.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f0a63c2eda1c1650a18fb8060e9cf6fb53c7c272 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-libraries-downloads.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"model-libraries-downloads.d.ts","sourceRoot":"","sources":["../../src/model-libraries-downloads.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/model-libraries-downloads.js b/node_modules/@huggingface/tasks/dist/esm/model-libraries-downloads.js new file mode 100644 index 0000000000000000000000000000000000000000..b5e06470edfefe3b3b690cb7caecc12f69347755 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-libraries-downloads.js @@ -0,0 +1,17 @@ +/** + * This file contains the (simplified) types used + * to represent queries that are made to Elastic + * in order to count number of model downloads + * + * Read this doc about download stats on the Hub: + * + * https://huggingface.co/docs/hub/models-download-stats + * Available fields: + * - path: the complete file path (relative) (e.g: "prefix/file.extension") + * - path_prefix: the prefix of the file path (e.g: "prefix/", empty if no prefix) + * - path_extension: the extension of the file path (e.g: "extension") + * - path_filename: the name of the file path (e.g: "file") + * see also: + * https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html + */ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.d.ts b/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..cb0d9d16a71eef68e697bbdbe0065d35ac434423 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.d.ts @@ -0,0 +1,116 @@ +import type { ModelData } from "./model-data.js"; +export declare const adapters: (model: ModelData) => string[]; +export declare const allennlp: (model: ModelData) => string[]; +export declare const araclip: (model: ModelData) => string[]; +export declare const asteroid: (model: ModelData) => string[]; +export declare const audioseal: (model: ModelData) => string[]; +export declare const ben2: (model: ModelData) => string[]; +export declare const bertopic: (model: ModelData) => string[]; +export declare const bm25s: (model: ModelData) => string[]; +export declare const chatterbox: () => string[]; +export declare const chronos_forecasting: (model: ModelData) => string[]; +export declare const collectorvision: (model: ModelData) => string[]; +export declare const colipri: (model: ModelData) => string[]; +export declare const sap_rpt_one_oss: () => string[]; +export declare const cxr_foundation: () => string[]; +export declare const depth_anything_v2: (model: ModelData) => string[]; +export declare const depth_pro: (model: ModelData) => string[]; +export declare const derm_foundation: () => string[]; +export declare const dia: (model: ModelData) => string[]; +export declare const dia2: (model: ModelData) => string[]; +export declare const describe_anything: (model: ModelData) => string[]; +export declare const diffusers: (model: ModelData) => string[]; +export declare const diffusionkit: (model: ModelData) => string[]; +export declare const cartesia_pytorch: (model: ModelData) => string[]; +export declare const cartesia_mlx: (model: ModelData) => string[]; +export declare const edsnlp: (model: ModelData) => string[]; +export declare const espnetTTS: (model: ModelData) => string[]; +export declare const espnetASR: (model: ModelData) => string[]; +export declare const espnet: (model: ModelData) => string[]; +export declare const fairseq: (model: ModelData) => string[]; +export declare const flair: (model: ModelData) => string[]; +export declare const gliner: (model: ModelData) => string[]; +export declare const gliner2: (model: ModelData) => string[]; +export declare const indextts: (model: ModelData) => string[]; +export declare const htrflow: (model: ModelData) => string[]; +export declare const keras: (model: ModelData) => string[]; +export declare const keras_hub: (model: ModelData) => string[]; +export declare const kernels: (model: ModelData) => string[]; +export declare const kimi_audio: (model: ModelData) => string[]; +export declare const kittentts: (model: ModelData) => string[]; +export declare const lightning_ir: (model: ModelData) => string[]; +export declare const llama_cpp_python: (model: ModelData) => string[]; +export declare const lerobot: (model: ModelData) => string[]; +export declare const litert_lm: (model: ModelData) => string[]; +export declare const tf_keras: (model: ModelData) => string[]; +export declare const mamba_ssm: (model: ModelData) => string[]; +export declare const mars5_tts: (model: ModelData) => string[]; +export declare const matanyone: (model: ModelData) => string[]; +export declare const mesh_anything: () => string[]; +export declare const multimolecule: (model: ModelData) => string[]; +export declare const open_clip: (model: ModelData) => string[]; +export declare const paddlenlp: (model: ModelData) => string[]; +export declare const paddleocr: (model: ModelData) => string[]; +export declare const perception_encoder: (model: ModelData) => string[]; +export declare const phantom_wan: (model: ModelData) => string[]; +export declare const pocket_tts: (model: ModelData) => string[]; +export declare const pyannote_audio_pipeline: (model: ModelData) => string[]; +export declare const pyannote_audio: (model: ModelData) => string[]; +export declare const relik: (model: ModelData) => string[]; +export declare const renderformer: (model: ModelData) => string[]; +export declare const tensorflowtts: (model: ModelData) => string[]; +export declare const timm: (model: ModelData) => string[]; +export declare const saelens: () => string[]; +export declare const seed_story: () => string[]; +export declare const sklearn: (model: ModelData) => string[]; +export declare const stable_audio_tools: (model: ModelData) => string[]; +export declare const fastai: (model: ModelData) => string[]; +export declare const sam2: (model: ModelData) => string[]; +export declare const sam_3d_objects: (model: ModelData) => string[]; +export declare const sam_3d_body: (model: ModelData) => string[]; +export declare const sampleFactory: (model: ModelData) => string[]; +export declare const sentenceTransformers: (model: ModelData) => string[]; +export declare const setfit: (model: ModelData) => string[]; +export declare const spacy: (model: ModelData) => string[]; +export declare const span_marker: (model: ModelData) => string[]; +export declare const stanza: (model: ModelData) => string[]; +export declare const speechbrain: (model: ModelData) => string[]; +export declare const terratorch: (model: ModelData) => string[]; +export declare const transformers: (model: ModelData) => string[]; +export declare const transformersJS: (model: ModelData) => string[]; +export declare const peft: (model: ModelData) => string[]; +export declare const fasttext: (model: ModelData) => string[]; +export declare const stableBaselines3: (model: ModelData) => string[]; +export declare const mlAgents: (model: ModelData) => string[]; +export declare const sentis: () => string[]; +export declare const sana: (model: ModelData) => string[]; +export declare const vibevoice: (model: ModelData) => string[]; +export declare const videoprism: (model: ModelData) => string[]; +export declare const vfimamba: (model: ModelData) => string[]; +export declare const lvface: (model: ModelData) => string[]; +export declare const voicecraft: (model: ModelData) => string[]; +export declare const voxcpm: (model: ModelData) => string[]; +export declare const vui: () => string[]; +export declare const chattts: () => string[]; +export declare const ultralytics: (model: ModelData) => string[]; +export declare const birefnet: (model: ModelData) => string[]; +export declare const supertonic: () => string[]; +export declare const swarmformer: (model: ModelData) => string[]; +export declare const univa: (model: ModelData) => string[]; +export declare const mlxim: (model: ModelData) => string[]; +export declare const mlx: (model: ModelData) => string[]; +export declare const model2vec: (model: ModelData) => string[]; +export declare const pruna: (model: ModelData) => string[]; +export declare const nemo: (model: ModelData) => string[]; +export declare const outetts: (model: ModelData) => string[]; +export declare const pxia: (model: ModelData) => string[]; +export declare const pythae: (model: ModelData) => string[]; +export declare const qwen3_tts: (model: ModelData) => string[]; +export declare const anemoi: (model: ModelData) => string[]; +export declare const audiocraft: (model: ModelData) => string[]; +export declare const whisperkit: () => string[]; +export declare const threedtopia_xl: (model: ModelData) => string[]; +export declare const hezar: (model: ModelData) => string[]; +export declare const zonos: (model: ModelData) => string[]; +export declare const moshi: (model: ModelData) => string[]; +//# sourceMappingURL=model-libraries-snippets.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..302eca3baf2bca8f5602ca5999586b5eaa1965b4 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"model-libraries-snippets.d.ts","sourceRoot":"","sources":["../../src/model-libraries-snippets.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAkBjD,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAKjD,CAAC;AAkBF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAKjD,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAIhD,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAIjD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAkBlD,CAAC;AAaF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EAY7C,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAIjD,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAI9C,CAAC;AAEF,eAAO,MAAM,UAAU,QAAO,MAAM,EAenC,CAAC;AAEF,eAAO,MAAM,mBAAmB,GAAI,OAAO,SAAS,KAAG,MAAM,EAsB5D,CAAC;AAEF,eAAO,MAAM,eAAe,GAAI,OAAO,SAAS,KAAG,MAAM,EAYxD,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAkBhD,CAAC;AAEF,eAAO,MAAM,eAAe,QAAO,MAAM,EAmDxC,CAAC;AAEF,eAAO,MAAM,cAAc,QAAO,MAAM,EAcvC,CAAC;AAEF,eAAO,MAAM,iBAAiB,GAAI,OAAO,SAAS,KAAG,MAAM,EA6C1D,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAwBlD,CAAC;AAEF,eAAO,MAAM,eAAe,QAAO,MAAM,EAoBxC,CAAC;AAEF,eAAO,MAAM,GAAG,GAAI,OAAO,SAAS,KAAG,MAAM,EAS5C,CAAC;AAEF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EAW7C,CAAC;AAEF,eAAO,MAAM,iBAAiB,GAAI,OAAO,SAAS,KAAG,MAAM,EAY1D,CAAC;AAqLF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAgClD,CAAC;AAEF,eAAO,MAAM,YAAY,GAAI,OAAO,SAAS,KAAG,MAAM,EAwCrD,CAAC;AAEF,eAAO,MAAM,gBAAgB,GAAI,OAAO,SAAS,KAAG,MAAM,EAgBzD,CAAC;AAEF,eAAO,MAAM,YAAY,GAAI,OAAO,SAAS,KAAG,MAAM,EAmBrD,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAgB/C,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAMlD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EASlD,CAAC;AAIF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAO/C,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAMhD,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAI9C,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAI/C,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAUhD,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAgBjD,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAehD,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAS9C,CAAC;AA4EF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAsBlD,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAMhD,CAAC;AAEF,eAAO,MAAM,UAAU,GAAI,OAAO,SAAS,KAAG,MAAM,EA+BnD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EASlD,CAAC;AAEF,eAAO,MAAM,YAAY,GAAI,OAAO,SAAS,KAAG,MAAM,EA+BrD,CAAC;AAEF,eAAO,MAAM,gBAAgB,GAAI,OAAO,SAAS,KAAG,MAAM,EA4BzD,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAuChD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAgBlD,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAOjD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAIlD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAKlD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAQlD,CAAC;AAEF,eAAO,MAAM,aAAa,QAAO,MAAM,EAQtC,CAAC;AAEF,eAAO,MAAM,aAAa,GAAI,OAAO,SAAS,KAAG,MAAM,EAgDtD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAKlD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAsBlD,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAuElD,CAAC;AAEF,eAAO,MAAM,kBAAkB,GAAI,OAAO,SAAS,KAAG,MAAM,EAgB3D,CAAC;AACF,eAAO,MAAM,WAAW,GAAI,OAAO,SAAS,KAAG,MAAM,EAUpD,CAAC;AAEF,eAAO,MAAM,UAAU,GAAI,OAAO,SAAS,KAAG,MAAM,EAWnD,CAAC;AAEF,eAAO,MAAM,uBAAuB,GAAI,OAAO,SAAS,KAAG,MAAM,EAehE,CAAC;AAiBF,eAAO,MAAM,cAAc,GAAI,OAAO,SAAS,KAAG,MAAM,EAKvD,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAI9C,CAAC;AAEF,eAAO,MAAM,YAAY,GAAI,OAAO,SAAS,KAAG,MAAM,EAKrD,CAAC;AAyBF,eAAO,MAAM,aAAa,GAAI,OAAO,SAAS,KAAG,MAAM,EAOtD,CAAC;AAEF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EAI7C,CAAC;AAEF,eAAO,MAAM,OAAO,QAA6B,MAAM,EAQtD,CAAC;AAEF,eAAO,MAAM,UAAU,QAAO,MAAM,EAanC,CAAC;AAsCF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAehD,CAAC;AAEF,eAAO,MAAM,kBAAkB,GAAI,OAAO,SAAS,KAAG,MAAM,EAmC3D,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAI/C,CAAC;AAEF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EA2B7C,CAAC;AAEF,eAAO,MAAM,cAAc,GAAI,OAAO,SAAS,KAAG,MAAM,EAWvD,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO,SAAS,KAAG,MAAM,EAMpD,CAAC;AAEF,eAAO,MAAM,aAAa,GAAI,OAAO,SAAS,KAAG,MAAM,EAEtD,CAAC;AASF,eAAO,MAAM,oBAAoB,GAAI,OAAO,SAAS,KAAG,MAAM,EA6D7D,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAI/C,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAU9C,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO,SAAS,KAAG,MAAM,EAIpD,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAK/C,CAAC;AAkBF,eAAO,MAAM,WAAW,GAAI,OAAO,SAAS,KAAG,MAAM,EAkBpD,CAAC;AAEF,eAAO,MAAM,UAAU,GAAI,OAAO,SAAS,KAAG,MAAM,EAInD,CAAC;AAOF,eAAO,MAAM,YAAY,GAAI,OAAO,SAAS,KAAG,MAAM,EAiHrD,CAAC;AAEF,eAAO,MAAM,cAAc,GAAI,OAAO,SAAS,KAAG,MAAM,EAcvD,CAAC;AAiBF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EAiB7C,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAKjD,CAAC;AAEF,eAAO,MAAM,gBAAgB,GAAI,OAAO,SAAS,KAAG,MAAM,EAMzD,CAAC;AAgBF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAEjD,CAAC;AAEF,eAAO,MAAM,MAAM,QAA6B,MAAM,EAMrD,CAAC;AAEF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EAkB7C,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAqBlD,CAAC;AAEF,eAAO,MAAM,UAAU,GAAI,OAAO,SAAS,KAAG,MAAM,EAWnD,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAIjD,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAQ/C,CAAC;AAEF,eAAO,MAAM,UAAU,GAAI,OAAO,SAAS,KAAG,MAAM,EAInD,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAqB/C,CAAC;AAEF,eAAO,MAAM,GAAG,QAAO,MAAM,EAgB5B,CAAC;AAEF,eAAO,MAAM,OAAO,QAAO,MAAM,EAYhC,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO,SAAS,KAAG,MAAM,EAiBpD,CAAC;AAEF,eAAO,MAAM,QAAQ,GAAI,OAAO,SAAS,KAAG,MAAM,EAYjD,CAAC;AAEF,eAAO,MAAM,UAAU,QAAO,MAAM,EAWnC,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,OAAO,SAAS,KAAG,MAAM,EAKpD,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAW9C,CAAC;AAmEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAI9C,CAAC;AAEF,eAAO,MAAM,GAAG,GAAI,OAAO,SAAS,KAAG,MAAM,EAY5C,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAIlD,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EA0B9C,CAAC;AAuDF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EAQ7C,CAAC;AAEF,eAAO,MAAM,OAAO,GAAI,OAAO,SAAS,KAAG,MAAM,EAyBhD,CAAC;AAEF,eAAO,MAAM,IAAI,GAAI,OAAO,SAAS,KAAG,MAAM,EAI7C,CAAC;AAEF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAI/C,CAAC;AAEF,eAAO,MAAM,SAAS,GAAI,OAAO,SAAS,KAAG,MAAM,EAqBlD,CAAC;AA4BF,eAAO,MAAM,MAAM,GAAI,OAAO,SAAS,KAAG,MAAM,EAO/C,CAAC;AAEF,eAAO,MAAM,UAAU,GAAI,OAAO,SAAS,KAAG,MAAM,EAUnD,CAAC;AAEF,eAAO,MAAM,UAAU,QAAO,MAAM,EAYnC,CAAC;AAEF,eAAO,MAAM,cAAc,GAAI,OAAO,SAAS,KAAG,MAAM,EAKvD,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAI9C,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAiB9C,CAAC;AAEF,eAAO,MAAM,KAAK,GAAI,OAAO,SAAS,KAAG,MAAM,EAgD9C,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.js b/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.js new file mode 100644 index 0000000000000000000000000000000000000000..30d3a92a07ef103babeb1637e250dd3e4b56a813 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.js @@ -0,0 +1,2265 @@ +import { LIBRARY_TASK_MAPPING, REMOVED_IN_V5_TRANSFORMERS_PIPELINES } from "./library-to-tasks.js"; +import { getModelInputSnippet } from "./snippets/inputs.js"; +import { stringifyMessages } from "./snippets/common.js"; +const TAG_CUSTOM_CODE = "custom_code"; +function nameWithoutNamespace(modelId) { + const splitted = modelId.split("/"); + return splitted.length === 1 ? splitted[0] : splitted[1]; +} +const escapeStringForJson = (str) => JSON.stringify(str).slice(1, -1); // slice is needed to remove surrounding quotes added by JSON.stringify +//#region snippets +export const adapters = (model) => [ + `from adapters import AutoAdapterModel + +model = AutoAdapterModel.from_pretrained("${model.config?.adapter_transformers?.model_name}") +model.load_adapter("${model.id}", set_active=True)`, +]; +const allennlpUnknown = (model) => [ + `import allennlp_models +from allennlp.predictors.predictor import Predictor + +predictor = Predictor.from_path("hf://${model.id}")`, +]; +const allennlpQuestionAnswering = (model) => [ + `import allennlp_models +from allennlp.predictors.predictor import Predictor + +predictor = Predictor.from_path("hf://${model.id}") +predictor_input = {"passage": "My name is Wolfgang and I live in Berlin", "question": "Where do I live?"} +predictions = predictor.predict_json(predictor_input)`, +]; +export const allennlp = (model) => { + if (model.tags.includes("question-answering")) { + return allennlpQuestionAnswering(model); + } + return allennlpUnknown(model); +}; +export const araclip = (model) => [ + `from araclip import AraClip + +model = AraClip.from_pretrained("${model.id}")`, +]; +export const asteroid = (model) => [ + `from asteroid.models import BaseModel + +model = BaseModel.from_pretrained("${model.id}")`, +]; +export const audioseal = (model) => { + const watermarkSnippet = `# Watermark Generator +from audioseal import AudioSeal + +model = AudioSeal.load_generator("${model.id}") +# pass a tensor (tensor_wav) of shape (batch, channels, samples) and a sample rate +wav, sr = tensor_wav, 16000 + +watermark = model.get_watermark(wav, sr) +watermarked_audio = wav + watermark`; + const detectorSnippet = `# Watermark Detector +from audioseal import AudioSeal + +detector = AudioSeal.load_detector("${model.id}") + +result, message = detector.detect_watermark(watermarked_audio, sr)`; + return [watermarkSnippet, detectorSnippet]; +}; +function get_base_diffusers_model(model) { + return model.cardData?.base_model?.toString() ?? "fill-in-base-model"; +} +function get_prompt_from_diffusers_model(model) { + const prompt = model.widgetData?.[0]?.text ?? model.cardData?.instance_prompt; + if (prompt) { + return escapeStringForJson(prompt); + } +} +export const ben2 = (model) => [ + `import requests +from PIL import Image +from ben2 import AutoModel + +url = "https://huggingface.co/datasets/mishig/sample_images/resolve/main/teapot.jpg" +image = Image.open(requests.get(url, stream=True).raw) + +model = AutoModel.from_pretrained("${model.id}") +model.to("cuda").eval() +foreground = model.inference(image) +`, +]; +export const bertopic = (model) => [ + `from bertopic import BERTopic + +model = BERTopic.load("${model.id}")`, +]; +export const bm25s = (model) => [ + `from bm25s.hf import BM25HF + +retriever = BM25HF.load_from_hub("${model.id}")`, +]; +export const chatterbox = () => [ + `# pip install chatterbox-tts +import torchaudio as ta +from chatterbox.tts import ChatterboxTTS + +model = ChatterboxTTS.from_pretrained(device="cuda") + +text = "Ezreal and Jinx teamed up with Ahri, Yasuo, and Teemo to take down the enemy's Nexus in an epic late-game pentakill." +wav = model.generate(text) +ta.save("test-1.wav", wav, model.sr) + +# If you want to synthesize with a different voice, specify the audio prompt +AUDIO_PROMPT_PATH="YOUR_FILE.wav" +wav = model.generate(text, audio_prompt_path=AUDIO_PROMPT_PATH) +ta.save("test-2.wav", wav, model.sr)`, +]; +export const chronos_forecasting = (model) => { + const installSnippet = `pip install chronos-forecasting`; + const exampleSnippet = `import pandas as pd +from chronos import BaseChronosPipeline + +pipeline = BaseChronosPipeline.from_pretrained("${model.id}", device_map="cuda") + +# Load historical data +context_df = pd.read_csv("https://autogluon.s3.us-west-2.amazonaws.com/datasets/timeseries/misc/AirPassengers.csv") + +# Generate predictions +pred_df = pipeline.predict_df( + context_df, + prediction_length=36, # Number of steps to forecast + quantile_levels=[0.1, 0.5, 0.9], # Quantiles for probabilistic forecast + id_column="item_id", # Column identifying different time series + timestamp_column="Month", # Column with datetime information + target="#Passengers", # Column(s) with time series values to predict +)`; + return [installSnippet, exampleSnippet]; +}; +export const collectorvision = (model) => [ + `pip install git+https://github.com/HanClinto/CollectorVision huggingface_hub`, + `from huggingface_hub import hf_hub_download +import collector_vision as cvg + +checkpoint = hf_hub_download(repo_id="${model.id}", filename="model.onnx") + +# Detector models, such as Cornelius: +detector = cvg.NeuralCornerDetector(checkpoint) + +# Embedder models, such as Milo: +embedder = cvg.NeuralEmbedder(checkpoint)`, +]; +export const colipri = (model) => { + const installSnippet = `pip install colipri`; + const exampleSnippet = `from colipri import get_model +from colipri import get_processor +from colipri import load_sample_ct +from colipri import ZeroShotImageClassificationPipeline + +model = get_model().cuda() +processor = get_processor() +pipeline = ZeroShotImageClassificationPipeline("${model.id}", processor) + +image = load_sample_ct() + +pipeline(image, ["No lung nodules", "Lung nodules"]) +`; + return [installSnippet, exampleSnippet]; +}; +export const sap_rpt_one_oss = () => { + const installSnippet = `pip install git+https://github.com/SAP-samples/sap-rpt-1-oss`; + const classificationSnippet = `# Run a classification task +from sklearn.datasets import load_breast_cancer +from sklearn.metrics import accuracy_score +from sklearn.model_selection import train_test_split + +from sap_rpt_oss import SAP_RPT_OSS_Classifier + +# Load sample data +X, y = load_breast_cancer(return_X_y=True) +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42) + +# Initialize a classifier, 8k context and 8-fold bagging gives best performance, reduce if running out of memory +clf = SAP_RPT_OSS_Classifier(max_context_size=8192, bagging=8) + +clf.fit(X_train, y_train) + +# Predict probabilities +prediction_probabilities = clf.predict_proba(X_test) +# Predict labels +predictions = clf.predict(X_test) +print("Accuracy", accuracy_score(y_test, predictions))`; + const regressionsSnippet = `# Run a regression task +from sklearn.datasets import fetch_openml +from sklearn.metrics import r2_score +from sklearn.model_selection import train_test_split + +from sap_rpt_oss import SAP_RPT_OSS_Regressor + +# Load sample data +df = fetch_openml(data_id=531, as_frame=True) +X = df.data +y = df.target.astype(float) + +# Train-test split +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42) + +# Initialize the regressor, 8k context and 8-fold bagging gives best performance, reduce if running out of memory +regressor = SAP_RPT_OSS_Regressor(max_context_size=8192, bagging=8) + +regressor.fit(X_train, y_train) + +# Predict on the test set +predictions = regressor.predict(X_test) + +r2 = r2_score(y_test, predictions) +print("R² Score:", r2)`; + return [installSnippet, classificationSnippet, regressionsSnippet]; +}; +export const cxr_foundation = () => [ + `# pip install git+https://github.com/Google-Health/cxr-foundation.git#subdirectory=python + +# Load image as grayscale (Stillwaterising, CC0, via Wikimedia Commons) +import requests +from PIL import Image +from io import BytesIO +image_url = "https://upload.wikimedia.org/wikipedia/commons/c/c8/Chest_Xray_PA_3-8-2010.png" +img = Image.open(requests.get(image_url, headers={'User-Agent': 'Demo'}, stream=True).raw).convert('L') + +# Run inference +from clientside.clients import make_hugging_face_client +cxr_client = make_hugging_face_client('cxr_model') +print(cxr_client.get_image_embeddings_from_images([img]))`, +]; +export const depth_anything_v2 = (model) => { + let encoder; + let features; + let out_channels; + encoder = ""; + features = ""; + out_channels = ""; + if (model.id === "depth-anything/Depth-Anything-V2-Small") { + encoder = "vits"; + features = "64"; + out_channels = "[48, 96, 192, 384]"; + } + else if (model.id === "depth-anything/Depth-Anything-V2-Base") { + encoder = "vitb"; + features = "128"; + out_channels = "[96, 192, 384, 768]"; + } + else if (model.id === "depth-anything/Depth-Anything-V2-Large") { + encoder = "vitl"; + features = "256"; + out_channels = "[256, 512, 1024, 1024"; + } + return [ + ` +# Install from https://github.com/DepthAnything/Depth-Anything-V2 + +# Load the model and infer depth from an image +import cv2 +import torch + +from depth_anything_v2.dpt import DepthAnythingV2 + +# instantiate the model +model = DepthAnythingV2(encoder="${encoder}", features=${features}, out_channels=${out_channels}) + +# load the weights +filepath = hf_hub_download(repo_id="${model.id}", filename="depth_anything_v2_${encoder}.pth", repo_type="model") +state_dict = torch.load(filepath, map_location="cpu") +model.load_state_dict(state_dict).eval() + +raw_img = cv2.imread("your/image/path") +depth = model.infer_image(raw_img) # HxW raw depth map in numpy + `, + ]; +}; +export const depth_pro = (model) => { + const installSnippet = `# Download checkpoint +pip install huggingface-hub +huggingface-cli download --local-dir checkpoints ${model.id}`; + const inferenceSnippet = `import depth_pro + +# Load model and preprocessing transform +model, transform = depth_pro.create_model_and_transforms() +model.eval() + +# Load and preprocess an image. +image, _, f_px = depth_pro.load_rgb("example.png") +image = transform(image) + +# Run inference. +prediction = model.infer(image, f_px=f_px) + +# Results: 1. Depth in meters +depth = prediction["depth"] +# Results: 2. Focal length in pixels +focallength_px = prediction["focallength_px"]`; + return [installSnippet, inferenceSnippet]; +}; +export const derm_foundation = () => [ + `from huggingface_hub import from_pretrained_keras +import tensorflow as tf, requests + +# Load and format input +IMAGE_URL = "https://storage.googleapis.com/dx-scin-public-data/dataset/images/3445096909671059178.png" +input_tensor = tf.train.Example( + features=tf.train.Features( + feature={ + "image/encoded": tf.train.Feature( + bytes_list=tf.train.BytesList(value=[requests.get(IMAGE_URL, stream=True).content]) + ) + } + ) +).SerializeToString() + +# Load model and run inference +loaded_model = from_pretrained_keras("google/derm-foundation") +infer = loaded_model.signatures["serving_default"] +print(infer(inputs=tf.constant([input_tensor])))`, +]; +export const dia = (model) => [ + `import soundfile as sf +from dia.model import Dia + +model = Dia.from_pretrained("${model.id}") +text = "[S1] Dia is an open weights text to dialogue model. [S2] You get full control over scripts and voices. [S1] Wow. Amazing. (laughs) [S2] Try it now on Git hub or Hugging Face." +output = model.generate(text) + +sf.write("simple.mp3", output, 44100)`, +]; +export const dia2 = (model) => [ + `from dia2 import Dia2, GenerationConfig, SamplingConfig + +dia = Dia2.from_repo("${model.id}", device="cuda", dtype="bfloat16") +config = GenerationConfig( + cfg_scale=2.0, + audio=SamplingConfig(temperature=0.8, top_k=50), + use_cuda_graph=True, +) +result = dia.generate("[S1] Hello Dia2!", config=config, output_wav="hello.wav", verbose=True) +`, +]; +export const describe_anything = (model) => [ + `# pip install git+https://github.com/NVlabs/describe-anything +from huggingface_hub import snapshot_download +from dam import DescribeAnythingModel + +snapshot_download(${model.id}, local_dir="checkpoints") + +dam = DescribeAnythingModel( + model_path="checkpoints", + conv_mode="v1", + prompt_mode="focal_prompt", +)`, +]; +const diffusers_install = "pip install -U diffusers transformers accelerate"; +const diffusersDefaultPrompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"; +const diffusersImg2ImgDefaultPrompt = "Turn this cat into a dog"; +const diffusersVideoDefaultPrompt = "A man with short gray hair plays a red electric guitar."; +const diffusers_default = (model) => [ + `import torch +from diffusers import DiffusionPipeline + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersDefaultPrompt}" +image = pipe(prompt).images[0]`, +]; +const diffusers_image_to_image = (model) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import load_image + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersImg2ImgDefaultPrompt}" +input_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png") + +image = pipe(image=input_image, prompt=prompt).images[0]`, +]; +const diffusers_image_to_video = (model) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import load_image, export_to_video + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda") +pipe.to("cuda") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersVideoDefaultPrompt}" +image = load_image( + "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/guitar-man.png" +) + +output = pipe(image=image, prompt=prompt).frames[0] +export_to_video(output, "output.mp4")`, +]; +const diffusers_controlnet = (model) => [ + `from diffusers import ControlNetModel, StableDiffusionControlNetPipeline + +controlnet = ControlNetModel.from_pretrained("${model.id}") +pipe = StableDiffusionControlNetPipeline.from_pretrained( + "${get_base_diffusers_model(model)}", controlnet=controlnet +)`, +]; +const diffusers_lora = (model) => [ + `import torch +from diffusers import DiffusionPipeline + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_lora_weights("${model.id}") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersDefaultPrompt}" +image = pipe(prompt).images[0]`, +]; +const diffusers_lora_image_to_image = (model) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import load_image + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_lora_weights("${model.id}") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersImg2ImgDefaultPrompt}" +input_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png") + +image = pipe(image=input_image, prompt=prompt).images[0]`, +]; +const diffusers_lora_text_to_video = (model) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import export_to_video + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_lora_weights("${model.id}") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersVideoDefaultPrompt}" + +output = pipe(prompt=prompt).frames[0] +export_to_video(output, "output.mp4")`, +]; +const diffusers_lora_image_to_video = (model) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import load_image, export_to_video + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_lora_weights("${model.id}") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersVideoDefaultPrompt}" +input_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/guitar-man.png") + +image = pipe(image=input_image, prompt=prompt).frames[0] +export_to_video(output, "output.mp4")`, +]; +const diffusers_textual_inversion = (model) => [ + `import torch +from diffusers import DiffusionPipeline + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_textual_inversion("${model.id}")`, +]; +const diffusers_flux_fill = (model) => [ + `import torch +from diffusers import FluxFillPipeline +from diffusers.utils import load_image + +image = load_image("https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup.png") +mask = load_image("https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup_mask.png") + +# switch to "mps" for apple devices +pipe = FluxFillPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda") +image = pipe( + prompt="a white paper cup", + image=image, + mask_image=mask, + height=1632, + width=1232, + guidance_scale=30, + num_inference_steps=50, + max_sequence_length=512, + generator=torch.Generator("cpu").manual_seed(0) +).images[0] +image.save(f"flux-fill-dev.png")`, +]; +const diffusers_inpainting = (model) => [ + `import torch +from diffusers import AutoPipelineForInpainting +from diffusers.utils import load_image + +# switch to "mps" for apple devices +pipe = AutoPipelineForInpainting.from_pretrained("${model.id}", dtype=torch.float16, variant="fp16", device_map="cuda") + +img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png" +mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png" + +image = load_image(img_url).resize((1024, 1024)) +mask_image = load_image(mask_url).resize((1024, 1024)) + +prompt = "a tiger sitting on a park bench" +generator = torch.Generator(device="cuda").manual_seed(0) + +image = pipe( + prompt=prompt, + image=image, + mask_image=mask_image, + guidance_scale=8.0, + num_inference_steps=20, # steps between 15 and 30 work well for us + strength=0.99, # make sure to use \`strength\` below 1.0 + generator=generator, +).images[0]`, +]; +export const diffusers = (model) => { + let codeSnippets; + if (model.tags.includes("StableDiffusionInpaintPipeline") || + model.tags.includes("StableDiffusionXLInpaintPipeline")) { + codeSnippets = diffusers_inpainting(model); + } + else if (model.tags.includes("controlnet")) { + codeSnippets = diffusers_controlnet(model); + } + else if (model.tags.includes("lora")) { + if (model.pipeline_tag === "image-to-image") { + codeSnippets = diffusers_lora_image_to_image(model); + } + else if (model.pipeline_tag === "image-to-video") { + codeSnippets = diffusers_lora_image_to_video(model); + } + else if (model.pipeline_tag === "text-to-video") { + codeSnippets = diffusers_lora_text_to_video(model); + } + else { + codeSnippets = diffusers_lora(model); + } + } + else if (model.tags.includes("textual_inversion")) { + codeSnippets = diffusers_textual_inversion(model); + } + else if (model.tags.includes("FluxFillPipeline")) { + codeSnippets = diffusers_flux_fill(model); + } + else if (model.pipeline_tag === "image-to-video") { + codeSnippets = diffusers_image_to_video(model); + } + else if (model.pipeline_tag === "image-to-image") { + codeSnippets = diffusers_image_to_image(model); + } + else { + codeSnippets = diffusers_default(model); + } + return [diffusers_install, ...codeSnippets]; +}; +export const diffusionkit = (model) => { + const sd3Snippet = `# Pipeline for Stable Diffusion 3 +from diffusionkit.mlx import DiffusionPipeline + +pipeline = DiffusionPipeline( + shift=3.0, + use_t5=False, + model_version=${model.id}, + low_memory_mode=True, + a16=True, + w16=True, +)`; + const fluxSnippet = `# Pipeline for Flux +from diffusionkit.mlx import FluxPipeline + +pipeline = FluxPipeline( + shift=1.0, + model_version=${model.id}, + low_memory_mode=True, + a16=True, + w16=True, +)`; + const generateSnippet = `# Image Generation +HEIGHT = 512 +WIDTH = 512 +NUM_STEPS = ${model.tags.includes("flux") ? 4 : 50} +CFG_WEIGHT = ${model.tags.includes("flux") ? 0 : 5} + +image, _ = pipeline.generate_image( + "a photo of a cat", + cfg_weight=CFG_WEIGHT, + num_steps=NUM_STEPS, + latent_size=(HEIGHT // 8, WIDTH // 8), +)`; + const pipelineSnippet = model.tags.includes("flux") ? fluxSnippet : sd3Snippet; + return [pipelineSnippet, generateSnippet]; +}; +export const cartesia_pytorch = (model) => [ + `# pip install --no-binary :all: cartesia-pytorch +from cartesia_pytorch import ReneLMHeadModel +from transformers import AutoTokenizer + +model = ReneLMHeadModel.from_pretrained("${model.id}") +tokenizer = AutoTokenizer.from_pretrained("allenai/OLMo-1B-hf") + +in_message = ["Rene Descartes was"] +inputs = tokenizer(in_message, return_tensors="pt") + +outputs = model.generate(inputs.input_ids, max_length=50, top_k=100, top_p=0.99) +out_message = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0] + +print(out_message) +)`, +]; +export const cartesia_mlx = (model) => [ + `import mlx.core as mx +import cartesia_mlx as cmx + +model = cmx.from_pretrained("${model.id}") +model.set_dtype(mx.float32) + +prompt = "Rene Descartes was" + +for text in model.generate( + prompt, + max_tokens=500, + eval_every_n=5, + verbose=True, + top_p=0.99, + temperature=0.85, +): + print(text, end="", flush=True) +`, +]; +export const edsnlp = (model) => { + const packageName = nameWithoutNamespace(model.id).replaceAll("-", "_"); + return [ + `# Load it from the Hub directly +import edsnlp +nlp = edsnlp.load("${model.id}") +`, + `# Or install it as a package +!pip install git+https://huggingface.co/${model.id} + +# and import it as a module +import ${packageName} + +nlp = ${packageName}.load() # or edsnlp.load("${packageName}") +`, + ]; +}; +export const espnetTTS = (model) => [ + `from espnet2.bin.tts_inference import Text2Speech + +model = Text2Speech.from_pretrained("${model.id}") + +speech, *_ = model("text to generate speech from")`, +]; +export const espnetASR = (model) => [ + `from espnet2.bin.asr_inference import Speech2Text + +model = Speech2Text.from_pretrained( + "${model.id}" +) + +speech, rate = soundfile.read("speech.wav") +text, *_ = model(speech)[0]`, +]; +const espnetUnknown = () => [`unknown model type (must be text-to-speech or automatic-speech-recognition)`]; +export const espnet = (model) => { + if (model.tags.includes("text-to-speech")) { + return espnetTTS(model); + } + else if (model.tags.includes("automatic-speech-recognition")) { + return espnetASR(model); + } + return espnetUnknown(); +}; +export const fairseq = (model) => [ + `from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub + +models, cfg, task = load_model_ensemble_and_task_from_hf_hub( + "${model.id}" +)`, +]; +export const flair = (model) => [ + `from flair.models import SequenceTagger + +tagger = SequenceTagger.load("${model.id}")`, +]; +export const gliner = (model) => [ + `from gliner import GLiNER + +model = GLiNER.from_pretrained("${model.id}")`, +]; +export const gliner2 = (model) => [ + `from gliner2 import GLiNER2 + +model = GLiNER2.from_pretrained("${model.id}") + +# Extract entities +text = "Apple CEO Tim Cook announced iPhone 15 in Cupertino yesterday." +result = extractor.extract_entities(text, ["company", "person", "product", "location"]) + +print(result)`, +]; +export const indextts = (model) => [ + `# Download model +from huggingface_hub import snapshot_download + +snapshot_download(${model.id}, local_dir="checkpoints") + +from indextts.infer import IndexTTS + +# Ensure config.yaml is present in the checkpoints directory +tts = IndexTTS(model_dir="checkpoints", cfg_path="checkpoints/config.yaml") + +voice = "path/to/your/reference_voice.wav" # Path to the voice reference audio file +text = "Hello, how are you?" +output_path = "output_index.wav" + +tts.infer(voice, text, output_path)`, +]; +export const htrflow = (model) => [ + `# CLI usage +# see docs: https://ai-riksarkivet.github.io/htrflow/latest/getting_started/quick_start.html +htrflow pipeline `, + `# Python usage +from htrflow.pipeline.pipeline import Pipeline +from htrflow.pipeline.steps import Task +from htrflow.models.framework.model import ModelClass + +pipeline = Pipeline( + [ + Task( + ModelClass, {"model": "${model.id}"}, {} + ), + ])`, +]; +export const keras = (model) => [ + `# Available backend options are: "jax", "torch", "tensorflow". +import os +os.environ["KERAS_BACKEND"] = "jax" + +import keras + +model = keras.saving.load_model("hf://${model.id}") +`, +]; +const _keras_hub_causal_lm = (modelId) => ` +import keras_hub + +# Load CausalLM model (optional: use half precision for inference) +causal_lm = keras_hub.models.CausalLM.from_preset("hf://${modelId}", dtype="bfloat16") +causal_lm.compile(sampler="greedy") # (optional) specify a sampler + +# Generate text +causal_lm.generate("Keras: deep learning for", max_length=64) +`; +const _keras_hub_text_to_image = (modelId) => ` +import keras_hub + +# Load TextToImage model (optional: use half precision for inference) +text_to_image = keras_hub.models.TextToImage.from_preset("hf://${modelId}", dtype="bfloat16") + +# Generate images with a TextToImage model. +text_to_image.generate("Astronaut in a jungle") +`; +const _keras_hub_text_classifier = (modelId) => ` +import keras_hub + +# Load TextClassifier model +text_classifier = keras_hub.models.TextClassifier.from_preset( + "hf://${modelId}", + num_classes=2, +) +# Fine-tune +text_classifier.fit(x=["Thilling adventure!", "Total snoozefest."], y=[1, 0]) +# Classify text +text_classifier.predict(["Not my cup of tea."]) +`; +const _keras_hub_image_classifier = (modelId) => ` +import keras_hub +import keras + +# Load ImageClassifier model +image_classifier = keras_hub.models.ImageClassifier.from_preset( + "hf://${modelId}", + num_classes=2, +) +# Fine-tune +image_classifier.fit( + x=keras.random.randint((32, 64, 64, 3), 0, 256), + y=keras.random.randint((32, 1), 0, 2), +) +# Classify image +image_classifier.predict(keras.random.randint((1, 64, 64, 3), 0, 256)) +`; +const _keras_hub_tasks_with_example = { + CausalLM: _keras_hub_causal_lm, + TextToImage: _keras_hub_text_to_image, + TextClassifier: _keras_hub_text_classifier, + ImageClassifier: _keras_hub_image_classifier, +}; +const _keras_hub_task_without_example = (task, modelId) => ` +import keras_hub + +# Create a ${task} model +task = keras_hub.models.${task}.from_preset("hf://${modelId}") +`; +const _keras_hub_generic_backbone = (modelId) => ` +import keras_hub + +# Create a Backbone model unspecialized for any task +backbone = keras_hub.models.Backbone.from_preset("hf://${modelId}") +`; +export const keras_hub = (model) => { + const modelId = model.id; + const tasks = model.config?.keras_hub?.tasks ?? []; + const snippets = []; + // First, generate tasks with examples + for (const [task, snippet] of Object.entries(_keras_hub_tasks_with_example)) { + if (tasks.includes(task)) { + snippets.push(snippet(modelId)); + } + } + // Then, add remaining tasks + for (const task of tasks) { + if (!Object.keys(_keras_hub_tasks_with_example).includes(task)) { + snippets.push(_keras_hub_task_without_example(task, modelId)); + } + } + // Finally, add generic backbone snippet + snippets.push(_keras_hub_generic_backbone(modelId)); + return snippets; +}; +export const kernels = (model) => [ + `# !pip install kernels + +from kernels import get_kernel + +kernel = get_kernel("${model.id}")`, +]; +export const kimi_audio = (model) => [ + `# Example usage for KimiAudio +# pip install git+https://github.com/MoonshotAI/Kimi-Audio.git + +from kimia_infer.api.kimia import KimiAudio + +model = KimiAudio(model_path="${model.id}", load_detokenizer=True) + +sampling_params = { + "audio_temperature": 0.8, + "audio_top_k": 10, + "text_temperature": 0.0, + "text_top_k": 5, +} + +# For ASR +asr_audio = "asr_example.wav" +messages_asr = [ + {"role": "user", "message_type": "text", "content": "Please transcribe the following audio:"}, + {"role": "user", "message_type": "audio", "content": asr_audio} +] +_, text = model.generate(messages_asr, **sampling_params, output_type="text") +print(text) + +# For Q&A +qa_audio = "qa_example.wav" +messages_conv = [{"role": "user", "message_type": "audio", "content": qa_audio}] +wav, text = model.generate(messages_conv, **sampling_params, output_type="both") +sf.write("output_audio.wav", wav.cpu().view(-1).numpy(), 24000) +print(text) +`, +]; +export const kittentts = (model) => [ + `from kittentts import KittenTTS +m = KittenTTS("${model.id}") + +audio = m.generate("This high quality TTS model works without a GPU") + +# Save the audio +import soundfile as sf +sf.write('output.wav', audio, 24000)`, +]; +export const lightning_ir = (model) => { + if (model.tags.includes("bi-encoder")) { + return [ + `#install from https://github.com/webis-de/lightning-ir + +from lightning_ir import BiEncoderModule +model = BiEncoderModule("${model.id}") + +model.score("query", ["doc1", "doc2", "doc3"])`, + ]; + } + else if (model.tags.includes("cross-encoder")) { + return [ + `#install from https://github.com/webis-de/lightning-ir + +from lightning_ir import CrossEncoderModule +model = CrossEncoderModule("${model.id}") + +model.score("query", ["doc1", "doc2", "doc3"])`, + ]; + } + return [ + `#install from https://github.com/webis-de/lightning-ir + +from lightning_ir import BiEncoderModule, CrossEncoderModule + +# depending on the model type, use either BiEncoderModule or CrossEncoderModule +model = BiEncoderModule("${model.id}") +# model = CrossEncoderModule("${model.id}") + +model.score("query", ["doc1", "doc2", "doc3"])`, + ]; +}; +export const llama_cpp_python = (model) => { + const snippets = [ + `# !pip install llama-cpp-python + +from llama_cpp import Llama + +llm = Llama.from_pretrained( + repo_id="${model.id}", + filename="{{GGUF_FILE}}", +) +`, + ]; + if (model.tags.includes("conversational")) { + const messages = getModelInputSnippet(model); + snippets.push(`llm.create_chat_completion( + messages = ${stringifyMessages(messages, { attributeKeyQuotes: true, indent: "\t" })} +)`); + } + else { + snippets.push(`output = llm( + "Once upon a time,", + max_tokens=512, + echo=True +) +print(output)`); + } + return snippets; +}; +export const lerobot = (model) => { + if (model.tags.includes("smolvla")) { + const smolvlaSnippets = [ + // Installation snippet + `# See https://github.com/huggingface/lerobot?tab=readme-ov-file#installation for more details +git clone https://github.com/huggingface/lerobot.git +cd lerobot +pip install -e .[smolvla]`, + // Finetune snippet + `# Launch finetuning on your dataset +python lerobot/scripts/train.py \\ +--policy.path=${model.id} \\ +--dataset.repo_id=lerobot/svla_so101_pickplace \\ +--batch_size=64 \\ +--steps=20000 \\ +--output_dir=outputs/train/my_smolvla \\ +--job_name=my_smolvla_training \\ +--policy.device=cuda \\ +--wandb.enable=true`, + ]; + if (model.id !== "lerobot/smolvla_base") { + // Inference snippet (only if not base model) + smolvlaSnippets.push(`# Run the policy using the record function +python -m lerobot.record \\ + --robot.type=so101_follower \\ + --robot.port=/dev/ttyACM0 \\ # <- Use your port + --robot.id=my_blue_follower_arm \\ # <- Use your robot id + --robot.cameras="{ front: {type: opencv, index_or_path: 8, width: 640, height: 480, fps: 30}}" \\ # <- Use your cameras + --dataset.single_task="Grasp a lego block and put it in the bin." \\ # <- Use the same task description you used in your dataset recording + --dataset.repo_id=HF_USER/dataset_name \\ # <- This will be the dataset name on HF Hub + --dataset.episode_time_s=50 \\ + --dataset.num_episodes=10 \\ + --policy.path=${model.id}`); + } + return smolvlaSnippets; + } + return []; +}; +export const litert_lm = (model) => [ + `# LiteRT-LM runs on various platforms (Android, iOS, Windows, Linux, macOS, IoT, Web/WASM) +# and supports many APIs (C++, Python, Kotlin, Swift, JavaScript, Flutter). +# For platform-specific integration guides, please refer to the official developer website: +# https://ai.google.dev/edge/litert-lm + +# To try LiteRT-LM, the easiest way is to use our CLI tool. +# 1. Install the LiteRT-LM CLI tool: +pip install litert-lm + +# 2. Download and run this model locally: +# See: https://ai.google.dev/edge/litert-lm/cli +litert-lm run \\ + --from-huggingface-repo=${model.id} \\ + model.litertlm \\ + --prompt="Write me a poem"`, +]; +export const tf_keras = (model) => [ + `# Note: 'keras<3.x' or 'tf_keras' must be installed (legacy) +# See https://github.com/keras-team/tf-keras for more details. +from huggingface_hub import from_pretrained_keras + +model = from_pretrained_keras("${model.id}") +`, +]; +export const mamba_ssm = (model) => [ + `from mamba_ssm import MambaLMHeadModel + +model = MambaLMHeadModel.from_pretrained("${model.id}")`, +]; +export const mars5_tts = (model) => [ + `# Install from https://github.com/Camb-ai/MARS5-TTS + +from inference import Mars5TTS +mars5 = Mars5TTS.from_pretrained("${model.id}")`, +]; +export const matanyone = (model) => [ + `# Install from https://github.com/pq-yang/MatAnyone.git + +from matanyone.model.matanyone import MatAnyone +model = MatAnyone.from_pretrained("${model.id}")`, + ` +from matanyone import InferenceCore +processor = InferenceCore("${model.id}")`, +]; +export const mesh_anything = () => [ + `# Install from https://github.com/buaacyw/MeshAnything.git + +from MeshAnything.models.meshanything import MeshAnything + +# refer to https://github.com/buaacyw/MeshAnything/blob/main/main.py#L91 on how to define args +# and https://github.com/buaacyw/MeshAnything/blob/main/app.py regarding usage +model = MeshAnything(args)`, +]; +export const multimolecule = (model) => { + const widgetExample = model.widgetData?.[0]; + const exampleText = widgetExample?.text; + const maskToken = model.mask_token ?? ""; + const sequence = exampleText?.replace(maskToken, "A"); + const snippets = [`pip install multimolecule`]; + if (sequence) { + snippets.push(`from multimolecule import AutoModel, AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained("${model.id}") +model = AutoModel.from_pretrained("${model.id}") + +inputs = tokenizer("${sequence}", return_tensors="pt") +outputs = model(**inputs) +embeddings = outputs.last_hidden_state`); + } + else { + snippets.push(`from multimolecule import AutoModel, AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained("${model.id}") +model = AutoModel.from_pretrained("${model.id}")`); + } + if (model.tags.includes("rna-secondary-structure") && exampleText) { + snippets.push(`import multimolecule +from transformers import pipeline + +predictor = pipeline("rna-secondary-structure", model="${model.id}") +output = predictor("${exampleText}") +print(output["secondary_structure"])`); + } + else if (model.pipeline_tag === "fill-mask" && exampleText) { + snippets.push(`import multimolecule +from transformers import pipeline + +predictor = pipeline("fill-mask", model="${model.id}") +output = predictor("${exampleText}")`); + } + return snippets; +}; +export const open_clip = (model) => [ + `import open_clip + +model, preprocess_train, preprocess_val = open_clip.create_model_and_transforms('hf-hub:${model.id}') +tokenizer = open_clip.get_tokenizer('hf-hub:${model.id}')`, +]; +export const paddlenlp = (model) => { + if (model.config?.architectures?.[0]) { + const architecture = model.config.architectures[0]; + return [ + [ + `from paddlenlp.transformers import AutoTokenizer, ${architecture}`, + "", + `tokenizer = AutoTokenizer.from_pretrained("${model.id}", from_hf_hub=True)`, + `model = ${architecture}.from_pretrained("${model.id}", from_hf_hub=True)`, + ].join("\n"), + ]; + } + else { + return [ + [ + `# ⚠️ Type of model unknown`, + `from paddlenlp.transformers import AutoTokenizer, AutoModel`, + "", + `tokenizer = AutoTokenizer.from_pretrained("${model.id}", from_hf_hub=True)`, + `model = AutoModel.from_pretrained("${model.id}", from_hf_hub=True)`, + ].join("\n"), + ]; + } +}; +export const paddleocr = (model) => { + const mapping = { + textline_detection: { className: "TextDetection" }, + textline_recognition: { className: "TextRecognition" }, + seal_text_detection: { className: "SealTextDetection" }, + doc_img_unwarping: { className: "TextImageUnwarping" }, + doc_img_orientation_classification: { className: "DocImgOrientationClassification" }, + textline_orientation_classification: { className: "TextLineOrientationClassification" }, + chart_parsing: { className: "ChartParsing" }, + formula_recognition: { className: "FormulaRecognition" }, + layout_detection: { className: "LayoutDetection" }, + table_cells_detection: { className: "TableCellsDetection" }, + wired_table_classification: { className: "TableClassification" }, + table_structure_recognition: { className: "TableStructureRecognition" }, + }; + if (model.tags.includes("doc_vlm")) { + return [ + `# 1. See https://www.paddlepaddle.org.cn/en/install to install paddlepaddle +# 2. pip install paddleocr + +from paddleocr import DocVLM +model = DocVLM(model_name="${nameWithoutNamespace(model.id)}") +output = model.predict( + input={"image": "path/to/image.png", "query": "Parsing this image and output the content in Markdown format."}, + batch_size=1 +) +for res in output: + res.print() + res.save_to_json(save_path="./output/res.json")`, + ]; + } + if (model.tags.includes("document-parse")) { + const rawVersion = model.id.replace("PaddlePaddle/PaddleOCR-VL-", "v"); + const version = rawVersion === "PaddlePaddle/PaddleOCR-VL" ? "v1" : rawVersion; + return [ + `# See https://www.paddleocr.ai/latest/version3.x/pipeline_usage/PaddleOCR-VL.html to installation + +from paddleocr import PaddleOCRVL +pipeline = PaddleOCRVL(pipeline_version="${version}") +output = pipeline.predict("path/to/document_image.png") +for res in output: + res.print() + res.save_to_json(save_path="output") + res.save_to_markdown(save_path="output")`, + ]; + } + for (const tag of model.tags) { + if (tag in mapping) { + const { className } = mapping[tag]; + return [ + `# 1. See https://www.paddlepaddle.org.cn/en/install to install paddlepaddle +# 2. pip install paddleocr + +from paddleocr import ${className} +model = ${className}(model_name="${nameWithoutNamespace(model.id)}") +output = model.predict(input="path/to/image.png", batch_size=1) +for res in output: + res.print() + res.save_to_img(save_path="./output/") + res.save_to_json(save_path="./output/res.json")`, + ]; + } + } + return [ + `# Please refer to the document for information on how to use the model. +# https://paddlepaddle.github.io/PaddleOCR/latest/en/version3.x/module_usage/module_overview.html`, + ]; +}; +export const perception_encoder = (model) => { + const clip_model = `# Use PE-Core models as CLIP models +import core.vision_encoder.pe as pe + +model = pe.CLIP.from_config("${model.id}", pretrained=True)`; + const vision_encoder = `# Use any PE model as a vision encoder +import core.vision_encoder.pe as pe + +model = pe.VisionTransformer.from_config("${model.id}", pretrained=True)`; + if (model.id.includes("Core")) { + return [clip_model, vision_encoder]; + } + else { + return [vision_encoder]; + } +}; +export const phantom_wan = (model) => [ + `from huggingface_hub import snapshot_download +from phantom_wan import WANI2V, configs + +checkpoint_dir = snapshot_download("${model.id}") +wan_i2v = WanI2V( + config=configs.WAN_CONFIGS['i2v-14B'], + checkpoint_dir=checkpoint_dir, + ) + video = wan_i2v.generate(text_prompt, image_prompt)`, +]; +export const pocket_tts = (model) => [ + `from pocket_tts import TTSModel +import scipy.io.wavfile + +tts_model = TTSModel.load_model("${model.id}") +voice_state = tts_model.get_state_for_audio_prompt( + "hf://kyutai/tts-voices/alba-mackenna/casual.wav" +) +audio = tts_model.generate_audio(voice_state, "Hello world, this is a test.") +# Audio is a 1D torch tensor containing PCM data. +scipy.io.wavfile.write("output.wav", tts_model.sample_rate, audio.numpy())`, +]; +export const pyannote_audio_pipeline = (model) => [ + `from pyannote.audio import Pipeline + +pipeline = Pipeline.from_pretrained("${model.id}") + +# inference on the whole file +pipeline("file.wav") + +# inference on an excerpt +from pyannote.core import Segment +excerpt = Segment(start=2.0, end=5.0) + +from pyannote.audio import Audio +waveform, sample_rate = Audio().crop("file.wav", excerpt) +pipeline({"waveform": waveform, "sample_rate": sample_rate})`, +]; +const pyannote_audio_model = (model) => [ + `from pyannote.audio import Model, Inference + +model = Model.from_pretrained("${model.id}") +inference = Inference(model) + +# inference on the whole file +inference("file.wav") + +# inference on an excerpt +from pyannote.core import Segment +excerpt = Segment(start=2.0, end=5.0) +inference.crop("file.wav", excerpt)`, +]; +export const pyannote_audio = (model) => { + if (model.tags.includes("pyannote-audio-pipeline")) { + return pyannote_audio_pipeline(model); + } + return pyannote_audio_model(model); +}; +export const relik = (model) => [ + `from relik import Relik + +relik = Relik.from_pretrained("${model.id}")`, +]; +export const renderformer = (model) => [ + `# Install from https://github.com/microsoft/renderformer + +from renderformer import RenderFormerRenderingPipeline +pipeline = RenderFormerRenderingPipeline.from_pretrained("${model.id}")`, +]; +const tensorflowttsTextToMel = (model) => [ + `from tensorflow_tts.inference import AutoProcessor, TFAutoModel + +processor = AutoProcessor.from_pretrained("${model.id}") +model = TFAutoModel.from_pretrained("${model.id}") +`, +]; +const tensorflowttsMelToWav = (model) => [ + `from tensorflow_tts.inference import TFAutoModel + +model = TFAutoModel.from_pretrained("${model.id}") +audios = model.inference(mels) +`, +]; +const tensorflowttsUnknown = (model) => [ + `from tensorflow_tts.inference import TFAutoModel + +model = TFAutoModel.from_pretrained("${model.id}") +`, +]; +export const tensorflowtts = (model) => { + if (model.tags.includes("text-to-mel")) { + return tensorflowttsTextToMel(model); + } + else if (model.tags.includes("mel-to-wav")) { + return tensorflowttsMelToWav(model); + } + return tensorflowttsUnknown(model); +}; +export const timm = (model) => [ + `import timm + +model = timm.create_model("hf_hub:${model.id}", pretrained=True)`, +]; +export const saelens = ( /* model: ModelData */) => [ + `# pip install sae-lens +from sae_lens import SAE + +sae, cfg_dict, sparsity = SAE.from_pretrained( + release = "RELEASE_ID", # e.g., "gpt2-small-res-jb". See other options in https://github.com/jbloomAus/SAELens/blob/main/sae_lens/pretrained_saes.yaml + sae_id = "SAE_ID", # e.g., "blocks.8.hook_resid_pre". Won't always be a hook point +)`, +]; +export const seed_story = () => [ + `# seed_story_cfg_path refers to 'https://github.com/TencentARC/SEED-Story/blob/master/configs/clm_models/agent_7b_sft.yaml' +# llm_cfg_path refers to 'https://github.com/TencentARC/SEED-Story/blob/master/configs/clm_models/llama2chat7b_lora.yaml' +from omegaconf import OmegaConf +import hydra + +# load Llama2 +llm_cfg = OmegaConf.load(llm_cfg_path) +llm = hydra.utils.instantiate(llm_cfg, torch_dtype="fp16") + +# initialize seed_story +seed_story_cfg = OmegaConf.load(seed_story_cfg_path) +seed_story = hydra.utils.instantiate(seed_story_cfg, llm=llm) `, +]; +const skopsPickle = (model, modelFile) => { + return [ + `import joblib +from skops.hub_utils import download +download("${model.id}", "path_to_folder") +model = joblib.load( + "${modelFile}" +) +# only load pickle files from sources you trust +# read more about it here https://skops.readthedocs.io/en/stable/persistence.html`, + ]; +}; +const skopsFormat = (model, modelFile) => { + return [ + `from skops.hub_utils import download +from skops.io import load +download("${model.id}", "path_to_folder") +# make sure model file is in skops format +# if model is a pickle file, make sure it's from a source you trust +model = load("path_to_folder/${modelFile}")`, + ]; +}; +const skopsJobLib = (model) => { + return [ + `from huggingface_hub import hf_hub_download +import joblib +model = joblib.load( + hf_hub_download("${model.id}", "sklearn_model.joblib") +) +# only load pickle files from sources you trust +# read more about it here https://skops.readthedocs.io/en/stable/persistence.html`, + ]; +}; +export const sklearn = (model) => { + if (model.tags.includes("skops")) { + const skopsmodelFile = model.config?.sklearn?.model?.file; + const skopssaveFormat = model.config?.sklearn?.model_format; + if (!skopsmodelFile) { + return [`# ⚠️ Model filename not specified in config.json`]; + } + if (skopssaveFormat === "pickle") { + return skopsPickle(model, skopsmodelFile); + } + else { + return skopsFormat(model, skopsmodelFile); + } + } + else { + return skopsJobLib(model); + } +}; +export const stable_audio_tools = (model) => [ + `import torch +import torchaudio +from einops import rearrange +from stable_audio_tools import get_pretrained_model +from stable_audio_tools.inference.generation import generate_diffusion_cond + +device = "cuda" if torch.cuda.is_available() else "cpu" + +# Download model +model, model_config = get_pretrained_model("${model.id}") +sample_rate = model_config["sample_rate"] +sample_size = model_config["sample_size"] + +model = model.to(device) + +# Set up text and timing conditioning +conditioning = [{ + "prompt": "128 BPM tech house drum loop", +}] + +# Generate stereo audio +output = generate_diffusion_cond( + model, + conditioning=conditioning, + sample_size=sample_size, + device=device +) + +# Rearrange audio batch to a single sequence +output = rearrange(output, "b d n -> d (b n)") + +# Peak normalize, clip, convert to int16, and save to file +output = output.to(torch.float32).div(torch.max(torch.abs(output))).clamp(-1, 1).mul(32767).to(torch.int16).cpu() +torchaudio.save("output.wav", output, sample_rate)`, +]; +export const fastai = (model) => [ + `from huggingface_hub import from_pretrained_fastai + +learn = from_pretrained_fastai("${model.id}")`, +]; +export const sam2 = (model) => { + const image_predictor = `# Use SAM2 with images +import torch +from sam2.sam2_image_predictor import SAM2ImagePredictor + +predictor = SAM2ImagePredictor.from_pretrained(${model.id}) + +with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16): + predictor.set_image() + masks, _, _ = predictor.predict()`; + const video_predictor = `# Use SAM2 with videos +import torch +from sam2.sam2_video_predictor import SAM2VideoPredictor + +predictor = SAM2VideoPredictor.from_pretrained(${model.id}) + +with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16): + state = predictor.init_state() + + # add new prompts and instantly get the output on the same frame + frame_idx, object_ids, masks = predictor.add_new_points(state, ): + + # propagate the prompts to get masklets throughout the video + for frame_idx, object_ids, masks in predictor.propagate_in_video(state): + ...`; + return [image_predictor, video_predictor]; +}; +export const sam_3d_objects = (model) => [ + `from inference import Inference, load_image, load_single_mask +from huggingface_hub import hf_hub_download + +path = hf_hub_download("${model.id}", "pipeline.yaml") +inference = Inference(path, compile=False) + +image = load_image("path_to_image.png") +mask = load_single_mask("path_to_mask.png", index=14) + +output = inference(image, mask)`, +]; +export const sam_3d_body = (model) => [ + `from notebook.utils import setup_sam_3d_body + +estimator = setup_sam_3d_body(${model.id}) +outputs = estimator.process_one_image(image) +rend_img = visualize_sample_together(image, outputs, estimator.faces)`, +]; +export const sampleFactory = (model) => [ + `python -m sample_factory.huggingface.load_from_hub -r ${model.id} -d ./train_dir`, +]; +function get_widget_examples_from_st_model(model) { + const widgetExample = model.widgetData?.[0]; + if (widgetExample?.source_sentence && widgetExample?.sentences?.length) { + return [widgetExample.source_sentence, ...widgetExample.sentences]; + } +} +export const sentenceTransformers = (model) => { + const remote_code_snippet = model.tags.includes(TAG_CUSTOM_CODE) ? ", trust_remote_code=True" : ""; + if (model.tags.includes("PyLate")) { + return [ + `from pylate import models + +queries = [ + "Which planet is known as the Red Planet?", + "What is the largest planet in our solar system?", +] + +documents = [ + ["Mars is the Red Planet.", "Venus is Earth's twin."], + ["Jupiter is the largest planet.", "Saturn has rings."], +] + +model = models.ColBERT(model_name_or_path="${model.id}") + +queries_emb = model.encode(queries, is_query=True) +docs_emb = model.encode(documents, is_query=False)`, + ]; + } + if (model.tags.includes("cross-encoder") || model.pipeline_tag == "text-ranking") { + return [ + `from sentence_transformers import CrossEncoder + +model = CrossEncoder("${model.id}"${remote_code_snippet}) + +query = "Which planet is known as the Red Planet?" +passages = [ + "Venus is often called Earth's twin because of its similar size and proximity.", + "Mars, known for its reddish appearance, is often referred to as the Red Planet.", + "Jupiter, the largest planet in our solar system, has a prominent red spot.", + "Saturn, famous for its rings, is sometimes mistaken for the Red Planet." +] + +scores = model.predict([(query, passage) for passage in passages]) +print(scores)`, + ]; + } + const exampleSentences = get_widget_examples_from_st_model(model) ?? [ + "The weather is lovely today.", + "It's so sunny outside!", + "He drove to the stadium.", + ]; + return [ + `from sentence_transformers import SentenceTransformer + +model = SentenceTransformer("${model.id}"${remote_code_snippet}) + +sentences = ${JSON.stringify(exampleSentences, null, 4)} +embeddings = model.encode(sentences) + +similarities = model.similarity(embeddings, embeddings) +print(similarities.shape) +# [${exampleSentences.length}, ${exampleSentences.length}]`, + ]; +}; +export const setfit = (model) => [ + `from setfit import SetFitModel + +model = SetFitModel.from_pretrained("${model.id}")`, +]; +export const spacy = (model) => [ + `!pip install https://huggingface.co/${model.id}/resolve/main/${nameWithoutNamespace(model.id)}-any-py3-none-any.whl + +# Using spacy.load(). +import spacy +nlp = spacy.load("${nameWithoutNamespace(model.id)}") + +# Importing as module. +import ${nameWithoutNamespace(model.id)} +nlp = ${nameWithoutNamespace(model.id)}.load()`, +]; +export const span_marker = (model) => [ + `from span_marker import SpanMarkerModel + +model = SpanMarkerModel.from_pretrained("${model.id}")`, +]; +export const stanza = (model) => [ + `import stanza + +stanza.download("${nameWithoutNamespace(model.id).replace("stanza-", "")}") +nlp = stanza.Pipeline("${nameWithoutNamespace(model.id).replace("stanza-", "")}")`, +]; +const speechBrainMethod = (speechbrainInterface) => { + switch (speechbrainInterface) { + case "EncoderClassifier": + return "classify_file"; + case "EncoderDecoderASR": + case "EncoderASR": + return "transcribe_file"; + case "SpectralMaskEnhancement": + return "enhance_file"; + case "SepformerSeparation": + return "separate_file"; + default: + return undefined; + } +}; +export const speechbrain = (model) => { + const speechbrainInterface = model.config?.speechbrain?.speechbrain_interface; + if (speechbrainInterface === undefined) { + return [`# interface not specified in config.json`]; + } + const speechbrainMethod = speechBrainMethod(speechbrainInterface); + if (speechbrainMethod === undefined) { + return [`# interface in config.json invalid`]; + } + return [ + `from speechbrain.pretrained import ${speechbrainInterface} +model = ${speechbrainInterface}.from_hparams( + "${model.id}" +) +model.${speechbrainMethod}("file.wav")`, + ]; +}; +export const terratorch = (model) => [ + `from terratorch.registry import BACKBONE_REGISTRY + +model = BACKBONE_REGISTRY.build("${model.id}")`, +]; +const hasChatTemplate = (model) => model.config?.tokenizer_config?.chat_template !== undefined || + model.config?.processor_config?.chat_template !== undefined || + model.config?.chat_template_jinja !== undefined; +export const transformers = (model) => { + const info = model.transformersInfo; + if (!info) { + return [`# ⚠️ Type of model unknown`]; + } + const remote_code_snippet = model.tags.includes(TAG_CUSTOM_CODE) ? ", trust_remote_code=True" : ""; + const autoSnippet = []; + if (info.processor) { + const processorVarName = info.processor === "AutoTokenizer" + ? "tokenizer" + : info.processor === "AutoFeatureExtractor" + ? "extractor" + : "processor"; + autoSnippet.push("# Load model directly", `from transformers import ${info.processor}, ${info.auto_model}`, "", `${processorVarName} = ${info.processor}.from_pretrained("${model.id}"` + remote_code_snippet + ")", `model = ${info.auto_model}.from_pretrained("${model.id}"` + remote_code_snippet + ")"); + if (model.tags.includes("conversational") && hasChatTemplate(model)) { + if (model.tags.includes("image-text-to-text")) { + autoSnippet.push("messages = [", [ + " {", + ' "role": "user",', + ' "content": [', + ' {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"},', + ' {"type": "text", "text": "What animal is on the candy?"}', + " ]", + " },", + ].join("\n"), "]"); + } + else { + autoSnippet.push("messages = [", ' {"role": "user", "content": "Who are you?"},', "]"); + } + autoSnippet.push(`inputs = ${processorVarName}.apply_chat_template(`, " messages,", " add_generation_prompt=True,", " tokenize=True,", " return_dict=True,", ' return_tensors="pt",', ").to(model.device)", "", "outputs = model.generate(**inputs, max_new_tokens=40)", `print(${processorVarName}.decode(outputs[0][inputs["input_ids"].shape[-1]:]))`); + } + } + else { + autoSnippet.push("# Load model directly", `from transformers import ${info.auto_model}`, `model = ${info.auto_model}.from_pretrained("${model.id}"` + remote_code_snippet + ', dtype="auto")'); + } + if (model.pipeline_tag && LIBRARY_TASK_MAPPING.transformers?.includes(model.pipeline_tag)) { + const pipelineSnippet = ["# Use a pipeline as a high-level helper"]; + if (REMOVED_IN_V5_TRANSFORMERS_PIPELINES.includes(model.pipeline_tag)) { + pipelineSnippet.push(`# Warning: Pipeline type "${model.pipeline_tag}" is no longer supported in transformers v5.`, `# You must load the model directly (see below) or downgrade to v4.x with:`, `# 'pip install "transformers<5.0.0'`); + } + pipelineSnippet.push("from transformers import pipeline", "", `pipe = pipeline("${model.pipeline_tag}", model="${model.id}"` + remote_code_snippet + ")"); + if (model.tags.includes("conversational")) { + if (model.tags.includes("image-text-to-text")) { + pipelineSnippet.push("messages = [", [ + " {", + ' "role": "user",', + ' "content": [', + ' {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"},', + ' {"type": "text", "text": "What animal is on the candy?"}', + " ]", + " },", + ].join("\n"), "]"); + pipelineSnippet.push("pipe(text=messages)"); + } + else { + pipelineSnippet.push("messages = [", ' {"role": "user", "content": "Who are you?"},', "]"); + pipelineSnippet.push("pipe(messages)"); + } + } + else if (model.pipeline_tag === "zero-shot-image-classification") { + pipelineSnippet.push("pipe(", ' "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/parrots.png",', ' candidate_labels=["animals", "humans", "landscape"],', ")"); + } + else if (model.pipeline_tag === "image-classification") { + pipelineSnippet.push('pipe("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/parrots.png")'); + } + return [pipelineSnippet.join("\n"), autoSnippet.join("\n")]; + } + return [autoSnippet.join("\n")]; +}; +export const transformersJS = (model) => { + if (!model.pipeline_tag) { + return [`// ⚠️ Unknown pipeline tag`]; + } + const libName = "@huggingface/transformers"; + return [ + `// npm i ${libName} +import { pipeline } from '${libName}'; + +// Allocate pipeline +const pipe = await pipeline('${model.pipeline_tag}', '${model.id}');`, + ]; +}; +const peftTask = (peftTaskType) => { + switch (peftTaskType) { + case "CAUSAL_LM": + return "CausalLM"; + case "SEQ_2_SEQ_LM": + return "Seq2SeqLM"; + case "TOKEN_CLS": + return "TokenClassification"; + case "SEQ_CLS": + return "SequenceClassification"; + default: + return undefined; + } +}; +export const peft = (model) => { + const { base_model_name_or_path: peftBaseModel, task_type: peftTaskType } = model.config?.peft ?? {}; + const pefttask = peftTask(peftTaskType); + if (!pefttask) { + return [`Task type is invalid.`]; + } + if (!peftBaseModel) { + return [`Base model is not found.`]; + } + return [ + `from peft import PeftModel +from transformers import AutoModelFor${pefttask} + +base_model = AutoModelFor${pefttask}.from_pretrained("${peftBaseModel}") +model = PeftModel.from_pretrained(base_model, "${model.id}")`, + ]; +}; +export const fasttext = (model) => [ + `from huggingface_hub import hf_hub_download +import fasttext + +model = fasttext.load_model(hf_hub_download("${model.id}", "model.bin"))`, +]; +export const stableBaselines3 = (model) => [ + `from huggingface_sb3 import load_from_hub +checkpoint = load_from_hub( + repo_id="${model.id}", + filename="{MODEL FILENAME}.zip", +)`, +]; +const nemoDomainResolver = (domain, model) => { + switch (domain) { + case "ASR": + return [ + `import nemo.collections.asr as nemo_asr +asr_model = nemo_asr.models.ASRModel.from_pretrained("${model.id}") + +transcriptions = asr_model.transcribe(["file.wav"])`, + ]; + default: + return undefined; + } +}; +export const mlAgents = (model) => [ + `mlagents-load-from-hf --repo-id="${model.id}" --local-dir="./download: string[]s"`, +]; +export const sentis = ( /* model: ModelData */) => [ + `string modelName = "[Your model name here].sentis"; +Model model = ModelLoader.Load(Application.streamingAssetsPath + "/" + modelName); +IWorker engine = WorkerFactory.CreateWorker(BackendType.GPUCompute, model); +// Please see provided C# file for more details +`, +]; +export const sana = (model) => [ + ` +# Load the model and infer image from text +import torch +from app.sana_pipeline import SanaPipeline +from torchvision.utils import save_image + +sana = SanaPipeline("configs/sana_config/1024ms/Sana_1600M_img1024.yaml") +sana.from_pretrained("hf://${model.id}") + +image = sana( + prompt='a cyberpunk cat with a neon sign that says "Sana"', + height=1024, + width=1024, + guidance_scale=5.0, + pag_guidance_scale=2.0, + num_inference_steps=18, +) `, +]; +export const vibevoice = (model) => [ + `import torch, soundfile as sf, librosa, numpy as np +from vibevoice.processor.vibevoice_processor import VibeVoiceProcessor +from vibevoice.modular.modeling_vibevoice_inference import VibeVoiceForConditionalGenerationInference + +# Load voice sample (should be 24kHz mono) +voice, sr = sf.read("path/to/voice_sample.wav") +if voice.ndim > 1: voice = voice.mean(axis=1) +if sr != 24000: voice = librosa.resample(voice, sr, 24000) + +processor = VibeVoiceProcessor.from_pretrained("${model.id}") +model = VibeVoiceForConditionalGenerationInference.from_pretrained( + "${model.id}", torch_dtype=torch.bfloat16 +).to("cuda").eval() +model.set_ddpm_inference_steps(5) + +inputs = processor(text=["Speaker 0: Hello!\\nSpeaker 1: Hi there!"], + voice_samples=[[voice]], return_tensors="pt") +audio = model.generate(**inputs, cfg_scale=1.3, + tokenizer=processor.tokenizer).speech_outputs[0] +sf.write("output.wav", audio.cpu().numpy().squeeze(), 24000)`, +]; +export const videoprism = (model) => [ + `# Install from https://github.com/google-deepmind/videoprism +import jax +from videoprism import models as vp + +flax_model = vp.get_model("${model.id}") +loaded_state = vp.load_pretrained_weights("${model.id}") + +@jax.jit +def forward_fn(inputs, train=False): + return flax_model.apply(loaded_state, inputs, train=train)`, +]; +export const vfimamba = (model) => [ + `from Trainer_finetune import Model + +model = Model.from_pretrained("${model.id}")`, +]; +export const lvface = (model) => [ + `from huggingface_hub import hf_hub_download + from inference_onnx import LVFaceONNXInferencer + +model_path = hf_hub_download("${model.id}", "LVFace-L_Glint360K/LVFace-L_Glint360K.onnx") +inferencer = LVFaceONNXInferencer(model_path, use_gpu=True, timeout=300) +img_path = 'path/to/image1.jpg' +embedding = inferencer.infer_from_image(img_path)`, +]; +export const voicecraft = (model) => [ + `from voicecraft import VoiceCraft + +model = VoiceCraft.from_pretrained("${model.id}")`, +]; +export const voxcpm = (model) => [ + `import soundfile as sf +from voxcpm import VoxCPM + +model = VoxCPM.from_pretrained("${model.id}") + +wav = model.generate( + text="VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly expressive speech.", + prompt_wav_path=None, # optional: path to a prompt speech for voice cloning + prompt_text=None, # optional: reference text + cfg_value=2.0, # LM guidance on LocDiT, higher for better adherence to the prompt, but maybe worse + inference_timesteps=10, # LocDiT inference timesteps, higher for better result, lower for fast speed + normalize=True, # enable external TN tool + denoise=True, # enable external Denoise tool + retry_badcase=True, # enable retrying mode for some bad cases (unstoppable) + retry_badcase_max_times=3, # maximum retrying times + retry_badcase_ratio_threshold=6.0, # maximum length restriction for bad case detection (simple but effective), it could be adjusted for slow pace speech +) + +sf.write("output.wav", wav, 16000) +print("saved: output.wav")`, +]; +export const vui = () => [ + `# !pip install git+https://github.com/fluxions-ai/vui + +import torchaudio + +from vui.inference import render +from vui.model import Vui, + +model = Vui.from_pretrained().cuda() +waveform = render( + model, + "Hey, here is some random stuff, usually something quite long as the shorter the text the less likely the model can cope!", +) +print(waveform.shape) +torchaudio.save("out.opus", waveform[0], 22050) +`, +]; +export const chattts = () => [ + `import ChatTTS +import torchaudio + +chat = ChatTTS.Chat() +chat.load_models(compile=False) # Set to True for better performance + +texts = ["PUT YOUR TEXT HERE",] + +wavs = chat.infer(texts, ) + +torchaudio.save("output1.wav", torch.from_numpy(wavs[0]), 24000)`, +]; +export const ultralytics = (model) => { + // ultralytics models must have a version tag (e.g. `yolov8`) + const versionTag = model.tags.find((tag) => tag.match(/^yolov\d+$/)); + const className = versionTag ? `YOLOv${versionTag.slice(4)}` : "YOLOvXX"; + const prefix = versionTag + ? "" + : `# Couldn't find a valid YOLO version tag.\n# Replace XX with the correct version.\n`; + return [ + prefix + + `from ultralytics import ${className} + +model = ${className}.from_pretrained("${model.id}") +source = 'http://images.cocodataset.org/val2017/000000039769.jpg' +model.predict(source=source, save=True)`, + ]; +}; +export const birefnet = (model) => [ + `# Option 1: use with transformers + +from transformers import AutoModelForImageSegmentation +birefnet = AutoModelForImageSegmentation.from_pretrained("${model.id}", trust_remote_code=True) +`, + `# Option 2: use with BiRefNet + +# Install from https://github.com/ZhengPeng7/BiRefNet + +from models.birefnet import BiRefNet +model = BiRefNet.from_pretrained("${model.id}")`, +]; +export const supertonic = () => [ + `from supertonic import TTS + +tts = TTS(auto_download=True) + +style = tts.get_voice_style(voice_name="M1") + +text = "The train delay was announced at 4:45 PM on Wed, Apr 3, 2024 due to track maintenance." +wav, duration = tts.synthesize(text, voice_style=style) + +tts.save_audio(wav, "output.wav")`, +]; +export const swarmformer = (model) => [ + `from swarmformer import SwarmFormerModel + +model = SwarmFormerModel.from_pretrained("${model.id}") +`, +]; +export const univa = (model) => [ + `# Follow installation instructions at https://github.com/PKU-YuanGroup/UniWorld-V1 + +from univa.models.qwen2p5vl.modeling_univa_qwen2p5vl import UnivaQwen2p5VLForConditionalGeneration + model = UnivaQwen2p5VLForConditionalGeneration.from_pretrained( + "${model.id}", + torch_dtype=torch.bfloat16, + attn_implementation="flash_attention_2", + ).to("cuda") + processor = AutoProcessor.from_pretrained("${model.id}") +`, +]; +const mlx_unknown = (model) => [ + `# Download the model from the Hub +pip install huggingface_hub[hf_xet] + +huggingface-cli download --local-dir ${nameWithoutNamespace(model.id)} ${model.id}`, +]; +const mlxlm = (model) => [ + `# Make sure mlx-lm is installed +# pip install --upgrade mlx-lm +# if on a CUDA device, also pip install mlx[cuda] + +# Generate text with mlx-lm +from mlx_lm import load, generate + +model, tokenizer = load("${model.id}") + +prompt = "Once upon a time in" +text = generate(model, tokenizer, prompt=prompt, verbose=True)`, +]; +const mlxchat = (model) => [ + `# Make sure mlx-lm is installed +# pip install --upgrade mlx-lm + +# Generate text with mlx-lm +from mlx_lm import load, generate + +model, tokenizer = load("${model.id}") + +prompt = "Write a story about Einstein" +messages = [{"role": "user", "content": prompt}] +prompt = tokenizer.apply_chat_template( + messages, add_generation_prompt=True +) + +text = generate(model, tokenizer, prompt=prompt, verbose=True)`, +]; +const mlxvlm = (model) => [ + `# Make sure mlx-vlm is installed +# pip install --upgrade mlx-vlm + +from mlx_vlm import load, generate +from mlx_vlm.prompt_utils import apply_chat_template +from mlx_vlm.utils import load_config + +# Load the model +model, processor = load("${model.id}") +config = load_config("${model.id}") + +# Prepare input +image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] +prompt = "Describe this image." + +# Apply chat template +formatted_prompt = apply_chat_template( + processor, config, prompt, num_images=1 +) + +# Generate output +output = generate(model, processor, formatted_prompt, image) +print(output)`, +]; +export const mlxim = (model) => [ + `from mlxim.model import create_model + +model = create_model(${model.id})`, +]; +export const mlx = (model) => { + if (model.pipeline_tag === "image-text-to-text") { + return mlxvlm(model); + } + if (model.pipeline_tag === "text-generation") { + if (model.tags.includes("conversational")) { + return mlxchat(model); + } + else { + return mlxlm(model); + } + } + return mlx_unknown(model); +}; +export const model2vec = (model) => [ + `from model2vec import StaticModel + +model = StaticModel.from_pretrained("${model.id}")`, +]; +export const pruna = (model) => { + let snippets; + if (model.tags.includes("diffusers")) { + snippets = pruna_diffusers(model); + } + else if (model.tags.includes("transformers")) { + snippets = pruna_transformers(model); + } + else { + snippets = pruna_default(model); + } + const ensurePrunaModelImport = (snippet) => { + if (!/^from pruna import PrunaModel/m.test(snippet)) { + return `from pruna import PrunaModel\n${snippet}`; + } + return snippet; + }; + snippets = snippets.map(ensurePrunaModelImport); + if (model.tags.includes("pruna_pro-ai")) { + return snippets.map((snippet) => snippet.replace(/\bpruna\b/g, "pruna_pro").replace(/\bPrunaModel\b/g, "PrunaProModel")); + } + return snippets; +}; +const pruna_diffusers = (model) => { + const diffusersSnippets = diffusers(model); + return diffusersSnippets.map((snippet) => snippet + // Replace pipeline classes with PrunaModel + .replace(/\b\w*Pipeline\w*\b/g, "PrunaModel") + // Clean up diffusers imports containing PrunaModel + .replace(/from diffusers import ([^,\n]*PrunaModel[^,\n]*)/g, "") + .replace(/from diffusers import ([^,\n]+),?\s*([^,\n]*PrunaModel[^,\n]*)/g, "from diffusers import $1") + .replace(/from diffusers import\s*(\n|$)/g, "") + // Fix PrunaModel imports + .replace(/from diffusers import PrunaModel/g, "from pruna import PrunaModel") + .replace(/from diffusers import ([^,\n]+), PrunaModel/g, "from diffusers import $1") + .replace(/from diffusers import PrunaModel, ([^,\n]+)/g, "from diffusers import $1") + // Clean up whitespace + .replace(/\n\n+/g, "\n") + .trim()); +}; +const pruna_transformers = (model) => { + const info = model.transformersInfo; + const transformersSnippets = transformers(model); + // Replace pipeline with PrunaModel + let processedSnippets = transformersSnippets.map((snippet) => snippet + .replace(/from transformers import pipeline/g, "from pruna import PrunaModel") + .replace(/pipeline\([^)]*\)/g, `PrunaModel.from_pretrained("${model.id}")`)); + // Additional cleanup if auto_model info is available + if (info?.auto_model) { + processedSnippets = processedSnippets.map((snippet) => snippet + .replace(new RegExp(`from transformers import ${info.auto_model}\n?`, "g"), "") + .replace(new RegExp(`${info.auto_model}.from_pretrained`, "g"), "PrunaModel.from_pretrained") + .replace(new RegExp(`^.*from.*import.*(, *${info.auto_model})+.*$`, "gm"), (line) => line.replace(new RegExp(`, *${info.auto_model}`, "g"), ""))); + } + return processedSnippets; +}; +const pruna_default = (model) => [ + `from pruna import PrunaModel +model = PrunaModel.from_pretrained("${model.id}") +`, +]; +export const nemo = (model) => { + let command = undefined; + // Resolve the tag to a nemo domain/sub-domain + if (model.tags.includes("automatic-speech-recognition")) { + command = nemoDomainResolver("ASR", model); + } + return command ?? [`# tag did not correspond to a valid NeMo domain.`]; +}; +export const outetts = (model) => { + // Don’t show this block on GGUF / ONNX mirrors + const t = model.tags ?? []; + if (t.includes("gguf") || t.includes("onnx")) { + return []; + } + // v1.0 HF → minimal runnable snippet + return [ + ` + import outetts + + enum = outetts.Models("${model.id}".split("/", 1)[1]) # VERSION_1_0_SIZE_1B + cfg = outetts.ModelConfig.auto_config(enum, outetts.Backend.HF) + tts = outetts.Interface(cfg) + + speaker = tts.load_default_speaker("EN-FEMALE-1-NEUTRAL") + tts.generate( + outetts.GenerationConfig( + text="Hello there, how are you doing?", + speaker=speaker, + ) + ).save("output.wav") + `, + ]; +}; +export const pxia = (model) => [ + `from pxia import AutoModel + +model = AutoModel.from_pretrained("${model.id}")`, +]; +export const pythae = (model) => [ + `from pythae.models import AutoModel + +model = AutoModel.load_from_hf_hub("${model.id}")`, +]; +export const qwen3_tts = (model) => [ + `# pip install qwen-tts +import torch +import soundfile as sf +from qwen_tts import Qwen3TTSModel + +model = Qwen3TTSModel.from_pretrained( + "${model.id}", + device_map="cuda:0", + dtype=torch.bfloat16, + attn_implementation="flash_attention_2", +) + +wavs, sr = model.generate_custom_voice( + text="Your text here.", + language="English", + speaker="Ryan", + instruct="Speak in a natural tone.", +) + +sf.write("output.wav", wavs[0], sr)`, +]; +const musicgen = (model) => [ + `from audiocraft.models import MusicGen + +model = MusicGen.get_pretrained("${model.id}") + +descriptions = ['happy rock', 'energetic EDM', 'sad jazz'] +wav = model.generate(descriptions) # generates 3 samples.`, +]; +const magnet = (model) => [ + `from audiocraft.models import MAGNeT + +model = MAGNeT.get_pretrained("${model.id}") + +descriptions = ['disco beat', 'energetic EDM', 'funky groove'] +wav = model.generate(descriptions) # generates 3 samples.`, +]; +const audiogen = (model) => [ + `from audiocraft.models import AudioGen + +model = AudioGen.get_pretrained("${model.id}") +model.set_generation_params(duration=5) # generate 5 seconds. +descriptions = ['dog barking', 'sirene of an emergency vehicle', 'footsteps in a corridor'] +wav = model.generate(descriptions) # generates 3 samples.`, +]; +export const anemoi = (model) => [ + `from anemoi.inference.runners.default import DefaultRunner +from anemoi.inference.config.run import RunConfiguration +# Create Configuration +config = RunConfiguration(checkpoint = {"huggingface":"${model.id}"}) +# Load Runner +runner = DefaultRunner(config)`, +]; +export const audiocraft = (model) => { + if (model.tags.includes("musicgen")) { + return musicgen(model); + } + else if (model.tags.includes("audiogen")) { + return audiogen(model); + } + else if (model.tags.includes("magnet")) { + return magnet(model); + } + else { + return [`# Type of model unknown.`]; + } +}; +export const whisperkit = () => [ + `# Install CLI with Homebrew on macOS device +brew install whisperkit-cli + +# View all available inference options +whisperkit-cli transcribe --help + +# Download and run inference using whisper base model +whisperkit-cli transcribe --audio-path /path/to/audio.mp3 + +# Or use your preferred model variant +whisperkit-cli transcribe --model "large-v3" --model-prefix "distil" --audio-path /path/to/audio.mp3 --verbose`, +]; +export const threedtopia_xl = (model) => [ + `from threedtopia_xl.models import threedtopia_xl + +model = threedtopia_xl.from_pretrained("${model.id}") +model.generate(cond="path/to/image.png")`, +]; +export const hezar = (model) => [ + `from hezar import Model + +model = Model.load("${model.id}")`, +]; +export const zonos = (model) => [ + `# pip install git+https://github.com/Zyphra/Zonos.git +import torchaudio +from zonos.model import Zonos +from zonos.conditioning import make_cond_dict + +model = Zonos.from_pretrained("${model.id}", device="cuda") + +wav, sr = torchaudio.load("speaker.wav") # 5-10s reference clip +speaker = model.make_speaker_embedding(wav, sr) + +cond = make_cond_dict(text="Hello, world!", speaker=speaker, language="en-us") +codes = model.generate(model.prepare_conditioning(cond)) + +audio = model.autoencoder.decode(codes)[0].cpu() +torchaudio.save("sample.wav", audio, model.autoencoder.sampling_rate) +`, +]; +export const moshi = (model) => { + // Detect backend from model name (no distinguishing tags available) + if (model.id.includes("-mlx")) { + // MLX backend (macOS Apple Silicon) + // -q flag only accepts 4 or 8, bf16 models don't use it + const quantFlag = model.id.includes("-q4") ? " -q 4" : model.id.includes("-q8") ? " -q 8" : ""; + return [ + `# pip install moshi_mlx +# Run local inference (macOS Apple Silicon) +python -m moshi_mlx.local${quantFlag} --hf-repo "${model.id}" + +# Or run with web UI +python -m moshi_mlx.local_web${quantFlag} --hf-repo "${model.id}"`, + ]; + } + if (model.id.includes("-candle")) { + // Rust/Candle backend + return [ + `# pip install rustymimi +# Candle backend - see https://github.com/kyutai-labs/moshi +# for Rust installation instructions`, + ]; + } + // PyTorch backend (default) + return [ + `# pip install moshi +# Run the interactive web server +python -m moshi.server --hf-repo "${model.id}" +# Then open https://localhost:8998 in your browser`, + `# pip install moshi +import torch +from moshi.models import loaders + +# Load checkpoint info from HuggingFace +checkpoint = loaders.CheckpointInfo.from_hf_repo("${model.id}") + +# Load the Mimi audio codec +mimi = checkpoint.get_mimi(device="cuda") +mimi.set_num_codebooks(8) + +# Encode audio (24kHz, mono) +wav = torch.randn(1, 1, 24000 * 10) # [batch, channels, samples] +with torch.no_grad(): + codes = mimi.encode(wav.cuda()) + decoded = mimi.decode(codes)`, + ]; +}; +//#endregion diff --git a/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.spec.d.ts b/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.spec.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..f4992762e83f6647ee2d130b8b7dc9a66c0d93af --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.spec.d.ts @@ -0,0 +1,2 @@ +export {}; +//# sourceMappingURL=model-libraries-snippets.spec.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.spec.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.spec.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c78932806acc5f8931c62472c22ccb33afa974da --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.spec.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"model-libraries-snippets.spec.d.ts","sourceRoot":"","sources":["../../src/model-libraries-snippets.spec.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.spec.js b/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.spec.js new file mode 100644 index 0000000000000000000000000000000000000000..4741c0d8b6af39108d16c0674181e6d47d629c9b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-libraries-snippets.spec.js @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import { llama_cpp_python } from "./model-libraries-snippets.js"; +describe("model-libraries-snippets", () => { + it("llama_cpp_python conversational", async () => { + const model = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + pipeline_tag: "text-generation", + tags: ["conversational"], + inference: "", + }; + const snippet = llama_cpp_python(model); + expect(snippet.join("\n")).toEqual(`# !pip install llama-cpp-python + +from llama_cpp import Llama + +llm = Llama.from_pretrained( + repo_id="bartowski/Llama-3.2-3B-Instruct-GGUF", + filename="{{GGUF_FILE}}", +) + +llm.create_chat_completion( + messages = [ + { + "role": "user", + "content": "What is the capital of France?" + } + ] +)`); + }); + it("llama_cpp_python non-conversational", async () => { + const model = { + id: "mlabonne/gemma-2b-GGUF", + tags: [""], + inference: "", + }; + const snippet = llama_cpp_python(model); + expect(snippet.join("\n")).toEqual(`# !pip install llama-cpp-python + +from llama_cpp import Llama + +llm = Llama.from_pretrained( + repo_id="mlabonne/gemma-2b-GGUF", + filename="{{GGUF_FILE}}", +) + +output = llm( + "Once upon a time,", + max_tokens=512, + echo=True +) +print(output)`); + }); +}); diff --git a/node_modules/@huggingface/tasks/dist/esm/model-libraries.d.ts b/node_modules/@huggingface/tasks/dist/esm/model-libraries.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..320f1da3af988daf046b9ba44473b8b0c11c32a1 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-libraries.d.ts @@ -0,0 +1,1705 @@ +import type { ModelData } from "./model-data.js"; +import type { ElasticSearchQuery } from "./model-libraries-downloads.js"; +/** + * Elements configurable by a model library. + */ +export interface LibraryUiElement { + /** + * Pretty name of the library. + * displayed in tags, and on the main + * call-to-action button on the model page. + */ + prettyLabel: string; + /** + * Repo name of the library's (usually on GitHub) code repo + */ + repoName: string; + /** + * URL to library's (usually on GitHub) code repo + */ + repoUrl: string; + /** + * URL to library's docs + */ + docsUrl?: string; + /** + * Code snippet(s) displayed on model page + */ + snippets?: (model: ModelData) => string[]; + /** + * Elastic query used to count this library's model downloads + * + * By default, those files are counted: + * "config.json", "config.yaml", "hyperparams.yaml", "params.json", "meta.yaml" + */ + countDownloads?: ElasticSearchQuery; + /** + * should we display this library in hf.co/models filter + * (only for popular libraries with > 100 models) + */ + filter?: boolean; +} +/** + * Add your new library here. + * + * This is for modeling (= architectures) libraries, not for file formats (like ONNX, etc). + * (unlike libraries, file formats live in an enum inside the internal codebase.) + * + * Doc on how to add a library to the Hub: + * + * https://huggingface.co/docs/hub/models-adding-libraries + * + * /!\ IMPORTANT + * + * The key you choose is the tag your models have in their library_name on the Hub. + */ +export declare const MODEL_LIBRARIES_UI_ELEMENTS: { + acestep: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "adapter-transformers": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + allennlp: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + anemoi: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + araclip: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + }; + "aviation-ner": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + countDownloads: string; + filter: false; + }; + asteroid: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + audiocraft: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + audioseal: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + "bagel-mot": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + bboxmaskpose: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + ben2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + bertopic: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + big_vision: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + bionemo: { + prettyLabel: string; + repoName: string; + filter: false; + repoUrl: string; + countDownloads: string; + }; + birder: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + birefnet: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + bm25s: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + boltzgen: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + cancertathomev2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + cartesia_pytorch: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + cartesia_mlx: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + champ: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + chatterbox: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: () => string[]; + countDownloads: string; + filter: false; + }; + chaossim: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + filter: false; + }; + chat_tts: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: () => string[]; + filter: false; + countDownloads: string; + }; + chexmix: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "chronos-forecasting": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + clara: { + prettyLabel: string; + repoName: string; + filter: false; + repoUrl: string; + countDownloads: string; + }; + clipscope: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "cloud-agents": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + collectorvision: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + colipri: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + cosyvoice: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + cotracker: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + colpali: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + comet: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + cosmos: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "cxr-foundation": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: () => string[]; + filter: false; + countDownloads: string; + }; + deepforest: { + prettyLabel: string; + repoName: string; + docsUrl: string; + repoUrl: string; + }; + "depth-anything-v2": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + "depth-pro": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "derm-foundation": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: () => string[]; + filter: false; + countDownloads: string; + }; + "describe-anything": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "dia-tts": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + dia2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "diff-interpretation-tuning": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + diffree: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + diffusers: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + diffusionkit: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + "docking-at-home": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + doctr: { + prettyLabel: string; + repoName: string; + repoUrl: string; + }; + edsnlp: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + countDownloads: string; + }; + elm: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + encoderfile: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + espnet: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + eupe: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + fairseq: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + fastai: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + fastprint: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + fasttext: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + fixer: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + flair: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + fme: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + }; + "gemma.cpp": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "geometry-crafter": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + gliner: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + gliner2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "glm-tts": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "glyph-byt5": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "granite-library": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + grok: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "habibi-tts": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + hallo: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + hermes: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + holomotion: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + hezar: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + countDownloads: string; + }; + htrflow: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + }; + "hunyuan-dit": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "hunyuan3d-2": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "hunyuanworld-voyager": { + prettyLabel: string; + repoName: string; + repoUrl: string; + }; + "hy-worldplay": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "hy-world-2": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "image-matching-models": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + imstoucan: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "index-tts": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + infinitetalk: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "infinite-you": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + intellifold: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "ising-decoding": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + keras: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "tf-keras": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + countDownloads: string; + }; + "keras-hub": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + kernels: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + countDownloads: string; + }; + "kimi-audio": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + kittentts: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + kronos: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + k2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + }; + "lyra-2.0": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + lagernvs: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "lightning-ir": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + litert: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "litert-lm": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + lerobot: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + }; + lightglue: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + liveportrait: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "longcat-video-avatar-1.5": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + }; + "llama-cpp-python": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + "mini-omni2": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + mindspore: { + prettyLabel: string; + repoName: string; + repoUrl: string; + }; + "magi-1": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "magenta-realtime": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "magenta-realtime-2": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "mamba-ssm": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + }; + "manas-1": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "mars5-tts": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + matanyone: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "mesh-anything": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + snippets: () => string[]; + }; + merlin: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + medvae: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + mitie: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "ml-agents": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "ml-sharp": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + mlx: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + "mlx-image": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + "mlc-llm": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + }; + model2vec: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + moshi: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + mtvcraft: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + multimolecule: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + nemo: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "nv-medtech": { + prettyLabel: string; + repoName: string; + filter: false; + repoUrl: string; + countDownloads: string; + }; + "open-oasis": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + open_clip: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + openpeerllm: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + countDownloads: string; + filter: false; + }; + "open-sora": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + outetts: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + paddlenlp: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + PaddleOCR: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + peft: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "perception-encoder": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + countDownloads: string; + }; + "phantom-wan": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + "pocket-tts": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + countDownloads: string; + }; + "pruna-ai": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + docsUrl: string; + }; + pxia: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "pyannote-audio": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + "py-feat": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + }; + pythae: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + quantumpeer: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + qwen3_tts: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + recurrentgemma: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + relik: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + refiners: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + }; + renderformer: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + reverb: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + }; + rkllm: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + "robo-orchard-lab": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + rwkv: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + }; + saelens: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: () => string[]; + filter: false; + }; + "scail-2": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + sam2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + countDownloads: string; + }; + "sam-3d-body": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + countDownloads: string; + }; + "sam-3d-objects": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + snippets: (model: ModelData) => string[]; + countDownloads: string; + }; + same: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "sample-factory": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "sap-rpt-1-oss": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + snippets: () => string[]; + }; + sapiens: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + sapiens2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + seedvr: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "self-forcing": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "sentence-transformers": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + setfit: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + sklearn: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + spacy: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "span-marker": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + speechbrain: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "ssr-speech": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "stable-audio-3": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "stable-audio-tools": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + monkeyocr: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "diffusion-single-file": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "seed-story": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + snippets: () => string[]; + }; + skala: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + soloaudio: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + songbloom: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "stable-baselines3": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + stanza: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + supertonic: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: () => string[]; + filter: false; + }; + swarmformer: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "synthefy-migas": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "f5-tts": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + genmo: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "tencent-song-generation": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + tensorflowtts: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + }; + tensorrt: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + tabpfn: { + prettyLabel: string; + repoName: string; + repoUrl: string; + }; + terratorch: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + "tic-clip": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + timesfm: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + timm: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + tirex: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + torchgeo: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + }; + transformers: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + "transformers.js": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + }; + trellis: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + trellis2: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + tunejury: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + ultralytics: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + filter: false; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + univa: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: true; + countDownloads: string; + }; + "uni-3dar": { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + countDownloads: string; + }; + "unity-sentis": { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: () => string[]; + filter: true; + countDownloads: string; + }; + sana: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + videoprism: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + "vfi-mamba": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + vismatch: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + lvface: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + voicecraft: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + }; + voxcpm: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + vui: { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + snippets: () => string[]; + }; + vibevoice: { + prettyLabel: string; + repoName: string; + repoUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + videox_fun: { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + }; + "wan2.2": { + prettyLabel: string; + repoName: string; + repoUrl: string; + countDownloads: string; + }; + wham: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + countDownloads: string; + }; + whisperkit: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: () => string[]; + countDownloads: string; + }; + yolov10: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; + yolov26: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + countDownloads: string; + }; + zonos: { + prettyLabel: string; + repoName: string; + repoUrl: string; + docsUrl: string; + snippets: (model: ModelData) => string[]; + filter: false; + }; + "3dtopia-xl": { + prettyLabel: string; + repoName: string; + repoUrl: string; + filter: false; + countDownloads: string; + snippets: (model: ModelData) => string[]; + }; +}; +export type ModelLibraryKey = keyof typeof MODEL_LIBRARIES_UI_ELEMENTS; +export declare const ALL_MODEL_LIBRARY_KEYS: ModelLibraryKey[]; +export declare const ALL_DISPLAY_MODEL_LIBRARY_KEYS: ("acestep" | "adapter-transformers" | "allennlp" | "anemoi" | "araclip" | "aviation-ner" | "asteroid" | "audiocraft" | "audioseal" | "bagel-mot" | "bboxmaskpose" | "ben2" | "bertopic" | "big_vision" | "bionemo" | "birder" | "birefnet" | "bm25s" | "boltzgen" | "cancertathomev2" | "cartesia_pytorch" | "cartesia_mlx" | "champ" | "chatterbox" | "chaossim" | "chat_tts" | "chexmix" | "chronos-forecasting" | "clara" | "clipscope" | "cloud-agents" | "collectorvision" | "colipri" | "cosyvoice" | "cotracker" | "colpali" | "comet" | "cosmos" | "cxr-foundation" | "deepforest" | "depth-anything-v2" | "depth-pro" | "derm-foundation" | "describe-anything" | "dia-tts" | "dia2" | "diff-interpretation-tuning" | "diffree" | "diffusers" | "diffusionkit" | "docking-at-home" | "doctr" | "edsnlp" | "elm" | "encoderfile" | "espnet" | "eupe" | "fairseq" | "fastai" | "fastprint" | "fasttext" | "fixer" | "flair" | "fme" | "gemma.cpp" | "geometry-crafter" | "gliner" | "gliner2" | "glm-tts" | "glyph-byt5" | "granite-library" | "grok" | "habibi-tts" | "hallo" | "hermes" | "holomotion" | "hezar" | "htrflow" | "hunyuan-dit" | "hunyuan3d-2" | "hunyuanworld-voyager" | "hy-worldplay" | "hy-world-2" | "image-matching-models" | "imstoucan" | "index-tts" | "infinitetalk" | "infinite-you" | "intellifold" | "ising-decoding" | "keras" | "tf-keras" | "keras-hub" | "kernels" | "kimi-audio" | "kittentts" | "kronos" | "k2" | "lyra-2.0" | "lagernvs" | "lightning-ir" | "litert" | "litert-lm" | "lerobot" | "lightglue" | "liveportrait" | "longcat-video-avatar-1.5" | "llama-cpp-python" | "mini-omni2" | "mindspore" | "magi-1" | "magenta-realtime" | "magenta-realtime-2" | "mamba-ssm" | "manas-1" | "mars5-tts" | "matanyone" | "mesh-anything" | "merlin" | "medvae" | "mitie" | "ml-agents" | "ml-sharp" | "mlx" | "mlx-image" | "mlc-llm" | "model2vec" | "moshi" | "mtvcraft" | "multimolecule" | "nemo" | "nv-medtech" | "open-oasis" | "open_clip" | "openpeerllm" | "open-sora" | "outetts" | "paddlenlp" | "PaddleOCR" | "peft" | "perception-encoder" | "phantom-wan" | "pocket-tts" | "pruna-ai" | "pxia" | "pyannote-audio" | "py-feat" | "pythae" | "quantumpeer" | "qwen3_tts" | "recurrentgemma" | "relik" | "refiners" | "renderformer" | "reverb" | "rkllm" | "robo-orchard-lab" | "rwkv" | "saelens" | "scail-2" | "sam2" | "sam-3d-body" | "sam-3d-objects" | "same" | "sample-factory" | "sap-rpt-1-oss" | "sapiens" | "sapiens2" | "seedvr" | "self-forcing" | "sentence-transformers" | "setfit" | "sklearn" | "spacy" | "span-marker" | "speechbrain" | "ssr-speech" | "stable-audio-3" | "stable-audio-tools" | "monkeyocr" | "diffusion-single-file" | "seed-story" | "skala" | "soloaudio" | "songbloom" | "stable-baselines3" | "stanza" | "supertonic" | "swarmformer" | "synthefy-migas" | "f5-tts" | "genmo" | "tencent-song-generation" | "tensorflowtts" | "tensorrt" | "tabpfn" | "terratorch" | "tic-clip" | "timesfm" | "timm" | "tirex" | "torchgeo" | "transformers" | "transformers.js" | "trellis" | "trellis2" | "tunejury" | "ultralytics" | "univa" | "uni-3dar" | "unity-sentis" | "sana" | "videoprism" | "vfi-mamba" | "vismatch" | "lvface" | "voicecraft" | "voxcpm" | "vui" | "vibevoice" | "videox_fun" | "wan2.2" | "wham" | "whisperkit" | "yolov10" | "yolov26" | "zonos" | "3dtopia-xl")[]; +//# sourceMappingURL=model-libraries.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/model-libraries.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/model-libraries.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f6ce894237e76719a0d6d8e17d9ab95c16bc841a --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-libraries.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"model-libraries.d.ts","sourceRoot":"","sources":["../../src/model-libraries.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAEzE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;;;OAIG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,MAAM,EAAE,CAAC;IAC1C;;;;;OAKG;IACH,cAAc,CAAC,EAAE,kBAAkB,CAAC;IACpC;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;;;;;;GAaG;AAEH,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAonDI,CAAC;AAE7C,MAAM,MAAM,eAAe,GAAG,MAAM,OAAO,2BAA2B,CAAC;AAEvE,eAAO,MAAM,sBAAsB,EAA+C,eAAe,EAAE,CAAC;AAEpG,eAAO,MAAM,8BAA8B,urGAQ1B,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/model-libraries.js b/node_modules/@huggingface/tasks/dist/esm/model-libraries.js new file mode 100644 index 0000000000000000000000000000000000000000..425edb7f8a94e28766b46f77bb99a25e594bae83 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/model-libraries.js @@ -0,0 +1,1672 @@ +import * as snippets from "./model-libraries-snippets.js"; +/** + * Add your new library here. + * + * This is for modeling (= architectures) libraries, not for file formats (like ONNX, etc). + * (unlike libraries, file formats live in an enum inside the internal codebase.) + * + * Doc on how to add a library to the Hub: + * + * https://huggingface.co/docs/hub/models-adding-libraries + * + * /!\ IMPORTANT + * + * The key you choose is the tag your models have in their library_name on the Hub. + */ +export const MODEL_LIBRARIES_UI_ELEMENTS = { + acestep: { + prettyLabel: "ACE-Step", + repoName: "ACE-Step", + repoUrl: "https://github.com/ace-step/ACE-Step", + filter: false, + countDownloads: `path:"ace_step_transformer/config.json"`, + }, + "adapter-transformers": { + prettyLabel: "Adapters", + repoName: "adapters", + repoUrl: "https://github.com/Adapter-Hub/adapters", + docsUrl: "https://huggingface.co/docs/hub/adapters", + snippets: snippets.adapters, + filter: true, + countDownloads: `path:"adapter_config.json"`, + }, + allennlp: { + prettyLabel: "AllenNLP", + repoName: "AllenNLP", + repoUrl: "https://github.com/allenai/allennlp", + docsUrl: "https://huggingface.co/docs/hub/allennlp", + snippets: snippets.allennlp, + filter: true, + }, + anemoi: { + prettyLabel: "AnemoI", + repoName: "AnemoI", + repoUrl: "https://github.com/ecmwf/anemoi-inference", + docsUrl: "https://anemoi.readthedocs.io/en/latest/", + filter: false, + countDownloads: `path_extension:"ckpt"`, + snippets: snippets.anemoi, + }, + araclip: { + prettyLabel: "AraClip", + repoName: "AraClip", + repoUrl: "https://huggingface.co/Arabic-Clip/araclip", + filter: false, + snippets: snippets.araclip, + }, + "aviation-ner": { + prettyLabel: "Aviation NER", + repoName: "Aviation NER", + repoUrl: "https://github.com/Boeing/aviation_ner_sdr", + docsUrl: "https://github.com/Boeing/aviation_ner_sdr", + countDownloads: `path:"gliner_config.json"`, + filter: false, + }, + asteroid: { + prettyLabel: "Asteroid", + repoName: "Asteroid", + repoUrl: "https://github.com/asteroid-team/asteroid", + docsUrl: "https://huggingface.co/docs/hub/asteroid", + snippets: snippets.asteroid, + filter: true, + countDownloads: `path:"pytorch_model.bin"`, + }, + audiocraft: { + prettyLabel: "Audiocraft", + repoName: "audiocraft", + repoUrl: "https://github.com/facebookresearch/audiocraft", + snippets: snippets.audiocraft, + filter: false, + countDownloads: `path:"state_dict.bin"`, + }, + audioseal: { + prettyLabel: "AudioSeal", + repoName: "audioseal", + repoUrl: "https://github.com/facebookresearch/audioseal", + filter: false, + countDownloads: `path_extension:"pth"`, + snippets: snippets.audioseal, + }, + "bagel-mot": { + prettyLabel: "Bagel", + repoName: "Bagel", + repoUrl: "https://github.com/ByteDance-Seed/Bagel/", + filter: false, + countDownloads: `path:"llm_config.json"`, + }, + bboxmaskpose: { + prettyLabel: "BBoxMaskPose", + repoName: "BBoxMaskPose", + repoUrl: "https://github.com/MiraPurkrabek/BBoxMaskPose", + filter: false, + countDownloads: `path_extension:"pth"`, + }, + ben2: { + prettyLabel: "BEN2", + repoName: "BEN2", + repoUrl: "https://github.com/PramaLLC/BEN2", + snippets: snippets.ben2, + filter: false, + }, + bertopic: { + prettyLabel: "BERTopic", + repoName: "BERTopic", + repoUrl: "https://github.com/MaartenGr/BERTopic", + snippets: snippets.bertopic, + filter: true, + }, + big_vision: { + prettyLabel: "Big Vision", + repoName: "big_vision", + repoUrl: "https://github.com/google-research/big_vision", + filter: false, + countDownloads: `path_extension:"npz"`, + }, + bionemo: { + prettyLabel: "BioNeMo", + repoName: "BioNeMo", + filter: false, + repoUrl: "https://github.com/nvidia/BioNeMo", + countDownloads: `path_extension:"ckpt" OR path:"config.json"`, + }, + birder: { + prettyLabel: "Birder", + repoName: "Birder", + repoUrl: "https://gitlab.com/birder/birder", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + birefnet: { + prettyLabel: "BiRefNet", + repoName: "BiRefNet", + repoUrl: "https://github.com/ZhengPeng7/BiRefNet", + snippets: snippets.birefnet, + filter: false, + }, + bm25s: { + prettyLabel: "BM25S", + repoName: "bm25s", + repoUrl: "https://github.com/xhluca/bm25s", + snippets: snippets.bm25s, + filter: false, + countDownloads: `path:"params.index.json"`, + }, + boltzgen: { + prettyLabel: "BoltzGen", + repoName: "BoltzGen", + repoUrl: "https://github.com/HannesStark/boltzgen", + filter: false, + countDownloads: `path:"boltzgen1_diverse.ckpt"`, + }, + cancertathomev2: { + prettyLabel: "Cancer@HomeV2", + repoName: "Cancer@HomeV2", + repoUrl: "https://huggingface.co/OpenPeerAI/CancerAtHomeV2", + filter: false, + countDownloads: `path:"run.py"`, + }, + cartesia_pytorch: { + prettyLabel: "Cartesia Pytorch", + repoName: "Cartesia Pytorch", + repoUrl: "https://github.com/cartesia-ai/cartesia_pytorch", + snippets: snippets.cartesia_pytorch, + }, + cartesia_mlx: { + prettyLabel: "Cartesia MLX", + repoName: "Cartesia MLX", + repoUrl: "https://github.com/cartesia-ai/cartesia_mlx", + snippets: snippets.cartesia_mlx, + }, + champ: { + prettyLabel: "Champ", + repoName: "Champ", + repoUrl: "https://github.com/fudan-generative-vision/champ", + countDownloads: `path:"champ/motion_module.pth"`, + }, + chatterbox: { + prettyLabel: "Chatterbox", + repoName: "Chatterbox", + repoUrl: "https://github.com/resemble-ai/chatterbox", + snippets: snippets.chatterbox, + countDownloads: `path:"tokenizer.json"`, + filter: false, + }, + chaossim: { + prettyLabel: "ChaosSIM", + repoName: "ChaosSIM", + repoUrl: "https://huggingface.co/OpenPeerAI/ChaosSIM/", + countDownloads: `path:"ChaosSim.nb"`, + filter: false, + }, + chat_tts: { + prettyLabel: "ChatTTS", + repoName: "ChatTTS", + repoUrl: "https://github.com/2noise/ChatTTS.git", + snippets: snippets.chattts, + filter: false, + countDownloads: `path:"asset/GPT.pt"`, + }, + chexmix: { + prettyLabel: "CheXmix", + repoName: "CheXmix", + repoUrl: "https://github.com/StanfordMIMI/CheXmix", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + "chronos-forecasting": { + prettyLabel: "Chronos", + repoName: "Chronos", + repoUrl: "https://github.com/amazon-science/chronos-forecasting", + snippets: snippets.chronos_forecasting, + }, + clara: { + prettyLabel: "Clara", + repoName: "Clara", + filter: false, + repoUrl: "https://github.com/nvidia/clara", + countDownloads: `path_extension:"ckpt" OR path:"config.json"`, + }, + clipscope: { + prettyLabel: "clipscope", + repoName: "clipscope", + repoUrl: "https://github.com/Lewington-pitsos/clipscope", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + "cloud-agents": { + prettyLabel: "Cloud Agents", + repoName: "Cloud Agents", + repoUrl: "https://huggingface.co/OpenPeerAI/Cloud-Agents", + filter: false, + countDownloads: `path:"setup.py"`, + }, + collectorvision: { + prettyLabel: "CollectorVision", + repoName: "CollectorVision", + repoUrl: "https://github.com/HanClinto/CollectorVision", + snippets: snippets.collectorvision, + filter: false, + countDownloads: `path_extension:"onnx"`, + }, + colipri: { + prettyLabel: "COLIPRI", + repoName: "COLIPRI", + repoUrl: "https://huggingface.co/microsoft/colipri", + snippets: snippets.colipri, + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + cosyvoice: { + prettyLabel: "CosyVoice", + repoName: "CosyVoice", + repoUrl: "https://github.com/FunAudioLLM/CosyVoice", + filter: false, + countDownloads: `path_extension:"onnx" OR path_extension:"pt"`, + }, + cotracker: { + prettyLabel: "CoTracker", + repoName: "CoTracker", + repoUrl: "https://github.com/facebookresearch/co-tracker", + filter: false, + countDownloads: `path_extension:"pth"`, + }, + colpali: { + prettyLabel: "ColPali", + repoName: "ColPali", + repoUrl: "https://github.com/ManuelFay/colpali", + filter: false, + countDownloads: `path:"adapter_config.json"`, + }, + comet: { + prettyLabel: "COMET", + repoName: "COMET", + repoUrl: "https://github.com/Unbabel/COMET/", + countDownloads: `path:"hparams.yaml"`, + }, + cosmos: { + prettyLabel: "Cosmos", + repoName: "Cosmos", + repoUrl: "https://github.com/NVIDIA/Cosmos", + countDownloads: `path:"config.json" OR path_extension:"pt"`, + }, + "cxr-foundation": { + prettyLabel: "CXR Foundation", + repoName: "cxr-foundation", + repoUrl: "https://github.com/google-health/cxr-foundation", + snippets: snippets.cxr_foundation, + filter: false, + countDownloads: `path:"precomputed_embeddings/embeddings.npz" OR path:"pax-elixr-b-text/saved_model.pb"`, + }, + deepforest: { + prettyLabel: "DeepForest", + repoName: "deepforest", + docsUrl: "https://deepforest.readthedocs.io/en/latest/", + repoUrl: "https://github.com/weecology/DeepForest", + }, + "depth-anything-v2": { + prettyLabel: "DepthAnythingV2", + repoName: "Depth Anything V2", + repoUrl: "https://github.com/DepthAnything/Depth-Anything-V2", + snippets: snippets.depth_anything_v2, + filter: false, + countDownloads: `path_extension:"pth"`, + }, + "depth-pro": { + prettyLabel: "Depth Pro", + repoName: "Depth Pro", + repoUrl: "https://github.com/apple/ml-depth-pro", + countDownloads: `path_extension:"pt"`, + snippets: snippets.depth_pro, + filter: false, + }, + "derm-foundation": { + prettyLabel: "Derm Foundation", + repoName: "derm-foundation", + repoUrl: "https://github.com/google-health/derm-foundation", + snippets: snippets.derm_foundation, + filter: false, + countDownloads: `path:"scin_dataset_precomputed_embeddings.npz" OR path:"saved_model.pb"`, + }, + "describe-anything": { + prettyLabel: "Describe Anything", + repoName: "Describe Anything", + repoUrl: "https://github.com/NVlabs/describe-anything", + snippets: snippets.describe_anything, + filter: false, + }, + "dia-tts": { + prettyLabel: "Dia", + repoName: "Dia", + repoUrl: "https://github.com/nari-labs/dia", + snippets: snippets.dia, + filter: false, + }, + dia2: { + prettyLabel: "Dia2", + repoName: "Dia2", + repoUrl: "https://github.com/nari-labs/dia2", + snippets: snippets.dia2, + filter: false, + }, + "diff-interpretation-tuning": { + prettyLabel: "Diff Interpretation Tuning", + repoName: "Diff Interpretation Tuning", + repoUrl: "https://github.com/Aviously/diff-interpretation-tuning", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + diffree: { + prettyLabel: "Diffree", + repoName: "Diffree", + repoUrl: "https://github.com/OpenGVLab/Diffree", + filter: false, + countDownloads: `path:"diffree-step=000010999.ckpt"`, + }, + diffusers: { + prettyLabel: "Diffusers", + repoName: "🤗/diffusers", + repoUrl: "https://github.com/huggingface/diffusers", + docsUrl: "https://huggingface.co/docs/hub/diffusers", + snippets: snippets.diffusers, + filter: true, + /// diffusers has its own more complex "countDownloads" query + }, + diffusionkit: { + prettyLabel: "DiffusionKit", + repoName: "DiffusionKit", + repoUrl: "https://github.com/argmaxinc/DiffusionKit", + snippets: snippets.diffusionkit, + }, + "docking-at-home": { + prettyLabel: "Docking@Home", + repoName: "Docking@Home", + repoUrl: "https://huggingface.co/OpenPeerAI/DockingAtHOME", + filter: false, + countDownloads: `path:"setup.py"`, + }, + doctr: { + prettyLabel: "docTR", + repoName: "doctr", + repoUrl: "https://github.com/mindee/doctr", + }, + edsnlp: { + prettyLabel: "EDS-NLP", + repoName: "edsnlp", + repoUrl: "https://github.com/aphp/edsnlp", + docsUrl: "https://aphp.github.io/edsnlp/latest/", + filter: false, + snippets: snippets.edsnlp, + countDownloads: `path_filename:"config" AND path_extension:"cfg"`, + }, + elm: { + prettyLabel: "ELM", + repoName: "elm", + repoUrl: "https://github.com/slicex-ai/elm", + filter: false, + countDownloads: `path_filename:"slicex_elm_config" AND path_extension:"json"`, + }, + encoderfile: { + prettyLabel: "encoderfile", + repoName: "encoderfile", + repoUrl: "https://github.com/mozilla-ai/encoderfile", + filter: false, + countDownloads: `path_extension:"encoderfile"`, + }, + espnet: { + prettyLabel: "ESPnet", + repoName: "ESPnet", + repoUrl: "https://github.com/espnet/espnet", + docsUrl: "https://huggingface.co/docs/hub/espnet", + snippets: snippets.espnet, + filter: true, + }, + eupe: { + prettyLabel: "EUPE", + repoName: "EUPE", + repoUrl: "https://github.com/facebookresearch/EUPE", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + fairseq: { + prettyLabel: "Fairseq", + repoName: "fairseq", + repoUrl: "https://github.com/pytorch/fairseq", + snippets: snippets.fairseq, + filter: true, + }, + fastai: { + prettyLabel: "fastai", + repoName: "fastai", + repoUrl: "https://github.com/fastai/fastai", + docsUrl: "https://huggingface.co/docs/hub/fastai", + snippets: snippets.fastai, + filter: true, + }, + fastprint: { + prettyLabel: "Fast Print", + repoName: "Fast Print", + repoUrl: "https://huggingface.co/OpenPeerAI/FastPrint", + countDownloads: `path_extension:"cs"`, + }, + fasttext: { + prettyLabel: "fastText", + repoName: "fastText", + repoUrl: "https://fasttext.cc/", + snippets: snippets.fasttext, + filter: true, + countDownloads: `path_extension:"bin"`, + }, + fixer: { + prettyLabel: "Fixer", + repoName: "Fixer", + repoUrl: "https://github.com/nv-tlabs/Fixer", + filter: false, + countDownloads: `path:"pretrained/pretrained_fixer.pkl"`, + }, + flair: { + prettyLabel: "Flair", + repoName: "Flair", + repoUrl: "https://github.com/flairNLP/flair", + docsUrl: "https://huggingface.co/docs/hub/flair", + snippets: snippets.flair, + filter: true, + countDownloads: `path:"pytorch_model.bin"`, + }, + fme: { + prettyLabel: "Full Model Emulation", + repoName: "Full Model Emulation", + repoUrl: "https://github.com/ai2cm/ace", + docsUrl: "https://ai2-climate-emulator.readthedocs.io/en/latest/", + filter: false, + countDownloads: `path_extension:"tar"`, + }, + "gemma.cpp": { + prettyLabel: "gemma.cpp", + repoName: "gemma.cpp", + repoUrl: "https://github.com/google/gemma.cpp", + filter: false, + countDownloads: `path_extension:"sbs"`, + }, + "geometry-crafter": { + prettyLabel: "GeometryCrafter", + repoName: "GeometryCrafter", + repoUrl: "https://github.com/TencentARC/GeometryCrafter", + countDownloads: `path:"point_map_vae/diffusion_pytorch_model.safetensors"`, + }, + gliner: { + prettyLabel: "GLiNER", + repoName: "GLiNER", + repoUrl: "https://github.com/urchade/GLiNER", + snippets: snippets.gliner, + filter: false, + countDownloads: `path:"gliner_config.json"`, + }, + gliner2: { + prettyLabel: "GLiNER2", + repoName: "GLiNER2", + repoUrl: "https://github.com/fastino-ai/GLiNER2", + snippets: snippets.gliner2, + filter: false, + }, + "glm-tts": { + prettyLabel: "GLM-TTS", + repoName: "GLM-TTS", + repoUrl: "https://github.com/zai-org/GLM-TTS", + filter: false, + countDownloads: `path:"flow/flow.pt"`, + }, + "glyph-byt5": { + prettyLabel: "Glyph-ByT5", + repoName: "Glyph-ByT5", + repoUrl: "https://github.com/AIGText/Glyph-ByT5", + filter: false, + countDownloads: `path:"checkpoints/byt5_model.pt"`, + }, + "granite-library": { + prettyLabel: "Granite Library", + repoName: "mellea", + repoUrl: "https://github.com/generative-computing/mellea", + filter: false, + countDownloads: `path_filename:"adapter_config" AND path_extension:"json"`, + }, + grok: { + prettyLabel: "Grok", + repoName: "Grok", + repoUrl: "https://github.com/xai-org/grok-1", + filter: false, + countDownloads: `path:"ckpt/tensor00000_000" OR path:"ckpt-0/tensor00000_000"`, + }, + "habibi-tts": { + prettyLabel: "Habibi-TTS", + repoName: "Habibi-TTS", + repoUrl: "https://github.com/SWivid/Habibi-TTS", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + hallo: { + prettyLabel: "Hallo", + repoName: "Hallo", + repoUrl: "https://github.com/fudan-generative-vision/hallo", + countDownloads: `path:"hallo/net.pth"`, + }, + hermes: { + prettyLabel: "HERMES", + repoName: "HERMES", + repoUrl: "https://github.com/LMD0311/HERMES", + filter: false, + countDownloads: `path:"ckpt/hermes_final.pth"`, + }, + holomotion: { + prettyLabel: "HoloMotion", + repoName: "HoloMotion", + repoUrl: "https://github.com/HorizonRobotics/HoloMotion", + filter: false, + countDownloads: `path_extension:"onnx"`, + }, + hezar: { + prettyLabel: "Hezar", + repoName: "Hezar", + repoUrl: "https://github.com/hezarai/hezar", + docsUrl: "https://hezarai.github.io/hezar", + countDownloads: `path:"model_config.yaml" OR path:"embedding/embedding_config.yaml"`, + }, + htrflow: { + prettyLabel: "HTRflow", + repoName: "HTRflow", + repoUrl: "https://github.com/AI-Riksarkivet/htrflow", + docsUrl: "https://ai-riksarkivet.github.io/htrflow", + snippets: snippets.htrflow, + }, + "hunyuan-dit": { + prettyLabel: "HunyuanDiT", + repoName: "HunyuanDiT", + repoUrl: "https://github.com/Tencent/HunyuanDiT", + countDownloads: `path:"pytorch_model_ema.pt" OR path:"pytorch_model_distill.pt"`, + }, + "hunyuan3d-2": { + prettyLabel: "Hunyuan3D-2", + repoName: "Hunyuan3D-2", + repoUrl: "https://github.com/Tencent/Hunyuan3D-2", + countDownloads: `path_filename:"model_index" OR path_filename:"config"`, + }, + "hunyuanworld-voyager": { + prettyLabel: "HunyuanWorld-voyager", + repoName: "HunyuanWorld-voyager", + repoUrl: "https://github.com/Tencent-Hunyuan/HunyuanWorld-Voyager", + }, + "hy-worldplay": { + prettyLabel: "HY-WorldPlay", + repoName: "HY-WorldPlay", + repoUrl: "https://github.com/Tencent-Hunyuan/HY-WorldPlay", + filter: false, + countDownloads: `path_extension:"json"`, + }, + "hy-world-2": { + prettyLabel: "HY-World-2.0", + repoName: "HY-World-2.0", + repoUrl: "https://github.com/Tencent-Hunyuan/HY-World-2.0", + filter: false, + countDownloads: `path_extension:"json"`, + }, + "image-matching-models": { + prettyLabel: "Image Matching Models", + repoName: "Image Matching Models", + repoUrl: "https://github.com/alexstoken/image-matching-models", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + imstoucan: { + prettyLabel: "IMS Toucan", + repoName: "IMS-Toucan", + repoUrl: "https://github.com/DigitalPhonetics/IMS-Toucan", + countDownloads: `path:"embedding_gan.pt" OR path:"Vocoder.pt" OR path:"ToucanTTS.pt"`, + }, + "index-tts": { + prettyLabel: "IndexTTS", + repoName: "IndexTTS", + repoUrl: "https://github.com/index-tts/index-tts", + snippets: snippets.indextts, + filter: false, + }, + infinitetalk: { + prettyLabel: "InfiniteTalk", + repoName: "InfiniteTalk", + repoUrl: "https://github.com/MeiGen-AI/InfiniteTalk", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + "infinite-you": { + prettyLabel: "InfiniteYou", + repoName: "InfiniteYou", + repoUrl: "https://github.com/bytedance/InfiniteYou", + filter: false, + countDownloads: `path:"infu_flux_v1.0/sim_stage1/image_proj_model.bin" OR path:"infu_flux_v1.0/aes_stage2/image_proj_model.bin"`, + }, + intellifold: { + prettyLabel: "IntelliFold", + repoName: "IntelliFold", + repoUrl: "https://github.com/IntelliGen-AI/IntelliFold", + filter: false, + countDownloads: `path_extension:"pt" OR path_extension:"zst"`, + }, + "ising-decoding": { + prettyLabel: "Ising Decoding", + repoName: "Ising-Decoding", + repoUrl: "https://github.com/NVIDIA/Ising-Decoding", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + keras: { + prettyLabel: "Keras", + repoName: "Keras", + repoUrl: "https://github.com/keras-team/keras", + docsUrl: "https://huggingface.co/docs/hub/keras", + snippets: snippets.keras, + filter: true, + countDownloads: `path:"config.json" OR path_extension:"keras"`, + }, + "tf-keras": { + // Legacy "Keras 2" library (tensorflow-only) + prettyLabel: "TF-Keras", + repoName: "TF-Keras", + repoUrl: "https://github.com/keras-team/tf-keras", + docsUrl: "https://huggingface.co/docs/hub/tf-keras", + snippets: snippets.tf_keras, + countDownloads: `path:"saved_model.pb"`, + }, + "keras-hub": { + prettyLabel: "KerasHub", + repoName: "KerasHub", + repoUrl: "https://github.com/keras-team/keras-hub", + docsUrl: "https://keras.io/keras_hub/", + snippets: snippets.keras_hub, + filter: true, + }, + kernels: { + prettyLabel: "Kernels", + repoName: "Kernels", + repoUrl: "https://github.com/huggingface/kernels", + docsUrl: "https://huggingface.co/docs/kernels", + snippets: snippets.kernels, + countDownloads: `path_filename:"_ops" AND path_extension:"py"`, + }, + "kimi-audio": { + prettyLabel: "KimiAudio", + repoName: "KimiAudio", + repoUrl: "https://github.com/MoonshotAI/Kimi-Audio", + snippets: snippets.kimi_audio, + filter: false, + }, + kittentts: { + prettyLabel: "KittenTTS", + repoName: "KittenTTS", + repoUrl: "https://github.com/KittenML/KittenTTS", + snippets: snippets.kittentts, + }, + kronos: { + prettyLabel: "KRONOS", + repoName: "KRONOS", + repoUrl: "https://github.com/mahmoodlab/KRONOS", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + k2: { + prettyLabel: "K2", + repoName: "k2", + repoUrl: "https://github.com/k2-fsa/k2", + }, + "lyra-2.0": { + prettyLabel: "Lyra-2.0", + repoName: "Lyra-2.0", + repoUrl: "https://github.com/nv-tlabs/lyra", + filter: false, + countDownloads: `path:"checkpoints/image_encoder/model.pth"`, + }, + lagernvs: { + prettyLabel: "LagerNVS", + repoName: "LagerNVS", + repoUrl: "https://github.com/facebookresearch/lagernvs", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + "lightning-ir": { + prettyLabel: "Lightning IR", + repoName: "Lightning IR", + repoUrl: "https://github.com/webis-de/lightning-ir", + snippets: snippets.lightning_ir, + }, + litert: { + prettyLabel: "LiteRT", + repoName: "LiteRT", + repoUrl: "https://github.com/google-ai-edge/LiteRT", + filter: false, + countDownloads: `path_extension:"tflite"`, + }, + "litert-lm": { + prettyLabel: "LiteRT-LM", + repoName: "LiteRT-LM", + repoUrl: "https://github.com/google-ai-edge/LiteRT-LM", + snippets: snippets.litert_lm, + filter: false, + countDownloads: `path_extension:"litertlm" OR path_extension:"task"`, + }, + lerobot: { + prettyLabel: "LeRobot", + repoName: "LeRobot", + repoUrl: "https://github.com/huggingface/lerobot", + docsUrl: "https://huggingface.co/docs/lerobot", + filter: false, + snippets: snippets.lerobot, + }, + lightglue: { + prettyLabel: "LightGlue", + repoName: "LightGlue", + repoUrl: "https://github.com/cvg/LightGlue", + filter: false, + countDownloads: `path_extension:"pth" OR path:"config.json"`, + }, + liveportrait: { + prettyLabel: "LivePortrait", + repoName: "LivePortrait", + repoUrl: "https://github.com/KwaiVGI/LivePortrait", + filter: false, + countDownloads: `path:"liveportrait/landmark.onnx"`, + }, + "longcat-video-avatar-1.5": { + prettyLabel: "LongCat-Video-Avatar 1.5", + repoName: "LongCat-Video-Avatar 1.5", + repoUrl: "https://github.com/meituan-longcat/LongCat-Video", + filter: false, + }, + "llama-cpp-python": { + prettyLabel: "llama-cpp-python", + repoName: "llama-cpp-python", + repoUrl: "https://github.com/abetlen/llama-cpp-python", + snippets: snippets.llama_cpp_python, + }, + "mini-omni2": { + prettyLabel: "Mini-Omni2", + repoName: "Mini-Omni2", + repoUrl: "https://github.com/gpt-omni/mini-omni2", + countDownloads: `path:"model_config.yaml"`, + }, + mindspore: { + prettyLabel: "MindSpore", + repoName: "mindspore", + repoUrl: "https://github.com/mindspore-ai/mindspore", + }, + "magi-1": { + prettyLabel: "MAGI-1", + repoName: "MAGI-1", + repoUrl: "https://github.com/SandAI-org/MAGI-1", + countDownloads: `path:"ckpt/vae/config.json"`, + }, + "magenta-realtime": { + prettyLabel: "Magenta RT", + repoName: "Magenta RT", + repoUrl: "https://github.com/magenta/magenta-realtime", + countDownloads: `path:"checkpoints/llm_base_x4286_c1860k.tar" OR path:"checkpoints/llm_large_x3047_c1860k.tar" OR path:"checkpoints/llm_large_x3047_c1860k/checkpoint"`, + }, + "magenta-realtime-2": { + prettyLabel: "Magenta RT 2", + repoName: "Magenta RT 2", + repoUrl: "https://github.com/magenta/magenta-realtime", + countDownloads: `path:"models/mrt2_base/mrt2_base.mlxfn" OR path:"models/mrt2_small/mrt2_small.mlxfn" OR path:"checkpoints/mrt2_base.safetensors" OR path:"checkpoints/mrt2_small.safetensors"`, + }, + "mamba-ssm": { + prettyLabel: "MambaSSM", + repoName: "MambaSSM", + repoUrl: "https://github.com/state-spaces/mamba", + filter: false, + snippets: snippets.mamba_ssm, + }, + "manas-1": { + prettyLabel: "MANAS-1", + repoName: "MANAS-1", + repoUrl: "https://github.com/NeurodxAI/manas-1", + countDownloads: `path_extension:"pt"`, + }, + "mars5-tts": { + prettyLabel: "MARS5-TTS", + repoName: "MARS5-TTS", + repoUrl: "https://github.com/Camb-ai/MARS5-TTS", + filter: false, + countDownloads: `path:"mars5_ar.safetensors"`, + snippets: snippets.mars5_tts, + }, + matanyone: { + prettyLabel: "MatAnyone", + repoName: "MatAnyone", + repoUrl: "https://github.com/pq-yang/MatAnyone", + snippets: snippets.matanyone, + filter: false, + }, + "mesh-anything": { + prettyLabel: "MeshAnything", + repoName: "MeshAnything", + repoUrl: "https://github.com/buaacyw/MeshAnything", + filter: false, + countDownloads: `path:"MeshAnything_350m.pth"`, + snippets: snippets.mesh_anything, + }, + merlin: { + prettyLabel: "Merlin", + repoName: "Merlin", + repoUrl: "https://github.com/StanfordMIMI/Merlin", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + medvae: { + prettyLabel: "MedVAE", + repoName: "MedVAE", + repoUrl: "https://github.com/StanfordMIMI/MedVAE", + filter: false, + countDownloads: `path_extension:"ckpt"`, + }, + mitie: { + prettyLabel: "MITIE", + repoName: "MITIE", + repoUrl: "https://github.com/mit-nlp/MITIE", + countDownloads: `path_filename:"total_word_feature_extractor"`, + }, + "ml-agents": { + prettyLabel: "ml-agents", + repoName: "ml-agents", + repoUrl: "https://github.com/Unity-Technologies/ml-agents", + docsUrl: "https://huggingface.co/docs/hub/ml-agents", + snippets: snippets.mlAgents, + filter: true, + countDownloads: `path_extension:"onnx"`, + }, + "ml-sharp": { + prettyLabel: "Sharp", + repoName: "Sharp", + repoUrl: "https://github.com/apple/ml-sharp", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + mlx: { + prettyLabel: "MLX", + repoName: "MLX", + repoUrl: "https://github.com/ml-explore/mlx-examples/tree/main", + snippets: snippets.mlx, + filter: true, + }, + "mlx-image": { + prettyLabel: "mlx-image", + repoName: "mlx-image", + repoUrl: "https://github.com/riccardomusmeci/mlx-image", + docsUrl: "https://huggingface.co/docs/hub/mlx-image", + snippets: snippets.mlxim, + filter: false, + countDownloads: `path:"model.safetensors"`, + }, + "mlc-llm": { + prettyLabel: "MLC-LLM", + repoName: "MLC-LLM", + repoUrl: "https://github.com/mlc-ai/mlc-llm", + docsUrl: "https://llm.mlc.ai/docs/", + filter: false, + countDownloads: `path:"mlc-chat-config.json"`, + }, + model2vec: { + prettyLabel: "Model2Vec", + repoName: "model2vec", + repoUrl: "https://github.com/MinishLab/model2vec", + snippets: snippets.model2vec, + filter: false, + }, + moshi: { + prettyLabel: "Moshi", + repoName: "Moshi", + repoUrl: "https://github.com/kyutai-labs/moshi", + snippets: snippets.moshi, + filter: false, + countDownloads: `path:"tokenizer-e351c8d8-checkpoint125.safetensors"`, + }, + mtvcraft: { + prettyLabel: "MTVCraft", + repoName: "MTVCraft", + repoUrl: "https://github.com/baaivision/MTVCraft", + filter: false, + countDownloads: `path:"vae/3d-vae.pt"`, + }, + multimolecule: { + prettyLabel: "MultiMolecule", + repoName: "MultiMolecule", + repoUrl: "https://github.com/MultiMolecule/multimolecule", + docsUrl: "https://multimolecule.danling.org", + snippets: snippets.multimolecule, + filter: false, + }, + nemo: { + prettyLabel: "NeMo", + repoName: "NeMo", + repoUrl: "https://github.com/NVIDIA/NeMo", + snippets: snippets.nemo, + filter: true, + countDownloads: `path_extension:"nemo" OR path:"model_config.yaml" OR path_extension:"json"`, + }, + "nv-medtech": { + prettyLabel: "NV-MedTech", + repoName: "NV-MedTech", + filter: false, + repoUrl: "https://github.com/nvidia-medtech", + countDownloads: `path_extension:"pt" OR path_extension:"safetensors" OR path:"config.json"`, + }, + "open-oasis": { + prettyLabel: "open-oasis", + repoName: "open-oasis", + repoUrl: "https://github.com/etched-ai/open-oasis", + countDownloads: `path:"oasis500m.safetensors"`, + }, + open_clip: { + prettyLabel: "OpenCLIP", + repoName: "OpenCLIP", + repoUrl: "https://github.com/mlfoundations/open_clip", + snippets: snippets.open_clip, + filter: true, + countDownloads: `path:"open_clip_model.safetensors" + OR path:"model.safetensors" + OR path:"open_clip_pytorch_model.bin" + OR path:"pytorch_model.bin"`, + }, + openpeerllm: { + prettyLabel: "OpenPeerLLM", + repoName: "OpenPeerLLM", + repoUrl: "https://huggingface.co/openpeerai/openpeerllm", + docsUrl: "https://huggingface.co/OpenPeerAI/OpenPeerLLM/blob/main/README.md", + countDownloads: `path:".meta-huggingface.json"`, + filter: false, + }, + "open-sora": { + prettyLabel: "Open-Sora", + repoName: "Open-Sora", + repoUrl: "https://github.com/hpcaitech/Open-Sora", + filter: false, + countDownloads: `path:"Open_Sora_v2.safetensors"`, + }, + outetts: { + prettyLabel: "OuteTTS", + repoName: "OuteTTS", + repoUrl: "https://github.com/edwko/OuteTTS", + snippets: snippets.outetts, + filter: false, + }, + paddlenlp: { + prettyLabel: "paddlenlp", + repoName: "PaddleNLP", + repoUrl: "https://github.com/PaddlePaddle/PaddleNLP", + docsUrl: "https://huggingface.co/docs/hub/paddlenlp", + snippets: snippets.paddlenlp, + filter: true, + countDownloads: `path:"model_config.json"`, + }, + PaddleOCR: { + prettyLabel: "PaddleOCR", + repoName: "PaddleOCR", + repoUrl: "https://github.com/PaddlePaddle/PaddleOCR", + docsUrl: "https://www.paddleocr.ai/", + snippets: snippets.paddleocr, + filter: true, + countDownloads: `path_extension:"safetensors" OR path:"inference.pdiparams" OR path:"inference.onnx"`, + }, + peft: { + prettyLabel: "PEFT", + repoName: "PEFT", + repoUrl: "https://github.com/huggingface/peft", + snippets: snippets.peft, + filter: true, + countDownloads: `path:"adapter_config.json"`, + }, + "perception-encoder": { + prettyLabel: "PerceptionEncoder", + repoName: "PerceptionModels", + repoUrl: "https://github.com/facebookresearch/perception_models", + filter: false, + snippets: snippets.perception_encoder, + countDownloads: `path_extension:"pt"`, + }, + "phantom-wan": { + prettyLabel: "Phantom", + repoName: "Phantom", + repoUrl: "https://github.com/Phantom-video/Phantom", + snippets: snippets.phantom_wan, + filter: false, + countDownloads: `path_extension:"pth"`, + }, + "pocket-tts": { + prettyLabel: "Pocket-TTS", + repoName: "PocketTTS", + repoUrl: "https://github.com/kyutai-labs/pocket-tts", + snippets: snippets.pocket_tts, + filter: false, + countDownloads: `path:"tts_b6369a24.safetensors"`, + }, + "pruna-ai": { + prettyLabel: "Pruna AI", + repoName: "Pruna AI", + repoUrl: "https://github.com/PrunaAI/pruna", + snippets: snippets.pruna, + docsUrl: "https://docs.pruna.ai", + }, + pxia: { + prettyLabel: "pxia", + repoName: "pxia", + repoUrl: "https://github.com/not-lain/pxia", + snippets: snippets.pxia, + filter: false, + }, + "pyannote-audio": { + prettyLabel: "pyannote.audio", + repoName: "pyannote-audio", + repoUrl: "https://github.com/pyannote/pyannote-audio", + snippets: snippets.pyannote_audio, + filter: true, + }, + "py-feat": { + prettyLabel: "Py-Feat", + repoName: "Py-Feat", + repoUrl: "https://github.com/cosanlab/py-feat", + docsUrl: "https://py-feat.org/", + filter: false, + }, + pythae: { + prettyLabel: "pythae", + repoName: "pythae", + repoUrl: "https://github.com/clementchadebec/benchmark_VAE", + snippets: snippets.pythae, + filter: false, + }, + quantumpeer: { + prettyLabel: "QuantumPeer", + repoName: "QuantumPeer", + repoUrl: "https://github.com/OpenPeer-AI/QuantumPeer", + filter: false, + countDownloads: `path_extension:"setup.py"`, + }, + qwen3_tts: { + prettyLabel: "Qwen3-TTS", + repoName: "Qwen3-TTS", + repoUrl: "https://github.com/QwenLM/Qwen3-TTS", + snippets: snippets.qwen3_tts, + filter: false, + }, + recurrentgemma: { + prettyLabel: "RecurrentGemma", + repoName: "recurrentgemma", + repoUrl: "https://github.com/google-deepmind/recurrentgemma", + filter: false, + countDownloads: `path:"tokenizer.model"`, + }, + relik: { + prettyLabel: "Relik", + repoName: "Relik", + repoUrl: "https://github.com/SapienzaNLP/relik", + snippets: snippets.relik, + filter: false, + }, + refiners: { + prettyLabel: "Refiners", + repoName: "Refiners", + repoUrl: "https://github.com/finegrain-ai/refiners", + docsUrl: "https://refine.rs/", + filter: false, + countDownloads: `path:"model.safetensors"`, + }, + renderformer: { + prettyLabel: "RenderFormer", + repoName: "RenderFormer", + repoUrl: "https://github.com/microsoft/renderformer", + snippets: snippets.renderformer, + filter: false, + }, + reverb: { + prettyLabel: "Reverb", + repoName: "Reverb", + repoUrl: "https://github.com/revdotcom/reverb", + filter: false, + }, + rkllm: { + prettyLabel: "RKLLM", + repoName: "RKLLM", + repoUrl: "https://github.com/airockchip/rknn-llm", + countDownloads: `path_extension:"rkllm"`, + }, + "robo-orchard-lab": { + prettyLabel: "RoboOrchardLab", + repoName: "RoboOrchardLab", + repoUrl: "https://github.com/HorizonRobotics/RoboOrchardLab", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + rwkv: { + prettyLabel: "RWKV", + repoName: "RWKV-LM", + repoUrl: "https://github.com/BlinkDL/RWKV-LM", + docsUrl: "https://rwkv.com/", + filter: false, + countDownloads: `path_extension:"pth"`, + }, + saelens: { + prettyLabel: "SAELens", + repoName: "SAELens", + repoUrl: "https://github.com/jbloomAus/SAELens", + snippets: snippets.saelens, + filter: false, + }, + "scail-2": { + prettyLabel: "SCAIL-2", + repoName: "SCAIL-2", + repoUrl: "https://github.com/zai-org/SCAIL-2", + filter: false, + countDownloads: `path:"model/1/fsdp2_rank_0000_checkpoint.pt"`, + }, + sam2: { + prettyLabel: "sam2", + repoName: "sam2", + repoUrl: "https://github.com/facebookresearch/segment-anything-2", + filter: false, + snippets: snippets.sam2, + countDownloads: `path_extension:"pt"`, + }, + "sam-3d-body": { + prettyLabel: "SAM 3D Body", + repoName: "SAM 3D Body", + repoUrl: "https://github.com/facebookresearch/sam-3d-body", + filter: false, + snippets: snippets.sam_3d_body, + countDownloads: `path:"model_config.yaml"`, + }, + "sam-3d-objects": { + prettyLabel: "SAM 3D Objects", + repoName: "SAM 3D Objects", + repoUrl: "https://github.com/facebookresearch/sam-3d-objects", + filter: false, + snippets: snippets.sam_3d_objects, + countDownloads: `path:"checkpoints/pipeline.yaml"`, + }, + same: { + prettyLabel: "SAME", + repoName: "SAME", + repoUrl: "https://github.com/GengzeZhou/SAME", + filter: false, + countDownloads: `path:"ckpt/SAME.pt" OR path:"pretrain/Attnq_pretrained_ckpt.pt"`, + }, + "sample-factory": { + prettyLabel: "sample-factory", + repoName: "sample-factory", + repoUrl: "https://github.com/alex-petrenko/sample-factory", + docsUrl: "https://huggingface.co/docs/hub/sample-factory", + snippets: snippets.sampleFactory, + filter: true, + countDownloads: `path:"cfg.json"`, + }, + "sap-rpt-1-oss": { + prettyLabel: "sap-rpt-1-oss", + repoName: "sap-rpt-1-oss", + repoUrl: "https://github.com/SAP-samples/sap-rpt-1-oss", + countDownloads: `path_extension:"pt"`, + snippets: snippets.sap_rpt_one_oss, + }, + sapiens: { + prettyLabel: "sapiens", + repoName: "sapiens", + repoUrl: "https://github.com/facebookresearch/sapiens", + filter: false, + countDownloads: `path_extension:"pt2" OR path_extension:"pth" OR path_extension:"onnx"`, + }, + sapiens2: { + prettyLabel: "sapiens2", + repoName: "sapiens2", + repoUrl: "https://github.com/facebookresearch/sapiens2", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + seedvr: { + prettyLabel: "SeedVR", + repoName: "SeedVR", + repoUrl: "https://github.com/ByteDance-Seed/SeedVR", + filter: false, + countDownloads: `path_extension:"pth"`, + }, + "self-forcing": { + prettyLabel: "SelfForcing", + repoName: "SelfForcing", + repoUrl: "https://github.com/guandeh17/Self-Forcing", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + "sentence-transformers": { + prettyLabel: "sentence-transformers", + repoName: "sentence-transformers", + repoUrl: "https://github.com/UKPLab/sentence-transformers", + docsUrl: "https://huggingface.co/docs/hub/sentence-transformers", + snippets: snippets.sentenceTransformers, + filter: true, + }, + setfit: { + prettyLabel: "setfit", + repoName: "setfit", + repoUrl: "https://github.com/huggingface/setfit", + docsUrl: "https://huggingface.co/docs/hub/setfit", + snippets: snippets.setfit, + filter: true, + }, + sklearn: { + prettyLabel: "Scikit-learn", + repoName: "Scikit-learn", + repoUrl: "https://github.com/scikit-learn/scikit-learn", + snippets: snippets.sklearn, + filter: true, + countDownloads: `path:"sklearn_model.joblib"`, + }, + spacy: { + prettyLabel: "spaCy", + repoName: "spaCy", + repoUrl: "https://github.com/explosion/spaCy", + docsUrl: "https://huggingface.co/docs/hub/spacy", + snippets: snippets.spacy, + filter: true, + countDownloads: `path_extension:"whl"`, + }, + "span-marker": { + prettyLabel: "SpanMarker", + repoName: "SpanMarkerNER", + repoUrl: "https://github.com/tomaarsen/SpanMarkerNER", + docsUrl: "https://huggingface.co/docs/hub/span_marker", + snippets: snippets.span_marker, + filter: true, + }, + speechbrain: { + prettyLabel: "speechbrain", + repoName: "speechbrain", + repoUrl: "https://github.com/speechbrain/speechbrain", + docsUrl: "https://huggingface.co/docs/hub/speechbrain", + snippets: snippets.speechbrain, + filter: true, + countDownloads: `path:"hyperparams.yaml"`, + }, + "ssr-speech": { + prettyLabel: "SSR-Speech", + repoName: "SSR-Speech", + repoUrl: "https://github.com/WangHelin1997/SSR-Speech", + filter: false, + countDownloads: `path_extension:".pth"`, + }, + "stable-audio-3": { + prettyLabel: "Stable Audio 3", + repoName: "stable-audio-3", + repoUrl: "https://github.com/Stability-AI/stable-audio-3", + filter: false, + countDownloads: `path:"model_config.json"`, + }, + "stable-audio-tools": { + prettyLabel: "Stable Audio Tools", + repoName: "stable-audio-tools", + repoUrl: "https://github.com/Stability-AI/stable-audio-tools.git", + filter: false, + countDownloads: `path:"model.safetensors"`, + snippets: snippets.stable_audio_tools, + }, + monkeyocr: { + prettyLabel: "MonkeyOCR", + repoName: "monkeyocr", + repoUrl: "https://github.com/Yuliang-Liu/MonkeyOCR", + filter: false, + countDownloads: `path:"Recognition/config.json"`, + }, + "diffusion-single-file": { + prettyLabel: "Diffusion Single File", + repoName: "diffusion-single-file", + repoUrl: "https://github.com/comfyanonymous/ComfyUI", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + "seed-story": { + prettyLabel: "SEED-Story", + repoName: "SEED-Story", + repoUrl: "https://github.com/TencentARC/SEED-Story", + filter: false, + countDownloads: `path:"cvlm_llama2_tokenizer/tokenizer.model"`, + snippets: snippets.seed_story, + }, + skala: { + prettyLabel: "Skala", + repoName: "Skala", + repoUrl: "https://github.com/microsoft/skala", + filter: false, + countDownloads: `path_extension:"fun"`, + }, + soloaudio: { + prettyLabel: "SoloAudio", + repoName: "SoloAudio", + repoUrl: "https://github.com/WangHelin1997/SoloAudio", + filter: false, + countDownloads: `path:"soloaudio_v2.pt"`, + }, + songbloom: { + prettyLabel: "SongBloom", + repoName: "SongBloom", + repoUrl: "https://github.com/Cypress-Yang/SongBloom", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + "stable-baselines3": { + prettyLabel: "stable-baselines3", + repoName: "stable-baselines3", + repoUrl: "https://github.com/huggingface/huggingface_sb3", + docsUrl: "https://huggingface.co/docs/hub/stable-baselines3", + snippets: snippets.stableBaselines3, + filter: true, + countDownloads: `path_extension:"zip"`, + }, + stanza: { + prettyLabel: "Stanza", + repoName: "stanza", + repoUrl: "https://github.com/stanfordnlp/stanza", + docsUrl: "https://huggingface.co/docs/hub/stanza", + snippets: snippets.stanza, + filter: true, + countDownloads: `path:"models/default.zip"`, + }, + supertonic: { + prettyLabel: "Supertonic", + repoName: "Supertonic", + repoUrl: "https://github.com/supertone-inc/supertonic", + snippets: snippets.supertonic, + filter: false, + }, + swarmformer: { + prettyLabel: "SwarmFormer", + repoName: "SwarmFormer", + repoUrl: "https://github.com/takara-ai/SwarmFormer", + snippets: snippets.swarmformer, + filter: false, + }, + "synthefy-migas": { + prettyLabel: "Migas", + repoName: "Migas", + repoUrl: "https://github.com/Synthefy/synthefy-migas", + filter: false, + countDownloads: `path:"model.pt"`, + }, + "f5-tts": { + prettyLabel: "F5-TTS", + repoName: "F5-TTS", + repoUrl: "https://github.com/SWivid/F5-TTS", + filter: false, + countDownloads: `path_extension:"safetensors" OR path_extension:"pt"`, + }, + genmo: { + prettyLabel: "Genmo", + repoName: "Genmo", + repoUrl: "https://github.com/genmoai/models", + filter: false, + countDownloads: `path:"vae_stats.json"`, + }, + "tencent-song-generation": { + prettyLabel: "SongGeneration", + repoName: "SongGeneration", + repoUrl: "https://github.com/tencent-ailab/songgeneration", + filter: false, + countDownloads: `path:"ckpt/songgeneration_base/model.pt"`, + }, + tensorflowtts: { + prettyLabel: "TensorFlowTTS", + repoName: "TensorFlowTTS", + repoUrl: "https://github.com/TensorSpeech/TensorFlowTTS", + snippets: snippets.tensorflowtts, + }, + tensorrt: { + prettyLabel: "TensorRT", + repoName: "TensorRT", + repoUrl: "https://github.com/NVIDIA/TensorRT", + countDownloads: `path_extension:"onnx"`, + }, + tabpfn: { + prettyLabel: "TabPFN", + repoName: "TabPFN", + repoUrl: "https://github.com/PriorLabs/TabPFN", + }, + terratorch: { + prettyLabel: "TerraTorch", + repoName: "TerraTorch", + repoUrl: "https://github.com/IBM/terratorch", + docsUrl: "https://ibm.github.io/terratorch/", + filter: false, + countDownloads: `path_extension:"pt" OR path_extension:"ckpt"`, + snippets: snippets.terratorch, + }, + "tic-clip": { + prettyLabel: "TiC-CLIP", + repoName: "TiC-CLIP", + repoUrl: "https://github.com/apple/ml-tic-clip", + filter: false, + countDownloads: `path_extension:"pt" AND path_prefix:"checkpoints/"`, + }, + timesfm: { + prettyLabel: "TimesFM", + repoName: "timesfm", + repoUrl: "https://github.com/google-research/timesfm", + filter: false, + countDownloads: `path:"checkpoints/checkpoint_1100000/state/checkpoint" OR path:"checkpoints/checkpoint_2150000/state/checkpoint" OR path_extension:"ckpt"`, + }, + timm: { + prettyLabel: "timm", + repoName: "pytorch-image-models", + repoUrl: "https://github.com/rwightman/pytorch-image-models", + docsUrl: "https://huggingface.co/docs/hub/timm", + snippets: snippets.timm, + filter: true, + countDownloads: `path:"pytorch_model.bin" OR path:"model.safetensors"`, + }, + tirex: { + prettyLabel: "TiRex", + repoName: "TiRex", + repoUrl: "https://github.com/NX-AI/tirex", + countDownloads: `path_extension:"ckpt"`, + }, + torchgeo: { + prettyLabel: "TorchGeo", + repoName: "TorchGeo", + repoUrl: "https://github.com/microsoft/torchgeo", + docsUrl: "https://torchgeo.readthedocs.io/", + filter: false, + countDownloads: `path_extension:"pt" OR path_extension:"pth"`, + }, + transformers: { + prettyLabel: "Transformers", + repoName: "🤗/transformers", + repoUrl: "https://github.com/huggingface/transformers", + docsUrl: "https://huggingface.co/docs/hub/transformers", + snippets: snippets.transformers, + filter: true, + }, + "transformers.js": { + prettyLabel: "Transformers.js", + repoName: "transformers.js", + repoUrl: "https://github.com/huggingface/transformers.js", + docsUrl: "https://huggingface.co/docs/hub/transformers-js", + snippets: snippets.transformersJS, + filter: true, + }, + trellis: { + prettyLabel: "Trellis", + repoName: "Trellis", + repoUrl: "https://github.com/microsoft/TRELLIS", + countDownloads: `path_extension:"safetensors"`, + }, + trellis2: { + prettyLabel: "TRELLIS.2", + repoName: "TRELLIS.2", + repoUrl: "https://github.com/microsoft/TRELLIS.2", + countDownloads: `path_extension:"safetensors"`, + }, + tunejury: { + prettyLabel: "TuneJury", + repoName: "TuneJury", + repoUrl: "https://github.com/yonghyunk1m/TuneJury", + countDownloads: `path_extension:"pt"`, + }, + ultralytics: { + prettyLabel: "ultralytics", + repoName: "ultralytics", + repoUrl: "https://github.com/ultralytics/ultralytics", + docsUrl: "https://github.com/ultralytics/ultralytics", + filter: false, + countDownloads: `path_extension:"pt"`, + snippets: snippets.ultralytics, + }, + univa: { + prettyLabel: "univa", + repoName: "univa", + repoUrl: "https://github.com/PKU-YuanGroup/UniWorld-V1", + snippets: snippets.univa, + filter: true, + countDownloads: `path:"config.json"`, + }, + "uni-3dar": { + prettyLabel: "Uni-3DAR", + repoName: "Uni-3DAR", + repoUrl: "https://github.com/dptech-corp/Uni-3DAR", + docsUrl: "https://github.com/dptech-corp/Uni-3DAR", + countDownloads: `path_extension:"pt"`, + }, + "unity-sentis": { + prettyLabel: "unity-sentis", + repoName: "unity-sentis", + repoUrl: "https://github.com/Unity-Technologies/sentis-samples", + snippets: snippets.sentis, + filter: true, + countDownloads: `path_extension:"sentis"`, + }, + sana: { + prettyLabel: "Sana", + repoName: "Sana", + repoUrl: "https://github.com/NVlabs/Sana", + countDownloads: `path_extension:"pth"`, + snippets: snippets.sana, + }, + videoprism: { + prettyLabel: "VideoPrism", + repoName: "VideoPrism", + repoUrl: "https://github.com/google-deepmind/videoprism", + countDownloads: `path_extension:"npz"`, + snippets: snippets.videoprism, + }, + "vfi-mamba": { + prettyLabel: "VFIMamba", + repoName: "VFIMamba", + repoUrl: "https://github.com/MCG-NJU/VFIMamba", + countDownloads: `path_extension:"pkl"`, + snippets: snippets.vfimamba, + }, + vismatch: { + prettyLabel: "VisMatch", + repoName: "VisMatch", + repoUrl: "https://github.com/gmberton/vismatch", + filter: false, + countDownloads: `path:"vismatch.yaml"`, + }, + lvface: { + prettyLabel: "LVFace", + repoName: "LVFace", + repoUrl: "https://github.com/bytedance/LVFace", + countDownloads: `path_extension:"pt" OR path_extension:"onnx"`, + snippets: snippets.lvface, + }, + voicecraft: { + prettyLabel: "VoiceCraft", + repoName: "VoiceCraft", + repoUrl: "https://github.com/jasonppy/VoiceCraft", + docsUrl: "https://github.com/jasonppy/VoiceCraft", + snippets: snippets.voicecraft, + }, + voxcpm: { + prettyLabel: "VoxCPM", + repoName: "VoxCPM", + repoUrl: "https://github.com/OpenBMB/VoxCPM", + snippets: snippets.voxcpm, + filter: false, + }, + vui: { + prettyLabel: "Vui", + repoName: "Vui", + repoUrl: "https://github.com/vui-ai/vui", + countDownloads: `path_extension:"pt"`, + snippets: snippets.vui, + }, + vibevoice: { + prettyLabel: "VibeVoice", + repoName: "VibeVoice", + repoUrl: "https://github.com/microsoft/VibeVoice", + snippets: snippets.vibevoice, + filter: false, + }, + videox_fun: { + prettyLabel: "VideoX Fun", + repoName: "VideoX Fun", + repoUrl: "https://github.com/aigc-apps/VideoX-Fun", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + "wan2.2": { + prettyLabel: "Wan2.2", + repoName: "Wan2.2", + repoUrl: "https://github.com/Wan-Video/Wan2.2", + countDownloads: `path_filename:"config" AND path_extension:"json"`, + }, + wham: { + prettyLabel: "WHAM", + repoName: "wham", + repoUrl: "https://huggingface.co/microsoft/wham", + docsUrl: "https://huggingface.co/microsoft/wham/blob/main/README.md", + countDownloads: `path_extension:"ckpt"`, + }, + whisperkit: { + prettyLabel: "WhisperKit", + repoName: "WhisperKit", + repoUrl: "https://github.com/argmaxinc/WhisperKit", + docsUrl: "https://github.com/argmaxinc/WhisperKit?tab=readme-ov-file#homebrew", + snippets: snippets.whisperkit, + countDownloads: `path_filename:"model" AND path_extension:"mil" AND _exists_:"path_prefix"`, + }, + yolov10: { + // YOLOv10 is a fork of ultraLytics. Code snippets and download count are the same but the repo is different. + prettyLabel: "YOLOv10", + repoName: "YOLOv10", + repoUrl: "https://github.com/THU-MIG/yolov10", + docsUrl: "https://github.com/THU-MIG/yolov10", + countDownloads: `path_extension:"pt" OR path_extension:"safetensors"`, + snippets: snippets.ultralytics, + }, + yolov26: { + prettyLabel: "YOLOv26", + repoName: "YOLOv26", + repoUrl: "https://github.com/ultralytics/ultralytics", + docsUrl: "https://docs.ultralytics.com/models/yolo26/", + countDownloads: `path_extension:"pt" OR path_extension:"safetensors"`, + }, + zonos: { + prettyLabel: "Zonos", + repoName: "Zonos", + repoUrl: "https://github.com/Zyphra/Zonos", + docsUrl: "https://github.com/Zyphra/Zonos", + snippets: snippets.zonos, + filter: false, + }, + "3dtopia-xl": { + prettyLabel: "3DTopia-XL", + repoName: "3DTopia-XL", + repoUrl: "https://github.com/3DTopia/3DTopia-XL", + filter: false, + countDownloads: `path:"model_vae_fp16.pt"`, + snippets: snippets.threedtopia_xl, + }, +}; +export const ALL_MODEL_LIBRARY_KEYS = Object.keys(MODEL_LIBRARIES_UI_ELEMENTS); +export const ALL_DISPLAY_MODEL_LIBRARY_KEYS = Object.entries(MODEL_LIBRARIES_UI_ELEMENTS) + // eslint-disable-next-line @typescript-eslint/no-unused-vars + .filter(([_, v]) => v.filter) + .map(([k]) => k); diff --git a/node_modules/@huggingface/tasks/dist/esm/package.json b/node_modules/@huggingface/tasks/dist/esm/package.json new file mode 100644 index 0000000000000000000000000000000000000000..3dbc1ca591c0557e35b6004aeba250e6a70b56e3 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/node_modules/@huggingface/tasks/dist/esm/pipelines.d.ts b/node_modules/@huggingface/tasks/dist/esm/pipelines.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..48b5bfceac989cb636f78400358077608a9f9af3 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/pipelines.d.ts @@ -0,0 +1,385 @@ +export declare const MODALITIES: readonly ["multimodal", "nlp", "cv", "audio", "tabular", "rl", "other"]; +export type Modality = (typeof MODALITIES)[number]; +export declare const MODALITY_LABELS: { + multimodal: string; + nlp: string; + audio: string; + cv: string; + rl: string; + tabular: string; + other: string; +}; +/** + * Public interface for a sub task. + * + * This can be used in a model card's `model-index` metadata. + * and is more granular classification that can grow significantly + * over time as new tasks are added. + */ +export interface SubTask { + /** + * type of the task (e.g. audio-source-separation) + */ + type: string; + /** + * displayed name of the task (e.g. Audio Source Separation) + */ + name: string; +} +/** + * Public interface for a PipelineData. + * + * This information corresponds to a pipeline type (aka task) + * in the Hub. + */ +export interface PipelineData { + /** + * displayed name of the task (e.g. Text Classification) + */ + name: string; + subtasks?: SubTask[]; + modality: Modality; + /** + * whether to hide in /models filters + */ + hideInModels?: boolean; + /** + * whether to hide in /datasets filters + */ + hideInDatasets?: boolean; +} +export declare const PIPELINE_DATA: { + "text-classification": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + }; + "token-classification": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + }; + "table-question-answering": { + name: string; + modality: "nlp"; + }; + "question-answering": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + }; + "zero-shot-classification": { + name: string; + modality: "nlp"; + }; + translation: { + name: string; + modality: "nlp"; + }; + summarization: { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + }; + "feature-extraction": { + name: string; + modality: "nlp"; + }; + "text-generation": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + }; + "fill-mask": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + }; + "sentence-similarity": { + name: string; + modality: "nlp"; + }; + "text-to-speech": { + name: string; + modality: "audio"; + }; + "text-to-audio": { + name: string; + modality: "audio"; + }; + "automatic-speech-recognition": { + name: string; + modality: "audio"; + }; + "audio-to-audio": { + name: string; + modality: "audio"; + }; + "audio-classification": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "audio"; + }; + "audio-text-to-text": { + name: string; + modality: "multimodal"; + hideInDatasets: true; + }; + "voice-activity-detection": { + name: string; + modality: "audio"; + }; + "depth-estimation": { + name: string; + modality: "cv"; + }; + "image-classification": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "cv"; + }; + "object-detection": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "cv"; + }; + "image-segmentation": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "cv"; + }; + "text-to-image": { + name: string; + modality: "cv"; + }; + "image-to-text": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "cv"; + }; + "image-to-image": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "cv"; + }; + "image-to-video": { + name: string; + modality: "cv"; + }; + "unconditional-image-generation": { + name: string; + modality: "cv"; + }; + "video-classification": { + name: string; + modality: "cv"; + }; + "reinforcement-learning": { + name: string; + modality: "rl"; + }; + robotics: { + name: string; + modality: "rl"; + subtasks: { + type: string; + name: string; + }[]; + }; + "tabular-classification": { + name: string; + modality: "tabular"; + subtasks: { + type: string; + name: string; + }[]; + }; + "tabular-regression": { + name: string; + modality: "tabular"; + subtasks: { + type: string; + name: string; + }[]; + }; + "tabular-to-text": { + name: string; + modality: "tabular"; + subtasks: { + type: string; + name: string; + }[]; + hideInModels: true; + }; + "table-to-text": { + name: string; + modality: "nlp"; + hideInModels: true; + }; + "multiple-choice": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + hideInModels: true; + }; + "text-ranking": { + name: string; + modality: "nlp"; + }; + "text-retrieval": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "nlp"; + hideInModels: true; + }; + "time-series-forecasting": { + name: string; + modality: "tabular"; + subtasks: { + type: string; + name: string; + }[]; + }; + "text-to-video": { + name: string; + modality: "cv"; + }; + "image-text-to-text": { + name: string; + modality: "multimodal"; + }; + "image-text-to-image": { + name: string; + modality: "multimodal"; + }; + "image-text-to-video": { + name: string; + modality: "multimodal"; + }; + "visual-question-answering": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "multimodal"; + }; + "document-question-answering": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "multimodal"; + hideInDatasets: true; + }; + "zero-shot-image-classification": { + name: string; + modality: "cv"; + }; + "graph-ml": { + name: string; + modality: "other"; + }; + "mask-generation": { + name: string; + modality: "cv"; + }; + "zero-shot-object-detection": { + name: string; + modality: "cv"; + }; + "text-to-3d": { + name: string; + modality: "cv"; + }; + "image-to-3d": { + name: string; + modality: "cv"; + }; + "image-feature-extraction": { + name: string; + modality: "cv"; + }; + "video-text-to-text": { + name: string; + modality: "multimodal"; + hideInDatasets: false; + }; + "keypoint-detection": { + name: string; + subtasks: { + type: string; + name: string; + }[]; + modality: "cv"; + hideInDatasets: true; + }; + "visual-document-retrieval": { + name: string; + modality: "multimodal"; + }; + "any-to-any": { + name: string; + modality: "multimodal"; + }; + "video-to-video": { + name: string; + modality: "cv"; + hideInDatasets: true; + }; + other: { + name: string; + modality: "other"; + hideInModels: true; + hideInDatasets: true; + }; +}; +export type PipelineType = keyof typeof PIPELINE_DATA; +export type WidgetType = PipelineType | "conversational"; +export declare const PIPELINE_TYPES: PipelineType[]; +export declare const SUBTASK_TYPES: string[]; +export declare const PIPELINE_TYPES_SET: Set<"other" | "text-classification" | "token-classification" | "table-question-answering" | "question-answering" | "zero-shot-classification" | "translation" | "summarization" | "feature-extraction" | "text-generation" | "fill-mask" | "sentence-similarity" | "text-to-speech" | "text-to-audio" | "automatic-speech-recognition" | "audio-to-audio" | "audio-classification" | "audio-text-to-text" | "voice-activity-detection" | "depth-estimation" | "image-classification" | "object-detection" | "image-segmentation" | "text-to-image" | "image-to-text" | "image-to-image" | "image-to-video" | "unconditional-image-generation" | "video-classification" | "reinforcement-learning" | "robotics" | "tabular-classification" | "tabular-regression" | "tabular-to-text" | "table-to-text" | "multiple-choice" | "text-ranking" | "text-retrieval" | "time-series-forecasting" | "text-to-video" | "image-text-to-text" | "image-text-to-image" | "image-text-to-video" | "visual-question-answering" | "document-question-answering" | "zero-shot-image-classification" | "graph-ml" | "mask-generation" | "zero-shot-object-detection" | "text-to-3d" | "image-to-3d" | "image-feature-extraction" | "video-text-to-text" | "keypoint-detection" | "visual-document-retrieval" | "any-to-any" | "video-to-video">; +//# sourceMappingURL=pipelines.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/pipelines.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/pipelines.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e44b58ed1d6363fdc5bd08b53ab435961d17e281 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/pipelines.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"pipelines.d.ts","sourceRoot":"","sources":["../../src/pipelines.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,UAAU,yEAA0E,CAAC;AAElG,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD,eAAO,MAAM,eAAe;;;;;;;;CAQQ,CAAC;AAErC;;;;;;GAMG;AACH,MAAM,WAAW,OAAO;IACvB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;CACb;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC5B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IACrB,QAAQ,EAAE,QAAQ,CAAC;IACnB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CACzB;AAcD,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwkBc,CAAC;AAEzC,MAAM,MAAM,YAAY,GAAG,MAAM,OAAO,aAAa,CAAC;AAEtD,MAAM,MAAM,UAAU,GAAG,YAAY,GAAG,gBAAgB,CAAC;AAEzD,eAAO,MAAM,cAAc,EAAiC,YAAY,EAAE,CAAC;AAE3E,eAAO,MAAM,aAAa,UAEN,CAAC;AAErB,eAAO,MAAM,kBAAkB,iwCAA0B,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/pipelines.js b/node_modules/@huggingface/tasks/dist/esm/pipelines.js new file mode 100644 index 0000000000000000000000000000000000000000..e1836aa9b7148f72dd53d31bccf07ccf13cefaf0 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/pipelines.js @@ -0,0 +1,612 @@ +export const MODALITIES = ["multimodal", "nlp", "cv", "audio", "tabular", "rl", "other"]; +export const MODALITY_LABELS = { + multimodal: "Multimodal", + nlp: "Natural Language Processing", + audio: "Audio", + cv: "Computer Vision", + rl: "Reinforcement Learning", + tabular: "Tabular", + other: "Other", +}; +/// Coarse-grained taxonomy of tasks +/// +/// This type is used in multiple places in the Hugging Face +/// ecosystem: +/// - To determine which widget to show. +/// - To determine which endpoint of Inference Endpoints to use. +/// - As filters at the left of models and datasets page. +/// +/// Note that this is sensitive to order. +/// For each domain, the order should be of decreasing specificity. +/// This will impact the default pipeline tag of a model when not +/// specified. +export const PIPELINE_DATA = { + "text-classification": { + name: "Text Classification", + subtasks: [ + { + type: "acceptability-classification", + name: "Acceptability Classification", + }, + { + type: "entity-linking-classification", + name: "Entity Linking Classification", + }, + { + type: "fact-checking", + name: "Fact Checking", + }, + { + type: "intent-classification", + name: "Intent Classification", + }, + { + type: "language-identification", + name: "Language Identification", + }, + { + type: "multi-class-classification", + name: "Multi Class Classification", + }, + { + type: "multi-label-classification", + name: "Multi Label Classification", + }, + { + type: "multi-input-text-classification", + name: "Multi-input Text Classification", + }, + { + type: "natural-language-inference", + name: "Natural Language Inference", + }, + { + type: "semantic-similarity-classification", + name: "Semantic Similarity Classification", + }, + { + type: "sentiment-classification", + name: "Sentiment Classification", + }, + { + type: "topic-classification", + name: "Topic Classification", + }, + { + type: "semantic-similarity-scoring", + name: "Semantic Similarity Scoring", + }, + { + type: "sentiment-scoring", + name: "Sentiment Scoring", + }, + { + type: "sentiment-analysis", + name: "Sentiment Analysis", + }, + { + type: "hate-speech-detection", + name: "Hate Speech Detection", + }, + { + type: "text-scoring", + name: "Text Scoring", + }, + ], + modality: "nlp", + }, + "token-classification": { + name: "Token Classification", + subtasks: [ + { + type: "named-entity-recognition", + name: "Named Entity Recognition", + }, + { + type: "part-of-speech", + name: "Part of Speech", + }, + { + type: "parsing", + name: "Parsing", + }, + { + type: "lemmatization", + name: "Lemmatization", + }, + { + type: "word-sense-disambiguation", + name: "Word Sense Disambiguation", + }, + { + type: "coreference-resolution", + name: "Coreference-resolution", + }, + ], + modality: "nlp", + }, + "table-question-answering": { + name: "Table Question Answering", + modality: "nlp", + }, + "question-answering": { + name: "Question Answering", + subtasks: [ + { + type: "extractive-qa", + name: "Extractive QA", + }, + { + type: "open-domain-qa", + name: "Open Domain QA", + }, + { + type: "closed-domain-qa", + name: "Closed Domain QA", + }, + ], + modality: "nlp", + }, + "zero-shot-classification": { + name: "Zero-Shot Classification", + modality: "nlp", + }, + translation: { + name: "Translation", + modality: "nlp", + }, + summarization: { + name: "Summarization", + subtasks: [ + { + type: "news-articles-summarization", + name: "News Articles Summarization", + }, + { + type: "news-articles-headline-generation", + name: "News Articles Headline Generation", + }, + ], + modality: "nlp", + }, + "feature-extraction": { + name: "Feature Extraction", + modality: "nlp", + }, + "text-generation": { + name: "Text Generation", + subtasks: [ + { + type: "dialogue-modeling", + name: "Dialogue Modeling", + }, + { + type: "dialogue-generation", + name: "Dialogue Generation", + }, + { + type: "conversational", + name: "Conversational", + }, + { + type: "language-modeling", + name: "Language Modeling", + }, + { + type: "text-simplification", + name: "Text simplification", + }, + { + type: "explanation-generation", + name: "Explanation Generation", + }, + { + type: "abstractive-qa", + name: "Abstractive QA", + }, + { + type: "open-domain-abstractive-qa", + name: "Open Domain Abstractive QA", + }, + { + type: "closed-domain-qa", + name: "Closed Domain QA", + }, + { + type: "open-book-qa", + name: "Open Book QA", + }, + { + type: "closed-book-qa", + name: "Closed Book QA", + }, + { + type: "text2text-generation", + name: "Text2Text Generation", + }, + ], + modality: "nlp", + }, + "fill-mask": { + name: "Fill-Mask", + subtasks: [ + { + type: "slot-filling", + name: "Slot Filling", + }, + { + type: "masked-language-modeling", + name: "Masked Language Modeling", + }, + ], + modality: "nlp", + }, + "sentence-similarity": { + name: "Sentence Similarity", + modality: "nlp", + }, + "text-to-speech": { + name: "Text-to-Speech", + modality: "audio", + }, + "text-to-audio": { + name: "Text-to-Audio", + modality: "audio", + }, + "automatic-speech-recognition": { + name: "Automatic Speech Recognition", + modality: "audio", + }, + "audio-to-audio": { + name: "Audio-to-Audio", + modality: "audio", + }, + "audio-classification": { + name: "Audio Classification", + subtasks: [ + { + type: "keyword-spotting", + name: "Keyword Spotting", + }, + { + type: "speaker-identification", + name: "Speaker Identification", + }, + { + type: "audio-intent-classification", + name: "Audio Intent Classification", + }, + { + type: "audio-emotion-recognition", + name: "Audio Emotion Recognition", + }, + { + type: "audio-language-identification", + name: "Audio Language Identification", + }, + ], + modality: "audio", + }, + "audio-text-to-text": { + name: "Audio-Text-to-Text", + modality: "multimodal", + hideInDatasets: true, + }, + "voice-activity-detection": { + name: "Voice Activity Detection", + modality: "audio", + }, + "depth-estimation": { + name: "Depth Estimation", + modality: "cv", + }, + "image-classification": { + name: "Image Classification", + subtasks: [ + { + type: "multi-label-image-classification", + name: "Multi Label Image Classification", + }, + { + type: "multi-class-image-classification", + name: "Multi Class Image Classification", + }, + ], + modality: "cv", + }, + "object-detection": { + name: "Object Detection", + subtasks: [ + { + type: "face-detection", + name: "Face Detection", + }, + { + type: "vehicle-detection", + name: "Vehicle Detection", + }, + ], + modality: "cv", + }, + "image-segmentation": { + name: "Image Segmentation", + subtasks: [ + { + type: "instance-segmentation", + name: "Instance Segmentation", + }, + { + type: "semantic-segmentation", + name: "Semantic Segmentation", + }, + { + type: "panoptic-segmentation", + name: "Panoptic Segmentation", + }, + ], + modality: "cv", + }, + "text-to-image": { + name: "Text-to-Image", + modality: "cv", + }, + "image-to-text": { + name: "Image-to-Text", + subtasks: [ + { + type: "image-captioning", + name: "Image Captioning", + }, + ], + modality: "cv", + }, + "image-to-image": { + name: "Image-to-Image", + subtasks: [ + { + type: "image-inpainting", + name: "Image Inpainting", + }, + { + type: "image-colorization", + name: "Image Colorization", + }, + { + type: "super-resolution", + name: "Super Resolution", + }, + ], + modality: "cv", + }, + "image-to-video": { + name: "Image-to-Video", + modality: "cv", + }, + "unconditional-image-generation": { + name: "Unconditional Image Generation", + modality: "cv", + }, + "video-classification": { + name: "Video Classification", + modality: "cv", + }, + "reinforcement-learning": { + name: "Reinforcement Learning", + modality: "rl", + }, + robotics: { + name: "Robotics", + modality: "rl", + subtasks: [ + { + type: "grasping", + name: "Grasping", + }, + { + type: "task-planning", + name: "Task Planning", + }, + ], + }, + "tabular-classification": { + name: "Tabular Classification", + modality: "tabular", + subtasks: [ + { + type: "tabular-multi-class-classification", + name: "Tabular Multi Class Classification", + }, + { + type: "tabular-multi-label-classification", + name: "Tabular Multi Label Classification", + }, + ], + }, + "tabular-regression": { + name: "Tabular Regression", + modality: "tabular", + subtasks: [ + { + type: "tabular-single-column-regression", + name: "Tabular Single Column Regression", + }, + ], + }, + "tabular-to-text": { + name: "Tabular to Text", + modality: "tabular", + subtasks: [ + { + type: "rdf-to-text", + name: "RDF to text", + }, + ], + hideInModels: true, + }, + "table-to-text": { + name: "Table to Text", + modality: "nlp", + hideInModels: true, + }, + "multiple-choice": { + name: "Multiple Choice", + subtasks: [ + { + type: "multiple-choice-qa", + name: "Multiple Choice QA", + }, + { + type: "multiple-choice-coreference-resolution", + name: "Multiple Choice Coreference Resolution", + }, + ], + modality: "nlp", + hideInModels: true, + }, + "text-ranking": { + name: "Text Ranking", + modality: "nlp", + }, + "text-retrieval": { + name: "Text Retrieval", + subtasks: [ + { + type: "document-retrieval", + name: "Document Retrieval", + }, + { + type: "utterance-retrieval", + name: "Utterance Retrieval", + }, + { + type: "entity-linking-retrieval", + name: "Entity Linking Retrieval", + }, + { + type: "fact-checking-retrieval", + name: "Fact Checking Retrieval", + }, + ], + modality: "nlp", + hideInModels: true, + }, + "time-series-forecasting": { + name: "Time Series Forecasting", + modality: "tabular", + subtasks: [ + { + type: "univariate-time-series-forecasting", + name: "Univariate Time Series Forecasting", + }, + { + type: "multivariate-time-series-forecasting", + name: "Multivariate Time Series Forecasting", + }, + ], + }, + "text-to-video": { + name: "Text-to-Video", + modality: "cv", + }, + "image-text-to-text": { + name: "Image-Text-to-Text", + modality: "multimodal", + }, + "image-text-to-image": { + name: "Image-Text-to-Image", + modality: "multimodal", + }, + "image-text-to-video": { + name: "Image-Text-to-Video", + modality: "multimodal", + }, + "visual-question-answering": { + name: "Visual Question Answering", + subtasks: [ + { + type: "visual-question-answering", + name: "Visual Question Answering", + }, + ], + modality: "multimodal", + }, + "document-question-answering": { + name: "Document Question Answering", + subtasks: [ + { + type: "document-question-answering", + name: "Document Question Answering", + }, + ], + modality: "multimodal", + hideInDatasets: true, + }, + "zero-shot-image-classification": { + name: "Zero-Shot Image Classification", + modality: "cv", + }, + "graph-ml": { + name: "Graph Machine Learning", + modality: "other", + }, + "mask-generation": { + name: "Mask Generation", + modality: "cv", + }, + "zero-shot-object-detection": { + name: "Zero-Shot Object Detection", + modality: "cv", + }, + "text-to-3d": { + name: "Text-to-3D", + modality: "cv", + }, + "image-to-3d": { + name: "Image-to-3D", + modality: "cv", + }, + "image-feature-extraction": { + name: "Image Feature Extraction", + modality: "cv", + }, + "video-text-to-text": { + name: "Video-Text-to-Text", + modality: "multimodal", + hideInDatasets: false, + }, + "keypoint-detection": { + name: "Keypoint Detection", + subtasks: [ + { + type: "pose-estimation", + name: "Pose Estimation", + }, + ], + modality: "cv", + hideInDatasets: true, + }, + "visual-document-retrieval": { + name: "Visual Document Retrieval", + modality: "multimodal", + }, + "any-to-any": { + name: "Any-to-Any", + modality: "multimodal", + }, + "video-to-video": { + name: "Video-to-Video", + modality: "cv", + hideInDatasets: true, + }, + other: { + name: "Other", + modality: "other", + hideInModels: true, + hideInDatasets: true, + }, +}; +export const PIPELINE_TYPES = Object.keys(PIPELINE_DATA); +export const SUBTASK_TYPES = Object.values(PIPELINE_DATA) + .flatMap((data) => ("subtasks" in data ? data.subtasks : [])) + .map((s) => s.type); +export const PIPELINE_TYPES_SET = new Set(PIPELINE_TYPES); diff --git a/node_modules/@huggingface/tasks/dist/esm/snippets/common.d.ts b/node_modules/@huggingface/tasks/dist/esm/snippets/common.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..bf5e9f386dcfeaef440383931a5892d91c9f01f8 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/snippets/common.d.ts @@ -0,0 +1,14 @@ +import type { ChatCompletionInputMessage, GenerationParameters } from "../tasks/index.js"; +export declare function stringifyMessages(messages: ChatCompletionInputMessage[], opts?: { + indent?: string; + attributeKeyQuotes?: boolean; + customContentEscaper?: (str: string) => string; +}): string; +type PartialGenerationParameters = Partial>; +export declare function stringifyGenerationConfig(config: PartialGenerationParameters, opts: { + indent: string; + attributeValueConnector: string; + attributeKeyQuotes?: boolean; +}): string; +export {}; +//# sourceMappingURL=common.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/snippets/common.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/snippets/common.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..22fc37561512c88d91a21b3ff56a7c29cdd680d6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/snippets/common.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../../src/snippets/common.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAE1F,wBAAgB,iBAAiB,CAChC,QAAQ,EAAE,0BAA0B,EAAE,EACtC,IAAI,CAAC,EAAE;IACN,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,oBAAoB,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;CAC/C,GACC,MAAM,CAYR;AAED,KAAK,2BAA2B,GAAG,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,aAAa,GAAG,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC;AAE/G,wBAAgB,yBAAyB,CACxC,MAAM,EAAE,2BAA2B,EACnC,IAAI,EAAE;IACL,MAAM,EAAE,MAAM,CAAC;IACf,uBAAuB,EAAE,MAAM,CAAC;IAChC,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC7B,GACC,MAAM,CAMR"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/snippets/common.js b/node_modules/@huggingface/tasks/dist/esm/snippets/common.js new file mode 100644 index 0000000000000000000000000000000000000000..d14cb0abac45d3e97b49c6934009f7f7c712f20a --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/snippets/common.js @@ -0,0 +1,19 @@ +export function stringifyMessages(messages, opts) { + let messagesStr = JSON.stringify(messages, null, "\t"); + if (opts?.indent) { + messagesStr = messagesStr.replaceAll("\n", `\n${opts.indent}`); + } + if (!opts?.attributeKeyQuotes) { + messagesStr = messagesStr.replace(/"([^"]+)":/g, "$1:"); + } + if (opts?.customContentEscaper) { + messagesStr = opts.customContentEscaper(messagesStr); + } + return messagesStr; +} +export function stringifyGenerationConfig(config, opts) { + const quote = opts.attributeKeyQuotes ? `"` : ""; + return Object.entries(config) + .map(([key, val]) => `${quote}${key}${quote}${opts.attributeValueConnector}${val},`) + .join(`${opts.indent}`); +} diff --git a/node_modules/@huggingface/tasks/dist/esm/snippets/index.d.ts b/node_modules/@huggingface/tasks/dist/esm/snippets/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..991b953e5f6ee7ddd737c3ad8c2a674f6197c98d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/snippets/index.d.ts @@ -0,0 +1,4 @@ +export * from "./common.js"; +export * from "./inputs.js"; +export * from "./types.js"; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/snippets/index.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/snippets/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..548a10edf879f1c90bf3b4e790a3ee666694d74f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/snippets/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/snippets/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/snippets/index.js b/node_modules/@huggingface/tasks/dist/esm/snippets/index.js new file mode 100644 index 0000000000000000000000000000000000000000..bd77dfd5213a3c73a1514ec0d14c87093b5bfb52 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/snippets/index.js @@ -0,0 +1,3 @@ +export * from "./common.js"; +export * from "./inputs.js"; +export * from "./types.js"; diff --git a/node_modules/@huggingface/tasks/dist/esm/snippets/inputs.d.ts b/node_modules/@huggingface/tasks/dist/esm/snippets/inputs.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b7195b500182bf3159236cc9a51dfd441b5c4743 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/snippets/inputs.d.ts @@ -0,0 +1,4 @@ +import type { ChatCompletionInputMessage } from "../tasks/index.js"; +import type { ModelDataMinimal } from "./types.js"; +export declare function getModelInputSnippet(model: ModelDataMinimal, noWrap?: boolean, noQuotes?: boolean): string | ChatCompletionInputMessage[]; +//# sourceMappingURL=inputs.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/snippets/inputs.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/snippets/inputs.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e78561c1f373ff55e20a0e5a281a40ddfcb99836 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/snippets/inputs.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inputs.d.ts","sourceRoot":"","sources":["../../../src/snippets/inputs.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAqKnD,wBAAgB,oBAAoB,CACnC,KAAK,EAAE,gBAAgB,EACvB,MAAM,UAAQ,EACd,QAAQ,UAAQ,GACd,MAAM,GAAG,0BAA0B,EAAE,CAmBvC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/snippets/inputs.js b/node_modules/@huggingface/tasks/dist/esm/snippets/inputs.js new file mode 100644 index 0000000000000000000000000000000000000000..c575168ffff7d4cb85cbf638645703ccb8da3832 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/snippets/inputs.js @@ -0,0 +1,144 @@ +const inputsZeroShotClassification = () => `"Hi, I recently bought a device from your company but it is not working as advertised and I would like to get reimbursed!"`; +const inputsTranslation = () => `"Меня зовут Вольфганг и я живу в Берлине"`; +const inputsSummarization = () => `"The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct."`; +const inputsTableQuestionAnswering = () => `{ + "query": "How many stars does the transformers repository have?", + "table": { + "Repository": ["Transformers", "Datasets", "Tokenizers"], + "Stars": ["36542", "4512", "3934"], + "Contributors": ["651", "77", "34"], + "Programming language": [ + "Python", + "Python", + "Rust, Python and NodeJS" + ] + } +}`; +const inputsVisualQuestionAnswering = () => `{ + "image": "cat.png", + "question": "What is in this image?" + }`; +const inputsQuestionAnswering = () => `{ + "question": "What is my name?", + "context": "My name is Clara and I live in Berkeley." +}`; +const inputsTextClassification = () => `"I like you. I love you"`; +const inputsTokenClassification = () => `"My name is Sarah Jessica Parker but you can call me Jessica"`; +const inputsTextGeneration = (model) => { + if (model.tags.includes("conversational")) { + return model.pipeline_tag === "text-generation" + ? [{ role: "user", content: "What is the capital of France?" }] + : [ + { + role: "user", + content: [ + { + type: "text", + text: "Describe this image in one sentence.", + }, + { + type: "image_url", + image_url: { + url: "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg", + }, + }, + ], + }, + ]; + } + return `"Can you please let us know more details about your "`; +}; +const inputsFillMask = (model) => `"The answer to the universe is ${model.mask_token}."`; +const inputsSentenceSimilarity = () => `{ + "source_sentence": "That is a happy person", + "sentences": [ + "That is a happy dog", + "That is a very happy person", + "Today is a sunny day" + ] +}`; +const inputsFeatureExtraction = () => `"Today is a sunny day and I will get some ice cream."`; +const inputsImageClassification = () => `"cats.jpg"`; +const inputsImageToText = () => `"cats.jpg"`; +const inputsImageToImage = () => `{ + "image": "cat.png", + "prompt": "Turn the cat into a tiger." +}`; +const inputsImageToVideo = () => `{ + "image": "cat.png", + "prompt": "The cat starts to dance" +}`; +const inputsImageTextToImage = () => `{ + "image": "cat.png", + "prompt": "Turn the cat into a tiger." +}`; +const inputsImageTextToVideo = () => `{ + "image": "cat.png", + "prompt": "The cat starts to dance" +}`; +const inputsImageSegmentation = () => `"cats.jpg"`; +const inputsObjectDetection = () => `"cats.jpg"`; +const inputsAudioToAudio = () => `"sample1.flac"`; +const inputsAudioClassification = () => `"sample1.flac"`; +const inputsTextToImage = () => `"Astronaut riding a horse"`; +const inputsTextToVideo = () => `"A young man walking on the street"`; +const inputsTextToSpeech = () => `"The answer to the universe is 42"`; +const inputsTextToAudio = () => `"liquid drum and bass, atmospheric synths, airy sounds"`; +const inputsAutomaticSpeechRecognition = () => `"sample1.flac"`; +const inputsTabularPrediction = () => `'{"Height":[11.52,12.48],"Length1":[23.2,24.0],"Length2":[25.4,26.3],"Species": ["Bream","Bream"]}'`; +const inputsZeroShotImageClassification = () => `"cats.jpg"`; +const modelInputSnippets = { + "audio-to-audio": inputsAudioToAudio, + "audio-classification": inputsAudioClassification, + "automatic-speech-recognition": inputsAutomaticSpeechRecognition, + "document-question-answering": inputsVisualQuestionAnswering, + "feature-extraction": inputsFeatureExtraction, + "fill-mask": inputsFillMask, + "image-classification": inputsImageClassification, + "image-to-text": inputsImageToText, + "image-to-image": inputsImageToImage, + "image-to-video": inputsImageToVideo, + "image-text-to-image": inputsImageTextToImage, + "image-text-to-video": inputsImageTextToVideo, + "image-segmentation": inputsImageSegmentation, + "object-detection": inputsObjectDetection, + "question-answering": inputsQuestionAnswering, + "sentence-similarity": inputsSentenceSimilarity, + summarization: inputsSummarization, + "table-question-answering": inputsTableQuestionAnswering, + "tabular-regression": inputsTabularPrediction, + "tabular-classification": inputsTabularPrediction, + "text-classification": inputsTextClassification, + "text-generation": inputsTextGeneration, + "image-text-to-text": inputsTextGeneration, + "text-to-image": inputsTextToImage, + "text-to-video": inputsTextToVideo, + "text-to-speech": inputsTextToSpeech, + "text-to-audio": inputsTextToAudio, + "token-classification": inputsTokenClassification, + translation: inputsTranslation, + "zero-shot-classification": inputsZeroShotClassification, + "zero-shot-image-classification": inputsZeroShotImageClassification, +}; +// Use noWrap to put the whole snippet on a single line (removing new lines and tabulations) +// Use noQuotes to strip quotes from start & end (example: "abc" -> abc) +export function getModelInputSnippet(model, noWrap = false, noQuotes = false) { + if (model.pipeline_tag) { + const inputs = modelInputSnippets[model.pipeline_tag]; + if (inputs) { + let result = inputs(model); + if (typeof result === "string") { + if (noWrap) { + result = result.replace(/(?:(?:\r?\n|\r)\t*)|\t+/g, " "); + } + if (noQuotes) { + const REGEX_QUOTES = /^"(.+)"$/s; + const match = result.match(REGEX_QUOTES); + result = match ? match[1] : result; + } + } + return result; + } + } + return "No input example has been defined for this model task."; +} diff --git a/node_modules/@huggingface/tasks/dist/esm/snippets/types.d.ts b/node_modules/@huggingface/tasks/dist/esm/snippets/types.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..684489ef8f7272cde9a9f5039481a92122373f5f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/snippets/types.d.ts @@ -0,0 +1,15 @@ +import type { ModelData } from "../model-data.js"; +/** + * Minimal model data required for snippets. + * + * Add more fields as needed. + */ +export type ModelDataMinimal = Pick; +export declare const inferenceSnippetLanguages: readonly ["python", "js", "sh"]; +export type InferenceSnippetLanguage = (typeof inferenceSnippetLanguages)[number]; +export interface InferenceSnippet { + language: InferenceSnippetLanguage; + client: string; + content: string; +} +//# sourceMappingURL=types.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/snippets/types.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/snippets/types.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..3e04ceef9eea3456ea042301bd21850e9794f79b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/snippets/types.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/snippets/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAElD;;;;GAIG;AACH,MAAM,MAAM,gBAAgB,GAAG,IAAI,CAClC,SAAS,EACT,IAAI,GAAG,cAAc,GAAG,YAAY,GAAG,cAAc,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CACvF,CAAC;AAGF,eAAO,MAAM,yBAAyB,iCAAkC,CAAC;AACzE,MAAM,MAAM,wBAAwB,GAAG,CAAC,OAAO,yBAAyB,CAAC,CAAC,MAAM,CAAC,CAAC;AAElF,MAAM,WAAW,gBAAgB;IAChC,QAAQ,EAAE,wBAAwB,CAAC;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CAChB"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/snippets/types.js b/node_modules/@huggingface/tasks/dist/esm/snippets/types.js new file mode 100644 index 0000000000000000000000000000000000000000..8a8ec2da8fb71da421bfdc5af018535e2356cae4 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/snippets/types.js @@ -0,0 +1,2 @@ +// Order of the elements in InferenceModal.svelte is determined by this const +export const inferenceSnippetLanguages = ["python", "js", "sh"]; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/any-to-any/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/any-to-any/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/any-to-any/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/any-to-any/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/any-to-any/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..316d468d5a30916ed526b156e185469fadbf2567 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/any-to-any/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/any-to-any/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA4Df,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/any-to-any/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/any-to-any/data.js new file mode 100644 index 0000000000000000000000000000000000000000..a2cc663cf9de2a15aec952874383817be297fe63 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/any-to-any/data.js @@ -0,0 +1,61 @@ +const taskData = { + datasets: [ + { + description: "A dataset with multiple modality input and output pairs.", + id: "PKU-Alignment/align-anything", + }, + ], + demo: { + inputs: [ + { + filename: "any-to-any-input.jpg", + type: "img", + }, + { + label: "Text Prompt", + content: "What is the significance of this place?", + type: "text", + }, + ], + outputs: [ + { + label: "Generated Text", + content: "The place in the picture is Osaka Castle, located in Osaka, Japan. Osaka Castle is a historic castle that was originally built in the 16th century by Toyotomi Hideyoshi, a powerful warlord of the time. It is one of the most famous landmarks in Osaka and is known for its distinctive white walls and black roof tiles. The castle has been rebuilt several times over the centuries and is now a popular tourist attraction, offering visitors a glimpse into Japan's rich history and culture.", + type: "text", + }, + { + filename: "any-to-any-output.wav", + type: "audio", + }, + ], + }, + metrics: [], + models: [ + { + description: "Strong model that can take in video, audio, image, text and output text and natural speech.", + id: "Qwen/Qwen2.5-Omni-7B", + }, + { + description: "Robust model that can take in image and text and generate image and text.", + id: "OmniGen2/OmniGen2", + }, + { + description: "Any-to-any model with speech, video, audio, image and text understanding capabilities.", + id: "openbmb/MiniCPM-o-2_6", + }, + { + description: "A model that can understand image and text and generate image and text.", + id: "ByteDance-Seed/BAGEL-7B-MoT", + }, + ], + spaces: [ + { + description: "An application to chat with an any-to-any (image & text) model.", + id: "OmniGen2/OmniGen2", + }, + ], + summary: "Any-to-any models can understand two or more modalities and output two or more modalities.", + widgetModels: [], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..bf29381612f3a120b322349e0cbf0b3d83167dc5 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/audio-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA4Ef,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..45edc3c5052228b01cdecf74cf7566278a15a092 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/data.js @@ -0,0 +1,77 @@ +const taskData = { + datasets: [ + { + description: "A benchmark of 10 different audio tasks.", + id: "s3prl/superb", + }, + { + description: "A dataset of YouTube clips and their sound categories.", + id: "agkphysics/AudioSet", + }, + ], + demo: { + inputs: [ + { + filename: "audio.wav", + type: "audio", + }, + ], + outputs: [ + { + data: [ + { + label: "Up", + score: 0.2, + }, + { + label: "Down", + score: 0.8, + }, + ], + type: "chart", + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + description: "An easy-to-use model for command recognition.", + id: "speechbrain/google_speech_command_xvector", + }, + { + description: "An emotion recognition model.", + id: "ehcalabres/wav2vec2-lg-xlsr-en-speech-emotion-recognition", + }, + { + description: "A language identification model.", + id: "facebook/mms-lid-126", + }, + ], + spaces: [ + { + description: "An application that can classify music into different genre.", + id: "kurianbenoy/audioclassification", + }, + ], + summary: "Audio classification is the task of assigning a label or class to a given audio. It can be used for recognizing which command a user is giving or the emotion of a statement, as well as identifying a speaker.", + widgetModels: ["MIT/ast-finetuned-audioset-10-10-0.4593"], + youtubeId: "KWwzcmG98Ds", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ddc1cffad496de5105c0ecf687f7302737086f82 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/inference.d.ts @@ -0,0 +1,54 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Audio Classification inference + */ +export interface AudioClassificationInput { + /** + * The input audio data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the audio data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Audio Classification + */ + parameters?: AudioClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Audio Classification + */ +export interface AudioClassificationParameters { + /** + * The function to apply to the model outputs in order to retrieve the scores. + */ + function_to_apply?: ClassificationOutputTransform; + /** + * When specified, limits the output to the top K most probable classes. + */ + top_k?: number; + [property: string]: unknown; +} +/** + * The function to apply to the model outputs in order to retrieve the scores. + */ +export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none"; +export type AudioClassificationOutput = AudioClassificationOutputElement[]; +/** + * Outputs for Audio Classification inference + */ +export interface AudioClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..36b8340d1109765339d12eda90f6e0d6c8be3e04 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/audio-classification/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACxC;;;OAGG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,6BAA6B,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC7C;;OAEG;IACH,iBAAiB,CAAC,EAAE,6BAA6B,CAAC;IAClD;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAC3E,MAAM,MAAM,yBAAyB,GAAG,gCAAgC,EAAE,CAAC;AAC3E;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-classification/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/audio-text-to-text/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-text-to-text/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-text-to-text/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/audio-text-to-text/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-text-to-text/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5a31cfccf308085ac90260ae5454e2c10df544b2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-text-to-text/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/audio-text-to-text/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAiEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/audio-text-to-text/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-text-to-text/data.js new file mode 100644 index 0000000000000000000000000000000000000000..c74e3dc069262e5e3602b807c523bac52860958d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-text-to-text/data.js @@ -0,0 +1,65 @@ +const taskData = { + datasets: [ + { + description: "A dataset containing audio conversations with question–answer pairs.", + id: "nvidia/AF-Think", + }, + { + description: "A more advanced and comprehensive dataset that contains characteristics of the audio as well", + id: "tsinghua-ee/QualiSpeech", + }, + ], + demo: { + inputs: [ + { + filename: "audio.wav", + type: "audio", + }, + { + label: "Text Prompt", + content: "What is the gender of the speaker?", + type: "text", + }, + ], + outputs: [ + { + label: "Generated Text", + content: "The gender of the speaker is female.", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "A lightweight model that has capabilities of taking both audio and text as inputs and generating responses.", + id: "fixie-ai/ultravox-v0_5-llama-3_2-1b", + }, + { + description: "A multimodal model that supports voice chat and audio analysis.", + id: "Qwen/Qwen2-Audio-7B-Instruct", + }, + { + description: "A model for audio understanding, speech translation, and transcription.", + id: "mistralai/Voxtral-Small-24B-2507", + }, + { + description: "A new model capable of audio question answering and reasoning.", + id: "nvidia/audio-flamingo-3", + }, + ], + spaces: [ + { + description: "A space that takes input as both audio and text and generates answers.", + id: "iamomtiwari/ATTT", + }, + { + description: "A web application that demonstrates chatting with the Qwen2Audio Model.", + id: "freddyaboulton/talk-to-qwen-webrtc", + }, + ], + summary: "Audio-text-to-text models take both an audio clip and a text prompt as input, and generate natural language text as output. These models can answer questions about spoken content, summarize meetings, analyze music, or interpret speech beyond simple transcription. They are useful for applications that combine speech understanding with reasoning or conversation.", + widgetModels: [], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/audio-to-audio/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-to-audio/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-to-audio/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/audio-to-audio/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-to-audio/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e20ea8823de86f46b9aa9c8658cf637fb2e9d198 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-to-audio/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/audio-to-audio/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA6Df,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/audio-to-audio/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-to-audio/data.js new file mode 100644 index 0000000000000000000000000000000000000000..98f95ed7c6081e3fedd784c6b304242ec022d948 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/audio-to-audio/data.js @@ -0,0 +1,60 @@ +const taskData = { + datasets: [ + { + description: "512-element X-vector embeddings of speakers from CMU ARCTIC dataset.", + id: "Matthijs/cmu-arctic-xvectors", + }, + ], + demo: { + inputs: [ + { + filename: "input.wav", + type: "audio", + }, + ], + outputs: [ + { + filename: "label-0.wav", + type: "audio", + }, + { + filename: "label-1.wav", + type: "audio", + }, + ], + }, + metrics: [ + { + description: "The Signal-to-Noise ratio is the relationship between the target signal level and the background noise level. It is calculated as the logarithm of the target signal divided by the background noise, in decibels.", + id: "snri", + }, + { + description: "The Signal-to-Distortion ratio is the relationship between the target signal and the sum of noise, interference, and artifact errors", + id: "sdri", + }, + ], + models: [ + { + description: "A speech enhancement model.", + id: "ResembleAI/resemble-enhance", + }, + { + description: "A model that can change the voice in a speech recording.", + id: "microsoft/speecht5_vc", + }, + ], + spaces: [ + { + description: "An application for speech separation.", + id: "younver/speechbrain-speech-separation", + }, + { + description: "An application for audio style transfer.", + id: "nakas/audio-diffusion_style_transfer", + }, + ], + summary: "Audio-to-Audio is a family of tasks in which the input is an audio and the output is one or multiple generated audios. Some example tasks are speech enhancement and source separation.", + widgetModels: ["speechbrain/sepformer-wham"], + youtubeId: "iohj7nCCYoM", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..987e72b47dbd3da5b501f1b9a5a875a736b3af59 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/automatic-speech-recognition/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAyFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/data.js new file mode 100644 index 0000000000000000000000000000000000000000..4d4b0bd7494b0c40729b64e48c5eb81a29c9b645 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/data.js @@ -0,0 +1,90 @@ +const taskData = { + datasets: [ + { + description: "31,175 hours of multilingual audio-text dataset in 108 languages.", + id: "mozilla-foundation/common_voice_17_0", + }, + { + description: "Multilingual and diverse audio dataset with 101k hours of audio.", + id: "amphion/Emilia-Dataset", + }, + { + description: "A dataset with 44.6k hours of English speaker data and 6k hours of other language speakers.", + id: "parler-tts/mls_eng", + }, + { + description: "A multilingual audio dataset with 370K hours of audio.", + id: "espnet/yodas", + }, + ], + demo: { + inputs: [ + { + filename: "input.flac", + type: "audio", + }, + ], + outputs: [ + { + /// GOING ALONG SLUSHY COUNTRY ROADS AND SPEAKING TO DAMP AUDIENCES I + label: "Transcript", + content: "Going along slushy country roads and speaking to damp audiences in...", + type: "text", + }, + ], + }, + metrics: [ + { + description: "", + id: "wer", + }, + { + description: "", + id: "cer", + }, + ], + models: [ + { + description: "A powerful ASR model by OpenAI.", + id: "openai/whisper-large-v3", + }, + { + description: "A good generic speech model by MetaAI for fine-tuning.", + id: "facebook/w2v-bert-2.0", + }, + { + description: "An end-to-end model that performs ASR and Speech Translation by MetaAI.", + id: "facebook/seamless-m4t-v2-large", + }, + { + description: "A powerful multilingual ASR and Speech Translation model by Nvidia.", + id: "nvidia/canary-1b", + }, + { + description: "Powerful speaker diarization model.", + id: "pyannote/speaker-diarization-3.1", + }, + ], + spaces: [ + { + description: "A powerful general-purpose speech recognition application.", + id: "hf-audio/whisper-large-v3", + }, + { + description: "Latest ASR model from Useful Sensors.", + id: "mrfakename/Moonshinex", + }, + { + description: "A high quality speech and text translation model by Meta.", + id: "facebook/seamless_m4t", + }, + { + description: "A powerful multilingual ASR and Speech Translation model by Nvidia", + id: "nvidia/canary-1b", + }, + ], + summary: "Automatic Speech Recognition (ASR), also known as Speech to Text (STT), is the task of transcribing a given audio to text. It has many applications, such as voice user interfaces.", + widgetModels: ["openai/whisper-large-v3"], + youtubeId: "TksaY_FDgnk", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..66b335f310af19f6c5741e82287897ecad32f647 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/inference.d.ts @@ -0,0 +1,151 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Automatic Speech Recognition inference + */ +export interface AutomaticSpeechRecognitionInput { + /** + * The input audio data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the audio data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Automatic Speech Recognition + */ + parameters?: AutomaticSpeechRecognitionParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Automatic Speech Recognition + */ +export interface AutomaticSpeechRecognitionParameters { + /** + * Parametrization of the text generation process + */ + generation_parameters?: GenerationParameters; + /** + * Whether to output corresponding timestamps with the generated text + */ + return_timestamps?: boolean; + [property: string]: unknown; +} +/** + * Parametrization of the text generation process + */ +export interface GenerationParameters { + /** + * Whether to use sampling instead of greedy decoding when generating new tokens. + */ + do_sample?: boolean; + /** + * Controls the stopping condition for beam-based methods. + */ + early_stopping?: EarlyStoppingUnion; + /** + * If set to float strictly between 0 and 1, only tokens with a conditional probability + * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + * Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + */ + epsilon_cutoff?: number; + /** + * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + * float strictly between 0 and 1, a token is only considered if it is greater than either + * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + * for more details. + */ + eta_cutoff?: number; + /** + * The maximum length (in tokens) of the generated text, including the input. + */ + max_length?: number; + /** + * The maximum number of tokens to generate. Takes precedence over max_length. + */ + max_new_tokens?: number; + /** + * The minimum length (in tokens) of the generated text, including the input. + */ + min_length?: number; + /** + * The minimum number of tokens to generate. Takes precedence over min_length. + */ + min_new_tokens?: number; + /** + * Number of groups to divide num_beams into in order to ensure diversity among different + * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + */ + num_beam_groups?: number; + /** + * Number of beams to use for beam search. + */ + num_beams?: number; + /** + * The value balances the model confidence and the degeneration penalty in contrastive + * search decoding. + */ + penalty_alpha?: number; + /** + * The value used to modulate the next token probabilities. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * If set to float < 1, only the smallest set of most probable tokens with probabilities + * that add up to top_p or higher are kept for generation. + */ + top_p?: number; + /** + * Local typicality measures how similar the conditional probability of predicting a target + * token next is to the expected conditional probability of predicting a random token next, + * given the partial text already generated. If set to float < 1, the smallest set of the + * most locally typical tokens with probabilities that add up to typical_p or higher are + * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + */ + typical_p?: number; + /** + * Whether the model should use the past last key/values attentions to speed up decoding + */ + use_cache?: boolean; + [property: string]: unknown; +} +/** + * Controls the stopping condition for beam-based methods. + */ +export type EarlyStoppingUnion = boolean | "never"; +/** + * Outputs of inference for the Automatic Speech Recognition task + */ +export interface AutomaticSpeechRecognitionOutput { + /** + * When returnTimestamps is enabled, chunks contains a list of audio chunks identified by + * the model. + */ + chunks?: AutomaticSpeechRecognitionOutputChunk[]; + /** + * The recognized text. + */ + text: string; + [property: string]: unknown; +} +export interface AutomaticSpeechRecognitionOutputChunk { + /** + * A chunk of text identified by the model + */ + text: string; + /** + * The start and end timestamps corresponding with the text + */ + timestamp: number[]; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..dece49d1b3a428996138b9dce1065a1c4b72d803 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/automatic-speech-recognition/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,+BAA+B;IAC/C;;;OAGG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,oCAAoC,CAAC;IAClD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,oCAAoC;IACpD;;OAEG;IACH,qBAAqB,CAAC,EAAE,oBAAoB,CAAC;IAC7C;;OAEG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,kBAAkB,CAAC;IACpC;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,OAAO,GAAG,OAAO,CAAC;AACnD;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;;OAGG;IACH,MAAM,CAAC,EAAE,qCAAqC,EAAE,CAAC;IACjD;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,qCAAqC;IACrD;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/automatic-speech-recognition/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/chat-completion/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/chat-completion/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..3f6f888f2a3d197dd648129c1d2dde804f6c248f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/chat-completion/inference.d.ts @@ -0,0 +1,334 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Chat Completion Input. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface ChatCompletionInput { + /** + * Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing + * frequency in the text so far, + * decreasing the model's likelihood to repeat the same line verbatim. + */ + frequency_penalty?: number; + /** + * UNUSED + * Modify the likelihood of specified tokens appearing in the completion. Accepts a JSON + * object that maps tokens + * (specified by their token ID in the tokenizer) to an associated bias value from -100 to + * 100. Mathematically, + * the bias is added to the logits generated by the model prior to sampling. The exact + * effect will vary per model, + * but values between -1 and 1 should decrease or increase likelihood of selection; values + * like -100 or 100 should + * result in a ban or exclusive selection of the relevant token. + */ + logit_bias?: number[]; + /** + * Whether to return log probabilities of the output tokens or not. If true, returns the log + * probabilities of each + * output token returned in the content of message. + */ + logprobs?: boolean; + /** + * The maximum number of tokens that can be generated in the chat completion. + */ + max_tokens?: number; + /** + * A list of messages comprising the conversation so far. + */ + messages: ChatCompletionInputMessage[]; + /** + * [UNUSED] ID of the model to use. See the model endpoint compatibility table for details + * on which models work with the Chat API. + */ + model?: string; + /** + * UNUSED + * How many chat completion choices to generate for each input message. Note that you will + * be charged based on the + * number of generated tokens across all of the choices. Keep n as 1 to minimize costs. + */ + n?: number; + /** + * Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they + * appear in the text so far, + * increasing the model's likelihood to talk about new topics + */ + presence_penalty?: number; + /** + * Optional. Constrains effort on reasoning for reasoning models. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning. Common values: none, minimal, low, medium, high, xhigh. Support and defaults are provider and model-dependent. + */ + reasoning_effort?: string; + response_format?: ChatCompletionInputGrammarType; + seed?: number; + /** + * Up to 4 sequences where the API will stop generating further tokens. + */ + stop?: string[]; + stream?: boolean; + stream_options?: ChatCompletionInputStreamOptions; + /** + * What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the + * output more random, while + * lower values like 0.2 will make it more focused and deterministic. + * + * We generally recommend altering this or `top_p` but not both. + */ + temperature?: number; + tool_choice?: ChatCompletionInputToolChoice; + /** + * A prompt to be appended before the tools + */ + tool_prompt?: string; + /** + * A list of tools the model may call. Currently, only functions are supported as a tool. + * Use this to provide a list of + * functions the model may generate JSON inputs for. + */ + tools?: ChatCompletionInputTool[]; + /** + * An integer between 0 and 5 specifying the number of most likely tokens to return at each + * token position, each with + * an associated log probability. logprobs must be set to true if this parameter is used. + */ + top_logprobs?: number; + /** + * An alternative to sampling with temperature, called nucleus sampling, where the model + * considers the results of the + * tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% + * probability mass are considered. + */ + top_p?: number; + [property: string]: unknown; +} +export interface ChatCompletionInputMessage { + content?: ChatCompletionInputMessageContent; + name?: string; + role: string; + tool_calls?: ChatCompletionInputToolCall[]; + [property: string]: unknown; +} +export type ChatCompletionInputMessageContent = ChatCompletionInputMessageChunk[] | string; +export interface ChatCompletionInputMessageChunk { + image_url?: ChatCompletionInputURL; + text?: string; + type: ChatCompletionInputMessageChunkType; + [property: string]: unknown; +} +export interface ChatCompletionInputURL { + url: string; + [property: string]: unknown; +} +export type ChatCompletionInputMessageChunkType = "text" | "image_url"; +export interface ChatCompletionInputToolCall { + function: ChatCompletionInputFunctionDefinition; + id: string; + type: string; + [property: string]: unknown; +} +export interface ChatCompletionInputFunctionDefinition { + description?: string; + name: string; + parameters?: unknown; + [property: string]: unknown; +} +export interface ChatCompletionInputGrammarType { + json_schema?: ChatCompletionInputJSONSchemaConfig; + type: ChatCompletionInputGrammarTypeType; + [property: string]: unknown; +} +export interface ChatCompletionInputJSONSchemaConfig { + /** + * A description of what the response format is for, used by the model to determine how to + * respond in the format. + */ + description?: string; + /** + * The name of the response format. + */ + name: string; + /** + * The schema for the response format, described as a JSON Schema object. Learn how to build + * JSON schemas [here](https://json-schema.org/). + */ + schema?: { + [key: string]: unknown; + }; + /** + * Whether to enable strict schema adherence when generating the output. If set to true, the + * model will always follow the exact schema defined in the `schema` field. + */ + strict?: boolean; + [property: string]: unknown; +} +export type ChatCompletionInputGrammarTypeType = "text" | "json_schema" | "json_object"; +export interface ChatCompletionInputStreamOptions { + /** + * If set, an additional chunk will be streamed before the data: [DONE] message. The usage + * field on this chunk shows the token usage statistics for the entire request, and the + * choices field will always be an empty array. All other chunks will also include a usage + * field, but with a null value. + */ + include_usage?: boolean; + [property: string]: unknown; +} +/** + * + * + */ +export type ChatCompletionInputToolChoice = ChatCompletionInputToolChoiceEnum | ChatCompletionInputToolChoiceObject; +/** + * Means the model can pick between generating a message or calling one or more tools. + * + * Means the model will not call any tool and instead generates a message. + * + * Means the model must call one or more tools. + */ +export type ChatCompletionInputToolChoiceEnum = "auto" | "none" | "required"; +export interface ChatCompletionInputToolChoiceObject { + function: ChatCompletionInputFunctionName; + [property: string]: unknown; +} +export interface ChatCompletionInputFunctionName { + name: string; + [property: string]: unknown; +} +export interface ChatCompletionInputTool { + function: ChatCompletionInputFunctionDefinition; + type: string; + [property: string]: unknown; +} +/** + * Chat Completion Output. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface ChatCompletionOutput { + choices: ChatCompletionOutputComplete[]; + created: number; + id: string; + model: string; + system_fingerprint: string; + usage: ChatCompletionOutputUsage; + [property: string]: unknown; +} +export interface ChatCompletionOutputComplete { + finish_reason: string; + index: number; + logprobs?: ChatCompletionOutputLogprobs; + message: ChatCompletionOutputMessage; + [property: string]: unknown; +} +export interface ChatCompletionOutputLogprobs { + content: ChatCompletionOutputLogprob[]; + [property: string]: unknown; +} +export interface ChatCompletionOutputLogprob { + logprob: number; + token: string; + top_logprobs: ChatCompletionOutputTopLogprob[]; + [property: string]: unknown; +} +export interface ChatCompletionOutputTopLogprob { + logprob: number; + token: string; + [property: string]: unknown; +} +export interface ChatCompletionOutputMessage { + content?: string; + role: string; + tool_call_id?: string; + tool_calls?: ChatCompletionOutputToolCall[]; + [property: string]: unknown; +} +export interface ChatCompletionOutputToolCall { + function: ChatCompletionOutputFunctionDefinition; + id: string; + type: string; + [property: string]: unknown; +} +export interface ChatCompletionOutputFunctionDefinition { + arguments: string; + description?: string; + name: string; + [property: string]: unknown; +} +export interface ChatCompletionOutputUsage { + completion_tokens: number; + prompt_tokens: number; + total_tokens: number; + [property: string]: unknown; +} +/** + * Chat Completion Stream Output. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface ChatCompletionStreamOutput { + choices: ChatCompletionStreamOutputChoice[]; + created: number; + id: string; + model: string; + system_fingerprint: string; + usage?: ChatCompletionStreamOutputUsage; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputChoice { + delta: ChatCompletionStreamOutputDelta; + finish_reason?: string; + index: number; + logprobs?: ChatCompletionStreamOutputLogprobs; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputDelta { + content?: string; + role: string; + tool_call_id?: string; + tool_calls?: ChatCompletionStreamOutputDeltaToolCall[]; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputDeltaToolCall { + function: ChatCompletionStreamOutputFunction; + id: string; + index: number; + type: string; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputFunction { + arguments: string; + name?: string; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputLogprobs { + content: ChatCompletionStreamOutputLogprob[]; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputLogprob { + logprob: number; + token: string; + top_logprobs: ChatCompletionStreamOutputTopLogprob[]; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputTopLogprob { + logprob: number; + token: string; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputUsage { + completion_tokens: number; + prompt_tokens: number; + total_tokens: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/chat-completion/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/chat-completion/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c38ba2fd0d997d34b0bb5ecdeae8cda391c784c7 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/chat-completion/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/chat-completion/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IACnC;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,QAAQ,EAAE,0BAA0B,EAAE,CAAC;IACvC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,CAAC,CAAC,EAAE,MAAM,CAAC;IACX;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,8BAA8B,CAAC;IACjD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE,gCAAgC,CAAC;IAClD;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,6BAA6B,CAAC;IAC5C;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,KAAK,CAAC,EAAE,uBAAuB,EAAE,CAAC;IAClC;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,0BAA0B;IAC1C,OAAO,CAAC,EAAE,iCAAiC,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,2BAA2B,EAAE,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,iCAAiC,GAAG,+BAA+B,EAAE,GAAG,MAAM,CAAC;AAC3F,MAAM,WAAW,+BAA+B;IAC/C,SAAS,CAAC,EAAE,sBAAsB,CAAC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,mCAAmC,CAAC;IAC1C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,sBAAsB;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,mCAAmC,GAAG,MAAM,GAAG,WAAW,CAAC;AACvE,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,EAAE,qCAAqC,CAAC;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,qCAAqC;IACrD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,8BAA8B;IAC9C,WAAW,CAAC,EAAE,mCAAmC,CAAC;IAClD,IAAI,EAAE,kCAAkC,CAAC;IACzC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,mCAAmC;IACnD;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,MAAM,CAAC,EAAE;QACR,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,kCAAkC,GAAG,MAAM,GAAG,aAAa,GAAG,aAAa,CAAC;AACxF,MAAM,WAAW,gCAAgC;IAChD;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,MAAM,6BAA6B,GAAG,iCAAiC,GAAG,mCAAmC,CAAC;AACpH;;;;;;GAMG;AACH,MAAM,MAAM,iCAAiC,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,CAAC;AAC7E,MAAM,WAAW,mCAAmC;IACnD,QAAQ,EAAE,+BAA+B,CAAC;IAC1C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,qCAAqC,CAAC;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACpC,OAAO,EAAE,4BAA4B,EAAE,CAAC;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,yBAAyB,CAAC;IACjC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,4BAA4B,CAAC;IACxC,OAAO,EAAE,2BAA2B,CAAC;IACrC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,OAAO,EAAE,2BAA2B,EAAE,CAAC;IACvC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,8BAA8B,EAAE,CAAC;IAC/C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,8BAA8B;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,4BAA4B,EAAE,CAAC;IAC5C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,4BAA4B;IAC5C,QAAQ,EAAE,sCAAsC,CAAC;IACjD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,sCAAsC;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,yBAAyB;IACzC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,0BAA0B;IAC1C,OAAO,EAAE,gCAAgC,EAAE,CAAC;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,CAAC,EAAE,+BAA+B,CAAC;IACxC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,gCAAgC;IAChD,KAAK,EAAE,+BAA+B,CAAC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,kCAAkC,CAAC;IAC9C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,uCAAuC,EAAE,CAAC;IACvD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,uCAAuC;IACvD,QAAQ,EAAE,kCAAkC,CAAC;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,kCAAkC;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,kCAAkC;IAClD,OAAO,EAAE,iCAAiC,EAAE,CAAC;IAC7C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,iCAAiC;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,oCAAoC,EAAE,CAAC;IACrD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,oCAAoC;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/chat-completion/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/chat-completion/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/chat-completion/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..cb232576320cf1d06f658b28334cf5d6afa01a01 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/depth-estimation/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAiEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/data.js new file mode 100644 index 0000000000000000000000000000000000000000..0380c00f386dac40d17dbca8d6fa3110cd05bb09 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/data.js @@ -0,0 +1,67 @@ +const taskData = { + datasets: [ + { + description: "NYU Depth V2 Dataset: Video dataset containing both RGB and depth sensor data.", + id: "sayakpaul/nyu_depth_v2", + }, + { + description: "Monocular depth estimation benchmark based without noise and errors.", + id: "depth-anything/DA-2K", + }, + ], + demo: { + inputs: [ + { + filename: "depth-estimation-input.jpg", + type: "img", + }, + ], + outputs: [ + { + filename: "depth-estimation-output.png", + type: "img", + }, + ], + }, + metrics: [], + models: [ + { + description: "Cutting-edge depth estimation model.", + id: "depth-anything/Depth-Anything-V2-Large", + }, + { + description: "A strong monocular depth estimation model.", + id: "jingheya/lotus-depth-g-v1-0", + }, + { + description: "A depth estimation model that predicts depth in videos.", + id: "tencent/DepthCrafter", + }, + { + description: "A robust depth estimation model.", + id: "apple/DepthPro-hf", + }, + ], + spaces: [ + { + description: "An application that predicts the depth of an image and then reconstruct the 3D model as voxels.", + id: "radames/dpt-depth-estimation-3d-voxels", + }, + { + description: "An application for bleeding-edge depth estimation.", + id: "akhaliq/depth-pro", + }, + { + description: "An application on cutting-edge depth estimation in videos.", + id: "tencent/DepthCrafter", + }, + { + description: "A human-centric depth estimation application.", + id: "facebook/sapiens-depth", + }, + ], + summary: "Depth estimation is the task of predicting depth of the objects present in an image.", + widgetModels: [""], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..7585f1ac33bb2d5ea520295d56da1ca4face71fb --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/inference.d.ts @@ -0,0 +1,36 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Depth Estimation inference + */ +export interface DepthEstimationInput { + /** + * The input image data + */ + inputs: unknown; + /** + * Additional inference parameters for Depth Estimation + */ + parameters?: { + [key: string]: unknown; + }; + [property: string]: unknown; +} +/** + * Outputs of inference for the Depth Estimation task + */ +export interface DepthEstimationOutput { + /** + * The predicted depth as an image + */ + depth?: unknown; + /** + * The predicted depth as a tensor + */ + predicted_depth?: unknown; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..033a915d9d5a928ebb48b36b6eda9f5e7cd5e4fb --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/depth-estimation/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC;IAChB;;OAEG;IACH,UAAU,CAAC,EAAE;QACZ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/depth-estimation/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..9c729fbee845c5ce760e5b2345f751973770add5 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/document-question-answering/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAgFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/data.js new file mode 100644 index 0000000000000000000000000000000000000000..a876b23f8b75874f66fccedac26623fcf2f03766 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/data.js @@ -0,0 +1,78 @@ +const taskData = { + datasets: [ + { + description: "Largest document understanding dataset.", + id: "HuggingFaceM4/Docmatix", + }, + { + description: "Dataset from the 2020 DocVQA challenge. The documents are taken from the UCSF Industry Documents Library.", + id: "eliolio/docvqa", + }, + ], + demo: { + inputs: [ + { + label: "Question", + content: "What is the idea behind the consumer relations efficiency team?", + type: "text", + }, + { + filename: "document-question-answering-input.png", + type: "img", + }, + ], + outputs: [ + { + label: "Answer", + content: "Balance cost efficiency with quality customer service", + type: "text", + }, + ], + }, + metrics: [ + { + description: "The evaluation metric for the DocVQA challenge is the Average Normalized Levenshtein Similarity (ANLS). This metric is flexible to character regognition errors and compares the predicted answer with the ground truth answer.", + id: "anls", + }, + { + description: "Exact Match is a metric based on the strict character match of the predicted answer and the right answer. For answers predicted correctly, the Exact Match will be 1. Even if only one character is different, Exact Match will be 0", + id: "exact-match", + }, + ], + models: [ + { + description: "A robust document question answering model.", + id: "impira/layoutlm-document-qa", + }, + { + description: "A document question answering model specialized in invoices.", + id: "impira/layoutlm-invoices", + }, + { + description: "A special model for OCR-free document question answering.", + id: "microsoft/udop-large", + }, + { + description: "A powerful model for document question answering.", + id: "google/pix2struct-docvqa-large", + }, + ], + spaces: [ + { + description: "A robust document question answering application.", + id: "impira/docquery", + }, + { + description: "An application that can answer questions from invoices.", + id: "impira/invoices", + }, + { + description: "An application to compare different document question answering models.", + id: "merve/compare_docvqa_models", + }, + ], + summary: "Document Question Answering (also known as Document Visual Question Answering) is the task of answering questions on document images. Document question answering models take a (document, question) pair as input and return an answer in natural language. Models usually rely on multi-modal features, combining text, position of words (bounding-boxes) and image.", + widgetModels: ["impira/layoutlm-invoices"], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d0201e7778a3a2003841e54a7e944a2b59a46653 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/inference.d.ts @@ -0,0 +1,105 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Document Question Answering inference + */ +export interface DocumentQuestionAnsweringInput { + /** + * One (document, question) pair to answer + */ + inputs: DocumentQuestionAnsweringInputData; + /** + * Additional inference parameters for Document Question Answering + */ + parameters?: DocumentQuestionAnsweringParameters; + [property: string]: unknown; +} +/** + * One (document, question) pair to answer + */ +export interface DocumentQuestionAnsweringInputData { + /** + * The image on which the question is asked + */ + image: unknown; + /** + * A question to ask of the document + */ + question: string; + [property: string]: unknown; +} +/** + * Additional inference parameters for Document Question Answering + */ +export interface DocumentQuestionAnsweringParameters { + /** + * If the words in the document are too long to fit with the question for the model, it will + * be split in several chunks with some overlap. This argument controls the size of that + * overlap. + */ + doc_stride?: number; + /** + * Whether to accept impossible as an answer + */ + handle_impossible_answer?: boolean; + /** + * Language to use while running OCR. Defaults to english. + */ + lang?: string; + /** + * The maximum length of predicted answers (e.g., only answers with a shorter length are + * considered). + */ + max_answer_len?: number; + /** + * The maximum length of the question after tokenization. It will be truncated if needed. + */ + max_question_len?: number; + /** + * The maximum length of the total sentence (context + question) in tokens of each chunk + * passed to the model. The context will be split in several chunks (using doc_stride as + * overlap) if needed. + */ + max_seq_len?: number; + /** + * The number of answers to return (will be chosen by order of likelihood). Can return less + * than top_k answers if there are not enough options available within the context. + */ + top_k?: number; + /** + * A list of words and bounding boxes (normalized 0->1000). If provided, the inference will + * skip the OCR step and use the provided bounding boxes instead. + */ + word_boxes?: WordBox[]; + [property: string]: unknown; +} +export type WordBox = number[] | string; +export type DocumentQuestionAnsweringOutput = DocumentQuestionAnsweringOutputElement[]; +/** + * Outputs of inference for the Document Question Answering task + */ +export interface DocumentQuestionAnsweringOutputElement { + /** + * The answer to the question. + */ + answer: string; + /** + * The end word index of the answer (in the OCR’d version of the input or provided word + * boxes). + */ + end: number; + /** + * The probability associated to the answer. + */ + score: number; + /** + * The start word index of the answer (in the OCR’d version of the input or provided word + * boxes). + */ + start: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f86c5635adab67721cb8843206eb5fc48b4c7d0e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/document-question-answering/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC9C;;OAEG;IACH,MAAM,EAAE,kCAAkC,CAAC;IAC3C;;OAEG;IACH,UAAU,CAAC,EAAE,mCAAmC,CAAC;IACjD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,kCAAkC;IAClD;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,mCAAmC;IACnD;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,EAAE,CAAC;IACvB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,OAAO,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC;AACxC,MAAM,MAAM,+BAA+B,GAAG,sCAAsC,EAAE,CAAC;AACvF;;GAEG;AACH,MAAM,WAAW,sCAAsC;IACtD;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/document-question-answering/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..1b7c67323ed90db6335e841bb8cdb74d2f429d4c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/feature-extraction/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAoDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/data.js new file mode 100644 index 0000000000000000000000000000000000000000..1e1a529bbeae16e98d798116b828090632e1f214 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/data.js @@ -0,0 +1,53 @@ +const taskData = { + datasets: [ + { + description: "Wikipedia dataset containing cleaned articles of all languages. Can be used to train `feature-extraction` models.", + id: "wikipedia", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "India, officially the Republic of India, is a country in South Asia.", + type: "text", + }, + ], + outputs: [ + { + table: [ + ["Dimension 1", "Dimension 2", "Dimension 3"], + ["2.583383083343506", "2.757075071334839", "0.9023529887199402"], + ["8.29393482208252", "1.1071064472198486", "2.03399395942688"], + ["-0.7754912972450256", "-1.647324562072754", "-0.6113331913948059"], + ["0.07087723910808563", "1.5942802429199219", "1.4610432386398315"], + ], + type: "tabular", + }, + ], + }, + metrics: [], + models: [ + { + description: "A powerful feature extraction model for natural language processing tasks.", + id: "thenlper/gte-large", + }, + { + description: "A strong feature extraction model for retrieval.", + id: "Alibaba-NLP/gte-Qwen1.5-7B-instruct", + }, + ], + spaces: [ + { + description: "A leaderboard to rank text feature extraction models based on a benchmark.", + id: "mteb/leaderboard", + }, + { + description: "A leaderboard to rank best feature extraction models based on human feedback.", + id: "mteb/arena", + }, + ], + summary: "Feature extraction is the task of extracting features learnt in a model.", + widgetModels: ["facebook/bart-base"], +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..688961c27e3dc737f10f5c7751b906eb109b4252 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/inference.d.ts @@ -0,0 +1,42 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +export type FeatureExtractionOutput = Array; +/** + * Feature Extraction Input. + * + * Auto-generated from TEI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tei-import.ts. + */ +export interface FeatureExtractionInput { + /** + * The text or list of texts to embed. + */ + inputs: FeatureExtractionInputs; + normalize?: boolean; + /** + * The name of the prompt that should be used by for encoding. If not set, no prompt + * will be applied. + * + * Must be a key in the `sentence-transformers` configuration `prompts` dictionary. + * + * For example if ``prompt_name`` is "query" and the ``prompts`` is {"query": "query: ", + * ...}, + * then the sentence "What is the capital of France?" will be encoded as + * "query: What is the capital of France?" because the prompt text will be prepended before + * any text to encode. + */ + prompt_name?: string; + truncate?: boolean; + truncation_direction?: FeatureExtractionInputTruncationDirection; + [property: string]: unknown; +} +/** + * The text or list of texts to embed. + */ +export type FeatureExtractionInputs = string[] | string; +export type FeatureExtractionInputTruncationDirection = "left" | "right"; +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6c2bd1f9aef9f4084c503ce5754da17e463ea4e7 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/feature-extraction/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,MAAM,uBAAuB,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;AACtD;;;;;;GAMG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,MAAM,EAAE,uBAAuB,CAAC;IAChC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;;;;;;;;OAWG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,oBAAoB,CAAC,EAAE,yCAAyC,CAAC;IACjE,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,uBAAuB,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC;AACxD,MAAM,MAAM,yCAAyC,GAAG,MAAM,GAAG,OAAO,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/feature-extraction/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d8447494880d737be45833c566928770eced4b7f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/fill-mask/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA0Ef,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/data.js new file mode 100644 index 0000000000000000000000000000000000000000..c7a47ab802fde362ada7e10a0219520dfca35718 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/data.js @@ -0,0 +1,73 @@ +const taskData = { + datasets: [ + { + description: "A common dataset that is used to train models for many languages.", + id: "wikipedia", + }, + { + description: "A large English dataset with text crawled from the web.", + id: "c4", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "The barked at me", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "wolf", + score: 0.487, + }, + { + label: "dog", + score: 0.061, + }, + { + label: "cat", + score: 0.058, + }, + { + label: "fox", + score: 0.047, + }, + { + label: "squirrel", + score: 0.025, + }, + ], + }, + ], + }, + metrics: [ + { + description: "Cross Entropy is a metric that calculates the difference between two probability distributions. Each probability distribution is the distribution of predicted words", + id: "cross_entropy", + }, + { + description: "Perplexity is the exponential of the cross-entropy loss. It evaluates the probabilities assigned to the next word by the model. Lower perplexity indicates better performance", + id: "perplexity", + }, + ], + models: [ + { + description: "State-of-the-art masked language model.", + id: "answerdotai/ModernBERT-large", + }, + { + description: "A multilingual model trained on 100 languages.", + id: "FacebookAI/xlm-roberta-base", + }, + ], + spaces: [], + summary: "Masked language modeling is the task of masking some of the words in a sentence and predicting which words should replace those masks. These models are useful when we want to get a statistical understanding of the language in which the model is trained in.", + widgetModels: ["distilroberta-base"], + youtubeId: "mqElG5QJWUg", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2d695741a7232bad148db87e1e55cf9825d20dcd --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/inference.d.ts @@ -0,0 +1,61 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Fill Mask inference + */ +export interface FillMaskInput { + /** + * The text with masked tokens + */ + inputs: string; + /** + * Additional inference parameters for Fill Mask + */ + parameters?: FillMaskParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Fill Mask + */ +export interface FillMaskParameters { + /** + * When passed, the model will limit the scores to the passed targets instead of looking up + * in the whole vocabulary. If the provided targets are not in the model vocab, they will be + * tokenized and the first resulting token will be used (with a warning, and that might be + * slower). + */ + targets?: string[]; + /** + * When passed, overrides the number of predictions to return. + */ + top_k?: number; + [property: string]: unknown; +} +export type FillMaskOutput = FillMaskOutputElement[]; +/** + * Outputs of inference for the Fill Mask task + */ +export interface FillMaskOutputElement { + /** + * The corresponding probability + */ + score: number; + /** + * The corresponding input with the mask token prediction. + */ + sequence: string; + /** + * The predicted token id (to replace the masked one). + */ + token: number; + tokenStr: unknown; + /** + * The predicted token (to replace the masked one). + */ + token_str?: string; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5d530f0a2d16b0ef869050b3c6edf02eb4922ebf --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/fill-mask/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,kBAAkB,CAAC;IAChC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,cAAc,GAAG,qBAAqB,EAAE,CAAC;AACrD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/fill-mask/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f157fe0bf42e631cd4810f5d9d3b88a1bd8cd488 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAkFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..6e86c7b6103393827d793cecf8ebafbe9ccb541c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/data.js @@ -0,0 +1,83 @@ +const taskData = { + datasets: [ + { + // TODO write proper description + description: "Benchmark dataset used for image classification with images that belong to 100 classes.", + id: "cifar100", + }, + { + // TODO write proper description + description: "Dataset consisting of images of garments.", + id: "fashion_mnist", + }, + ], + demo: { + inputs: [ + { + filename: "image-classification-input.jpeg", + type: "img", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Egyptian cat", + score: 0.514, + }, + { + label: "Tabby cat", + score: 0.193, + }, + { + label: "Tiger cat", + score: 0.068, + }, + ], + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + description: "A strong image classification model.", + id: "google/vit-base-patch16-224", + }, + { + description: "A robust image classification model.", + id: "facebook/deit-base-distilled-patch16-224", + }, + { + description: "A strong image classification model.", + id: "facebook/convnext-large-224", + }, + ], + spaces: [ + { + description: "A leaderboard to evaluate different image classification models.", + id: "timm/leaderboard", + }, + ], + summary: "Image classification is the task of assigning a label or class to an entire image. Images are expected to have only one class for each image. Image classification models take an image as input and return a prediction about which class the image belongs to.", + widgetModels: ["google/vit-base-patch16-224"], + youtubeId: "tjAIM7BOYhw", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..cea2bf9efe0f9dfa738b46d40fb17dcaba022793 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/inference.d.ts @@ -0,0 +1,54 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image Classification inference + */ +export interface ImageClassificationInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Image Classification + */ + parameters?: ImageClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image Classification + */ +export interface ImageClassificationParameters { + /** + * The function to apply to the model outputs in order to retrieve the scores. + */ + function_to_apply?: ClassificationOutputTransform; + /** + * When specified, limits the output to the top K most probable classes. + */ + top_k?: number; + [property: string]: unknown; +} +/** + * The function to apply to the model outputs in order to retrieve the scores. + */ +export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none"; +export type ImageClassificationOutput = ImageClassificationOutputElement[]; +/** + * Outputs of inference for the Image Classification task + */ +export interface ImageClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..037c9227770f76072b14980b75b858c399c7a54a --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-classification/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACxC;;;OAGG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,6BAA6B,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC7C;;OAEG;IACH,iBAAiB,CAAC,EAAE,6BAA6B,CAAC;IAClD;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAC3E,MAAM,MAAM,yBAAyB,GAAG,gCAAgC,EAAE,CAAC;AAC3E;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-classification/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-feature-extraction/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-feature-extraction/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-feature-extraction/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-feature-extraction/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-feature-extraction/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..90bc944282b5d5de9e390cc55fa1f326f5f1b6bf --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-feature-extraction/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-feature-extraction/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA2Df,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-feature-extraction/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-feature-extraction/data.js new file mode 100644 index 0000000000000000000000000000000000000000..51a48eaff60c91e5c4c12f8db1d8761f501a98a1 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-feature-extraction/data.js @@ -0,0 +1,60 @@ +const taskData = { + datasets: [ + { + description: "ImageNet-1K is a image classification dataset in which images are used to train image-feature-extraction models.", + id: "imagenet-1k", + }, + ], + demo: { + inputs: [ + { + filename: "mask-generation-input.png", + type: "img", + }, + ], + outputs: [ + { + table: [ + ["Dimension 1", "Dimension 2", "Dimension 3"], + ["0.21236686408519745", "1.0919708013534546", "0.8512550592422485"], + ["0.809657871723175", "-0.18544459342956543", "-0.7851548194885254"], + ["1.3103108406066895", "-0.2479034662246704", "-0.9107287526130676"], + ["1.8536205291748047", "-0.36419737339019775", "0.09717650711536407"], + ], + type: "tabular", + }, + ], + }, + metrics: [], + models: [ + { + description: "A powerful image feature extraction model.", + id: "timm/vit_large_patch14_dinov2.lvd142m", + }, + { + description: "A strong image feature extraction model.", + id: "nvidia/MambaVision-T-1K", + }, + { + description: "A robust image feature extraction model.", + id: "facebook/dino-vitb16", + }, + { + description: "Cutting-edge image feature extraction model.", + id: "apple/aimv2-large-patch14-336-distilled", + }, + { + description: "Strong image feature extraction model that can be used on images and documents.", + id: "OpenGVLab/InternViT-6B-448px-V1-2", + }, + ], + spaces: [ + { + description: "A leaderboard to evaluate different image-feature-extraction models on classification performances", + id: "timm/leaderboard", + }, + ], + summary: "Image feature extraction is the task of extracting features learnt in a computer vision model.", + widgetModels: [], +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..23ff6576f4a0f6e8cee58c7500193af6426f7738 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-segmentation/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA8Ff,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/data.js new file mode 100644 index 0000000000000000000000000000000000000000..378202a2bc1bd8c4958972289cb9e763026d348c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/data.js @@ -0,0 +1,93 @@ +const taskData = { + datasets: [ + { + description: "Scene segmentation dataset.", + id: "scene_parse_150", + }, + ], + demo: { + inputs: [ + { + filename: "image-segmentation-input.jpeg", + type: "img", + }, + ], + outputs: [ + { + filename: "image-segmentation-output.png", + type: "img", + }, + ], + }, + metrics: [ + { + description: "Average Precision (AP) is the Area Under the PR Curve (AUC-PR). It is calculated for each semantic class separately", + id: "Average Precision", + }, + { + description: "Mean Average Precision (mAP) is the overall average of the AP values", + id: "Mean Average Precision", + }, + { + description: "Intersection over Union (IoU) is the overlap of segmentation masks. Mean IoU is the average of the IoU of all semantic classes", + id: "Mean Intersection over Union", + }, + { + description: "APα is the Average Precision at the IoU threshold of a α value, for example, AP50 and AP75", + id: "APα", + }, + ], + models: [ + { + // TO DO: write description + description: "Solid panoptic segmentation model trained on COCO.", + id: "tue-mps/coco_panoptic_eomt_large_640", + }, + { + description: "Background removal model.", + id: "briaai/RMBG-1.4", + }, + { + description: "A multipurpose image segmentation model for high resolution images.", + id: "ZhengPeng7/BiRefNet", + }, + { + description: "Powerful human-centric image segmentation model.", + id: "facebook/sapiens-seg-1b", + }, + { + description: "Panoptic segmentation model trained on the COCO (common objects) dataset.", + id: "facebook/mask2former-swin-large-coco-panoptic", + }, + ], + spaces: [ + { + description: "A semantic segmentation application that can predict unseen instances out of the box.", + id: "facebook/ov-seg", + }, + { + description: "One of the strongest segmentation applications.", + id: "jbrinkma/segment-anything", + }, + { + description: "A human-centric segmentation model.", + id: "facebook/sapiens-pose", + }, + { + description: "An instance segmentation application to predict neuronal cell types from microscopy images.", + id: "rashmi/sartorius-cell-instance-segmentation", + }, + { + description: "An application that segments videos.", + id: "ArtGAN/Segment-Anything-Video", + }, + { + description: "An panoptic segmentation application built for outdoor environments.", + id: "segments/panoptic-segment-anything", + }, + ], + summary: "Image Segmentation divides an image into segments where each pixel in the image is mapped to an object. This task has multiple variants such as instance segmentation, panoptic segmentation and semantic segmentation.", + widgetModels: ["nvidia/segformer-b0-finetuned-ade-512-512"], + youtubeId: "dKE8SIt9C-w", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..62893d683d831473f37662e89e8742a0582f6d4f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/inference.d.ts @@ -0,0 +1,68 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image Segmentation inference + */ +export interface ImageSegmentationInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Image Segmentation + */ + parameters?: ImageSegmentationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image Segmentation + */ +export interface ImageSegmentationParameters { + /** + * Threshold to use when turning the predicted masks into binary values. + */ + mask_threshold?: number; + /** + * Mask overlap threshold to eliminate small, disconnected segments. + */ + overlap_mask_area_threshold?: number; + /** + * Segmentation task to be performed, depending on model capabilities. + */ + subtask?: ImageSegmentationSubtask; + /** + * Probability threshold to filter out predicted masks. + */ + threshold?: number; + [property: string]: unknown; +} +/** + * Segmentation task to be performed, depending on model capabilities. + */ +export type ImageSegmentationSubtask = "instance" | "panoptic" | "semantic"; +export type ImageSegmentationOutput = ImageSegmentationOutputElement[]; +/** + * Outputs of inference for the Image Segmentation task + * + * A predicted mask / segment + */ +export interface ImageSegmentationOutputElement { + /** + * The label of the predicted segment. + */ + label: string; + /** + * The corresponding mask as a black-and-white image (base64-encoded). + */ + mask: string; + /** + * The score or confidence degree the model has. + */ + score?: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..002f53684e81edb0b5ac7dd0ed180a404781f3e2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-segmentation/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;;OAGG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,2BAA2B,CAAC;IACzC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC3C;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC;;OAEG;IACH,OAAO,CAAC,EAAE,wBAAwB,CAAC;IACnC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAC5E,MAAM,MAAM,uBAAuB,GAAG,8BAA8B,EAAE,CAAC;AACvE;;;;GAIG;AACH,MAAM,WAAW,8BAA8B;IAC9C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-segmentation/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..53a0eeeace77cd5956a7331e9ab7cb5c8f3fd1fb --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-text-to-image/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAiDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/data.js new file mode 100644 index 0000000000000000000000000000000000000000..a171a1390db0bdb9bdaeabe80ff411b211e00410 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/data.js @@ -0,0 +1,48 @@ +const taskData = { + datasets: [], + demo: { + inputs: [ + { + filename: "image-text-to-image-input.jpeg", + type: "img", + }, + { + label: "Input", + content: "A city above clouds, pastel colors, Victorian style", + type: "text", + }, + ], + outputs: [ + { + filename: "image-text-to-image-output.png", + type: "img", + }, + ], + }, + metrics: [ + { + description: "The Fréchet Inception Distance (FID) calculates the distance between distributions between synthetic and real samples. A lower FID score indicates better similarity between the distributions of real and generated images.", + id: "FID", + }, + { + description: "CLIP Score measures the similarity between the generated image and the text prompt using CLIP embeddings. A higher score indicates better alignment with the text prompt.", + id: "CLIP", + }, + ], + models: [ + { + description: "A powerful model for image-text-to-image generation.", + id: "black-forest-labs/FLUX.2-dev", + }, + ], + spaces: [ + { + description: "An application for image-text-to-image generation.", + id: "black-forest-labs/FLUX.2-dev", + }, + ], + summary: "Image-text-to-image models take an image and a text prompt as input and generate a new image based on the reference image and text instructions. These models are useful for image editing, style transfer, image variations, and guided image generation tasks.", + widgetModels: ["black-forest-labs/FLUX.2-dev"], + youtubeId: undefined, +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d6d2dab3a0b73c4600be87602613629b9d288570 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/inference.d.ts @@ -0,0 +1,76 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image Text To Image inference. Either inputs (image) or prompt (in parameters) + * must be provided, or both. + */ +export interface ImageTextToImageInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. Either this or prompt must be + * provided. + */ + inputs?: Blob; + /** + * Additional inference parameters for Image Text To Image + */ + parameters?: ImageTextToImageParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image Text To Image + */ +export interface ImageTextToImageParameters { + /** + * For diffusion models. A higher guidance scale value encourages the model to generate + * images closely linked to the text prompt at the expense of lower image quality. + */ + guidance_scale?: number; + /** + * One prompt to guide what NOT to include in image generation. + */ + negative_prompt?: string; + /** + * For diffusion models. The number of denoising steps. More denoising steps usually lead to + * a higher quality image at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * The text prompt to guide the image generation. Either this or inputs (image) must be + * provided. + */ + prompt?: string; + /** + * Seed for the random number generator. + */ + seed?: number; + /** + * The size in pixels of the output image. This parameter is only supported by some + * providers and for specific models. It will be ignored when unsupported. + */ + target_size?: TargetSize; + [property: string]: unknown; +} +/** + * The size in pixels of the output image. This parameter is only supported by some + * providers and for specific models. It will be ignored when unsupported. + */ +export interface TargetSize { + height: number; + width: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Image Text To Image task + */ +export interface ImageTextToImageOutput { + /** + * The generated image returned as raw bytes in the payload. + */ + image: unknown; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e0caa1a268aacfd85428f46c8615abf68d4f0606 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-text-to-image/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACrC;;;;OAIG;IACH,MAAM,CAAC,EAAE,IAAI,CAAC;IACd;;OAEG;IACH,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,0BAA0B;IAC1C;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,WAAW,UAAU;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-image/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-text/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-text/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-text/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-text/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-text/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..b600eb9ac66eb184e991671ecca7a6e8d4b6e8ab --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-text/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-text-to-text/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAiFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-text/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-text/data.js new file mode 100644 index 0000000000000000000000000000000000000000..7c3b498b4143f7f4b5b83001fe42402dbc91296e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-text/data.js @@ -0,0 +1,81 @@ +const taskData = { + datasets: [ + { + description: "Instructions composed of image and text.", + id: "liuhaotian/LLaVA-Instruct-150K", + }, + { + description: "Collection of image-text pairs on scientific topics.", + id: "DAMO-NLP-SG/multimodal_textbook", + }, + { + description: "A collection of datasets made for model fine-tuning.", + id: "HuggingFaceM4/the_cauldron", + }, + { + description: "Screenshots of websites with their HTML/CSS codes.", + id: "HuggingFaceM4/WebSight", + }, + ], + demo: { + inputs: [ + { + filename: "image-text-to-text-input.png", + type: "img", + }, + { + label: "Text Prompt", + content: "Describe the position of the bee in detail.", + type: "text", + }, + ], + outputs: [ + { + label: "Answer", + content: "The bee is sitting on a pink flower, surrounded by other flowers. The bee is positioned in the center of the flower, with its head and front legs sticking out.", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "Small and efficient yet powerful vision language model.", + id: "HuggingFaceTB/SmolVLM-Instruct", + }, + { + description: "Cutting-edge reasoning vision language model.", + id: "zai-org/GLM-4.5V", + }, + { + description: "Cutting-edge small vision language model to convert documents to text.", + id: "rednote-hilab/dots.ocr", + }, + { + description: "Small yet powerful model.", + id: "Qwen/Qwen2.5-VL-3B-Instruct", + }, + { + description: "Image-text-to-text model with agentic capabilities.", + id: "microsoft/Magma-8B", + }, + ], + spaces: [ + { + description: "Leaderboard to evaluate vision language models.", + id: "opencompass/open_vlm_leaderboard", + }, + { + description: "An application that compares object detection capabilities of different vision language models.", + id: "sergiopaniego/vlm_object_understanding", + }, + { + description: "An application to compare different OCR models.", + id: "prithivMLmods/Multimodal-OCR", + }, + ], + summary: "Image-text-to-text models take in an image and text prompt and output text. These models are also called vision-language models, or VLMs. The difference from image-to-text models is that these models take an additional text input, not restricting the model to certain use cases like image captioning, and may also be trained to accept a conversation as input.", + widgetModels: ["zai-org/GLM-4.5V"], + youtubeId: "IoGaGfU1CIg", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..4b9e273281951252ef914c15fc3f2df7c63fdfbd --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-text-to-video/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAiDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/data.js new file mode 100644 index 0000000000000000000000000000000000000000..3dc8ae377373ff0068695d7ebe9b64bd89ee6a8e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/data.js @@ -0,0 +1,48 @@ +const taskData = { + datasets: [], + demo: { + inputs: [ + { + filename: "image-text-to-video-input.jpg", + type: "img", + }, + { + label: "Input", + content: "Darth Vader is surfing on the waves.", + type: "text", + }, + ], + outputs: [ + { + filename: "image-text-to-video-output.gif", + type: "img", + }, + ], + }, + metrics: [ + { + description: "Frechet Video Distance uses a model that captures coherence for changes in frames and the quality of each frame. A smaller score indicates better video generation.", + id: "fvd", + }, + { + description: "CLIPSIM measures similarity between video frames and text using an image-text similarity model. A higher score indicates better video generation.", + id: "clipsim", + }, + ], + models: [ + { + description: "A powerful model for image-text-to-video generation.", + id: "Lightricks/LTX-Video", + }, + ], + spaces: [ + { + description: "An application for image-text-to-video generation.", + id: "Lightricks/ltx-video-distilled", + }, + ], + summary: "Image-text-to-video models take an reference image and a text instructions as and generate a video based on them. These models are useful for animating still images, creating dynamic content from static references, and generating videos with specific motion or transformation guidance.", + widgetModels: ["Lightricks/LTX-Video"], + youtubeId: undefined, +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..069db05bcf626591087431e5284525e1b189cc41 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/inference.d.ts @@ -0,0 +1,78 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image Text To Video inference. Either inputs (image) or prompt (in parameters) + * must be provided, or both. + */ +export interface ImageTextToVideoInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. Either this or prompt must be + * provided. + */ + inputs?: Blob; + /** + * Additional inference parameters for Image Text To Video + */ + parameters?: ImageTextToVideoParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image Text To Video + */ +export interface ImageTextToVideoParameters { + /** + * For diffusion models. A higher guidance scale value encourages the model to generate + * videos closely linked to the text prompt at the expense of lower image quality. + */ + guidance_scale?: number; + /** + * One prompt to guide what NOT to include in video generation. + */ + negative_prompt?: string; + /** + * The num_frames parameter determines how many video frames are generated. + */ + num_frames?: number; + /** + * The number of denoising steps. More denoising steps usually lead to a higher quality + * video at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * The text prompt to guide the video generation. Either this or inputs (image) must be + * provided. + */ + prompt?: string; + /** + * Seed for the random number generator. + */ + seed?: number; + /** + * The size in pixel of the output video frames. + */ + target_size?: TargetSize; + [property: string]: unknown; +} +/** + * The size in pixel of the output video frames. + */ +export interface TargetSize { + height: number; + width: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Image Text To Video task + */ +export interface ImageTextToVideoOutput { + /** + * The generated video returned as raw bytes in the payload. + */ + video: unknown; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..dddebe7ae1d8e967527b72e26ba1a9b5f1544c2c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-text-to-video/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACrC;;;;OAIG;IACH,MAAM,CAAC,EAAE,IAAI,CAAC;IACd;;OAEG;IACH,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,0BAA0B;IAC1C;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-text-to-video/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-3d/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-3d/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-3d/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-3d/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-3d/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..8c8b2f7cc89572fcc9c798d136a3bc242d3d98bd --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-3d/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-to-3d/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAsEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-3d/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-3d/data.js new file mode 100644 index 0000000000000000000000000000000000000000..2382dec02e4181451cd45d5c2c21be5ac9851702 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-3d/data.js @@ -0,0 +1,72 @@ +const taskData = { + datasets: [ + { + description: "A large dataset of over 10 million 3D objects.", + id: "allenai/objaverse-xl", + }, + { + description: "A dataset of isolated object images for evaluating image-to-3D models.", + id: "dylanebert/iso3d", + }, + ], + demo: { + inputs: [ + { + filename: "image-to-3d-image-input.png", + type: "img", + }, + ], + outputs: [ + { + label: "Result", + content: "image-to-3d-3d-output-filename.glb", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "Fast image-to-3D mesh model by Tencent.", + id: "TencentARC/InstantMesh", + }, + { + description: "3D world generation model.", + id: "tencent/HunyuanWorld-1", + }, + { + description: "A scaled up image-to-3D mesh model derived from TripoSR.", + id: "hwjiang/Real3D", + }, + { + description: "Consistent image-to-3d generation model.", + id: "stabilityai/stable-point-aware-3d", + }, + ], + spaces: [ + { + description: "Leaderboard to evaluate image-to-3D models.", + id: "dylanebert/3d-arena", + }, + { + description: "Image-to-3D demo with mesh outputs.", + id: "TencentARC/InstantMesh", + }, + { + description: "Image-to-3D demo.", + id: "stabilityai/stable-point-aware-3d", + }, + { + description: "Image-to-3D demo with mesh outputs.", + id: "hwjiang/Real3D", + }, + { + description: "Image-to-3D demo with splat outputs.", + id: "dylanebert/LGM-mini", + }, + ], + summary: "Image-to-3D models take in image input and produce 3D output.", + widgetModels: [], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..df6672e2ac2e6e69cdfdc88c3e5300bb1a441eb8 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-to-image/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA2Ff,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/data.js new file mode 100644 index 0000000000000000000000000000000000000000..0244d38c4c9be4d6f45298cdae10c76273f72f9b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/data.js @@ -0,0 +1,89 @@ +const taskData = { + datasets: [ + { + description: "Synthetic dataset, for image relighting", + id: "VIDIT", + }, + { + description: "Multiple images of celebrities, used for facial expression translation", + id: "huggan/CelebA-faces", + }, + { + description: "12M image-caption pairs.", + id: "Spawning/PD12M", + }, + ], + demo: { + inputs: [ + { + filename: "image-to-image-input.jpeg", + type: "img", + }, + ], + outputs: [ + { + filename: "image-to-image-output.png", + type: "img", + }, + ], + }, + isPlaceholder: false, + metrics: [ + { + description: "Peak Signal to Noise Ratio (PSNR) is an approximation of the human perception, considering the ratio of the absolute intensity with respect to the variations. Measured in dB, a high value indicates a high fidelity.", + id: "PSNR", + }, + { + description: "Structural Similarity Index (SSIM) is a perceptual metric which compares the luminance, contrast and structure of two images. The values of SSIM range between -1 and 1, and higher values indicate closer resemblance to the original image.", + id: "SSIM", + }, + { + description: "Inception Score (IS) is an analysis of the labels predicted by an image classification model when presented with a sample of the generated images.", + id: "IS", + }, + ], + models: [ + { + description: "An image-to-image model to improve image resolution.", + id: "fal/AuraSR-v2", + }, + { + description: "Powerful image editing model.", + id: "black-forest-labs/FLUX.1-Kontext-dev", + }, + { + description: "Virtual try-on model.", + id: "yisol/IDM-VTON", + }, + { + description: "Image re-lighting model.", + id: "kontext-community/relighting-kontext-dev-lora-v3", + }, + { + description: "Strong model for inpainting and outpainting.", + id: "black-forest-labs/FLUX.1-Fill-dev", + }, + { + description: "Strong model for image editing using depth maps.", + id: "black-forest-labs/FLUX.1-Depth-dev-lora", + }, + ], + spaces: [ + { + description: "Image editing application.", + id: "black-forest-labs/FLUX.1-Kontext-Dev", + }, + { + description: "Image relighting application.", + id: "lllyasviel/iclight-v2-vary", + }, + { + description: "An application for image upscaling.", + id: "jasperai/Flux.1-dev-Controlnet-Upscaler", + }, + ], + summary: "Image-to-image is the task of transforming an input image through a variety of possible manipulations and enhancements, such as super-resolution, image inpainting, colorization, and more.", + widgetModels: ["Qwen/Qwen-Image"], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..c5c7a1dc660bc32cac43d7d9298c4710c82fc186 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/inference.d.ts @@ -0,0 +1,69 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image To Image inference + */ +export interface ImageToImageInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Image To Image + */ + parameters?: ImageToImageParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image To Image + */ +export interface ImageToImageParameters { + /** + * For diffusion models. A higher guidance scale value encourages the model to generate + * images closely linked to the text prompt at the expense of lower image quality. + */ + guidance_scale?: number; + /** + * One prompt to guide what NOT to include in image generation. + */ + negative_prompt?: string; + /** + * For diffusion models. The number of denoising steps. More denoising steps usually lead to + * a higher quality image at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * The text prompt to guide the image generation. + */ + prompt?: string; + /** + * The size in pixels of the output image. This parameter is only supported by some + * providers and for specific models. It will be ignored when unsupported. + */ + target_size?: TargetSize; + [property: string]: unknown; +} +/** + * The size in pixels of the output image. This parameter is only supported by some + * providers and for specific models. It will be ignored when unsupported. + */ +export interface TargetSize { + height: number; + width: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Image To Image task + */ +export interface ImageToImageOutput { + /** + * The output image returned as raw bytes in the payload. + */ + image?: unknown; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6a40c1d8e247354fc5934c1b438879b3888011e6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-to-image/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;;OAGG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,WAAW,UAAU;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-image/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..aa4d932b0466467f4b50fda065faf7fd02b50f87 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-to-text/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAyDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/data.js new file mode 100644 index 0000000000000000000000000000000000000000..2f2ab05e7c120985ebb55428eb532c66250f88d1 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/data.js @@ -0,0 +1,58 @@ +const taskData = { + datasets: [ + { + // TODO write proper description + description: "Dataset from 12M image-text of Reddit", + id: "red_caps", + }, + { + // TODO write proper description + description: "Dataset from 3.3M images of Google", + id: "datasets/conceptual_captions", + }, + ], + demo: { + inputs: [ + { + filename: "savanna.jpg", + type: "img", + }, + ], + outputs: [ + { + label: "Detailed description", + content: "a herd of giraffes and zebras grazing in a field", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "Strong OCR model.", + id: "allenai/olmOCR-7B-0725", + }, + { + description: "Powerful image captioning model.", + id: "fancyfeast/llama-joycaption-beta-one-hf-llava", + }, + ], + spaces: [ + { + description: "SVG generator app from images.", + id: "multimodalart/OmniSVG-3B", + }, + { + description: "An application that converts documents to markdown.", + id: "numind/NuMarkdown-8B-Thinking", + }, + { + description: "An application that can caption images.", + id: "fancyfeast/joy-caption-beta-one", + }, + ], + summary: "Image to text models output a text from a given image. Image captioning or optical character recognition can be considered as the most common applications of image to text.", + widgetModels: ["Salesforce/blip-image-captioning-large"], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..9944a906d4fa9529af0e38ef2473969de421da4c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/inference.d.ts @@ -0,0 +1,135 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image To Text inference + */ +export interface ImageToTextInput { + /** + * The input image data + */ + inputs: Blob; + /** + * Additional inference parameters for Image To Text + */ + parameters?: ImageToTextParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image To Text + */ +export interface ImageToTextParameters { + /** + * Parametrization of the text generation process + */ + generation_parameters?: GenerationParameters; + /** + * The amount of maximum tokens to generate. + */ + max_new_tokens?: number; + [property: string]: unknown; +} +/** + * Parametrization of the text generation process + */ +export interface GenerationParameters { + /** + * Whether to use sampling instead of greedy decoding when generating new tokens. + */ + do_sample?: boolean; + /** + * Controls the stopping condition for beam-based methods. + */ + early_stopping?: EarlyStoppingUnion; + /** + * If set to float strictly between 0 and 1, only tokens with a conditional probability + * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + * Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + */ + epsilon_cutoff?: number; + /** + * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + * float strictly between 0 and 1, a token is only considered if it is greater than either + * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + * for more details. + */ + eta_cutoff?: number; + /** + * The maximum length (in tokens) of the generated text, including the input. + */ + max_length?: number; + /** + * The maximum number of tokens to generate. Takes precedence over max_length. + */ + max_new_tokens?: number; + /** + * The minimum length (in tokens) of the generated text, including the input. + */ + min_length?: number; + /** + * The minimum number of tokens to generate. Takes precedence over min_length. + */ + min_new_tokens?: number; + /** + * Number of groups to divide num_beams into in order to ensure diversity among different + * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + */ + num_beam_groups?: number; + /** + * Number of beams to use for beam search. + */ + num_beams?: number; + /** + * The value balances the model confidence and the degeneration penalty in contrastive + * search decoding. + */ + penalty_alpha?: number; + /** + * The value used to modulate the next token probabilities. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * If set to float < 1, only the smallest set of most probable tokens with probabilities + * that add up to top_p or higher are kept for generation. + */ + top_p?: number; + /** + * Local typicality measures how similar the conditional probability of predicting a target + * token next is to the expected conditional probability of predicting a random token next, + * given the partial text already generated. If set to float < 1, the smallest set of the + * most locally typical tokens with probabilities that add up to typical_p or higher are + * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + */ + typical_p?: number; + /** + * Whether the model should use the past last key/values attentions to speed up decoding + */ + use_cache?: boolean; + [property: string]: unknown; +} +/** + * Controls the stopping condition for beam-based methods. + */ +export type EarlyStoppingUnion = boolean | "never"; +/** + * Outputs of inference for the Image To Text task + */ +export interface ImageToTextOutput { + generatedText: unknown; + /** + * The generated text. + */ + generated_text?: string; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..b4d812f2f06619307ca7ebf5c53526474c51ab2d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-to-text/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,qBAAqB,CAAC,EAAE,oBAAoB,CAAC;IAC7C;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,kBAAkB,CAAC;IACpC;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,OAAO,GAAG,OAAO,CAAC;AACnD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,aAAa,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-text/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e5d0273d48ff15ed08a8d8bcd0d85efb1ba1a091 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-to-video/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAyHf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/data.js new file mode 100644 index 0000000000000000000000000000000000000000..153be3c7ca096b06a8bd1239dfb7390d99b8be3f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/data.js @@ -0,0 +1,117 @@ +const taskData = { + datasets: [ + { + description: "A benchmark dataset for reference image controlled video generation.", + id: "ali-vilab/VACE-Benchmark", + }, + { + description: "A dataset of video generation style preferences.", + id: "Rapidata/sora-video-generation-style-likert-scoring", + }, + { + description: "A dataset with videos and captions throughout the videos.", + id: "BestWishYsh/ChronoMagic", + }, + ], + demo: { + inputs: [ + { + filename: "image-to-video-input.jpg", + type: "img", + }, + { + label: "Optional Text Prompt", + content: "This penguin is dancing", + type: "text", + }, + ], + outputs: [ + { + filename: "image-to-video-output.gif", + type: "img", + }, + ], + }, + metrics: [ + { + description: "Fréchet Video Distance (FVD) measures the perceptual similarity between the distributions of generated videos and a set of real videos, assessing overall visual quality and temporal coherence of the video generated from an input image.", + id: "fvd", + }, + { + description: "CLIP Score measures the semantic similarity between a textual prompt (if provided alongside the input image) and the generated video frames. It evaluates how well the video's generated content and motion align with the textual description, conditioned on the initial image.", + id: "clip_score", + }, + { + description: "First Frame Fidelity, often measured using LPIPS (Learned Perceptual Image Patch Similarity), PSNR, or SSIM, quantifies how closely the first frame of the generated video matches the input conditioning image.", + id: "lpips", + }, + { + description: "Identity Preservation Score measures the consistency of identity (e.g., a person's face or a specific object's characteristics) between the input image and throughout the generated video frames, often calculated using features from specialized models like face recognition (e.g., ArcFace) or re-identification models.", + id: "identity_preservation", + }, + { + description: "Motion Score evaluates the quality, realism, and temporal consistency of motion in the video generated from a static image. This can be based on optical flow analysis (e.g., smoothness, magnitude), consistency of object trajectories, or specific motion plausibility assessments.", + id: "motion_score", + }, + ], + models: [ + { + description: "LTX-Video, a 13B parameter model for high quality video generation", + id: "Lightricks/LTX-Video-0.9.7-dev", + }, + { + description: "A 14B parameter model for reference image controlled video generation", + id: "Wan-AI/Wan2.1-VACE-14B", + }, + { + description: "An image-to-video generation model using FramePack F1 methodology with Hunyuan-DiT architecture", + id: "lllyasviel/FramePack_F1_I2V_HY_20250503", + }, + { + description: "A distilled version of the LTX-Video-0.9.7-dev model for faster inference", + id: "Lightricks/LTX-Video-0.9.7-distilled", + }, + { + description: "An image-to-video generation model by Skywork AI, 14B parameters, producing 720p videos.", + id: "Skywork/SkyReels-V2-I2V-14B-720P", + }, + { + description: "Image-to-video variant of Tencent's HunyuanVideo.", + id: "tencent/HunyuanVideo-I2V", + }, + { + description: "A 14B parameter model for 720p image-to-video generation by Wan-AI.", + id: "Wan-AI/Wan2.1-I2V-14B-720P", + }, + { + description: "A Diffusers version of the Wan2.1-I2V-14B-720P model for 720p image-to-video generation.", + id: "Wan-AI/Wan2.1-I2V-14B-720P-Diffusers", + }, + ], + spaces: [ + { + description: "An application to generate videos fast.", + id: "Lightricks/ltx-video-distilled", + }, + { + description: "Generate videos with the FramePack-F1", + id: "linoyts/FramePack-F1", + }, + { + description: "Generate videos with the FramePack", + id: "lisonallen/framepack-i2v", + }, + { + description: "Wan2.1 with CausVid LoRA", + id: "multimodalart/wan2-1-fast", + }, + { + description: "A demo for Stable Video Diffusion", + id: "multimodalart/stable-video-diffusion", + }, + ], + summary: "Image-to-video models take a still image as input and generate a video. These models can be guided by text prompts to influence the content and style of the output video.", + widgetModels: [], + youtubeId: undefined, +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..9d6c62757ec0115df3abf86575056526232d0a7b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/inference.d.ts @@ -0,0 +1,75 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image To Video inference + */ +export interface ImageToVideoInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Image To Video + */ + parameters?: ImageToVideoParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image To Video + */ +export interface ImageToVideoParameters { + /** + * For diffusion models. A higher guidance scale value encourages the model to generate + * videos closely linked to the text prompt at the expense of lower image quality. + */ + guidance_scale?: number; + /** + * One prompt to guide what NOT to include in video generation. + */ + negative_prompt?: string; + /** + * The num_frames parameter determines how many video frames are generated. + */ + num_frames?: number; + /** + * The number of denoising steps. More denoising steps usually lead to a higher quality + * video at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * The text prompt to guide the video generation. + */ + prompt?: string; + /** + * Seed for the random number generator. + */ + seed?: number; + /** + * The size in pixel of the output video frames. + */ + target_size?: TargetSize; + [property: string]: unknown; +} +/** + * The size in pixel of the output video frames. + */ +export interface TargetSize { + height: number; + width: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Image To Video task + */ +export interface ImageToVideoOutput { + /** + * The generated video returned as raw bytes in the payload. + */ + video: unknown; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..81dcff95e010b3e686370e299598f3966b398664 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-to-video/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;;OAGG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/image-to-video/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/index.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..a29860d78e4b4dd742787d45f069abfc39f3ade2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/index.d.ts @@ -0,0 +1,92 @@ +import type { PipelineType } from "../pipelines.js"; +export type * from "./audio-classification/inference.js"; +export type * from "./automatic-speech-recognition/inference.js"; +export type { ChatCompletionInput, ChatCompletionInputMessage, ChatCompletionInputMessageChunkType, ChatCompletionOutput, ChatCompletionOutputComplete, ChatCompletionOutputMessage, ChatCompletionStreamOutput, ChatCompletionStreamOutputChoice, ChatCompletionStreamOutputDelta, } from "./chat-completion/inference.js"; +export type * from "./document-question-answering/inference.js"; +export type * from "./feature-extraction/inference.js"; +export type * from "./fill-mask/inference.js"; +export type { ImageClassificationInput, ImageClassificationOutput, ImageClassificationOutputElement, ImageClassificationParameters, } from "./image-classification/inference.js"; +export type * from "./image-to-image/inference.js"; +export type { ImageToTextInput, ImageToTextOutput, ImageToTextParameters } from "./image-to-text/inference.js"; +export type * from "./image-segmentation/inference.js"; +export type { ImageToVideoInput, ImageToVideoOutput, ImageToVideoParameters } from "./image-to-video/inference.js"; +export type { ImageTextToImageInput, ImageTextToImageOutput, ImageTextToImageParameters, } from "./image-text-to-image/inference.js"; +export type { ImageTextToVideoInput, ImageTextToVideoOutput, ImageTextToVideoParameters, } from "./image-text-to-video/inference.js"; +export type * from "./object-detection/inference.js"; +export type * from "./depth-estimation/inference.js"; +export type * from "./question-answering/inference.js"; +export type * from "./sentence-similarity/inference.js"; +export type * from "./summarization/inference.js"; +export type * from "./table-question-answering/inference.js"; +export type { TextToImageInput, TextToImageOutput, TextToImageParameters } from "./text-to-image/inference.js"; +export type { TextToVideoParameters, TextToVideoOutput, TextToVideoInput } from "./text-to-video/inference.js"; +export type { TextToSpeechParameters, TextToSpeechInput, TextToSpeechOutput } from "./text-to-speech/inference.js"; +export type { TextToAudioInput, TextToAudioOutput, TextToAudioParameters } from "./text-to-audio/inference.js"; +export type * from "./token-classification/inference.js"; +export type { TranslationInput, TranslationOutput } from "./translation/inference.js"; +export type { ClassificationOutputTransform, TextClassificationInput, TextClassificationOutput, TextClassificationOutputElement, TextClassificationParameters, } from "./text-classification/inference.js"; +export type { TextGenerationOutputFinishReason, TextGenerationOutputPrefillToken, TextGenerationInput, TextGenerationOutput, TextGenerationOutputDetails, TextGenerationInputGenerateParameters, TextGenerationOutputBestOfSequence, TextGenerationOutputToken, TextGenerationStreamOutputStreamDetails, TextGenerationStreamOutput, } from "./text-generation/inference.js"; +export type * from "./video-classification/inference.js"; +export type * from "./visual-question-answering/inference.js"; +export type * from "./zero-shot-classification/inference.js"; +export type * from "./zero-shot-image-classification/inference.js"; +export type { BoundingBox, ZeroShotObjectDetectionInput, ZeroShotObjectDetectionOutput, ZeroShotObjectDetectionOutputElement, } from "./zero-shot-object-detection/inference.js"; +import type { ModelLibraryKey } from "../model-libraries.js"; +/** + * Model libraries compatible with each ML task + */ +export declare const TASKS_MODEL_LIBRARIES: Record; +export declare const TASKS_DATA: Record; +export interface ExampleRepo { + description: string; + id: string; +} +export type TaskDemoEntry = { + filename: string; + type: "audio"; +} | { + data: Array<{ + label: string; + score: number; + }>; + type: "chart"; +} | { + filename: string; + type: "img"; +} | { + table: string[][]; + type: "tabular"; +} | { + content: string; + label: string; + type: "text"; +} | { + text: string; + tokens: Array<{ + end: number; + start: number; + type: string; + }>; + type: "text-with-tokens"; +}; +export interface TaskDemo { + inputs: TaskDemoEntry[]; + outputs: TaskDemoEntry[]; +} +export interface TaskData { + datasets: ExampleRepo[]; + demo: TaskDemo; + id: PipelineType; + canonicalId?: PipelineType; + isPlaceholder?: boolean; + label: string; + libraries: ModelLibraryKey[]; + metrics: ExampleRepo[]; + models: ExampleRepo[]; + spaces: ExampleRepo[]; + summary: string; + widgetModels: string[]; + youtubeId?: string; +} +export type TaskDataCustom = Omit; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/index.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5f05d142917b923971ede20c87bb836a755daaad --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tasks/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAoDpD,mBAAmB,qCAAqC,CAAC;AACzD,mBAAmB,6CAA6C,CAAC;AACjE,YAAY,EACX,mBAAmB,EACnB,0BAA0B,EAC1B,mCAAmC,EACnC,oBAAoB,EACpB,4BAA4B,EAC5B,2BAA2B,EAC3B,0BAA0B,EAC1B,gCAAgC,EAChC,+BAA+B,GAC/B,MAAM,gCAAgC,CAAC;AACxC,mBAAmB,4CAA4C,CAAC;AAChE,mBAAmB,mCAAmC,CAAC;AACvD,mBAAmB,0BAA0B,CAAC;AAC9C,YAAY,EACX,wBAAwB,EACxB,yBAAyB,EACzB,gCAAgC,EAChC,6BAA6B,GAC7B,MAAM,qCAAqC,CAAC;AAC7C,mBAAmB,+BAA+B,CAAC;AACnD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAC/G,mBAAmB,mCAAmC,CAAC;AACvD,YAAY,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACnH,YAAY,EACX,qBAAqB,EACrB,sBAAsB,EACtB,0BAA0B,GAC1B,MAAM,oCAAoC,CAAC;AAC5C,YAAY,EACX,qBAAqB,EACrB,sBAAsB,EACtB,0BAA0B,GAC1B,MAAM,oCAAoC,CAAC;AAC5C,mBAAmB,iCAAiC,CAAC;AACrD,mBAAmB,iCAAiC,CAAC;AACrD,mBAAmB,mCAAmC,CAAC;AACvD,mBAAmB,oCAAoC,CAAC;AACxD,mBAAmB,8BAA8B,CAAC;AAClD,mBAAmB,yCAAyC,CAAC;AAC7D,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAC/G,YAAY,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAC/G,YAAY,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnH,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAC/G,mBAAmB,qCAAqC,CAAC;AACzD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AACtF,YAAY,EACX,6BAA6B,EAC7B,uBAAuB,EACvB,wBAAwB,EACxB,+BAA+B,EAC/B,4BAA4B,GAC5B,MAAM,oCAAoC,CAAC;AAC5C,YAAY,EACX,gCAAgC,EAChC,gCAAgC,EAChC,mBAAmB,EACnB,oBAAoB,EACpB,2BAA2B,EAC3B,qCAAqC,EACrC,kCAAkC,EAClC,yBAAyB,EACzB,uCAAuC,EACvC,0BAA0B,GAC1B,MAAM,gCAAgC,CAAC;AACxC,mBAAmB,qCAAqC,CAAC;AACzD,mBAAmB,0CAA0C,CAAC;AAC9D,mBAAmB,yCAAyC,CAAC;AAC7D,mBAAmB,+CAA+C,CAAC;AACnE,YAAY,EACX,WAAW,EACX,4BAA4B,EAC5B,6BAA6B,EAC7B,oCAAoC,GACpC,MAAM,2CAA2C,CAAC;AAEnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,YAAY,EAAE,eAAe,EAAE,CAkEzE,CAAC;AAoBF,eAAO,MAAM,UAAU,EAAE,MAAM,CAAC,YAAY,EAAE,QAAQ,GAAG,SAAS,CA0DxD,CAAC;AAEX,MAAM,WAAW,WAAW;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,EAAE,EAAE,MAAM,CAAC;CACX;AAED,MAAM,MAAM,aAAa,GACtB;IACA,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;CACb,GACD;IACA,IAAI,EAAE,KAAK,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACd,CAAC,CAAC;IACH,IAAI,EAAE,OAAO,CAAC;CACb,GACD;IACA,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,KAAK,CAAC;CACX,GACD;IACA,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC;IAClB,IAAI,EAAE,SAAS,CAAC;CACf,GACD;IACA,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACZ,GACD;IACA,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,KAAK,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;KACb,CAAC,CAAC;IACH,IAAI,EAAE,kBAAkB,CAAC;CACxB,CAAC;AAEL,MAAM,WAAW,QAAQ;IACxB,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,OAAO,EAAE,aAAa,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACxB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,IAAI,EAAE,QAAQ,CAAC;IACf,EAAE,EAAE,YAAY,CAAC;IACjB,WAAW,CAAC,EAAE,YAAY,CAAC;IAC3B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,eAAe,EAAE,CAAC;IAC7B,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,GAAG,WAAW,CAAC,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/index.js b/node_modules/@huggingface/tasks/dist/esm/tasks/index.js new file mode 100644 index 0000000000000000000000000000000000000000..db0af44e7f1f527b240c206bcd91430c05c2615a --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/index.js @@ -0,0 +1,195 @@ +import { PIPELINE_DATA } from "../pipelines.js"; +import anyToAny from "./any-to-any/data.js"; +import audioClassification from "./audio-classification/data.js"; +import audioTextToText from "./audio-text-to-text/data.js"; +import audioToAudio from "./audio-to-audio/data.js"; +import automaticSpeechRecognition from "./automatic-speech-recognition/data.js"; +import documentQuestionAnswering from "./document-question-answering/data.js"; +import featureExtraction from "./feature-extraction/data.js"; +import fillMask from "./fill-mask/data.js"; +import imageClassification from "./image-classification/data.js"; +import imageFeatureExtraction from "./image-feature-extraction/data.js"; +import imageToImage from "./image-to-image/data.js"; +import imageToText from "./image-to-text/data.js"; +import imageTextToText from "./image-text-to-text/data.js"; +import imageTextToImage from "./image-text-to-image/data.js"; +import imageTextToVideo from "./image-text-to-video/data.js"; +import imageSegmentation from "./image-segmentation/data.js"; +import imageToVideo from "./image-to-video/data.js"; +import maskGeneration from "./mask-generation/data.js"; +import objectDetection from "./object-detection/data.js"; +import depthEstimation from "./depth-estimation/data.js"; +import placeholder from "./placeholder/data.js"; +import reinforcementLearning from "./reinforcement-learning/data.js"; +import questionAnswering from "./question-answering/data.js"; +import sentenceSimilarity from "./sentence-similarity/data.js"; +import summarization from "./summarization/data.js"; +import tableQuestionAnswering from "./table-question-answering/data.js"; +import tabularClassification from "./tabular-classification/data.js"; +import tabularRegression from "./tabular-regression/data.js"; +import textToImage from "./text-to-image/data.js"; +import textToSpeech from "./text-to-speech/data.js"; +import tokenClassification from "./token-classification/data.js"; +import translation from "./translation/data.js"; +import textClassification from "./text-classification/data.js"; +import textGeneration from "./text-generation/data.js"; +import textRanking from "./text-ranking/data.js"; +import textToVideo from "./text-to-video/data.js"; +import unconditionalImageGeneration from "./unconditional-image-generation/data.js"; +import videoClassification from "./video-classification/data.js"; +import visualDocumentRetrieval from "./visual-document-retrieval/data.js"; +import visualQuestionAnswering from "./visual-question-answering/data.js"; +import zeroShotClassification from "./zero-shot-classification/data.js"; +import zeroShotImageClassification from "./zero-shot-image-classification/data.js"; +import zeroShotObjectDetection from "./zero-shot-object-detection/data.js"; +import imageTo3D from "./image-to-3d/data.js"; +import textTo3D from "./text-to-3d/data.js"; +import keypointDetection from "./keypoint-detection/data.js"; +import videoTextToText from "./video-text-to-text/data.js"; +import videoToVideo from "./video-to-video/data.js"; +/** + * Model libraries compatible with each ML task + */ +export const TASKS_MODEL_LIBRARIES = { + "audio-classification": ["speechbrain", "transformers", "transformers.js"], + "audio-to-audio": ["asteroid", "fairseq", "speechbrain"], + "automatic-speech-recognition": ["espnet", "nemo", "speechbrain", "transformers", "transformers.js"], + "audio-text-to-text": ["transformers"], + "depth-estimation": ["transformers", "transformers.js"], + "document-question-answering": ["transformers", "transformers.js"], + "feature-extraction": ["sentence-transformers", "transformers", "transformers.js"], + "fill-mask": ["transformers", "transformers.js"], + "graph-ml": ["transformers"], + "image-classification": ["keras", "timm", "transformers", "transformers.js"], + "image-feature-extraction": ["timm", "transformers"], + "image-segmentation": ["transformers", "transformers.js"], + "image-text-to-text": ["transformers"], + "image-text-to-image": ["diffusers"], + "image-text-to-video": ["diffusers"], + "image-to-image": ["diffusers", "transformers", "transformers.js"], + "image-to-text": ["transformers", "transformers.js"], + "image-to-video": ["diffusers"], + "keypoint-detection": ["transformers"], + "video-classification": ["transformers"], + "mask-generation": ["transformers"], + "multiple-choice": ["transformers"], + "object-detection": ["transformers", "transformers.js", "ultralytics"], + other: [], + "question-answering": ["adapter-transformers", "allennlp", "transformers", "transformers.js"], + robotics: [], + "reinforcement-learning": ["transformers", "stable-baselines3", "ml-agents", "sample-factory"], + "sentence-similarity": ["sentence-transformers", "spacy", "transformers.js"], + summarization: ["transformers", "transformers.js"], + "table-question-answering": ["transformers"], + "table-to-text": ["transformers"], + "tabular-classification": ["sklearn"], + "tabular-regression": ["sklearn"], + "tabular-to-text": ["transformers"], + "text-classification": ["adapter-transformers", "setfit", "spacy", "transformers", "transformers.js"], + "text-generation": ["transformers", "transformers.js"], + "text-ranking": ["sentence-transformers", "transformers"], + "text-retrieval": [], + "text-to-image": ["diffusers"], + "text-to-speech": ["espnet", "tensorflowtts", "transformers", "transformers.js"], + "text-to-audio": ["transformers", "transformers.js"], + "text-to-video": ["diffusers"], + "time-series-forecasting": [], + "token-classification": [ + "adapter-transformers", + "flair", + "spacy", + "span-marker", + "stanza", + "transformers", + "transformers.js", + ], + translation: ["transformers", "transformers.js"], + "unconditional-image-generation": ["diffusers"], + "video-text-to-text": ["transformers"], + "visual-question-answering": ["transformers", "transformers.js"], + "voice-activity-detection": [], + "zero-shot-classification": ["transformers", "transformers.js"], + "zero-shot-image-classification": ["transformers", "transformers.js"], + "zero-shot-object-detection": ["transformers", "transformers.js"], + "text-to-3d": ["diffusers"], + "image-to-3d": ["diffusers"], + "any-to-any": ["transformers"], + "visual-document-retrieval": ["transformers"], + "video-to-video": ["diffusers"], +}; +/** + * Return the whole TaskData object for a certain task. + * If the partialTaskData argument is left undefined, + * the default placeholder data will be used. + */ +function getData(type, partialTaskData = placeholder) { + return { + ...partialTaskData, + id: type, + label: PIPELINE_DATA[type].name, + libraries: TASKS_MODEL_LIBRARIES[type], + }; +} +// To make comparisons easier, task order is the same as in const.ts +// Tasks set to undefined won't have an associated task page. +// Tasks that call getData() without the second argument will +// have a "placeholder" page. +export const TASKS_DATA = { + "any-to-any": getData("any-to-any", anyToAny), + "audio-classification": getData("audio-classification", audioClassification), + "audio-to-audio": getData("audio-to-audio", audioToAudio), + "audio-text-to-text": getData("audio-text-to-text", audioTextToText), + "automatic-speech-recognition": getData("automatic-speech-recognition", automaticSpeechRecognition), + "depth-estimation": getData("depth-estimation", depthEstimation), + "document-question-answering": getData("document-question-answering", documentQuestionAnswering), + "visual-document-retrieval": getData("visual-document-retrieval", visualDocumentRetrieval), + "feature-extraction": getData("feature-extraction", featureExtraction), + "fill-mask": getData("fill-mask", fillMask), + "graph-ml": undefined, + "image-classification": getData("image-classification", imageClassification), + "image-feature-extraction": getData("image-feature-extraction", imageFeatureExtraction), + "image-segmentation": getData("image-segmentation", imageSegmentation), + "image-to-image": getData("image-to-image", imageToImage), + "image-text-to-text": getData("image-text-to-text", imageTextToText), + "image-text-to-image": getData("image-text-to-image", imageTextToImage), + "image-text-to-video": getData("image-text-to-video", imageTextToVideo), + "image-to-text": getData("image-to-text", imageToText), + "image-to-video": getData("image-to-video", imageToVideo), + "keypoint-detection": getData("keypoint-detection", keypointDetection), + "mask-generation": getData("mask-generation", maskGeneration), + "multiple-choice": undefined, + "object-detection": getData("object-detection", objectDetection), + "video-classification": getData("video-classification", videoClassification), + other: undefined, + "question-answering": getData("question-answering", questionAnswering), + "reinforcement-learning": getData("reinforcement-learning", reinforcementLearning), + robotics: undefined, + "sentence-similarity": getData("sentence-similarity", sentenceSimilarity), + summarization: getData("summarization", summarization), + "table-question-answering": getData("table-question-answering", tableQuestionAnswering), + "table-to-text": undefined, + "tabular-classification": getData("tabular-classification", tabularClassification), + "tabular-regression": getData("tabular-regression", tabularRegression), + "tabular-to-text": undefined, + "text-classification": getData("text-classification", textClassification), + "text-generation": getData("text-generation", textGeneration), + "text-ranking": getData("text-ranking", textRanking), + "text-retrieval": undefined, + "text-to-image": getData("text-to-image", textToImage), + "text-to-speech": getData("text-to-speech", textToSpeech), + "text-to-audio": undefined, + "text-to-video": getData("text-to-video", textToVideo), + "time-series-forecasting": undefined, + "token-classification": getData("token-classification", tokenClassification), + translation: getData("translation", translation), + "unconditional-image-generation": getData("unconditional-image-generation", unconditionalImageGeneration), + "video-text-to-text": getData("video-text-to-text", videoTextToText), + "video-to-video": getData("video-to-video", videoToVideo), + "visual-question-answering": getData("visual-question-answering", visualQuestionAnswering), + "voice-activity-detection": undefined, + "zero-shot-classification": getData("zero-shot-classification", zeroShotClassification), + "zero-shot-image-classification": getData("zero-shot-image-classification", zeroShotImageClassification), + "zero-shot-object-detection": getData("zero-shot-object-detection", zeroShotObjectDetection), + "text-to-3d": getData("text-to-3d", textTo3D), + "image-to-3d": getData("image-to-3d", imageTo3D), +}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/keypoint-detection/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/keypoint-detection/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/keypoint-detection/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/keypoint-detection/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/keypoint-detection/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..eb823f21e8304e3e89965c2360ab9594e5d59139 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/keypoint-detection/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/keypoint-detection/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAqDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/keypoint-detection/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/keypoint-detection/data.js new file mode 100644 index 0000000000000000000000000000000000000000..667a7c1e36e849486f73ebcc3dddc84c72d1c56e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/keypoint-detection/data.js @@ -0,0 +1,55 @@ +const taskData = { + datasets: [ + { + description: "A dataset of hand keypoints of over 500k examples.", + id: "Vincent-luo/hagrid-mediapipe-hands", + }, + ], + demo: { + inputs: [ + { + filename: "keypoint-detection-input.png", + type: "img", + }, + ], + outputs: [ + { + filename: "keypoint-detection-output.png", + type: "img", + }, + ], + }, + metrics: [], + models: [ + { + description: "A robust keypoint detection model.", + id: "magic-leap-community/superpoint", + }, + { + description: "A robust keypoint matching model.", + id: "magic-leap-community/superglue_outdoor", + }, + { + description: "Strong keypoint detection model used to detect human pose.", + id: "qualcomm/RTMPose-Body2d", + }, + { + description: "Powerful keypoint matching model.", + id: "ETH-CVG/lightglue_disk", + }, + ], + spaces: [ + { + description: "An application that detects hand keypoints in real-time.", + id: "datasciencedojo/Hand-Keypoint-Detection-Realtime", + }, + { + description: "An application for keypoint detection and matching.", + id: "ETH-CVG/LightGlue", + }, + ], + summary: "Keypoint detection is the task of identifying meaningful distinctive points or features in an image.", + widgetModels: [], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/mask-generation/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/mask-generation/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/mask-generation/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/mask-generation/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/mask-generation/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..4c15a98fdabb31e019d38076d691da7391cefa47 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/mask-generation/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/mask-generation/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAgEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/mask-generation/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/mask-generation/data.js new file mode 100644 index 0000000000000000000000000000000000000000..2013d36d168346be75ffdb1651778dd4df3abfb5 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/mask-generation/data.js @@ -0,0 +1,64 @@ +const taskData = { + datasets: [ + { + description: "Widely used benchmark dataset for multiple Vision tasks.", + id: "merve/coco2017", + }, + { + description: "Medical Imaging dataset of the Human Brain for segmentation and mask generating tasks", + id: "rocky93/BraTS_segmentation", + }, + ], + demo: { + inputs: [ + { + filename: "mask-generation-input.png", + type: "img", + }, + ], + outputs: [ + { + filename: "mask-generation-output.png", + type: "img", + }, + ], + }, + metrics: [ + { + description: "IoU is used to measure the overlap between predicted mask and the ground truth mask.", + id: "Intersection over Union (IoU)", + }, + ], + models: [ + { + description: "Small yet powerful mask generation model.", + id: "Zigeng/SlimSAM-uniform-50", + }, + { + description: "Very strong mask generation model.", + id: "facebook/sam2-hiera-large", + }, + ], + spaces: [ + { + description: "An application that combines a mask generation model with a zero-shot object detection model for text-guided image segmentation.", + id: "merve/OWLSAM2", + }, + { + description: "An application that compares the performance of a large and a small mask generation model.", + id: "merve/slimsam", + }, + { + description: "An application based on an improved mask generation model.", + id: "SkalskiP/segment-anything-model-2", + }, + { + description: "An application to remove objects from videos using mask generation models.", + id: "SkalskiP/SAM_and_ProPainter", + }, + ], + summary: "Mask generation is the task of generating masks that identify a specific object or region of interest in a given image. Masks are often used in segmentation tasks, where they provide a precise way to isolate the object of interest for further processing or analysis.", + widgetModels: [], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..628b2c78e9a1f523dcab289b02218d52718855ac --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/object-detection/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAqFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/data.js new file mode 100644 index 0000000000000000000000000000000000000000..2722a892f542f7214b9d1080c0fc851c21b22287 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/data.js @@ -0,0 +1,84 @@ +const taskData = { + datasets: [ + { + description: "Widely used benchmark dataset for multiple vision tasks.", + id: "merve/coco2017", + }, + { + description: "Multi-task computer vision benchmark.", + id: "merve/pascal-voc", + }, + ], + demo: { + inputs: [ + { + filename: "object-detection-input.jpg", + type: "img", + }, + ], + outputs: [ + { + filename: "object-detection-output.jpg", + type: "img", + }, + ], + }, + metrics: [ + { + description: "The Average Precision (AP) metric is the Area Under the PR Curve (AUC-PR). It is calculated for each class separately", + id: "Average Precision", + }, + { + description: "The Mean Average Precision (mAP) metric is the overall average of the AP values", + id: "Mean Average Precision", + }, + { + description: "The APα metric is the Average Precision at the IoU threshold of a α value, for example, AP50 and AP75", + id: "APα", + }, + ], + models: [ + { + description: "Solid object detection model pre-trained on the COCO 2017 dataset.", + id: "facebook/detr-resnet-50", + }, + { + description: "Accurate object detection model.", + id: "IDEA-Research/dab-detr-resnet-50", + }, + { + description: "Fast and accurate object detection model.", + id: "PekingU/rtdetr_v2_r50vd", + }, + { + description: "Object detection model for low-lying objects.", + id: "StephanST/WALDO30", + }, + ], + spaces: [ + { + description: "Real-time object detection demo.", + id: "Roboflow/RF-DETR", + }, + { + description: "An application that contains various object detection models to try from.", + id: "Gradio-Blocks/Object-Detection-With-DETR-and-YOLOS", + }, + { + description: "A cutting-edge object detection application.", + id: "sunsmarterjieleaf/yolov12", + }, + { + description: "An object tracking, segmentation and inpainting application.", + id: "VIPLab/Track-Anything", + }, + { + description: "Very fast object tracking application based on object detection.", + id: "merve/RT-DETR-tracking-coco", + }, + ], + summary: "Object Detection models allow users to identify objects of certain defined classes. Object detection models receive an image as input and output the images with bounding boxes and labels on detected objects.", + widgetModels: ["facebook/detr-resnet-50"], + youtubeId: "WdAeKSOpxhw", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..14661180189671f77efac91846a902f1f85d91a9 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/inference.d.ts @@ -0,0 +1,74 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Object Detection inference + */ +export interface ObjectDetectionInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Object Detection + */ + parameters?: ObjectDetectionParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Object Detection + */ +export interface ObjectDetectionParameters { + /** + * The probability necessary to make a prediction. + */ + threshold?: number; + [property: string]: unknown; +} +/** + * The predicted bounding box. Coordinates are relative to the top left corner of the input + * image. + */ +export interface BoundingBox { + /** + * The x-coordinate of the bottom-right corner of the bounding box. + */ + xmax: number; + /** + * The x-coordinate of the top-left corner of the bounding box. + */ + xmin: number; + /** + * The y-coordinate of the bottom-right corner of the bounding box. + */ + ymax: number; + /** + * The y-coordinate of the top-left corner of the bounding box. + */ + ymin: number; + [property: string]: unknown; +} +export type ObjectDetectionOutput = ObjectDetectionOutputElement[]; +/** + * Outputs of inference for the Object Detection task + */ +export interface ObjectDetectionOutputElement { + /** + * The predicted bounding box. Coordinates are relative to the top left corner of the input + * image. + */ + box: BoundingBox; + /** + * The predicted label for the bounding box. + */ + label: string; + /** + * The associated score / probability. + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..64ac1c743fb4d62b1c12cceb58c692b409d268e1 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/object-detection/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;;OAGG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,CAAC,EAAE,yBAAyB,CAAC;IACvC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACzC;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC3B;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,qBAAqB,GAAG,4BAA4B,EAAE,CAAC;AACnE;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC5C;;;OAGG;IACH,GAAG,EAAE,WAAW,CAAC;IACjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/object-detection/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/placeholder/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/placeholder/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/placeholder/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/placeholder/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/placeholder/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5db21ad2a46706da5b2235bfb24ee2a5a9cc0757 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/placeholder/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/placeholder/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAgBf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/placeholder/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/placeholder/data.js new file mode 100644 index 0000000000000000000000000000000000000000..6ce240dc187bfc5153e73c451aa90fef21e42ad5 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/placeholder/data.js @@ -0,0 +1,18 @@ +const taskData = { + datasets: [], + demo: { + inputs: [], + outputs: [], + }, + isPlaceholder: true, + metrics: [], + models: [], + spaces: [], + summary: "", + widgetModels: [], + youtubeId: undefined, + /// If this is a subtask, link to the most general task ID + /// (eg, text-generation is the canonical ID of text-simplification) + canonicalId: undefined, +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..796067455ff80c17a422ec70958bc4311e8124e3 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/question-answering/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAsEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/data.js new file mode 100644 index 0000000000000000000000000000000000000000..558d87c71d03d87bf4c0080f5a411e2df8c8adfd --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/data.js @@ -0,0 +1,69 @@ +const taskData = { + datasets: [ + { + // TODO write proper description + description: "A famous question answering dataset based on English articles from Wikipedia.", + id: "squad_v2", + }, + { + // TODO write proper description + description: "A dataset of aggregated anonymized actual queries issued to the Google search engine.", + id: "natural_questions", + }, + ], + demo: { + inputs: [ + { + label: "Question", + content: "Which name is also used to describe the Amazon rainforest in English?", + type: "text", + }, + { + label: "Context", + content: "The Amazon rainforest, also known in English as Amazonia or the Amazon Jungle", + type: "text", + }, + ], + outputs: [ + { + label: "Answer", + content: "Amazonia", + type: "text", + }, + ], + }, + metrics: [ + { + description: "Exact Match is a metric based on the strict character match of the predicted answer and the right answer. For answers predicted correctly, the Exact Match will be 1. Even if only one character is different, Exact Match will be 0", + id: "exact-match", + }, + { + description: " The F1-Score metric is useful if we value both false positives and false negatives equally. The F1-Score is calculated on each word in the predicted sequence against the correct answer", + id: "f1", + }, + ], + models: [ + { + description: "A robust baseline model for most question answering domains.", + id: "deepset/roberta-base-squad2", + }, + { + description: "Small yet robust model that can answer questions.", + id: "distilbert/distilbert-base-cased-distilled-squad", + }, + { + description: "A special model that can answer questions from tables.", + id: "google/tapas-base-finetuned-wtq", + }, + ], + spaces: [ + { + description: "An application that can answer a long question from Wikipedia.", + id: "deepset/wikipedia-assistant", + }, + ], + summary: "Question Answering models can retrieve the answer to a question from a given text, which is useful for searching for an answer in a document. Some question answering models can generate answers without context!", + widgetModels: ["deepset/roberta-base-squad2"], + youtubeId: "ajPx5LwJD-I", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..0956db6c47a1bb07ee6c2bd1fd5603b4d66b406f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/inference.d.ts @@ -0,0 +1,98 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Question Answering inference + */ +export interface QuestionAnsweringInput { + /** + * One (context, question) pair to answer + */ + inputs: QuestionAnsweringInputData; + /** + * Additional inference parameters for Question Answering + */ + parameters?: QuestionAnsweringParameters; + [property: string]: unknown; +} +/** + * One (context, question) pair to answer + */ +export interface QuestionAnsweringInputData { + /** + * The context to be used for answering the question + */ + context: string; + /** + * The question to be answered + */ + question: string; + [property: string]: unknown; +} +/** + * Additional inference parameters for Question Answering + */ +export interface QuestionAnsweringParameters { + /** + * Attempts to align the answer to real words. Improves quality on space separated + * languages. Might hurt on non-space-separated languages (like Japanese or Chinese) + */ + align_to_words?: boolean; + /** + * If the context is too long to fit with the question for the model, it will be split in + * several chunks with some overlap. This argument controls the size of that overlap. + */ + doc_stride?: number; + /** + * Whether to accept impossible as an answer. + */ + handle_impossible_answer?: boolean; + /** + * The maximum length of predicted answers (e.g., only answers with a shorter length are + * considered). + */ + max_answer_len?: number; + /** + * The maximum length of the question after tokenization. It will be truncated if needed. + */ + max_question_len?: number; + /** + * The maximum length of the total sentence (context + question) in tokens of each chunk + * passed to the model. The context will be split in several chunks (using docStride as + * overlap) if needed. + */ + max_seq_len?: number; + /** + * The number of answers to return (will be chosen by order of likelihood). Note that we + * return less than topk answers if there are not enough options available within the + * context. + */ + top_k?: number; + [property: string]: unknown; +} +export type QuestionAnsweringOutput = QuestionAnsweringOutputElement[]; +/** + * Outputs of inference for the Question Answering task + */ +export interface QuestionAnsweringOutputElement { + /** + * The answer to the question. + */ + answer: string; + /** + * The character position in the input where the answer ends. + */ + end: number; + /** + * The probability associated to the answer. + */ + score: number; + /** + * The character position in the input where the answer begins. + */ + start: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..bf237c1fb8f8f2de5f02091faf25bed254771a75 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/question-answering/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,MAAM,EAAE,0BAA0B,CAAC;IACnC;;OAEG;IACH,UAAU,CAAC,EAAE,2BAA2B,CAAC;IACzC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,0BAA0B;IAC1C;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC3C;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,uBAAuB,GAAG,8BAA8B,EAAE,CAAC;AACvE;;GAEG;AACH,MAAM,WAAW,8BAA8B;IAC9C;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/question-answering/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/reinforcement-learning/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/reinforcement-learning/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/reinforcement-learning/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/reinforcement-learning/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/reinforcement-learning/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..da6859fc13463382ff95b67b4ee618679d06845c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/reinforcement-learning/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/reinforcement-learning/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAsEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/reinforcement-learning/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/reinforcement-learning/data.js new file mode 100644 index 0000000000000000000000000000000000000000..54e250c9ccb5507b7941e74eaf3658f40b2578ca --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/reinforcement-learning/data.js @@ -0,0 +1,67 @@ +const taskData = { + datasets: [ + { + description: "A curation of widely used datasets for Data Driven Deep Reinforcement Learning (D4RL)", + id: "edbeeching/decision_transformer_gym_replay", + }, + ], + demo: { + inputs: [ + { + label: "State", + content: "Red traffic light, pedestrians are about to pass.", + type: "text", + }, + ], + outputs: [ + { + label: "Action", + content: "Stop the car.", + type: "text", + }, + { + label: "Next State", + content: "Yellow light, pedestrians have crossed.", + type: "text", + }, + ], + }, + metrics: [ + { + description: "Accumulated reward across all time steps discounted by a factor that ranges between 0 and 1 and determines how much the agent optimizes for future relative to immediate rewards. Measures how good is the policy ultimately found by a given algorithm considering uncertainty over the future.", + id: "Discounted Total Reward", + }, + { + description: "Average return obtained after running the policy for a certain number of evaluation episodes. As opposed to total reward, mean reward considers how much reward a given algorithm receives while learning.", + id: "Mean Reward", + }, + { + description: "Measures how good a given algorithm is after a predefined time. Some algorithms may be guaranteed to converge to optimal behavior across many time steps. However, an agent that reaches an acceptable level of optimality after a given time horizon may be preferable to one that ultimately reaches optimality but takes a long time.", + id: "Level of Performance After Some Time", + }, + ], + models: [ + { + description: "A Reinforcement Learning model trained on expert data from the Gym Hopper environment", + id: "edbeeching/decision-transformer-gym-hopper-expert", + }, + { + description: "A PPO agent playing seals/CartPole-v0 using the stable-baselines3 library and the RL Zoo.", + id: "HumanCompatibleAI/ppo-seals-CartPole-v0", + }, + ], + spaces: [ + { + description: "An application for a cute puppy agent learning to catch a stick.", + id: "ThomasSimonini/Huggy", + }, + { + description: "An application to play Snowball Fight with a reinforcement learning agent.", + id: "ThomasSimonini/SnowballFight", + }, + ], + summary: "Reinforcement learning is the computational approach of learning from action by interacting with an environment through trial and error and receiving rewards (negative or positive) as feedback", + widgetModels: [], + youtubeId: "q0BiUn5LiBc", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d457bc40bbb8d21b76a24e583381c7ea694b063e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/sentence-similarity/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAoGf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/data.js new file mode 100644 index 0000000000000000000000000000000000000000..3dca75956ece6eee1ee2275a3a7c82b4e96f3a88 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/data.js @@ -0,0 +1,97 @@ +const taskData = { + datasets: [ + { + description: "Bing queries with relevant passages from various web sources.", + id: "microsoft/ms_marco", + }, + ], + demo: { + inputs: [ + { + label: "Source sentence", + content: "Machine learning is so easy.", + type: "text", + }, + { + label: "Sentences to compare to", + content: "Deep learning is so straightforward.", + type: "text", + }, + { + label: "", + content: "This is so difficult, like rocket science.", + type: "text", + }, + { + label: "", + content: "I can't believe how much I struggled with this.", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Deep learning is so straightforward.", + score: 0.623, + }, + { + label: "This is so difficult, like rocket science.", + score: 0.413, + }, + { + label: "I can't believe how much I struggled with this.", + score: 0.256, + }, + ], + }, + ], + }, + metrics: [ + { + description: "Reciprocal Rank is a measure used to rank the relevancy of documents given a set of documents. Reciprocal Rank is the reciprocal of the rank of the document retrieved, meaning, if the rank is 3, the Reciprocal Rank is 0.33. If the rank is 1, the Reciprocal Rank is 1", + id: "Mean Reciprocal Rank", + }, + { + description: "The similarity of the embeddings is evaluated mainly on cosine similarity. It is calculated as the cosine of the angle between two vectors. It is particularly useful when your texts are not the same length", + id: "Cosine Similarity", + }, + ], + models: [ + { + description: "This model works well for sentences and paragraphs and can be used for clustering/grouping and semantic searches.", + id: "sentence-transformers/all-mpnet-base-v2", + }, + { + description: "A multilingual robust sentence similarity model.", + id: "BAAI/bge-m3", + }, + { + description: "A robust sentence similarity model.", + id: "HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v1.5", + }, + ], + spaces: [ + { + description: "An application that leverages sentence similarity to answer questions from YouTube videos.", + id: "Gradio-Blocks/Ask_Questions_To_YouTube_Videos", + }, + { + description: "An application that retrieves relevant PubMed abstracts for a given online article which can be used as further references.", + id: "Gradio-Blocks/pubmed-abstract-retriever", + }, + { + description: "An application that leverages sentence similarity to summarize text.", + id: "nickmuchi/article-text-summarizer", + }, + { + description: "A guide that explains how Sentence Transformers can be used for semantic search.", + id: "sentence-transformers/Sentence_Transformers_for_semantic_search", + }, + ], + summary: "Sentence Similarity is the task of determining how similar two texts are. Sentence similarity models convert input texts into vectors (embeddings) that capture semantic information and calculate how close (similar) they are between them. This task is particularly useful for information retrieval and clustering/grouping.", + widgetModels: ["sentence-transformers/all-MiniLM-L6-v2"], + youtubeId: "VCZq5AkbNEU", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..7836ed35d431c28daa542326aca7ea75563f5b9c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/inference.d.ts @@ -0,0 +1,32 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +export type SentenceSimilarityOutput = number[]; +/** + * Inputs for Sentence similarity inference + */ +export interface SentenceSimilarityInput { + inputs: SentenceSimilarityInputData; + /** + * Additional inference parameters for Sentence Similarity + */ + parameters?: { + [key: string]: unknown; + }; + [property: string]: unknown; +} +export interface SentenceSimilarityInputData { + /** + * A list of strings which will be compared against the source_sentence. + */ + sentences: string[]; + /** + * The string that you wish to compare the other strings with. This can be a phrase, + * sentence, or longer passage, depending on the model being used. + */ + source_sentence: string; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..1a31ca5b3e6cbd66e566a52e86cba1a024684a97 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/sentence-similarity/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,MAAM,wBAAwB,GAAG,MAAM,EAAE,CAAC;AAChD;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC,MAAM,EAAE,2BAA2B,CAAC;IACpC;;OAEG;IACH,UAAU,CAAC,EAAE;QACZ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C;;OAEG;IACH,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB;;;OAGG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/sentence-similarity/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7f112b24d58cd35588a17df77cc61bad268f5b37 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/summarization/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAuEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/data.js new file mode 100644 index 0000000000000000000000000000000000000000..6e984a19398bec160a6bf2414110ee132d5f92b3 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/data.js @@ -0,0 +1,67 @@ +const taskData = { + canonicalId: "text-generation", + datasets: [ + { + description: "News articles in five different languages along with their summaries. Widely used for benchmarking multilingual summarization models.", + id: "mlsum", + }, + { + description: "English conversations and their summaries. Useful for benchmarking conversational agents.", + id: "samsum", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. It was the first structure to reach a height of 300 metres. Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct.", + type: "text", + }, + ], + outputs: [ + { + label: "Output", + content: "The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building. It was the first structure to reach a height of 300 metres.", + type: "text", + }, + ], + }, + metrics: [ + { + description: "The generated sequence is compared against its summary, and the overlap of tokens are counted. ROUGE-N refers to overlap of N subsequent tokens, ROUGE-1 refers to overlap of single tokens and ROUGE-2 is the overlap of two subsequent tokens.", + id: "rouge", + }, + ], + models: [ + { + description: "A strong summarization model trained on English news articles. Excels at generating factual summaries.", + id: "facebook/bart-large-cnn", + }, + { + description: "A summarization model trained on medical articles.", + id: "Falconsai/medical_summarization", + }, + ], + spaces: [ + { + description: "An application that can summarize long paragraphs.", + id: "pszemraj/summarize-long-text", + }, + { + description: "A much needed summarization application for terms and conditions.", + id: "ml6team/distilbart-tos-summarizer-tosdr", + }, + { + description: "An application that summarizes long documents.", + id: "pszemraj/document-summarization", + }, + { + description: "An application that can detect errors in abstractive summarization.", + id: "ml6team/post-processing-summarization", + }, + ], + summary: "Summarization is the task of producing a shorter version of a document while preserving its important information. Some models can extract text from the original input, while other models can generate entirely new text.", + widgetModels: ["facebook/bart-large-cnn"], + youtubeId: "yHnr5Dk2zCI", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b3153fbb4fdb5eeb040516b68b90dfdda2057798 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/inference.d.ts @@ -0,0 +1,54 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Summarization inference + */ +export interface SummarizationInput { + /** + * The input text to summarize. + */ + inputs: string; + /** + * Additional inference parameters for summarization. + */ + parameters?: SummarizationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for summarization. + */ +export interface SummarizationParameters { + /** + * Whether to clean up the potential extra spaces in the text output. + */ + clean_up_tokenization_spaces?: boolean; + /** + * Additional parametrization of the text generation algorithm. + */ + generate_parameters?: { + [key: string]: unknown; + }; + /** + * The truncation strategy to use. + */ + truncation?: SummarizationTruncationStrategy; + [property: string]: unknown; +} +/** + * The truncation strategy to use. + */ +export type SummarizationTruncationStrategy = "do_not_truncate" | "longest_first" | "only_first" | "only_second"; +/** + * Outputs of inference for the Summarization task + */ +export interface SummarizationOutput { + /** + * The summarized text. + */ + summary_text: string; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..e2834ebbaa672e6227241c2f2c7fad3e67a586ba --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/summarization/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,uBAAuB,CAAC;IACrC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC;;OAEG;IACH,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC;;OAEG;IACH,mBAAmB,CAAC,EAAE;QACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF;;OAEG;IACH,UAAU,CAAC,EAAE,+BAA+B,CAAC;IAC7C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,+BAA+B,GAAG,iBAAiB,GAAG,eAAe,GAAG,YAAY,GAAG,aAAa,CAAC;AACjH;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/summarization/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7df5547173450d1e1d1218adc4b5e3c9a2d90075 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/table-question-answering/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAsDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/data.js new file mode 100644 index 0000000000000000000000000000000000000000..7afd54c44761c1c33a555717c3254c51bd022d8e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/data.js @@ -0,0 +1,52 @@ +const taskData = { + datasets: [ + { + description: "The WikiTableQuestions dataset is a large-scale dataset for the task of question answering on semi-structured tables.", + id: "wikitablequestions", + }, + { + description: "WikiSQL is a dataset of 80654 hand-annotated examples of questions and SQL queries distributed across 24241 tables from Wikipedia.", + id: "wikisql", + }, + ], + demo: { + inputs: [ + { + table: [ + ["Rank", "Name", "No.of reigns", "Combined days"], + ["1", "lou Thesz", "3", "3749"], + ["2", "Ric Flair", "8", "3103"], + ["3", "Harley Race", "7", "1799"], + ], + type: "tabular", + }, + { label: "Question", content: "What is the number of reigns for Harley Race?", type: "text" }, + ], + outputs: [{ label: "Result", content: "7", type: "text" }], + }, + metrics: [ + { + description: "Checks whether the predicted answer(s) is the same as the ground-truth answer(s).", + id: "Denotation Accuracy", + }, + ], + models: [ + { + description: "A table question answering model that is capable of neural SQL execution, i.e., employ TAPEX to execute a SQL query on a given table.", + id: "microsoft/tapex-base", + }, + { + description: "A robust table question answering model.", + id: "google/tapas-base-finetuned-wtq", + }, + ], + spaces: [ + { + description: "An application that answers questions based on table CSV files.", + id: "katanaml/table-query", + }, + ], + summary: "Table Question Answering (Table QA) is the answering a question about an information on a given table.", + widgetModels: ["google/tapas-base-finetuned-wtq"], +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..bfe5048481141e19695ee9459efa19ceda69949d --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/inference.d.ts @@ -0,0 +1,84 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Table Question Answering inference + */ +export interface TableQuestionAnsweringInput { + /** + * One (table, question) pair to answer + */ + inputs: TableQuestionAnsweringInputData; + /** + * Additional inference parameters for Table Question Answering + */ + parameters?: TableQuestionAnsweringParameters; + [property: string]: unknown; +} +/** + * One (table, question) pair to answer + */ +export interface TableQuestionAnsweringInputData { + /** + * The question to be answered about the table + */ + question: string; + /** + * The table to serve as context for the questions + */ + table: { + [key: string]: string[]; + }; + [property: string]: unknown; +} +/** + * Additional inference parameters for Table Question Answering + */ +export interface TableQuestionAnsweringParameters { + /** + * Activates and controls padding. + */ + padding?: Padding; + /** + * Whether to do inference sequentially or as a batch. Batching is faster, but models like + * SQA require the inference to be done sequentially to extract relations within sequences, + * given their conversational nature. + */ + sequential?: boolean; + /** + * Activates and controls truncation. + */ + truncation?: boolean; + [property: string]: unknown; +} +/** + * Activates and controls padding. + */ +export type Padding = "do_not_pad" | "longest" | "max_length"; +export type TableQuestionAnsweringOutput = TableQuestionAnsweringOutputElement[]; +/** + * Outputs of inference for the Table Question Answering task + */ +export interface TableQuestionAnsweringOutputElement { + /** + * If the model has an aggregator, this returns the aggregator. + */ + aggregator?: string; + /** + * The answer of the question given the table. If there is an aggregator, the answer will be + * preceded by `AGGREGATOR >`. + */ + answer: string; + /** + * List of strings made up of the answer cell values. + */ + cells: string[]; + /** + * Coordinates of the cells of the answers. + */ + coordinates: Array; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..662dc6b5c7390184441f95d326a449f226b02dbb --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/table-question-answering/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC3C;;OAEG;IACH,MAAM,EAAE,+BAA+B,CAAC;IACxC;;OAEG;IACH,UAAU,CAAC,EAAE,gCAAgC,CAAC;IAC9C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,+BAA+B;IAC/C;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,KAAK,EAAE;QACN,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;KACxB,CAAC;IACF,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,OAAO,GAAG,YAAY,GAAG,SAAS,GAAG,YAAY,CAAC;AAC9D,MAAM,MAAM,4BAA4B,GAAG,mCAAmC,EAAE,CAAC;AACjF;;GAEG;AACH,MAAM,WAAW,mCAAmC;IACnD;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB;;OAEG;IACH,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/table-question-answering/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5a327b8a6c01a349e8a3eb677fa7d63a604fd2a6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/tabular-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA+Df,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-classification/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..4897081943f0c5206b1fa979aed8e8756339a671 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-classification/data.js @@ -0,0 +1,65 @@ +const taskData = { + datasets: [ + { + description: "A comprehensive curation of datasets covering all benchmarks.", + id: "inria-soda/tabular-benchmark", + }, + ], + demo: { + inputs: [ + { + table: [ + ["Glucose", "Blood Pressure ", "Skin Thickness", "Insulin", "BMI"], + ["148", "72", "35", "0", "33.6"], + ["150", "50", "30", "0", "35.1"], + ["141", "60", "29", "1", "39.2"], + ], + type: "tabular", + }, + ], + outputs: [ + { + table: [["Diabetes"], ["1"], ["1"], ["0"]], + type: "tabular", + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + description: "Breast cancer prediction model based on decision trees.", + id: "scikit-learn/cancer-prediction-trees", + }, + ], + spaces: [ + { + description: "An application that can predict defective products on a production line.", + id: "scikit-learn/tabular-playground", + }, + { + description: "An application that compares various tabular classification techniques on different datasets.", + id: "scikit-learn/classification", + }, + ], + summary: "Tabular classification is the task of classifying a target category (a group) based on set of attributes.", + widgetModels: ["scikit-learn/tabular-playground"], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-regression/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-regression/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-regression/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-regression/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-regression/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..8603a1fb917be27322c628610908f41d7228a969 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-regression/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/tabular-regression/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAoDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-regression/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-regression/data.js new file mode 100644 index 0000000000000000000000000000000000000000..d283cd9f04db9934267603a3aa1b5df58f8ba361 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/tabular-regression/data.js @@ -0,0 +1,53 @@ +const taskData = { + datasets: [ + { + description: "A comprehensive curation of datasets covering all benchmarks.", + id: "inria-soda/tabular-benchmark", + }, + ], + demo: { + inputs: [ + { + table: [ + ["Car Name", "Horsepower", "Weight"], + ["ford torino", "140", "3,449"], + ["amc hornet", "97", "2,774"], + ["toyota corolla", "65", "1,773"], + ], + type: "tabular", + }, + ], + outputs: [ + { + table: [["MPG (miles per gallon)"], ["17"], ["18"], ["31"]], + type: "tabular", + }, + ], + }, + metrics: [ + { + description: "", + id: "mse", + }, + { + description: "Coefficient of determination (or R-squared) is a measure of how well the model fits the data. Higher R-squared is considered a better fit.", + id: "r-squared", + }, + ], + models: [ + { + description: "Fish weight prediction based on length measurements and species.", + id: "scikit-learn/Fish-Weight", + }, + ], + spaces: [ + { + description: "An application that can predict weight of a fish based on set of attributes.", + id: "scikit-learn/fish-weight-prediction", + }, + ], + summary: "Tabular regression is the task of predicting a numerical value given a set of attributes.", + widgetModels: ["scikit-learn/Fish-Weight"], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..90bc031f2e0d672008b73b37088e7779fba975a2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAkGf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..d7d1664356ff91dc99f9ee31504dd097d84b5d56 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/data.js @@ -0,0 +1,98 @@ +const taskData = { + datasets: [ + { + description: "A widely used dataset used to benchmark multiple variants of text classification.", + id: "nyu-mll/glue", + }, + { + description: "A text classification dataset used to benchmark natural language inference models", + id: "stanfordnlp/snli", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "I love Hugging Face!", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "POSITIVE", + score: 0.9, + }, + { + label: "NEUTRAL", + score: 0.1, + }, + { + label: "NEGATIVE", + score: 0.0, + }, + ], + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "The F1 metric is the harmonic mean of the precision and recall. It can be calculated as: F1 = 2 * (precision * recall) / (precision + recall)", + id: "f1", + }, + ], + models: [ + { + description: "A robust model trained for sentiment analysis.", + id: "distilbert/distilbert-base-uncased-finetuned-sst-2-english", + }, + { + description: "A sentiment analysis model specialized in financial sentiment.", + id: "ProsusAI/finbert", + }, + { + description: "A sentiment analysis model specialized in analyzing tweets.", + id: "cardiffnlp/twitter-roberta-base-sentiment-latest", + }, + { + description: "A model that can classify languages.", + id: "papluca/xlm-roberta-base-language-detection", + }, + { + description: "A model that can classify text generation attacks.", + id: "meta-llama/Prompt-Guard-86M", + }, + ], + spaces: [ + { + description: "An application that can classify financial sentiment.", + id: "IoannisTr/Tech_Stocks_Trading_Assistant", + }, + { + description: "A dashboard that contains various text classification tasks.", + id: "miesnerjacob/Multi-task-NLP", + }, + { + description: "An application that analyzes user reviews in healthcare.", + id: "spacy/healthsea-demo", + }, + ], + summary: "Text Classification is the task of assigning a label or class to a given text. Some use cases are sentiment analysis, natural language inference, and assessing grammatical correctness.", + widgetModels: ["distilbert/distilbert-base-uncased-finetuned-sst-2-english"], + youtubeId: "leNG9fN9FQU", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..ff29b09e9a909bb94b21f96d8a939dadcd7deb3b --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/inference.d.ts @@ -0,0 +1,53 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text Classification inference + */ +export interface TextClassificationInput { + /** + * The text to classify + */ + inputs: string; + /** + * Additional inference parameters for Text Classification + */ + parameters?: TextClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text Classification + */ +export interface TextClassificationParameters { + /** + * The function to apply to the model outputs in order to retrieve the scores. + */ + function_to_apply?: ClassificationOutputTransform; + /** + * When specified, limits the output to the top K most probable classes. + */ + top_k?: number; + [property: string]: unknown; +} +/** + * The function to apply to the model outputs in order to retrieve the scores. + */ +export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none"; +export type TextClassificationOutput = TextClassificationOutputElement[]; +/** + * Outputs of inference for the Text Classification task + */ +export interface TextClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..328ab1be6f6979d09ce733cb3208285bba6b5fba --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-classification/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,4BAA4B,CAAC;IAC1C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC5C;;OAEG;IACH,iBAAiB,CAAC,EAAE,6BAA6B,CAAC;IAClD;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAC3E,MAAM,MAAM,wBAAwB,GAAG,+BAA+B,EAAE,CAAC;AACzE;;GAEG;AACH,MAAM,WAAW,+BAA+B;IAC/C;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-classification/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..1cb375d4130e99f5926dd0eacd8fe90366df7b10 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-generation/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA6Hf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/data.js new file mode 100644 index 0000000000000000000000000000000000000000..557887b6b3b6b450edc3abc8178ede1539619d90 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/data.js @@ -0,0 +1,123 @@ +const taskData = { + datasets: [ + { + description: "Multilingual dataset used to evaluate text generation models.", + id: "CohereForAI/Global-MMLU", + }, + { + description: "High quality multilingual data used to train text-generation models.", + id: "HuggingFaceFW/fineweb-2", + }, + { + description: "Truly open-source, curated and cleaned dialogue dataset.", + id: "HuggingFaceH4/ultrachat_200k", + }, + { + description: "A reasoning dataset.", + id: "open-r1/OpenThoughts-114k-math", + }, + { + description: "A multilingual instruction dataset with preference ratings on responses.", + id: "allenai/tulu-3-sft-mixture", + }, + { + description: "A large synthetic dataset for alignment of text generation models.", + id: "HuggingFaceTB/smoltalk", + }, + { + description: "A dataset made for training text generation models solving math questions.", + id: "HuggingFaceTB/finemath", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "Once upon a time,", + type: "text", + }, + ], + outputs: [ + { + label: "Output", + content: "Once upon a time, we knew that our ancestors were on the verge of extinction. The great explorers and poets of the Old World, from Alexander the Great to Chaucer, are dead and gone. A good many of our ancient explorers and poets have", + type: "text", + }, + ], + }, + metrics: [ + { + description: "Cross Entropy is a metric that calculates the difference between two probability distributions. Each probability distribution is the distribution of predicted words", + id: "Cross Entropy", + }, + { + description: "The Perplexity metric is the exponential of the cross-entropy loss. It evaluates the probabilities assigned to the next word by the model. Lower perplexity indicates better performance", + id: "Perplexity", + }, + ], + models: [ + { description: "A text-generation model trained to follow instructions.", id: "google/gemma-2-2b-it" }, + { + description: "Powerful text generation model for coding.", + id: "Qwen/Qwen3-Coder-480B-A35B-Instruct", + }, + { + description: "Great text generation model with top-notch tool calling capabilities.", + id: "openai/gpt-oss-120b", + }, + { + description: "Powerful text generation model.", + id: "zai-org/GLM-4.5", + }, + { + description: "A powerful small model with reasoning capabilities.", + id: "Qwen/Qwen3-4B-Thinking-2507", + }, + { + description: "Strong conversational model that supports very long instructions.", + id: "Qwen/Qwen2.5-7B-Instruct-1M", + }, + { + description: "Text generation model used to write code.", + id: "Qwen/Qwen2.5-Coder-32B-Instruct", + }, + { + description: "Powerful reasoning based open large language model.", + id: "deepseek-ai/DeepSeek-R1", + }, + ], + spaces: [ + { + description: "An application that writes and executes code from text instructions and supports many models.", + id: "akhaliq/anycoder", + }, + { + description: "An application that builds websites from natural language prompts.", + id: "enzostvs/deepsite", + }, + { + description: "A leaderboard for comparing chain-of-thought performance of models.", + id: "logikon/open_cot_leaderboard", + }, + { + description: "An text generation based application based on a very powerful LLaMA2 model.", + id: "ysharma/Explore_llamav2_with_TGI", + }, + { + description: "An text generation based application to converse with Zephyr model.", + id: "HuggingFaceH4/zephyr-chat", + }, + { + description: "A leaderboard that ranks text generation models based on blind votes from people.", + id: "lmsys/chatbot-arena-leaderboard", + }, + { + description: "An chatbot to converse with a very powerful text generation model.", + id: "mlabonne/phixtral-chat", + }, + ], + summary: "Generating text is the task of generating new text given another text. These models can, for example, fill in incomplete text or paraphrase.", + widgetModels: ["mistralai/Mistral-Nemo-Instruct-2407"], + youtubeId: "e9gNEAlsOvU", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..0495ad64fc55455d8b2a2933ae99b37d86775f1f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/inference.d.ts @@ -0,0 +1,188 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Text Generation Input. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface TextGenerationInput { + inputs: string; + parameters?: TextGenerationInputGenerateParameters; + stream?: boolean; + [property: string]: unknown; +} +export interface TextGenerationInputGenerateParameters { + /** + * Lora adapter id + */ + adapter_id?: string; + /** + * Generate best_of sequences and return the one if the highest token logprobs. + */ + best_of?: number; + /** + * Whether to return decoder input token logprobs and ids. + */ + decoder_input_details?: boolean; + /** + * Whether to return generation details. + */ + details?: boolean; + /** + * Activate logits sampling. + */ + do_sample?: boolean; + /** + * The parameter for frequency penalty. 1.0 means no penalty + * Penalize new tokens based on their existing frequency in the text so far, + * decreasing the model's likelihood to repeat the same line verbatim. + */ + frequency_penalty?: number; + grammar?: TextGenerationInputGrammarType; + /** + * Maximum number of tokens to generate. + */ + max_new_tokens?: number; + /** + * The parameter for repetition penalty. 1.0 means no penalty. + * See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. + */ + repetition_penalty?: number; + /** + * Whether to prepend the prompt to the generated text + */ + return_full_text?: boolean; + /** + * Random sampling seed. + */ + seed?: number; + /** + * Stop generating tokens if a member of `stop` is generated. + */ + stop?: string[]; + /** + * The value used to module the logits distribution. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-n-filtering. + */ + top_n_tokens?: number; + /** + * Top-p value for nucleus sampling. + */ + top_p?: number; + /** + * Truncate inputs tokens to the given size. + */ + truncate?: number; + /** + * Typical Decoding mass + * See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) + * for more information. + */ + typical_p?: number; + /** + * Watermarking with [A Watermark for Large Language + * Models](https://arxiv.org/abs/2301.10226). + */ + watermark?: boolean; + [property: string]: unknown; +} +export interface TextGenerationInputGrammarType { + type: Type; + /** + * A string that represents a [JSON Schema](https://json-schema.org/). + * + * JSON Schema is a declarative language that allows to annotate JSON documents + * with types and descriptions. + */ + value: unknown; + [property: string]: unknown; +} +export type Type = "json" | "regex" | "json_schema"; +/** + * Text Generation Output. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface TextGenerationOutput { + details?: TextGenerationOutputDetails; + generated_text: string; + [property: string]: unknown; +} +export interface TextGenerationOutputDetails { + best_of_sequences?: TextGenerationOutputBestOfSequence[]; + finish_reason: TextGenerationOutputFinishReason; + generated_tokens: number; + prefill: TextGenerationOutputPrefillToken[]; + seed?: number; + tokens: TextGenerationOutputToken[]; + top_tokens?: Array; + [property: string]: unknown; +} +export interface TextGenerationOutputBestOfSequence { + finish_reason: TextGenerationOutputFinishReason; + generated_text: string; + generated_tokens: number; + prefill: TextGenerationOutputPrefillToken[]; + seed?: number; + tokens: TextGenerationOutputToken[]; + top_tokens?: Array; + [property: string]: unknown; +} +export type TextGenerationOutputFinishReason = "length" | "eos_token" | "stop_sequence"; +export interface TextGenerationOutputPrefillToken { + id: number; + logprob: number; + text: string; + [property: string]: unknown; +} +export interface TextGenerationOutputToken { + id: number; + logprob: number; + special: boolean; + text: string; + [property: string]: unknown; +} +/** + * Text Generation Stream Output. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface TextGenerationStreamOutput { + details?: TextGenerationStreamOutputStreamDetails; + generated_text?: string; + index: number; + token: TextGenerationStreamOutputToken; + top_tokens?: TextGenerationStreamOutputToken[]; + [property: string]: unknown; +} +export interface TextGenerationStreamOutputStreamDetails { + finish_reason: TextGenerationOutputFinishReason; + generated_tokens: number; + input_length: number; + seed?: number; + [property: string]: unknown; +} +export interface TextGenerationStreamOutputToken { + id: number; + logprob: number; + special: boolean; + text: string; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..44c5f235fb539f2e65e39fae7af67582b0bf0a04 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-generation/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;;;;;GAMG;AACH,MAAM,WAAW,mBAAmB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,qCAAqC,CAAC;IACnD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,qCAAqC;IACrD;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,8BAA8B,CAAC;IACzC;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;OAEG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,8BAA8B;IAC9C,IAAI,EAAE,IAAI,CAAC;IACX;;;;;OAKG;IACH,KAAK,EAAE,OAAO,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,IAAI,GAAG,MAAM,GAAG,OAAO,GAAG,aAAa,CAAC;AACpD;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACpC,OAAO,CAAC,EAAE,2BAA2B,CAAC;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,2BAA2B;IAC3C,iBAAiB,CAAC,EAAE,kCAAkC,EAAE,CAAC;IACzD,aAAa,EAAE,gCAAgC,CAAC;IAChD,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,gCAAgC,EAAE,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,yBAAyB,EAAE,CAAC;IACpC,UAAU,CAAC,EAAE,KAAK,CAAC,yBAAyB,EAAE,CAAC,CAAC;IAChD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,kCAAkC;IAClD,aAAa,EAAE,gCAAgC,CAAC;IAChD,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,gCAAgC,EAAE,CAAC;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,yBAAyB,EAAE,CAAC;IACpC,UAAU,CAAC,EAAE,KAAK,CAAC,yBAAyB,EAAE,CAAC,CAAC;IAChD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,gCAAgC,GAAG,QAAQ,GAAG,WAAW,GAAG,eAAe,CAAC;AACxF,MAAM,WAAW,gCAAgC;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,yBAAyB;IACzC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;GAMG;AACH,MAAM,WAAW,0BAA0B;IAC1C,OAAO,CAAC,EAAE,uCAAuC,CAAC;IAClD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,+BAA+B,CAAC;IACvC,UAAU,CAAC,EAAE,+BAA+B,EAAE,CAAC;IAC/C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,uCAAuC;IACvD,aAAa,EAAE,gCAAgC,CAAC;IAChD,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,WAAW,+BAA+B;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-generation/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-ranking/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/text-ranking/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-ranking/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-ranking/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/text-ranking/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..4e751a3bbd1ecc3171184413375e276af5c186d3 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-ranking/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-ranking/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAsFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-ranking/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/text-ranking/data.js new file mode 100644 index 0000000000000000000000000000000000000000..75c662056be4d92a33b647048b078ff2e5b70a48 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-ranking/data.js @@ -0,0 +1,84 @@ +const taskData = { + datasets: [ + { + description: "Bing queries with relevant passages from various web sources.", + id: "microsoft/ms_marco", + }, + ], + demo: { + inputs: [ + { + label: "Source sentence", + content: "Machine learning is so easy.", + type: "text", + }, + { + label: "Sentences to compare to", + content: "Deep learning is so straightforward.", + type: "text", + }, + { + label: "", + content: "This is so difficult, like rocket science.", + type: "text", + }, + { + label: "", + content: "I can't believe how much I struggled with this.", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Deep learning is so straightforward.", + score: 2.2006407, + }, + { + label: "This is so difficult, like rocket science.", + score: -6.2634873, + }, + { + label: "I can't believe how much I struggled with this.", + score: -10.251488, + }, + ], + }, + ], + }, + metrics: [ + { + description: "Discounted Cumulative Gain (DCG) measures the gain, or usefulness, of search results discounted by their position. The normalization is done by dividing the DCG by the ideal DCG, which is the DCG of the perfect ranking.", + id: "Normalized Discounted Cumulative Gain", + }, + { + description: "Reciprocal Rank is a measure used to rank the relevancy of documents given a set of documents. Reciprocal Rank is the reciprocal of the rank of the document retrieved, meaning, if the rank is 3, the Reciprocal Rank is 0.33. If the rank is 1, the Reciprocal Rank is 1", + id: "Mean Reciprocal Rank", + }, + { + description: "Mean Average Precision (mAP) is the overall average of the Average Precision (AP) values, where AP is the Area Under the PR Curve (AUC-PR)", + id: "Mean Average Precision", + }, + ], + models: [ + { + description: "An extremely efficient text ranking model trained on a web search dataset.", + id: "cross-encoder/ms-marco-MiniLM-L6-v2", + }, + { + description: "A strong multilingual text reranker model.", + id: "Alibaba-NLP/gte-multilingual-reranker-base", + }, + { + description: "An efficient text ranking model that punches above its weight.", + id: "Alibaba-NLP/gte-reranker-modernbert-base", + }, + ], + spaces: [], + summary: "Text Ranking is the task of ranking a set of texts based on their relevance to a query. Text ranking models are trained on large datasets of queries and relevant documents to learn how to rank documents based on their relevance to the query. This task is particularly useful for search engines and information retrieval systems.", + widgetModels: ["cross-encoder/ms-marco-MiniLM-L6-v2"], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-3d/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-3d/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-3d/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-3d/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-3d/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a56f91f9a5ea203fe8a085e3c5d555f5306b119e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-3d/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-3d/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAmDf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-3d/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-3d/data.js new file mode 100644 index 0000000000000000000000000000000000000000..bac716d4308cbd938b635061e83ecd6d34d277d2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-3d/data.js @@ -0,0 +1,53 @@ +const taskData = { + datasets: [ + { + description: "A large dataset of over 10 million 3D objects.", + id: "allenai/objaverse-xl", + }, + { + description: "Descriptive captions for 3D objects in Objaverse.", + id: "tiange/Cap3D", + }, + ], + demo: { + inputs: [ + { + label: "Prompt", + content: "a cat statue", + type: "text", + }, + ], + outputs: [ + { + label: "Result", + content: "text-to-3d-3d-output-filename.glb", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "Text-to-3D mesh model by OpenAI", + id: "openai/shap-e", + }, + { + description: "Generative 3D gaussian splatting model.", + id: "ashawkey/LGM", + }, + ], + spaces: [ + { + description: "Text-to-3D demo with mesh outputs.", + id: "hysts/Shap-E", + }, + { + description: "Text/image-to-3D demo with splat outputs.", + id: "ashawkey/LGM", + }, + ], + summary: "Text-to-3D models take in text input and produce 3D output.", + widgetModels: [], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-audio/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-audio/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..811850173233fed08a447e510bede7a576e09822 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-audio/inference.d.ts @@ -0,0 +1,134 @@ +/** + * Outputs of inference for the Text To Audio task + */ +export interface TextToAudioOutput { + /** + * The generated audio waveform. + */ + audio: Blob; + /** + * The sampling rate of the generated audio waveform. + */ + sampling_rate: number; + [property: string]: unknown; +} +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text To Audio inference + */ +export interface TextToAudioInput { + /** + * The input text data + */ + inputs: string; + /** + * Additional inference parameters for Text To Audio + */ + parameters?: TextToAudioParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text To Audio + */ +export interface TextToAudioParameters { + /** + * Parametrization of the text generation process + */ + generation_parameters?: GenerationParameters; + [property: string]: unknown; +} +/** + * Parametrization of the text generation process + */ +export interface GenerationParameters { + /** + * Whether to use sampling instead of greedy decoding when generating new tokens. + */ + do_sample?: boolean; + /** + * Controls the stopping condition for beam-based methods. + */ + early_stopping?: EarlyStoppingUnion; + /** + * If set to float strictly between 0 and 1, only tokens with a conditional probability + * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + * Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + */ + epsilon_cutoff?: number; + /** + * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + * float strictly between 0 and 1, a token is only considered if it is greater than either + * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + * for more details. + */ + eta_cutoff?: number; + /** + * The maximum length (in tokens) of the generated text, including the input. + */ + max_length?: number; + /** + * The maximum number of tokens to generate. Takes precedence over max_length. + */ + max_new_tokens?: number; + /** + * The minimum length (in tokens) of the generated text, including the input. + */ + min_length?: number; + /** + * The minimum number of tokens to generate. Takes precedence over min_length. + */ + min_new_tokens?: number; + /** + * Number of groups to divide num_beams into in order to ensure diversity among different + * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + */ + num_beam_groups?: number; + /** + * Number of beams to use for beam search. + */ + num_beams?: number; + /** + * The value balances the model confidence and the degeneration penalty in contrastive + * search decoding. + */ + penalty_alpha?: number; + /** + * The value used to modulate the next token probabilities. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * If set to float < 1, only the smallest set of most probable tokens with probabilities + * that add up to top_p or higher are kept for generation. + */ + top_p?: number; + /** + * Local typicality measures how similar the conditional probability of predicting a target + * token next is to the expected conditional probability of predicting a random token next, + * given the partial text already generated. If set to float < 1, the smallest set of the + * most locally typical tokens with probabilities that add up to typical_p or higher are + * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + */ + typical_p?: number; + /** + * Whether the model should use the past last key/values attentions to speed up decoding + */ + use_cache?: boolean; + [property: string]: unknown; +} +/** + * Controls the stopping condition for beam-based methods. + */ +export type EarlyStoppingUnion = boolean | "never"; +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-audio/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-audio/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..eec3725373942525186fac3145889c21d7eab2d7 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-audio/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-audio/inference.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,KAAK,EAAE,IAAI,CAAC;IACZ;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,qBAAqB,CAAC,EAAE,oBAAoB,CAAC;IAC7C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,kBAAkB,CAAC;IACpC;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,OAAO,GAAG,OAAO,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-audio/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-audio/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-audio/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..9f3b6ad845035578c7d5be1c8af56971c67a1f3f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-image/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAmGf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/data.js new file mode 100644 index 0000000000000000000000000000000000000000..9ab998a4f0f3ab6f71b55c27ac9ab6cc5232e0b6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/data.js @@ -0,0 +1,97 @@ +const taskData = { + datasets: [ + { + description: "RedCaps is a large-scale dataset of 12M image-text pairs collected from Reddit.", + id: "red_caps", + }, + { + description: "Conceptual Captions is a dataset consisting of ~3.3M images annotated with captions.", + id: "conceptual_captions", + }, + { + description: "12M image-caption pairs.", + id: "Spawning/PD12M", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "A city above clouds, pastel colors, Victorian style", + type: "text", + }, + ], + outputs: [ + { + filename: "image.jpeg", + type: "img", + }, + ], + }, + metrics: [ + { + description: "The Inception Score (IS) measure assesses diversity and meaningfulness. It uses a generated image sample to predict its label. A higher score signifies more diverse and meaningful images.", + id: "IS", + }, + { + description: "The Fréchet Inception Distance (FID) calculates the distance between distributions between synthetic and real samples. A lower FID score indicates better similarity between the distributions of real and generated images.", + id: "FID", + }, + { + description: "R-precision assesses how the generated image aligns with the provided text description. It uses the generated images as queries to retrieve relevant text descriptions. The top 'r' relevant descriptions are selected and used to calculate R-precision as r/R, where 'R' is the number of ground truth descriptions associated with the generated images. A higher R-precision value indicates a better model.", + id: "R-Precision", + }, + ], + models: [ + { + description: "One of the most powerful image generation models that can generate realistic outputs.", + id: "black-forest-labs/FLUX.1-Krea-dev", + }, + { + description: "A powerful image generation model.", + id: "Qwen/Qwen-Image", + }, + { + description: "Powerful and fast image generation model.", + id: "ByteDance/SDXL-Lightning", + }, + { + description: "A powerful text-to-image model.", + id: "ByteDance/Hyper-SD", + }, + ], + spaces: [ + { + description: "A powerful text-to-image application.", + id: "stabilityai/stable-diffusion-3-medium", + }, + { + description: "A text-to-image application to generate comics.", + id: "jbilcke-hf/ai-comic-factory", + }, + { + description: "An application to match multiple custom image generation models.", + id: "multimodalart/flux-lora-lab", + }, + { + description: "A powerful yet very fast image generation application.", + id: "latent-consistency/lcm-lora-for-sdxl", + }, + { + description: "A gallery to explore various text-to-image models.", + id: "multimodalart/LoraTheExplorer", + }, + { + description: "An application for `text-to-image`, `image-to-image` and image inpainting.", + id: "ArtGAN/Stable-Diffusion-ControlNet-WebUI", + }, + { + description: "An application to generate realistic images given photos of a person and a prompt.", + id: "InstantX/InstantID", + }, + ], + summary: "Text-to-image is the task of generating images from input text. These pipelines can also be used to modify and edit images based on text prompts.", + widgetModels: ["black-forest-labs/FLUX.1-dev"], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..af711fe85ee463c28cdd3f514466a6344a3a2cdb --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/inference.d.ts @@ -0,0 +1,66 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text To Image inference + */ +export interface TextToImageInput { + /** + * The input text data (sometimes called "prompt") + */ + inputs: string; + /** + * Additional inference parameters for Text To Image + */ + parameters?: TextToImageParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text To Image + */ +export interface TextToImageParameters { + /** + * A higher guidance scale value encourages the model to generate images closely linked to + * the text prompt, but values too high may cause saturation and other artifacts. + */ + guidance_scale?: number; + /** + * The height in pixels of the output image + */ + height?: number; + /** + * One prompt to guide what NOT to include in image generation. + */ + negative_prompt?: string; + /** + * The number of denoising steps. More denoising steps usually lead to a higher quality + * image at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * Override the scheduler with a compatible one. + */ + scheduler?: string; + /** + * Seed for the random number generator. + */ + seed?: number; + /** + * The width in pixels of the output image + */ + width?: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Text To Image task + */ +export interface TextToImageOutput { + /** + * The generated image returned as raw bytes in the payload. + */ + image: unknown; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..03467437eb89c69ccf8ef265aa92d9e6eae427e5 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-image/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-image/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..ed4c56895bf73bf3252bfe24ad8c688f910332fc --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-speech/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAiFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/data.js new file mode 100644 index 0000000000000000000000000000000000000000..76d375ab5181baff63bfaa6dc812ad2c845831bd --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/data.js @@ -0,0 +1,82 @@ +const taskData = { + canonicalId: "text-to-audio", + datasets: [ + { + description: "10K hours of multi-speaker English dataset.", + id: "parler-tts/mls_eng_10k", + }, + { + description: "Multi-speaker English dataset.", + id: "mythicinfinity/libritts_r", + }, + { + description: "Multi-lingual dataset.", + id: "facebook/multilingual_librispeech", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "I love audio models on the Hub!", + type: "text", + }, + ], + outputs: [ + { + filename: "audio.wav", + type: "audio", + }, + ], + }, + metrics: [ + { + description: "The Mel Cepstral Distortion (MCD) metric is used to calculate the quality of generated speech.", + id: "mel cepstral distortion", + }, + ], + models: [ + { + description: "Small yet powerful TTS model.", + id: "KittenML/kitten-tts-nano-0.1", + }, + { + description: "Bleeding edge TTS model.", + id: "ResembleAI/chatterbox", + }, + { + description: "A massively multi-lingual TTS model.", + id: "fishaudio/fish-speech-1.5", + }, + { + description: "A text-to-dialogue model.", + id: "nari-labs/Dia-1.6B-0626", + }, + ], + spaces: [ + { + description: "An application for generate high quality speech in different languages.", + id: "hexgrad/Kokoro-TTS", + }, + { + description: "A multilingual text-to-speech application.", + id: "fishaudio/fish-speech-1", + }, + { + description: "Performant TTS application.", + id: "ResembleAI/Chatterbox", + }, + { + description: "An application to compare different TTS models.", + id: "TTS-AGI/TTS-Arena-V2", + }, + { + description: "An application that generates podcast episodes.", + id: "ngxson/kokoro-podcast-generator", + }, + ], + summary: "Text-to-Speech (TTS) is the task of generating natural sounding speech given text input. TTS models can be extended to have a single model that generates speech for multiple speakers and multiple languages.", + widgetModels: ["suno/bark"], + youtubeId: "NW62DpzJ274", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..6cc131914ae1bf76593f4a2958c5cda23baff842 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/inference.d.ts @@ -0,0 +1,134 @@ +/** + * Outputs of inference for the Text To Speech task + */ +export interface TextToSpeechOutput { + /** + * The generated audio + */ + audio: Blob; + /** + * The sampling rate of the generated audio waveform. + */ + sampling_rate?: number; + [property: string]: unknown; +} +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text To Speech inference + */ +export interface TextToSpeechInput { + /** + * The input text data + */ + inputs: string; + /** + * Additional inference parameters for Text To Speech + */ + parameters?: TextToSpeechParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text To Speech + */ +export interface TextToSpeechParameters { + /** + * Parametrization of the text generation process + */ + generation_parameters?: GenerationParameters; + [property: string]: unknown; +} +/** + * Parametrization of the text generation process + */ +export interface GenerationParameters { + /** + * Whether to use sampling instead of greedy decoding when generating new tokens. + */ + do_sample?: boolean; + /** + * Controls the stopping condition for beam-based methods. + */ + early_stopping?: EarlyStoppingUnion; + /** + * If set to float strictly between 0 and 1, only tokens with a conditional probability + * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + * Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + */ + epsilon_cutoff?: number; + /** + * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + * float strictly between 0 and 1, a token is only considered if it is greater than either + * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + * for more details. + */ + eta_cutoff?: number; + /** + * The maximum length (in tokens) of the generated text, including the input. + */ + max_length?: number; + /** + * The maximum number of tokens to generate. Takes precedence over max_length. + */ + max_new_tokens?: number; + /** + * The minimum length (in tokens) of the generated text, including the input. + */ + min_length?: number; + /** + * The minimum number of tokens to generate. Takes precedence over min_length. + */ + min_new_tokens?: number; + /** + * Number of groups to divide num_beams into in order to ensure diversity among different + * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + */ + num_beam_groups?: number; + /** + * Number of beams to use for beam search. + */ + num_beams?: number; + /** + * The value balances the model confidence and the degeneration penalty in contrastive + * search decoding. + */ + penalty_alpha?: number; + /** + * The value used to modulate the next token probabilities. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * If set to float < 1, only the smallest set of most probable tokens with probabilities + * that add up to top_p or higher are kept for generation. + */ + top_p?: number; + /** + * Local typicality measures how similar the conditional probability of predicting a target + * token next is to the expected conditional probability of predicting a random token next, + * given the partial text already generated. If set to float < 1, the smallest set of the + * most locally typical tokens with probabilities that add up to typical_p or higher are + * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + */ + typical_p?: number; + /** + * Whether the model should use the past last key/values attentions to speed up decoding + */ + use_cache?: boolean; + [property: string]: unknown; +} +/** + * Controls the stopping condition for beam-based methods. + */ +export type EarlyStoppingUnion = boolean | "never"; +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..47fb15af47a221be88e3d57de5fdb4b950d5af25 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-speech/inference.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,KAAK,EAAE,IAAI,CAAC;IACZ;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,qBAAqB,CAAC,EAAE,oBAAoB,CAAC;IAC7C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,kBAAkB,CAAC;IACpC;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,OAAO,GAAG,OAAO,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-speech/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..15f63d971713a969dbf522ff03184c46e982cf0e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-video/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAqGf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/data.js new file mode 100644 index 0000000000000000000000000000000000000000..6659ef5d32e775ce846aa8faa0b6932c8fd2e0d1 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/data.js @@ -0,0 +1,97 @@ +const taskData = { + datasets: [ + { + description: "Microsoft Research Video to Text is a large-scale dataset for open domain video captioning", + id: "iejMac/CLIP-MSR-VTT", + }, + { + description: "UCF101 Human Actions dataset consists of 13,320 video clips from YouTube, with 101 classes.", + id: "quchenyuan/UCF101-ZIP", + }, + { + description: "A high-quality dataset for human action recognition in YouTube videos.", + id: "nateraw/kinetics", + }, + { + description: "A dataset of video clips of humans performing pre-defined basic actions with everyday objects.", + id: "HuggingFaceM4/something_something_v2", + }, + { + description: "This dataset consists of text-video pairs and contains noisy samples with irrelevant video descriptions", + id: "HuggingFaceM4/webvid", + }, + { + description: "A dataset of short Flickr videos for the temporal localization of events with descriptions.", + id: "iejMac/CLIP-DiDeMo", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "Darth Vader is surfing on the waves.", + type: "text", + }, + ], + outputs: [ + { + filename: "text-to-video-output.gif", + type: "img", + }, + ], + }, + metrics: [ + { + description: "Inception Score uses an image classification model that predicts class labels and evaluates how distinct and diverse the images are. A higher score indicates better video generation.", + id: "is", + }, + { + description: "Frechet Inception Distance uses an image classification model to obtain image embeddings. The metric compares mean and standard deviation of the embeddings of real and generated images. A smaller score indicates better video generation.", + id: "fid", + }, + { + description: "Frechet Video Distance uses a model that captures coherence for changes in frames and the quality of each frame. A smaller score indicates better video generation.", + id: "fvd", + }, + { + description: "CLIPSIM measures similarity between video frames and text using an image-text similarity model. A higher score indicates better video generation.", + id: "clipsim", + }, + ], + models: [ + { + description: "A strong model for consistent video generation.", + id: "tencent/HunyuanVideo", + }, + { + description: "A text-to-video model with high fidelity motion and strong prompt adherence.", + id: "Lightricks/LTX-Video", + }, + { + description: "A text-to-video model focusing on physics-aware applications like robotics.", + id: "nvidia/Cosmos-1.0-Diffusion-7B-Text2World", + }, + { + description: "Very fast model for video generation.", + id: "Lightricks/LTX-Video-0.9.8-13B-distilled", + }, + ], + spaces: [ + { + description: "An application that generates video from text.", + id: "VideoCrafter/VideoCrafter", + }, + { + description: "Consistent video generation application.", + id: "Wan-AI/Wan2.1", + }, + { + description: "A cutting edge video generation application.", + id: "Pyramid-Flow/pyramid-flow", + }, + ], + summary: "Text-to-video models can be used in any application that requires generating consistent sequence of images from text. ", + widgetModels: ["Wan-AI/Wan2.2-TI2V-5B"], + youtubeId: undefined, +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4c3791350c974ef6ec40fc61866deb614344b143 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/inference.d.ts @@ -0,0 +1,58 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text To Video inference + */ +export interface TextToVideoInput { + /** + * The input text data (sometimes called "prompt") + */ + inputs: string; + /** + * Additional inference parameters for Text To Video + */ + parameters?: TextToVideoParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text To Video + */ +export interface TextToVideoParameters { + /** + * A higher guidance scale value encourages the model to generate videos closely linked to + * the text prompt, but values too high may cause saturation and other artifacts. + */ + guidance_scale?: number; + /** + * One or several prompt to guide what NOT to include in video generation. + */ + negative_prompt?: string[]; + /** + * The num_frames parameter determines how many video frames are generated. + */ + num_frames?: number; + /** + * The number of denoising steps. More denoising steps usually lead to a higher quality + * video at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * Seed for the random number generator. + */ + seed?: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Text To Video task + */ +export interface TextToVideoOutput { + /** + * The generated video returned as raw bytes in the payload. + */ + video: unknown; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..420f50f10ede66c1dfc384c98c82e226bb0a2f14 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/text-to-video/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/text-to-video/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..be4523e103c5a787df5558ad8ae91a69cda08bb5 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/token-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAuFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..35308663b96f0600598a2092a9592e1c98f257d8 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/data.js @@ -0,0 +1,85 @@ +const taskData = { + datasets: [ + { + description: "A widely used dataset useful to benchmark named entity recognition models.", + id: "eriktks/conll2003", + }, + { + description: "A multilingual dataset of Wikipedia articles annotated for named entity recognition in over 150 different languages.", + id: "unimelb-nlp/wikiann", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "My name is Omar and I live in Zürich.", + type: "text", + }, + ], + outputs: [ + { + text: "My name is Omar and I live in Zürich.", + tokens: [ + { + type: "PERSON", + start: 11, + end: 15, + }, + { + type: "GPE", + start: 30, + end: 36, + }, + ], + type: "text-with-tokens", + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + description: "A robust performance model to identify people, locations, organizations and names of miscellaneous entities.", + id: "dslim/bert-base-NER", + }, + { + description: "A strong model to identify people, locations, organizations and names in multiple languages.", + id: "FacebookAI/xlm-roberta-large-finetuned-conll03-english", + }, + { + description: "A token classification model specialized on medical entity recognition.", + id: "blaze999/Medical-NER", + }, + { + description: "Flair models are typically the state of the art in named entity recognition tasks.", + id: "flair/ner-english", + }, + ], + spaces: [ + { + description: "An application that can recognizes entities, extracts noun chunks and recognizes various linguistic features of each token.", + id: "spacy/gradio_pipeline_visualizer", + }, + ], + summary: "Token classification is a natural language understanding task in which a label is assigned to some tokens in a text. Some popular token classification subtasks are Named Entity Recognition (NER) and Part-of-Speech (PoS) tagging. NER models could be trained to identify specific entities in a text, such as dates, individuals and places; and PoS tagging would identify, for example, which words in a text are verbs, nouns, and punctuation marks.", + widgetModels: ["FacebookAI/xlm-roberta-large-finetuned-conll03-english"], + youtubeId: "wVHdVlPScxA", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..eac5ea3f3006f8f8909e67692888a5290d2c766f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/inference.d.ts @@ -0,0 +1,84 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Token Classification inference + */ +export interface TokenClassificationInput { + /** + * The input text data + */ + inputs: string; + /** + * Additional inference parameters for Token Classification + */ + parameters?: TokenClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Token Classification + */ +export interface TokenClassificationParameters { + /** + * The strategy used to fuse tokens based on model predictions + */ + aggregation_strategy?: TokenClassificationAggregationStrategy; + /** + * A list of labels to ignore + */ + ignore_labels?: string[]; + /** + * The number of overlapping tokens between chunks when splitting the input text. + */ + stride?: number; + [property: string]: unknown; +} +/** + * Do not aggregate tokens + * + * Group consecutive tokens with the same label in a single entity. + * + * Similar to "simple", also preserves word integrity (use the label predicted for the first + * token in a word). + * + * Similar to "simple", also preserves word integrity (uses the label with the highest + * score, averaged across the word's tokens). + * + * Similar to "simple", also preserves word integrity (uses the label with the highest score + * across the word's tokens). + */ +export type TokenClassificationAggregationStrategy = "none" | "simple" | "first" | "average" | "max"; +export type TokenClassificationOutput = TokenClassificationOutputElement[]; +/** + * Outputs of inference for the Token Classification task + */ +export interface TokenClassificationOutputElement { + /** + * The character position in the input where this group ends. + */ + end: number; + /** + * The predicted label for a single token + */ + entity?: string; + /** + * The predicted label for a group of one or more tokens + */ + entity_group?: string; + /** + * The associated score / probability + */ + score: number; + /** + * The character position in the input where this group begins. + */ + start: number; + /** + * The corresponding text + */ + word: string; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..c8afeae4c523c21a681a7a3453d1df2ec382d7a3 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/token-classification/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACxC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,6BAA6B,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC7C;;OAEG;IACH,oBAAoB,CAAC,EAAE,sCAAsC,CAAC;IAC9D;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,sCAAsC,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,GAAG,KAAK,CAAC;AACrG,MAAM,MAAM,yBAAyB,GAAG,gCAAgC,EAAE,CAAC;AAC3E;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IACZ;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/token-classification/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/translation/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/translation/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/translation/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/translation/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/translation/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..2db6b74c8d7f7cb9ecc51018f83b40c1749d641f --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/translation/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/translation/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAiEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/translation/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/translation/data.js new file mode 100644 index 0000000000000000000000000000000000000000..1c7e0afcaf4e3e97d16a23aed68479beea1f55f0 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/translation/data.js @@ -0,0 +1,63 @@ +const taskData = { + canonicalId: "text-generation", + datasets: [ + { + description: "A dataset of copyright-free books translated into 16 different languages.", + id: "Helsinki-NLP/opus_books", + }, + { + description: "An example of translation between programming languages. This dataset consists of functions in Java and C#.", + id: "google/code_x_glue_cc_code_to_code_trans", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "My name is Omar and I live in Zürich.", + type: "text", + }, + ], + outputs: [ + { + label: "Output", + content: "Mein Name ist Omar und ich wohne in Zürich.", + type: "text", + }, + ], + }, + metrics: [ + { + description: "BLEU score is calculated by counting the number of shared single or subsequent tokens between the generated sequence and the reference. Subsequent n tokens are called “n-grams”. Unigram refers to a single token while bi-gram refers to token pairs and n-grams refer to n subsequent tokens. The score ranges from 0 to 1, where 1 means the translation perfectly matched and 0 did not match at all", + id: "bleu", + }, + { + description: "", + id: "sacrebleu", + }, + ], + models: [ + { + description: "Very powerful model that can translate many languages between each other, especially low-resource languages.", + id: "facebook/nllb-200-1.3B", + }, + { + description: "A general-purpose Transformer that can be used to translate from English to German, French, or Romanian.", + id: "google-t5/t5-base", + }, + ], + spaces: [ + { + description: "An application that can translate between 100 languages.", + id: "Iker/Translate-100-languages", + }, + { + description: "An application that can translate between many languages.", + id: "Geonmo/nllb-translation-demo", + }, + ], + summary: "Translation is the task of converting text from one language to another.", + widgetModels: ["facebook/mbart-large-50-many-to-many-mmt"], + youtubeId: "1JvfrvZgi6c", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/translation/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/translation/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4df641a2cb758ae4ef24e0313e9dec7f9a4d8059 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/translation/inference.d.ts @@ -0,0 +1,64 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Translation inference + */ +export interface TranslationInput { + /** + * The text to translate. + */ + inputs: string; + /** + * Additional inference parameters for Translation + */ + parameters?: TranslationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Translation + */ +export interface TranslationParameters { + /** + * Whether to clean up the potential extra spaces in the text output. + */ + clean_up_tokenization_spaces?: boolean; + /** + * Additional parametrization of the text generation algorithm. + */ + generate_parameters?: { + [key: string]: unknown; + }; + /** + * The source language of the text. Required for models that can translate from multiple + * languages. + */ + src_lang?: string; + /** + * Target language to translate to. Required for models that can translate to multiple + * languages. + */ + tgt_lang?: string; + /** + * The truncation strategy to use. + */ + truncation?: TranslationTruncationStrategy; + [property: string]: unknown; +} +/** + * The truncation strategy to use. + */ +export type TranslationTruncationStrategy = "do_not_truncate" | "longest_first" | "only_first" | "only_second"; +/** + * Outputs of inference for the Translation task + */ +export interface TranslationOutput { + /** + * The translated text. + */ + translation_text: string; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/translation/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/translation/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..03d7567511144732e069f733012ac65822b2e539 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/translation/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/translation/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC;;OAEG;IACH,mBAAmB,CAAC,EAAE;QACrB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;IACF;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,UAAU,CAAC,EAAE,6BAA6B,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG,iBAAiB,GAAG,eAAe,GAAG,YAAY,GAAG,aAAa,CAAC;AAC/G;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/translation/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/translation/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/translation/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/unconditional-image-generation/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/unconditional-image-generation/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/unconditional-image-generation/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/unconditional-image-generation/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/unconditional-image-generation/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7ae1ca65807d46bdc5161ad0d750274be15306cb --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/unconditional-image-generation/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/unconditional-image-generation/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAmEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/unconditional-image-generation/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/unconditional-image-generation/data.js new file mode 100644 index 0000000000000000000000000000000000000000..9e388e1a329274d8d0eae24827a45a2802df50be --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/unconditional-image-generation/data.js @@ -0,0 +1,63 @@ +const taskData = { + datasets: [ + { + description: "The CIFAR-100 dataset consists of 60000 32x32 colour images in 100 classes, with 600 images per class.", + id: "cifar100", + }, + { + description: "Multiple images of celebrities, used for facial expression translation.", + id: "CelebA", + }, + ], + demo: { + inputs: [ + { + label: "Seed", + content: "42", + type: "text", + }, + { + label: "Number of images to generate:", + content: "4", + type: "text", + }, + ], + outputs: [ + { + filename: "unconditional-image-generation-output.jpeg", + type: "img", + }, + ], + }, + metrics: [ + { + description: "The inception score (IS) evaluates the quality of generated images. It measures the diversity of the generated images (the model predictions are evenly distributed across all possible labels) and their 'distinction' or 'sharpness' (the model confidently predicts a single label for each image).", + id: "Inception score (IS)", + }, + { + description: "The Fréchet Inception Distance (FID) evaluates the quality of images created by a generative model by calculating the distance between feature vectors for real and generated images.", + id: "Frećhet Inception Distance (FID)", + }, + ], + models: [ + { + description: "High-quality image generation model trained on the CIFAR-10 dataset. It synthesizes images of the ten classes presented in the dataset using diffusion probabilistic models, a class of latent variable models inspired by considerations from nonequilibrium thermodynamics.", + id: "google/ddpm-cifar10-32", + }, + { + description: "High-quality image generation model trained on the 256x256 CelebA-HQ dataset. It synthesizes images of faces using diffusion probabilistic models, a class of latent variable models inspired by considerations from nonequilibrium thermodynamics.", + id: "google/ddpm-celebahq-256", + }, + ], + spaces: [ + { + description: "An application that can generate realistic faces.", + id: "CompVis/celeba-latent-diffusion", + }, + ], + summary: "Unconditional image generation is the task of generating images with no condition in any context (like a prompt text or another image). Once trained, the model will create images that resemble its training data distribution.", + widgetModels: [""], + // TODO: Add related video + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7ddd46629dd3f6fdc2e7bebd42b3cbe2cb701d6e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/video-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA+Ef,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..70fba6ea1c544dfbc3ed0cd730848c6344b39f56 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/data.js @@ -0,0 +1,80 @@ +const taskData = { + datasets: [ + { + // TODO write proper description + description: "Benchmark dataset used for video classification with videos that belong to 400 classes.", + id: "kinetics400", + }, + ], + demo: { + inputs: [ + { + filename: "video-classification-input.gif", + type: "img", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Playing Guitar", + score: 0.514, + }, + { + label: "Playing Tennis", + score: 0.193, + }, + { + label: "Cooking", + score: 0.068, + }, + ], + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + // TO DO: write description + description: "Strong Video Classification model trained on the Kinetics 400 dataset.", + id: "google/vivit-b-16x2-kinetics400", + }, + { + // TO DO: write description + description: "Strong Video Classification model trained on the Kinetics 400 dataset.", + id: "microsoft/xclip-base-patch32", + }, + ], + spaces: [ + { + description: "An application that classifies video at different timestamps.", + id: "nateraw/lavila", + }, + { + description: "An application that classifies video.", + id: "fcakyon/video-classification", + }, + ], + summary: "Video classification is the task of assigning a label or class to an entire video. Videos are expected to have only one class for each video. Video classification models take a video as input and return a prediction about which class the video belongs to.", + widgetModels: [], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..97499d022d9a2406ecf3ae55c64b7855f16fcb62 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/inference.d.ts @@ -0,0 +1,61 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Video Classification inference + */ +export interface VideoClassificationInput { + /** + * The input video data + */ + inputs: unknown; + /** + * Additional inference parameters for Video Classification + */ + parameters?: VideoClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Video Classification + */ +export interface VideoClassificationParameters { + /** + * The sampling rate used to select frames from the video. + */ + frame_sampling_rate?: number; + /** + * The function to apply to the model outputs in order to retrieve the scores. + */ + function_to_apply?: ClassificationOutputTransform; + /** + * The number of sampled frames to consider for classification. + */ + num_frames?: number; + /** + * When specified, limits the output to the top K most probable classes. + */ + top_k?: number; + [property: string]: unknown; +} +/** + * The function to apply to the model outputs in order to retrieve the scores. + */ +export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none"; +export type VideoClassificationOutput = VideoClassificationOutputElement[]; +/** + * Outputs of inference for the Video Classification task + */ +export interface VideoClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..becc24194891238147e7e67309e2d194a029bce1 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/video-classification/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACxC;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC;IAChB;;OAEG;IACH,UAAU,CAAC,EAAE,6BAA6B,CAAC;IAC3C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC7C;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;OAEG;IACH,iBAAiB,CAAC,EAAE,6BAA6B,CAAC;IAClD;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAC3E,MAAM,MAAM,yBAAyB,GAAG,gCAAgC,EAAE,CAAC;AAC3E;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/video-classification/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/video-text-to-text/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/video-text-to-text/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/video-text-to-text/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/video-text-to-text/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/video-text-to-text/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..9fd072183cf99733f8fa8c8d9917610b5984eca7 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/video-text-to-text/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/video-text-to-text/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAqEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/video-text-to-text/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/video-text-to-text/data.js new file mode 100644 index 0000000000000000000000000000000000000000..bf19fbd5b0642e9070c1430d2938b838e3301eb6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/video-text-to-text/data.js @@ -0,0 +1,69 @@ +const taskData = { + datasets: [ + { + description: "Multiple-choice questions and answers about videos.", + id: "lmms-lab/Video-MME", + }, + { + description: "A dataset of instructions and question-answer pairs about videos.", + id: "lmms-lab/VideoChatGPT", + }, + { + description: "Large video understanding dataset.", + id: "HuggingFaceFV/finevideo", + }, + ], + demo: { + inputs: [ + { + filename: "video-text-to-text-input.gif", + type: "img", + }, + { + label: "Text Prompt", + content: "What is happening in this video?", + type: "text", + }, + ], + outputs: [ + { + label: "Answer", + content: "The video shows a series of images showing a fountain with water jets and a variety of colorful flowers and butterflies in the background.", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "A robust video-text-to-text model.", + id: "Vision-CAIR/LongVU_Qwen2_7B", + }, + { + description: "Strong video-text-to-text model with reasoning capabilities.", + id: "GoodiesHere/Apollo-LMMs-Apollo-7B-t32", + }, + { + description: "Strong video-text-to-text model.", + id: "HuggingFaceTB/SmolVLM2-2.2B-Instruct", + }, + ], + spaces: [ + { + description: "An application to chat with a video-text-to-text model.", + id: "llava-hf/video-llava", + }, + { + description: "A leaderboard for various video-text-to-text models.", + id: "opencompass/openvlm_video_leaderboard", + }, + { + description: "An application to generate highlights from a video.", + id: "HuggingFaceTB/SmolVLM2-HighlightGenerator", + }, + ], + summary: "Video-text-to-text models take in a video and a text prompt and output text. These models are also called video-language models.", + widgetModels: [""], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/video-to-video/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/video-to-video/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/video-to-video/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/video-to-video/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/video-to-video/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..69a3a40fc74e02b3eae44a8bc24cd05d4de41b61 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/video-to-video/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/video-to-video/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA8Df,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/video-to-video/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/video-to-video/data.js new file mode 100644 index 0000000000000000000000000000000000000000..7b73d0271d5e409b5b37eefd3030d5fe1b62a6c0 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/video-to-video/data.js @@ -0,0 +1,63 @@ +const taskData = { + datasets: [ + { + description: "Dataset with detailed annotations for training and benchmarking video instance editing.", + id: "suimu/VIRESET", + }, + { + description: "Dataset to evaluate models on long video generation and understanding.", + id: "zhangsh2001/LongV-EVAL", + }, + { + description: "Collection of 104 demo videos from the SeedVR/SeedVR2 series showcasing model outputs.", + id: "Iceclear/SeedVR_VideoDemos", + }, + ], + demo: { + inputs: [ + { + filename: "input.gif", + type: "img", + }, + ], + outputs: [ + { + filename: "output.gif", + type: "img", + }, + ], + }, + metrics: [], + models: [ + { + description: "Model for editing outfits, character, and scenery in videos.", + id: "decart-ai/Lucy-Edit-Dev", + }, + { + description: "Framework that uses 3D mesh proxies for precise, consistent video editing.", + id: "LeoLau/Shape-for-Motion", + }, + { + description: "Model for generating physics-aware videos from input videos and control conditions.", + id: "nvidia/Cosmos-Transfer2.5-2B", + }, + { + description: "A model to upscale videos at input, designed for seamless use with ComfyUI.", + id: "numz/SeedVR2_comfyUI", + }, + ], + spaces: [ + { + description: "Interactive demo space for Lucy-Edit-Dev video editing.", + id: "decart-ai/lucy-edit-dev", + }, + { + description: "Demo space for SeedVR2-3B showcasing video upscaling and restoration.", + id: "ByteDance-Seed/SeedVR2-3B", + }, + ], + summary: "Video-to-video models take one or more videos as input and generate new videos as output. They can enhance quality, interpolate frames, modify styles, or create new motion dynamics, enabling creative applications, video production, and research.", + widgetModels: [], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/visual-document-retrieval/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-document-retrieval/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-document-retrieval/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/visual-document-retrieval/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-document-retrieval/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..186ad37def7b38501f210bec15304d1c534ebdd4 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-document-retrieval/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/visual-document-retrieval/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAuEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/visual-document-retrieval/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-document-retrieval/data.js new file mode 100644 index 0000000000000000000000000000000000000000..d13d117acbc5ede912f623d62bebf3f623d927c2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-document-retrieval/data.js @@ -0,0 +1,71 @@ +const taskData = { + datasets: [ + { + description: "A large dataset used to train visual document retrieval models.", + id: "vidore/colpali_train_set", + }, + ], + demo: { + inputs: [ + { + filename: "input.png", + type: "img", + }, + { + label: "Question", + content: "Is the model in this paper the fastest for inference?", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Page 10", + score: 0.7, + }, + { + label: "Page 11", + score: 0.06, + }, + { + label: "Page 9", + score: 0.003, + }, + ], + }, + ], + }, + isPlaceholder: false, + metrics: [ + { + description: "NDCG@k scores ranked recommendation lists for top-k results. 0 is the worst, 1 is the best.", + id: "Normalized Discounted Cumulative Gain at K", + }, + ], + models: [ + { + description: "Very accurate visual document retrieval model for multilingual queries and documents.", + id: "vidore/colqwen2-v1.0", + }, + { + description: "Very fast and efficient visual document retrieval model that can also take in other modalities like audio.", + id: "Tevatron/OmniEmbed-v0.1", + }, + ], + spaces: [ + { + description: "A leaderboard of visual document retrieval models.", + id: "vidore/vidore-leaderboard", + }, + { + description: "Visual retrieval augmented generation demo based on ColQwen2 model.", + id: "vidore/visual-rag-tool", + }, + ], + summary: "Visual document retrieval is the task of searching for relevant image-based documents, such as PDFs. These models take a text query and multiple documents as input and return the top-most relevant documents and relevancy scores as output.", + widgetModels: [""], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a5bb4ed58442694fd0d8a32ae92012c6083a496c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/visual-question-answering/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA4Ff,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/data.js new file mode 100644 index 0000000000000000000000000000000000000000..2482cfd2a35a51534d185c4963e4c14fcf8a2512 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/data.js @@ -0,0 +1,91 @@ +const taskData = { + datasets: [ + { + description: "A widely used dataset containing questions (with answers) about images.", + id: "Graphcore/vqa", + }, + { + description: "A dataset to benchmark visual reasoning based on text in images.", + id: "facebook/textvqa", + }, + ], + demo: { + inputs: [ + { + filename: "elephant.jpeg", + type: "img", + }, + { + label: "Question", + content: "What is in this image?", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "elephant", + score: 0.97, + }, + { + label: "elephants", + score: 0.06, + }, + { + label: "animal", + score: 0.003, + }, + ], + }, + ], + }, + isPlaceholder: false, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "Measures how much a predicted answer differs from the ground truth based on the difference in their semantic meaning.", + id: "wu-palmer similarity", + }, + ], + models: [ + { + description: "A visual question answering model trained to convert charts and plots to text.", + id: "google/deplot", + }, + { + description: "A visual question answering model trained for mathematical reasoning and chart derendering from images.", + id: "google/matcha-base", + }, + { + description: "A strong visual question answering that answers questions from book covers.", + id: "google/pix2struct-ocrvqa-large", + }, + ], + spaces: [ + { + description: "An application that compares visual question answering models across different tasks.", + id: "merve/pix2struct", + }, + { + description: "An application that can answer questions based on images.", + id: "nielsr/vilt-vqa", + }, + { + description: "An application that can caption images and answer questions about a given image. ", + id: "Salesforce/BLIP", + }, + { + description: "An application that can caption images and answer questions about a given image. ", + id: "vumichien/Img2Prompt", + }, + ], + summary: "Visual Question Answering is the task of answering open-ended questions based on an image. They output natural language responses to natural language questions.", + widgetModels: ["dandelin/vilt-b32-finetuned-vqa"], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..4736a2d592969c569d706924af56da6583ee0891 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/inference.d.ts @@ -0,0 +1,61 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Visual Question Answering inference + */ +export interface VisualQuestionAnsweringInput { + /** + * One (image, question) pair to answer + */ + inputs: VisualQuestionAnsweringInputData; + /** + * Additional inference parameters for Visual Question Answering + */ + parameters?: VisualQuestionAnsweringParameters; + [property: string]: unknown; +} +/** + * One (image, question) pair to answer + */ +export interface VisualQuestionAnsweringInputData { + /** + * The image. + */ + image: unknown; + /** + * The question to answer based on the image. + */ + question: string; + [property: string]: unknown; +} +/** + * Additional inference parameters for Visual Question Answering + */ +export interface VisualQuestionAnsweringParameters { + /** + * The number of answers to return (will be chosen by order of likelihood). Note that we + * return less than topk answers if there are not enough options available within the + * context. + */ + top_k?: number; + [property: string]: unknown; +} +export type VisualQuestionAnsweringOutput = VisualQuestionAnsweringOutputElement[]; +/** + * Outputs of inference for the Visual Question Answering task + */ +export interface VisualQuestionAnsweringOutputElement { + /** + * The answer to the question + */ + answer?: string; + /** + * The associated score / probability + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..a0f2508331d6046894c38d3b358c1ff70aef4bb5 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/visual-question-answering/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC5C;;OAEG;IACH,MAAM,EAAE,gCAAgC,CAAC;IACzC;;OAEG;IACH,UAAU,CAAC,EAAE,iCAAiC,CAAC;IAC/C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,iCAAiC;IACjD;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,6BAA6B,GAAG,oCAAoC,EAAE,CAAC;AACnF;;GAEG;AACH,MAAM,WAAW,oCAAoC;IACpD;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/visual-question-answering/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..8ae7284ee6e360d3a874ce894594cd62659f2cf9 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/zero-shot-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAqEf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..539edc5c2c588358bd971031628012c0720fd416 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/data.js @@ -0,0 +1,68 @@ +const taskData = { + datasets: [ + { + description: "A widely used dataset used to benchmark multiple variants of text classification.", + id: "nyu-mll/glue", + }, + { + description: "The Multi-Genre Natural Language Inference (MultiNLI) corpus is a crowd-sourced collection of 433k sentence pairs annotated with textual entailment information.", + id: "nyu-mll/multi_nli", + }, + { + description: "FEVER is a publicly available dataset for fact extraction and verification against textual sources.", + id: "fever/fever", + }, + ], + demo: { + inputs: [ + { + label: "Text Input", + content: "Dune is the best movie ever.", + type: "text", + }, + { + label: "Candidate Labels", + content: "CINEMA, ART, MUSIC", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "CINEMA", + score: 0.9, + }, + { + label: "ART", + score: 0.1, + }, + { + label: "MUSIC", + score: 0.0, + }, + ], + }, + ], + }, + metrics: [], + models: [ + { + description: "Powerful zero-shot text classification model.", + id: "facebook/bart-large-mnli", + }, + { + description: "Cutting-edge zero-shot multilingual text classification model.", + id: "MoritzLaurer/ModernBERT-large-zeroshot-v2.0", + }, + { + description: "Zero-shot text classification model that can be used for topic and sentiment classification.", + id: "knowledgator/gliclass-modern-base-v2.0-init", + }, + ], + spaces: [], + summary: "Zero-shot text classification is a task in natural language processing where a model is trained on a set of labeled examples but is then able to classify new examples from previously unseen classes.", + widgetModels: ["facebook/bart-large-mnli"], +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2d9281733c9a213395c423d02a1e1e11b3ebf8d2 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/inference.d.ts @@ -0,0 +1,56 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Zero Shot Classification inference + */ +export interface ZeroShotClassificationInput { + /** + * The text to classify + */ + inputs: string; + /** + * Additional inference parameters for Zero Shot Classification + */ + parameters: ZeroShotClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Zero Shot Classification + */ +export interface ZeroShotClassificationParameters { + /** + * The set of possible class labels to classify the text into. + */ + candidate_labels: string[]; + /** + * The sentence used in conjunction with `candidate_labels` to attempt the text + * classification by replacing the placeholder with the candidate labels. + */ + hypothesis_template?: string; + /** + * Whether multiple candidate labels can be true. If false, the scores are normalized such + * that the sum of the label likelihoods for each sequence is 1. If true, the labels are + * considered independent and probabilities are normalized for each candidate. + */ + multi_label?: boolean; + [property: string]: unknown; +} +export type ZeroShotClassificationOutput = ZeroShotClassificationOutputElement[]; +/** + * Outputs of inference for the Zero Shot Classification task + */ +export interface ZeroShotClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..1a132031034157cd5ba0a7b20a870ef33c337044 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/zero-shot-classification/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC3C;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,UAAU,EAAE,gCAAgC,CAAC;IAC7C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,4BAA4B,GAAG,mCAAmC,EAAE,CAAC;AACjF;;GAEG;AACH,MAAM,WAAW,mCAAmC;IACnD;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-classification/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..18ffbbd656c7012e0d48ef08fb8065a700213622 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/zero-shot-image-classification/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cAmFf,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/data.js new file mode 100644 index 0000000000000000000000000000000000000000..340ee53ccde9a7ca4299dda737355f9a37d06431 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/data.js @@ -0,0 +1,83 @@ +const taskData = { + datasets: [ + { + // TODO write proper description + description: "", + id: "", + }, + ], + demo: { + inputs: [ + { + filename: "image-classification-input.jpeg", + type: "img", + }, + { + label: "Classes", + content: "cat, dog, bird", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Cat", + score: 0.664, + }, + { + label: "Dog", + score: 0.329, + }, + { + label: "Bird", + score: 0.008, + }, + ], + }, + ], + }, + metrics: [ + { + description: "Computes the number of times the correct label appears in top K labels predicted", + id: "top-K accuracy", + }, + ], + models: [ + { + description: "Multilingual image classification model for 80 languages.", + id: "visheratin/mexma-siglip", + }, + { + description: "Strong zero-shot image classification model.", + id: "google/siglip2-base-patch16-224", + }, + { + description: "Robust zero-shot image classification model.", + id: "intfloat/mmE5-mllama-11b-instruct", + }, + { + description: "Powerful zero-shot image classification model supporting 94 languages.", + id: "jinaai/jina-clip-v2", + }, + { + description: "Strong image classification model for biomedical domain.", + id: "microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224", + }, + ], + spaces: [ + { + description: "An application that leverages zero-shot image classification to find best captions to generate an image. ", + id: "pharma/CLIP-Interrogator", + }, + { + description: "An application to compare different zero-shot image classification models. ", + id: "merve/compare_clip_siglip", + }, + ], + summary: "Zero-shot image classification is the task of classifying previously unseen classes during training of a model.", + widgetModels: ["google/siglip-so400m-patch14-224"], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..8e80503ecf772e75dbaa641b0533c1f7d61ed22c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/inference.d.ts @@ -0,0 +1,50 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Zero Shot Image Classification inference + */ +export interface ZeroShotImageClassificationInput { + /** + * The input image data to classify as a base64-encoded string. + */ + inputs: Blob; + /** + * Additional inference parameters for Zero Shot Image Classification + */ + parameters: ZeroShotImageClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Zero Shot Image Classification + */ +export interface ZeroShotImageClassificationParameters { + /** + * The candidate labels for this image + */ + candidate_labels: string[]; + /** + * The sentence used in conjunction with `candidate_labels` to attempt the image + * classification by replacing the placeholder with the candidate labels. + */ + hypothesis_template?: string; + [property: string]: unknown; +} +export type ZeroShotImageClassificationOutput = ZeroShotImageClassificationOutputElement[]; +/** + * Outputs of inference for the Zero Shot Image Classification task + */ +export interface ZeroShotImageClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6c9de1ceb345e5457fd3755db377e73e3ac9b836 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/zero-shot-image-classification/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAChD;;OAEG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,EAAE,qCAAqC,CAAC;IAClD,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,qCAAqC;IACrD;;OAEG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,iCAAiC,GAAG,wCAAwC,EAAE,CAAC;AAC3F;;GAEG;AACH,MAAM,WAAW,wCAAwC;IACxD;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-image-classification/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc8794d0d9385cdff0fd88a7c158222897e8ea50 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/data.d.ts @@ -0,0 +1,4 @@ +import type { TaskDataCustom } from "../index.js"; +declare const taskData: TaskDataCustom; +export default taskData; +//# sourceMappingURL=data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d3836f53f66b8db2c829bfe01d8fe3505d975832 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"data.d.ts","sourceRoot":"","sources":["../../../../src/tasks/zero-shot-object-detection/data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,QAAA,MAAM,QAAQ,EAAE,cA8Df,CAAC;AAEF,eAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/data.js b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/data.js new file mode 100644 index 0000000000000000000000000000000000000000..3c4601325d9991ba356865903eb1b8b3e2858602 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/data.js @@ -0,0 +1,60 @@ +const taskData = { + datasets: [], + demo: { + inputs: [ + { + filename: "zero-shot-object-detection-input.jpg", + type: "img", + }, + { + label: "Classes", + content: "cat, dog, bird", + type: "text", + }, + ], + outputs: [ + { + filename: "zero-shot-object-detection-output.jpg", + type: "img", + }, + ], + }, + metrics: [ + { + description: "The Average Precision (AP) metric is the Area Under the PR Curve (AUC-PR). It is calculated for each class separately", + id: "Average Precision", + }, + { + description: "The Mean Average Precision (mAP) metric is the overall average of the AP values", + id: "Mean Average Precision", + }, + { + description: "The APα metric is the Average Precision at the IoU threshold of a α value, for example, AP50 and AP75", + id: "APα", + }, + ], + models: [ + { + description: "Solid zero-shot object detection model.", + id: "openmmlab-community/mm_grounding_dino_large_all", + }, + { + description: "Cutting-edge zero-shot object detection model.", + id: "fushh7/LLMDet", + }, + ], + spaces: [ + { + description: "A demo to compare different zero-shot object detection models per output and latency.", + id: "ariG23498/zero-shot-od", + }, + { + description: "A demo that combines a zero-shot object detection and mask generation model for zero-shot segmentation.", + id: "merve/OWLSAM", + }, + ], + summary: "Zero-shot object detection is a computer vision task to detect objects and their classes in images, without any prior training or knowledge of the classes. Zero-shot object detection models receive an image as input, as well as a list of candidate classes, and output the bounding boxes and labels where the objects have been detected.", + widgetModels: [], + youtubeId: "", +}; +export default taskData; diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/inference.d.ts b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/inference.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..00bfad1dfb69b256e202c541e3c00d200be46ca6 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/inference.d.ts @@ -0,0 +1,61 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Zero Shot Object Detection inference + */ +export interface ZeroShotObjectDetectionInput { + /** + * The input image data as a base64-encoded string. + */ + inputs: Blob; + /** + * Additional inference parameters for Zero Shot Object Detection + */ + parameters: ZeroShotObjectDetectionParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Zero Shot Object Detection + */ +export interface ZeroShotObjectDetectionParameters { + /** + * The candidate labels for this image + */ + candidate_labels: string[]; + [property: string]: unknown; +} +/** + * The predicted bounding box. Coordinates are relative to the top left corner of the input + * image. + */ +export interface BoundingBox { + xmax: number; + xmin: number; + ymax: number; + ymin: number; + [property: string]: unknown; +} +export type ZeroShotObjectDetectionOutput = ZeroShotObjectDetectionOutputElement[]; +/** + * Outputs of inference for the Zero Shot Object Detection task + */ +export interface ZeroShotObjectDetectionOutputElement { + /** + * The predicted bounding box. Coordinates are relative to the top left corner of the input + * image. + */ + box: BoundingBox; + /** + * A candidate label + */ + label: string; + /** + * The associated score / probability + */ + score: number; + [property: string]: unknown; +} +//# sourceMappingURL=inference.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/inference.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/inference.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..d31807a4e74fc4e9f381f911e9838ca83df9e351 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/inference.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/zero-shot-object-detection/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC5C;;OAEG;IACH,MAAM,EAAE,IAAI,CAAC;IACb;;OAEG;IACH,UAAU,EAAE,iCAAiC,CAAC;IAC9C,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,iCAAiC;IACjD;;OAEG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD,MAAM,MAAM,6BAA6B,GAAG,oCAAoC,EAAE,CAAC;AACnF;;GAEG;AACH,MAAM,WAAW,oCAAoC;IACpD;;;OAGG;IACH,GAAG,EAAE,WAAW,CAAC;IACjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/inference.js b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/inference.js new file mode 100644 index 0000000000000000000000000000000000000000..cb0ff5c3b541f646105198ee23ac0fc3d805023e --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tasks/zero-shot-object-detection/inference.js @@ -0,0 +1 @@ +export {}; diff --git a/node_modules/@huggingface/tasks/dist/esm/tokenizer-data.d.ts b/node_modules/@huggingface/tasks/dist/esm/tokenizer-data.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..5863744080832186bd5f7482bd3f55c446872e84 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tokenizer-data.d.ts @@ -0,0 +1,26 @@ +export declare const SPECIAL_TOKENS_ATTRIBUTES: readonly ["bos_token", "eos_token", "unk_token", "sep_token", "pad_token", "cls_token", "mask_token"]; +/** + * Public interface for a tokenizer's special tokens mapping + */ +export interface AddedToken { + __type: "AddedToken"; + content?: string; + lstrip?: boolean; + normalized?: boolean; + rstrip?: boolean; + single_word?: boolean; +} +export type SpecialTokensMap = { + [key in (typeof SPECIAL_TOKENS_ATTRIBUTES)[number]]?: string | AddedToken | null; +}; +/** + * Public interface for tokenizer config + */ +export interface TokenizerConfig extends SpecialTokensMap { + use_default_system_prompt?: boolean; + chat_template?: string | Array<{ + name: string; + template: string; + }>; +} +//# sourceMappingURL=tokenizer-data.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tokenizer-data.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/tokenizer-data.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..8f5b0a57b829b2f6c0060ac3f36d00e093d8195c --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tokenizer-data.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"tokenizer-data.d.ts","sourceRoot":"","sources":["../../src/tokenizer-data.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,yBAAyB,uGAS5B,CAAC;AAEX;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,CAAC;CACtB;AACD,MAAM,MAAM,gBAAgB,GAAG;KAC7B,GAAG,IAAI,CAAC,OAAO,yBAAyB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI;CAChF,CAAC;AACF;;GAEG;AACH,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACxD,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,aAAa,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACnE"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/tokenizer-data.js b/node_modules/@huggingface/tasks/dist/esm/tokenizer-data.js new file mode 100644 index 0000000000000000000000000000000000000000..c77687634546d37b3467e5721eed57bf45a52644 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/tokenizer-data.js @@ -0,0 +1,10 @@ +export const SPECIAL_TOKENS_ATTRIBUTES = [ + "bos_token", + "eos_token", + "unk_token", + "sep_token", + "pad_token", + "cls_token", + "mask_token", + // additional_special_tokens (TODO) +]; diff --git a/node_modules/@huggingface/tasks/dist/esm/widget-example.d.ts b/node_modules/@huggingface/tasks/dist/esm/widget-example.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..3e99fa12e050a01ce6c7e167346e18184dde2f96 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/widget-example.d.ts @@ -0,0 +1,83 @@ +/** + * See default-widget-inputs.ts for the default widget inputs, this files only contains the types + */ +import type { ChatCompletionInputMessage } from "./tasks/index.js"; +type TableData = Record; +export type WidgetExampleOutputLabels = Array<{ + label: string; + score: number; +}>; +export interface WidgetExampleOutputAnswerScore { + answer: string; + score: number; +} +export interface WidgetExampleOutputText { + text: string; +} +export interface WidgetExampleOutputUrl { + url: string; +} +export type WidgetExampleOutput = WidgetExampleOutputLabels | WidgetExampleOutputAnswerScore | WidgetExampleOutputText | WidgetExampleOutputUrl; +export interface WidgetExampleBase { + example_title?: string; + group?: string; + /** + * Potential overrides to API parameters for this specific example + * (takes precedences over the model card metadata's inference.parameters) + */ + parameters?: { + aggregation_strategy?: string; + top_k?: number; + top_p?: number; + temperature?: number; + max_new_tokens?: number; + do_sample?: boolean; + negative_prompt?: string; + guidance_scale?: number; + num_inference_steps?: number; + }; + /** + * Optional output + */ + output?: TOutput; +} +export interface WidgetExampleChatInput extends WidgetExampleBase { + messages: ChatCompletionInputMessage[]; +} +export interface WidgetExampleTextInput extends WidgetExampleBase { + text: string; +} +export interface WidgetExampleTextAndContextInput extends WidgetExampleTextInput { + context: string; +} +export interface WidgetExampleTextAndTableInput extends WidgetExampleTextInput { + table: TableData; +} +export interface WidgetExampleAssetInput extends WidgetExampleBase { + src: string; +} +export interface WidgetExampleAssetAndPromptInput extends WidgetExampleAssetInput { + prompt: string; +} +export type WidgetExampleAssetAndTextInput = WidgetExampleAssetInput & WidgetExampleTextInput; +export type WidgetExampleAssetAndZeroShotInput = WidgetExampleAssetInput & WidgetExampleZeroShotTextInput; +export interface WidgetExampleStructuredDataInput extends WidgetExampleBase { + structured_data: TableData; +} +export interface WidgetExampleTableDataInput extends WidgetExampleBase { + table: TableData; +} +export interface WidgetExampleZeroShotTextInput extends WidgetExampleTextInput { + text: string; + candidate_labels: string; + multi_class: boolean; +} +export interface WidgetExampleSentenceSimilarityInput extends WidgetExampleBase { + source_sentence: string; + sentences: string[]; +} +export type WidgetExample = WidgetExampleChatInput | WidgetExampleTextInput | WidgetExampleTextAndContextInput | WidgetExampleTextAndTableInput | WidgetExampleAssetInput | WidgetExampleAssetAndPromptInput | WidgetExampleAssetAndTextInput | WidgetExampleAssetAndZeroShotInput | WidgetExampleStructuredDataInput | WidgetExampleTableDataInput | WidgetExampleZeroShotTextInput | WidgetExampleSentenceSimilarityInput; +type KeysOfUnion = T extends unknown ? keyof T : never; +export type WidgetExampleAttribute = KeysOfUnion; +export {}; +//# sourceMappingURL=widget-example.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/widget-example.d.ts.map b/node_modules/@huggingface/tasks/dist/esm/widget-example.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..2a7075f6b0bbf8789a174607be1ffbb685c86987 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/widget-example.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"widget-example.d.ts","sourceRoot":"","sources":["../../src/widget-example.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,kBAAkB,CAAC;AAEnE,KAAK,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;AAGrD,MAAM,MAAM,yBAAyB,GAAG,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAChF,MAAM,WAAW,8BAA8B;IAC9C,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACd;AACD,MAAM,WAAW,uBAAuB;IACvC,IAAI,EAAE,MAAM,CAAC;CACb;AACD,MAAM,WAAW,sBAAsB;IACtC,GAAG,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,MAAM,mBAAmB,GAC5B,yBAAyB,GACzB,8BAA8B,GAC9B,uBAAuB,GACvB,sBAAsB,CAAC;AAG1B,MAAM,WAAW,iBAAiB,CAAC,OAAO;IACzC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,UAAU,CAAC,EAAE;QAEZ,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAE9B,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,SAAS,CAAC,EAAE,OAAO,CAAC;QAEpB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,mBAAmB,CAAC,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,sBAAsB,CAAC,OAAO,GAAG,mBAAmB,CAAE,SAAQ,iBAAiB,CAAC,OAAO,CAAC;IACxG,QAAQ,EAAE,0BAA0B,EAAE,CAAC;CACvC;AAED,MAAM,WAAW,sBAAsB,CAAC,OAAO,GAAG,mBAAmB,CAAE,SAAQ,iBAAiB,CAAC,OAAO,CAAC;IACxG,IAAI,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,gCAAgC,CAChD,OAAO,GAAG,mBAAmB,CAC5B,SAAQ,sBAAsB,CAAC,OAAO,CAAC;IACxC,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,8BAA8B,CAAC,OAAO,GAAG,mBAAmB,CAAE,SAAQ,sBAAsB,CAAC,OAAO,CAAC;IACrH,KAAK,EAAE,SAAS,CAAC;CACjB;AAED,MAAM,WAAW,uBAAuB,CAAC,OAAO,GAAG,mBAAmB,CAAE,SAAQ,iBAAiB,CAAC,OAAO,CAAC;IACzG,GAAG,EAAE,MAAM,CAAC;CACZ;AACD,MAAM,WAAW,gCAAgC,CAChD,OAAO,GAAG,mBAAmB,CAC5B,SAAQ,uBAAuB,CAAC,OAAO,CAAC;IACzC,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,8BAA8B,CAAC,OAAO,GAAG,mBAAmB,IAAI,uBAAuB,CAAC,OAAO,CAAC,GAC3G,sBAAsB,CAAC,OAAO,CAAC,CAAC;AAEjC,MAAM,MAAM,kCAAkC,CAAC,OAAO,GAAG,mBAAmB,IAAI,uBAAuB,CAAC,OAAO,CAAC,GAC/G,8BAA8B,CAAC,OAAO,CAAC,CAAC;AAEzC,MAAM,WAAW,gCAAgC,CAAC,OAAO,GAAG,mBAAmB,CAAE,SAAQ,iBAAiB,CAAC,OAAO,CAAC;IAClH,eAAe,EAAE,SAAS,CAAC;CAC3B;AAED,MAAM,WAAW,2BAA2B,CAAC,OAAO,GAAG,mBAAmB,CAAE,SAAQ,iBAAiB,CAAC,OAAO,CAAC;IAC7G,KAAK,EAAE,SAAS,CAAC;CACjB;AAED,MAAM,WAAW,8BAA8B,CAAC,OAAO,GAAG,mBAAmB,CAAE,SAAQ,sBAAsB,CAAC,OAAO,CAAC;IACrH,IAAI,EAAE,MAAM,CAAC;IACb,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,oCAAoC,CACpD,OAAO,GAAG,mBAAmB,CAC5B,SAAQ,iBAAiB,CAAC,OAAO,CAAC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,EAAE,CAAC;CACpB;AAID,MAAM,MAAM,aAAa,CAAC,OAAO,GAAG,mBAAmB,IACpD,sBAAsB,CAAC,OAAO,CAAC,GAC/B,sBAAsB,CAAC,OAAO,CAAC,GAC/B,gCAAgC,CAAC,OAAO,CAAC,GACzC,8BAA8B,CAAC,OAAO,CAAC,GACvC,uBAAuB,CAAC,OAAO,CAAC,GAChC,gCAAgC,CAAC,OAAO,CAAC,GACzC,8BAA8B,CAAC,OAAO,CAAC,GACvC,kCAAkC,CAAC,OAAO,CAAC,GAC3C,gCAAgC,CAAC,OAAO,CAAC,GACzC,2BAA2B,CAAC,OAAO,CAAC,GACpC,8BAA8B,CAAC,OAAO,CAAC,GACvC,oCAAoC,CAAC,OAAO,CAAC,CAAC;AAEjD,KAAK,WAAW,CAAC,CAAC,IAAI,CAAC,SAAS,OAAO,GAAG,MAAM,CAAC,GAAG,KAAK,CAAC;AAE1D,MAAM,MAAM,sBAAsB,GAAG,WAAW,CAAC,aAAa,CAAC,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/dist/esm/widget-example.js b/node_modules/@huggingface/tasks/dist/esm/widget-example.js new file mode 100644 index 0000000000000000000000000000000000000000..7cb6cbe5c996656d6eea133f9dc21f6a2899f607 --- /dev/null +++ b/node_modules/@huggingface/tasks/dist/esm/widget-example.js @@ -0,0 +1,4 @@ +/** + * See default-widget-inputs.ts for the default widget inputs, this files only contains the types + */ +export {}; diff --git a/node_modules/@huggingface/tasks/package.json b/node_modules/@huggingface/tasks/package.json new file mode 100644 index 0000000000000000000000000000000000000000..b7d19d410a4c2781c8a5cb29848b93a1555eb0c4 --- /dev/null +++ b/node_modules/@huggingface/tasks/package.json @@ -0,0 +1,57 @@ +{ + "name": "@huggingface/tasks", + "version": "0.21.20", + "description": "List of ML tasks for huggingface.co/tasks", + "keywords": [ + "hub", + "huggingface", + "languages" + ], + "license": "MIT", + "author": "Hugging Face", + "repository": "https://github.com/huggingface/huggingface.js.git", + "source": "./src/index.ts", + "files": [ + "dist", + "src", + "tsconfig.json" + ], + "type": "module", + "main": "./dist/commonjs/index.js", + "module": "./dist/esm/index.js", + "types": "./dist/commonjs/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/index.js" + }, + "require": { + "types": "./dist/commonjs/index.d.ts", + "default": "./dist/commonjs/index.js" + } + }, + "./package.json": "./package.json" + }, + "publishConfig": { + "access": "public" + }, + "tshy": { + "exports": { + ".": "./src/index.ts", + "./package.json": "./package.json" + } + }, + "scripts": { + "lint": "eslint --quiet --fix --ext .cjs,.ts .", + "lint:check": "eslint --ext .cjs,.ts .", + "format": "oxfmt .", + "format:check": "oxfmt --check .", + "build": "tshy", + "watch:cjs": "tsc --declaration --outdir dist/commonjs --module commonjs --watch", + "watch:esm": "tsc --declaration --outdir dist/esm --watch", + "watch": "npm-run-all --parallel watch:esm watch:cjs", + "check": "tsc", + "test": "vitest run" + } +} \ No newline at end of file diff --git a/node_modules/@huggingface/tasks/src/agent-harnesses.ts b/node_modules/@huggingface/tasks/src/agent-harnesses.ts new file mode 100644 index 0000000000000000000000000000000000000000..fb230aa0ac5d14017641aa0fa2b3ee6c012082cf --- /dev/null +++ b/node_modules/@huggingface/tasks/src/agent-harnesses.ts @@ -0,0 +1,222 @@ +/** + * Registry of AI coding agents / harnesses known to use the Hugging Face Hub. + * + * To add your harness, append an entry below keyed by its `id` (the name used + * when reporting Hub activity), and list the environment variable(s) that + * identify it. + */ +export interface AgentHarness { + /** + * Human-readable name of the harness, e.g. displayed in a leaderboard. + */ + prettyLabel: string; + /** + * URL to the harness's code repository (usually on GitHub). + */ + repoUrl?: string; + /** + * URL to the harness's documentation or website. + */ + docsUrl?: string; + /** + * Short description of the harness. + */ + description?: string; + /** + * Environment variable(s) that identify this harness, mapped to the value + * pattern they must match. Detection matches if ANY entry matches. + * + * The value pattern is one of: + * - `"*"`: the variable is set to any (non-empty) value + * - `""`: the variable equals this exact value + * - `"*"`: the variable value starts with `` (fuzzy match, resolved client-side) + * + * If not provided, the harness is detected through the standard AI_AGENT / AGENT variables only. + */ + envVars?: Record; +} + +/** + * Standard environment variables that any tool can set to identify itself. + * When one of these is set, its value is used directly as the agent `id` + * (matched against the keys of `AGENT_HARNESSES`); unrecognized values are + * reported as `"unknown"`. + */ +export const STANDARD_AGENT_ENV_VARS = ["AI_AGENT", "AGENT"] as const; + +/** + * Add your new agent harness here. + * + * /!\ IMPORTANT + * + * Insertion order matters for detection priority: harnesses are checked from + * top to bottom and the first match wins. In particular, `cowork` must stay + * before `claude-code` so the more specific signal takes priority when both + * `CLAUDE_CODE` and `CLAUDE_CODE_IS_COWORK` are set. + */ +export const AGENT_HARNESSES = { + antigravity: { + prettyLabel: "Antigravity", + docsUrl: "https://antigravity.google", + description: "Agentic development platform from Google built around Gemini.", + envVars: { ANTIGRAVITY_AGENT: "*" }, + }, + "augment-cli": { + prettyLabel: "Augment CLI", + repoUrl: "https://github.com/augmentcode/auggie", + docsUrl: "https://www.augmentcode.com", + description: "Auggie, the command-line coding agent from Augment Code.", + envVars: { AUGMENT_AGENT: "*" }, + }, + cline: { + prettyLabel: "Cline", + repoUrl: "https://github.com/cline/cline", + docsUrl: "https://cline.bot", + description: "Open-source autonomous coding agent for VS Code.", + envVars: { CLINE_ACTIVE: "*" }, + }, + cowork: { + // must stay before `claude-code` so the more specific signal takes priority when both `CLAUDE_CODE` and `CLAUDE_CODE_IS_COWORK` are set. + prettyLabel: "Cowork", + docsUrl: "https://claude.com/product/cowork", + description: "Anthropic's agent for autonomous knowledge work, built on top of Claude Code.", + envVars: { CLAUDE_CODE_IS_COWORK: "*" }, + }, + "claude-code": { + prettyLabel: "Claude Code", + repoUrl: "https://github.com/anthropics/claude-code", + docsUrl: "https://code.claude.com/docs", + description: "Anthropic's agentic coding tool that lives in your terminal.", + envVars: { CLAUDECODE: "*", CLAUDE_CODE: "*" }, + }, + codex: { + prettyLabel: "Codex", + repoUrl: "https://github.com/openai/codex", + docsUrl: "https://developers.openai.com/codex", + description: "OpenAI's lightweight coding agent that runs in your terminal.", + envVars: { CODEX_SANDBOX: "*", CODEX_CI: "*", CODEX_THREAD_ID: "*" }, + }, + crush: { + prettyLabel: "Crush", + repoUrl: "https://github.com/charmbracelet/crush", + docsUrl: "https://github.com/charmbracelet/crush", + description: "Charm's open-source AI coding agent for the terminal.", + envVars: { CRUSH: "*" }, + }, + "gemini-cli": { + prettyLabel: "Gemini CLI", + repoUrl: "https://github.com/google-gemini/gemini-cli", + docsUrl: "https://geminicli.com", + description: "Google's open-source terminal AI coding agent powered by Gemini models.", + envVars: { GEMINI_CLI: "*" }, + }, + "github-copilot": { + prettyLabel: "GitHub Copilot", + docsUrl: "https://docs.github.com/copilot", + description: "GitHub's AI coding assistant.", + envVars: { COPILOT_MODEL: "*", COPILOT_ALLOW_ALL: "*", COPILOT_GITHUB_TOKEN: "*" }, + }, + goose: { + prettyLabel: "Goose", + repoUrl: "https://github.com/aaif-goose/goose", + docsUrl: "https://goose-docs.ai/", + description: "Open-source, extensible AI agent, originally from Block and now part of the Agentic AI Foundation.", + envVars: { GOOSE_TERMINAL: "*" }, + }, + "hermes-agent": { + prettyLabel: "Hermes Agent", + repoUrl: "https://github.com/NousResearch/hermes-agent", + docsUrl: "https://hermes-agent.nousresearch.com/docs", + description: "Nous Research's self-improving, multi-provider terminal AI agent.", + envVars: { HERMES_SESSION_ID: "*" }, + }, + hi: { + prettyLabel: "hi", + repoUrl: "https://github.com/PipeNetwork/hi", + docsUrl: "https://github.com/PipeNetwork/hi#readme", + description: "Rust terminal coding agent with verification-in-the-loop.", + }, + "kilo-code": { + prettyLabel: "Kilo Code", + repoUrl: "https://github.com/Kilo-Org/kilocode", + docsUrl: "https://kilocode.ai/docs", + description: "Open-source agentic coding agent for VS Code, JetBrains, and the terminal.", + envVars: { KILOCODE_FEATURE: "*" }, + }, + kiro: { + prettyLabel: "Kiro", + docsUrl: "https://kiro.dev", + description: "AWS's agentic IDE for spec-driven AI software development.", + envVars: { AGENT_CONTEXT_OUT: "*" }, + }, + openclaw: { + prettyLabel: "OpenClaw", + repoUrl: "https://github.com/openclaw/openclaw", + docsUrl: "https://openclaw.ai", + description: "Open-source, self-hosted personal AI assistant that runs on your own devices.", + envVars: { OPENCLAW_SHELL: "*" }, + }, + opencode: { + prettyLabel: "opencode", + repoUrl: "https://github.com/anomalyco/opencode", + docsUrl: "https://opencode.ai", + description: "Open-source AI coding agent built for the terminal.", + envVars: { OPENCODE_CLIENT: "*" }, + }, + pi: { + prettyLabel: "Pi", + repoUrl: "https://github.com/earendil-works/pi", + docsUrl: "https://pi.dev", + description: "Minimal, self-extensible terminal coding agent with a unified multi-provider LLM API.", + envVars: { PI_CODING_AGENT: "*" }, + }, + replit: { + prettyLabel: "Replit", + docsUrl: "https://replit.com", + description: "Cloud development environment with an AI coding agent.", + envVars: { REPL_ID: "*" }, + }, + trae: { + prettyLabel: "Trae", + docsUrl: "https://trae.ai", + description: "AI-powered IDE from ByteDance.", + envVars: { TRAE_AI_SHELL_ID: "*" }, + }, + warp: { + prettyLabel: "Warp", + repoUrl: "https://github.com/warpdotdev/Warp", + docsUrl: "https://docs.warp.dev", + description: "AI-powered terminal with an agentic Agent Mode.", + envVars: { TERM_PROGRAM: "WarpTerminal" }, + }, + zed: { + prettyLabel: "Zed", + repoUrl: "https://github.com/zed-industries/zed", + docsUrl: "https://zed.dev", + description: "High-performance code editor with an integrated AI agent panel and terminal.", + envVars: { ZED_TERM: "*" }, + }, + "cursor-cli": { + // Kept near the bottom (and before `cursor`): when another agent runs inside the Cursor editor's terminal, + // its child processes inherit `CURSOR_TRACE_ID`, so `cursor` must stay a low-priority fallback and lose to + // the agent's own marker. `cursor-cli` is the more specific Cursor signal (`CURSOR_AGENT`), so it comes first. + prettyLabel: "Cursor CLI", + docsUrl: "https://cursor.com/docs/cli/overview", + description: "Cursor's coding agent for the command line.", + envVars: { CURSOR_AGENT: "*" }, + }, + cursor: { + prettyLabel: "Cursor", + docsUrl: "https://cursor.com", + description: "AI-powered code editor.", + envVars: { CURSOR_TRACE_ID: "*" }, + }, + devin: { + prettyLabel: "Devin", + docsUrl: "https://devin.ai", + description: "Autonomous AI software engineer from Cognition.", + }, +} satisfies Record; + +/// List of the agent harnesses known to the Hub +export type AgentHarnessKey = keyof typeof AGENT_HARNESSES; diff --git a/node_modules/@huggingface/tasks/src/dataset-libraries.ts b/node_modules/@huggingface/tasks/src/dataset-libraries.ts new file mode 100644 index 0000000000000000000000000000000000000000..fbe85fdb3b94c5e5d4b0bad837d08e5817d45096 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/dataset-libraries.ts @@ -0,0 +1,101 @@ +/** + * Elements configurable by a dataset library. + */ +export interface DatasetLibraryUiElement { + /** + * Pretty name of the library. + * displayed (in tags?, and) on the main + * call-to-action button on the dataset page. + */ + prettyLabel: string; + /** + * Repo name of the library's (usually on GitHub) code repo + */ + repoName: string; + /** + * URL to library's (usually on GitHub) code repo + */ + repoUrl: string; + /** + * URL to library's docs + */ + docsUrl?: string; +} + +export const DATASET_LIBRARIES_UI_ELEMENTS = { + mlcroissant: { + prettyLabel: "Croissant", + repoName: "croissant", + repoUrl: "https://github.com/mlcommons/croissant/tree/main/python/mlcroissant", + docsUrl: "https://huggingface.co/docs/dataset-viewer/mlcroissant", + }, + webdataset: { + prettyLabel: "WebDataset", + repoName: "webdataset", + repoUrl: "https://github.com/webdataset/webdataset", + docsUrl: "https://huggingface.co/docs/hub/datasets-webdataset", + }, + datasets: { + prettyLabel: "Datasets", + repoName: "datasets", + repoUrl: "https://github.com/huggingface/datasets", + docsUrl: "https://huggingface.co/docs/hub/datasets-usage", + }, + pandas: { + prettyLabel: "pandas", + repoName: "pandas", + repoUrl: "https://github.com/pandas-dev/pandas", + docsUrl: "https://huggingface.co/docs/hub/datasets-pandas", + }, + dask: { + prettyLabel: "Dask", + repoName: "dask", + repoUrl: "https://github.com/dask/dask", + docsUrl: "https://huggingface.co/docs/hub/datasets-dask", + }, + distilabel: { + prettyLabel: "Distilabel", + repoName: "distilabel", + repoUrl: "https://github.com/argilla-io/distilabel", + docsUrl: "https://huggingface.co/docs/hub/datasets-distilabel", + }, + fiftyone: { + prettyLabel: "FiftyOne", + repoName: "fiftyone", + repoUrl: "https://github.com/voxel51/fiftyone", + docsUrl: "https://huggingface.co/docs/hub/datasets-fiftyone", + }, + lance: { + prettyLabel: "Lance", + repoName: "lance", + repoUrl: "https://github.com/lance-format/lance", + docsUrl: "https://huggingface.co/docs/hub/datasets-lance", + }, + argilla: { + prettyLabel: "Argilla", + repoName: "argilla", + repoUrl: "https://github.com/argilla-io/argilla", + docsUrl: "https://huggingface.co/docs/hub/datasets-argilla", + }, + polars: { + prettyLabel: "Polars", + repoName: "polars", + repoUrl: "https://github.com/pola-rs/polars", + docsUrl: "https://huggingface.co/docs/hub/datasets-polars", + }, + duckdb: { + prettyLabel: "DuckDB", + repoName: "duckdb", + repoUrl: "https://github.com/duckdb/duckdb", + docsUrl: "https://huggingface.co/docs/hub/datasets-duckdb", + }, + datadesigner: { + prettyLabel: "NeMo Data Designer", + repoName: "datadesigner", + repoUrl: "https://github.com/NVIDIA-NeMo/DataDesigner", + docsUrl: "https://nvidia-nemo.github.io/DataDesigner/", + }, +} satisfies Record; + +/// List of the dataset libraries supported by the Hub +export type DatasetLibraryKey = keyof typeof DATASET_LIBRARIES_UI_ELEMENTS; diff --git a/node_modules/@huggingface/tasks/src/default-widget-inputs.ts b/node_modules/@huggingface/tasks/src/default-widget-inputs.ts new file mode 100644 index 0000000000000000000000000000000000000000..7185abe6daa6fdfb50aa562f50585fdcee6aed5d --- /dev/null +++ b/node_modules/@huggingface/tasks/src/default-widget-inputs.ts @@ -0,0 +1,697 @@ +import type { WidgetExample } from "./widget-example.js"; +import type { WidgetType } from "./pipelines.js"; + +type LanguageCode = string; + +type PerLanguageMapping = Map; + +/// NOTE TO CONTRIBUTORS: +/// +/// When adding sample inputs for a new language, you don't +/// necessarily have to translate the inputs from existing languages. +/// (which were quite random to begin with) +/// +/// i.e. Feel free to be creative and provide better samples. +// + +/// The placeholder will be replaced by the correct mask token +/// in the following examples, depending on the model type +/// +/// see [INTERNAL] github.com/huggingface/moon-landing/blob/c5c3d45fe0ab27347b3ab27bdad646ef20732351/server/lib/App.ts#L254 +// + +const MAPPING_EN: PerLanguageMapping = new Map([ + ["text-classification", [`I like you. I love you`]], + [ + "token-classification", + [ + `My name is Wolfgang and I live in Berlin`, + `My name is Sarah and I live in London`, + `My name is Clara and I live in Berkeley, California.`, + ], + ], + [ + "table-question-answering", + [ + { + text: `How many stars does the transformers repository have?`, + table: { + Repository: ["Transformers", "Datasets", "Tokenizers"], + Stars: [36542, 4512, 3934], + Contributors: [651, 77, 34], + "Programming language": ["Python", "Python", "Rust, Python and NodeJS"], + }, + }, + ], + ], + [ + "question-answering", + [ + { + text: `Where do I live?`, + context: `My name is Wolfgang and I live in Berlin`, + }, + { + text: `Where do I live?`, + context: `My name is Sarah and I live in London`, + }, + { + text: `What's my name?`, + context: `My name is Clara and I live in Berkeley.`, + }, + { + text: `Which name is also used to describe the Amazon rainforest in English?`, + context: `The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain "Amazonas" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species.`, + }, + ], + ], + [ + "zero-shot-classification", + [ + { + text: "I have a problem with my iphone that needs to be resolved asap!", + candidate_labels: "urgent, not urgent, phone, tablet, computer", + multi_class: true, + }, + { + text: "Last week I upgraded my iOS version and ever since then my phone has been overheating whenever I use your app.", + candidate_labels: "mobile, website, billing, account access", + multi_class: false, + }, + { + text: "A new model offers an explanation for how the Galilean satellites formed around the solar system’s largest world. Konstantin Batygin did not set out to solve one of the solar system’s most puzzling mysteries when he went for a run up a hill in Nice, France. Dr. Batygin, a Caltech researcher, best known for his contributions to the search for the solar system’s missing “Planet Nine,” spotted a beer bottle. At a steep, 20 degree grade, he wondered why it wasn’t rolling down the hill. He realized there was a breeze at his back holding the bottle in place. Then he had a thought that would only pop into the mind of a theoretical astrophysicist: “Oh! This is how Europa formed.” Europa is one of Jupiter’s four large Galilean moons. And in a paper published Monday in the Astrophysical Journal, Dr. Batygin and a co-author, Alessandro Morbidelli, a planetary scientist at the Côte d’Azur Observatory in France, present a theory explaining how some moons form around gas giants like Jupiter and Saturn, suggesting that millimeter-sized grains of hail produced during the solar system’s formation became trapped around these massive worlds, taking shape one at a time into the potentially habitable moons we know today.", + candidate_labels: "space & cosmos, scientific discovery, microbiology, robots, archeology", + multi_class: true, + }, + ], + ], + ["translation", [`My name is Wolfgang and I live in Berlin`, `My name is Sarah and I live in London`]], + [ + "summarization", + [ + `The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct.`, + ], + ], + [ + "conversational", + [ + `Hi, what can you help me with?`, + `What is 84 * 3 / 2?`, + `Tell me an interesting fact about the universe!`, + `Explain quantum computing in simple terms.`, + ], + ], + [ + "text-generation", + [ + `My name is Julien and I like to`, + `I like traveling by train because`, + `Paris is an amazing place to visit,`, + `Once upon a time,`, + ], + ], + ["fill-mask", [`Paris is the of France.`, `The goal of life is .`]], + [ + "sentence-similarity", + [ + { + source_sentence: "That is a happy person", + sentences: ["That is a happy dog", "That is a very happy person", "Today is a sunny day"], + }, + ], + ], +]); + +const MAPPING_ZH: PerLanguageMapping = new Map([ + ["text-classification", [`我喜欢你。 我爱你`]], + ["token-classification", [`我叫沃尔夫冈，我住在柏林。`, `我叫萨拉，我住在伦敦。`, `我叫克拉拉，我住在加州伯克利。`]], + [ + "question-answering", + [ + { + text: `我住在哪里？`, + context: `我叫沃尔夫冈，我住在柏林。`, + }, + { + text: `我住在哪里？`, + context: `我叫萨拉，我住在伦敦。`, + }, + { + text: `我的名字是什么？`, + context: `我叫克拉拉，我住在伯克利。`, + }, + ], + ], + ["translation", [`我叫沃尔夫冈，我住在柏林。`, `我叫萨拉，我住在伦敦。`]], + [ + "zero-shot-classification", + [ + { + text: "房间干净明亮，非常不错", + candidate_labels: "这是一条差评, 这是一条好评", + }, + ], + ], + [ + "summarization", + [ + `该塔高324米（1063英尺），与一幢81层的建筑物一样高，是巴黎最高的建筑物。 它的底座是方形的，每边长125米（410英尺）。 在建造过程中，艾菲尔铁塔超过了华盛顿纪念碑，成为世界上最高的人造结构，它保持了41年的头衔，直到1930年纽约市的克莱斯勒大楼竣工。这是第一个到达300米高度的结构。 由于1957年在塔顶增加了广播天线，因此它现在比克莱斯勒大厦高5.2米（17英尺）。 除发射器外，艾菲尔铁塔是法国第二高的独立式建筑，仅次于米劳高架桥。`, + ], + ], + [ + "text-generation", + [`我叫朱利安，我喜欢`, `我叫托马斯，我的主要`, `我叫玛丽亚，我最喜欢的`, `我叫克拉拉，我是`, `从前，`], + ], + ["fill-mask", [`巴黎是国的首都。`, `生活的真谛是。`]], + [ + "sentence-similarity", + [ + { + source_sentence: "那是 個快樂的人", + sentences: ["那是 條快樂的狗", "那是 個非常幸福的人", "今天是晴天"], + }, + ], + ], +]); + +const MAPPING_FR: PerLanguageMapping = new Map([ + ["text-classification", [`Je t'apprécie beaucoup. Je t'aime.`]], + ["token-classification", [`Mon nom est Wolfgang et je vis à Berlin`]], + [ + "question-answering", + [ + { + text: `Où est-ce que je vis?`, + context: `Mon nom est Wolfgang et je vis à Berlin`, + }, + ], + ], + ["translation", [`Mon nom est Wolfgang et je vis à Berlin`]], + [ + "summarization", + [ + `La tour fait 324 mètres (1,063 pieds) de haut, environ la même hauteur qu'un immeuble de 81 étages, et est la plus haute structure de Paris. Sa base est carrée, mesurant 125 mètres (410 pieds) sur chaque côté. Durant sa construction, la tour Eiffel surpassa le Washington Monument pour devenir la plus haute structure construite par l'homme dans le monde, un titre qu'elle conserva pendant 41 ans jusqu'à l'achèvement du Chrysler Building à New-York City en 1930. Ce fut la première structure à atteindre une hauteur de 300 mètres. Avec l'ajout d'une antenne de radiodiffusion au sommet de la tour Eiffel en 1957, celle-ci redevint plus haute que le Chrysler Building de 5,2 mètres (17 pieds). En excluant les transmetteurs, elle est la seconde plus haute structure autoportante de France après le viaduc de Millau.`, + ], + ], + ["text-generation", [`Mon nom est Julien et j'aime`, `Mon nom est Thomas et mon principal`, `Il était une fois`]], + ["fill-mask", [`Paris est la de la France.`]], + [ + "sentence-similarity", + [ + { + source_sentence: "C'est une personne heureuse", + sentences: [ + "C'est un chien heureux", + "C'est une personne très heureuse", + "Aujourd'hui est une journée ensoleillée", + ], + }, + ], + ], +]); + +const MAPPING_ES: PerLanguageMapping = new Map([ + ["text-classification", [`Te quiero. Te amo.`]], + ["token-classification", [`Me llamo Wolfgang y vivo en Berlin`]], + [ + "question-answering", + [ + { + text: `¿Dónde vivo?`, + context: `Me llamo Wolfgang y vivo en Berlin`, + }, + { + text: `¿Quién inventó el submarino?`, + context: `Isaac Peral fue un murciano que inventó el submarino`, + }, + { + text: `¿Cuántas personas hablan español?`, + context: `El español es el segundo idioma más hablado del mundo con más de 442 millones de hablantes`, + }, + ], + ], + [ + "translation", + [ + `Me llamo Wolfgang y vivo en Berlin`, + `Los ingredientes de una tortilla de patatas son: huevos, patatas y cebolla`, + ], + ], + [ + "summarization", + [ + `La torre tiene 324 metros (1.063 pies) de altura, aproximadamente la misma altura que un edificio de 81 pisos y la estructura más alta de París. Su base es cuadrada, mide 125 metros (410 pies) a cada lado. Durante su construcción, la Torre Eiffel superó al Washington Monument para convertirse en la estructura artificial más alta del mundo, un título que mantuvo durante 41 años hasta que el Chrysler Building en la ciudad de Nueva York se terminó en 1930. Fue la primera estructura en llegar Una altura de 300 metros. Debido a la adición de una antena de transmisión en la parte superior de la torre en 1957, ahora es más alta que el Chrysler Building en 5,2 metros (17 pies). Excluyendo los transmisores, la Torre Eiffel es la segunda estructura independiente más alta de Francia después del Viaducto de Millau.`, + ], + ], + [ + "text-generation", + [ + `Me llamo Julien y me gusta`, + `Me llamo Thomas y mi principal`, + `Me llamo Manuel y trabajo en`, + `Érase una vez,`, + `Si tú me dices ven, `, + ], + ], + ["fill-mask", [`Mi nombre es y vivo en Nueva York.`, `El español es un idioma muy en el mundo.`]], + [ + "sentence-similarity", + [ + { + source_sentence: "Esa es una persona feliz", + sentences: ["Ese es un perro feliz", "Esa es una persona muy feliz", "Hoy es un día soleado"], + }, + ], + ], +]); + +const MAPPING_RU: PerLanguageMapping = new Map([ + ["text-classification", [`Ты мне нравишься. Я тебя люблю`]], + ["token-classification", [`Меня зовут Вольфганг и я живу в Берлине`]], + [ + "question-answering", + [ + { + text: `Где живу?`, + context: `Меня зовут Вольфганг и я живу в Берлине`, + }, + ], + ], + ["translation", [`Меня зовут Вольфганг и я живу в Берлине`]], + [ + "summarization", + [ + `Высота башни составляет 324 метра (1063 фута), примерно такая же высота, как у 81-этажного здания, и самое высокое сооружение в Париже. Его основание квадратно, размером 125 метров (410 футов) с любой стороны. Во время строительства Эйфелева башня превзошла монумент Вашингтона, став самым высоким искусственным сооружением в мире, и этот титул она удерживала в течение 41 года до завершения строительство здания Крайслер в Нью-Йорке в 1930 году. Это первое сооружение которое достигло высоты 300 метров. Из-за добавления вещательной антенны на вершине башни в 1957 году она сейчас выше здания Крайслер на 5,2 метра (17 футов). За исключением передатчиков, Эйфелева башня является второй самой высокой отдельно стоящей структурой во Франции после виадука Мийо.`, + ], + ], + ["text-generation", [`Меня зовут Жюльен и`, `Меня зовут Томас и мой основной`, `Однажды`]], + ["fill-mask", [`Меня зовут и я инженер живущий в Нью-Йорке.`]], + [ + "sentence-similarity", + [ + { + source_sentence: "Это счастливый человек", + sentences: ["Это счастливая собака", "Это очень счастливый человек", "Сегодня солнечный день"], + }, + ], + ], +]); + +const MAPPING_UK: PerLanguageMapping = new Map([ + ["translation", [`Мене звати Вольфґанґ і я живу в Берліні.`]], + ["fill-mask", [`Мене звати .`]], +]); + +const MAPPING_IT: PerLanguageMapping = new Map([ + ["text-classification", [`Mi piaci. Ti amo`]], + [ + "token-classification", + [ + `Mi chiamo Wolfgang e vivo a Berlino`, + `Mi chiamo Sarah e vivo a Londra`, + `Mi chiamo Clara e vivo a Berkeley in California.`, + ], + ], + [ + "question-answering", + [ + { + text: `Dove vivo?`, + context: `Mi chiamo Wolfgang e vivo a Berlino`, + }, + { + text: `Dove vivo?`, + context: `Mi chiamo Sarah e vivo a Londra`, + }, + { + text: `Come mio chiamo?`, + context: `Mi chiamo Clara e vivo a Berkeley.`, + }, + ], + ], + ["translation", [`Mi chiamo Wolfgang e vivo a Berlino`, `Mi chiamo Sarah e vivo a Londra`]], + [ + "summarization", + [ + `La torre degli Asinelli è una delle cosiddette due torri di Bologna, simbolo della città, situate in piazza di porta Ravegnana, all'incrocio tra le antiche strade San Donato (ora via Zamboni), San Vitale, Maggiore e Castiglione. Eretta, secondo la tradizione, fra il 1109 e il 1119 dal nobile Gherardo Asinelli, la torre è alta 97,20 metri, pende verso ovest per 2,23 metri e presenta all'interno una scalinata composta da 498 gradini. Ancora non si può dire con certezza quando e da chi fu costruita la torre degli Asinelli. Si presume che la torre debba il proprio nome a Gherardo Asinelli, il nobile cavaliere di fazione ghibellina al quale se ne attribuisce la costruzione, iniziata secondo una consolidata tradizione l'11 ottobre 1109 e terminata dieci anni dopo, nel 1119.`, + ], + ], + [ + "text-generation", + [ + `Mi chiamo Loreto e mi piace`, + `Mi chiamo Thomas e il mio principale`, + `Mi chiamo Marianna, la mia cosa preferita`, + `Mi chiamo Clara e sono`, + `C'era una volta`, + ], + ], + ["fill-mask", [`Roma è la d'Italia.`, `Lo scopo della vita è .`]], + [ + "sentence-similarity", + [ + { + source_sentence: "Questa è una persona felice", + sentences: ["Questo è un cane felice", "Questa è una persona molto felice", "Oggi è una giornata di sole"], + }, + ], + ], +]); + +const MAPPING_FA: PerLanguageMapping = new Map([ + [ + "text-classification", + [`پروژه به موقع تحویل شد و همه چیز خوب بود.`, `سیب‌زمینی بی‌کیفیت بود.`, `قیمت و کیفیت عالی`, `خوب نبود اصلا`], + ], + [ + "token-classification", + [ + `این سریال به صورت رسمی در تاریخ دهم می ۲۰۱۱ توسط شبکه فاکس برای پخش رزرو شد.`, + `دفتر مرکزی شرکت پارس‌مینو در شهر اراک در استان مرکزی قرار دارد.`, + `وی در سال ۲۰۱۳ درگذشت و مسئول خاکسپاری و اقوامش برای او مراسم یادبود گرفتند.`, + ], + ], + [ + "question-answering", + [ + { + text: `من کجا زندگی میکنم؟`, + context: `نام من پژمان است و در گرگان زندگی میکنم.`, + }, + { + text: `نامم چیست و کجا زندگی می‌کنم؟`, + context: `اسمم سارا است و در آفریقای جنوبی زندگی میکنم.`, + }, + { + text: `نام من چیست؟`, + context: `من مریم هستم و در تبریز زندگی می‌کنم.`, + }, + { + text: `بیشترین مساحت جنگل آمازون در کدام کشور است؟`, + context: [ + "آمازون نام بزرگ‌ترین جنگل بارانی جهان است که در شمال آمریکای جنوبی قرار گرفته و بیشتر آن در خاک برزیل و پرو", + "جای دارد. بیش از نیمی از همه جنگل‌های بارانی باقی‌مانده در جهان در آمازون قرار دارد.", + "مساحت جنگل‌های آمازون ۵٫۵ میلیون کیلومتر مربع است که بین ۹ کشور تقسیم شده‌است.", + ].join("\n"), + }, + ], + ], + [ + "translation", + [ + "بیشتر مساحت جنگل‌های آمازون در حوضه آبریز رود آمازون و ۱۱۰۰ شاخه آن واقع شده‌است.", + "مردمان نَبَطی از هزاره‌های یکم و دوم پیش از میلاد در این منطقه زندگی می‌کردند.", + ], + ], + [ + "summarization", + [ + [ + "شاهنامه اثر حکیم ابوالقاسم فردوسی توسی، حماسه‌ای منظوم، بر حسب دست نوشته‌های ", + "موجود دربرگیرنده نزدیک به ۵۰٬۰۰۰ بیت تا نزدیک به ۶۱٬۰۰۰ بیت و یکی از ", + "بزرگ‌ترین و برجسته‌ترین سروده‌های حماسی جهان است که سرایش آن دست‌آوردِ ", + "دست‌کم سی سال کارِ پیوستهٔ این سخن‌سرای نامدار ایرانی است. موضوع این شاهکار ادبی،", + " افسانه‌ها و تاریخ ایران از آغاز تا حملهٔ عرب‌ها به ایران در سدهٔ هفتم میلادی است", + " (شاهنامه از سه بخش اسطوره، پهلوانی و تاریخی تشکیل شده‌است) که در چهار", + " دودمان پادشاهیِ پیشدادیان، کیانیان، اشکانیان و ساسانیان گنجانده می‌شود.", + " شاهنامه بر وزن «فَعولُن فعولن فعولن فَعَلْ»، در بحرِ مُتَقارِبِ مثمَّنِ محذوف نگاشته شده‌است.", + "هنگامی که زبان دانش و ادبیات در ایران زبان عربی بود، فردوسی، با سرودن شاهنامه", + " با ویژگی‌های هدف‌مندی که داشت، زبان پارسی را زنده و پایدار کرد. یکی از ", + " بن‌مایه‌های مهمی که فردوسی برای سرودن شاهنامه از آن استفاده کرد،", + " شاهنامهٔ ابومنصوری بود. شاهنامه نفوذ بسیاری در جهت‌گیری ", + " فرهنگ فارسی و نیز بازتاب‌های شکوه‌مندی در ادبیات جهان داشته‌است و شاعران ", + " بزرگی مانند گوته و ویکتور هوگو از آن به نیکی یاد کرده‌اند.", + ].join("\n"), + ], + ], + ["text-generation", ["اسم من نازنین است و من", "روزی روزگاری"]], + [ + "fill-mask", + [ + `زندگی یک سوال است و این که چگونه کنیم پاسخ این سوال!`, + `زندگی از مرگ پرسید: چرا همه من را دارند اما از تو متنفرند؟`, + ], + ], +]); + +const MAPPING_AR: PerLanguageMapping = new Map([ + ["text-classification", [`أحبك. أهواك`]], + [ + "token-classification", + [`إسمي محمد وأسكن في برلين`, `إسمي ساره وأسكن في لندن`, `إسمي سامي وأسكن في القدس في فلسطين.`], + ], + [ + "question-answering", + [ + { + text: `أين أسكن؟`, + context: `إسمي محمد وأسكن في بيروت`, + }, + { + text: `أين أسكن؟`, + context: `إسمي ساره وأسكن في لندن`, + }, + { + text: `ما اسمي؟`, + context: `اسمي سعيد وأسكن في حيفا.`, + }, + { + text: `ما لقب خالد بن الوليد بالعربية؟`, + context: `خالد بن الوليد من أبطال وقادة الفتح الإسلامي وقد تحدثت عنه اللغات الإنجليزية والفرنسية والإسبانية ولقب بسيف الله المسلول.`, + }, + ], + ], + ["translation", [`إسمي محمد وأسكن في برلين`, `إسمي ساره وأسكن في لندن`]], + [ + "summarization", + [ + `تقع الأهرامات في الجيزة قرب القاهرة في مصر وقد بنيت منذ عدة قرون، وقيل إنها كانت قبورا للفراعنة وتم بناؤها بعملية هندسية رائعة واستقدمت حجارتها من جبل المقطم وتم نقلها بالسفن أو على الرمل، وما تزال شامخة ويقصدها السياح من كافة أرجاء المعمورة.`, + ], + ], + [ + "text-generation", + [ + `إسمي محمد وأحب أن`, + `دع المكارم لا ترحل لبغيتها - واقعد فإنك أنت الطاعم الكاسي.`, + `لماذا نحن هنا؟`, + `القدس مدينة تاريخية، بناها الكنعانيون في`, + `كان يا ما كان في قديم الزمان`, + ], + ], + ["fill-mask", [`باريس فرنسا.`, `فلسفة الحياة هي .`]], + [ + "sentence-similarity", + [ + { + source_sentence: "هذا شخص سعيد", + sentences: ["هذا كلب سعيد", "هذا شخص سعيد جدا", "اليوم هو يوم مشمس"], + }, + ], + ], +]); + +const MAPPING_BN: PerLanguageMapping = new Map([ + ["text-classification", [`বাঙালির ঘরে ঘরে আজ নবান্ন উৎসব।`]], + [ + "token-classification", + [`আমার নাম জাহিদ এবং আমি ঢাকায় বাস করি।`, `তিনি গুগলে চাকরী করেন।`, `আমার নাম সুস্মিতা এবং আমি কলকাতায় বাস করি।`], + ], + ["translation", [`আমার নাম জাহিদ, আমি রংপুরে বাস করি।`, `আপনি কী আজকে বাসায় আসবেন?`]], + [ + "summarization", + [ + `‘ইকোনমিস্ট’ লিখেছে, অ্যান্টিবডির চার মাস স্থায়ী হওয়ার খবরটি দুই কারণে আনন্দের। অ্যান্টিবডি যত দিন পর্যন্ত শরীরে টিকবে, তত দিন সংক্রমণ থেকে সুরক্ষিত থাকা সম্ভব। অর্থাৎ, এমন এক টিকার প্রয়োজন হবে, যা অ্যান্টিবডির উত্পাদনকে প্ররোচিত করতে পারে এবং দীর্ঘস্থায়ী সুরক্ষা দিতে পারে। এগুলো খুঁজে বের করাও সহজ। এটি আভাস দেয়, ব্যাপক হারে অ্যান্টিবডি শনাক্তকরণ ফলাফল মোটামুটি নির্ভুল হওয়া উচিত। দ্বিতীয় আরেকটি গবেষণার নেতৃত্ব দিয়েছেন যুক্তরাজ্যের মেডিকেল রিসার্চ কাউন্সিলের (এমআরসি) ইমিউনোলজিস্ট তাও দং। তিনি টি-সেল শনাক্তকরণে কাজ করেছেন। টি-সেল শনাক্তকরণের প্রক্রিয়া অবশ্য অ্যান্টিবডির মতো এত আলোচিত নয়। তবে সংক্রমণের বিরুদ্ধে লড়াই এবং দীর্ঘমেয়াদি সুরক্ষায় সমান গুরুত্বপূর্ণ ভূমিকা পালন করে। গবেষণাসংক্রান্ত নিবন্ধ প্রকাশিত হয়েছে ‘নেচার ইমিউনোলজি’ সাময়িকীতে। তাঁরা বলছেন, গবেষণার ক্ষেত্রে কোভিড-১৯ মৃদু সংক্রমণের শিকার ২৮ ব্যক্তির রক্তের নমুনা, ১৪ জন গুরুতর অসুস্থ ও ১৬ জন সুস্থ ব্যক্তির রক্তের নমুনা পরীক্ষা করেছেন। গবেষণা নিবন্ধে বলা হয়, সংক্রমিত ব্যক্তিদের ক্ষেত্রে টি-সেলের তীব্র প্রতিক্রিয়া তাঁরা দেখেছেন। এ ক্ষেত্রে মৃদু ও গুরুতর অসুস্থ ব্যক্তিদের ক্ষেত্রে প্রতিক্রিয়ার ভিন্নতা পাওয়া গেছে।`, + ], + ], + ["text-generation", [`আমি রতন এবং আমি`, `তুমি যদি চাও তবে`, `মিথিলা আজকে বড্ড`]], + ["fill-mask", [`আমি বাংলায় গাই।`, `আমি খুব ভালোবাসি। `]], + [ + "question-answering", + [ + { + text: `প্রথম এশিয়া কাপ ক্রিকেট টুর্নামেন্ট কোথায় অনুষ্ঠিত হয় ?`, + context: `প্রথম টুর্নামেন্ট অনুষ্ঠিত হয় ১৯৮৪ সালে সংযুক্ত আরব আমিরাত এর শারজাহ তে যেখানে কাউন্সিলের মূল অফিস ছিল (১৯৯৫ পর্যন্ত)। ভারত শ্রীলঙ্কার সাথে আন্তরিকতাহীন ক্রিকেট সম্পর্কের কারণে ১৯৮৬ সালের টুর্নামেন্ট বর্জন করে। ১৯৯৩ সালে ভারত ও পাকিস্তান এর মধ্যে রাজনৈতিক অস্থিরতার কারণে এটি বাতিল হয়ে যায়। শ্রীলঙ্কা এশিয়া কাপ শুরু থেকে অংশ গ্রহণ করে আসছে। আন্তর্জাতিক ক্রিকেট কাউন্সিল নিয়ম করে দিয়েছে যে এশিয়া কাপের সকল খেলা অনুষ্ঠিত হবে অফিসিয়াল একদিনের আন্তর্জাতিক ক্রিকেট হিসেবে। এসিসি ঘোষনা অনুযায়ী প্রতি দুই বছর পর পর টুর্নামেন্ট অনুষ্ঠিত হয় ২০০৮ সাল থেকে।`, + }, + { + text: `ভারতীয় বাঙালি কথাসাহিত্যিক মহাশ্বেতা দেবীর মৃত্যু কবে হয় ?`, + context: `২০১৬ সালের ২৩ জুলাই হৃদরোগে আক্রান্ত হয়ে মহাশ্বেতা দেবী কলকাতার বেল ভিউ ক্লিনিকে ভর্তি হন। সেই বছরই ২৮ জুলাই একাধিক অঙ্গ বিকল হয়ে তাঁর মৃত্যু ঘটে। তিনি মধুমেহ, সেপ্টিসেমিয়া ও মূত্র সংক্রমণ রোগেও ভুগছিলেন।`, + }, + { + text: `মাস্টারদা সূর্যকুমার সেনের বাবার নাম কী ছিল ?`, + context: `সূর্য সেন ১৮৯৪ সালের ২২ মার্চ চট্টগ্রামের রাউজান থানার নোয়াপাড়ায় অর্থনৈতিক ভাবে অস্বচ্ছল পরিবারে জন্মগ্রহণ করেন। তাঁর পিতার নাম রাজমনি সেন এবং মাতার নাম শশী বালা সেন। রাজমনি সেনের দুই ছেলে আর চার মেয়ে। সূর্য সেন তাঁদের পরিবারের চতুর্থ সন্তান। দুই ছেলের নাম সূর্য ও কমল। চার মেয়ের নাম বরদাসুন্দরী, সাবিত্রী, ভানুমতী ও প্রমিলা। শৈশবে পিতা মাতাকে হারানো সূর্য সেন কাকা গৌরমনি সেনের কাছে মানুষ হয়েছেন। সূর্য সেন ছেলেবেলা থেকেই খুব মনোযোগী ভাল ছাত্র ছিলেন এবং ধর্মভাবাপন্ন গম্ভীর প্রকৃতির ছিলেন।`, + }, + ], + ], + [ + "sentence-similarity", + [ + { + source_sentence: "সে একজন সুখী ব্যক্তি", + sentences: ["সে হ্যাপি কুকুর", "সে খুব সুখী মানুষ", "আজ একটি রৌদ্রোজ্জ্বল দিন"], + }, + ], + ], +]); + +const MAPPING_MN: PerLanguageMapping = new Map([ + ["text-classification", [`Би чамд хайртай`]], + [ + "token-classification", + [ + `Намайг Дорж гэдэг. Би Улаанбаатарт амьдардаг.`, + `Намайг Ганбат гэдэг. Би Увс аймагт төрсөн.`, + `Манай улс таван хошуу малтай.`, + ], + ], + [ + "question-answering", + [ + { + text: `Та хаана амьдардаг вэ?`, + context: `Намайг Дорж гэдэг. Би Улаанбаатарт амьдардаг.`, + }, + { + text: `Таныг хэн гэдэг вэ?`, + context: `Намайг Дорж гэдэг. Би Улаанбаатарт амьдардаг.`, + }, + { + text: `Миний нэрийг хэн гэдэг вэ?`, + context: `Намайг Ганбат гэдэг. Би Увс аймагт төрсөн.`, + }, + ], + ], + ["translation", [`Намайг Дорж гэдэг. Би Улаанбаатарт амьдардаг.`, `Намайг Ганбат гэдэг. Би Увс аймагт төрсөн.`]], + [ + "summarization", + [ + `Монгол Улс (1992 оноос хойш) — дорно болон төв Азид оршдог бүрэн эрхт улс. Хойд талаараа Орос, бусад талаараа Хятад улстай хиллэдэг далайд гарцгүй орон. Нийслэл — Улаанбаатар хот. Алтайн нуруунаас Хянган, Соёноос Говь хүрсэн 1 сая 566 мянган км2 уудам нутагтай, дэлхийд нутаг дэвсгэрийн хэмжээгээр 19-рт жагсдаг. 2015 оны эхэнд Монгол Улсын хүн ам 3 сая хүрсэн (135-р олон). Үндсэндээ монгол үндэстэн (95 хувь), мөн хасаг, тува хүн байна. 16-р зуунаас хойш буддын шашин, 20-р зуунаас шашингүй байдал дэлгэрсэн ба албан хэрэгт монгол хэлээр харилцана.`, + ], + ], + [ + "text-generation", + [`Намайг Дорж гэдэг. Би`, `Хамгийн сайн дуучин бол`, `Миний дуртай хамтлаг бол`, `Эрт урьдын цагт`], + ], + ["fill-mask", [`Монгол улсын Улаанбаатар хотоос ярьж байна.`, `Миний амьдралын зорилго бол .`]], + [ + "automatic-speech-recognition", + [ + { + label: `Common Voice Train Example`, + src: `https://cdn-media.huggingface.co/common_voice/train/common_voice_mn_18577472.wav`, + }, + { + label: `Common Voice Test Example`, + src: `https://cdn-media.huggingface.co/common_voice/test/common_voice_mn_18577346.wav`, + }, + ], + ], + [ + "text-to-speech", + [ + `Би Монгол улсын иргэн.`, + `Энэхүү жишээ нь цаанаа ямар ч утга агуулаагүй болно`, + `Сар шинэдээ сайхан шинэлэж байна уу?`, + ], + ], + [ + "sentence-similarity", + [ + { + source_sentence: "Энэ бол аз жаргалтай хүн юм", + sentences: ["Энэ бол аз жаргалтай нохой юм", "Энэ бол маш их аз жаргалтай хүн юм", "Өнөөдөр нарлаг өдөр байна"], + }, + ], + ], +]); + +const MAPPING_SI: PerLanguageMapping = new Map([ + ["translation", [`සිංහල ඉතා අලංකාර භාෂාවකි.`, `මෙම තාක්ෂණය භාවිතා කරන ඔබට ස්තූතියි.`]], + ["fill-mask", [`මම ගෙදර .`, ` ඉගෙනීමට ගියාය.`]], +]); + +const MAPPING_DE: PerLanguageMapping = new Map([ + [ + "question-answering", + [ + { + text: `Wo wohne ich?`, + context: `Mein Name ist Wolfgang und ich lebe in Berlin`, + }, + { + text: `Welcher Name wird auch verwendet, um den Amazonas-Regenwald auf Englisch zu beschreiben?`, + context: `Der Amazonas-Regenwald, auf Englisch auch als Amazonien oder Amazonas-Dschungel bekannt, ist ein feuchter Laubwald, der den größten Teil des Amazonas-Beckens Südamerikas bedeckt. Dieses Becken umfasst 7.000.000 Quadratkilometer (2.700.000 Quadratmeilen), von denen 5.500.000 Quadratkilometer (2.100.000 Quadratmeilen) vom Regenwald bedeckt sind. Diese Region umfasst Gebiete von neun Nationen. Der größte Teil des Waldes befindet sich in Brasilien mit 60% des Regenwaldes, gefolgt von Peru mit 13%, Kolumbien mit 10% und geringen Mengen in Venezuela, Ecuador, Bolivien, Guyana, Suriname und Französisch-Guayana. Staaten oder Abteilungen in vier Nationen enthalten "Amazonas" in ihren Namen. Der Amazonas repräsentiert mehr als die Hälfte der verbleibenden Regenwälder des Planeten und umfasst den größten und artenreichsten tropischen Regenwald der Welt mit geschätzten 390 Milliarden Einzelbäumen, die in 16.000 Arten unterteilt sind.`, + }, + ], + ], + [ + "sentence-similarity", + [ + { + source_sentence: "Das ist eine glückliche Person", + sentences: [ + "Das ist ein glücklicher Hund", + "Das ist eine sehr glückliche Person", + "Heute ist ein sonniger Tag", + ], + }, + ], + ], +]); + +const MAPPING_DV: PerLanguageMapping = new Map([ + ["text-classification", [`އަހަރެން ގަޔާވޭ. އަހަރެން ލޯބިވޭ`]], + [ + "token-classification", + [`އަހަރެންގެ ނަމަކީ އަހުމަދު އަދި އަހަރެން ދިރިއުޅެނީ މާލޭގަ`, `އަހަރެންގެ ނަމަކީ ސާރާ އަދި އަހަރެން ދިރިއުޅެނީ އުތީމުގަ`, `އަހަރެންގެ ނަމަކީ އައިޝާ އަދި އަހަރެން ދިރިއުޅެނީ ފޭދޫ، އައްޑޫގަ`], + ], + [ + "question-answering", + [ + { + text: `އަހަރެން ދިރިއުޅެނީ ކޮންތާކު؟`, + context: `އަހަރެންގެ ނަމަކީ އަހުމަދު އަދި އަހަރެން ދިރިއުޅެނީ މާލޭގަ`, + }, + { + text: `އަހަރެން ދިރިއުޅެނީ ކޮންތާކު؟`, + context: `އަހަރެންގެ ނަމަކީ ސާރާ އަދި އަހަރެން ދިރިއުޅެނީ އުތީމުގަ`, + }, + { + text: `އަހަރެންގެ ނަމަކީ ކޮބާ؟`, + context: `އަހަރެންގެ ނަމަކީ އައިޝާ އަދި އަހަރެން ދިރިއުޅެނީ ފޭދޫގަ`, + }, + { + text: `އެމޭޒަން ރެއިންފޮރެސްޓް ސިފަކޮށްދިނުމަށް އިނގިރޭސި ބަހުން ބޭނުންކުރާނީ ކޮންނަމެއް؟`, + context: `އެމޭޒަން ރެއިންފޮރެސްޓް (ޕޯޗުޖީޒް: ފްލޮރެސްޓާ އެމަސޮނިކާ ނުވަތަ އެމަސޮނިއާ؛ ސްޕެނިޝް: ސެލްވާ އެމަސޮނިކާ, އެމަސޮނިއާ ނޫނީ އާންމުކޮށް އެމަޒޯނިއާ؛ ފްރެންޗް: ފޮރޭ އެމެޒޮނިއެން؛ ޑަޗް: އެމެޒޯންރޭގެވައުޑް)، އިގިރޭސި ބަހުން ބުނާ އެމެޒޯނިއާ ނުވަތަ ދަ އެމޭޒަން ޖަންގަލް އަކީ, ސައުތު އެމެރިކާގެ އެމޭޒަން ބޭސިން ސަރަހައްދުގެ ބޮޑުބައެއްގައި ހިމެނޭ މޮއިސްޓް ބޮރޯޑްލީފް ފޮރެސްޓެއެކެވެ. އެމޭޒަން ބޭސިން ސަރަހައްދުގެ ބޮޑު މިނަކީ 7 މިލިއަން އަކަ ކިލޯމީޓަރ (2.7 މިލިއަން އަކަ މައިލް(. މީގެ ތެރެއިން 5.5 މިލިއަން އަކަ ކިލޯމީޓަރ (2.1 މިލިއަން އަކަ މައިލް) އަކީ މި ފޮރެސްޓެވެ. މި ސަރަހައްދުގައި 9 ގައުމަކަށް ނިސްބަތްވާ ޓެރިޓަރީ ހިމެނެއެވެ. 60% އާއިއެކެ އެންމެ ބޮޑު ބައެއް ނިސްބަތްވަނީ ބްރެޒިލްއަށެވެ. އޭގެ ފަހުތުން 13% އާއެކު ޕެރޫ އާއި 10% އާއެކު ކޮލަމްބިއާ އަދި ކުޑަ ބައެއް ހިމެނޭ ގޮތުން ވެނެޒުއެލާ, އެކްއަޑޯ, ބޮލިވިއާ, ގުޔާނާ, ސުރިނާމް އަދި ފްރެންޗް ގްއާނާ އަށް ވެސް ނިސްބަތްވެއެވެ. މީގެ ތެރެއިން 4 ގައުމެއްގައި "އެމެޒޮނާސް" ހިމަނައިގެން ސްޓޭޓް ނުވަތަ ޑިޕާޓްމަންޓް އަކަށް ނަންދީފައިވެއެވެ. މުޅި ދުނިޔޭގައި ބާކީ ހުރި ރެއިންފޮރެސްޓްގެ ތެރެއިން ދެބައިކުޅަ އެއްބަޔަށްވުރެބޮޑުވަރެއް އެމޭޒޮން ރެއިންފޮރެސްޓް ހިއްސާކުރެއެވެ. މިއީ މުޅި ދުނިޔެއިން އެންމޮ ބޮޑު އަދި އެންމެ ބައޮޑައިވަރސް ރެއިންފޮރެސްޓް ޓްރެކްޓެވެ. ލަފާކުރެވޭ ގޮތުން 16 ހާސް ސްޕީޝީސްއަށް ބެހިގެންވާ 390 މިލިއަން ވައްތަރުގެ ގަސް މިތާގައި ހިމެނެއެވެ`, + }, + ], + ], + ["translation", [`އަހަރެންގެ ނަމަކީ އަހުމަދު އަދި އަހަރެން ދިރިއުޅެނީ މާލޭގަ`, `އަހަރެންގެ ނަމަކީ ސާރާ އަދި އަހަރެން ދިރިއުޅެނީ އުތީމުގަ`]], + [ + "summarization", + [ + `ޓަވަރުގެ އުސްމިނަކީ 324 މީޓަރު، އެއީ ގާތްގަނޑަކަށް 81 ބުރީގެ އިމާރާތަކާއި އެއްވަރެވެ. އެއީ ޕެރިސްގައި ހުރި އެންމެ އުސް އިމާރާތެވެ. އޭގެ ހަތަރެސްކަނަށް ހުރި ބުޑުގެ ދިގުމިނަކީ ކޮންމެ ފަރާތަކުން 125 މީޓަރެވެ. (410 ފޫޓު) އައިފިލް ޓަވަރު ބިނާކުރި އިރު، ވޮޝިންގްޓަން މޮނިއުމެންޓްގެ އުސްމިން ފަހަނައަޅާ ގޮސް، ދުނިޔޭގައި މީހުން އުފެއްދި ތަންތަނުގެ ތެރެއިން އެންމެ އުސް ތަނުގެ ލަގަބު ލިބުނެވެ. އަދި 1930 ގައި ނިއު ޔޯކްގެ ކްރައިސްލަރ ބިލްޑިންގް ބިނާކުރުމާއި ހަމައަށް 41 އަހަރު ވަންދެން މިލަގަބު ހިފެހެއްޓިއެވެ. މިއީ 300 މީޓަރަށް ވުރެ އުސްކޮށް އިމާރާތްކުރެވުނު ފުރަތަމަ ތަނެވެ. 1957 ގައި ޓަވަރުގެ އެންމެ މަތީގައި ހަރުކުރެވުނު ބްރޯޑްކާސްޓިންގ އޭރިއަލްގެ ސަބަބުން މިހާރު މި ޓަވަރު ކްރައިސްލަރ ބިލްޑިންގއަށް ވުރެ 5.2 މީޓަރ (17 ފޫޓު) އުހެވެ. މި ޓްރާންސްމިޓަރު ނުލާ، އައިފިލް ޓަވަރަކީ، މިލާއު ވިއާޑަކްޓަށް ފަހު ފްރާންސްގައި ހުރި 2 ވަނައަށް އެންމެ އުސް ފްރީސްޓޭންޑިންގ އިމާރާތެވެ`, + ], + ], + [ + "text-generation", + [`އަހަރެންގެ ނަމަކީ ޔޫސުފް އަދި އަހަރެންގެ މައިގަނޑު`, `އަހަރެންގެ ނަމަކީ މަރިއަމް، އަހަރެން އެންމެ ގަޔާވާ`, `އަހަރެންގެ ނަމަކީ ފާތުމަތު އަދި އަހަރެން`, `،އެއް ޒަމާނެއްގައި`], + ], + ["fill-mask", [`. މާލެ އަކީ ދިވެހިރާއްޖޭގެ`, `ގަރުދިޔައަކީ ދިވެހިންގެ މެދުގައި ކެއުމެއް.`]], +]); + +export const MAPPING_DEFAULT_WIDGET = new Map([ + ["en", MAPPING_EN], + ["zh", MAPPING_ZH], + ["fr", MAPPING_FR], + ["es", MAPPING_ES], + ["ru", MAPPING_RU], + ["uk", MAPPING_UK], + ["it", MAPPING_IT], + ["fa", MAPPING_FA], + ["ar", MAPPING_AR], + ["bn", MAPPING_BN], + ["mn", MAPPING_MN], + ["si", MAPPING_SI], + ["de", MAPPING_DE], + ["dv", MAPPING_DV], +]); diff --git a/node_modules/@huggingface/tasks/src/eval.ts b/node_modules/@huggingface/tasks/src/eval.ts new file mode 100644 index 0000000000000000000000000000000000000000..12fa80bc50cfb1e3af70e4ce36765a882ddee8df --- /dev/null +++ b/node_modules/@huggingface/tasks/src/eval.ts @@ -0,0 +1,164 @@ +/** + * List of supported Evaluation Frameworks supported in the `eval.yaml` file in benchmarks datasets. + */ +export const EVALUATION_FRAMEWORKS = { + exgentic: { + name: "exgentic", + description: + "Exgentic is an open evaluation framework for general-purpose AI agents across diverse domains and benchmarks.", + url: "https://github.com/Exgentic/exgentic", + }, + "inspect-ai": { + name: "inspect-ai", + description: "Inspect AI is an open-source framework for large language model evaluations.", + url: "https://inspect.aisi.org.uk/", + }, + "math-arena": { + name: "math-arena", + description: "MathArena is a platform for evaluation of LLMs on latest math competitions and olympiads.", + url: "https://github.com/eth-sri/matharena", + }, + mteb: { + name: "mteb", + description: "Multimodal toolbox for evaluating embeddings and retrieval systems.", + url: "https://github.com/embeddings-benchmark/mteb", + }, + "olmocr-bench": { + name: "olmocr-bench", + description: "olmOCR-Bench is a framework for evaluating document-level OCR of various tools.", + url: "https://github.com/allenai/olmocr/tree/main/olmocr/bench", + }, + harbor: { + name: "harbor", + description: "Harbor is a framework for evaluating and optimizing agents and language models.", + url: "https://github.com/laude-institute/harbor", + }, + ifstruct: { + name: "ifstruct", + description: + "IFStruct is a benchmark for structured-output compliance: whether a model produces valid JSON/YAML that follows a requested schema, scored without constrained decoding.", + url: "https://github.com/Liquid4All/ifstruct", + }, + pier: { + name: "pier", + description: + "Pier is a Harbor fork built for DeepSWE, with stronger support for CLI agents in no-internet tasks and more faithful, consistent agent trajectories.", + url: "https://github.com/datacurve-ai/pier", + }, + "redline-bench": { + name: "redline-bench", + description: + "RedlineBench measures multi-turn contract redlining: agents produce tracked-change .docx edits that are graded against attorney-authored weighted rubrics by an LLM judge panel across five dimensions. Report: https://intelligence.crosby.ai/benchmark/", + url: "https://github.com/crosbylegal/redline-bench", + }, + archipelago: { + name: "archipelago", + description: "Archipelago is a system for running and evaluating AI agents against MCP applications.", + url: "https://github.com/Mercor-Intelligence/archipelago", + }, + benchflow: { + name: "benchflow", + description: + "BenchFlow is an evaluation framework for AI agents on professional, skill-aware workflows. It powers SkillsBench and runs containerized agent trials with paired with-skills / without-skills configurations.", + url: "https://github.com/benchflow-ai/benchflow", + }, + "apex-evals": { + name: "apex-evals", + description: "APEX Evals is a benchmark suite and evaluation harness for evaluating large language models.", + url: "https://github.com/Mercor-Intelligence/apex-evals", + }, + "screenspot-pro": { + name: "screenspot-pro", + description: + "ScreenSpot-Pro is a GUI grounding benchmark designed to evaluate how well AI agents can locate and identify UI elements across professional software applications in high-resolution screenshots, covering 1,585 annotated images from 26 professional tools.", + url: "https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding", + }, + "swe-bench": { + name: "swe-bench", + description: "SWE Bench is a framework for evaluating the performance of LLMs on software engineering tasks.", + url: "https://github.com/swe-bench/swe-bench", + }, + "swe-bench-pro": { + name: "swe-bench-pro", + description: + "SWE-Bench Pro is a challenging benchmark evaluating LLMs/Agents on long-horizon software engineering tasks.", + url: "https://github.com/scaleapi/SWE-bench_Pro-os", + }, + "nemo-evaluator": { + name: "nemo-evaluator", + description: + "NeMo Evaluator is an open-source platform for robust, reproducible, and scalable evaluation of Large Language Models across 100+ benchmarks.", + url: "https://github.com/NVIDIA-NeMo/Evaluator", + }, + "yc-bench": { + name: "yc-bench", + description: + "YC Bench is a long-horizon deterministic benchmark for LLM agents. The agent plays CEO of an AI startup over a simulated 1–3 year run.", + url: "https://github.com/collinear-ai/yc-bench", + }, + "open-asr-leaderboard": { + name: "open-asr-leaderboard", + description: "The Open ASR Leaderboard ranks and evaluates speech recognition models.", + url: "https://github.com/huggingface/open_asr_leaderboard", + }, + mdpbench: { + name: "mdpbench", + description: + "MDPBench is a benchmark for evaluating multilingual document parsing across digital, photographed, Latin, and non-Latin document subsets.", + url: "https://github.com/Yuliang-Liu/MultimodalOCR", + }, + parsebench: { + name: "parsebench", + description: + "ParseBench is a benchmark for evaluating document parsing systems on real-world enterprise documents across tables, charts, content faithfulness, semantic formatting, and visual grounding.", + url: "https://github.com/run-llama/ParseBench", + }, + "video-mme-v2": { + name: "video-mme-v2", + description: + "Video-MME-v2 is a benchmark for evaluating the next stage of video understanding capabilities of multimodal large language models.", + url: "https://github.com/MME-Benchmarks/Video-MME-v2", + }, + "claw-eval": { + name: "claw-eval", + description: + "CLAW-Eval is an evaluation framework for assessing LLMs as autonomous agents across 300 human-verified tasks covering communication, finance, and productivity domains.", + url: "https://github.com/claw-eval/claw-eval", + }, + researchclawbench: { + name: "researchclawbench", + description: + "ResearchClawBench is a benchmark for evaluating AI agents on end-to-end scientific research tasks, from reading data and related work to producing code, figures, and publication-style reports.", + url: "https://github.com/InternScience/ResearchClawBench", + }, + pbench: { + name: "pbench", + description: + "PBench is a multi-level referring expression segmentation benchmark for evaluating vision-language perception across a structured hierarchy of skills.", + url: "https://github.com/tiiuae/Falcon-Perception", + }, + wildclawbench: { + name: "wildclawbench", + description: + "WildClawBench is an in-the-wild benchmark for evaluating AI agents in the OpenClaw environment across 60 hand-built, end-to-end tasks spanning productivity, code intelligence, social interaction, search, creative synthesis, and safety domains.", + url: "https://github.com/InternLM/WildClawBench", + }, + wbench: { + name: "wbench", + description: + "WBench is a comprehensive multi-turn benchmark for interactive video world model evaluation, assessing models across 5 dimensions (video quality, setting adherence, interaction adherence, consistency, physics compliance) and 22 metrics over 289 multi-turn interaction cases.", + url: "https://github.com/meituan-longcat/WBench", + }, + nanofold: { + name: "nanofold", + description: + "nanoFold is a data-efficiency benchmark for protein structure prediction. Its goal is to evaluate models on scenarios with scarce data.", + url: "https://github.com/ChrisHayduk/nanoFold-Competition", + }, + mmmu: { + name: "mmmu", + description: + "MMMU is a new benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning.", + url: "https://mmmu-benchmark.github.io/", + }, +} as const; diff --git a/node_modules/@huggingface/tasks/src/gguf.ts b/node_modules/@huggingface/tasks/src/gguf.ts new file mode 100644 index 0000000000000000000000000000000000000000..bfbb3f7fb523aaf2e06c0e90ed389e9f5635a7ca --- /dev/null +++ b/node_modules/@huggingface/tasks/src/gguf.ts @@ -0,0 +1,211 @@ +// This list is copied from gguf/types.ts, but will all types available (for backward compatibility) +// NOT to be confused with GGMLQuantizationType, a FileQuantization can contain multiple GGMLQuantizationType +// For example, Q4_K_M model can contains Q4_K and Q6_K tensors +export enum GGMLFileQuantizationType { + F32 = 0, + F16 = 1, + Q4_0 = 2, + Q4_1 = 3, + Q4_1_SOME_F16 = 4, + Q4_2 = 5, + Q4_3 = 6, + Q8_0 = 7, + Q5_0 = 8, + Q5_1 = 9, + Q2_K = 10, + Q3_K_S = 11, + Q3_K_M = 12, + Q3_K_L = 13, + Q4_K_S = 14, + Q4_K_M = 15, + Q5_K_S = 16, + Q5_K_M = 17, + Q6_K = 18, + IQ2_XXS = 19, + IQ2_XS = 20, + Q2_K_S = 21, + IQ3_XS = 22, + IQ3_XXS = 23, + IQ1_S = 24, + IQ4_NL = 25, + IQ3_S = 26, + IQ3_M = 27, + IQ2_S = 28, + IQ2_M = 29, + IQ4_XS = 30, + IQ1_M = 31, + BF16 = 32, + Q4_0_4_4 = 33, + Q4_0_4_8 = 34, + Q4_0_8_8 = 35, + TQ1_0 = 36, + TQ2_0 = 37, + MXFP4_MOE = 38, + NVFP4 = 39, + Q1_0 = 40, + + // custom quants used by unsloth + // they are not officially a scheme enum value in GGUF, but only here for naming + Q2_K_XL = 1000, + Q3_K_XL = 1001, + Q4_K_XL = 1002, + Q5_K_XL = 1003, + Q6_K_XL = 1004, + Q8_K_XL = 1005, +} + +const ggufQuants = Object.values(GGMLFileQuantizationType).filter((v): v is string => typeof v === "string"); +export const GGUF_QUANT_RE = new RegExp( + "(?UD-)?" + `(?${ggufQuants.join("|")})` + "(_(?[A-Z]+))?", +); +export const GGUF_QUANT_RE_GLOBAL = new RegExp(GGUF_QUANT_RE, "g"); + +export function parseGGUFQuantLabel(fname: string): string | undefined { + const quantLabel = fname.toUpperCase().match(GGUF_QUANT_RE_GLOBAL)?.at(-1); // if there is multiple quant substrings in a name, we prefer the last one + return quantLabel; +} + +// order of quantization, from biggest to smallest +// this list must be in sync with the order in GGMLFileQuantizationType +// the gguf.spec.ts tests are using verify if the order is correct +export const GGUF_QUANT_ORDER: GGMLFileQuantizationType[] = [ + GGMLFileQuantizationType.F32, + GGMLFileQuantizationType.BF16, + GGMLFileQuantizationType.F16, + GGMLFileQuantizationType.Q8_K_XL, + GGMLFileQuantizationType.Q8_0, + + // 6-bit quantizations + GGMLFileQuantizationType.Q6_K_XL, + GGMLFileQuantizationType.Q6_K, + + // 5-bit quantizations + GGMLFileQuantizationType.Q5_K_XL, + GGMLFileQuantizationType.Q5_K_M, + GGMLFileQuantizationType.Q5_K_S, + GGMLFileQuantizationType.Q5_0, + GGMLFileQuantizationType.Q5_1, + + // 4-bit quantizations + GGMLFileQuantizationType.Q4_K_XL, + GGMLFileQuantizationType.Q4_K_M, + GGMLFileQuantizationType.Q4_K_S, + GGMLFileQuantizationType.IQ4_NL, + GGMLFileQuantizationType.IQ4_XS, + GGMLFileQuantizationType.Q4_0_4_4, + GGMLFileQuantizationType.Q4_0_4_8, + GGMLFileQuantizationType.Q4_0_8_8, + GGMLFileQuantizationType.Q4_1_SOME_F16, + GGMLFileQuantizationType.Q4_0, + GGMLFileQuantizationType.Q4_1, + GGMLFileQuantizationType.Q4_2, + GGMLFileQuantizationType.Q4_3, + GGMLFileQuantizationType.MXFP4_MOE, + GGMLFileQuantizationType.NVFP4, + + // 3-bit quantizations + GGMLFileQuantizationType.Q3_K_XL, + GGMLFileQuantizationType.Q3_K_L, + GGMLFileQuantizationType.Q3_K_M, + GGMLFileQuantizationType.Q3_K_S, + GGMLFileQuantizationType.IQ3_M, + GGMLFileQuantizationType.IQ3_S, + GGMLFileQuantizationType.IQ3_XS, + GGMLFileQuantizationType.IQ3_XXS, + + // 2-bit quantizations + GGMLFileQuantizationType.Q2_K_XL, + GGMLFileQuantizationType.Q2_K, + GGMLFileQuantizationType.Q2_K_S, + GGMLFileQuantizationType.IQ2_M, + GGMLFileQuantizationType.IQ2_S, + GGMLFileQuantizationType.IQ2_XS, + GGMLFileQuantizationType.IQ2_XXS, + + // 1-bit quantizations + GGMLFileQuantizationType.IQ1_S, + GGMLFileQuantizationType.IQ1_M, + GGMLFileQuantizationType.TQ1_0, + GGMLFileQuantizationType.TQ2_0, + GGMLFileQuantizationType.Q1_0, +]; + +// This function finds the nearest quantization type that is less than or equal to the given quantization type. +// It returns undefined if no such quantization type is found. +export function findNearestQuantType( + quant: GGMLFileQuantizationType, + availableQuants: GGMLFileQuantizationType[], +): GGMLFileQuantizationType | undefined { + // Create a map for quick index lookup from the defined order + const orderMap = new Map(); + GGUF_QUANT_ORDER.forEach((q, index) => { + orderMap.set(q, index); + }); + + const targetIndex = orderMap.get(quant) ?? 0; // the 0 case should never happen + + // Filter the available quantizations to include only those defined in the order map, + // then sort them according to the GGUF_QUANT_ORDER (from largest/index 0 to smallest/highest index). + const sortedAvailable = availableQuants + .filter((q) => orderMap.has(q)) + .sort((a, b) => (orderMap.get(a) ?? Infinity) - (orderMap.get(b) ?? Infinity)); + + // If no valid quantizations are available after filtering + if (sortedAvailable.length === 0) { + return undefined; + } + + // Iterate through the sorted available quantizations (largest to smallest). + // Find the first one whose order index is >= the target index. + // This means finding the largest quantization that is smaller than or equal to the target. + for (const availableQuant of sortedAvailable) { + // We know the key exists due to the filter above. + const availableIndex = orderMap.get(availableQuant) ?? 0; + if (availableIndex >= targetIndex) { + return availableQuant; + } + } + + // If the loop completes, it means all available quantizations are larger (have a smaller index) + // than the target quantization. In this case, return the "smallest" available quantization, + // which is the last element in the sorted list (highest index among available). + return sortedAvailable[sortedAvailable.length - 1]; +} + +// This list is only used to calculate the size of the model, NOT to be confused with the quantization FILE type +export enum GGMLQuantizationType { + F32 = 0, + F16 = 1, + Q4_0 = 2, + Q4_1 = 3, + Q5_0 = 6, + Q5_1 = 7, + Q8_0 = 8, + Q8_1 = 9, + Q2_K = 10, + Q3_K = 11, + Q4_K = 12, + Q5_K = 13, + Q6_K = 14, + Q8_K = 15, + IQ2_XXS = 16, + IQ2_XS = 17, + IQ3_XXS = 18, + IQ1_S = 19, + IQ4_NL = 20, + IQ3_S = 21, + IQ2_S = 22, + IQ4_XS = 23, + I8 = 24, + I16 = 25, + I32 = 26, + I64 = 27, + F64 = 28, + IQ1_M = 29, + BF16 = 30, + TQ1_0 = 34, + TQ2_0 = 35, + MXFP4 = 39, + NVFP4 = 40, + Q1_0 = 41, +} diff --git a/node_modules/@huggingface/tasks/src/hardware-amd.ts b/node_modules/@huggingface/tasks/src/hardware-amd.ts new file mode 100644 index 0000000000000000000000000000000000000000..1e12db0ab8afe0c200e6f7563ad8586460f6d7cf --- /dev/null +++ b/node_modules/@huggingface/tasks/src/hardware-amd.ts @@ -0,0 +1,335 @@ +import type { HardwareSpec } from "./hardware.js"; + +export interface AmdGpuHardwareSpec extends HardwareSpec { + /** + * GFX version / LLVM ISA target (AMD GPUs only), e.g. "gfx1100" + * + * potential source https://llvm.org/docs/AMDGPUUsage.html#processors + */ + gfxVersion: string; +} + +const AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS = [16, 24, 32, 48, 64, 96]; + +export const AMD_GPU_SKUS: Record = { + MI300: { + tflops: 383.0, + memory: [192], + gfxVersion: "gfx942", + msrp: 15_000, + power: 750, + releaseYear: 2023, + }, + MI250: { + tflops: 362.1, + memory: [128], + gfxVersion: "gfx90a", + msrp: 10_000, + power: 560, + releaseYear: 2021, + }, + MI210: { + tflops: 181.0, + memory: [64], + gfxVersion: "gfx90a", + msrp: 8_000, + power: 300, + releaseYear: 2022, + }, + MI100: { + tflops: 184.6, + memory: [32], + gfxVersion: "gfx908", + msrp: 6_400, + power: 300, + releaseYear: 2020, + }, + MI60: { + tflops: 29.5, + memory: [32], + gfxVersion: "gfx906", + msrp: 3_000, + power: 300, + releaseYear: 2018, + }, + MI50: { + tflops: 26.5, + memory: [16, 32], + gfxVersion: "gfx906", + msrp: 1_800, + power: 300, + releaseYear: 2018, + }, + "R9700 PRO": { + tflops: 95.7, + memory: [32], + gfxVersion: "gfx1201", + msrp: 1_250, + power: 300, + releaseYear: 2025, + }, + "RX 9070 XT": { + tflops: 97.32, + memory: [16], + gfxVersion: "gfx1201", + msrp: 600, + power: 304, + releaseYear: 2025, + }, + "RX 9070": { + tflops: 72.25, + memory: [16], + gfxVersion: "gfx1201", + msrp: 550, + power: 220, + releaseYear: 2025, + }, + "RX 9060 XT": { + tflops: 51.28, + memory: [8, 16], + gfxVersion: "gfx1200", + msrp: 350, + power: 160, + releaseYear: 2025, + }, + "PRO W7900": { + tflops: 122.6, + memory: [48], + gfxVersion: "gfx1100", + msrp: 4_000, + power: 295, + releaseYear: 2023, + }, + "PRO W7800": { + tflops: 90.5, + memory: [32, 48], + gfxVersion: "gfx1100", + msrp: 2_500, + power: 260, + releaseYear: 2023, + }, + "RX 7900 XTX": { + tflops: 122.8, + memory: [24], + gfxVersion: "gfx1100", + msrp: 1_000, + power: 355, + releaseYear: 2022, + }, + "RX 7900 XT": { + tflops: 103.0, + memory: [20], + gfxVersion: "gfx1100", + msrp: 900, + power: 315, + releaseYear: 2022, + }, + "RX 7900 GRE": { + tflops: 91.96, + memory: [16], + gfxVersion: "gfx1100", + msrp: 550, + power: 260, + releaseYear: 2023, + }, + "RX 7800 XT": { + tflops: 74.65, + memory: [16], + gfxVersion: "gfx1101", + msrp: 500, + power: 263, + releaseYear: 2023, + }, + "RX 7700 XT": { + tflops: 70.34, + memory: [12], + gfxVersion: "gfx1101", + msrp: 450, + power: 245, + releaseYear: 2023, + }, + "RX 7600 XT": { + tflops: 45.14, + memory: [16, 8], + gfxVersion: "gfx1102", + msrp: 350, + power: 190, + releaseYear: 2024, + }, + "RX 6950 XT": { + tflops: 47.31, + memory: [16], + gfxVersion: "gfx1030", + msrp: 1_100, + power: 335, + releaseYear: 2022, + }, + "RX 6800": { + tflops: 32.33, + memory: [16], + gfxVersion: "gfx1030", + msrp: 600, + power: 250, + releaseYear: 2020, + }, + "RX 6700 XT": { + tflops: 26.43, + memory: [12], + gfxVersion: "gfx1031", + msrp: 500, + power: 230, + releaseYear: 2021, + }, + "RX 6700": { + tflops: 22.58, + memory: [10], + gfxVersion: "gfx1031", + msrp: 329, + power: 175, + releaseYear: 2022, + }, + "RX 6650 XT": { + tflops: 21.59, + memory: [8], + gfxVersion: "gfx1032", + msrp: 400, + power: 180, + releaseYear: 2022, + }, + "RX 6600 XT": { + tflops: 21.21, + memory: [8], + gfxVersion: "gfx1032", + msrp: 400, + power: 160, + releaseYear: 2021, + }, + "RX 6600": { + tflops: 17.86, + memory: [8], + gfxVersion: "gfx1032", + msrp: 350, + power: 132, + releaseYear: 2021, + }, + "RX 5700 XT": { + tflops: 19.51, + memory: [8], + gfxVersion: "gfx1010", + msrp: 399, + power: 225, + releaseYear: 2019, + }, + "RX 5700": { + tflops: 15.9, + memory: [8], + gfxVersion: "gfx1010", + msrp: 349, + power: 180, + releaseYear: 2019, + }, + "RX 5500 XT": { + tflops: 10.39, + memory: [4, 8], + gfxVersion: "gfx1012", + msrp: 200, + power: 130, + releaseYear: 2019, + }, + "Radeon Pro V620": { + tflops: 40.55, + memory: [32], + gfxVersion: "gfx1030", + msrp: 3_000, + power: 300, + releaseYear: 2021, + }, + "Radeon Pro VII": { + tflops: 26.11, + memory: [16, 32], + gfxVersion: "gfx906", + msrp: 1_900, + power: 250, + releaseYear: 2020, + }, + "Radeon 610M": { + tflops: 0.97, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1037", + msrp: 300, + power: 15, + releaseYear: 2022, + }, + "Radeon 740M": { + tflops: 5.12, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1103", + msrp: 179, + power: 15, + releaseYear: 2023, + }, + "Radeon 760M": { + tflops: 10.65, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1103", + msrp: 229, + power: 15, + releaseYear: 2023, + }, + "Radeon 780M": { + tflops: 16.59, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1103", + msrp: 329, + power: 15, + releaseYear: 2023, + }, + "Radeon 820M": { + tflops: 1.434, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1152", + msrp: 200, + power: 15, + releaseYear: 2025, + }, + "Radeon 840M": { + tflops: 2.97, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1152", + msrp: 250, + power: 15, + releaseYear: 2025, + }, + "Radeon 860M": { + tflops: 6.14, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1152", + msrp: 300, + power: 15, + releaseYear: 2025, + }, + "Radeon 880M": { + tflops: 8.91, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1150", + msrp: 400, + power: 15, + releaseYear: 2024, + }, + "Radeon 890M": { + tflops: 11.88, + memory: AMD_GPU_INTEGRATED_SHARED_MEMORY_OPTIONS, + gfxVersion: "gfx1150", + msrp: 450, + power: 15, + releaseYear: 2024, + }, + "Ryzen AI Max+ 395": { + tflops: 29.7, + memory: [64, 96, 128], + gfxVersion: "gfx1151", + msrp: 1_500, + power: 120, + releaseYear: 2025, + }, +}; diff --git a/node_modules/@huggingface/tasks/src/hardware-nvidia.ts b/node_modules/@huggingface/tasks/src/hardware-nvidia.ts new file mode 100644 index 0000000000000000000000000000000000000000..f3442fcb5a6a787b868d403d89727b7974289675 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/hardware-nvidia.ts @@ -0,0 +1,1007 @@ +import type { HardwareSpec } from "./hardware.js"; + +export interface NvidiaHardwareSpec extends HardwareSpec { + /** + * CUDA Compute Capability (NVIDIA GPUs only) + * + * potential source https://developer.nvidia.com/cuda/gpus + */ + computeCapability: number; +} + +export enum NvidiaComputeCapabilities { + BLACKWELL_ULTRA = 12.1, + BLACKWELL_RTX = 12.0, + BLACKWELL = 10.0, + HOPPER = 9.0, + ADA_LOVELACE = 8.9, + ORIN = 8.7, + AMPERE_RTX = 8.6, + AMPERE = 8.0, + TURING = 7.5, + XAVIER = 7.2, + VOLTA = 7.0, + PASCAL_TEGRA = 6.2, + PASCAL = 6.1, + PASCAL_DATACENTER = 6.0, + MAXWELL = 5.3, +} + +export const NVIDIA_SKUS: Record = { + B300: { + tflops: 1232, + memory: [288], + computeCapability: 10.0, + msrp: 45_000, + power: 1400, + releaseYear: 2026, + }, + B200: { + tflops: 496.6, + memory: [192], + computeCapability: 10.0, + msrp: 40_000, + power: 1000, + releaseYear: 2024, + }, + H200: { + tflops: 241.3, + memory: [141], + computeCapability: 9.0, + msrp: 32_000, + power: 700, + releaseYear: 2024, + }, + H100: { + tflops: 267.6, + memory: [80], + computeCapability: 9.0, + msrp: 30_000, + power: 700, + releaseYear: 2022, + }, + H800: { + tflops: 237.2, + memory: [80], + computeCapability: 9.0, + msrp: 30_000, + power: 700, + releaseYear: 2023, + }, + H20: { + tflops: 148, + memory: [96], + computeCapability: 9.0, + msrp: 13_500, + power: 400, + releaseYear: 2024, + }, + L40s: { + tflops: 91.61, + memory: [48], + computeCapability: 8.9, + msrp: 8_500, + power: 350, + releaseYear: 2023, + }, + L40: { + tflops: 90.52, + memory: [48], + computeCapability: 8.9, + msrp: 7_500, + power: 300, + releaseYear: 2022, + }, + L20: { + tflops: 59.35, + memory: [48], + computeCapability: 8.9, + msrp: 5_000, + power: 275, + releaseYear: 2023, + }, + L4: { + tflops: 30.29, + memory: [24], + computeCapability: 8.9, + msrp: 2_500, + power: 72, + releaseYear: 2023, + }, + GB10: { + tflops: 29.71, + memory: [128], + computeCapability: 12.1, + msrp: 3_999, + power: 140, + releaseYear: 2025, + }, + "RTX PRO 6000 WS": { + tflops: 126, + memory: [96], + computeCapability: 12.0, + msrp: 8_600, + power: 600, + releaseYear: 2025, + }, + "RTX PRO 6000 Max-Q": { + tflops: 116, + memory: [96], + computeCapability: 12.0, + msrp: 8_600, + power: 300, + releaseYear: 2025, + }, + "RTX PRO 5000": { + tflops: 66.94, + memory: [48, 72], + computeCapability: 12.0, + msrp: 4_500, + power: 300, + releaseYear: 2025, + }, + "RTX PRO 4500 WS": { + tflops: 50.53, + memory: [32], + computeCapability: 12.0, + msrp: 2_800, + power: 200, + releaseYear: 2025, + }, + "RTX PRO 4000": { + tflops: 36.83, + memory: [24], + computeCapability: 12.0, + msrp: 1_500, + power: 140, + releaseYear: 2025, + }, + "RTX PRO 4000 SFF": { + tflops: 24.05, + memory: [24], + computeCapability: 12.0, + msrp: 1_500, + power: 70, + releaseYear: 2025, + }, + "RTX PRO 2000": { + tflops: 17.03, + memory: [16], + computeCapability: 12.0, + msrp: 700, + power: 70, + releaseYear: 2025, + }, + "RTX 6000 Ada": { + tflops: 91.1, + memory: [48], + computeCapability: 8.9, + msrp: 6_800, + power: 300, + releaseYear: 2022, + }, + "RTX 5880 Ada": { + tflops: 69.3, + memory: [48], + computeCapability: 8.9, + msrp: 6_000, + power: 285, + releaseYear: 2024, + }, + "RTX 5000 Ada": { + tflops: 65.3, + memory: [32], + computeCapability: 8.9, + msrp: 4_000, + power: 250, + releaseYear: 2023, + }, + "RTX 4500 Ada": { + tflops: 39.6, + memory: [24], + computeCapability: 8.9, + msrp: 2_250, + power: 210, + releaseYear: 2023, + }, + "RTX 4000 Ada": { + tflops: 26.7, + memory: [20], + computeCapability: 8.9, + msrp: 1_250, + power: 130, + releaseYear: 2023, + }, + "RTX 4000 SFF Ada": { + tflops: 19.2, + memory: [20], + computeCapability: 8.9, + msrp: 1_250, + power: 70, + releaseYear: 2023, + }, + "RTX 3500 Ada Mobile": { + tflops: 15.8, + memory: [12], + computeCapability: 8.9, + msrp: 1_500, + power: 150, + releaseYear: 2023, + }, + "RTX 2000 Ada": { + tflops: 12.0, + memory: [16], + computeCapability: 8.9, + msrp: 650, + power: 70, + releaseYear: 2024, + }, + "RTX A6000": { + tflops: 38.7, + memory: [48], + computeCapability: 8.6, + msrp: 4_650, + power: 300, + releaseYear: 2020, + }, + "RTX A5000": { + tflops: 27.77, + memory: [8, 12, 24], + computeCapability: 8.6, + msrp: 2_250, + power: 230, + releaseYear: 2021, + }, + "RTX A5000 Max-Q": { + tflops: 16.59, + memory: [16], + computeCapability: 8.6, + msrp: 2_000, + power: 80, + releaseYear: 2021, + }, + "RTX A5000 Mobile": { + tflops: 19.35, + memory: [16], + computeCapability: 8.6, + msrp: 2_000, + power: 165, + releaseYear: 2021, + }, + "RTX A4000": { + tflops: 19.17, + memory: [16], + computeCapability: 8.6, + msrp: 1_000, + power: 140, + releaseYear: 2021, + }, + "RTX A4000 Max-Q": { + tflops: 14.28, + memory: [8], + computeCapability: 8.6, + msrp: 1_000, + power: 35, + releaseYear: 2021, + }, + "RTX A4000 Mobile": { + tflops: 17.2, + memory: [8], + computeCapability: 8.6, + msrp: 1_000, + power: 80, + releaseYear: 2021, + }, + "RTX A3000 Mobile": { + tflops: 10.9, + memory: [6, 12], + computeCapability: 8.6, + msrp: 700, + power: 80, + releaseYear: 2021, + }, + "RTX A2000": { + tflops: 7.987, + memory: [6, 12], + computeCapability: 8.6, + msrp: 450, + power: 70, + releaseYear: 2021, + }, + "RTX A2000 Embedded": { + tflops: 6.026, + memory: [4], + computeCapability: 8.6, + msrp: 400, + power: 70, + releaseYear: 2022, + }, + "RTX A2000 Max-Q": { + tflops: 6.1, + memory: [4, 8], + computeCapability: 8.6, + msrp: 450, + power: 35, + releaseYear: 2021, + }, + "RTX A2000 Mobile": { + tflops: 8.4, + memory: [4, 8], + computeCapability: 8.6, + msrp: 450, + power: 95, + releaseYear: 2021, + }, + A800: { + tflops: 77.97, + memory: [40, 80], + computeCapability: 8.0, + msrp: 12_000, + power: 400, + releaseYear: 2022, + }, + A100: { + tflops: 77.97, + memory: [80, 40], + computeCapability: 8.0, + msrp: 15_000, + power: 400, + releaseYear: 2020, + }, + A40: { + tflops: 37.42, + memory: [48], + computeCapability: 8.6, + msrp: 5_500, + power: 300, + releaseYear: 2020, + }, + A30: { + tflops: 10.32, + memory: [24], + computeCapability: 8.0, + msrp: 5_000, + power: 165, + releaseYear: 2021, + }, + A10: { + tflops: 31.24, + memory: [24], + computeCapability: 8.6, + msrp: 3_200, + power: 150, + releaseYear: 2021, + }, + A2: { + tflops: 4.531, + memory: [16], + computeCapability: 8.6, + msrp: 1_000, + power: 60, + releaseYear: 2021, + }, + "RTX 5090": { + tflops: 104.8, + memory: [32], + computeCapability: 12.0, + msrp: 2_000, + power: 575, + releaseYear: 2025, + }, + "RTX 5090 D": { + tflops: 104.8, + memory: [32], + computeCapability: 12.0, + msrp: 2_000, + power: 575, + releaseYear: 2025, + }, + "RTX 5090 Mobile": { + tflops: 31.8, + memory: [24], + computeCapability: 12.0, + msrp: 1_500, + power: 175, + releaseYear: 2025, + }, + "RTX 5080": { + tflops: 56.28, + memory: [16], + computeCapability: 12.0, + msrp: 1_000, + power: 360, + releaseYear: 2025, + }, + "RTX 5080 Mobile": { + tflops: 23.04, + memory: [16], + computeCapability: 12.0, + msrp: 1_000, + power: 175, + releaseYear: 2025, + }, + "RTX 5070": { + tflops: 30.84, + memory: [12], + computeCapability: 12.0, + msrp: 550, + power: 250, + releaseYear: 2025, + }, + "RTX 5070 Mobile": { + tflops: 13.13, + memory: [8], + computeCapability: 12.0, + msrp: 500, + power: 100, + releaseYear: 2025, + }, + "RTX 5070 Ti": { + tflops: 43.94, + memory: [16], + computeCapability: 12.0, + msrp: 750, + power: 300, + releaseYear: 2025, + }, + "RTX 5070 Ti Mobile": { + tflops: 17.04, + memory: [12], + computeCapability: 12.0, + msrp: 700, + power: 140, + releaseYear: 2025, + }, + "RTX 5060 Ti": { + tflops: 23.7, + memory: [16, 8], + computeCapability: 12.0, + msrp: 450, + power: 180, + releaseYear: 2025, + }, + "RTX 5060": { + tflops: 19.18, + memory: [8], + computeCapability: 12.0, + msrp: 300, + power: 150, + releaseYear: 2025, + }, + "RTX 5060 Mobile": { + tflops: 9.684, + memory: [8], + computeCapability: 12.0, + msrp: 300, + power: 100, + releaseYear: 2025, + }, + "RTX 5050": { + tflops: 13.17, + memory: [8], + computeCapability: 12.0, + msrp: 249, + power: 130, + releaseYear: 2025, + }, + "RTX 5050 Mobile": { + tflops: 7.7, + memory: [8], + computeCapability: 12.0, + msrp: 250, + power: 100, + releaseYear: 2025, + }, + "RTX 4090": { + tflops: 82.58, + memory: [24], + computeCapability: 8.9, + msrp: 1_600, + power: 450, + releaseYear: 2022, + }, + "RTX 4090D": { + tflops: 79.49, + memory: [24, 48], + computeCapability: 8.9, + msrp: 1_600, + power: 425, + releaseYear: 2023, + }, + "RTX 4090 Mobile": { + tflops: 32.98, + memory: [16], + computeCapability: 8.9, + msrp: 1_500, + power: 150, + releaseYear: 2023, + }, + "RTX 4080 SUPER": { + tflops: 52.2, + memory: [16], + computeCapability: 8.9, + msrp: 1_000, + power: 320, + releaseYear: 2024, + }, + "RTX 4080": { + tflops: 48.7, + memory: [16], + computeCapability: 8.9, + msrp: 1_200, + power: 320, + releaseYear: 2022, + }, + "RTX 4080 Mobile": { + tflops: 24.72, + memory: [12], + computeCapability: 8.9, + msrp: 1_000, + power: 150, + releaseYear: 2023, + }, + "RTX 4070": { + tflops: 29.15, + memory: [12], + computeCapability: 8.9, + msrp: 600, + power: 200, + releaseYear: 2023, + }, + "RTX 4070 Mobile": { + tflops: 15.62, + memory: [8], + computeCapability: 8.9, + msrp: 500, + power: 115, + releaseYear: 2023, + }, + "RTX 4070 Ti": { + tflops: 40.09, + memory: [12], + computeCapability: 8.9, + msrp: 800, + power: 285, + releaseYear: 2023, + }, + "RTX 4070 Super": { + tflops: 35.48, + memory: [12], + computeCapability: 8.9, + msrp: 600, + power: 220, + releaseYear: 2024, + }, + "RTX 4070 Ti Super": { + tflops: 44.1, + memory: [16], + computeCapability: 8.9, + msrp: 800, + power: 285, + releaseYear: 2024, + }, + "RTX 4060": { + tflops: 15.11, + memory: [8], + computeCapability: 8.9, + msrp: 300, + power: 115, + releaseYear: 2023, + }, + "RTX 4060 Ti": { + tflops: 22.06, + memory: [8, 16], + computeCapability: 8.9, + msrp: 500, + power: 165, + releaseYear: 2023, + }, + "RTX 4090 Laptop": { + tflops: 32.98, + memory: [16], + computeCapability: 8.9, + msrp: 1_500, + power: 150, + releaseYear: 2023, + }, + "RTX 4080 Laptop": { + tflops: 24.72, + memory: [12], + computeCapability: 8.9, + msrp: 1_000, + power: 150, + releaseYear: 2023, + }, + "RTX 4070 Laptop": { + tflops: 15.62, + memory: [8], + computeCapability: 8.9, + msrp: 500, + power: 115, + releaseYear: 2023, + }, + "RTX 4060 Laptop": { + tflops: 11.61, + memory: [8], + computeCapability: 8.9, + msrp: 300, + power: 115, + releaseYear: 2023, + }, + "RTX 4050 Laptop": { + tflops: 8.9, + memory: [6], + computeCapability: 8.9, + msrp: 250, + power: 115, + releaseYear: 2023, + }, + "RTX 3090": { + tflops: 35.58, + memory: [24], + computeCapability: 8.6, + msrp: 1_500, + power: 350, + releaseYear: 2020, + }, + "RTX 3090 Ti": { + tflops: 40, + memory: [24], + computeCapability: 8.6, + msrp: 2_000, + power: 450, + releaseYear: 2022, + }, + "RTX 3080": { + tflops: 30.6, + memory: [12, 10], + computeCapability: 8.6, + msrp: 800, + power: 350, + releaseYear: 2020, + }, + "RTX 3080 Ti": { + tflops: 34.1, + memory: [12], + computeCapability: 8.6, + msrp: 1_200, + power: 350, + releaseYear: 2021, + }, + "RTX 3080 Mobile": { + tflops: 18.98, + memory: [16, 8], + computeCapability: 8.6, + msrp: 800, + power: 150, + releaseYear: 2021, + }, + "RTX 3070": { + tflops: 20.31, + memory: [8], + computeCapability: 8.6, + msrp: 500, + power: 220, + releaseYear: 2020, + }, + "RTX 3070 Ti": { + tflops: 21.75, + memory: [8], + computeCapability: 8.6, + msrp: 600, + power: 290, + releaseYear: 2021, + }, + "RTX 3070 Ti Mobile": { + tflops: 16.6, + memory: [8], + computeCapability: 8.6, + msrp: 700, + power: 125, + releaseYear: 2022, + }, + "RTX 3060 Ti": { + tflops: 16.2, + memory: [8], + computeCapability: 8.6, + msrp: 400, + power: 200, + releaseYear: 2020, + }, + "RTX 3060": { + tflops: 12.74, + memory: [12, 8], + computeCapability: 8.6, + msrp: 350, + power: 170, + releaseYear: 2021, + }, + "RTX 2080 Ti": { + tflops: 26.9, + memory: [11, 22], // 22GB: modded 2080ti + computeCapability: 7.5, + msrp: 1_000, + power: 250, + releaseYear: 2018, + }, + "RTX 2080": { + tflops: 20.14, + memory: [8], + computeCapability: 7.5, + msrp: 700, + power: 215, + releaseYear: 2018, + }, + "RTX 2070": { + tflops: 14.93, + memory: [8], + computeCapability: 7.5, + msrp: 500, + power: 175, + releaseYear: 2018, + }, + "RTX 2070 SUPER Mobile": { + tflops: 14.13, + memory: [8], + computeCapability: 7.5, + msrp: 600, + power: 115, + releaseYear: 2020, + }, + "RTX 2070 SUPER": { + tflops: 18.12, + memory: [8], + computeCapability: 7.5, + msrp: 500, + power: 215, + releaseYear: 2019, + }, + "RTX 3060 Mobile": { + tflops: 10.94, + memory: [6], + computeCapability: 8.6, + msrp: 400, + power: 115, + releaseYear: 2021, + }, + "RTX 3050 Mobile": { + tflops: 7.639, + memory: [4, 6], + computeCapability: 8.6, + msrp: 250, + power: 95, + releaseYear: 2022, + }, + "RTX 2060": { + tflops: 12.9, + memory: [6], + computeCapability: 7.5, + msrp: 350, + power: 160, + releaseYear: 2019, + }, + "RTX 2060 12GB": { + tflops: 14.36, + memory: [12], + computeCapability: 7.5, + msrp: 300, + power: 184, + releaseYear: 2021, + }, + "RTX 2060 Mobile": { + tflops: 9.22, + memory: [6], + computeCapability: 7.5, + msrp: 350, + power: 90, + releaseYear: 2019, + }, + "RTX 2050 Mobile": { + tflops: 10.2, + memory: [4], + computeCapability: 8.6, // Ampere (outlier GPU in the 20xx series) + msrp: 250, + power: 45, + releaseYear: 2021, + }, + "GTX 1080 Ti": { + tflops: 11.34, // float32 (GPU does not support native float16) + memory: [11], + computeCapability: 6.1, + msrp: 700, + power: 250, + releaseYear: 2017, + }, + "GTX 1080": { + tflops: 8.87, // float32 (GPU does not support native float16) + memory: [8], + computeCapability: 6.1, + msrp: 599, + power: 180, + releaseYear: 2016, + }, + "GTX 1070 Ti": { + tflops: 8.2, // float32 (GPU does not support native float16) + memory: [8], + computeCapability: 6.1, + msrp: 450, + power: 180, + releaseYear: 2017, + }, + "GTX 1070": { + tflops: 6.46, // float32 (GPU does not support native float16) + memory: [8], + computeCapability: 6.1, + msrp: 379, + power: 150, + releaseYear: 2016, + }, + "GTX 1060": { + tflops: 3.9, // float32 (GPU does not support native float16) + memory: [3, 6], + computeCapability: 6.1, + msrp: 300, + power: 120, + releaseYear: 2016, + }, + "GTX 1050 Ti": { + tflops: 2.1, // float32 (GPU does not support native float16) + memory: [4], + computeCapability: 6.1, + msrp: 150, + power: 75, + releaseYear: 2016, + }, + "RTX Titan": { + tflops: 32.62, + memory: [24], + computeCapability: 7.5, + msrp: 2_500, + power: 280, + releaseYear: 2018, + }, + "GTX 1660": { + tflops: 10.05, + memory: [6], + computeCapability: 7.5, + msrp: 200, + power: 120, + releaseYear: 2019, + }, + "GTX 1650 Mobile": { + tflops: 6.39, + memory: [4], + computeCapability: 7.5, + msrp: 150, + power: 50, + releaseYear: 2019, + }, + T4: { + tflops: 65.13, + memory: [16], + computeCapability: 7.5, + msrp: 2_000, + power: 70, + releaseYear: 2018, + }, + T10: { + tflops: 20.0, + memory: [16], + computeCapability: 7.5, + msrp: 2_000, + power: 150, + releaseYear: 2020, + }, + V100: { + tflops: 28.26, + memory: [32, 16], + computeCapability: 7.0, + msrp: 10_000, + power: 300, + releaseYear: 2017, + }, + "Quadro P6000": { + tflops: 12.63, // float32 (GPU does not support native float16) + memory: [24], + computeCapability: 6.1, + msrp: 5_000, + power: 250, + releaseYear: 2016, + }, + P40: { + tflops: 11.76, // float32 (GPU does not support native float16) + memory: [24], + computeCapability: 6.1, + msrp: 5_700, + power: 250, + releaseYear: 2016, + }, + P100: { + tflops: 19.05, + memory: [16], + computeCapability: 6.0, + msrp: 7_000, + power: 300, + releaseYear: 2016, + }, + "Jetson AGX Orin 64GB": { + tflops: 10.65, + memory: [64], + computeCapability: 8.7, + msrp: 2_000, + power: 60, + releaseYear: 2022, + }, + "Jetson AGX Orin 32GB": { + tflops: 6.66, + memory: [32], + computeCapability: 8.7, + msrp: 999, + power: 40, + releaseYear: 2022, + }, + "Jetson Orin NX 16GB": { + tflops: 3.76, + memory: [16], + computeCapability: 8.7, + msrp: 600, + power: 25, + releaseYear: 2023, + }, + "Jetson Orin NX 8GB": { + tflops: 3.13, + memory: [8], + computeCapability: 8.7, + msrp: 400, + power: 20, + releaseYear: 2023, + }, + "Jetson Orin Nano 8GB": { + tflops: 2.56, + memory: [8], + computeCapability: 8.7, + msrp: 500, + power: 15, + releaseYear: 2023, + }, + "Jetson Orin Nano 4GB": { + tflops: 1.28, + memory: [4], + computeCapability: 8.7, + msrp: 200, + power: 10, + releaseYear: 2023, + }, + "Jetson AGX Xavier": { + tflops: 2.82, + memory: [32, 64], + computeCapability: 7.2, + msrp: 1_100, + power: 30, + releaseYear: 2018, + }, + "Jetson Xavier NX": { + tflops: 1.69, + memory: [8, 16], + computeCapability: 7.2, + msrp: 400, + power: 20, + releaseYear: 2020, + }, + "Jetson TX2": { + tflops: 1.33, + memory: [4, 8], + computeCapability: 6.2, + msrp: 400, + power: 15, + releaseYear: 2017, + }, + "Jetson Nano": { + tflops: 0.47, + memory: [4], + computeCapability: 5.3, + msrp: 100, + power: 10, + releaseYear: 2019, + }, +}; diff --git a/node_modules/@huggingface/tasks/src/hardware.ts b/node_modules/@huggingface/tasks/src/hardware.ts new file mode 100644 index 0000000000000000000000000000000000000000..b447bc27b4ebcc45cbd31bff5576eae3a3625683 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/hardware.ts @@ -0,0 +1,625 @@ +import { AMD_GPU_SKUS } from "./hardware-amd.js"; +import { NVIDIA_SKUS } from "./hardware-nvidia.js"; + +/** + * Biden AI Executive Order (since revoked by President Trump): + * https://web.archive.org/web/20250105222429/https://www.whitehouse.gov/briefing-room/presidential-actions/2023/10/30/executive-order-on-the-safe-secure-and-trustworthy-development-and-use-of-artificial-intelligence/ + */ +export const TFLOPS_THRESHOLD_WHITE_HOUSE_MODEL_TRAINING_TOTAL = 10 ** 14; +export const TFLOPS_THRESHOLD_WHITE_HOUSE_MODEL_TRAINING_TOTAL_BIOLOGY = 10 ** 11; +export const TFLOPS_THRESHOLD_WHITE_HOUSE_CLUSTER = 10 ** 8; + +/** + * EU AI Act + * https://ec.europa.eu/commission/presscorner/detail/en/qanda_21_1683 + */ +export const TFLOPS_THRESHOLD_EU_AI_ACT_MODEL_TRAINING_TOTAL = 10 ** 13; + +export interface HardwareSpec { + /** + * Approximate value, in FP16 whenever possible for GPUs and FP32 for CPUs. + * This is only approximate/theoretical and shouldn't be taken too seriously. + * Currently the CPU values are from cpu-monkey.com + * while the GPU values are from techpowerup.com + * + * Note to reviewers: I got fed up with data entry, + * and HuggingChat running Llama3 with Web search was failing a bit, + * so some of those values might be slightly inaccurate. Forgive me and please feel free to improve. + */ + tflops: number; + /** + * If an array is specified, options of memory size (can be VRAM, unified RAM) + * e.g. an A100 exists in 40 or 80 GB. + */ + memory?: number[]; + /** + * Approximate MSRP in USD at launch. For SKUs with multiple memory variants, + * the price corresponds to the largest memory variant. For datacenter GPUs + * sold via OEMs without a public MSRP (H100, MI300X, ...), this is a + * widely-reported street price. For mobile/laptop GPUs that are not sold + * standalone, this is the approximate module/BOM cost. For Apple Silicon + * SoCs, this is the price of a Mac configured with that chip and the + * largest memory option. For CPU "family" entries (e.g. "Xeon 4th Gen", + * "Ryzen Zen 4 7000 (Ryzen 9)"), this is the tray/box price of a + * representative flagship SKU at launch. + */ + msrp: number; + /** + * Approximate maximum sustained power draw in watts. For GPUs with multiple + * form factors (e.g. H100 SXM vs PCIe), uses the highest variant. For CPUs, + * uses max turbo power (PL2 / MTP for Intel, PPT for AMD), not base TDP. + * For Apple Silicon and Snapdragon SoCs, an estimated package power based + * on benchmarks/teardowns (Apple does not publish TDP). + */ + power: number; + /** + * Year the SKU first became available. For SKUs refreshed later with + * additional memory variants (e.g. A100 40GB → 80GB, RTX 2060 → 12GB), + * this is the original launch year. For CPU "family" entries, this is + * the year the family debuted. + */ + releaseYear: number; +} + +export const DEFAULT_MEMORY_OPTIONS = [ + 8, 16, 24, 32, 40, 48, 64, 80, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, 2048, +]; + +export const SKUS = { + GPU: { + NVIDIA: NVIDIA_SKUS, + AMD: AMD_GPU_SKUS, + INTEL: { + "Arc A750": { + tflops: 34.41, + memory: [8], + msrp: 250, + power: 225, + releaseYear: 2022, + }, + "Arc A770": { + tflops: 39.32, + memory: [8, 16], + msrp: 350, + power: 225, + releaseYear: 2022, + }, + "Arc B570": { + tflops: 23.04, + memory: [10], + msrp: 200, + power: 150, + releaseYear: 2025, + }, + "Arc B580": { + tflops: 27.34, + memory: [12], + msrp: 250, + power: 190, + releaseYear: 2024, + }, + "Arc B50": { + tflops: 21.3, + memory: [16], + msrp: 350, + power: 70, + releaseYear: 2025, + }, + "Arc B60": { + tflops: 24.58, + memory: [24, 48], + msrp: 1_200, + power: 200, + releaseYear: 2025, + }, + "Arc Pro B70": { + tflops: 45.88, + memory: [32], + msrp: 949, + power: 230, + releaseYear: 2026, + }, + }, + QUALCOMM: { + "Snapdragon X Elite X1E-00-1DE": { + tflops: 4.6, + msrp: 900, + power: 80, + releaseYear: 2024, + }, + "Snapdragon X Elite X1E-84-100": { + tflops: 4.6, + msrp: 1_700, + power: 30, + releaseYear: 2024, + }, + "Snapdragon X Elite X1E-80-100": { + tflops: 3.8, + msrp: 1_300, + power: 23, + releaseYear: 2024, + }, + "Snapdragon X Elite X1E-78-100": { + tflops: 3.8, + msrp: 1_200, + power: 23, + releaseYear: 2024, + }, + "Snapdragon X Plus X1P-64-100": { + tflops: 3.8, + msrp: 1_000, + power: 23, + releaseYear: 2024, + }, + }, + }, + CPU: { + Intel: { + "Xeon 4th Generation (Sapphire Rapids)": { + tflops: 1.3, + msrp: 10_500, + power: 350, + releaseYear: 2023, + }, + "Xeon 3th Generation (Ice Lake)": { + tflops: 0.8, + msrp: 8_000, + power: 270, + releaseYear: 2021, + }, + "Xeon 2th Generation (Cascade Lake)": { + tflops: 0.55, + msrp: 10_000, + power: 205, + releaseYear: 2019, + }, + "Xeon E5v4 (Broadwell)": { + tflops: 0.25, + msrp: 4_000, + power: 145, + releaseYear: 2016, + }, + "Xeon E5v3 (Haswell)": { + tflops: 0.2, + msrp: 4_000, + power: 145, + releaseYear: 2014, + }, + "Xeon E5v2 (Ivy Bridge)": { + tflops: 0.15, + msrp: 2_500, + power: 130, + releaseYear: 2013, + }, + "Intel Core Ultra 9 275HX": { + tflops: 1.89, + msrp: 700, + power: 160, + releaseYear: 2025, + }, + "Intel Core Ultra 7 255HX": { + tflops: 1.62, + msrp: 583, + power: 160, + releaseYear: 2025, + }, + "Intel Core Ultra 7 265KF": { + tflops: 1.53, + msrp: 400, + power: 250, + releaseYear: 2024, + }, + "Intel Core 14th Generation (i7)": { + tflops: 0.8, + msrp: 400, + power: 253, + releaseYear: 2023, + }, + "Intel Core 13th Generation (i9)": { + tflops: 0.85, + msrp: 600, + power: 253, + releaseYear: 2022, + }, + "Intel Core 13th Generation (i7)": { + tflops: 0.82, + msrp: 400, + power: 253, + releaseYear: 2022, + }, + "Intel Core 13th Generation (i5)": { + tflops: 0.68, + msrp: 300, + power: 181, + releaseYear: 2022, + }, + "Intel Core 13th Generation (i3)": { + tflops: 0.57, + msrp: 150, + power: 89, + releaseYear: 2023, + }, + "Intel Core 12th Generation (i9)": { + tflops: 0.79, + msrp: 600, + power: 241, + releaseYear: 2021, + }, + "Intel Core 12th Generation (i7)": { + tflops: 0.77, + msrp: 400, + power: 190, + releaseYear: 2021, + }, + "Intel Core 12th Generation (i5)": { + tflops: 0.65, + msrp: 300, + power: 150, + releaseYear: 2021, + }, + "Intel Core 12th Generation (i3)": { + tflops: 0.53, + msrp: 150, + power: 89, + releaseYear: 2022, + }, + "Intel Core 11th Generation (i9)": { + tflops: 0.7, + msrp: 550, + power: 251, + releaseYear: 2021, + }, + "Intel Core 11th Generation (i7)": { + tflops: 0.6, + msrp: 400, + power: 251, + releaseYear: 2021, + }, + "Intel Core 11th Generation (i5)": { + tflops: 0.5, + msrp: 250, + power: 251, + releaseYear: 2021, + }, + "Intel Core 11th Generation (i3)": { + tflops: 0.35, + msrp: 150, + power: 90, + releaseYear: 2021, + }, + "Intel Core 10th Generation (i9)": { + tflops: 0.46, + msrp: 500, + power: 250, + releaseYear: 2020, + }, + "Intel Core 10th Generation (i7)": { + tflops: 0.46, + msrp: 400, + power: 215, + releaseYear: 2020, + }, + "Intel Core 10th Generation (i5)": { + tflops: 0.46, + msrp: 250, + power: 182, + releaseYear: 2020, + }, + "Intel Core 10th Generation (i3)": { + tflops: 0.44, + msrp: 150, + power: 90, + releaseYear: 2020, + }, + }, + AMD: { + "EPYC 5th Generation Zen 5 (Turin)": { + tflops: 13.8, + msrp: 13_000, + power: 500, + releaseYear: 2024, + }, + "EPYC 4th Generation Zen 4 (Genoa)": { + tflops: 5, + msrp: 11_500, + power: 360, + releaseYear: 2022, + }, + "EPYC 3th Generation Zen 3 (Milan)": { + tflops: 2.4, + msrp: 8_000, + power: 280, + releaseYear: 2021, + }, + "EPYC 2th Generation Zen 2 (Rome)": { + tflops: 0.6, + msrp: 7_000, + power: 225, + releaseYear: 2019, + }, + "EPYC 1st Generation Zen (Naples)": { + tflops: 0.6, + msrp: 4_000, + power: 180, + releaseYear: 2017, + }, + "Ryzen Threadripper Zen 5 9000 (Shimada Peak)": { + tflops: 14.0, + msrp: 5_000, + power: 350, + releaseYear: 2025, + }, + "Ryzen Threadripper Zen 4 7000 (Storm Peak)": { + tflops: 10.0, + msrp: 5_000, + power: 350, + releaseYear: 2023, + }, + "Ryzen Threadripper Zen 3 5000 (Chagall)": { + tflops: 4.6, + msrp: 6_500, + power: 280, + releaseYear: 2022, + }, + "Ryzen Threadripper Zen 2 3000 (Castle Peak)": { + tflops: 3.2, + msrp: 4_000, + power: 280, + releaseYear: 2019, + }, + "Ryzen Threadripper Zen 1000 (Whitehaven)": { + tflops: 0.6, + msrp: 1_000, + power: 180, + releaseYear: 2017, + }, + "Ryzen 7 3800X (16)": { + tflops: 1.15, + msrp: 400, + power: 142, + releaseYear: 2019, + }, + "Ryzen Zen 5 9000 (Ryzen 9)": { + tflops: 0.56, + msrp: 650, + power: 230, + releaseYear: 2024, + }, + "Ryzen Zen 5 9000 (Ryzen 7)": { + tflops: 0.56, + msrp: 350, + power: 88, + releaseYear: 2024, + }, + "Ryzen Zen 5 9000 (Ryzen 5)": { + tflops: 0.56, + msrp: 300, + power: 88, + releaseYear: 2024, + }, + "Ryzen Zen 4 7000 (Ryzen 9)": { + tflops: 0.56, + msrp: 700, + power: 230, + releaseYear: 2022, + }, + "Ryzen Zen 4 7000 (Ryzen 7)": { + tflops: 0.56, + msrp: 400, + power: 142, + releaseYear: 2022, + }, + "Ryzen Zen 4 7000 (Ryzen 5)": { + tflops: 0.56, + msrp: 300, + power: 142, + releaseYear: 2022, + }, + "Ryzen Zen 3 5000 (Ryzen 9)": { + tflops: 1.33, + msrp: 800, + power: 142, + releaseYear: 2020, + }, + "Ryzen Zen 3 5000 (Ryzen 7)": { + tflops: 1.33, + msrp: 450, + power: 142, + releaseYear: 2020, + }, + "Ryzen Zen 3 5000 (Ryzen 5)": { + tflops: 0.72, + msrp: 300, + power: 88, + releaseYear: 2020, + }, + "Ryzen Zen 2 3000 (Ryzen 9)": { + tflops: 0.72, + msrp: 750, + power: 142, + releaseYear: 2019, + }, + "Ryzen Zen 2 3000 (Ryzen 7)": { + tflops: 0.72, + msrp: 400, + power: 142, + releaseYear: 2019, + }, + "Ryzen Zen 2 3000 (Ryzen 5)": { + tflops: 0.72, + msrp: 250, + power: 88, + releaseYear: 2019, + }, + "Ryzen Zen 2 3000 (Ryzen 3)": { + tflops: 0.72, + msrp: 150, + power: 88, + releaseYear: 2020, + }, + "Ryzen AI 300 (Ryzen AI 9 HX)": { + tflops: 5.52, + msrp: 500, + power: 54, + releaseYear: 2024, + }, + "Ryzen AI 300 (Ryzen AI 9)": { + tflops: 5.2, + msrp: 450, + power: 54, + releaseYear: 2024, + }, + "Ryzen AI 300 (Ryzen AI 7)": { + tflops: 4.34, + msrp: 350, + power: 54, + releaseYear: 2024, + }, + "Ryzen AI 300 (Ryzen AI 5)": { + tflops: 1.57, + msrp: 250, + power: 28, + releaseYear: 2024, + }, + }, + }, + "Apple Silicon": { + "-": { + "Apple MacBook Neo": { + tflops: 1.9, + memory: [8], + msrp: 700, + power: 10, + releaseYear: 2026, + }, + "Apple M1": { + tflops: 2.6, + memory: [8, 16], + msrp: 1_250, + power: 15, + releaseYear: 2020, + }, + "Apple M1 Pro": { + tflops: 5.2, + memory: [16, 24, 32], + msrp: 2_900, + power: 30, + releaseYear: 2021, + }, + "Apple M1 Max": { + tflops: 10.4, + memory: [16, 24, 32, 64], + msrp: 3_900, + power: 60, + releaseYear: 2021, + }, + "Apple M1 Ultra": { + tflops: 21, + memory: [16, 24, 32, 64, 96, 128], + msrp: 6_200, + power: 120, + releaseYear: 2022, + }, + "Apple M2": { + tflops: 3.6, + memory: [8, 16, 24], + msrp: 1_500, + power: 20, + releaseYear: 2022, + }, + "Apple M2 Pro": { + tflops: 6.8, + memory: [16, 24, 32], + msrp: 2_800, + power: 35, + releaseYear: 2023, + }, + "Apple M2 Max": { + tflops: 13.49, + memory: [32, 64, 96], + msrp: 4_500, + power: 80, + releaseYear: 2023, + }, + "Apple M2 Ultra": { + tflops: 27.2, + memory: [64, 96, 128, 192], + msrp: 7_000, + power: 150, + releaseYear: 2023, + }, + "Apple M3": { + tflops: 4.1, + memory: [8, 16, 24], + msrp: 1_500, + power: 22, + releaseYear: 2023, + }, + "Apple M3 Pro": { + tflops: 7.4, + memory: [18, 36], + msrp: 2_400, + power: 40, + releaseYear: 2023, + }, + "Apple M3 Max": { + tflops: 14.2, + memory: [36, 48, 64, 96, 128], + msrp: 5_000, + power: 90, + releaseYear: 2023, + }, + "Apple M3 Ultra": { + tflops: 28.4, + memory: [96, 256, 512], + msrp: 9_500, + power: 180, + releaseYear: 2025, + }, + "Apple M4": { + tflops: 4.6, + memory: [16, 24, 32], + msrp: 1_600, + power: 22, + releaseYear: 2024, + }, + "Apple M4 Pro": { + tflops: 9.2, + memory: [24, 48, 64], + msrp: 2_600, + power: 45, + releaseYear: 2024, + }, + "Apple M4 Max": { + tflops: 18.4, + memory: [36, 48, 64, 128], + msrp: 5_000, + power: 100, + releaseYear: 2024, + }, + "Apple M5": { + tflops: 5.7, + memory: [16, 24, 32], + msrp: 2_000, + power: 25, + releaseYear: 2025, + }, + "Apple M5 Pro": { + tflops: 11.4, + memory: [24, 36, 48, 64], + msrp: 2_900, + power: 50, + releaseYear: 2026, + }, + "Apple M5 Max": { + tflops: 22.8, + memory: [36, 48, 64, 128], + msrp: 5_000, + power: 110, + releaseYear: 2026, + }, + }, + }, +} satisfies Record>>; + +export type SkuType = keyof typeof SKUS; diff --git a/node_modules/@huggingface/tasks/src/index.ts b/node_modules/@huggingface/tasks/src/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..769f6f248831569ac53a11b5476eac60c070bfae --- /dev/null +++ b/node_modules/@huggingface/tasks/src/index.ts @@ -0,0 +1,78 @@ +export { LIBRARY_TASK_MAPPING } from "./library-to-tasks.js"; +export { MAPPING_DEFAULT_WIDGET } from "./default-widget-inputs.js"; +export type { TaskData, TaskDemo, TaskDemoEntry, ExampleRepo } from "./tasks/index.js"; +export * from "./tasks/index.js"; +export { + PIPELINE_DATA, + PIPELINE_TYPES, + type WidgetType, + type PipelineType, + type PipelineData, + type Modality, + MODALITIES, + MODALITY_LABELS, + SUBTASK_TYPES, + PIPELINE_TYPES_SET, +} from "./pipelines.js"; +export { + ALL_DISPLAY_MODEL_LIBRARY_KEYS, + ALL_MODEL_LIBRARY_KEYS, + MODEL_LIBRARIES_UI_ELEMENTS, +} from "./model-libraries.js"; +export type { LibraryUiElement, ModelLibraryKey } from "./model-libraries.js"; +export type { ModelData, TransformersInfo } from "./model-data.js"; +export type { AddedToken, SpecialTokensMap, TokenizerConfig } from "./tokenizer-data.js"; +export type { + WidgetExample, + WidgetExampleAttribute, + WidgetExampleAssetAndPromptInput, + WidgetExampleAssetAndTextInput, + WidgetExampleAssetAndZeroShotInput, + WidgetExampleAssetInput, + WidgetExampleChatInput, + WidgetExampleSentenceSimilarityInput, + WidgetExampleStructuredDataInput, + WidgetExampleTableDataInput, + WidgetExampleTextAndContextInput, + WidgetExampleTextAndTableInput, + WidgetExampleTextInput, + WidgetExampleZeroShotTextInput, + WidgetExampleOutput, + WidgetExampleOutputUrl, + WidgetExampleOutputLabels, + WidgetExampleOutputAnswerScore, + WidgetExampleOutputText, +} from "./widget-example.js"; +export { SPECIAL_TOKENS_ATTRIBUTES } from "./tokenizer-data.js"; + +export * from "./gguf.js"; + +export { + type InferenceSnippet, + type InferenceSnippetLanguage, + type ModelDataMinimal, + inferenceSnippetLanguages, + stringifyGenerationConfig, + stringifyMessages, + getModelInputSnippet, +} from "./snippets/index.js"; + +export { SKUS, DEFAULT_MEMORY_OPTIONS } from "./hardware.js"; +export type { HardwareSpec, SkuType } from "./hardware.js"; +export type { AmdGpuHardwareSpec } from "./hardware-amd.js"; +export type { NvidiaHardwareSpec } from "./hardware-nvidia.js"; +export { LOCAL_APPS } from "./local-apps.js"; +export type { LocalApp, LocalAppKey, LocalAppSnippet } from "./local-apps.js"; + +export { DATASET_LIBRARIES_UI_ELEMENTS } from "./dataset-libraries.js"; +export type { DatasetLibraryUiElement, DatasetLibraryKey } from "./dataset-libraries.js"; + +export { KERNEL_LIBRARIES_UI_ELEMENTS } from "./kernel-libraries.js"; +export type { KernelLibraryKey, KernelLibraryUiElement } from "./kernel-libraries.js"; + +export * from "./inference-providers.js"; + +export { EVALUATION_FRAMEWORKS } from "./eval.js"; + +export { AGENT_HARNESSES, STANDARD_AGENT_ENV_VARS } from "./agent-harnesses.js"; +export type { AgentHarness, AgentHarnessKey } from "./agent-harnesses.js"; diff --git a/node_modules/@huggingface/tasks/src/inference-providers.ts b/node_modules/@huggingface/tasks/src/inference-providers.ts new file mode 100644 index 0000000000000000000000000000000000000000..1c82ad17eedd9b28486d0cc3b91b25f9f9863fff --- /dev/null +++ b/node_modules/@huggingface/tasks/src/inference-providers.ts @@ -0,0 +1,27 @@ +/// This list is for illustration purposes only. +/// in the `tasks` sub-package, we do not need actual strong typing of the inference providers. +const INFERENCE_PROVIDERS = [ + "cerebras", + "cohere", + "deepinfra", + "fal-ai", + "fireworks-ai", + "hf-inference", + "ovhcloud", + "replicate", + "together", +] as const; + +export type SnippetInferenceProvider = (typeof INFERENCE_PROVIDERS)[number] | string; + +export const HF_HUB_INFERENCE_PROXY_TEMPLATE = `https://router.huggingface.co/{{PROVIDER}}`; + +/** + * URL to set as baseUrl in the OpenAI SDK. + * + * TODO(Expose this from InferenceClient in the future?) + */ +export function openAIbaseUrl(provider: SnippetInferenceProvider): string { + const url = HF_HUB_INFERENCE_PROXY_TEMPLATE.replace("{{PROVIDER}}", provider); + return provider === "hf-inference" ? `${url}/v1` : url; +} diff --git a/node_modules/@huggingface/tasks/src/kernel-libraries.ts b/node_modules/@huggingface/tasks/src/kernel-libraries.ts new file mode 100644 index 0000000000000000000000000000000000000000..e0856969561edafe4ccb9ae76582ae199147c4c8 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/kernel-libraries.ts @@ -0,0 +1,43 @@ +/** + * Elements configurable by a kernel library. + */ +export interface KernelLibraryUiElement { + /** + * Pretty name of the library. + */ + prettyLabel: string; + /** + * Repo name of the library's (usually on GitHub) code repo + */ + repoName: string; + /** + * URL to library's (usually on GitHub) code repo + */ + repoUrl: string; + /** + * URL to library's docs + */ + docsUrl?: string; + /** + * Code snippet(s) displayed + */ + snippets?: (kernelId: string, version?: number) => string[]; +} + +export const KERNEL_LIBRARIES_UI_ELEMENTS = { + kernels: { + prettyLabel: "Kernels", + repoName: "Kernels", + repoUrl: "https://github.com/huggingface/kernels", + docsUrl: "https://huggingface.co/docs/kernels", + snippets: (kernelId: string, version?: number) => [ + `# !pip install kernels + +from kernels import get_kernel + +kernel = get_kernel("${kernelId}"${version !== undefined ? `, version=${version}` : ""})`, + ], + }, +} satisfies Record; + +export type KernelLibraryKey = keyof typeof KERNEL_LIBRARIES_UI_ELEMENTS; diff --git a/node_modules/@huggingface/tasks/src/library-to-tasks.ts b/node_modules/@huggingface/tasks/src/library-to-tasks.ts new file mode 100644 index 0000000000000000000000000000000000000000..af2d6894ba0edc272193377325da478c5545d601 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/library-to-tasks.ts @@ -0,0 +1,75 @@ +import type { ModelLibraryKey } from "./model-libraries.js"; +import type { PipelineType } from "./pipelines.js"; + +/** + * Mapping from library name to its supported tasks. + * HF-Inference API (serverless) should be disabled for all other (library, task) pairs beyond this mapping. + * This mapping is partially generated automatically by "python-api-export-tasks" action in + * huggingface/api-inference-community repo upon merge. For transformers, the mapping is manually + * based on api-inference (hf_types.rs). + */ +export const LIBRARY_TASK_MAPPING: Partial> = { + "adapter-transformers": ["question-answering", "text-classification", "token-classification"], + allennlp: ["question-answering"], + asteroid: [ + // "audio-source-separation", + "audio-to-audio", + ], + bertopic: ["text-classification"], + diffusers: ["image-to-image", "text-to-image"], + doctr: ["object-detection"], + espnet: ["text-to-speech", "automatic-speech-recognition"], + fairseq: ["text-to-speech", "audio-to-audio"], + fastai: ["image-classification"], + fasttext: ["feature-extraction", "text-classification"], + flair: ["token-classification"], + k2: ["automatic-speech-recognition"], + keras: ["image-classification"], + nemo: ["automatic-speech-recognition"], + open_clip: ["zero-shot-classification", "zero-shot-image-classification"], + paddlenlp: ["fill-mask", "summarization", "zero-shot-classification"], + peft: ["text-generation"], + "pyannote-audio": ["automatic-speech-recognition"], + "sentence-transformers": ["feature-extraction", "sentence-similarity"], + setfit: ["text-classification"], + sklearn: ["tabular-classification", "tabular-regression", "text-classification"], + spacy: ["token-classification", "text-classification", "sentence-similarity"], + "span-marker": ["token-classification"], + speechbrain: ["audio-classification", "audio-to-audio", "automatic-speech-recognition", "text-to-speech"], + stanza: ["token-classification"], + timm: ["image-classification", "image-feature-extraction"], + transformers: [ + "audio-classification", + "automatic-speech-recognition", + "depth-estimation", + "document-question-answering", + "feature-extraction", + "fill-mask", + "image-classification", + "image-feature-extraction", + "image-segmentation", + "image-to-image", + "image-to-text", + "image-text-to-text", + "mask-generation", + "object-detection", + "question-answering", + "summarization", + "table-question-answering", + "text-classification", + "text-generation", + "text-to-audio", + "text-to-speech", + "token-classification", + "translation", + "video-classification", + "visual-question-answering", + "zero-shot-classification", + "zero-shot-image-classification", + "zero-shot-object-detection", + ], + mindspore: ["image-classification"], +}; + +// Pipeline types that were supported in legacy transformers versions (<5.0.0) +export const REMOVED_IN_V5_TRANSFORMERS_PIPELINES: PipelineType[] = ["image-to-text", "summarization", "translation"]; diff --git a/node_modules/@huggingface/tasks/src/local-apps.spec.ts b/node_modules/@huggingface/tasks/src/local-apps.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..4732cd4ecf347fd20c939da3c0124af202ceb447 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/local-apps.spec.ts @@ -0,0 +1,366 @@ +import { describe, expect, it } from "vitest"; +import { LOCAL_APPS } from "./local-apps.js"; +import type { ModelData } from "./model-data.js"; + +describe("local-apps", () => { + it("llama.cpp conversational", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["llama.cpp"]; + const model: ModelData = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + inference: "", + }; + const snippet = snippetFunc(model); + + expect(snippet[0].content).toEqual([ + `# Start a local OpenAI-compatible server with a web UI: +llama serve -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`, + `# Run inference directly in the terminal: +llama cli -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`, + ]); + }); + + it("llama.cpp non-conversational", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["llama.cpp"]; + const model: ModelData = { + id: "mlabonne/gemma-2b-GGUF", + tags: [], + inference: "", + }; + const snippet = snippetFunc(model); + + expect(snippet[0].content).toEqual([ + `# Start a local OpenAI-compatible server with a web UI: +llama serve -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}}`, + `# Run inference directly in the terminal: +llama cli -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}}`, + ]); + }); + + it("vLLM conversational llm", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["vllm"]; + const model: ModelData = { + id: "meta-llama/Llama-3.2-3B-Instruct", + pipeline_tag: "text-generation", + tags: ["conversational"], + inference: "", + }; + const snippet = snippetFunc(model); + + expect((snippet[0].content as string[]).join("\n")).toEqual(`# Start the vLLM server: +vllm serve "meta-llama/Llama-3.2-3B-Instruct" +# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/chat/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "meta-llama/Llama-3.2-3B-Instruct", + "messages": [ + { + "role": "user", + "content": "What is the capital of France?" + } + ] + }'`); + }); + + it("vLLM non-conversational llm", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["vllm"]; + const model: ModelData = { + id: "meta-llama/Llama-3.2-3B", + tags: [""], + inference: "", + }; + const snippet = snippetFunc(model); + + expect((snippet[0].content as string[]).join("\n")).toEqual(`# Start the vLLM server: +vllm serve "meta-llama/Llama-3.2-3B" +# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "meta-llama/Llama-3.2-3B", + "prompt": "Once upon a time,", + "max_tokens": 512, + "temperature": 0.5 + }'`); + }); + + it("vLLM conversational vlm", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["vllm"]; + const model: ModelData = { + id: "meta-llama/Llama-3.2-11B-Vision-Instruct", + pipeline_tag: "image-text-to-text", + tags: ["conversational"], + inference: "", + }; + const snippet = snippetFunc(model); + + expect((snippet[0].content as string[]).join("\n")).toEqual(`# Start the vLLM server: +vllm serve "meta-llama/Llama-3.2-11B-Vision-Instruct" +# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/chat/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "meta-llama/Llama-3.2-11B-Vision-Instruct", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Describe this image in one sentence." + }, + { + "type": "image_url", + "image_url": { + "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" + } + } + ] + } + ] + }'`); + }); + + it("pi", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["pi"]; + const model: ModelData = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096, chat_template: "{% if tools %}" }, + inference: "", + }; + const snippet = snippetFunc(model); + + expect(snippet[0].content).toContain(`llama serve -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); + expect(snippet[1].setup).toContain("npm install -g @mariozechner/pi-coding-agent"); + expect(snippet[1].content).toContain(`"id": "bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}"`); + expect(snippet[2].content).toContain("pi"); + }); + + it("pi - mlx", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["pi"]; + const model: ModelData = { + id: "mlx-community/Llama-3.2-3B-Instruct-mlx", + tags: ["mlx", "conversational"], + pipeline_tag: "text-generation", + config: { + tokenizer_config: { + chat_template: "{% if tools %}...{% endif %}", + }, + }, + inference: "", + }; + const snippet = snippetFunc(model); + + expect(snippet[0].setup).toContain("uv tool install mlx-lm"); + expect(snippet[0].content).toContain('mlx_lm.server --model "mlx-community/Llama-3.2-3B-Instruct-mlx"'); + expect(snippet[1].setup).toContain("npm install -g @mariozechner/pi-coding-agent"); + expect(snippet[1].content).toContain('"baseUrl": "http://localhost:8080/v1"'); + expect(snippet[1].content).toContain('"id": "mlx-community/Llama-3.2-3B-Instruct-mlx"'); + expect(snippet[2].content).toContain("pi"); + }); + + it("hermes-agent", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["hermes-agent"]; + const model: ModelData = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096, chat_template: "{% if tools %}" }, + inference: "", + }; + const snippet = snippetFunc(model); + + expect(snippet[0].content).toContain(`llama serve -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); + expect(snippet[1].content).toContain("hermes config set model.provider custom"); + expect(snippet[1].content).toContain("hermes config set model.base_url http://127.0.0.1:8080/v1"); + expect(snippet[1].content).toContain( + "hermes config set model.default bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}", + ); + expect(snippet[2].content).toContain("hermes"); + }); + + it("hermes-agent - mlx", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["hermes-agent"]; + const model: ModelData = { + id: "mlx-community/Llama-3.2-3B-Instruct-mlx", + tags: ["mlx", "conversational"], + pipeline_tag: "text-generation", + config: { + tokenizer_config: { + chat_template: "{% if tools %}...{% endif %}", + }, + }, + inference: "", + }; + const snippet = snippetFunc(model); + + expect(snippet[0].setup).toContain("uv tool install mlx-lm"); + expect(snippet[1].content).toContain("hermes config set model.provider custom"); + expect(snippet[1].content).toContain("hermes config set model.default mlx-community/Llama-3.2-3B-Instruct-mlx"); + expect(snippet[2].content).toContain("hermes"); + }); + + it("openclaw", async () => { + const { snippet: snippetFunc } = LOCAL_APPS.openclaw; + const model: ModelData = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096, chat_template: "{% if tools %}" }, + inference: "", + }; + const snippet = snippetFunc(model); + + expect(snippet[0].content).toContain(`llama-server -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); + expect(snippet[1].setup).toContain("npm install -g openclaw@latest"); + expect(snippet[1].content).toContain("openclaw onboard --non-interactive --mode local"); + expect(snippet[1].content).toContain("--auth-choice custom-api-key"); + expect(snippet[1].content).toContain("--custom-base-url http://127.0.0.1:8080/v1"); + expect(snippet[1].content).toContain('--custom-model-id "bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}"'); + expect(snippet[1].content).toContain("--custom-provider-id llama-cpp"); + expect(snippet[1].content).toContain("--custom-compatibility openai"); + expect(snippet[1].content).not.toContain("--custom-api-key"); + expect(snippet[1].content).toContain("--custom-text-input"); + expect(snippet[1].content).toContain("--accept-risk"); + expect(snippet[1].content).toContain("--skip-health"); + expect(snippet[2].content).toContain('openclaw agent --local --agent main --message "Hello from Hugging Face"'); + }); + + it("openclaw - mlx", async () => { + const { snippet: snippetFunc } = LOCAL_APPS.openclaw; + const model: ModelData = { + id: "mlx-community/Llama-3.2-3B-Instruct-mlx", + tags: ["mlx", "conversational"], + pipeline_tag: "text-generation", + config: { + tokenizer_config: { + chat_template: "{% if tools %}...{% endif %}", + }, + }, + inference: "", + }; + const snippet = snippetFunc(model); + + expect(snippet[0].setup).toContain("uv tool install mlx-lm"); + expect(snippet[1].content).toContain("openclaw onboard --non-interactive --mode local"); + expect(snippet[1].content).toContain('--custom-model-id "mlx-community/Llama-3.2-3B-Instruct-mlx"'); + expect(snippet[1].content).toContain("--custom-provider-id mlx-lm"); + expect(snippet[1].content).toContain("--custom-text-input"); + expect(snippet[2].content).toContain('openclaw agent --local --agent main --message "Hello from Hugging Face"'); + }); + + it("docker model runner", async () => { + const { snippet: snippetFunc } = LOCAL_APPS["docker-model-runner"]; + const model: ModelData = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096 }, + inference: "", + }; + const snippet = snippetFunc(model); + + expect(snippet).toEqual(`docker model run hf.co/bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); + }); + + it("atomic chat deeplink", async () => { + const { displayOnModelPage, deeplink } = LOCAL_APPS["atomic-chat"]; + const model: ModelData = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096 }, + inference: "", + }; + + expect(displayOnModelPage(model)).toBe(true); + expect(deeplink(model).href).toBe("atomic-chat://models/huggingface/bartowski/Llama-3.2-3B-Instruct-GGUF"); + }); + + it("unsloth tagged model", async () => { + const { displayOnModelPage, snippet: snippetFunc } = LOCAL_APPS.unsloth; + const model: ModelData = { + id: "some-user/my-unsloth-finetune", + tags: ["unsloth", "conversational"], + inference: "", + }; + + expect(displayOnModelPage(model)).toBe(true); + const snippet = snippetFunc(model); + expect(snippet[0].setup).toBe("curl -fsSL https://unsloth.ai/install.sh | sh"); + expect(snippet[0].content).toBe( + "# Run unsloth studio\nunsloth studio -H 0.0.0.0 -p 8888\n# Then open http://localhost:8888 in your browser\n# Search for some-user/my-unsloth-finetune to start chatting", + ); + expect(snippet[1].setup).toBe("irm https://unsloth.ai/install.ps1 | iex"); + expect(snippet[1].content).toBe(snippet[0].content); + expect(snippet[2].setup).toBe("# No setup required"); + expect(snippet[2].content).toBe( + "# Open https://huggingface.co/spaces/unsloth/studio in your browser\n# Search for some-user/my-unsloth-finetune to start chatting", + ); + expect(snippet[3].setup).toBe("pip install unsloth"); + expect(snippet[3].content).toBe( + 'from unsloth import FastModel\nmodel, tokenizer = FastModel.from_pretrained(\n model_name="some-user/my-unsloth-finetune",\n max_seq_length=2048,\n)', + ); + }); + + it("unsloth namespace gguf model", async () => { + const { displayOnModelPage, snippet: snippetFunc } = LOCAL_APPS.unsloth; + const model: ModelData = { + id: "unsloth/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096 }, + inference: "", + }; + + expect(displayOnModelPage(model)).toBe(true); + const snippet = snippetFunc(model); + expect(snippet[0].setup).toBe("curl -fsSL https://unsloth.ai/install.sh | sh"); + expect(snippet[0].content).toBe( + "# Run unsloth studio\nunsloth studio -H 0.0.0.0 -p 8888\n# Then open http://localhost:8888 in your browser\n# Search for unsloth/Llama-3.2-3B-Instruct-GGUF to start chatting", + ); + expect(snippet[1].setup).toBe("irm https://unsloth.ai/install.ps1 | iex"); + expect(snippet[1].content).toBe(snippet[0].content); + expect(snippet[2].setup).toBe("# No setup required"); + expect(snippet[2].content).toBe( + "# Open https://huggingface.co/spaces/unsloth/studio in your browser\n# Search for unsloth/Llama-3.2-3B-Instruct-GGUF to start chatting", + ); + expect(snippet).toHaveLength(3); // GGUF models only get 3 snippets + }); + + it("non unsloth namespace gguf model", async () => { + const { displayOnModelPage } = LOCAL_APPS.unsloth; + const model: ModelData = { + id: "dummy/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + gguf: { total: 1, context_length: 4096 }, + inference: "", + }; + + expect(displayOnModelPage(model)).toBe(true); + }); + + it("unsloth not shown for unrelated model", async () => { + const { displayOnModelPage } = LOCAL_APPS.unsloth; + const model: ModelData = { + id: "meta-llama/Llama-3.2-3B-Instruct", + tags: ["conversational"], + inference: "", + }; + + expect(displayOnModelPage(model)).toBe(false); + }); + + it("links as a function", async () => { + const model: ModelData = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + tags: ["conversational"], + inference: "", + }; + const appWithFnLinks = { + ...LOCAL_APPS["llama.cpp"], + links: (m: ModelData) => [{ label: "Releases", url: `https://github.com/${m.id}/releases` }], + }; + + expect(appWithFnLinks.links(model)).toEqual([ + { label: "Releases", url: "https://github.com/bartowski/Llama-3.2-3B-Instruct-GGUF/releases" }, + ]); + }); +}); diff --git a/node_modules/@huggingface/tasks/src/local-apps.ts b/node_modules/@huggingface/tasks/src/local-apps.ts new file mode 100644 index 0000000000000000000000000000000000000000..08a3a6620b6d06ac073a64c0ad57a72d3e20ca83 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/local-apps.ts @@ -0,0 +1,844 @@ +import { parseGGUFQuantLabel } from "./gguf.js"; +import type { ModelData } from "./model-data.js"; +import type { PipelineType } from "./pipelines.js"; +import { stringifyMessages } from "./snippets/common.js"; +import { getModelInputSnippet } from "./snippets/inputs.js"; +import type { ChatCompletionInputMessage } from "./tasks/index.js"; + +export interface LocalAppSnippet { + /** + * Title of the snippet + */ + title: string; + /** + * Optional setup guide + */ + setup?: string; + /** + * Content (or command) to be run + */ + content: string | string[]; +} + +/** + * Elements configurable by a local app. + */ +export type LocalApp = { + /** + * Name that appears in buttons + */ + prettyLabel: string; + /** + * Link to get more info about a local app (website etc) + */ + docsUrl: string; + /** + * Additional links to display (max 2) + */ + links?: { label: string; url: string }[] | ((model: ModelData) => { label: string; url: string }[]); + /** + * main category of app + */ + mainTask: PipelineType; + /** + * Whether to display a pill "macOS-only" + */ + macOSOnly?: boolean; + + comingSoon?: boolean; + /** + * IMPORTANT: function to figure out whether to display the button on a model page's main "Use this model" dropdown. + */ + displayOnModelPage: (model: ModelData) => boolean; +} & ( + | { + /** + * If the app supports deeplink, URL to open. + */ + deeplink: (model: ModelData, filepath?: string) => URL; + } + | { + /** + * And if not (mostly llama.cpp), snippet to copy/paste in your terminal + * Support the placeholder {{GGUF_FILE}} that will be replaced by the gguf file path or the list of available files. + * Support the placeholder {{QUANT_TAG}} that will be replaced by the list of available quant tags or will be removed if there are no multiple quant files in a same repo. + */ + snippet: (model: ModelData, filepath?: string) => string | string[] | LocalAppSnippet | LocalAppSnippet[]; + } +); + +function isAwqModel(model: ModelData): boolean { + return model.config?.quantization_config?.quant_method === "awq"; +} + +function isGptqModel(model: ModelData): boolean { + return model.config?.quantization_config?.quant_method === "gptq"; +} + +function isAqlmModel(model: ModelData): boolean { + return model.config?.quantization_config?.quant_method === "aqlm"; +} + +function isMarlinModel(model: ModelData): boolean { + return model.config?.quantization_config?.quant_method === "marlin"; +} + +function isTransformersModel(model: ModelData): boolean { + return model.tags.includes("transformers"); +} + +function isTgiModel(model: ModelData): boolean { + return model.tags.includes("text-generation-inference"); +} + +function isLlamaCppGgufModel(model: ModelData) { + return !!model.gguf?.context_length; +} + +function isVllmModel(model: ModelData): boolean { + return ( + (isAwqModel(model) || + isGptqModel(model) || + isAqlmModel(model) || + isMarlinModel(model) || + isLlamaCppGgufModel(model) || + isTransformersModel(model)) && + (model.pipeline_tag === "text-generation" || model.pipeline_tag === "image-text-to-text") + ); +} + +function isDockerModelRunnerModel(model: ModelData): boolean { + return isLlamaCppGgufModel(model) || isVllmModel(model); +} + +function isAmdRyzenModel(model: ModelData) { + return model.tags.includes("ryzenai-hybrid") || model.tags.includes("ryzenai-npu"); +} + +function isMlxModel(model: ModelData) { + return model.tags.includes("mlx"); +} + +/** + * Returns the model's chat template string, coalescing across sources: + * GGUF metadata > chat_template_jinja file > tokenizer_config.json + */ +function getChatTemplate(model: ModelData): string | undefined { + const ct = + model.gguf?.chat_template ?? model.config?.chat_template_jinja ?? model.config?.tokenizer_config?.chat_template; + if (typeof ct === "string") { + return ct; + } + if (Array.isArray(ct)) { + return ct[0]?.template; + } + return undefined; +} + +function isUnslothModel(model: ModelData) { + return model.tags.includes("unsloth") || isLlamaCppGgufModel(model); +} + +function isToolCallingLocalAgentModel(model: ModelData): boolean { + return ( + (isLlamaCppGgufModel(model) || isMlxModel(model)) && + model.tags.includes("conversational") && + !!getChatTemplate(model)?.includes("tools") + ); +} + +function getQuantTag(filepath?: string): string { + const defaultTag = ":{{QUANT_TAG}}"; + + if (!filepath) { + return defaultTag; + } + + const quantLabel = parseGGUFQuantLabel(filepath); + return quantLabel ? `:${quantLabel}` : defaultTag; +} + +const snippetLlamacpp = (model: ModelData, filepath?: string): LocalAppSnippet[] => { + const serverCommand = (binary: string) => { + const snippet = [ + "# Start a local OpenAI-compatible server with a web UI:", + `${binary} -hf ${model.id}${getQuantTag(filepath)}`, + ]; + return snippet.join("\n"); + }; + const cliCommand = (binary: string) => { + const snippet = ["# Run inference directly in the terminal:", `${binary} -hf ${model.id}${getQuantTag(filepath)}`]; + return snippet.join("\n"); + }; + return [ + { + title: "Install (macOS, Linux)", + setup: "curl -LsSf https://llama.app/install.sh | sh", + content: [serverCommand("llama serve"), cliCommand("llama cli")], + }, + { + title: "Install from WinGet (Windows)", + setup: "winget install llama.cpp", + content: [serverCommand("llama serve"), cliCommand("llama cli")], + }, + { + title: "Use pre-built binary", + setup: [ + // prettier-ignore + "# Download pre-built binary from:", + "# https://github.com/ggerganov/llama.cpp/releases", + ].join("\n"), + content: [serverCommand("./llama-server"), cliCommand("./llama-cli")], + }, + { + title: "Build from source code", + setup: [ + "git clone https://github.com/ggerganov/llama.cpp.git", + "cd llama.cpp", + "cmake -B build", + "cmake --build build -j --target llama-server llama-cli", + ].join("\n"), + content: [serverCommand("./build/bin/llama-server"), cliCommand("./build/bin/llama-cli")], + }, + { + title: "Use Docker", + content: snippetDockerModelRunner(model, filepath), + }, + ]; +}; + +const snippetNodeLlamaCppCli = (model: ModelData, filepath?: string): LocalAppSnippet[] => { + const tagName = getQuantTag(filepath); + return [ + { + title: "Chat with the model", + content: `npx -y node-llama-cpp chat hf:${model.id}${tagName}`, + }, + { + title: "Estimate the model compatibility with your hardware", + content: `npx -y node-llama-cpp inspect estimate hf:${model.id}${tagName}`, + }, + ]; +}; + +const snippetOllama = (model: ModelData, filepath?: string): string => { + return `ollama run hf.co/${model.id}${getQuantTag(filepath)}`; +}; + +const snippetUnsloth = (model: ModelData): LocalAppSnippet[] => { + const isGguf = isLlamaCppGgufModel(model); + + const studio_content = [ + "# Run unsloth studio", + "unsloth studio -H 0.0.0.0 -p 8888", + "# Then open http://localhost:8888 in your browser", + "# Search for " + model.id + " to start chatting", + ].join("\n"); + + const studio_instructions: LocalAppSnippet = { + title: "Install Unsloth Studio (macOS, Linux, WSL)", + setup: "curl -fsSL https://unsloth.ai/install.sh | sh", + content: studio_content, + }; + + const studio_instructions_windows: LocalAppSnippet = { + title: "Install Unsloth Studio (Windows)", + setup: "irm https://unsloth.ai/install.ps1 | iex", + content: studio_content, + }; + + const hf_spaces_instructions: LocalAppSnippet = { + title: "Using HuggingFace Spaces for Unsloth", + setup: "# No setup required", + content: + "# Open https://huggingface.co/spaces/unsloth/studio in your browser\n# Search for " + + model.id + + " to start chatting", + }; + + const fastmodel_instructions: LocalAppSnippet = { + title: "Load model with FastModel", + setup: "pip install unsloth", + content: [ + "from unsloth import FastModel", + "model, tokenizer = FastModel.from_pretrained(", + ' model_name="' + model.id + '",', + " max_seq_length=2048,", + ")", + ].join("\n"), + }; + + if (isGguf) { + return [studio_instructions, studio_instructions_windows, hf_spaces_instructions]; + } else { + return [studio_instructions, studio_instructions_windows, hf_spaces_instructions, fastmodel_instructions]; + } +}; + +const snippetLocalAI = (model: ModelData, filepath?: string): LocalAppSnippet[] => { + const command = (binary: string) => + ["# Load and run the model:", `${binary} huggingface://${model.id}/${filepath ?? "{{GGUF_FILE}}"}`].join("\n"); + return [ + { + title: "Install from binary", + setup: "curl https://localai.io/install.sh | sh", + content: command("local-ai run"), + }, + { + title: "Use Docker images", + setup: [ + // prettier-ignore + "# Pull the image:", + "docker pull localai/localai:latest-cpu", + ].join("\n"), + content: command( + "docker run -p 8080:8080 --name localai -v $PWD/models:/build/models localai/localai:latest-cpu", + ), + }, + ]; +}; + +const snippetVllm = (model: ModelData): LocalAppSnippet[] => { + const messages = getModelInputSnippet(model) as ChatCompletionInputMessage[]; + + const isMistral = model.tags.includes("mistral-common"); + const mistralFlags = isMistral + ? " --tokenizer_mode mistral --config_format mistral --load_format mistral --tool-call-parser mistral --enable-auto-tool-choice" + : ""; + + const setup = isMistral + ? [ + "# Install vLLM from pip:", + "pip install vllm", + "# Install mistral-common:", + "pip install --upgrade mistral-common", + ].join("\n") + : ["# Install vLLM from pip:", "pip install vllm"].join("\n"); + + const serverCommand = `# Start the vLLM server: +vllm serve "${model.id}"${mistralFlags}`; + + const runCommandInstruct = `# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/chat/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "${model.id}", + "messages": ${stringifyMessages(messages, { + indent: "\t\t", + attributeKeyQuotes: true, + customContentEscaper: (str) => str.replace(/'/g, "'\\''"), + })} + }'`; + const runCommandNonInstruct = `# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:8000/v1/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "${model.id}", + "prompt": "Once upon a time,", + "max_tokens": 512, + "temperature": 0.5 + }'`; + const runCommand = model.tags.includes("conversational") ? runCommandInstruct : runCommandNonInstruct; + + return [ + { + title: "Install from pip and serve model", + setup: setup, + content: [serverCommand, runCommand], + }, + { + title: "Use Docker", + content: snippetDockerModelRunner(model), + }, + ]; +}; +const snippetSglang = (model: ModelData): LocalAppSnippet[] => { + const messages = getModelInputSnippet(model) as ChatCompletionInputMessage[]; + + const setup = ["# Install SGLang from pip:", "pip install sglang"].join("\n"); + const serverCommand = `# Start the SGLang server: +python3 -m sglang.launch_server \\ + --model-path "${model.id}" \\ + --host 0.0.0.0 \\ + --port 30000`; + const dockerCommand = `docker run --gpus all \\ + --shm-size 32g \\ + -p 30000:30000 \\ + -v ~/.cache/huggingface:/root/.cache/huggingface \\ + --env "HF_TOKEN=" \\ + --ipc=host \\ + lmsysorg/sglang:latest \\ + python3 -m sglang.launch_server \\ + --model-path "${model.id}" \\ + --host 0.0.0.0 \\ + --port 30000`; + const runCommandInstruct = `# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:30000/v1/chat/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "${model.id}", + "messages": ${stringifyMessages(messages, { + indent: "\t\t", + attributeKeyQuotes: true, + customContentEscaper: (str) => str.replace(/'/g, "'\\''"), + })} + }'`; + const runCommandNonInstruct = `# Call the server using curl (OpenAI-compatible API): +curl -X POST "http://localhost:30000/v1/completions" \\ + -H "Content-Type: application/json" \\ + --data '{ + "model": "${model.id}", + "prompt": "Once upon a time,", + "max_tokens": 512, + "temperature": 0.5 + }'`; + const runCommand = model.tags.includes("conversational") ? runCommandInstruct : runCommandNonInstruct; + + return [ + { + title: "Install from pip and serve model", + setup: setup, + content: [serverCommand, runCommand], + }, + { + title: "Use Docker images", + setup: dockerCommand, + content: [runCommand], + }, + ]; +}; +const snippetTgi = (model: ModelData): LocalAppSnippet[] => { + const runCommand = [ + "# Call the server using curl:", + `curl -X POST "http://localhost:8000/v1/chat/completions" \\`, + ` -H "Content-Type: application/json" \\`, + ` --data '{`, + ` "model": "${model.id}",`, + ` "messages": [`, + ` {"role": "user", "content": "What is the capital of France?"}`, + ` ]`, + ` }'`, + ]; + return [ + { + title: "Use Docker images", + setup: [ + "# Deploy with docker on Linux:", + `docker run --gpus all \\`, + ` -v ~/.cache/huggingface:/root/.cache/huggingface \\`, + ` -e HF_TOKEN="" \\`, + ` -p 8000:80 \\`, + ` ghcr.io/huggingface/text-generation-inference:latest \\`, + ` --model-id ${model.id}`, + ].join("\n"), + content: [runCommand.join("\n")], + }, + ]; +}; + +const snippetMlxLm = (model: ModelData): LocalAppSnippet[] => { + const openaiCurl = [ + "# Calling the OpenAI-compatible server with curl", + `curl -X POST "http://localhost:8000/v1/chat/completions" \\`, + ` -H "Content-Type: application/json" \\`, + ` --data '{`, + ` "model": "${model.id}",`, + ` "messages": [`, + ` {"role": "user", "content": "Hello"}`, + ` ]`, + ` }'`, + ]; + + return [ + { + title: "Generate or start a chat session", + setup: ["# Install MLX LM", "uv tool install mlx-lm"].join("\n"), + content: [ + ...(model.tags.includes("conversational") + ? ["# Interactive chat REPL", `mlx_lm.chat --model "${model.id}"`] + : ["# Generate some text", `mlx_lm.generate --model "${model.id}" --prompt "Once upon a time"`]), + ].join("\n"), + }, + ...(model.tags.includes("conversational") + ? [ + { + title: "Run an OpenAI-compatible server", + setup: ["# Install MLX LM", "uv tool install mlx-lm"].join("\n"), + content: ["# Start the server", `mlx_lm.server --model "${model.id}"`, ...openaiCurl].join("\n"), + }, + ] + : []), + ]; +}; + +const getLocalServerStep = (model: ModelData, filepath?: string): LocalAppSnippet => { + return isMlxModel(model) + ? { + title: "Start the MLX server", + setup: "# Install MLX LM:\nuv tool install mlx-lm", + content: `# Start a local OpenAI-compatible server:\nmlx_lm.server --model "${model.id}"`, + } + : { + title: "Start the llama.cpp server", + setup: "# Install llama.cpp:\nbrew install llama.cpp", + content: `# Start a local OpenAI-compatible server:\nllama serve -hf ${model.id}${getQuantTag(filepath)}`, + }; +}; + +const snippetPi = (model: ModelData, filepath?: string): LocalAppSnippet[] => { + const isMLX = isMlxModel(model); + const modelId = isMLX ? model.id : `${model.id}${getQuantTag(filepath)}`; + const serverStep = getLocalServerStep(model, filepath); + + const modelsJson = JSON.stringify( + { + providers: { + [isMLX ? "mlx-lm" : "llama-cpp"]: { + baseUrl: "http://localhost:8080/v1", + api: "openai-completions", + apiKey: "none", + models: [{ id: modelId }], + }, + }, + }, + null, + 2, + ); + + return [ + serverStep, + { + title: "Configure the model in Pi", + setup: "# Install Pi:\nnpm install -g @mariozechner/pi-coding-agent", + content: `# Add to ~/.pi/agent/models.json:\n${modelsJson}`, + }, + { + title: "Run Pi", + content: "# Start Pi in your project directory:\npi", + }, + ]; +}; + +const snippetHermesAgent = (model: ModelData, filepath?: string): LocalAppSnippet[] => { + const modelId = isMlxModel(model) ? model.id : `${model.id}${getQuantTag(filepath)}`; + const serverStep = getLocalServerStep(model, filepath); + + return [ + serverStep, + { + title: "Configure Hermes", + setup: [ + "# Install Hermes:", + "curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash", + "hermes setup", + ].join("\n"), + content: [ + "# Point Hermes at the local server:", + "hermes config set model.provider custom", + "hermes config set model.base_url http://127.0.0.1:8080/v1", + `hermes config set model.default ${modelId}`, + ].join("\n"), + }, + { + title: "Run Hermes", + content: "hermes", + }, + ]; +}; + +const snippetOpenClaw = (model: ModelData, filepath?: string): LocalAppSnippet[] => { + const isMLX = isMlxModel(model); + const providerId = isMLX ? "mlx-lm" : "llama-cpp"; + const modelId = isMLX ? model.id : `${model.id}${getQuantTag(filepath)}`; + const serverStep = getLocalServerStep(model, filepath); + + return [ + serverStep, + { + title: "Configure OpenClaw", + setup: "# Install OpenClaw:\nnpm install -g openclaw@latest", + content: [ + "# Register the local server and set it as the default model:", + "openclaw onboard --non-interactive --mode local \\", + " --auth-choice custom-api-key \\", + " --custom-base-url http://127.0.0.1:8080/v1 \\", + ` --custom-model-id "${modelId}" \\`, + ` --custom-provider-id ${providerId} \\`, + " --custom-compatibility openai \\", + " --custom-text-input \\", + " --accept-risk \\", + " --skip-health", + ].join("\n"), + }, + { + title: "Run OpenClaw", + content: `openclaw agent --local --agent main --message "Hello from Hugging Face"`, + }, + ]; +}; + +const snippetDockerModelRunner = (model: ModelData, filepath?: string): string => { + // Only add quant tag for GGUF models, not safetensors + const quantTag = isLlamaCppGgufModel(model) ? getQuantTag(filepath) : ""; + return `docker model run hf.co/${model.id}${quantTag}`; +}; + +const snippetLemonade = (model: ModelData, filepath?: string): LocalAppSnippet[] => { + const modelName = model.id.includes("/") ? model.id.split("/")[1] : model.id; + const isRyzenAI = model.tags.some((tag) => ["ryzenai-npu", "ryzenai-hybrid"].includes(tag)); + + // Lemonade auto-registers pulled models as `user.[-]`. + // For GGUF/llamacpp: suggested_name is the repo name and variant is the quant tag. + // For RyzenAI ONNX: there is no per-variant suffix. + let pullArg: string; + let runName: string; + let requirements: string; + if (isRyzenAI) { + pullArg = model.id; + runName = `user.${modelName}`; + requirements = " (requires XDNA 2 NPU)"; + } else { + const tagName = getQuantTag(filepath); + pullArg = `${model.id}${tagName}`; + runName = `user.${modelName}${tagName.replace(":", "-")}`; + requirements = ""; + } + + return [ + { + title: "Pull the model", + setup: "# Download Lemonade from https://lemonade-server.ai/", + content: `lemonade pull ${pullArg}`, + }, + { + title: `Run and chat with the model${requirements}`, + content: `lemonade run ${runName}`, + }, + { + title: "List all available models", + content: "lemonade list", + }, + ]; +}; + +/** + * Add your new local app here. + * + * This is open to new suggestions and awesome upcoming apps. + * + * /!\ IMPORTANT + * + * If possible, you need to support deeplinks and be as cross-platform as possible. + * + * Ping the HF team if we can help with anything! + */ +export const LOCAL_APPS = { + "llama.cpp": { + prettyLabel: "llama.cpp", + docsUrl: "https://github.com/ggerganov/llama.cpp", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + snippet: snippetLlamacpp, + }, + "node-llama-cpp": { + prettyLabel: "node-llama-cpp", + docsUrl: "https://node-llama-cpp.withcat.ai", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + snippet: snippetNodeLlamaCppCli, + }, + vllm: { + prettyLabel: "vLLM", + docsUrl: "https://docs.vllm.ai", + mainTask: "text-generation", + displayOnModelPage: isVllmModel, + snippet: snippetVllm, + }, + sglang: { + prettyLabel: "SGLang", + docsUrl: "https://docs.sglang.io", + mainTask: "text-generation", + displayOnModelPage: (model: ModelData) => + (isAwqModel(model) || + isGptqModel(model) || + isAqlmModel(model) || + isMarlinModel(model) || + isTransformersModel(model)) && + (model.pipeline_tag === "text-generation" || model.pipeline_tag === "image-text-to-text"), + snippet: snippetSglang, + }, + "mlx-lm": { + prettyLabel: "MLX LM", + docsUrl: "https://github.com/ml-explore/mlx-lm", + mainTask: "text-generation", + displayOnModelPage: (model) => model.pipeline_tag === "text-generation" && isMlxModel(model), + snippet: snippetMlxLm, + }, + tgi: { + prettyLabel: "TGI", + docsUrl: "https://huggingface.co/docs/text-generation-inference/", + mainTask: "text-generation", + displayOnModelPage: isTgiModel, + snippet: snippetTgi, + }, + lmstudio: { + prettyLabel: "LM Studio", + docsUrl: "https://lmstudio.ai", + mainTask: "text-generation", + displayOnModelPage: (model) => isLlamaCppGgufModel(model) || isMlxModel(model), + deeplink: (model, filepath) => + new URL(`lmstudio://open_from_hf?model=${model.id}${filepath ? `&file=${filepath}` : ""}`), + }, + localai: { + prettyLabel: "LocalAI", + docsUrl: "https://github.com/mudler/LocalAI", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + snippet: snippetLocalAI, + }, + jan: { + prettyLabel: "Jan", + docsUrl: "https://jan.ai", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`jan://models/huggingface/${model.id}`), + }, + "atomic-chat": { + prettyLabel: "Atomic Chat", + docsUrl: "https://atomic.chat", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`atomic-chat://models/huggingface/${model.id}`), + }, + backyard: { + prettyLabel: "Backyard AI", + docsUrl: "https://backyard.ai", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`https://backyard.ai/hf/model/${model.id}`), + }, + sanctum: { + prettyLabel: "Sanctum", + docsUrl: "https://sanctum.ai", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`sanctum://open_from_hf?model=${model.id}`), + }, + jellybox: { + prettyLabel: "Jellybox", + docsUrl: "https://jellybox.com", + mainTask: "text-generation", + displayOnModelPage: (model) => + isLlamaCppGgufModel(model) || + (model.library_name === "diffusers" && + model.tags.includes("safetensors") && + (model.pipeline_tag === "text-to-image" || model.tags.includes("lora"))), + deeplink: (model) => { + if (isLlamaCppGgufModel(model)) { + return new URL(`jellybox://llm/models/huggingface/LLM/${model.id}`); + } else if (model.tags.includes("lora")) { + return new URL(`jellybox://image/models/huggingface/ImageLora/${model.id}`); + } else { + return new URL(`jellybox://image/models/huggingface/Image/${model.id}`); + } + }, + }, + msty: { + prettyLabel: "Msty", + docsUrl: "https://msty.app", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`msty://models/search/hf/${model.id}`), + }, + recursechat: { + prettyLabel: "RecurseChat", + docsUrl: "https://recurse.chat", + mainTask: "text-generation", + macOSOnly: true, + displayOnModelPage: isLlamaCppGgufModel, + deeplink: (model) => new URL(`recursechat://new-hf-gguf-model?hf-model-id=${model.id}`), + }, + drawthings: { + prettyLabel: "Draw Things", + docsUrl: "https://drawthings.ai", + mainTask: "text-to-image", + macOSOnly: true, + displayOnModelPage: (model) => + model.library_name === "diffusers" && (model.pipeline_tag === "text-to-image" || model.tags.includes("lora")), + deeplink: (model) => { + if (model.tags.includes("lora")) { + return new URL(`https://drawthings.ai/import/diffusers/pipeline.load_lora_weights?repo_id=${model.id}`); + } else { + return new URL(`https://drawthings.ai/import/diffusers/pipeline.from_pretrained?repo_id=${model.id}`); + } + }, + }, + diffusionbee: { + prettyLabel: "DiffusionBee", + docsUrl: "https://diffusionbee.com", + mainTask: "text-to-image", + macOSOnly: true, + displayOnModelPage: (model) => model.library_name === "diffusers" && model.pipeline_tag === "text-to-image", + deeplink: (model) => new URL(`https://diffusionbee.com/huggingface_import?model_id=${model.id}`), + }, + joyfusion: { + prettyLabel: "JoyFusion", + docsUrl: "https://joyfusion.app", + mainTask: "text-to-image", + macOSOnly: true, + displayOnModelPage: (model) => + model.tags.includes("coreml") && model.tags.includes("joyfusion") && model.pipeline_tag === "text-to-image", + deeplink: (model) => new URL(`https://joyfusion.app/import_from_hf?repo_id=${model.id}`), + }, + ollama: { + prettyLabel: "Ollama", + docsUrl: "https://ollama.com", + mainTask: "text-generation", + displayOnModelPage: isLlamaCppGgufModel, + snippet: snippetOllama, + }, + unsloth: { + prettyLabel: "Unsloth Studio", + docsUrl: "https://unsloth.ai/docs/new/studio", + mainTask: "text-generation", + displayOnModelPage: isUnslothModel, + snippet: snippetUnsloth, + }, + "docker-model-runner": { + prettyLabel: "Docker Model Runner", + docsUrl: "https://docs.docker.com/ai/model-runner/", + mainTask: "text-generation", + displayOnModelPage: isDockerModelRunnerModel, + snippet: snippetDockerModelRunner, + }, + lemonade: { + prettyLabel: "Lemonade", + docsUrl: "https://lemonade-server.ai", + mainTask: "text-generation", + displayOnModelPage: (model) => isLlamaCppGgufModel(model) || isAmdRyzenModel(model), + snippet: snippetLemonade, + }, + pi: { + prettyLabel: "Pi", + docsUrl: "https://github.com/badlogic/pi-mono", + mainTask: "text-generation", + displayOnModelPage: isToolCallingLocalAgentModel, + snippet: snippetPi, + }, + "hermes-agent": { + prettyLabel: "Hermes Agent", + docsUrl: "https://hermes-agent.nousresearch.com/", + mainTask: "text-generation", + displayOnModelPage: isToolCallingLocalAgentModel, + snippet: snippetHermesAgent, + }, + openclaw: { + prettyLabel: "OpenClaw", + docsUrl: "https://github.com/openclaw/openclaw", + mainTask: "text-generation", + displayOnModelPage: isToolCallingLocalAgentModel, + snippet: snippetOpenClaw, + }, +} satisfies Record; + +export type LocalAppKey = keyof typeof LOCAL_APPS; diff --git a/node_modules/@huggingface/tasks/src/model-data.ts b/node_modules/@huggingface/tasks/src/model-data.ts new file mode 100644 index 0000000000000000000000000000000000000000..3a043f51741c27526dfae784ced9cdd740216f4d --- /dev/null +++ b/node_modules/@huggingface/tasks/src/model-data.ts @@ -0,0 +1,157 @@ +import type { PipelineType } from "./pipelines.js"; +import type { WidgetExample } from "./widget-example.js"; +import type { TokenizerConfig } from "./tokenizer-data.js"; + +/** + * Public interface for model metadata + */ +export interface ModelData { + /** + * id of model (e.g. 'user/repo_name') + */ + id: string; + /** + * Whether or not to enable inference widget for this model + * TODO(type it) + */ + inference: string; + /** + * is this model private? + */ + private?: boolean; + /** + * this dictionary has useful information about the model configuration + */ + config?: { + architectures?: string[]; + /** + * Dict of AutoModel or Auto… class name to local import path in the repo + */ + auto_map?: { + /** + * String Property + */ + [x: string]: string; + }; + model_type?: string; + quantization_config?: { + bits?: number; + load_in_4bit?: boolean; + load_in_8bit?: boolean; + /** + * awq, gptq, aqlm, marlin, … Used by vLLM + */ + quant_method?: string; + }; + tokenizer_config?: TokenizerConfig; + processor_config?: { + chat_template?: string; + }; + chat_template_jinja?: string; + adapter_transformers?: { + model_name?: string; + model_class?: string; + }; + diffusers?: { + _class_name?: string; + }; + sklearn?: { + model?: { + file?: string; + }; + model_format?: string; + }; + speechbrain?: { + speechbrain_interface?: string; + vocoder_interface?: string; + vocoder_model_id?: string; + }; + peft?: { + base_model_name_or_path?: string; + task_type?: string; + }; + keras_hub?: { + tasks?: string[]; + }; + }; + /** + * all the model tags + */ + tags: string[]; + /** + * transformers-specific info to display in the code sample. + */ + transformersInfo?: TransformersInfo; + /** + * Pipeline type + */ + pipeline_tag?: PipelineType | undefined; + /** + * for relevant models, get mask token + */ + mask_token?: string | undefined; + /** + * Example data that will be fed into the widget. + * + * can be set in the model card metadata (under `widget`), + * or by default in `DefaultWidget.ts` + */ + widgetData?: WidgetExample[] | undefined; + /** + * Parameters that will be used by the widget when calling Inference API (serverless) + * https://huggingface.co/docs/api-inference/detailed_parameters + * + * can be set in the model card metadata (under `inference/parameters`) + * Example: + * inference: + * parameters: + * key: val + */ + cardData?: { + inference?: + | boolean + | { + parameters?: Record; + }; + base_model?: string | string[]; + instance_prompt?: string | null; + }; + /** + * Library name + * Example: transformers, SpeechBrain, Stanza, etc. + */ + library_name?: string; + safetensors?: { + parameters: Record; + total: number; + sharded: boolean; + }; + gguf?: { + total: number; + architecture?: string; + context_length?: number; + chat_template?: string; + }; +} + +/** + * transformers-specific info to display in the code sample. + */ +export interface TransformersInfo { + /** + * e.g. AutoModelForSequenceClassification + */ + auto_model: string; + /** + * if set in config.json's auto_map + */ + custom_class?: string; + /** + * e.g. text-classification + */ + pipeline_tag?: PipelineType; + /** + * e.g. "AutoTokenizer" | "AutoFeatureExtractor" | "AutoProcessor" + */ + processor?: string; +} diff --git a/node_modules/@huggingface/tasks/src/model-libraries-downloads.ts b/node_modules/@huggingface/tasks/src/model-libraries-downloads.ts new file mode 100644 index 0000000000000000000000000000000000000000..70fc3bface26cdf0ad939623921435724deb8ced --- /dev/null +++ b/node_modules/@huggingface/tasks/src/model-libraries-downloads.ts @@ -0,0 +1,18 @@ +/** + * This file contains the (simplified) types used + * to represent queries that are made to Elastic + * in order to count number of model downloads + * + * Read this doc about download stats on the Hub: + * + * https://huggingface.co/docs/hub/models-download-stats + * Available fields: + * - path: the complete file path (relative) (e.g: "prefix/file.extension") + * - path_prefix: the prefix of the file path (e.g: "prefix/", empty if no prefix) + * - path_extension: the extension of the file path (e.g: "extension") + * - path_filename: the name of the file path (e.g: "file") + * see also: + * https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html + */ + +export type ElasticSearchQuery = string; diff --git a/node_modules/@huggingface/tasks/src/model-libraries-snippets.spec.ts b/node_modules/@huggingface/tasks/src/model-libraries-snippets.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..81d28f5540d51715b640248e46681ed19358f715 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/model-libraries-snippets.spec.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import type { ModelData } from "./model-data.js"; +import { llama_cpp_python } from "./model-libraries-snippets.js"; + +describe("model-libraries-snippets", () => { + it("llama_cpp_python conversational", async () => { + const model: ModelData = { + id: "bartowski/Llama-3.2-3B-Instruct-GGUF", + pipeline_tag: "text-generation", + tags: ["conversational"], + inference: "", + }; + const snippet = llama_cpp_python(model); + + expect(snippet.join("\n")).toEqual(`# !pip install llama-cpp-python + +from llama_cpp import Llama + +llm = Llama.from_pretrained( + repo_id="bartowski/Llama-3.2-3B-Instruct-GGUF", + filename="{{GGUF_FILE}}", +) + +llm.create_chat_completion( + messages = [ + { + "role": "user", + "content": "What is the capital of France?" + } + ] +)`); + }); + + it("llama_cpp_python non-conversational", async () => { + const model: ModelData = { + id: "mlabonne/gemma-2b-GGUF", + tags: [""], + inference: "", + }; + const snippet = llama_cpp_python(model); + + expect(snippet.join("\n")).toEqual(`# !pip install llama-cpp-python + +from llama_cpp import Llama + +llm = Llama.from_pretrained( + repo_id="mlabonne/gemma-2b-GGUF", + filename="{{GGUF_FILE}}", +) + +output = llm( + "Once upon a time,", + max_tokens=512, + echo=True +) +print(output)`); + }); +}); diff --git a/node_modules/@huggingface/tasks/src/model-libraries-snippets.ts b/node_modules/@huggingface/tasks/src/model-libraries-snippets.ts new file mode 100644 index 0000000000000000000000000000000000000000..8bfe373644278e3aade8baf475c247624b5faf70 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/model-libraries-snippets.ts @@ -0,0 +1,2530 @@ +import type { ModelData } from "./model-data.js"; +import type { WidgetExampleTextInput, WidgetExampleSentenceSimilarityInput } from "./widget-example.js"; +import { LIBRARY_TASK_MAPPING, REMOVED_IN_V5_TRANSFORMERS_PIPELINES } from "./library-to-tasks.js"; +import { getModelInputSnippet } from "./snippets/inputs.js"; +import type { ChatCompletionInputMessage } from "./tasks/index.js"; +import { stringifyMessages } from "./snippets/common.js"; + +const TAG_CUSTOM_CODE = "custom_code"; + +function nameWithoutNamespace(modelId: string): string { + const splitted = modelId.split("/"); + return splitted.length === 1 ? splitted[0] : splitted[1]; +} + +const escapeStringForJson = (str: string): string => JSON.stringify(str).slice(1, -1); // slice is needed to remove surrounding quotes added by JSON.stringify + +//#region snippets + +export const adapters = (model: ModelData): string[] => [ + `from adapters import AutoAdapterModel + +model = AutoAdapterModel.from_pretrained("${model.config?.adapter_transformers?.model_name}") +model.load_adapter("${model.id}", set_active=True)`, +]; + +const allennlpUnknown = (model: ModelData) => [ + `import allennlp_models +from allennlp.predictors.predictor import Predictor + +predictor = Predictor.from_path("hf://${model.id}")`, +]; + +const allennlpQuestionAnswering = (model: ModelData) => [ + `import allennlp_models +from allennlp.predictors.predictor import Predictor + +predictor = Predictor.from_path("hf://${model.id}") +predictor_input = {"passage": "My name is Wolfgang and I live in Berlin", "question": "Where do I live?"} +predictions = predictor.predict_json(predictor_input)`, +]; + +export const allennlp = (model: ModelData): string[] => { + if (model.tags.includes("question-answering")) { + return allennlpQuestionAnswering(model); + } + return allennlpUnknown(model); +}; + +export const araclip = (model: ModelData): string[] => [ + `from araclip import AraClip + +model = AraClip.from_pretrained("${model.id}")`, +]; + +export const asteroid = (model: ModelData): string[] => [ + `from asteroid.models import BaseModel + +model = BaseModel.from_pretrained("${model.id}")`, +]; + +export const audioseal = (model: ModelData): string[] => { + const watermarkSnippet = `# Watermark Generator +from audioseal import AudioSeal + +model = AudioSeal.load_generator("${model.id}") +# pass a tensor (tensor_wav) of shape (batch, channels, samples) and a sample rate +wav, sr = tensor_wav, 16000 + +watermark = model.get_watermark(wav, sr) +watermarked_audio = wav + watermark`; + + const detectorSnippet = `# Watermark Detector +from audioseal import AudioSeal + +detector = AudioSeal.load_detector("${model.id}") + +result, message = detector.detect_watermark(watermarked_audio, sr)`; + return [watermarkSnippet, detectorSnippet]; +}; + +function get_base_diffusers_model(model: ModelData): string { + return model.cardData?.base_model?.toString() ?? "fill-in-base-model"; +} + +function get_prompt_from_diffusers_model(model: ModelData): string | undefined { + const prompt = (model.widgetData?.[0] as WidgetExampleTextInput | undefined)?.text ?? model.cardData?.instance_prompt; + if (prompt) { + return escapeStringForJson(prompt); + } +} + +export const ben2 = (model: ModelData): string[] => [ + `import requests +from PIL import Image +from ben2 import AutoModel + +url = "https://huggingface.co/datasets/mishig/sample_images/resolve/main/teapot.jpg" +image = Image.open(requests.get(url, stream=True).raw) + +model = AutoModel.from_pretrained("${model.id}") +model.to("cuda").eval() +foreground = model.inference(image) +`, +]; + +export const bertopic = (model: ModelData): string[] => [ + `from bertopic import BERTopic + +model = BERTopic.load("${model.id}")`, +]; + +export const bm25s = (model: ModelData): string[] => [ + `from bm25s.hf import BM25HF + +retriever = BM25HF.load_from_hub("${model.id}")`, +]; + +export const chatterbox = (): string[] => [ + `# pip install chatterbox-tts +import torchaudio as ta +from chatterbox.tts import ChatterboxTTS + +model = ChatterboxTTS.from_pretrained(device="cuda") + +text = "Ezreal and Jinx teamed up with Ahri, Yasuo, and Teemo to take down the enemy's Nexus in an epic late-game pentakill." +wav = model.generate(text) +ta.save("test-1.wav", wav, model.sr) + +# If you want to synthesize with a different voice, specify the audio prompt +AUDIO_PROMPT_PATH="YOUR_FILE.wav" +wav = model.generate(text, audio_prompt_path=AUDIO_PROMPT_PATH) +ta.save("test-2.wav", wav, model.sr)`, +]; + +export const chronos_forecasting = (model: ModelData): string[] => { + const installSnippet = `pip install chronos-forecasting`; + + const exampleSnippet = `import pandas as pd +from chronos import BaseChronosPipeline + +pipeline = BaseChronosPipeline.from_pretrained("${model.id}", device_map="cuda") + +# Load historical data +context_df = pd.read_csv("https://autogluon.s3.us-west-2.amazonaws.com/datasets/timeseries/misc/AirPassengers.csv") + +# Generate predictions +pred_df = pipeline.predict_df( + context_df, + prediction_length=36, # Number of steps to forecast + quantile_levels=[0.1, 0.5, 0.9], # Quantiles for probabilistic forecast + id_column="item_id", # Column identifying different time series + timestamp_column="Month", # Column with datetime information + target="#Passengers", # Column(s) with time series values to predict +)`; + + return [installSnippet, exampleSnippet]; +}; + +export const collectorvision = (model: ModelData): string[] => [ + `pip install git+https://github.com/HanClinto/CollectorVision huggingface_hub`, + `from huggingface_hub import hf_hub_download +import collector_vision as cvg + +checkpoint = hf_hub_download(repo_id="${model.id}", filename="model.onnx") + +# Detector models, such as Cornelius: +detector = cvg.NeuralCornerDetector(checkpoint) + +# Embedder models, such as Milo: +embedder = cvg.NeuralEmbedder(checkpoint)`, +]; + +export const colipri = (model: ModelData): string[] => { + const installSnippet = `pip install colipri`; + + const exampleSnippet = `from colipri import get_model +from colipri import get_processor +from colipri import load_sample_ct +from colipri import ZeroShotImageClassificationPipeline + +model = get_model().cuda() +processor = get_processor() +pipeline = ZeroShotImageClassificationPipeline("${model.id}", processor) + +image = load_sample_ct() + +pipeline(image, ["No lung nodules", "Lung nodules"]) +`; + + return [installSnippet, exampleSnippet]; +}; + +export const sap_rpt_one_oss = (): string[] => { + const installSnippet = `pip install git+https://github.com/SAP-samples/sap-rpt-1-oss`; + + const classificationSnippet = `# Run a classification task +from sklearn.datasets import load_breast_cancer +from sklearn.metrics import accuracy_score +from sklearn.model_selection import train_test_split + +from sap_rpt_oss import SAP_RPT_OSS_Classifier + +# Load sample data +X, y = load_breast_cancer(return_X_y=True) +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42) + +# Initialize a classifier, 8k context and 8-fold bagging gives best performance, reduce if running out of memory +clf = SAP_RPT_OSS_Classifier(max_context_size=8192, bagging=8) + +clf.fit(X_train, y_train) + +# Predict probabilities +prediction_probabilities = clf.predict_proba(X_test) +# Predict labels +predictions = clf.predict(X_test) +print("Accuracy", accuracy_score(y_test, predictions))`; + + const regressionsSnippet = `# Run a regression task +from sklearn.datasets import fetch_openml +from sklearn.metrics import r2_score +from sklearn.model_selection import train_test_split + +from sap_rpt_oss import SAP_RPT_OSS_Regressor + +# Load sample data +df = fetch_openml(data_id=531, as_frame=True) +X = df.data +y = df.target.astype(float) + +# Train-test split +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42) + +# Initialize the regressor, 8k context and 8-fold bagging gives best performance, reduce if running out of memory +regressor = SAP_RPT_OSS_Regressor(max_context_size=8192, bagging=8) + +regressor.fit(X_train, y_train) + +# Predict on the test set +predictions = regressor.predict(X_test) + +r2 = r2_score(y_test, predictions) +print("R² Score:", r2)`; + return [installSnippet, classificationSnippet, regressionsSnippet]; +}; + +export const cxr_foundation = (): string[] => [ + `# pip install git+https://github.com/Google-Health/cxr-foundation.git#subdirectory=python + +# Load image as grayscale (Stillwaterising, CC0, via Wikimedia Commons) +import requests +from PIL import Image +from io import BytesIO +image_url = "https://upload.wikimedia.org/wikipedia/commons/c/c8/Chest_Xray_PA_3-8-2010.png" +img = Image.open(requests.get(image_url, headers={'User-Agent': 'Demo'}, stream=True).raw).convert('L') + +# Run inference +from clientside.clients import make_hugging_face_client +cxr_client = make_hugging_face_client('cxr_model') +print(cxr_client.get_image_embeddings_from_images([img]))`, +]; + +export const depth_anything_v2 = (model: ModelData): string[] => { + let encoder: string; + let features: string; + let out_channels: string; + + encoder = ""; + features = ""; + out_channels = ""; + + if (model.id === "depth-anything/Depth-Anything-V2-Small") { + encoder = "vits"; + features = "64"; + out_channels = "[48, 96, 192, 384]"; + } else if (model.id === "depth-anything/Depth-Anything-V2-Base") { + encoder = "vitb"; + features = "128"; + out_channels = "[96, 192, 384, 768]"; + } else if (model.id === "depth-anything/Depth-Anything-V2-Large") { + encoder = "vitl"; + features = "256"; + out_channels = "[256, 512, 1024, 1024"; + } + + return [ + ` +# Install from https://github.com/DepthAnything/Depth-Anything-V2 + +# Load the model and infer depth from an image +import cv2 +import torch + +from depth_anything_v2.dpt import DepthAnythingV2 + +# instantiate the model +model = DepthAnythingV2(encoder="${encoder}", features=${features}, out_channels=${out_channels}) + +# load the weights +filepath = hf_hub_download(repo_id="${model.id}", filename="depth_anything_v2_${encoder}.pth", repo_type="model") +state_dict = torch.load(filepath, map_location="cpu") +model.load_state_dict(state_dict).eval() + +raw_img = cv2.imread("your/image/path") +depth = model.infer_image(raw_img) # HxW raw depth map in numpy + `, + ]; +}; + +export const depth_pro = (model: ModelData): string[] => { + const installSnippet = `# Download checkpoint +pip install huggingface-hub +huggingface-cli download --local-dir checkpoints ${model.id}`; + + const inferenceSnippet = `import depth_pro + +# Load model and preprocessing transform +model, transform = depth_pro.create_model_and_transforms() +model.eval() + +# Load and preprocess an image. +image, _, f_px = depth_pro.load_rgb("example.png") +image = transform(image) + +# Run inference. +prediction = model.infer(image, f_px=f_px) + +# Results: 1. Depth in meters +depth = prediction["depth"] +# Results: 2. Focal length in pixels +focallength_px = prediction["focallength_px"]`; + + return [installSnippet, inferenceSnippet]; +}; + +export const derm_foundation = (): string[] => [ + `from huggingface_hub import from_pretrained_keras +import tensorflow as tf, requests + +# Load and format input +IMAGE_URL = "https://storage.googleapis.com/dx-scin-public-data/dataset/images/3445096909671059178.png" +input_tensor = tf.train.Example( + features=tf.train.Features( + feature={ + "image/encoded": tf.train.Feature( + bytes_list=tf.train.BytesList(value=[requests.get(IMAGE_URL, stream=True).content]) + ) + } + ) +).SerializeToString() + +# Load model and run inference +loaded_model = from_pretrained_keras("google/derm-foundation") +infer = loaded_model.signatures["serving_default"] +print(infer(inputs=tf.constant([input_tensor])))`, +]; + +export const dia = (model: ModelData): string[] => [ + `import soundfile as sf +from dia.model import Dia + +model = Dia.from_pretrained("${model.id}") +text = "[S1] Dia is an open weights text to dialogue model. [S2] You get full control over scripts and voices. [S1] Wow. Amazing. (laughs) [S2] Try it now on Git hub or Hugging Face." +output = model.generate(text) + +sf.write("simple.mp3", output, 44100)`, +]; + +export const dia2 = (model: ModelData): string[] => [ + `from dia2 import Dia2, GenerationConfig, SamplingConfig + +dia = Dia2.from_repo("${model.id}", device="cuda", dtype="bfloat16") +config = GenerationConfig( + cfg_scale=2.0, + audio=SamplingConfig(temperature=0.8, top_k=50), + use_cuda_graph=True, +) +result = dia.generate("[S1] Hello Dia2!", config=config, output_wav="hello.wav", verbose=True) +`, +]; + +export const describe_anything = (model: ModelData): string[] => [ + `# pip install git+https://github.com/NVlabs/describe-anything +from huggingface_hub import snapshot_download +from dam import DescribeAnythingModel + +snapshot_download(${model.id}, local_dir="checkpoints") + +dam = DescribeAnythingModel( + model_path="checkpoints", + conv_mode="v1", + prompt_mode="focal_prompt", +)`, +]; + +const diffusers_install = "pip install -U diffusers transformers accelerate"; + +const diffusersDefaultPrompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"; + +const diffusersImg2ImgDefaultPrompt = "Turn this cat into a dog"; + +const diffusersVideoDefaultPrompt = "A man with short gray hair plays a red electric guitar."; + +const diffusers_default = (model: ModelData) => [ + `import torch +from diffusers import DiffusionPipeline + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersDefaultPrompt}" +image = pipe(prompt).images[0]`, +]; + +const diffusers_image_to_image = (model: ModelData) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import load_image + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersImg2ImgDefaultPrompt}" +input_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png") + +image = pipe(image=input_image, prompt=prompt).images[0]`, +]; + +const diffusers_image_to_video = (model: ModelData) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import load_image, export_to_video + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda") +pipe.to("cuda") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersVideoDefaultPrompt}" +image = load_image( + "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/guitar-man.png" +) + +output = pipe(image=image, prompt=prompt).frames[0] +export_to_video(output, "output.mp4")`, +]; + +const diffusers_controlnet = (model: ModelData) => [ + `from diffusers import ControlNetModel, StableDiffusionControlNetPipeline + +controlnet = ControlNetModel.from_pretrained("${model.id}") +pipe = StableDiffusionControlNetPipeline.from_pretrained( + "${get_base_diffusers_model(model)}", controlnet=controlnet +)`, +]; + +const diffusers_lora = (model: ModelData) => [ + `import torch +from diffusers import DiffusionPipeline + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_lora_weights("${model.id}") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersDefaultPrompt}" +image = pipe(prompt).images[0]`, +]; + +const diffusers_lora_image_to_image = (model: ModelData) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import load_image + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_lora_weights("${model.id}") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersImg2ImgDefaultPrompt}" +input_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png") + +image = pipe(image=input_image, prompt=prompt).images[0]`, +]; + +const diffusers_lora_text_to_video = (model: ModelData) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import export_to_video + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_lora_weights("${model.id}") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersVideoDefaultPrompt}" + +output = pipe(prompt=prompt).frames[0] +export_to_video(output, "output.mp4")`, +]; + +const diffusers_lora_image_to_video = (model: ModelData) => [ + `import torch +from diffusers import DiffusionPipeline +from diffusers.utils import load_image, export_to_video + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_lora_weights("${model.id}") + +prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersVideoDefaultPrompt}" +input_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/guitar-man.png") + +image = pipe(image=input_image, prompt=prompt).frames[0] +export_to_video(output, "output.mp4")`, +]; + +const diffusers_textual_inversion = (model: ModelData) => [ + `import torch +from diffusers import DiffusionPipeline + +# switch to "mps" for apple devices +pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda") +pipe.load_textual_inversion("${model.id}")`, +]; + +const diffusers_flux_fill = (model: ModelData) => [ + `import torch +from diffusers import FluxFillPipeline +from diffusers.utils import load_image + +image = load_image("https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup.png") +mask = load_image("https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup_mask.png") + +# switch to "mps" for apple devices +pipe = FluxFillPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda") +image = pipe( + prompt="a white paper cup", + image=image, + mask_image=mask, + height=1632, + width=1232, + guidance_scale=30, + num_inference_steps=50, + max_sequence_length=512, + generator=torch.Generator("cpu").manual_seed(0) +).images[0] +image.save(f"flux-fill-dev.png")`, +]; + +const diffusers_inpainting = (model: ModelData) => [ + `import torch +from diffusers import AutoPipelineForInpainting +from diffusers.utils import load_image + +# switch to "mps" for apple devices +pipe = AutoPipelineForInpainting.from_pretrained("${model.id}", dtype=torch.float16, variant="fp16", device_map="cuda") + +img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png" +mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png" + +image = load_image(img_url).resize((1024, 1024)) +mask_image = load_image(mask_url).resize((1024, 1024)) + +prompt = "a tiger sitting on a park bench" +generator = torch.Generator(device="cuda").manual_seed(0) + +image = pipe( + prompt=prompt, + image=image, + mask_image=mask_image, + guidance_scale=8.0, + num_inference_steps=20, # steps between 15 and 30 work well for us + strength=0.99, # make sure to use \`strength\` below 1.0 + generator=generator, +).images[0]`, +]; + +export const diffusers = (model: ModelData): string[] => { + let codeSnippets: string[]; + if ( + model.tags.includes("StableDiffusionInpaintPipeline") || + model.tags.includes("StableDiffusionXLInpaintPipeline") + ) { + codeSnippets = diffusers_inpainting(model); + } else if (model.tags.includes("controlnet")) { + codeSnippets = diffusers_controlnet(model); + } else if (model.tags.includes("lora")) { + if (model.pipeline_tag === "image-to-image") { + codeSnippets = diffusers_lora_image_to_image(model); + } else if (model.pipeline_tag === "image-to-video") { + codeSnippets = diffusers_lora_image_to_video(model); + } else if (model.pipeline_tag === "text-to-video") { + codeSnippets = diffusers_lora_text_to_video(model); + } else { + codeSnippets = diffusers_lora(model); + } + } else if (model.tags.includes("textual_inversion")) { + codeSnippets = diffusers_textual_inversion(model); + } else if (model.tags.includes("FluxFillPipeline")) { + codeSnippets = diffusers_flux_fill(model); + } else if (model.pipeline_tag === "image-to-video") { + codeSnippets = diffusers_image_to_video(model); + } else if (model.pipeline_tag === "image-to-image") { + codeSnippets = diffusers_image_to_image(model); + } else { + codeSnippets = diffusers_default(model); + } + + return [diffusers_install, ...codeSnippets]; +}; + +export const diffusionkit = (model: ModelData): string[] => { + const sd3Snippet = `# Pipeline for Stable Diffusion 3 +from diffusionkit.mlx import DiffusionPipeline + +pipeline = DiffusionPipeline( + shift=3.0, + use_t5=False, + model_version=${model.id}, + low_memory_mode=True, + a16=True, + w16=True, +)`; + + const fluxSnippet = `# Pipeline for Flux +from diffusionkit.mlx import FluxPipeline + +pipeline = FluxPipeline( + shift=1.0, + model_version=${model.id}, + low_memory_mode=True, + a16=True, + w16=True, +)`; + + const generateSnippet = `# Image Generation +HEIGHT = 512 +WIDTH = 512 +NUM_STEPS = ${model.tags.includes("flux") ? 4 : 50} +CFG_WEIGHT = ${model.tags.includes("flux") ? 0 : 5} + +image, _ = pipeline.generate_image( + "a photo of a cat", + cfg_weight=CFG_WEIGHT, + num_steps=NUM_STEPS, + latent_size=(HEIGHT // 8, WIDTH // 8), +)`; + + const pipelineSnippet = model.tags.includes("flux") ? fluxSnippet : sd3Snippet; + + return [pipelineSnippet, generateSnippet]; +}; + +export const cartesia_pytorch = (model: ModelData): string[] => [ + `# pip install --no-binary :all: cartesia-pytorch +from cartesia_pytorch import ReneLMHeadModel +from transformers import AutoTokenizer + +model = ReneLMHeadModel.from_pretrained("${model.id}") +tokenizer = AutoTokenizer.from_pretrained("allenai/OLMo-1B-hf") + +in_message = ["Rene Descartes was"] +inputs = tokenizer(in_message, return_tensors="pt") + +outputs = model.generate(inputs.input_ids, max_length=50, top_k=100, top_p=0.99) +out_message = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0] + +print(out_message) +)`, +]; + +export const cartesia_mlx = (model: ModelData): string[] => [ + `import mlx.core as mx +import cartesia_mlx as cmx + +model = cmx.from_pretrained("${model.id}") +model.set_dtype(mx.float32) + +prompt = "Rene Descartes was" + +for text in model.generate( + prompt, + max_tokens=500, + eval_every_n=5, + verbose=True, + top_p=0.99, + temperature=0.85, +): + print(text, end="", flush=True) +`, +]; + +export const edsnlp = (model: ModelData): string[] => { + const packageName = nameWithoutNamespace(model.id).replaceAll("-", "_"); + return [ + `# Load it from the Hub directly +import edsnlp +nlp = edsnlp.load("${model.id}") +`, + `# Or install it as a package +!pip install git+https://huggingface.co/${model.id} + +# and import it as a module +import ${packageName} + +nlp = ${packageName}.load() # or edsnlp.load("${packageName}") +`, + ]; +}; + +export const espnetTTS = (model: ModelData): string[] => [ + `from espnet2.bin.tts_inference import Text2Speech + +model = Text2Speech.from_pretrained("${model.id}") + +speech, *_ = model("text to generate speech from")`, +]; + +export const espnetASR = (model: ModelData): string[] => [ + `from espnet2.bin.asr_inference import Speech2Text + +model = Speech2Text.from_pretrained( + "${model.id}" +) + +speech, rate = soundfile.read("speech.wav") +text, *_ = model(speech)[0]`, +]; + +const espnetUnknown = () => [`unknown model type (must be text-to-speech or automatic-speech-recognition)`]; + +export const espnet = (model: ModelData): string[] => { + if (model.tags.includes("text-to-speech")) { + return espnetTTS(model); + } else if (model.tags.includes("automatic-speech-recognition")) { + return espnetASR(model); + } + return espnetUnknown(); +}; + +export const fairseq = (model: ModelData): string[] => [ + `from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub + +models, cfg, task = load_model_ensemble_and_task_from_hf_hub( + "${model.id}" +)`, +]; + +export const flair = (model: ModelData): string[] => [ + `from flair.models import SequenceTagger + +tagger = SequenceTagger.load("${model.id}")`, +]; + +export const gliner = (model: ModelData): string[] => [ + `from gliner import GLiNER + +model = GLiNER.from_pretrained("${model.id}")`, +]; + +export const gliner2 = (model: ModelData): string[] => [ + `from gliner2 import GLiNER2 + +model = GLiNER2.from_pretrained("${model.id}") + +# Extract entities +text = "Apple CEO Tim Cook announced iPhone 15 in Cupertino yesterday." +result = extractor.extract_entities(text, ["company", "person", "product", "location"]) + +print(result)`, +]; + +export const indextts = (model: ModelData): string[] => [ + `# Download model +from huggingface_hub import snapshot_download + +snapshot_download(${model.id}, local_dir="checkpoints") + +from indextts.infer import IndexTTS + +# Ensure config.yaml is present in the checkpoints directory +tts = IndexTTS(model_dir="checkpoints", cfg_path="checkpoints/config.yaml") + +voice = "path/to/your/reference_voice.wav" # Path to the voice reference audio file +text = "Hello, how are you?" +output_path = "output_index.wav" + +tts.infer(voice, text, output_path)`, +]; + +export const htrflow = (model: ModelData): string[] => [ + `# CLI usage +# see docs: https://ai-riksarkivet.github.io/htrflow/latest/getting_started/quick_start.html +htrflow pipeline `, + `# Python usage +from htrflow.pipeline.pipeline import Pipeline +from htrflow.pipeline.steps import Task +from htrflow.models.framework.model import ModelClass + +pipeline = Pipeline( + [ + Task( + ModelClass, {"model": "${model.id}"}, {} + ), + ])`, +]; + +export const keras = (model: ModelData): string[] => [ + `# Available backend options are: "jax", "torch", "tensorflow". +import os +os.environ["KERAS_BACKEND"] = "jax" + +import keras + +model = keras.saving.load_model("hf://${model.id}") +`, +]; + +const _keras_hub_causal_lm = (modelId: string): string => ` +import keras_hub + +# Load CausalLM model (optional: use half precision for inference) +causal_lm = keras_hub.models.CausalLM.from_preset("hf://${modelId}", dtype="bfloat16") +causal_lm.compile(sampler="greedy") # (optional) specify a sampler + +# Generate text +causal_lm.generate("Keras: deep learning for", max_length=64) +`; + +const _keras_hub_text_to_image = (modelId: string): string => ` +import keras_hub + +# Load TextToImage model (optional: use half precision for inference) +text_to_image = keras_hub.models.TextToImage.from_preset("hf://${modelId}", dtype="bfloat16") + +# Generate images with a TextToImage model. +text_to_image.generate("Astronaut in a jungle") +`; + +const _keras_hub_text_classifier = (modelId: string): string => ` +import keras_hub + +# Load TextClassifier model +text_classifier = keras_hub.models.TextClassifier.from_preset( + "hf://${modelId}", + num_classes=2, +) +# Fine-tune +text_classifier.fit(x=["Thilling adventure!", "Total snoozefest."], y=[1, 0]) +# Classify text +text_classifier.predict(["Not my cup of tea."]) +`; + +const _keras_hub_image_classifier = (modelId: string): string => ` +import keras_hub +import keras + +# Load ImageClassifier model +image_classifier = keras_hub.models.ImageClassifier.from_preset( + "hf://${modelId}", + num_classes=2, +) +# Fine-tune +image_classifier.fit( + x=keras.random.randint((32, 64, 64, 3), 0, 256), + y=keras.random.randint((32, 1), 0, 2), +) +# Classify image +image_classifier.predict(keras.random.randint((1, 64, 64, 3), 0, 256)) +`; + +const _keras_hub_tasks_with_example = { + CausalLM: _keras_hub_causal_lm, + TextToImage: _keras_hub_text_to_image, + TextClassifier: _keras_hub_text_classifier, + ImageClassifier: _keras_hub_image_classifier, +}; + +const _keras_hub_task_without_example = (task: string, modelId: string): string => ` +import keras_hub + +# Create a ${task} model +task = keras_hub.models.${task}.from_preset("hf://${modelId}") +`; + +const _keras_hub_generic_backbone = (modelId: string): string => ` +import keras_hub + +# Create a Backbone model unspecialized for any task +backbone = keras_hub.models.Backbone.from_preset("hf://${modelId}") +`; + +export const keras_hub = (model: ModelData): string[] => { + const modelId = model.id; + const tasks = model.config?.keras_hub?.tasks ?? []; + + const snippets: string[] = []; + + // First, generate tasks with examples + for (const [task, snippet] of Object.entries(_keras_hub_tasks_with_example)) { + if (tasks.includes(task)) { + snippets.push(snippet(modelId)); + } + } + // Then, add remaining tasks + for (const task of tasks) { + if (!Object.keys(_keras_hub_tasks_with_example).includes(task)) { + snippets.push(_keras_hub_task_without_example(task, modelId)); + } + } + // Finally, add generic backbone snippet + snippets.push(_keras_hub_generic_backbone(modelId)); + + return snippets; +}; + +export const kernels = (model: ModelData): string[] => [ + `# !pip install kernels + +from kernels import get_kernel + +kernel = get_kernel("${model.id}")`, +]; + +export const kimi_audio = (model: ModelData): string[] => [ + `# Example usage for KimiAudio +# pip install git+https://github.com/MoonshotAI/Kimi-Audio.git + +from kimia_infer.api.kimia import KimiAudio + +model = KimiAudio(model_path="${model.id}", load_detokenizer=True) + +sampling_params = { + "audio_temperature": 0.8, + "audio_top_k": 10, + "text_temperature": 0.0, + "text_top_k": 5, +} + +# For ASR +asr_audio = "asr_example.wav" +messages_asr = [ + {"role": "user", "message_type": "text", "content": "Please transcribe the following audio:"}, + {"role": "user", "message_type": "audio", "content": asr_audio} +] +_, text = model.generate(messages_asr, **sampling_params, output_type="text") +print(text) + +# For Q&A +qa_audio = "qa_example.wav" +messages_conv = [{"role": "user", "message_type": "audio", "content": qa_audio}] +wav, text = model.generate(messages_conv, **sampling_params, output_type="both") +sf.write("output_audio.wav", wav.cpu().view(-1).numpy(), 24000) +print(text) +`, +]; + +export const kittentts = (model: ModelData): string[] => [ + `from kittentts import KittenTTS +m = KittenTTS("${model.id}") + +audio = m.generate("This high quality TTS model works without a GPU") + +# Save the audio +import soundfile as sf +sf.write('output.wav', audio, 24000)`, +]; + +export const lightning_ir = (model: ModelData): string[] => { + if (model.tags.includes("bi-encoder")) { + return [ + `#install from https://github.com/webis-de/lightning-ir + +from lightning_ir import BiEncoderModule +model = BiEncoderModule("${model.id}") + +model.score("query", ["doc1", "doc2", "doc3"])`, + ]; + } else if (model.tags.includes("cross-encoder")) { + return [ + `#install from https://github.com/webis-de/lightning-ir + +from lightning_ir import CrossEncoderModule +model = CrossEncoderModule("${model.id}") + +model.score("query", ["doc1", "doc2", "doc3"])`, + ]; + } + return [ + `#install from https://github.com/webis-de/lightning-ir + +from lightning_ir import BiEncoderModule, CrossEncoderModule + +# depending on the model type, use either BiEncoderModule or CrossEncoderModule +model = BiEncoderModule("${model.id}") +# model = CrossEncoderModule("${model.id}") + +model.score("query", ["doc1", "doc2", "doc3"])`, + ]; +}; + +export const llama_cpp_python = (model: ModelData): string[] => { + const snippets = [ + `# !pip install llama-cpp-python + +from llama_cpp import Llama + +llm = Llama.from_pretrained( + repo_id="${model.id}", + filename="{{GGUF_FILE}}", +) +`, + ]; + + if (model.tags.includes("conversational")) { + const messages = getModelInputSnippet(model) as ChatCompletionInputMessage[]; + snippets.push(`llm.create_chat_completion( + messages = ${stringifyMessages(messages, { attributeKeyQuotes: true, indent: "\t" })} +)`); + } else { + snippets.push(`output = llm( + "Once upon a time,", + max_tokens=512, + echo=True +) +print(output)`); + } + + return snippets; +}; + +export const lerobot = (model: ModelData): string[] => { + if (model.tags.includes("smolvla")) { + const smolvlaSnippets = [ + // Installation snippet + `# See https://github.com/huggingface/lerobot?tab=readme-ov-file#installation for more details +git clone https://github.com/huggingface/lerobot.git +cd lerobot +pip install -e .[smolvla]`, + // Finetune snippet + `# Launch finetuning on your dataset +python lerobot/scripts/train.py \\ +--policy.path=${model.id} \\ +--dataset.repo_id=lerobot/svla_so101_pickplace \\ +--batch_size=64 \\ +--steps=20000 \\ +--output_dir=outputs/train/my_smolvla \\ +--job_name=my_smolvla_training \\ +--policy.device=cuda \\ +--wandb.enable=true`, + ]; + if (model.id !== "lerobot/smolvla_base") { + // Inference snippet (only if not base model) + smolvlaSnippets.push( + `# Run the policy using the record function +python -m lerobot.record \\ + --robot.type=so101_follower \\ + --robot.port=/dev/ttyACM0 \\ # <- Use your port + --robot.id=my_blue_follower_arm \\ # <- Use your robot id + --robot.cameras="{ front: {type: opencv, index_or_path: 8, width: 640, height: 480, fps: 30}}" \\ # <- Use your cameras + --dataset.single_task="Grasp a lego block and put it in the bin." \\ # <- Use the same task description you used in your dataset recording + --dataset.repo_id=HF_USER/dataset_name \\ # <- This will be the dataset name on HF Hub + --dataset.episode_time_s=50 \\ + --dataset.num_episodes=10 \\ + --policy.path=${model.id}`, + ); + } + return smolvlaSnippets; + } + return []; +}; + +export const litert_lm = (model: ModelData): string[] => [ + `# LiteRT-LM runs on various platforms (Android, iOS, Windows, Linux, macOS, IoT, Web/WASM) +# and supports many APIs (C++, Python, Kotlin, Swift, JavaScript, Flutter). +# For platform-specific integration guides, please refer to the official developer website: +# https://ai.google.dev/edge/litert-lm + +# To try LiteRT-LM, the easiest way is to use our CLI tool. +# 1. Install the LiteRT-LM CLI tool: +pip install litert-lm + +# 2. Download and run this model locally: +# See: https://ai.google.dev/edge/litert-lm/cli +litert-lm run \\ + --from-huggingface-repo=${model.id} \\ + model.litertlm \\ + --prompt="Write me a poem"`, +]; + +export const tf_keras = (model: ModelData): string[] => [ + `# Note: 'keras<3.x' or 'tf_keras' must be installed (legacy) +# See https://github.com/keras-team/tf-keras for more details. +from huggingface_hub import from_pretrained_keras + +model = from_pretrained_keras("${model.id}") +`, +]; + +export const mamba_ssm = (model: ModelData): string[] => [ + `from mamba_ssm import MambaLMHeadModel + +model = MambaLMHeadModel.from_pretrained("${model.id}")`, +]; + +export const mars5_tts = (model: ModelData): string[] => [ + `# Install from https://github.com/Camb-ai/MARS5-TTS + +from inference import Mars5TTS +mars5 = Mars5TTS.from_pretrained("${model.id}")`, +]; + +export const matanyone = (model: ModelData): string[] => [ + `# Install from https://github.com/pq-yang/MatAnyone.git + +from matanyone.model.matanyone import MatAnyone +model = MatAnyone.from_pretrained("${model.id}")`, + ` +from matanyone import InferenceCore +processor = InferenceCore("${model.id}")`, +]; + +export const mesh_anything = (): string[] => [ + `# Install from https://github.com/buaacyw/MeshAnything.git + +from MeshAnything.models.meshanything import MeshAnything + +# refer to https://github.com/buaacyw/MeshAnything/blob/main/main.py#L91 on how to define args +# and https://github.com/buaacyw/MeshAnything/blob/main/app.py regarding usage +model = MeshAnything(args)`, +]; + +export const multimolecule = (model: ModelData): string[] => { + const widgetExample = model.widgetData?.[0] as WidgetExampleTextInput | undefined; + const exampleText = widgetExample?.text; + const maskToken = model.mask_token ?? ""; + const sequence = exampleText?.replace(maskToken, "A"); + + const snippets = [`pip install multimolecule`]; + + if (sequence) { + snippets.push( + `from multimolecule import AutoModel, AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained("${model.id}") +model = AutoModel.from_pretrained("${model.id}") + +inputs = tokenizer("${sequence}", return_tensors="pt") +outputs = model(**inputs) +embeddings = outputs.last_hidden_state`, + ); + } else { + snippets.push( + `from multimolecule import AutoModel, AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained("${model.id}") +model = AutoModel.from_pretrained("${model.id}")`, + ); + } + + if (model.tags.includes("rna-secondary-structure") && exampleText) { + snippets.push( + `import multimolecule +from transformers import pipeline + +predictor = pipeline("rna-secondary-structure", model="${model.id}") +output = predictor("${exampleText}") +print(output["secondary_structure"])`, + ); + } else if (model.pipeline_tag === "fill-mask" && exampleText) { + snippets.push( + `import multimolecule +from transformers import pipeline + +predictor = pipeline("fill-mask", model="${model.id}") +output = predictor("${exampleText}")`, + ); + } + + return snippets; +}; + +export const open_clip = (model: ModelData): string[] => [ + `import open_clip + +model, preprocess_train, preprocess_val = open_clip.create_model_and_transforms('hf-hub:${model.id}') +tokenizer = open_clip.get_tokenizer('hf-hub:${model.id}')`, +]; + +export const paddlenlp = (model: ModelData): string[] => { + if (model.config?.architectures?.[0]) { + const architecture = model.config.architectures[0]; + return [ + [ + `from paddlenlp.transformers import AutoTokenizer, ${architecture}`, + "", + `tokenizer = AutoTokenizer.from_pretrained("${model.id}", from_hf_hub=True)`, + `model = ${architecture}.from_pretrained("${model.id}", from_hf_hub=True)`, + ].join("\n"), + ]; + } else { + return [ + [ + `# ⚠️ Type of model unknown`, + `from paddlenlp.transformers import AutoTokenizer, AutoModel`, + "", + `tokenizer = AutoTokenizer.from_pretrained("${model.id}", from_hf_hub=True)`, + `model = AutoModel.from_pretrained("${model.id}", from_hf_hub=True)`, + ].join("\n"), + ]; + } +}; + +export const paddleocr = (model: ModelData): string[] => { + const mapping: Record = { + textline_detection: { className: "TextDetection" }, + textline_recognition: { className: "TextRecognition" }, + seal_text_detection: { className: "SealTextDetection" }, + doc_img_unwarping: { className: "TextImageUnwarping" }, + doc_img_orientation_classification: { className: "DocImgOrientationClassification" }, + textline_orientation_classification: { className: "TextLineOrientationClassification" }, + chart_parsing: { className: "ChartParsing" }, + formula_recognition: { className: "FormulaRecognition" }, + layout_detection: { className: "LayoutDetection" }, + table_cells_detection: { className: "TableCellsDetection" }, + wired_table_classification: { className: "TableClassification" }, + table_structure_recognition: { className: "TableStructureRecognition" }, + }; + + if (model.tags.includes("doc_vlm")) { + return [ + `# 1. See https://www.paddlepaddle.org.cn/en/install to install paddlepaddle +# 2. pip install paddleocr + +from paddleocr import DocVLM +model = DocVLM(model_name="${nameWithoutNamespace(model.id)}") +output = model.predict( + input={"image": "path/to/image.png", "query": "Parsing this image and output the content in Markdown format."}, + batch_size=1 +) +for res in output: + res.print() + res.save_to_json(save_path="./output/res.json")`, + ]; + } + + if (model.tags.includes("document-parse")) { + const rawVersion = model.id.replace("PaddlePaddle/PaddleOCR-VL-", "v"); + const version = rawVersion === "PaddlePaddle/PaddleOCR-VL" ? "v1" : rawVersion; + return [ + `# See https://www.paddleocr.ai/latest/version3.x/pipeline_usage/PaddleOCR-VL.html to installation + +from paddleocr import PaddleOCRVL +pipeline = PaddleOCRVL(pipeline_version="${version}") +output = pipeline.predict("path/to/document_image.png") +for res in output: + res.print() + res.save_to_json(save_path="output") + res.save_to_markdown(save_path="output")`, + ]; + } + + for (const tag of model.tags) { + if (tag in mapping) { + const { className } = mapping[tag]; + return [ + `# 1. See https://www.paddlepaddle.org.cn/en/install to install paddlepaddle +# 2. pip install paddleocr + +from paddleocr import ${className} +model = ${className}(model_name="${nameWithoutNamespace(model.id)}") +output = model.predict(input="path/to/image.png", batch_size=1) +for res in output: + res.print() + res.save_to_img(save_path="./output/") + res.save_to_json(save_path="./output/res.json")`, + ]; + } + } + + return [ + `# Please refer to the document for information on how to use the model. +# https://paddlepaddle.github.io/PaddleOCR/latest/en/version3.x/module_usage/module_overview.html`, + ]; +}; + +export const perception_encoder = (model: ModelData): string[] => { + const clip_model = `# Use PE-Core models as CLIP models +import core.vision_encoder.pe as pe + +model = pe.CLIP.from_config("${model.id}", pretrained=True)`; + + const vision_encoder = `# Use any PE model as a vision encoder +import core.vision_encoder.pe as pe + +model = pe.VisionTransformer.from_config("${model.id}", pretrained=True)`; + + if (model.id.includes("Core")) { + return [clip_model, vision_encoder]; + } else { + return [vision_encoder]; + } +}; +export const phantom_wan = (model: ModelData): string[] => [ + `from huggingface_hub import snapshot_download +from phantom_wan import WANI2V, configs + +checkpoint_dir = snapshot_download("${model.id}") +wan_i2v = WanI2V( + config=configs.WAN_CONFIGS['i2v-14B'], + checkpoint_dir=checkpoint_dir, + ) + video = wan_i2v.generate(text_prompt, image_prompt)`, +]; + +export const pocket_tts = (model: ModelData): string[] => [ + `from pocket_tts import TTSModel +import scipy.io.wavfile + +tts_model = TTSModel.load_model("${model.id}") +voice_state = tts_model.get_state_for_audio_prompt( + "hf://kyutai/tts-voices/alba-mackenna/casual.wav" +) +audio = tts_model.generate_audio(voice_state, "Hello world, this is a test.") +# Audio is a 1D torch tensor containing PCM data. +scipy.io.wavfile.write("output.wav", tts_model.sample_rate, audio.numpy())`, +]; + +export const pyannote_audio_pipeline = (model: ModelData): string[] => [ + `from pyannote.audio import Pipeline + +pipeline = Pipeline.from_pretrained("${model.id}") + +# inference on the whole file +pipeline("file.wav") + +# inference on an excerpt +from pyannote.core import Segment +excerpt = Segment(start=2.0, end=5.0) + +from pyannote.audio import Audio +waveform, sample_rate = Audio().crop("file.wav", excerpt) +pipeline({"waveform": waveform, "sample_rate": sample_rate})`, +]; + +const pyannote_audio_model = (model: ModelData): string[] => [ + `from pyannote.audio import Model, Inference + +model = Model.from_pretrained("${model.id}") +inference = Inference(model) + +# inference on the whole file +inference("file.wav") + +# inference on an excerpt +from pyannote.core import Segment +excerpt = Segment(start=2.0, end=5.0) +inference.crop("file.wav", excerpt)`, +]; + +export const pyannote_audio = (model: ModelData): string[] => { + if (model.tags.includes("pyannote-audio-pipeline")) { + return pyannote_audio_pipeline(model); + } + return pyannote_audio_model(model); +}; + +export const relik = (model: ModelData): string[] => [ + `from relik import Relik + +relik = Relik.from_pretrained("${model.id}")`, +]; + +export const renderformer = (model: ModelData): string[] => [ + `# Install from https://github.com/microsoft/renderformer + +from renderformer import RenderFormerRenderingPipeline +pipeline = RenderFormerRenderingPipeline.from_pretrained("${model.id}")`, +]; + +const tensorflowttsTextToMel = (model: ModelData): string[] => [ + `from tensorflow_tts.inference import AutoProcessor, TFAutoModel + +processor = AutoProcessor.from_pretrained("${model.id}") +model = TFAutoModel.from_pretrained("${model.id}") +`, +]; + +const tensorflowttsMelToWav = (model: ModelData): string[] => [ + `from tensorflow_tts.inference import TFAutoModel + +model = TFAutoModel.from_pretrained("${model.id}") +audios = model.inference(mels) +`, +]; + +const tensorflowttsUnknown = (model: ModelData): string[] => [ + `from tensorflow_tts.inference import TFAutoModel + +model = TFAutoModel.from_pretrained("${model.id}") +`, +]; + +export const tensorflowtts = (model: ModelData): string[] => { + if (model.tags.includes("text-to-mel")) { + return tensorflowttsTextToMel(model); + } else if (model.tags.includes("mel-to-wav")) { + return tensorflowttsMelToWav(model); + } + return tensorflowttsUnknown(model); +}; + +export const timm = (model: ModelData): string[] => [ + `import timm + +model = timm.create_model("hf_hub:${model.id}", pretrained=True)`, +]; + +export const saelens = (/* model: ModelData */): string[] => [ + `# pip install sae-lens +from sae_lens import SAE + +sae, cfg_dict, sparsity = SAE.from_pretrained( + release = "RELEASE_ID", # e.g., "gpt2-small-res-jb". See other options in https://github.com/jbloomAus/SAELens/blob/main/sae_lens/pretrained_saes.yaml + sae_id = "SAE_ID", # e.g., "blocks.8.hook_resid_pre". Won't always be a hook point +)`, +]; + +export const seed_story = (): string[] => [ + `# seed_story_cfg_path refers to 'https://github.com/TencentARC/SEED-Story/blob/master/configs/clm_models/agent_7b_sft.yaml' +# llm_cfg_path refers to 'https://github.com/TencentARC/SEED-Story/blob/master/configs/clm_models/llama2chat7b_lora.yaml' +from omegaconf import OmegaConf +import hydra + +# load Llama2 +llm_cfg = OmegaConf.load(llm_cfg_path) +llm = hydra.utils.instantiate(llm_cfg, torch_dtype="fp16") + +# initialize seed_story +seed_story_cfg = OmegaConf.load(seed_story_cfg_path) +seed_story = hydra.utils.instantiate(seed_story_cfg, llm=llm) `, +]; + +const skopsPickle = (model: ModelData, modelFile: string) => { + return [ + `import joblib +from skops.hub_utils import download +download("${model.id}", "path_to_folder") +model = joblib.load( + "${modelFile}" +) +# only load pickle files from sources you trust +# read more about it here https://skops.readthedocs.io/en/stable/persistence.html`, + ]; +}; + +const skopsFormat = (model: ModelData, modelFile: string) => { + return [ + `from skops.hub_utils import download +from skops.io import load +download("${model.id}", "path_to_folder") +# make sure model file is in skops format +# if model is a pickle file, make sure it's from a source you trust +model = load("path_to_folder/${modelFile}")`, + ]; +}; + +const skopsJobLib = (model: ModelData) => { + return [ + `from huggingface_hub import hf_hub_download +import joblib +model = joblib.load( + hf_hub_download("${model.id}", "sklearn_model.joblib") +) +# only load pickle files from sources you trust +# read more about it here https://skops.readthedocs.io/en/stable/persistence.html`, + ]; +}; + +export const sklearn = (model: ModelData): string[] => { + if (model.tags.includes("skops")) { + const skopsmodelFile = model.config?.sklearn?.model?.file; + const skopssaveFormat = model.config?.sklearn?.model_format; + if (!skopsmodelFile) { + return [`# ⚠️ Model filename not specified in config.json`]; + } + if (skopssaveFormat === "pickle") { + return skopsPickle(model, skopsmodelFile); + } else { + return skopsFormat(model, skopsmodelFile); + } + } else { + return skopsJobLib(model); + } +}; + +export const stable_audio_tools = (model: ModelData): string[] => [ + `import torch +import torchaudio +from einops import rearrange +from stable_audio_tools import get_pretrained_model +from stable_audio_tools.inference.generation import generate_diffusion_cond + +device = "cuda" if torch.cuda.is_available() else "cpu" + +# Download model +model, model_config = get_pretrained_model("${model.id}") +sample_rate = model_config["sample_rate"] +sample_size = model_config["sample_size"] + +model = model.to(device) + +# Set up text and timing conditioning +conditioning = [{ + "prompt": "128 BPM tech house drum loop", +}] + +# Generate stereo audio +output = generate_diffusion_cond( + model, + conditioning=conditioning, + sample_size=sample_size, + device=device +) + +# Rearrange audio batch to a single sequence +output = rearrange(output, "b d n -> d (b n)") + +# Peak normalize, clip, convert to int16, and save to file +output = output.to(torch.float32).div(torch.max(torch.abs(output))).clamp(-1, 1).mul(32767).to(torch.int16).cpu() +torchaudio.save("output.wav", output, sample_rate)`, +]; + +export const fastai = (model: ModelData): string[] => [ + `from huggingface_hub import from_pretrained_fastai + +learn = from_pretrained_fastai("${model.id}")`, +]; + +export const sam2 = (model: ModelData): string[] => { + const image_predictor = `# Use SAM2 with images +import torch +from sam2.sam2_image_predictor import SAM2ImagePredictor + +predictor = SAM2ImagePredictor.from_pretrained(${model.id}) + +with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16): + predictor.set_image() + masks, _, _ = predictor.predict()`; + + const video_predictor = `# Use SAM2 with videos +import torch +from sam2.sam2_video_predictor import SAM2VideoPredictor + +predictor = SAM2VideoPredictor.from_pretrained(${model.id}) + +with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16): + state = predictor.init_state() + + # add new prompts and instantly get the output on the same frame + frame_idx, object_ids, masks = predictor.add_new_points(state, ): + + # propagate the prompts to get masklets throughout the video + for frame_idx, object_ids, masks in predictor.propagate_in_video(state): + ...`; + return [image_predictor, video_predictor]; +}; + +export const sam_3d_objects = (model: ModelData): string[] => [ + `from inference import Inference, load_image, load_single_mask +from huggingface_hub import hf_hub_download + +path = hf_hub_download("${model.id}", "pipeline.yaml") +inference = Inference(path, compile=False) + +image = load_image("path_to_image.png") +mask = load_single_mask("path_to_mask.png", index=14) + +output = inference(image, mask)`, +]; + +export const sam_3d_body = (model: ModelData): string[] => [ + `from notebook.utils import setup_sam_3d_body + +estimator = setup_sam_3d_body(${model.id}) +outputs = estimator.process_one_image(image) +rend_img = visualize_sample_together(image, outputs, estimator.faces)`, +]; + +export const sampleFactory = (model: ModelData): string[] => [ + `python -m sample_factory.huggingface.load_from_hub -r ${model.id} -d ./train_dir`, +]; + +function get_widget_examples_from_st_model(model: ModelData): string[] | undefined { + const widgetExample = model.widgetData?.[0] as WidgetExampleSentenceSimilarityInput | undefined; + if (widgetExample?.source_sentence && widgetExample?.sentences?.length) { + return [widgetExample.source_sentence, ...widgetExample.sentences]; + } +} + +export const sentenceTransformers = (model: ModelData): string[] => { + const remote_code_snippet = model.tags.includes(TAG_CUSTOM_CODE) ? ", trust_remote_code=True" : ""; + + if (model.tags.includes("PyLate")) { + return [ + `from pylate import models + +queries = [ + "Which planet is known as the Red Planet?", + "What is the largest planet in our solar system?", +] + +documents = [ + ["Mars is the Red Planet.", "Venus is Earth's twin."], + ["Jupiter is the largest planet.", "Saturn has rings."], +] + +model = models.ColBERT(model_name_or_path="${model.id}") + +queries_emb = model.encode(queries, is_query=True) +docs_emb = model.encode(documents, is_query=False)`, + ]; + } + + if (model.tags.includes("cross-encoder") || model.pipeline_tag == "text-ranking") { + return [ + `from sentence_transformers import CrossEncoder + +model = CrossEncoder("${model.id}"${remote_code_snippet}) + +query = "Which planet is known as the Red Planet?" +passages = [ + "Venus is often called Earth's twin because of its similar size and proximity.", + "Mars, known for its reddish appearance, is often referred to as the Red Planet.", + "Jupiter, the largest planet in our solar system, has a prominent red spot.", + "Saturn, famous for its rings, is sometimes mistaken for the Red Planet." +] + +scores = model.predict([(query, passage) for passage in passages]) +print(scores)`, + ]; + } + + const exampleSentences = get_widget_examples_from_st_model(model) ?? [ + "The weather is lovely today.", + "It's so sunny outside!", + "He drove to the stadium.", + ]; + + return [ + `from sentence_transformers import SentenceTransformer + +model = SentenceTransformer("${model.id}"${remote_code_snippet}) + +sentences = ${JSON.stringify(exampleSentences, null, 4)} +embeddings = model.encode(sentences) + +similarities = model.similarity(embeddings, embeddings) +print(similarities.shape) +# [${exampleSentences.length}, ${exampleSentences.length}]`, + ]; +}; + +export const setfit = (model: ModelData): string[] => [ + `from setfit import SetFitModel + +model = SetFitModel.from_pretrained("${model.id}")`, +]; + +export const spacy = (model: ModelData): string[] => [ + `!pip install https://huggingface.co/${model.id}/resolve/main/${nameWithoutNamespace(model.id)}-any-py3-none-any.whl + +# Using spacy.load(). +import spacy +nlp = spacy.load("${nameWithoutNamespace(model.id)}") + +# Importing as module. +import ${nameWithoutNamespace(model.id)} +nlp = ${nameWithoutNamespace(model.id)}.load()`, +]; + +export const span_marker = (model: ModelData): string[] => [ + `from span_marker import SpanMarkerModel + +model = SpanMarkerModel.from_pretrained("${model.id}")`, +]; + +export const stanza = (model: ModelData): string[] => [ + `import stanza + +stanza.download("${nameWithoutNamespace(model.id).replace("stanza-", "")}") +nlp = stanza.Pipeline("${nameWithoutNamespace(model.id).replace("stanza-", "")}")`, +]; + +const speechBrainMethod = (speechbrainInterface: string) => { + switch (speechbrainInterface) { + case "EncoderClassifier": + return "classify_file"; + case "EncoderDecoderASR": + case "EncoderASR": + return "transcribe_file"; + case "SpectralMaskEnhancement": + return "enhance_file"; + case "SepformerSeparation": + return "separate_file"; + default: + return undefined; + } +}; + +export const speechbrain = (model: ModelData): string[] => { + const speechbrainInterface = model.config?.speechbrain?.speechbrain_interface; + if (speechbrainInterface === undefined) { + return [`# interface not specified in config.json`]; + } + + const speechbrainMethod = speechBrainMethod(speechbrainInterface); + if (speechbrainMethod === undefined) { + return [`# interface in config.json invalid`]; + } + + return [ + `from speechbrain.pretrained import ${speechbrainInterface} +model = ${speechbrainInterface}.from_hparams( + "${model.id}" +) +model.${speechbrainMethod}("file.wav")`, + ]; +}; + +export const terratorch = (model: ModelData): string[] => [ + `from terratorch.registry import BACKBONE_REGISTRY + +model = BACKBONE_REGISTRY.build("${model.id}")`, +]; + +const hasChatTemplate = (model: ModelData): boolean => + model.config?.tokenizer_config?.chat_template !== undefined || + model.config?.processor_config?.chat_template !== undefined || + model.config?.chat_template_jinja !== undefined; + +export const transformers = (model: ModelData): string[] => { + const info = model.transformersInfo; + if (!info) { + return [`# ⚠️ Type of model unknown`]; + } + const remote_code_snippet = model.tags.includes(TAG_CUSTOM_CODE) ? ", trust_remote_code=True" : ""; + + const autoSnippet = []; + if (info.processor) { + const processorVarName = + info.processor === "AutoTokenizer" + ? "tokenizer" + : info.processor === "AutoFeatureExtractor" + ? "extractor" + : "processor"; + autoSnippet.push( + "# Load model directly", + `from transformers import ${info.processor}, ${info.auto_model}`, + "", + `${processorVarName} = ${info.processor}.from_pretrained("${model.id}"` + remote_code_snippet + ")", + `model = ${info.auto_model}.from_pretrained("${model.id}"` + remote_code_snippet + ")", + ); + if (model.tags.includes("conversational") && hasChatTemplate(model)) { + if (model.tags.includes("image-text-to-text")) { + autoSnippet.push( + "messages = [", + [ + " {", + ' "role": "user",', + ' "content": [', + ' {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"},', + ' {"type": "text", "text": "What animal is on the candy?"}', + " ]", + " },", + ].join("\n"), + "]", + ); + } else { + autoSnippet.push("messages = [", ' {"role": "user", "content": "Who are you?"},', "]"); + } + autoSnippet.push( + `inputs = ${processorVarName}.apply_chat_template(`, + " messages,", + " add_generation_prompt=True,", + " tokenize=True,", + " return_dict=True,", + ' return_tensors="pt",', + ").to(model.device)", + "", + "outputs = model.generate(**inputs, max_new_tokens=40)", + `print(${processorVarName}.decode(outputs[0][inputs["input_ids"].shape[-1]:]))`, + ); + } + } else { + autoSnippet.push( + "# Load model directly", + `from transformers import ${info.auto_model}`, + `model = ${info.auto_model}.from_pretrained("${model.id}"` + remote_code_snippet + ', dtype="auto")', + ); + } + + if (model.pipeline_tag && LIBRARY_TASK_MAPPING.transformers?.includes(model.pipeline_tag)) { + const pipelineSnippet = ["# Use a pipeline as a high-level helper"]; + if (REMOVED_IN_V5_TRANSFORMERS_PIPELINES.includes(model.pipeline_tag)) { + pipelineSnippet.push( + `# Warning: Pipeline type "${model.pipeline_tag}" is no longer supported in transformers v5.`, + `# You must load the model directly (see below) or downgrade to v4.x with:`, + `# 'pip install "transformers<5.0.0'`, + ); + } + + pipelineSnippet.push( + "from transformers import pipeline", + "", + `pipe = pipeline("${model.pipeline_tag}", model="${model.id}"` + remote_code_snippet + ")", + ); + + if (model.tags.includes("conversational")) { + if (model.tags.includes("image-text-to-text")) { + pipelineSnippet.push( + "messages = [", + [ + " {", + ' "role": "user",', + ' "content": [', + ' {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"},', + ' {"type": "text", "text": "What animal is on the candy?"}', + " ]", + " },", + ].join("\n"), + "]", + ); + pipelineSnippet.push("pipe(text=messages)"); + } else { + pipelineSnippet.push("messages = [", ' {"role": "user", "content": "Who are you?"},', "]"); + pipelineSnippet.push("pipe(messages)"); + } + } else if (model.pipeline_tag === "zero-shot-image-classification") { + pipelineSnippet.push( + "pipe(", + ' "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/parrots.png",', + ' candidate_labels=["animals", "humans", "landscape"],', + ")", + ); + } else if (model.pipeline_tag === "image-classification") { + pipelineSnippet.push( + 'pipe("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/parrots.png")', + ); + } + + return [pipelineSnippet.join("\n"), autoSnippet.join("\n")]; + } + return [autoSnippet.join("\n")]; +}; + +export const transformersJS = (model: ModelData): string[] => { + if (!model.pipeline_tag) { + return [`// ⚠️ Unknown pipeline tag`]; + } + + const libName = "@huggingface/transformers"; + + return [ + `// npm i ${libName} +import { pipeline } from '${libName}'; + +// Allocate pipeline +const pipe = await pipeline('${model.pipeline_tag}', '${model.id}');`, + ]; +}; + +const peftTask = (peftTaskType?: string) => { + switch (peftTaskType) { + case "CAUSAL_LM": + return "CausalLM"; + case "SEQ_2_SEQ_LM": + return "Seq2SeqLM"; + case "TOKEN_CLS": + return "TokenClassification"; + case "SEQ_CLS": + return "SequenceClassification"; + default: + return undefined; + } +}; + +export const peft = (model: ModelData): string[] => { + const { base_model_name_or_path: peftBaseModel, task_type: peftTaskType } = model.config?.peft ?? {}; + const pefttask = peftTask(peftTaskType); + if (!pefttask) { + return [`Task type is invalid.`]; + } + if (!peftBaseModel) { + return [`Base model is not found.`]; + } + + return [ + `from peft import PeftModel +from transformers import AutoModelFor${pefttask} + +base_model = AutoModelFor${pefttask}.from_pretrained("${peftBaseModel}") +model = PeftModel.from_pretrained(base_model, "${model.id}")`, + ]; +}; + +export const fasttext = (model: ModelData): string[] => [ + `from huggingface_hub import hf_hub_download +import fasttext + +model = fasttext.load_model(hf_hub_download("${model.id}", "model.bin"))`, +]; + +export const stableBaselines3 = (model: ModelData): string[] => [ + `from huggingface_sb3 import load_from_hub +checkpoint = load_from_hub( + repo_id="${model.id}", + filename="{MODEL FILENAME}.zip", +)`, +]; + +const nemoDomainResolver = (domain: string, model: ModelData): string[] | undefined => { + switch (domain) { + case "ASR": + return [ + `import nemo.collections.asr as nemo_asr +asr_model = nemo_asr.models.ASRModel.from_pretrained("${model.id}") + +transcriptions = asr_model.transcribe(["file.wav"])`, + ]; + default: + return undefined; + } +}; + +export const mlAgents = (model: ModelData): string[] => [ + `mlagents-load-from-hf --repo-id="${model.id}" --local-dir="./download: string[]s"`, +]; + +export const sentis = (/* model: ModelData */): string[] => [ + `string modelName = "[Your model name here].sentis"; +Model model = ModelLoader.Load(Application.streamingAssetsPath + "/" + modelName); +IWorker engine = WorkerFactory.CreateWorker(BackendType.GPUCompute, model); +// Please see provided C# file for more details +`, +]; + +export const sana = (model: ModelData): string[] => [ + ` +# Load the model and infer image from text +import torch +from app.sana_pipeline import SanaPipeline +from torchvision.utils import save_image + +sana = SanaPipeline("configs/sana_config/1024ms/Sana_1600M_img1024.yaml") +sana.from_pretrained("hf://${model.id}") + +image = sana( + prompt='a cyberpunk cat with a neon sign that says "Sana"', + height=1024, + width=1024, + guidance_scale=5.0, + pag_guidance_scale=2.0, + num_inference_steps=18, +) `, +]; + +export const vibevoice = (model: ModelData): string[] => [ + `import torch, soundfile as sf, librosa, numpy as np +from vibevoice.processor.vibevoice_processor import VibeVoiceProcessor +from vibevoice.modular.modeling_vibevoice_inference import VibeVoiceForConditionalGenerationInference + +# Load voice sample (should be 24kHz mono) +voice, sr = sf.read("path/to/voice_sample.wav") +if voice.ndim > 1: voice = voice.mean(axis=1) +if sr != 24000: voice = librosa.resample(voice, sr, 24000) + +processor = VibeVoiceProcessor.from_pretrained("${model.id}") +model = VibeVoiceForConditionalGenerationInference.from_pretrained( + "${model.id}", torch_dtype=torch.bfloat16 +).to("cuda").eval() +model.set_ddpm_inference_steps(5) + +inputs = processor(text=["Speaker 0: Hello!\\nSpeaker 1: Hi there!"], + voice_samples=[[voice]], return_tensors="pt") +audio = model.generate(**inputs, cfg_scale=1.3, + tokenizer=processor.tokenizer).speech_outputs[0] +sf.write("output.wav", audio.cpu().numpy().squeeze(), 24000)`, +]; + +export const videoprism = (model: ModelData): string[] => [ + `# Install from https://github.com/google-deepmind/videoprism +import jax +from videoprism import models as vp + +flax_model = vp.get_model("${model.id}") +loaded_state = vp.load_pretrained_weights("${model.id}") + +@jax.jit +def forward_fn(inputs, train=False): + return flax_model.apply(loaded_state, inputs, train=train)`, +]; + +export const vfimamba = (model: ModelData): string[] => [ + `from Trainer_finetune import Model + +model = Model.from_pretrained("${model.id}")`, +]; + +export const lvface = (model: ModelData): string[] => [ + `from huggingface_hub import hf_hub_download + from inference_onnx import LVFaceONNXInferencer + +model_path = hf_hub_download("${model.id}", "LVFace-L_Glint360K/LVFace-L_Glint360K.onnx") +inferencer = LVFaceONNXInferencer(model_path, use_gpu=True, timeout=300) +img_path = 'path/to/image1.jpg' +embedding = inferencer.infer_from_image(img_path)`, +]; + +export const voicecraft = (model: ModelData): string[] => [ + `from voicecraft import VoiceCraft + +model = VoiceCraft.from_pretrained("${model.id}")`, +]; + +export const voxcpm = (model: ModelData): string[] => [ + `import soundfile as sf +from voxcpm import VoxCPM + +model = VoxCPM.from_pretrained("${model.id}") + +wav = model.generate( + text="VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly expressive speech.", + prompt_wav_path=None, # optional: path to a prompt speech for voice cloning + prompt_text=None, # optional: reference text + cfg_value=2.0, # LM guidance on LocDiT, higher for better adherence to the prompt, but maybe worse + inference_timesteps=10, # LocDiT inference timesteps, higher for better result, lower for fast speed + normalize=True, # enable external TN tool + denoise=True, # enable external Denoise tool + retry_badcase=True, # enable retrying mode for some bad cases (unstoppable) + retry_badcase_max_times=3, # maximum retrying times + retry_badcase_ratio_threshold=6.0, # maximum length restriction for bad case detection (simple but effective), it could be adjusted for slow pace speech +) + +sf.write("output.wav", wav, 16000) +print("saved: output.wav")`, +]; + +export const vui = (): string[] => [ + `# !pip install git+https://github.com/fluxions-ai/vui + +import torchaudio + +from vui.inference import render +from vui.model import Vui, + +model = Vui.from_pretrained().cuda() +waveform = render( + model, + "Hey, here is some random stuff, usually something quite long as the shorter the text the less likely the model can cope!", +) +print(waveform.shape) +torchaudio.save("out.opus", waveform[0], 22050) +`, +]; + +export const chattts = (): string[] => [ + `import ChatTTS +import torchaudio + +chat = ChatTTS.Chat() +chat.load_models(compile=False) # Set to True for better performance + +texts = ["PUT YOUR TEXT HERE",] + +wavs = chat.infer(texts, ) + +torchaudio.save("output1.wav", torch.from_numpy(wavs[0]), 24000)`, +]; + +export const ultralytics = (model: ModelData): string[] => { + // ultralytics models must have a version tag (e.g. `yolov8`) + const versionTag = model.tags.find((tag) => tag.match(/^yolov\d+$/)); + + const className = versionTag ? `YOLOv${versionTag.slice(4)}` : "YOLOvXX"; + const prefix = versionTag + ? "" + : `# Couldn't find a valid YOLO version tag.\n# Replace XX with the correct version.\n`; + + return [ + prefix + + `from ultralytics import ${className} + +model = ${className}.from_pretrained("${model.id}") +source = 'http://images.cocodataset.org/val2017/000000039769.jpg' +model.predict(source=source, save=True)`, + ]; +}; + +export const birefnet = (model: ModelData): string[] => [ + `# Option 1: use with transformers + +from transformers import AutoModelForImageSegmentation +birefnet = AutoModelForImageSegmentation.from_pretrained("${model.id}", trust_remote_code=True) +`, + `# Option 2: use with BiRefNet + +# Install from https://github.com/ZhengPeng7/BiRefNet + +from models.birefnet import BiRefNet +model = BiRefNet.from_pretrained("${model.id}")`, +]; + +export const supertonic = (): string[] => [ + `from supertonic import TTS + +tts = TTS(auto_download=True) + +style = tts.get_voice_style(voice_name="M1") + +text = "The train delay was announced at 4:45 PM on Wed, Apr 3, 2024 due to track maintenance." +wav, duration = tts.synthesize(text, voice_style=style) + +tts.save_audio(wav, "output.wav")`, +]; + +export const swarmformer = (model: ModelData): string[] => [ + `from swarmformer import SwarmFormerModel + +model = SwarmFormerModel.from_pretrained("${model.id}") +`, +]; + +export const univa = (model: ModelData): string[] => [ + `# Follow installation instructions at https://github.com/PKU-YuanGroup/UniWorld-V1 + +from univa.models.qwen2p5vl.modeling_univa_qwen2p5vl import UnivaQwen2p5VLForConditionalGeneration + model = UnivaQwen2p5VLForConditionalGeneration.from_pretrained( + "${model.id}", + torch_dtype=torch.bfloat16, + attn_implementation="flash_attention_2", + ).to("cuda") + processor = AutoProcessor.from_pretrained("${model.id}") +`, +]; + +const mlx_unknown = (model: ModelData): string[] => [ + `# Download the model from the Hub +pip install huggingface_hub[hf_xet] + +huggingface-cli download --local-dir ${nameWithoutNamespace(model.id)} ${model.id}`, +]; + +const mlxlm = (model: ModelData): string[] => [ + `# Make sure mlx-lm is installed +# pip install --upgrade mlx-lm +# if on a CUDA device, also pip install mlx[cuda] + +# Generate text with mlx-lm +from mlx_lm import load, generate + +model, tokenizer = load("${model.id}") + +prompt = "Once upon a time in" +text = generate(model, tokenizer, prompt=prompt, verbose=True)`, +]; + +const mlxchat = (model: ModelData): string[] => [ + `# Make sure mlx-lm is installed +# pip install --upgrade mlx-lm + +# Generate text with mlx-lm +from mlx_lm import load, generate + +model, tokenizer = load("${model.id}") + +prompt = "Write a story about Einstein" +messages = [{"role": "user", "content": prompt}] +prompt = tokenizer.apply_chat_template( + messages, add_generation_prompt=True +) + +text = generate(model, tokenizer, prompt=prompt, verbose=True)`, +]; + +const mlxvlm = (model: ModelData): string[] => [ + `# Make sure mlx-vlm is installed +# pip install --upgrade mlx-vlm + +from mlx_vlm import load, generate +from mlx_vlm.prompt_utils import apply_chat_template +from mlx_vlm.utils import load_config + +# Load the model +model, processor = load("${model.id}") +config = load_config("${model.id}") + +# Prepare input +image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] +prompt = "Describe this image." + +# Apply chat template +formatted_prompt = apply_chat_template( + processor, config, prompt, num_images=1 +) + +# Generate output +output = generate(model, processor, formatted_prompt, image) +print(output)`, +]; + +export const mlxim = (model: ModelData): string[] => [ + `from mlxim.model import create_model + +model = create_model(${model.id})`, +]; + +export const mlx = (model: ModelData): string[] => { + if (model.pipeline_tag === "image-text-to-text") { + return mlxvlm(model); + } + if (model.pipeline_tag === "text-generation") { + if (model.tags.includes("conversational")) { + return mlxchat(model); + } else { + return mlxlm(model); + } + } + return mlx_unknown(model); +}; + +export const model2vec = (model: ModelData): string[] => [ + `from model2vec import StaticModel + +model = StaticModel.from_pretrained("${model.id}")`, +]; + +export const pruna = (model: ModelData): string[] => { + let snippets: string[]; + + if (model.tags.includes("diffusers")) { + snippets = pruna_diffusers(model); + } else if (model.tags.includes("transformers")) { + snippets = pruna_transformers(model); + } else { + snippets = pruna_default(model); + } + + const ensurePrunaModelImport = (snippet: string): string => { + if (!/^from pruna import PrunaModel/m.test(snippet)) { + return `from pruna import PrunaModel\n${snippet}`; + } + return snippet; + }; + snippets = snippets.map(ensurePrunaModelImport); + + if (model.tags.includes("pruna_pro-ai")) { + return snippets.map((snippet) => + snippet.replace(/\bpruna\b/g, "pruna_pro").replace(/\bPrunaModel\b/g, "PrunaProModel"), + ); + } + + return snippets; +}; + +const pruna_diffusers = (model: ModelData): string[] => { + const diffusersSnippets = diffusers(model); + + return diffusersSnippets.map((snippet) => + snippet + // Replace pipeline classes with PrunaModel + .replace(/\b\w*Pipeline\w*\b/g, "PrunaModel") + // Clean up diffusers imports containing PrunaModel + .replace(/from diffusers import ([^,\n]*PrunaModel[^,\n]*)/g, "") + .replace(/from diffusers import ([^,\n]+),?\s*([^,\n]*PrunaModel[^,\n]*)/g, "from diffusers import $1") + .replace(/from diffusers import\s*(\n|$)/g, "") + // Fix PrunaModel imports + .replace(/from diffusers import PrunaModel/g, "from pruna import PrunaModel") + .replace(/from diffusers import ([^,\n]+), PrunaModel/g, "from diffusers import $1") + .replace(/from diffusers import PrunaModel, ([^,\n]+)/g, "from diffusers import $1") + // Clean up whitespace + .replace(/\n\n+/g, "\n") + .trim(), + ); +}; + +const pruna_transformers = (model: ModelData): string[] => { + const info = model.transformersInfo; + const transformersSnippets = transformers(model); + + // Replace pipeline with PrunaModel + let processedSnippets = transformersSnippets.map((snippet) => + snippet + .replace(/from transformers import pipeline/g, "from pruna import PrunaModel") + .replace(/pipeline\([^)]*\)/g, `PrunaModel.from_pretrained("${model.id}")`), + ); + + // Additional cleanup if auto_model info is available + if (info?.auto_model) { + processedSnippets = processedSnippets.map((snippet) => + snippet + .replace(new RegExp(`from transformers import ${info.auto_model}\n?`, "g"), "") + .replace(new RegExp(`${info.auto_model}.from_pretrained`, "g"), "PrunaModel.from_pretrained") + .replace(new RegExp(`^.*from.*import.*(, *${info.auto_model})+.*$`, "gm"), (line) => + line.replace(new RegExp(`, *${info.auto_model}`, "g"), ""), + ), + ); + } + + return processedSnippets; +}; + +const pruna_default = (model: ModelData): string[] => [ + `from pruna import PrunaModel +model = PrunaModel.from_pretrained("${model.id}") +`, +]; + +export const nemo = (model: ModelData): string[] => { + let command: string[] | undefined = undefined; + // Resolve the tag to a nemo domain/sub-domain + if (model.tags.includes("automatic-speech-recognition")) { + command = nemoDomainResolver("ASR", model); + } + + return command ?? [`# tag did not correspond to a valid NeMo domain.`]; +}; + +export const outetts = (model: ModelData): string[] => { + // Don’t show this block on GGUF / ONNX mirrors + const t = model.tags ?? []; + if (t.includes("gguf") || t.includes("onnx")) { + return []; + } + + // v1.0 HF → minimal runnable snippet + return [ + ` + import outetts + + enum = outetts.Models("${model.id}".split("/", 1)[1]) # VERSION_1_0_SIZE_1B + cfg = outetts.ModelConfig.auto_config(enum, outetts.Backend.HF) + tts = outetts.Interface(cfg) + + speaker = tts.load_default_speaker("EN-FEMALE-1-NEUTRAL") + tts.generate( + outetts.GenerationConfig( + text="Hello there, how are you doing?", + speaker=speaker, + ) + ).save("output.wav") + `, + ]; +}; + +export const pxia = (model: ModelData): string[] => [ + `from pxia import AutoModel + +model = AutoModel.from_pretrained("${model.id}")`, +]; + +export const pythae = (model: ModelData): string[] => [ + `from pythae.models import AutoModel + +model = AutoModel.load_from_hf_hub("${model.id}")`, +]; + +export const qwen3_tts = (model: ModelData): string[] => [ + `# pip install qwen-tts +import torch +import soundfile as sf +from qwen_tts import Qwen3TTSModel + +model = Qwen3TTSModel.from_pretrained( + "${model.id}", + device_map="cuda:0", + dtype=torch.bfloat16, + attn_implementation="flash_attention_2", +) + +wavs, sr = model.generate_custom_voice( + text="Your text here.", + language="English", + speaker="Ryan", + instruct="Speak in a natural tone.", +) + +sf.write("output.wav", wavs[0], sr)`, +]; + +const musicgen = (model: ModelData): string[] => [ + `from audiocraft.models import MusicGen + +model = MusicGen.get_pretrained("${model.id}") + +descriptions = ['happy rock', 'energetic EDM', 'sad jazz'] +wav = model.generate(descriptions) # generates 3 samples.`, +]; + +const magnet = (model: ModelData): string[] => [ + `from audiocraft.models import MAGNeT + +model = MAGNeT.get_pretrained("${model.id}") + +descriptions = ['disco beat', 'energetic EDM', 'funky groove'] +wav = model.generate(descriptions) # generates 3 samples.`, +]; + +const audiogen = (model: ModelData): string[] => [ + `from audiocraft.models import AudioGen + +model = AudioGen.get_pretrained("${model.id}") +model.set_generation_params(duration=5) # generate 5 seconds. +descriptions = ['dog barking', 'sirene of an emergency vehicle', 'footsteps in a corridor'] +wav = model.generate(descriptions) # generates 3 samples.`, +]; +export const anemoi = (model: ModelData): string[] => [ + `from anemoi.inference.runners.default import DefaultRunner +from anemoi.inference.config.run import RunConfiguration +# Create Configuration +config = RunConfiguration(checkpoint = {"huggingface":"${model.id}"}) +# Load Runner +runner = DefaultRunner(config)`, +]; + +export const audiocraft = (model: ModelData): string[] => { + if (model.tags.includes("musicgen")) { + return musicgen(model); + } else if (model.tags.includes("audiogen")) { + return audiogen(model); + } else if (model.tags.includes("magnet")) { + return magnet(model); + } else { + return [`# Type of model unknown.`]; + } +}; + +export const whisperkit = (): string[] => [ + `# Install CLI with Homebrew on macOS device +brew install whisperkit-cli + +# View all available inference options +whisperkit-cli transcribe --help + +# Download and run inference using whisper base model +whisperkit-cli transcribe --audio-path /path/to/audio.mp3 + +# Or use your preferred model variant +whisperkit-cli transcribe --model "large-v3" --model-prefix "distil" --audio-path /path/to/audio.mp3 --verbose`, +]; + +export const threedtopia_xl = (model: ModelData): string[] => [ + `from threedtopia_xl.models import threedtopia_xl + +model = threedtopia_xl.from_pretrained("${model.id}") +model.generate(cond="path/to/image.png")`, +]; + +export const hezar = (model: ModelData): string[] => [ + `from hezar import Model + +model = Model.load("${model.id}")`, +]; + +export const zonos = (model: ModelData): string[] => [ + `# pip install git+https://github.com/Zyphra/Zonos.git +import torchaudio +from zonos.model import Zonos +from zonos.conditioning import make_cond_dict + +model = Zonos.from_pretrained("${model.id}", device="cuda") + +wav, sr = torchaudio.load("speaker.wav") # 5-10s reference clip +speaker = model.make_speaker_embedding(wav, sr) + +cond = make_cond_dict(text="Hello, world!", speaker=speaker, language="en-us") +codes = model.generate(model.prepare_conditioning(cond)) + +audio = model.autoencoder.decode(codes)[0].cpu() +torchaudio.save("sample.wav", audio, model.autoencoder.sampling_rate) +`, +]; + +export const moshi = (model: ModelData): string[] => { + // Detect backend from model name (no distinguishing tags available) + if (model.id.includes("-mlx")) { + // MLX backend (macOS Apple Silicon) + // -q flag only accepts 4 or 8, bf16 models don't use it + const quantFlag = model.id.includes("-q4") ? " -q 4" : model.id.includes("-q8") ? " -q 8" : ""; + return [ + `# pip install moshi_mlx +# Run local inference (macOS Apple Silicon) +python -m moshi_mlx.local${quantFlag} --hf-repo "${model.id}" + +# Or run with web UI +python -m moshi_mlx.local_web${quantFlag} --hf-repo "${model.id}"`, + ]; + } + + if (model.id.includes("-candle")) { + // Rust/Candle backend + return [ + `# pip install rustymimi +# Candle backend - see https://github.com/kyutai-labs/moshi +# for Rust installation instructions`, + ]; + } + + // PyTorch backend (default) + return [ + `# pip install moshi +# Run the interactive web server +python -m moshi.server --hf-repo "${model.id}" +# Then open https://localhost:8998 in your browser`, + `# pip install moshi +import torch +from moshi.models import loaders + +# Load checkpoint info from HuggingFace +checkpoint = loaders.CheckpointInfo.from_hf_repo("${model.id}") + +# Load the Mimi audio codec +mimi = checkpoint.get_mimi(device="cuda") +mimi.set_num_codebooks(8) + +# Encode audio (24kHz, mono) +wav = torch.randn(1, 1, 24000 * 10) # [batch, channels, samples] +with torch.no_grad(): + codes = mimi.encode(wav.cuda()) + decoded = mimi.decode(codes)`, + ]; +}; + +//#endregion diff --git a/node_modules/@huggingface/tasks/src/model-libraries.ts b/node_modules/@huggingface/tasks/src/model-libraries.ts new file mode 100644 index 0000000000000000000000000000000000000000..edb56af1202a3d428ed379e37b3cccaf597a6042 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/model-libraries.ts @@ -0,0 +1,1726 @@ +import * as snippets from "./model-libraries-snippets.js"; +import type { ModelData } from "./model-data.js"; +import type { ElasticSearchQuery } from "./model-libraries-downloads.js"; + +/** + * Elements configurable by a model library. + */ +export interface LibraryUiElement { + /** + * Pretty name of the library. + * displayed in tags, and on the main + * call-to-action button on the model page. + */ + prettyLabel: string; + /** + * Repo name of the library's (usually on GitHub) code repo + */ + repoName: string; + /** + * URL to library's (usually on GitHub) code repo + */ + repoUrl: string; + /** + * URL to library's docs + */ + docsUrl?: string; + /** + * Code snippet(s) displayed on model page + */ + snippets?: (model: ModelData) => string[]; + /** + * Elastic query used to count this library's model downloads + * + * By default, those files are counted: + * "config.json", "config.yaml", "hyperparams.yaml", "params.json", "meta.yaml" + */ + countDownloads?: ElasticSearchQuery; + /** + * should we display this library in hf.co/models filter + * (only for popular libraries with > 100 models) + */ + filter?: boolean; +} + +/** + * Add your new library here. + * + * This is for modeling (= architectures) libraries, not for file formats (like ONNX, etc). + * (unlike libraries, file formats live in an enum inside the internal codebase.) + * + * Doc on how to add a library to the Hub: + * + * https://huggingface.co/docs/hub/models-adding-libraries + * + * /!\ IMPORTANT + * + * The key you choose is the tag your models have in their library_name on the Hub. + */ + +export const MODEL_LIBRARIES_UI_ELEMENTS = { + acestep: { + prettyLabel: "ACE-Step", + repoName: "ACE-Step", + repoUrl: "https://github.com/ace-step/ACE-Step", + filter: false, + countDownloads: `path:"ace_step_transformer/config.json"`, + }, + "adapter-transformers": { + prettyLabel: "Adapters", + repoName: "adapters", + repoUrl: "https://github.com/Adapter-Hub/adapters", + docsUrl: "https://huggingface.co/docs/hub/adapters", + snippets: snippets.adapters, + filter: true, + countDownloads: `path:"adapter_config.json"`, + }, + allennlp: { + prettyLabel: "AllenNLP", + repoName: "AllenNLP", + repoUrl: "https://github.com/allenai/allennlp", + docsUrl: "https://huggingface.co/docs/hub/allennlp", + snippets: snippets.allennlp, + filter: true, + }, + anemoi: { + prettyLabel: "AnemoI", + repoName: "AnemoI", + repoUrl: "https://github.com/ecmwf/anemoi-inference", + docsUrl: "https://anemoi.readthedocs.io/en/latest/", + filter: false, + countDownloads: `path_extension:"ckpt"`, + snippets: snippets.anemoi, + }, + araclip: { + prettyLabel: "AraClip", + repoName: "AraClip", + repoUrl: "https://huggingface.co/Arabic-Clip/araclip", + filter: false, + snippets: snippets.araclip, + }, + "aviation-ner": { + prettyLabel: "Aviation NER", + repoName: "Aviation NER", + repoUrl: "https://github.com/Boeing/aviation_ner_sdr", + docsUrl: "https://github.com/Boeing/aviation_ner_sdr", + countDownloads: `path:"gliner_config.json"`, + filter: false, + }, + asteroid: { + prettyLabel: "Asteroid", + repoName: "Asteroid", + repoUrl: "https://github.com/asteroid-team/asteroid", + docsUrl: "https://huggingface.co/docs/hub/asteroid", + snippets: snippets.asteroid, + filter: true, + countDownloads: `path:"pytorch_model.bin"`, + }, + audiocraft: { + prettyLabel: "Audiocraft", + repoName: "audiocraft", + repoUrl: "https://github.com/facebookresearch/audiocraft", + snippets: snippets.audiocraft, + filter: false, + countDownloads: `path:"state_dict.bin"`, + }, + audioseal: { + prettyLabel: "AudioSeal", + repoName: "audioseal", + repoUrl: "https://github.com/facebookresearch/audioseal", + filter: false, + countDownloads: `path_extension:"pth"`, + snippets: snippets.audioseal, + }, + "bagel-mot": { + prettyLabel: "Bagel", + repoName: "Bagel", + repoUrl: "https://github.com/ByteDance-Seed/Bagel/", + filter: false, + countDownloads: `path:"llm_config.json"`, + }, + bboxmaskpose: { + prettyLabel: "BBoxMaskPose", + repoName: "BBoxMaskPose", + repoUrl: "https://github.com/MiraPurkrabek/BBoxMaskPose", + filter: false, + countDownloads: `path_extension:"pth"`, + }, + ben2: { + prettyLabel: "BEN2", + repoName: "BEN2", + repoUrl: "https://github.com/PramaLLC/BEN2", + snippets: snippets.ben2, + filter: false, + }, + bertopic: { + prettyLabel: "BERTopic", + repoName: "BERTopic", + repoUrl: "https://github.com/MaartenGr/BERTopic", + snippets: snippets.bertopic, + filter: true, + }, + big_vision: { + prettyLabel: "Big Vision", + repoName: "big_vision", + repoUrl: "https://github.com/google-research/big_vision", + filter: false, + countDownloads: `path_extension:"npz"`, + }, + bionemo: { + prettyLabel: "BioNeMo", + repoName: "BioNeMo", + filter: false, + repoUrl: "https://github.com/nvidia/BioNeMo", + countDownloads: `path_extension:"ckpt" OR path:"config.json"`, + }, + birder: { + prettyLabel: "Birder", + repoName: "Birder", + repoUrl: "https://gitlab.com/birder/birder", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + birefnet: { + prettyLabel: "BiRefNet", + repoName: "BiRefNet", + repoUrl: "https://github.com/ZhengPeng7/BiRefNet", + snippets: snippets.birefnet, + filter: false, + }, + bm25s: { + prettyLabel: "BM25S", + repoName: "bm25s", + repoUrl: "https://github.com/xhluca/bm25s", + snippets: snippets.bm25s, + filter: false, + countDownloads: `path:"params.index.json"`, + }, + boltzgen: { + prettyLabel: "BoltzGen", + repoName: "BoltzGen", + repoUrl: "https://github.com/HannesStark/boltzgen", + filter: false, + countDownloads: `path:"boltzgen1_diverse.ckpt"`, + }, + cancertathomev2: { + prettyLabel: "Cancer@HomeV2", + repoName: "Cancer@HomeV2", + repoUrl: "https://huggingface.co/OpenPeerAI/CancerAtHomeV2", + filter: false, + countDownloads: `path:"run.py"`, + }, + cartesia_pytorch: { + prettyLabel: "Cartesia Pytorch", + repoName: "Cartesia Pytorch", + repoUrl: "https://github.com/cartesia-ai/cartesia_pytorch", + snippets: snippets.cartesia_pytorch, + }, + cartesia_mlx: { + prettyLabel: "Cartesia MLX", + repoName: "Cartesia MLX", + repoUrl: "https://github.com/cartesia-ai/cartesia_mlx", + snippets: snippets.cartesia_mlx, + }, + champ: { + prettyLabel: "Champ", + repoName: "Champ", + repoUrl: "https://github.com/fudan-generative-vision/champ", + countDownloads: `path:"champ/motion_module.pth"`, + }, + chatterbox: { + prettyLabel: "Chatterbox", + repoName: "Chatterbox", + repoUrl: "https://github.com/resemble-ai/chatterbox", + snippets: snippets.chatterbox, + countDownloads: `path:"tokenizer.json"`, + filter: false, + }, + chaossim: { + prettyLabel: "ChaosSIM", + repoName: "ChaosSIM", + repoUrl: "https://huggingface.co/OpenPeerAI/ChaosSIM/", + countDownloads: `path:"ChaosSim.nb"`, + filter: false, + }, + chat_tts: { + prettyLabel: "ChatTTS", + repoName: "ChatTTS", + repoUrl: "https://github.com/2noise/ChatTTS.git", + snippets: snippets.chattts, + filter: false, + countDownloads: `path:"asset/GPT.pt"`, + }, + chexmix: { + prettyLabel: "CheXmix", + repoName: "CheXmix", + repoUrl: "https://github.com/StanfordMIMI/CheXmix", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + "chronos-forecasting": { + prettyLabel: "Chronos", + repoName: "Chronos", + repoUrl: "https://github.com/amazon-science/chronos-forecasting", + snippets: snippets.chronos_forecasting, + }, + clara: { + prettyLabel: "Clara", + repoName: "Clara", + filter: false, + repoUrl: "https://github.com/nvidia/clara", + countDownloads: `path_extension:"ckpt" OR path:"config.json"`, + }, + clipscope: { + prettyLabel: "clipscope", + repoName: "clipscope", + repoUrl: "https://github.com/Lewington-pitsos/clipscope", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + "cloud-agents": { + prettyLabel: "Cloud Agents", + repoName: "Cloud Agents", + repoUrl: "https://huggingface.co/OpenPeerAI/Cloud-Agents", + filter: false, + countDownloads: `path:"setup.py"`, + }, + collectorvision: { + prettyLabel: "CollectorVision", + repoName: "CollectorVision", + repoUrl: "https://github.com/HanClinto/CollectorVision", + snippets: snippets.collectorvision, + filter: false, + countDownloads: `path_extension:"onnx"`, + }, + colipri: { + prettyLabel: "COLIPRI", + repoName: "COLIPRI", + repoUrl: "https://huggingface.co/microsoft/colipri", + snippets: snippets.colipri, + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + cosyvoice: { + prettyLabel: "CosyVoice", + repoName: "CosyVoice", + repoUrl: "https://github.com/FunAudioLLM/CosyVoice", + filter: false, + countDownloads: `path_extension:"onnx" OR path_extension:"pt"`, + }, + cotracker: { + prettyLabel: "CoTracker", + repoName: "CoTracker", + repoUrl: "https://github.com/facebookresearch/co-tracker", + filter: false, + countDownloads: `path_extension:"pth"`, + }, + colpali: { + prettyLabel: "ColPali", + repoName: "ColPali", + repoUrl: "https://github.com/ManuelFay/colpali", + filter: false, + countDownloads: `path:"adapter_config.json"`, + }, + comet: { + prettyLabel: "COMET", + repoName: "COMET", + repoUrl: "https://github.com/Unbabel/COMET/", + countDownloads: `path:"hparams.yaml"`, + }, + cosmos: { + prettyLabel: "Cosmos", + repoName: "Cosmos", + repoUrl: "https://github.com/NVIDIA/Cosmos", + countDownloads: `path:"config.json" OR path_extension:"pt"`, + }, + "cxr-foundation": { + prettyLabel: "CXR Foundation", + repoName: "cxr-foundation", + repoUrl: "https://github.com/google-health/cxr-foundation", + snippets: snippets.cxr_foundation, + filter: false, + countDownloads: `path:"precomputed_embeddings/embeddings.npz" OR path:"pax-elixr-b-text/saved_model.pb"`, + }, + deepforest: { + prettyLabel: "DeepForest", + repoName: "deepforest", + docsUrl: "https://deepforest.readthedocs.io/en/latest/", + repoUrl: "https://github.com/weecology/DeepForest", + }, + "depth-anything-v2": { + prettyLabel: "DepthAnythingV2", + repoName: "Depth Anything V2", + repoUrl: "https://github.com/DepthAnything/Depth-Anything-V2", + snippets: snippets.depth_anything_v2, + filter: false, + countDownloads: `path_extension:"pth"`, + }, + "depth-pro": { + prettyLabel: "Depth Pro", + repoName: "Depth Pro", + repoUrl: "https://github.com/apple/ml-depth-pro", + countDownloads: `path_extension:"pt"`, + snippets: snippets.depth_pro, + filter: false, + }, + "derm-foundation": { + prettyLabel: "Derm Foundation", + repoName: "derm-foundation", + repoUrl: "https://github.com/google-health/derm-foundation", + snippets: snippets.derm_foundation, + filter: false, + countDownloads: `path:"scin_dataset_precomputed_embeddings.npz" OR path:"saved_model.pb"`, + }, + "describe-anything": { + prettyLabel: "Describe Anything", + repoName: "Describe Anything", + repoUrl: "https://github.com/NVlabs/describe-anything", + snippets: snippets.describe_anything, + filter: false, + }, + "dia-tts": { + prettyLabel: "Dia", + repoName: "Dia", + repoUrl: "https://github.com/nari-labs/dia", + snippets: snippets.dia, + filter: false, + }, + dia2: { + prettyLabel: "Dia2", + repoName: "Dia2", + repoUrl: "https://github.com/nari-labs/dia2", + snippets: snippets.dia2, + filter: false, + }, + "diff-interpretation-tuning": { + prettyLabel: "Diff Interpretation Tuning", + repoName: "Diff Interpretation Tuning", + repoUrl: "https://github.com/Aviously/diff-interpretation-tuning", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + diffree: { + prettyLabel: "Diffree", + repoName: "Diffree", + repoUrl: "https://github.com/OpenGVLab/Diffree", + filter: false, + countDownloads: `path:"diffree-step=000010999.ckpt"`, + }, + diffusers: { + prettyLabel: "Diffusers", + repoName: "🤗/diffusers", + repoUrl: "https://github.com/huggingface/diffusers", + docsUrl: "https://huggingface.co/docs/hub/diffusers", + snippets: snippets.diffusers, + filter: true, + /// diffusers has its own more complex "countDownloads" query + }, + diffusionkit: { + prettyLabel: "DiffusionKit", + repoName: "DiffusionKit", + repoUrl: "https://github.com/argmaxinc/DiffusionKit", + snippets: snippets.diffusionkit, + }, + "docking-at-home": { + prettyLabel: "Docking@Home", + repoName: "Docking@Home", + repoUrl: "https://huggingface.co/OpenPeerAI/DockingAtHOME", + filter: false, + countDownloads: `path:"setup.py"`, + }, + doctr: { + prettyLabel: "docTR", + repoName: "doctr", + repoUrl: "https://github.com/mindee/doctr", + }, + edsnlp: { + prettyLabel: "EDS-NLP", + repoName: "edsnlp", + repoUrl: "https://github.com/aphp/edsnlp", + docsUrl: "https://aphp.github.io/edsnlp/latest/", + filter: false, + snippets: snippets.edsnlp, + countDownloads: `path_filename:"config" AND path_extension:"cfg"`, + }, + elm: { + prettyLabel: "ELM", + repoName: "elm", + repoUrl: "https://github.com/slicex-ai/elm", + filter: false, + countDownloads: `path_filename:"slicex_elm_config" AND path_extension:"json"`, + }, + encoderfile: { + prettyLabel: "encoderfile", + repoName: "encoderfile", + repoUrl: "https://github.com/mozilla-ai/encoderfile", + filter: false, + countDownloads: `path_extension:"encoderfile"`, + }, + espnet: { + prettyLabel: "ESPnet", + repoName: "ESPnet", + repoUrl: "https://github.com/espnet/espnet", + docsUrl: "https://huggingface.co/docs/hub/espnet", + snippets: snippets.espnet, + filter: true, + }, + eupe: { + prettyLabel: "EUPE", + repoName: "EUPE", + repoUrl: "https://github.com/facebookresearch/EUPE", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + fairseq: { + prettyLabel: "Fairseq", + repoName: "fairseq", + repoUrl: "https://github.com/pytorch/fairseq", + snippets: snippets.fairseq, + filter: true, + }, + fastai: { + prettyLabel: "fastai", + repoName: "fastai", + repoUrl: "https://github.com/fastai/fastai", + docsUrl: "https://huggingface.co/docs/hub/fastai", + snippets: snippets.fastai, + filter: true, + }, + fastprint: { + prettyLabel: "Fast Print", + repoName: "Fast Print", + repoUrl: "https://huggingface.co/OpenPeerAI/FastPrint", + countDownloads: `path_extension:"cs"`, + }, + fasttext: { + prettyLabel: "fastText", + repoName: "fastText", + repoUrl: "https://fasttext.cc/", + snippets: snippets.fasttext, + filter: true, + countDownloads: `path_extension:"bin"`, + }, + fixer: { + prettyLabel: "Fixer", + repoName: "Fixer", + repoUrl: "https://github.com/nv-tlabs/Fixer", + filter: false, + countDownloads: `path:"pretrained/pretrained_fixer.pkl"`, + }, + flair: { + prettyLabel: "Flair", + repoName: "Flair", + repoUrl: "https://github.com/flairNLP/flair", + docsUrl: "https://huggingface.co/docs/hub/flair", + snippets: snippets.flair, + filter: true, + countDownloads: `path:"pytorch_model.bin"`, + }, + fme: { + prettyLabel: "Full Model Emulation", + repoName: "Full Model Emulation", + repoUrl: "https://github.com/ai2cm/ace", + docsUrl: "https://ai2-climate-emulator.readthedocs.io/en/latest/", + filter: false, + countDownloads: `path_extension:"tar"`, + }, + "gemma.cpp": { + prettyLabel: "gemma.cpp", + repoName: "gemma.cpp", + repoUrl: "https://github.com/google/gemma.cpp", + filter: false, + countDownloads: `path_extension:"sbs"`, + }, + "geometry-crafter": { + prettyLabel: "GeometryCrafter", + repoName: "GeometryCrafter", + repoUrl: "https://github.com/TencentARC/GeometryCrafter", + countDownloads: `path:"point_map_vae/diffusion_pytorch_model.safetensors"`, + }, + gliner: { + prettyLabel: "GLiNER", + repoName: "GLiNER", + repoUrl: "https://github.com/urchade/GLiNER", + snippets: snippets.gliner, + filter: false, + countDownloads: `path:"gliner_config.json"`, + }, + gliner2: { + prettyLabel: "GLiNER2", + repoName: "GLiNER2", + repoUrl: "https://github.com/fastino-ai/GLiNER2", + snippets: snippets.gliner2, + filter: false, + }, + "glm-tts": { + prettyLabel: "GLM-TTS", + repoName: "GLM-TTS", + repoUrl: "https://github.com/zai-org/GLM-TTS", + filter: false, + countDownloads: `path:"flow/flow.pt"`, + }, + "glyph-byt5": { + prettyLabel: "Glyph-ByT5", + repoName: "Glyph-ByT5", + repoUrl: "https://github.com/AIGText/Glyph-ByT5", + filter: false, + countDownloads: `path:"checkpoints/byt5_model.pt"`, + }, + "granite-library": { + prettyLabel: "Granite Library", + repoName: "mellea", + repoUrl: "https://github.com/generative-computing/mellea", + filter: false, + countDownloads: `path_filename:"adapter_config" AND path_extension:"json"`, + }, + grok: { + prettyLabel: "Grok", + repoName: "Grok", + repoUrl: "https://github.com/xai-org/grok-1", + filter: false, + countDownloads: `path:"ckpt/tensor00000_000" OR path:"ckpt-0/tensor00000_000"`, + }, + "habibi-tts": { + prettyLabel: "Habibi-TTS", + repoName: "Habibi-TTS", + repoUrl: "https://github.com/SWivid/Habibi-TTS", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + hallo: { + prettyLabel: "Hallo", + repoName: "Hallo", + repoUrl: "https://github.com/fudan-generative-vision/hallo", + countDownloads: `path:"hallo/net.pth"`, + }, + hermes: { + prettyLabel: "HERMES", + repoName: "HERMES", + repoUrl: "https://github.com/LMD0311/HERMES", + filter: false, + countDownloads: `path:"ckpt/hermes_final.pth"`, + }, + holomotion: { + prettyLabel: "HoloMotion", + repoName: "HoloMotion", + repoUrl: "https://github.com/HorizonRobotics/HoloMotion", + filter: false, + countDownloads: `path_extension:"onnx"`, + }, + hezar: { + prettyLabel: "Hezar", + repoName: "Hezar", + repoUrl: "https://github.com/hezarai/hezar", + docsUrl: "https://hezarai.github.io/hezar", + countDownloads: `path:"model_config.yaml" OR path:"embedding/embedding_config.yaml"`, + }, + htrflow: { + prettyLabel: "HTRflow", + repoName: "HTRflow", + repoUrl: "https://github.com/AI-Riksarkivet/htrflow", + docsUrl: "https://ai-riksarkivet.github.io/htrflow", + snippets: snippets.htrflow, + }, + "hunyuan-dit": { + prettyLabel: "HunyuanDiT", + repoName: "HunyuanDiT", + repoUrl: "https://github.com/Tencent/HunyuanDiT", + countDownloads: `path:"pytorch_model_ema.pt" OR path:"pytorch_model_distill.pt"`, + }, + "hunyuan3d-2": { + prettyLabel: "Hunyuan3D-2", + repoName: "Hunyuan3D-2", + repoUrl: "https://github.com/Tencent/Hunyuan3D-2", + countDownloads: `path_filename:"model_index" OR path_filename:"config"`, + }, + "hunyuanworld-voyager": { + prettyLabel: "HunyuanWorld-voyager", + repoName: "HunyuanWorld-voyager", + repoUrl: "https://github.com/Tencent-Hunyuan/HunyuanWorld-Voyager", + }, + "hy-worldplay": { + prettyLabel: "HY-WorldPlay", + repoName: "HY-WorldPlay", + repoUrl: "https://github.com/Tencent-Hunyuan/HY-WorldPlay", + filter: false, + countDownloads: `path_extension:"json"`, + }, + "hy-world-2": { + prettyLabel: "HY-World-2.0", + repoName: "HY-World-2.0", + repoUrl: "https://github.com/Tencent-Hunyuan/HY-World-2.0", + filter: false, + countDownloads: `path_extension:"json"`, + }, + "image-matching-models": { + prettyLabel: "Image Matching Models", + repoName: "Image Matching Models", + repoUrl: "https://github.com/alexstoken/image-matching-models", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + imstoucan: { + prettyLabel: "IMS Toucan", + repoName: "IMS-Toucan", + repoUrl: "https://github.com/DigitalPhonetics/IMS-Toucan", + countDownloads: `path:"embedding_gan.pt" OR path:"Vocoder.pt" OR path:"ToucanTTS.pt"`, + }, + "index-tts": { + prettyLabel: "IndexTTS", + repoName: "IndexTTS", + repoUrl: "https://github.com/index-tts/index-tts", + snippets: snippets.indextts, + filter: false, + }, + infinitetalk: { + prettyLabel: "InfiniteTalk", + repoName: "InfiniteTalk", + repoUrl: "https://github.com/MeiGen-AI/InfiniteTalk", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + "infinite-you": { + prettyLabel: "InfiniteYou", + repoName: "InfiniteYou", + repoUrl: "https://github.com/bytedance/InfiniteYou", + filter: false, + countDownloads: `path:"infu_flux_v1.0/sim_stage1/image_proj_model.bin" OR path:"infu_flux_v1.0/aes_stage2/image_proj_model.bin"`, + }, + intellifold: { + prettyLabel: "IntelliFold", + repoName: "IntelliFold", + repoUrl: "https://github.com/IntelliGen-AI/IntelliFold", + filter: false, + countDownloads: `path_extension:"pt" OR path_extension:"zst"`, + }, + "ising-decoding": { + prettyLabel: "Ising Decoding", + repoName: "Ising-Decoding", + repoUrl: "https://github.com/NVIDIA/Ising-Decoding", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + keras: { + prettyLabel: "Keras", + repoName: "Keras", + repoUrl: "https://github.com/keras-team/keras", + docsUrl: "https://huggingface.co/docs/hub/keras", + snippets: snippets.keras, + filter: true, + countDownloads: `path:"config.json" OR path_extension:"keras"`, + }, + "tf-keras": { + // Legacy "Keras 2" library (tensorflow-only) + prettyLabel: "TF-Keras", + repoName: "TF-Keras", + repoUrl: "https://github.com/keras-team/tf-keras", + docsUrl: "https://huggingface.co/docs/hub/tf-keras", + snippets: snippets.tf_keras, + countDownloads: `path:"saved_model.pb"`, + }, + "keras-hub": { + prettyLabel: "KerasHub", + repoName: "KerasHub", + repoUrl: "https://github.com/keras-team/keras-hub", + docsUrl: "https://keras.io/keras_hub/", + snippets: snippets.keras_hub, + filter: true, + }, + kernels: { + prettyLabel: "Kernels", + repoName: "Kernels", + repoUrl: "https://github.com/huggingface/kernels", + docsUrl: "https://huggingface.co/docs/kernels", + snippets: snippets.kernels, + countDownloads: `path_filename:"_ops" AND path_extension:"py"`, + }, + "kimi-audio": { + prettyLabel: "KimiAudio", + repoName: "KimiAudio", + repoUrl: "https://github.com/MoonshotAI/Kimi-Audio", + snippets: snippets.kimi_audio, + filter: false, + }, + kittentts: { + prettyLabel: "KittenTTS", + repoName: "KittenTTS", + repoUrl: "https://github.com/KittenML/KittenTTS", + snippets: snippets.kittentts, + }, + kronos: { + prettyLabel: "KRONOS", + repoName: "KRONOS", + repoUrl: "https://github.com/mahmoodlab/KRONOS", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + k2: { + prettyLabel: "K2", + repoName: "k2", + repoUrl: "https://github.com/k2-fsa/k2", + }, + "lyra-2.0": { + prettyLabel: "Lyra-2.0", + repoName: "Lyra-2.0", + repoUrl: "https://github.com/nv-tlabs/lyra", + filter: false, + countDownloads: `path:"checkpoints/image_encoder/model.pth"`, + }, + lagernvs: { + prettyLabel: "LagerNVS", + repoName: "LagerNVS", + repoUrl: "https://github.com/facebookresearch/lagernvs", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + "lightning-ir": { + prettyLabel: "Lightning IR", + repoName: "Lightning IR", + repoUrl: "https://github.com/webis-de/lightning-ir", + snippets: snippets.lightning_ir, + }, + litert: { + prettyLabel: "LiteRT", + repoName: "LiteRT", + repoUrl: "https://github.com/google-ai-edge/LiteRT", + filter: false, + countDownloads: `path_extension:"tflite"`, + }, + "litert-lm": { + prettyLabel: "LiteRT-LM", + repoName: "LiteRT-LM", + repoUrl: "https://github.com/google-ai-edge/LiteRT-LM", + snippets: snippets.litert_lm, + filter: false, + countDownloads: `path_extension:"litertlm" OR path_extension:"task"`, + }, + lerobot: { + prettyLabel: "LeRobot", + repoName: "LeRobot", + repoUrl: "https://github.com/huggingface/lerobot", + docsUrl: "https://huggingface.co/docs/lerobot", + filter: false, + snippets: snippets.lerobot, + }, + lightglue: { + prettyLabel: "LightGlue", + repoName: "LightGlue", + repoUrl: "https://github.com/cvg/LightGlue", + filter: false, + countDownloads: `path_extension:"pth" OR path:"config.json"`, + }, + liveportrait: { + prettyLabel: "LivePortrait", + repoName: "LivePortrait", + repoUrl: "https://github.com/KwaiVGI/LivePortrait", + filter: false, + countDownloads: `path:"liveportrait/landmark.onnx"`, + }, + "longcat-video-avatar-1.5": { + prettyLabel: "LongCat-Video-Avatar 1.5", + repoName: "LongCat-Video-Avatar 1.5", + repoUrl: "https://github.com/meituan-longcat/LongCat-Video", + filter: false, + }, + "llama-cpp-python": { + prettyLabel: "llama-cpp-python", + repoName: "llama-cpp-python", + repoUrl: "https://github.com/abetlen/llama-cpp-python", + snippets: snippets.llama_cpp_python, + }, + "mini-omni2": { + prettyLabel: "Mini-Omni2", + repoName: "Mini-Omni2", + repoUrl: "https://github.com/gpt-omni/mini-omni2", + countDownloads: `path:"model_config.yaml"`, + }, + mindspore: { + prettyLabel: "MindSpore", + repoName: "mindspore", + repoUrl: "https://github.com/mindspore-ai/mindspore", + }, + "magi-1": { + prettyLabel: "MAGI-1", + repoName: "MAGI-1", + repoUrl: "https://github.com/SandAI-org/MAGI-1", + countDownloads: `path:"ckpt/vae/config.json"`, + }, + "magenta-realtime": { + prettyLabel: "Magenta RT", + repoName: "Magenta RT", + repoUrl: "https://github.com/magenta/magenta-realtime", + countDownloads: `path:"checkpoints/llm_base_x4286_c1860k.tar" OR path:"checkpoints/llm_large_x3047_c1860k.tar" OR path:"checkpoints/llm_large_x3047_c1860k/checkpoint"`, + }, + "magenta-realtime-2": { + prettyLabel: "Magenta RT 2", + repoName: "Magenta RT 2", + repoUrl: "https://github.com/magenta/magenta-realtime", + countDownloads: `path:"models/mrt2_base/mrt2_base.mlxfn" OR path:"models/mrt2_small/mrt2_small.mlxfn" OR path:"checkpoints/mrt2_base.safetensors" OR path:"checkpoints/mrt2_small.safetensors"`, + }, + "mamba-ssm": { + prettyLabel: "MambaSSM", + repoName: "MambaSSM", + repoUrl: "https://github.com/state-spaces/mamba", + filter: false, + snippets: snippets.mamba_ssm, + }, + "manas-1": { + prettyLabel: "MANAS-1", + repoName: "MANAS-1", + repoUrl: "https://github.com/NeurodxAI/manas-1", + countDownloads: `path_extension:"pt"`, + }, + "mars5-tts": { + prettyLabel: "MARS5-TTS", + repoName: "MARS5-TTS", + repoUrl: "https://github.com/Camb-ai/MARS5-TTS", + filter: false, + countDownloads: `path:"mars5_ar.safetensors"`, + snippets: snippets.mars5_tts, + }, + matanyone: { + prettyLabel: "MatAnyone", + repoName: "MatAnyone", + repoUrl: "https://github.com/pq-yang/MatAnyone", + snippets: snippets.matanyone, + filter: false, + }, + "mesh-anything": { + prettyLabel: "MeshAnything", + repoName: "MeshAnything", + repoUrl: "https://github.com/buaacyw/MeshAnything", + filter: false, + countDownloads: `path:"MeshAnything_350m.pth"`, + snippets: snippets.mesh_anything, + }, + merlin: { + prettyLabel: "Merlin", + repoName: "Merlin", + repoUrl: "https://github.com/StanfordMIMI/Merlin", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + medvae: { + prettyLabel: "MedVAE", + repoName: "MedVAE", + repoUrl: "https://github.com/StanfordMIMI/MedVAE", + filter: false, + countDownloads: `path_extension:"ckpt"`, + }, + mitie: { + prettyLabel: "MITIE", + repoName: "MITIE", + repoUrl: "https://github.com/mit-nlp/MITIE", + countDownloads: `path_filename:"total_word_feature_extractor"`, + }, + "ml-agents": { + prettyLabel: "ml-agents", + repoName: "ml-agents", + repoUrl: "https://github.com/Unity-Technologies/ml-agents", + docsUrl: "https://huggingface.co/docs/hub/ml-agents", + snippets: snippets.mlAgents, + filter: true, + countDownloads: `path_extension:"onnx"`, + }, + "ml-sharp": { + prettyLabel: "Sharp", + repoName: "Sharp", + repoUrl: "https://github.com/apple/ml-sharp", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + mlx: { + prettyLabel: "MLX", + repoName: "MLX", + repoUrl: "https://github.com/ml-explore/mlx-examples/tree/main", + snippets: snippets.mlx, + filter: true, + }, + "mlx-image": { + prettyLabel: "mlx-image", + repoName: "mlx-image", + repoUrl: "https://github.com/riccardomusmeci/mlx-image", + docsUrl: "https://huggingface.co/docs/hub/mlx-image", + snippets: snippets.mlxim, + filter: false, + countDownloads: `path:"model.safetensors"`, + }, + "mlc-llm": { + prettyLabel: "MLC-LLM", + repoName: "MLC-LLM", + repoUrl: "https://github.com/mlc-ai/mlc-llm", + docsUrl: "https://llm.mlc.ai/docs/", + filter: false, + countDownloads: `path:"mlc-chat-config.json"`, + }, + model2vec: { + prettyLabel: "Model2Vec", + repoName: "model2vec", + repoUrl: "https://github.com/MinishLab/model2vec", + snippets: snippets.model2vec, + filter: false, + }, + moshi: { + prettyLabel: "Moshi", + repoName: "Moshi", + repoUrl: "https://github.com/kyutai-labs/moshi", + snippets: snippets.moshi, + filter: false, + countDownloads: `path:"tokenizer-e351c8d8-checkpoint125.safetensors"`, + }, + mtvcraft: { + prettyLabel: "MTVCraft", + repoName: "MTVCraft", + repoUrl: "https://github.com/baaivision/MTVCraft", + filter: false, + countDownloads: `path:"vae/3d-vae.pt"`, + }, + multimolecule: { + prettyLabel: "MultiMolecule", + repoName: "MultiMolecule", + repoUrl: "https://github.com/MultiMolecule/multimolecule", + docsUrl: "https://multimolecule.danling.org", + snippets: snippets.multimolecule, + filter: false, + }, + nemo: { + prettyLabel: "NeMo", + repoName: "NeMo", + repoUrl: "https://github.com/NVIDIA/NeMo", + snippets: snippets.nemo, + filter: true, + countDownloads: `path_extension:"nemo" OR path:"model_config.yaml" OR path_extension:"json"`, + }, + "nv-medtech": { + prettyLabel: "NV-MedTech", + repoName: "NV-MedTech", + filter: false, + repoUrl: "https://github.com/nvidia-medtech", + countDownloads: `path_extension:"pt" OR path_extension:"safetensors" OR path:"config.json"`, + }, + "open-oasis": { + prettyLabel: "open-oasis", + repoName: "open-oasis", + repoUrl: "https://github.com/etched-ai/open-oasis", + countDownloads: `path:"oasis500m.safetensors"`, + }, + open_clip: { + prettyLabel: "OpenCLIP", + repoName: "OpenCLIP", + repoUrl: "https://github.com/mlfoundations/open_clip", + snippets: snippets.open_clip, + filter: true, + countDownloads: `path:"open_clip_model.safetensors" + OR path:"model.safetensors" + OR path:"open_clip_pytorch_model.bin" + OR path:"pytorch_model.bin"`, + }, + openpeerllm: { + prettyLabel: "OpenPeerLLM", + repoName: "OpenPeerLLM", + repoUrl: "https://huggingface.co/openpeerai/openpeerllm", + docsUrl: "https://huggingface.co/OpenPeerAI/OpenPeerLLM/blob/main/README.md", + countDownloads: `path:".meta-huggingface.json"`, + filter: false, + }, + "open-sora": { + prettyLabel: "Open-Sora", + repoName: "Open-Sora", + repoUrl: "https://github.com/hpcaitech/Open-Sora", + filter: false, + countDownloads: `path:"Open_Sora_v2.safetensors"`, + }, + outetts: { + prettyLabel: "OuteTTS", + repoName: "OuteTTS", + repoUrl: "https://github.com/edwko/OuteTTS", + snippets: snippets.outetts, + filter: false, + }, + paddlenlp: { + prettyLabel: "paddlenlp", + repoName: "PaddleNLP", + repoUrl: "https://github.com/PaddlePaddle/PaddleNLP", + docsUrl: "https://huggingface.co/docs/hub/paddlenlp", + snippets: snippets.paddlenlp, + filter: true, + countDownloads: `path:"model_config.json"`, + }, + PaddleOCR: { + prettyLabel: "PaddleOCR", + repoName: "PaddleOCR", + repoUrl: "https://github.com/PaddlePaddle/PaddleOCR", + docsUrl: "https://www.paddleocr.ai/", + snippets: snippets.paddleocr, + filter: true, + countDownloads: `path_extension:"safetensors" OR path:"inference.pdiparams" OR path:"inference.onnx"`, + }, + peft: { + prettyLabel: "PEFT", + repoName: "PEFT", + repoUrl: "https://github.com/huggingface/peft", + snippets: snippets.peft, + filter: true, + countDownloads: `path:"adapter_config.json"`, + }, + "perception-encoder": { + prettyLabel: "PerceptionEncoder", + repoName: "PerceptionModels", + repoUrl: "https://github.com/facebookresearch/perception_models", + filter: false, + snippets: snippets.perception_encoder, + countDownloads: `path_extension:"pt"`, + }, + "phantom-wan": { + prettyLabel: "Phantom", + repoName: "Phantom", + repoUrl: "https://github.com/Phantom-video/Phantom", + snippets: snippets.phantom_wan, + filter: false, + countDownloads: `path_extension:"pth"`, + }, + "pocket-tts": { + prettyLabel: "Pocket-TTS", + repoName: "PocketTTS", + repoUrl: "https://github.com/kyutai-labs/pocket-tts", + snippets: snippets.pocket_tts, + filter: false, + countDownloads: `path:"tts_b6369a24.safetensors"`, + }, + "pruna-ai": { + prettyLabel: "Pruna AI", + repoName: "Pruna AI", + repoUrl: "https://github.com/PrunaAI/pruna", + snippets: snippets.pruna, + docsUrl: "https://docs.pruna.ai", + }, + pxia: { + prettyLabel: "pxia", + repoName: "pxia", + repoUrl: "https://github.com/not-lain/pxia", + snippets: snippets.pxia, + filter: false, + }, + "pyannote-audio": { + prettyLabel: "pyannote.audio", + repoName: "pyannote-audio", + repoUrl: "https://github.com/pyannote/pyannote-audio", + snippets: snippets.pyannote_audio, + filter: true, + }, + "py-feat": { + prettyLabel: "Py-Feat", + repoName: "Py-Feat", + repoUrl: "https://github.com/cosanlab/py-feat", + docsUrl: "https://py-feat.org/", + filter: false, + }, + pythae: { + prettyLabel: "pythae", + repoName: "pythae", + repoUrl: "https://github.com/clementchadebec/benchmark_VAE", + snippets: snippets.pythae, + filter: false, + }, + quantumpeer: { + prettyLabel: "QuantumPeer", + repoName: "QuantumPeer", + repoUrl: "https://github.com/OpenPeer-AI/QuantumPeer", + filter: false, + countDownloads: `path_extension:"setup.py"`, + }, + qwen3_tts: { + prettyLabel: "Qwen3-TTS", + repoName: "Qwen3-TTS", + repoUrl: "https://github.com/QwenLM/Qwen3-TTS", + snippets: snippets.qwen3_tts, + filter: false, + }, + recurrentgemma: { + prettyLabel: "RecurrentGemma", + repoName: "recurrentgemma", + repoUrl: "https://github.com/google-deepmind/recurrentgemma", + filter: false, + countDownloads: `path:"tokenizer.model"`, + }, + relik: { + prettyLabel: "Relik", + repoName: "Relik", + repoUrl: "https://github.com/SapienzaNLP/relik", + snippets: snippets.relik, + filter: false, + }, + refiners: { + prettyLabel: "Refiners", + repoName: "Refiners", + repoUrl: "https://github.com/finegrain-ai/refiners", + docsUrl: "https://refine.rs/", + filter: false, + countDownloads: `path:"model.safetensors"`, + }, + renderformer: { + prettyLabel: "RenderFormer", + repoName: "RenderFormer", + repoUrl: "https://github.com/microsoft/renderformer", + snippets: snippets.renderformer, + filter: false, + }, + reverb: { + prettyLabel: "Reverb", + repoName: "Reverb", + repoUrl: "https://github.com/revdotcom/reverb", + filter: false, + }, + rkllm: { + prettyLabel: "RKLLM", + repoName: "RKLLM", + repoUrl: "https://github.com/airockchip/rknn-llm", + countDownloads: `path_extension:"rkllm"`, + }, + "robo-orchard-lab": { + prettyLabel: "RoboOrchardLab", + repoName: "RoboOrchardLab", + repoUrl: "https://github.com/HorizonRobotics/RoboOrchardLab", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + rwkv: { + prettyLabel: "RWKV", + repoName: "RWKV-LM", + repoUrl: "https://github.com/BlinkDL/RWKV-LM", + docsUrl: "https://rwkv.com/", + filter: false, + countDownloads: `path_extension:"pth"`, + }, + saelens: { + prettyLabel: "SAELens", + repoName: "SAELens", + repoUrl: "https://github.com/jbloomAus/SAELens", + snippets: snippets.saelens, + filter: false, + }, + "scail-2": { + prettyLabel: "SCAIL-2", + repoName: "SCAIL-2", + repoUrl: "https://github.com/zai-org/SCAIL-2", + filter: false, + countDownloads: `path:"model/1/fsdp2_rank_0000_checkpoint.pt"`, + }, + sam2: { + prettyLabel: "sam2", + repoName: "sam2", + repoUrl: "https://github.com/facebookresearch/segment-anything-2", + filter: false, + snippets: snippets.sam2, + countDownloads: `path_extension:"pt"`, + }, + "sam-3d-body": { + prettyLabel: "SAM 3D Body", + repoName: "SAM 3D Body", + repoUrl: "https://github.com/facebookresearch/sam-3d-body", + filter: false, + snippets: snippets.sam_3d_body, + countDownloads: `path:"model_config.yaml"`, + }, + "sam-3d-objects": { + prettyLabel: "SAM 3D Objects", + repoName: "SAM 3D Objects", + repoUrl: "https://github.com/facebookresearch/sam-3d-objects", + filter: false, + snippets: snippets.sam_3d_objects, + countDownloads: `path:"checkpoints/pipeline.yaml"`, + }, + same: { + prettyLabel: "SAME", + repoName: "SAME", + repoUrl: "https://github.com/GengzeZhou/SAME", + filter: false, + countDownloads: `path:"ckpt/SAME.pt" OR path:"pretrain/Attnq_pretrained_ckpt.pt"`, + }, + "sample-factory": { + prettyLabel: "sample-factory", + repoName: "sample-factory", + repoUrl: "https://github.com/alex-petrenko/sample-factory", + docsUrl: "https://huggingface.co/docs/hub/sample-factory", + snippets: snippets.sampleFactory, + filter: true, + countDownloads: `path:"cfg.json"`, + }, + "sap-rpt-1-oss": { + prettyLabel: "sap-rpt-1-oss", + repoName: "sap-rpt-1-oss", + repoUrl: "https://github.com/SAP-samples/sap-rpt-1-oss", + countDownloads: `path_extension:"pt"`, + snippets: snippets.sap_rpt_one_oss, + }, + sapiens: { + prettyLabel: "sapiens", + repoName: "sapiens", + repoUrl: "https://github.com/facebookresearch/sapiens", + filter: false, + countDownloads: `path_extension:"pt2" OR path_extension:"pth" OR path_extension:"onnx"`, + }, + sapiens2: { + prettyLabel: "sapiens2", + repoName: "sapiens2", + repoUrl: "https://github.com/facebookresearch/sapiens2", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + seedvr: { + prettyLabel: "SeedVR", + repoName: "SeedVR", + repoUrl: "https://github.com/ByteDance-Seed/SeedVR", + filter: false, + countDownloads: `path_extension:"pth"`, + }, + "self-forcing": { + prettyLabel: "SelfForcing", + repoName: "SelfForcing", + repoUrl: "https://github.com/guandeh17/Self-Forcing", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + "sentence-transformers": { + prettyLabel: "sentence-transformers", + repoName: "sentence-transformers", + repoUrl: "https://github.com/UKPLab/sentence-transformers", + docsUrl: "https://huggingface.co/docs/hub/sentence-transformers", + snippets: snippets.sentenceTransformers, + filter: true, + }, + setfit: { + prettyLabel: "setfit", + repoName: "setfit", + repoUrl: "https://github.com/huggingface/setfit", + docsUrl: "https://huggingface.co/docs/hub/setfit", + snippets: snippets.setfit, + filter: true, + }, + sklearn: { + prettyLabel: "Scikit-learn", + repoName: "Scikit-learn", + repoUrl: "https://github.com/scikit-learn/scikit-learn", + snippets: snippets.sklearn, + filter: true, + countDownloads: `path:"sklearn_model.joblib"`, + }, + spacy: { + prettyLabel: "spaCy", + repoName: "spaCy", + repoUrl: "https://github.com/explosion/spaCy", + docsUrl: "https://huggingface.co/docs/hub/spacy", + snippets: snippets.spacy, + filter: true, + countDownloads: `path_extension:"whl"`, + }, + "span-marker": { + prettyLabel: "SpanMarker", + repoName: "SpanMarkerNER", + repoUrl: "https://github.com/tomaarsen/SpanMarkerNER", + docsUrl: "https://huggingface.co/docs/hub/span_marker", + snippets: snippets.span_marker, + filter: true, + }, + speechbrain: { + prettyLabel: "speechbrain", + repoName: "speechbrain", + repoUrl: "https://github.com/speechbrain/speechbrain", + docsUrl: "https://huggingface.co/docs/hub/speechbrain", + snippets: snippets.speechbrain, + filter: true, + countDownloads: `path:"hyperparams.yaml"`, + }, + "ssr-speech": { + prettyLabel: "SSR-Speech", + repoName: "SSR-Speech", + repoUrl: "https://github.com/WangHelin1997/SSR-Speech", + filter: false, + countDownloads: `path_extension:".pth"`, + }, + "stable-audio-3": { + prettyLabel: "Stable Audio 3", + repoName: "stable-audio-3", + repoUrl: "https://github.com/Stability-AI/stable-audio-3", + filter: false, + countDownloads: `path:"model_config.json"`, + }, + "stable-audio-tools": { + prettyLabel: "Stable Audio Tools", + repoName: "stable-audio-tools", + repoUrl: "https://github.com/Stability-AI/stable-audio-tools.git", + filter: false, + countDownloads: `path:"model.safetensors"`, + snippets: snippets.stable_audio_tools, + }, + monkeyocr: { + prettyLabel: "MonkeyOCR", + repoName: "monkeyocr", + repoUrl: "https://github.com/Yuliang-Liu/MonkeyOCR", + filter: false, + countDownloads: `path:"Recognition/config.json"`, + }, + "diffusion-single-file": { + prettyLabel: "Diffusion Single File", + repoName: "diffusion-single-file", + repoUrl: "https://github.com/comfyanonymous/ComfyUI", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + "seed-story": { + prettyLabel: "SEED-Story", + repoName: "SEED-Story", + repoUrl: "https://github.com/TencentARC/SEED-Story", + filter: false, + countDownloads: `path:"cvlm_llama2_tokenizer/tokenizer.model"`, + snippets: snippets.seed_story, + }, + skala: { + prettyLabel: "Skala", + repoName: "Skala", + repoUrl: "https://github.com/microsoft/skala", + filter: false, + countDownloads: `path_extension:"fun"`, + }, + soloaudio: { + prettyLabel: "SoloAudio", + repoName: "SoloAudio", + repoUrl: "https://github.com/WangHelin1997/SoloAudio", + filter: false, + countDownloads: `path:"soloaudio_v2.pt"`, + }, + songbloom: { + prettyLabel: "SongBloom", + repoName: "SongBloom", + repoUrl: "https://github.com/Cypress-Yang/SongBloom", + filter: false, + countDownloads: `path_extension:"pt"`, + }, + "stable-baselines3": { + prettyLabel: "stable-baselines3", + repoName: "stable-baselines3", + repoUrl: "https://github.com/huggingface/huggingface_sb3", + docsUrl: "https://huggingface.co/docs/hub/stable-baselines3", + snippets: snippets.stableBaselines3, + filter: true, + countDownloads: `path_extension:"zip"`, + }, + stanza: { + prettyLabel: "Stanza", + repoName: "stanza", + repoUrl: "https://github.com/stanfordnlp/stanza", + docsUrl: "https://huggingface.co/docs/hub/stanza", + snippets: snippets.stanza, + filter: true, + countDownloads: `path:"models/default.zip"`, + }, + supertonic: { + prettyLabel: "Supertonic", + repoName: "Supertonic", + repoUrl: "https://github.com/supertone-inc/supertonic", + snippets: snippets.supertonic, + filter: false, + }, + swarmformer: { + prettyLabel: "SwarmFormer", + repoName: "SwarmFormer", + repoUrl: "https://github.com/takara-ai/SwarmFormer", + snippets: snippets.swarmformer, + filter: false, + }, + "synthefy-migas": { + prettyLabel: "Migas", + repoName: "Migas", + repoUrl: "https://github.com/Synthefy/synthefy-migas", + filter: false, + countDownloads: `path:"model.pt"`, + }, + "f5-tts": { + prettyLabel: "F5-TTS", + repoName: "F5-TTS", + repoUrl: "https://github.com/SWivid/F5-TTS", + filter: false, + countDownloads: `path_extension:"safetensors" OR path_extension:"pt"`, + }, + genmo: { + prettyLabel: "Genmo", + repoName: "Genmo", + repoUrl: "https://github.com/genmoai/models", + filter: false, + countDownloads: `path:"vae_stats.json"`, + }, + "tencent-song-generation": { + prettyLabel: "SongGeneration", + repoName: "SongGeneration", + repoUrl: "https://github.com/tencent-ailab/songgeneration", + filter: false, + countDownloads: `path:"ckpt/songgeneration_base/model.pt"`, + }, + tensorflowtts: { + prettyLabel: "TensorFlowTTS", + repoName: "TensorFlowTTS", + repoUrl: "https://github.com/TensorSpeech/TensorFlowTTS", + snippets: snippets.tensorflowtts, + }, + tensorrt: { + prettyLabel: "TensorRT", + repoName: "TensorRT", + repoUrl: "https://github.com/NVIDIA/TensorRT", + countDownloads: `path_extension:"onnx"`, + }, + tabpfn: { + prettyLabel: "TabPFN", + repoName: "TabPFN", + repoUrl: "https://github.com/PriorLabs/TabPFN", + }, + terratorch: { + prettyLabel: "TerraTorch", + repoName: "TerraTorch", + repoUrl: "https://github.com/IBM/terratorch", + docsUrl: "https://ibm.github.io/terratorch/", + filter: false, + countDownloads: `path_extension:"pt" OR path_extension:"ckpt"`, + snippets: snippets.terratorch, + }, + "tic-clip": { + prettyLabel: "TiC-CLIP", + repoName: "TiC-CLIP", + repoUrl: "https://github.com/apple/ml-tic-clip", + filter: false, + countDownloads: `path_extension:"pt" AND path_prefix:"checkpoints/"`, + }, + timesfm: { + prettyLabel: "TimesFM", + repoName: "timesfm", + repoUrl: "https://github.com/google-research/timesfm", + filter: false, + countDownloads: `path:"checkpoints/checkpoint_1100000/state/checkpoint" OR path:"checkpoints/checkpoint_2150000/state/checkpoint" OR path_extension:"ckpt"`, + }, + timm: { + prettyLabel: "timm", + repoName: "pytorch-image-models", + repoUrl: "https://github.com/rwightman/pytorch-image-models", + docsUrl: "https://huggingface.co/docs/hub/timm", + snippets: snippets.timm, + filter: true, + countDownloads: `path:"pytorch_model.bin" OR path:"model.safetensors"`, + }, + tirex: { + prettyLabel: "TiRex", + repoName: "TiRex", + repoUrl: "https://github.com/NX-AI/tirex", + countDownloads: `path_extension:"ckpt"`, + }, + torchgeo: { + prettyLabel: "TorchGeo", + repoName: "TorchGeo", + repoUrl: "https://github.com/microsoft/torchgeo", + docsUrl: "https://torchgeo.readthedocs.io/", + filter: false, + countDownloads: `path_extension:"pt" OR path_extension:"pth"`, + }, + transformers: { + prettyLabel: "Transformers", + repoName: "🤗/transformers", + repoUrl: "https://github.com/huggingface/transformers", + docsUrl: "https://huggingface.co/docs/hub/transformers", + snippets: snippets.transformers, + filter: true, + }, + "transformers.js": { + prettyLabel: "Transformers.js", + repoName: "transformers.js", + repoUrl: "https://github.com/huggingface/transformers.js", + docsUrl: "https://huggingface.co/docs/hub/transformers-js", + snippets: snippets.transformersJS, + filter: true, + }, + trellis: { + prettyLabel: "Trellis", + repoName: "Trellis", + repoUrl: "https://github.com/microsoft/TRELLIS", + countDownloads: `path_extension:"safetensors"`, + }, + trellis2: { + prettyLabel: "TRELLIS.2", + repoName: "TRELLIS.2", + repoUrl: "https://github.com/microsoft/TRELLIS.2", + countDownloads: `path_extension:"safetensors"`, + }, + tunejury: { + prettyLabel: "TuneJury", + repoName: "TuneJury", + repoUrl: "https://github.com/yonghyunk1m/TuneJury", + countDownloads: `path_extension:"pt"`, + }, + + ultralytics: { + prettyLabel: "ultralytics", + repoName: "ultralytics", + repoUrl: "https://github.com/ultralytics/ultralytics", + docsUrl: "https://github.com/ultralytics/ultralytics", + filter: false, + countDownloads: `path_extension:"pt"`, + snippets: snippets.ultralytics, + }, + univa: { + prettyLabel: "univa", + repoName: "univa", + repoUrl: "https://github.com/PKU-YuanGroup/UniWorld-V1", + snippets: snippets.univa, + filter: true, + countDownloads: `path:"config.json"`, + }, + "uni-3dar": { + prettyLabel: "Uni-3DAR", + repoName: "Uni-3DAR", + repoUrl: "https://github.com/dptech-corp/Uni-3DAR", + docsUrl: "https://github.com/dptech-corp/Uni-3DAR", + countDownloads: `path_extension:"pt"`, + }, + "unity-sentis": { + prettyLabel: "unity-sentis", + repoName: "unity-sentis", + repoUrl: "https://github.com/Unity-Technologies/sentis-samples", + snippets: snippets.sentis, + filter: true, + countDownloads: `path_extension:"sentis"`, + }, + sana: { + prettyLabel: "Sana", + repoName: "Sana", + repoUrl: "https://github.com/NVlabs/Sana", + countDownloads: `path_extension:"pth"`, + snippets: snippets.sana, + }, + videoprism: { + prettyLabel: "VideoPrism", + repoName: "VideoPrism", + repoUrl: "https://github.com/google-deepmind/videoprism", + countDownloads: `path_extension:"npz"`, + snippets: snippets.videoprism, + }, + "vfi-mamba": { + prettyLabel: "VFIMamba", + repoName: "VFIMamba", + repoUrl: "https://github.com/MCG-NJU/VFIMamba", + countDownloads: `path_extension:"pkl"`, + snippets: snippets.vfimamba, + }, + vismatch: { + prettyLabel: "VisMatch", + repoName: "VisMatch", + repoUrl: "https://github.com/gmberton/vismatch", + filter: false, + countDownloads: `path:"vismatch.yaml"`, + }, + lvface: { + prettyLabel: "LVFace", + repoName: "LVFace", + repoUrl: "https://github.com/bytedance/LVFace", + countDownloads: `path_extension:"pt" OR path_extension:"onnx"`, + snippets: snippets.lvface, + }, + voicecraft: { + prettyLabel: "VoiceCraft", + repoName: "VoiceCraft", + repoUrl: "https://github.com/jasonppy/VoiceCraft", + docsUrl: "https://github.com/jasonppy/VoiceCraft", + snippets: snippets.voicecraft, + }, + voxcpm: { + prettyLabel: "VoxCPM", + repoName: "VoxCPM", + repoUrl: "https://github.com/OpenBMB/VoxCPM", + snippets: snippets.voxcpm, + filter: false, + }, + vui: { + prettyLabel: "Vui", + repoName: "Vui", + repoUrl: "https://github.com/vui-ai/vui", + countDownloads: `path_extension:"pt"`, + snippets: snippets.vui, + }, + vibevoice: { + prettyLabel: "VibeVoice", + repoName: "VibeVoice", + repoUrl: "https://github.com/microsoft/VibeVoice", + snippets: snippets.vibevoice, + filter: false, + }, + videox_fun: { + prettyLabel: "VideoX Fun", + repoName: "VideoX Fun", + repoUrl: "https://github.com/aigc-apps/VideoX-Fun", + filter: false, + countDownloads: `path_extension:"safetensors"`, + }, + "wan2.2": { + prettyLabel: "Wan2.2", + repoName: "Wan2.2", + repoUrl: "https://github.com/Wan-Video/Wan2.2", + countDownloads: `path_filename:"config" AND path_extension:"json"`, + }, + wham: { + prettyLabel: "WHAM", + repoName: "wham", + repoUrl: "https://huggingface.co/microsoft/wham", + docsUrl: "https://huggingface.co/microsoft/wham/blob/main/README.md", + countDownloads: `path_extension:"ckpt"`, + }, + whisperkit: { + prettyLabel: "WhisperKit", + repoName: "WhisperKit", + repoUrl: "https://github.com/argmaxinc/WhisperKit", + docsUrl: "https://github.com/argmaxinc/WhisperKit?tab=readme-ov-file#homebrew", + snippets: snippets.whisperkit, + countDownloads: `path_filename:"model" AND path_extension:"mil" AND _exists_:"path_prefix"`, + }, + yolov10: { + // YOLOv10 is a fork of ultraLytics. Code snippets and download count are the same but the repo is different. + prettyLabel: "YOLOv10", + repoName: "YOLOv10", + repoUrl: "https://github.com/THU-MIG/yolov10", + docsUrl: "https://github.com/THU-MIG/yolov10", + countDownloads: `path_extension:"pt" OR path_extension:"safetensors"`, + snippets: snippets.ultralytics, + }, + yolov26: { + prettyLabel: "YOLOv26", + repoName: "YOLOv26", + repoUrl: "https://github.com/ultralytics/ultralytics", + docsUrl: "https://docs.ultralytics.com/models/yolo26/", + countDownloads: `path_extension:"pt" OR path_extension:"safetensors"`, + }, + zonos: { + prettyLabel: "Zonos", + repoName: "Zonos", + repoUrl: "https://github.com/Zyphra/Zonos", + docsUrl: "https://github.com/Zyphra/Zonos", + snippets: snippets.zonos, + filter: false, + }, + "3dtopia-xl": { + prettyLabel: "3DTopia-XL", + repoName: "3DTopia-XL", + repoUrl: "https://github.com/3DTopia/3DTopia-XL", + filter: false, + countDownloads: `path:"model_vae_fp16.pt"`, + snippets: snippets.threedtopia_xl, + }, +} satisfies Record; + +export type ModelLibraryKey = keyof typeof MODEL_LIBRARIES_UI_ELEMENTS; + +export const ALL_MODEL_LIBRARY_KEYS = Object.keys(MODEL_LIBRARIES_UI_ELEMENTS) as ModelLibraryKey[]; + +export const ALL_DISPLAY_MODEL_LIBRARY_KEYS = ( + Object.entries(MODEL_LIBRARIES_UI_ELEMENTS as Record) as [ + ModelLibraryKey, + LibraryUiElement, + ][] +) + // eslint-disable-next-line @typescript-eslint/no-unused-vars + .filter(([_, v]) => v.filter) + .map(([k]) => k); diff --git a/node_modules/@huggingface/tasks/src/pipelines.ts b/node_modules/@huggingface/tasks/src/pipelines.ts new file mode 100644 index 0000000000000000000000000000000000000000..36c9c90fcad11941c485ae829cd86b398e29ac56 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/pipelines.ts @@ -0,0 +1,664 @@ +export const MODALITIES = ["multimodal", "nlp", "cv", "audio", "tabular", "rl", "other"] as const; + +export type Modality = (typeof MODALITIES)[number]; + +export const MODALITY_LABELS = { + multimodal: "Multimodal", + nlp: "Natural Language Processing", + audio: "Audio", + cv: "Computer Vision", + rl: "Reinforcement Learning", + tabular: "Tabular", + other: "Other", +} satisfies Record; + +/** + * Public interface for a sub task. + * + * This can be used in a model card's `model-index` metadata. + * and is more granular classification that can grow significantly + * over time as new tasks are added. + */ +export interface SubTask { + /** + * type of the task (e.g. audio-source-separation) + */ + type: string; + /** + * displayed name of the task (e.g. Audio Source Separation) + */ + name: string; +} + +/** + * Public interface for a PipelineData. + * + * This information corresponds to a pipeline type (aka task) + * in the Hub. + */ +export interface PipelineData { + /** + * displayed name of the task (e.g. Text Classification) + */ + name: string; + subtasks?: SubTask[]; + modality: Modality; + /** + * whether to hide in /models filters + */ + hideInModels?: boolean; + /** + * whether to hide in /datasets filters + */ + hideInDatasets?: boolean; +} + +/// Coarse-grained taxonomy of tasks +/// +/// This type is used in multiple places in the Hugging Face +/// ecosystem: +/// - To determine which widget to show. +/// - To determine which endpoint of Inference Endpoints to use. +/// - As filters at the left of models and datasets page. +/// +/// Note that this is sensitive to order. +/// For each domain, the order should be of decreasing specificity. +/// This will impact the default pipeline tag of a model when not +/// specified. +export const PIPELINE_DATA = { + "text-classification": { + name: "Text Classification", + subtasks: [ + { + type: "acceptability-classification", + name: "Acceptability Classification", + }, + { + type: "entity-linking-classification", + name: "Entity Linking Classification", + }, + { + type: "fact-checking", + name: "Fact Checking", + }, + { + type: "intent-classification", + name: "Intent Classification", + }, + { + type: "language-identification", + name: "Language Identification", + }, + { + type: "multi-class-classification", + name: "Multi Class Classification", + }, + { + type: "multi-label-classification", + name: "Multi Label Classification", + }, + { + type: "multi-input-text-classification", + name: "Multi-input Text Classification", + }, + { + type: "natural-language-inference", + name: "Natural Language Inference", + }, + { + type: "semantic-similarity-classification", + name: "Semantic Similarity Classification", + }, + { + type: "sentiment-classification", + name: "Sentiment Classification", + }, + { + type: "topic-classification", + name: "Topic Classification", + }, + { + type: "semantic-similarity-scoring", + name: "Semantic Similarity Scoring", + }, + { + type: "sentiment-scoring", + name: "Sentiment Scoring", + }, + { + type: "sentiment-analysis", + name: "Sentiment Analysis", + }, + { + type: "hate-speech-detection", + name: "Hate Speech Detection", + }, + { + type: "text-scoring", + name: "Text Scoring", + }, + ], + modality: "nlp", + }, + "token-classification": { + name: "Token Classification", + subtasks: [ + { + type: "named-entity-recognition", + name: "Named Entity Recognition", + }, + { + type: "part-of-speech", + name: "Part of Speech", + }, + { + type: "parsing", + name: "Parsing", + }, + { + type: "lemmatization", + name: "Lemmatization", + }, + { + type: "word-sense-disambiguation", + name: "Word Sense Disambiguation", + }, + { + type: "coreference-resolution", + name: "Coreference-resolution", + }, + ], + modality: "nlp", + }, + "table-question-answering": { + name: "Table Question Answering", + modality: "nlp", + }, + "question-answering": { + name: "Question Answering", + subtasks: [ + { + type: "extractive-qa", + name: "Extractive QA", + }, + { + type: "open-domain-qa", + name: "Open Domain QA", + }, + { + type: "closed-domain-qa", + name: "Closed Domain QA", + }, + ], + modality: "nlp", + }, + "zero-shot-classification": { + name: "Zero-Shot Classification", + modality: "nlp", + }, + translation: { + name: "Translation", + modality: "nlp", + }, + summarization: { + name: "Summarization", + subtasks: [ + { + type: "news-articles-summarization", + name: "News Articles Summarization", + }, + { + type: "news-articles-headline-generation", + name: "News Articles Headline Generation", + }, + ], + modality: "nlp", + }, + "feature-extraction": { + name: "Feature Extraction", + modality: "nlp", + }, + "text-generation": { + name: "Text Generation", + subtasks: [ + { + type: "dialogue-modeling", + name: "Dialogue Modeling", + }, + { + type: "dialogue-generation", + name: "Dialogue Generation", + }, + { + type: "conversational", + name: "Conversational", + }, + { + type: "language-modeling", + name: "Language Modeling", + }, + { + type: "text-simplification", + name: "Text simplification", + }, + { + type: "explanation-generation", + name: "Explanation Generation", + }, + { + type: "abstractive-qa", + name: "Abstractive QA", + }, + { + type: "open-domain-abstractive-qa", + name: "Open Domain Abstractive QA", + }, + { + type: "closed-domain-qa", + name: "Closed Domain QA", + }, + { + type: "open-book-qa", + name: "Open Book QA", + }, + { + type: "closed-book-qa", + name: "Closed Book QA", + }, + { + type: "text2text-generation", + name: "Text2Text Generation", + }, + ], + modality: "nlp", + }, + "fill-mask": { + name: "Fill-Mask", + subtasks: [ + { + type: "slot-filling", + name: "Slot Filling", + }, + { + type: "masked-language-modeling", + name: "Masked Language Modeling", + }, + ], + modality: "nlp", + }, + "sentence-similarity": { + name: "Sentence Similarity", + modality: "nlp", + }, + "text-to-speech": { + name: "Text-to-Speech", + modality: "audio", + }, + "text-to-audio": { + name: "Text-to-Audio", + modality: "audio", + }, + "automatic-speech-recognition": { + name: "Automatic Speech Recognition", + modality: "audio", + }, + "audio-to-audio": { + name: "Audio-to-Audio", + modality: "audio", + }, + "audio-classification": { + name: "Audio Classification", + subtasks: [ + { + type: "keyword-spotting", + name: "Keyword Spotting", + }, + { + type: "speaker-identification", + name: "Speaker Identification", + }, + { + type: "audio-intent-classification", + name: "Audio Intent Classification", + }, + { + type: "audio-emotion-recognition", + name: "Audio Emotion Recognition", + }, + { + type: "audio-language-identification", + name: "Audio Language Identification", + }, + ], + modality: "audio", + }, + "audio-text-to-text": { + name: "Audio-Text-to-Text", + modality: "multimodal", + hideInDatasets: true, + }, + "voice-activity-detection": { + name: "Voice Activity Detection", + modality: "audio", + }, + "depth-estimation": { + name: "Depth Estimation", + modality: "cv", + }, + "image-classification": { + name: "Image Classification", + subtasks: [ + { + type: "multi-label-image-classification", + name: "Multi Label Image Classification", + }, + { + type: "multi-class-image-classification", + name: "Multi Class Image Classification", + }, + ], + modality: "cv", + }, + "object-detection": { + name: "Object Detection", + subtasks: [ + { + type: "face-detection", + name: "Face Detection", + }, + { + type: "vehicle-detection", + name: "Vehicle Detection", + }, + ], + modality: "cv", + }, + "image-segmentation": { + name: "Image Segmentation", + subtasks: [ + { + type: "instance-segmentation", + name: "Instance Segmentation", + }, + { + type: "semantic-segmentation", + name: "Semantic Segmentation", + }, + { + type: "panoptic-segmentation", + name: "Panoptic Segmentation", + }, + ], + modality: "cv", + }, + "text-to-image": { + name: "Text-to-Image", + modality: "cv", + }, + "image-to-text": { + name: "Image-to-Text", + subtasks: [ + { + type: "image-captioning", + name: "Image Captioning", + }, + ], + modality: "cv", + }, + "image-to-image": { + name: "Image-to-Image", + subtasks: [ + { + type: "image-inpainting", + name: "Image Inpainting", + }, + { + type: "image-colorization", + name: "Image Colorization", + }, + { + type: "super-resolution", + name: "Super Resolution", + }, + ], + modality: "cv", + }, + "image-to-video": { + name: "Image-to-Video", + modality: "cv", + }, + "unconditional-image-generation": { + name: "Unconditional Image Generation", + modality: "cv", + }, + "video-classification": { + name: "Video Classification", + modality: "cv", + }, + "reinforcement-learning": { + name: "Reinforcement Learning", + modality: "rl", + }, + robotics: { + name: "Robotics", + modality: "rl", + subtasks: [ + { + type: "grasping", + name: "Grasping", + }, + { + type: "task-planning", + name: "Task Planning", + }, + ], + }, + "tabular-classification": { + name: "Tabular Classification", + modality: "tabular", + subtasks: [ + { + type: "tabular-multi-class-classification", + name: "Tabular Multi Class Classification", + }, + { + type: "tabular-multi-label-classification", + name: "Tabular Multi Label Classification", + }, + ], + }, + "tabular-regression": { + name: "Tabular Regression", + modality: "tabular", + subtasks: [ + { + type: "tabular-single-column-regression", + name: "Tabular Single Column Regression", + }, + ], + }, + "tabular-to-text": { + name: "Tabular to Text", + modality: "tabular", + subtasks: [ + { + type: "rdf-to-text", + name: "RDF to text", + }, + ], + hideInModels: true, + }, + "table-to-text": { + name: "Table to Text", + modality: "nlp", + hideInModels: true, + }, + "multiple-choice": { + name: "Multiple Choice", + subtasks: [ + { + type: "multiple-choice-qa", + name: "Multiple Choice QA", + }, + { + type: "multiple-choice-coreference-resolution", + name: "Multiple Choice Coreference Resolution", + }, + ], + modality: "nlp", + hideInModels: true, + }, + "text-ranking": { + name: "Text Ranking", + modality: "nlp", + }, + "text-retrieval": { + name: "Text Retrieval", + subtasks: [ + { + type: "document-retrieval", + name: "Document Retrieval", + }, + { + type: "utterance-retrieval", + name: "Utterance Retrieval", + }, + { + type: "entity-linking-retrieval", + name: "Entity Linking Retrieval", + }, + { + type: "fact-checking-retrieval", + name: "Fact Checking Retrieval", + }, + ], + modality: "nlp", + hideInModels: true, + }, + "time-series-forecasting": { + name: "Time Series Forecasting", + modality: "tabular", + subtasks: [ + { + type: "univariate-time-series-forecasting", + name: "Univariate Time Series Forecasting", + }, + { + type: "multivariate-time-series-forecasting", + name: "Multivariate Time Series Forecasting", + }, + ], + }, + "text-to-video": { + name: "Text-to-Video", + modality: "cv", + }, + "image-text-to-text": { + name: "Image-Text-to-Text", + modality: "multimodal", + }, + "image-text-to-image": { + name: "Image-Text-to-Image", + modality: "multimodal", + }, + "image-text-to-video": { + name: "Image-Text-to-Video", + modality: "multimodal", + }, + "visual-question-answering": { + name: "Visual Question Answering", + subtasks: [ + { + type: "visual-question-answering", + name: "Visual Question Answering", + }, + ], + modality: "multimodal", + }, + "document-question-answering": { + name: "Document Question Answering", + subtasks: [ + { + type: "document-question-answering", + name: "Document Question Answering", + }, + ], + modality: "multimodal", + hideInDatasets: true, + }, + "zero-shot-image-classification": { + name: "Zero-Shot Image Classification", + modality: "cv", + }, + "graph-ml": { + name: "Graph Machine Learning", + modality: "other", + }, + "mask-generation": { + name: "Mask Generation", + modality: "cv", + }, + "zero-shot-object-detection": { + name: "Zero-Shot Object Detection", + modality: "cv", + }, + "text-to-3d": { + name: "Text-to-3D", + modality: "cv", + }, + "image-to-3d": { + name: "Image-to-3D", + modality: "cv", + }, + "image-feature-extraction": { + name: "Image Feature Extraction", + modality: "cv", + }, + "video-text-to-text": { + name: "Video-Text-to-Text", + modality: "multimodal", + hideInDatasets: false, + }, + "keypoint-detection": { + name: "Keypoint Detection", + subtasks: [ + { + type: "pose-estimation", + name: "Pose Estimation", + }, + ], + modality: "cv", + hideInDatasets: true, + }, + "visual-document-retrieval": { + name: "Visual Document Retrieval", + modality: "multimodal", + }, + "any-to-any": { + name: "Any-to-Any", + modality: "multimodal", + }, + "video-to-video": { + name: "Video-to-Video", + modality: "cv", + hideInDatasets: true, + }, + other: { + name: "Other", + modality: "other", + hideInModels: true, + hideInDatasets: true, + }, +} satisfies Record; + +export type PipelineType = keyof typeof PIPELINE_DATA; + +export type WidgetType = PipelineType | "conversational"; + +export const PIPELINE_TYPES = Object.keys(PIPELINE_DATA) as PipelineType[]; + +export const SUBTASK_TYPES = Object.values(PIPELINE_DATA) + .flatMap((data) => ("subtasks" in data ? data.subtasks : [])) + .map((s) => s.type); + +export const PIPELINE_TYPES_SET = new Set(PIPELINE_TYPES); diff --git a/node_modules/@huggingface/tasks/src/snippets/common.ts b/node_modules/@huggingface/tasks/src/snippets/common.ts new file mode 100644 index 0000000000000000000000000000000000000000..5b325d35f7571276741cfab9bfb3f802d99f5d8a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/snippets/common.ts @@ -0,0 +1,39 @@ +import type { ChatCompletionInputMessage, GenerationParameters } from "../tasks/index.js"; + +export function stringifyMessages( + messages: ChatCompletionInputMessage[], + opts?: { + indent?: string; + attributeKeyQuotes?: boolean; + customContentEscaper?: (str: string) => string; + }, +): string { + let messagesStr = JSON.stringify(messages, null, "\t"); + if (opts?.indent) { + messagesStr = messagesStr.replaceAll("\n", `\n${opts.indent}`); + } + if (!opts?.attributeKeyQuotes) { + messagesStr = messagesStr.replace(/"([^"]+)":/g, "$1:"); + } + if (opts?.customContentEscaper) { + messagesStr = opts.customContentEscaper(messagesStr); + } + return messagesStr; +} + +type PartialGenerationParameters = Partial>; + +export function stringifyGenerationConfig( + config: PartialGenerationParameters, + opts: { + indent: string; + attributeValueConnector: string; + attributeKeyQuotes?: boolean; + }, +): string { + const quote = opts.attributeKeyQuotes ? `"` : ""; + + return Object.entries(config) + .map(([key, val]) => `${quote}${key}${quote}${opts.attributeValueConnector}${val},`) + .join(`${opts.indent}`); +} diff --git a/node_modules/@huggingface/tasks/src/snippets/index.ts b/node_modules/@huggingface/tasks/src/snippets/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..bd77dfd5213a3c73a1514ec0d14c87093b5bfb52 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/snippets/index.ts @@ -0,0 +1,3 @@ +export * from "./common.js"; +export * from "./inputs.js"; +export * from "./types.js"; diff --git a/node_modules/@huggingface/tasks/src/snippets/inputs.ts b/node_modules/@huggingface/tasks/src/snippets/inputs.ts new file mode 100644 index 0000000000000000000000000000000000000000..cffcde67adf7295b5ceae4dfca727eb8503fdd10 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/snippets/inputs.ts @@ -0,0 +1,191 @@ +import type { PipelineType } from "../pipelines.js"; +import type { ChatCompletionInputMessage } from "../tasks/index.js"; +import type { ModelDataMinimal } from "./types.js"; + +const inputsZeroShotClassification = () => + `"Hi, I recently bought a device from your company but it is not working as advertised and I would like to get reimbursed!"`; + +const inputsTranslation = () => `"Меня зовут Вольфганг и я живу в Берлине"`; + +const inputsSummarization = () => + `"The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct."`; + +const inputsTableQuestionAnswering = () => + `{ + "query": "How many stars does the transformers repository have?", + "table": { + "Repository": ["Transformers", "Datasets", "Tokenizers"], + "Stars": ["36542", "4512", "3934"], + "Contributors": ["651", "77", "34"], + "Programming language": [ + "Python", + "Python", + "Rust, Python and NodeJS" + ] + } +}`; + +const inputsVisualQuestionAnswering = () => + `{ + "image": "cat.png", + "question": "What is in this image?" + }`; + +const inputsQuestionAnswering = () => + `{ + "question": "What is my name?", + "context": "My name is Clara and I live in Berkeley." +}`; + +const inputsTextClassification = () => `"I like you. I love you"`; + +const inputsTokenClassification = () => `"My name is Sarah Jessica Parker but you can call me Jessica"`; + +const inputsTextGeneration = (model: ModelDataMinimal): string | ChatCompletionInputMessage[] => { + if (model.tags.includes("conversational")) { + return model.pipeline_tag === "text-generation" + ? [{ role: "user", content: "What is the capital of France?" }] + : [ + { + role: "user", + content: [ + { + type: "text", + text: "Describe this image in one sentence.", + }, + { + type: "image_url", + image_url: { + url: "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg", + }, + }, + ], + }, + ]; + } + return `"Can you please let us know more details about your "`; +}; + +const inputsFillMask = (model: ModelDataMinimal) => `"The answer to the universe is ${model.mask_token}."`; + +const inputsSentenceSimilarity = () => + `{ + "source_sentence": "That is a happy person", + "sentences": [ + "That is a happy dog", + "That is a very happy person", + "Today is a sunny day" + ] +}`; + +const inputsFeatureExtraction = () => `"Today is a sunny day and I will get some ice cream."`; + +const inputsImageClassification = () => `"cats.jpg"`; + +const inputsImageToText = () => `"cats.jpg"`; + +const inputsImageToImage = () => `{ + "image": "cat.png", + "prompt": "Turn the cat into a tiger." +}`; + +const inputsImageToVideo = () => `{ + "image": "cat.png", + "prompt": "The cat starts to dance" +}`; + +const inputsImageTextToImage = () => `{ + "image": "cat.png", + "prompt": "Turn the cat into a tiger." +}`; + +const inputsImageTextToVideo = () => `{ + "image": "cat.png", + "prompt": "The cat starts to dance" +}`; + +const inputsImageSegmentation = () => `"cats.jpg"`; + +const inputsObjectDetection = () => `"cats.jpg"`; + +const inputsAudioToAudio = () => `"sample1.flac"`; + +const inputsAudioClassification = () => `"sample1.flac"`; + +const inputsTextToImage = () => `"Astronaut riding a horse"`; + +const inputsTextToVideo = () => `"A young man walking on the street"`; + +const inputsTextToSpeech = () => `"The answer to the universe is 42"`; + +const inputsTextToAudio = () => `"liquid drum and bass, atmospheric synths, airy sounds"`; + +const inputsAutomaticSpeechRecognition = () => `"sample1.flac"`; + +const inputsTabularPrediction = () => + `'{"Height":[11.52,12.48],"Length1":[23.2,24.0],"Length2":[25.4,26.3],"Species": ["Bream","Bream"]}'`; + +const inputsZeroShotImageClassification = () => `"cats.jpg"`; + +const modelInputSnippets: { + [key in PipelineType]?: (model: ModelDataMinimal) => string | ChatCompletionInputMessage[]; +} = { + "audio-to-audio": inputsAudioToAudio, + "audio-classification": inputsAudioClassification, + "automatic-speech-recognition": inputsAutomaticSpeechRecognition, + "document-question-answering": inputsVisualQuestionAnswering, + "feature-extraction": inputsFeatureExtraction, + "fill-mask": inputsFillMask, + "image-classification": inputsImageClassification, + "image-to-text": inputsImageToText, + "image-to-image": inputsImageToImage, + "image-to-video": inputsImageToVideo, + "image-text-to-image": inputsImageTextToImage, + "image-text-to-video": inputsImageTextToVideo, + "image-segmentation": inputsImageSegmentation, + "object-detection": inputsObjectDetection, + "question-answering": inputsQuestionAnswering, + "sentence-similarity": inputsSentenceSimilarity, + summarization: inputsSummarization, + "table-question-answering": inputsTableQuestionAnswering, + "tabular-regression": inputsTabularPrediction, + "tabular-classification": inputsTabularPrediction, + "text-classification": inputsTextClassification, + "text-generation": inputsTextGeneration, + "image-text-to-text": inputsTextGeneration, + "text-to-image": inputsTextToImage, + "text-to-video": inputsTextToVideo, + "text-to-speech": inputsTextToSpeech, + "text-to-audio": inputsTextToAudio, + "token-classification": inputsTokenClassification, + translation: inputsTranslation, + "zero-shot-classification": inputsZeroShotClassification, + "zero-shot-image-classification": inputsZeroShotImageClassification, +}; + +// Use noWrap to put the whole snippet on a single line (removing new lines and tabulations) +// Use noQuotes to strip quotes from start & end (example: "abc" -> abc) +export function getModelInputSnippet( + model: ModelDataMinimal, + noWrap = false, + noQuotes = false, +): string | ChatCompletionInputMessage[] { + if (model.pipeline_tag) { + const inputs = modelInputSnippets[model.pipeline_tag]; + if (inputs) { + let result = inputs(model); + if (typeof result === "string") { + if (noWrap) { + result = result.replace(/(?:(?:\r?\n|\r)\t*)|\t+/g, " "); + } + if (noQuotes) { + const REGEX_QUOTES = /^"(.+)"$/s; + const match = result.match(REGEX_QUOTES); + result = match ? match[1] : result; + } + } + return result; + } + } + return "No input example has been defined for this model task."; +} diff --git a/node_modules/@huggingface/tasks/src/snippets/types.ts b/node_modules/@huggingface/tasks/src/snippets/types.ts new file mode 100644 index 0000000000000000000000000000000000000000..22400d3ae11adf9f1eaa5f71dada70056e4a51e8 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/snippets/types.ts @@ -0,0 +1,21 @@ +import type { ModelData } from "../model-data.js"; + +/** + * Minimal model data required for snippets. + * + * Add more fields as needed. + */ +export type ModelDataMinimal = Pick< + ModelData, + "id" | "pipeline_tag" | "mask_token" | "library_name" | "config" | "tags" | "inference" +>; + +// Order of the elements in InferenceModal.svelte is determined by this const +export const inferenceSnippetLanguages = ["python", "js", "sh"] as const; +export type InferenceSnippetLanguage = (typeof inferenceSnippetLanguages)[number]; + +export interface InferenceSnippet { + language: InferenceSnippetLanguage; // e.g. `python`, `curl`, `js` + client: string; // e.g. `huggingface_hub`, `openai`, `fetch`, etc. + content: string; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/any-to-any/about.md b/node_modules/@huggingface/tasks/src/tasks/any-to-any/about.md new file mode 100644 index 0000000000000000000000000000000000000000..6e7c42430ad76f75966ae1b5f57640f3fd6452dc --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/any-to-any/about.md @@ -0,0 +1,71 @@ +## Use Cases + +### Embodied Agents + +Any-to-any models can help embodied agents operate in multi-sensory environments, such as video games or physical robots. The model can take in an image or video of a scene, text prompts, and audio, and respond by generating text, actions, predict next frames, or generate speech commands. + +### Real-time Accessibility Systems + +Vision-language based any-to-any models can be used to aid visually impaired people. A real-time on-device any-to-any model can take a real-world video stream from wearable glasses, and describe the scene in audio (e.g., "A person in a red coat is walking toward you"), or provide real-time closed captions and environmental sound cues. + +### Multimodal Content Creation + +One can use any-to-any models to generate multimodal content. For example, given a video and an outline, the model can generate speech, better videos, or a descriptive blog post. Moreover, these models can sync narration timing with visual transitions. + +## Inference + +You can infer with any-to-any models using transformers. Below is an example that passes a video as part of a chat conversation to the Qwen2.5-Omni-7B model, and retrieves text and audio responses. Make sure to check the model you're inferring with. + +```python +import soundfile as sf +from transformers import Qwen2_5OmniForConditionalGeneration, Qwen2_5OmniProcessor + +model = Qwen2_5OmniForConditionalGeneration.from_pretrained( + "Qwen/Qwen2.5-Omni-7B", + torch_dtype="auto", + device_map="auto", + attn_implementation="flash_attention_2", +) +processor = Qwen2_5OmniProcessor.from_pretrained("Qwen/Qwen2.5-Omni-7B") + +conversation = [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech."} + ], + }, + { + "role": "user", + "content": [ + {"type": "video", "video": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw.mp4"}, + {"type": "text", "text": "What can you hear and see in this video?"}, + ], + }, +] + +inputs = processor.apply_chat_template( + conversation, + load_audio_from_video=True, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + video_fps=2, + + # kwargs to be passed to `Qwen2-5-OmniProcessor` + padding=True, + use_audio_in_video=True, +) + +# Inference: Generation of the output text and audio +text_ids, audio = model.generate(**inputs, use_audio_in_video=True) + +text = processor.batch_decode(text_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False) +print(text) +sf.write( + "output.wav", + audio.reshape(-1).detach().cpu().numpy(), + samplerate=24000, +) +``` diff --git a/node_modules/@huggingface/tasks/src/tasks/any-to-any/data.ts b/node_modules/@huggingface/tasks/src/tasks/any-to-any/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..2c7351e545d29fb525d758c05ac2d6b32a9ba4d4 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/any-to-any/data.ts @@ -0,0 +1,65 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A dataset with multiple modality input and output pairs.", + id: "PKU-Alignment/align-anything", + }, + ], + demo: { + inputs: [ + { + filename: "any-to-any-input.jpg", + type: "img", + }, + { + label: "Text Prompt", + content: "What is the significance of this place?", + type: "text", + }, + ], + outputs: [ + { + label: "Generated Text", + content: + "The place in the picture is Osaka Castle, located in Osaka, Japan. Osaka Castle is a historic castle that was originally built in the 16th century by Toyotomi Hideyoshi, a powerful warlord of the time. It is one of the most famous landmarks in Osaka and is known for its distinctive white walls and black roof tiles. The castle has been rebuilt several times over the centuries and is now a popular tourist attraction, offering visitors a glimpse into Japan's rich history and culture.", + type: "text", + }, + { + filename: "any-to-any-output.wav", + type: "audio", + }, + ], + }, + metrics: [], + models: [ + { + description: "Strong model that can take in video, audio, image, text and output text and natural speech.", + id: "Qwen/Qwen2.5-Omni-7B", + }, + { + description: "Robust model that can take in image and text and generate image and text.", + id: "OmniGen2/OmniGen2", + }, + { + description: "Any-to-any model with speech, video, audio, image and text understanding capabilities.", + id: "openbmb/MiniCPM-o-2_6", + }, + { + description: "A model that can understand image and text and generate image and text.", + id: "ByteDance-Seed/BAGEL-7B-MoT", + }, + ], + spaces: [ + { + description: "An application to chat with an any-to-any (image & text) model.", + id: "OmniGen2/OmniGen2", + }, + ], + summary: "Any-to-any models can understand two or more modalities and output two or more modalities.", + widgetModels: [], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/audio-classification/about.md b/node_modules/@huggingface/tasks/src/tasks/audio-classification/about.md new file mode 100644 index 0000000000000000000000000000000000000000..dfeab08f05ba1c3b6b2040094bb254b26be93379 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/audio-classification/about.md @@ -0,0 +1,86 @@ +## Use Cases + +### Command Recognition + +Command recognition or keyword spotting classifies utterances into a predefined set of commands. This is often done on-device for fast response time. + +As an example, using the Google Speech Commands dataset, given an input, a model can classify which of the following commands the user is typing: + +``` +'yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go', 'unknown', 'silence' +``` + +Speechbrain models can easily perform this task with just a couple of lines of code! + +```python +from speechbrain.pretrained import EncoderClassifier +model = EncoderClassifier.from_hparams( + "speechbrain/google_speech_command_xvector" +) +model.classify_file("file.wav") +``` + +### Language Identification + +Datasets such as VoxLingua107 allow anyone to train language identification models for up to 107 languages! This can be extremely useful as a preprocessing step for other systems. Here's an example [model](https://huggingface.co/TalTechNLP/voxlingua107-epaca-tdnn)trained on VoxLingua107. + +### Emotion recognition + +Emotion recognition is self explanatory. In addition to trying the widgets, you can use Inference Endpoints to perform audio classification. Here is a simple example that uses a [HuBERT](https://huggingface.co/superb/hubert-large-superb-er) model fine-tuned for this task. + +```python +import json +import requests + +headers = {"Authorization": f"Bearer {API_TOKEN}"} +API_URL = "https://router.huggingface.co/hf-inference/models/superb/hubert-large-superb-er" + +def query(filename): + with open(filename, "rb") as f: + data = f.read() + response = requests.request("POST", API_URL, headers=headers, data=data) + return json.loads(response.content.decode("utf-8")) + +data = query("sample1.flac") +# [{'label': 'neu', 'score': 0.60}, +# {'label': 'hap', 'score': 0.20}, +# {'label': 'ang', 'score': 0.13}, +# {'label': 'sad', 'score': 0.07}] +``` + +You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer with audio classification models on Hugging Face Hub. + +```javascript +import { InferenceClient } from "@huggingface/inference"; + +const inference = new InferenceClient(HF_TOKEN); +await inference.audioClassification({ + data: await (await fetch("sample.flac")).blob(), + model: "facebook/mms-lid-126", +}); +``` + +### Speaker Identification + +Speaker Identification is classifying the audio of the person speaking. Speakers are usually predefined. You can try out this task with [this model](https://huggingface.co/superb/wav2vec2-base-superb-sid). A useful dataset for this task is VoxCeleb1. + +## Solving audio classification for your own data + +We have some great news! You can do fine-tuning (transfer learning) to train a well-performing model without requiring as much data. Pretrained models such as Wav2Vec2 and HuBERT exist. [Facebook's Wav2Vec2 XLS-R model](https://huggingface.co/docs/transformers/model_doc/xlsr_wav2vec2) is a large multilingual model trained on 128 languages and with 436K hours of speech. Similarly, you can also use [OpenAI's Whisper](https://huggingface.co/docs/transformers/model_doc/whisper) trained on up to 4 Million hours of multilingual speech data for this task too! + +## Useful Resources + +Would you like to learn more about the topic? Awesome! Here you can find some curated resources that you may find helpful! + +### Notebooks + +- [PyTorch](https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/audio_classification.ipynb) + +### Scripts for training + +- [PyTorch](https://github.com/huggingface/transformers/tree/main/examples/pytorch/audio-classification) + +### Documentation + +- [Hugging Face Audio Course](https://huggingface.co/learn/audio-course/chapter4/introduction) +- [Audio classification task guide](https://huggingface.co/docs/transformers/tasks/audio_classification) diff --git a/node_modules/@huggingface/tasks/src/tasks/audio-classification/data.ts b/node_modules/@huggingface/tasks/src/tasks/audio-classification/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..8919f81c0eb0770fdbe91f10f40cd6e4e9b61047 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/audio-classification/data.ts @@ -0,0 +1,81 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A benchmark of 10 different audio tasks.", + id: "s3prl/superb", + }, + { + description: "A dataset of YouTube clips and their sound categories.", + id: "agkphysics/AudioSet", + }, + ], + demo: { + inputs: [ + { + filename: "audio.wav", + type: "audio", + }, + ], + outputs: [ + { + data: [ + { + label: "Up", + score: 0.2, + }, + { + label: "Down", + score: 0.8, + }, + ], + type: "chart", + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + description: "An easy-to-use model for command recognition.", + id: "speechbrain/google_speech_command_xvector", + }, + { + description: "An emotion recognition model.", + id: "ehcalabres/wav2vec2-lg-xlsr-en-speech-emotion-recognition", + }, + { + description: "A language identification model.", + id: "facebook/mms-lid-126", + }, + ], + spaces: [ + { + description: "An application that can classify music into different genre.", + id: "kurianbenoy/audioclassification", + }, + ], + summary: + "Audio classification is the task of assigning a label or class to a given audio. It can be used for recognizing which command a user is giving or the emotion of a statement, as well as identifying a speaker.", + widgetModels: ["MIT/ast-finetuned-audioset-10-10-0.4593"], + youtubeId: "KWwzcmG98Ds", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/audio-classification/inference.ts b/node_modules/@huggingface/tasks/src/tasks/audio-classification/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..5a87b2e46c61aad7d73c704486e130a39d92566f --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/audio-classification/inference.ts @@ -0,0 +1,53 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Audio Classification inference + */ +export interface AudioClassificationInput { + /** + * The input audio data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the audio data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Audio Classification + */ + parameters?: AudioClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Audio Classification + */ +export interface AudioClassificationParameters { + /** + * The function to apply to the model outputs in order to retrieve the scores. + */ + function_to_apply?: ClassificationOutputTransform; + /** + * When specified, limits the output to the top K most probable classes. + */ + top_k?: number; + [property: string]: unknown; +} +/** + * The function to apply to the model outputs in order to retrieve the scores. + */ +export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none"; +export type AudioClassificationOutput = AudioClassificationOutputElement[]; +/** + * Outputs for Audio Classification inference + */ +export interface AudioClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/audio-classification/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/audio-classification/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..df86e333b4f05aeb8f9268edb6c646d4b56fd03e --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/audio-classification/spec/input.json @@ -0,0 +1,36 @@ +{ + "$id": "/inference/schemas/audio-classification/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Audio Classification inference", + "title": "AudioClassificationInput", + "type": "object", + "properties": { + "inputs": { + "description": "The input audio data as a base64-encoded string. If no `parameters` are provided, you can also provide the audio data as a raw bytes payload.", + "type": "string", + "comment": "type=binary" + }, + "parameters": { + "description": "Additional inference parameters for Audio Classification", + "$ref": "#/$defs/AudioClassificationParameters" + } + }, + "$defs": { + "AudioClassificationParameters": { + "title": "AudioClassificationParameters", + "type": "object", + "properties": { + "function_to_apply": { + "title": "AudioClassificationOutputTransform", + "$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutputTransform", + "description": "The function to apply to the model outputs in order to retrieve the scores." + }, + "top_k": { + "type": "integer", + "description": "When specified, limits the output to the top K most probable classes." + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/audio-classification/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/audio-classification/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..f1f2dfe8ef36519f30898d6ad5e6924112e5e860 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/audio-classification/spec/output.json @@ -0,0 +1,11 @@ +{ + "$id": "/inference/schemas/audio-classification/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "title": "AudioClassificationOutput", + "description": "Outputs for Audio Classification inference", + "type": "array", + "items": { + "type": "object", + "$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput" + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/audio-text-to-text/about.md b/node_modules/@huggingface/tasks/src/tasks/audio-text-to-text/about.md new file mode 100644 index 0000000000000000000000000000000000000000..685385e25ce815731bacd5dd916deeccca7ea9a2 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/audio-text-to-text/about.md @@ -0,0 +1,139 @@ +## Use Cases + +> This task takes `audio` and a `text prompt` and returns `text` (answers, summaries, structured notes, etc.). + +### Audio question answering + +Ask targeted questions about lectures, podcasts, or calls and get context-aware answers. +**Example:** Audio: physics lecture → Prompt: “What did the teacher say about gravity and how is it measured?” + +### Meeting notes & action items + +Turn multi-speaker meetings into concise minutes with decisions, owners, and deadlines. +**Example:** Audio: weekly stand-up → Prompt: “Summarize key decisions and list action items with assignees.” + +### Speech understanding & intent + +Go beyond transcription to extract intent, sentiment, uncertainty, or emotion from spoken language. +**Example:** “I’m not sure I can finish this on time.” → Prompt: “Describe speaker intent and confidence.” + +### Music & sound analysis (textual) + +Describe instrumentation, genre, tempo, or sections, and suggest edits or techniques (text output only). +**Example:** Song demo → Prompt: “Identify key and tempo, then suggest jazz reharmonization ideas for the chorus.” + +## Inference + +You can use the 'transformers' library, and your audio file to any of the `audio-text-to-text` model, with instructions and get text responses. Following code examples show how to do so. + +### Speech Transcription and Analysis + +These models don’t just turn speech into text—they also capture tone, emotion, and speaker traits. This makes them useful for tasks like sentiment analysis or identifying speaker profiles. + +You can try audio transcription with [Voxtral Mini](https://huggingface.co/mistralai/Voxtral-Mini-3B-2507) using the following code. + +```python +from transformers import VoxtralForConditionalGeneration, AutoProcessor +import torch + +device = "cuda" +repo_id = "mistralai/Voxtral-Mini-3B-2507" + +processor = AutoProcessor.from_pretrained(repo_id) +model = VoxtralForConditionalGeneration.from_pretrained(repo_id, dtype=torch.bfloat16, device_map=device) + +inputs = processor.apply_transcription_request(language="en", audio="https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/obama.mp3", model_id=repo_id) +inputs = inputs.to(device, dtype=torch.bfloat16) + +outputs = model.generate(**inputs, max_new_tokens=500) +decoded_outputs = processor.batch_decode(outputs[:, inputs.input_ids.shape[1]:], skip_special_tokens=True) + +print("\nGenerated responses:") +print("=" * 80) +for decoded_output in decoded_outputs: + print(decoded_output) + print("=" * 80) +``` + +### Audio Question Answering + +These models can understand audio directly and answer questions about it. For example, summarizing a podcast clip or explaining parts of a recorded conversation. + +You can experiment with [Qwen2-Audio-Instruct-Demo](https://huggingface.co/Qwen/Qwen2-Audio-Instruct-Demo) for conversations with both text and audio inputs, letting you ask follow-up questions about different sounds or speech clips. + +```python +from io import BytesIO +from urllib.request import urlopen +import librosa +from transformers import Qwen2AudioForConditionalGeneration, AutoProcessor + +processor = AutoProcessor.from_pretrained("Qwen/Qwen2-Audio-7B-Instruct") +model = Qwen2AudioForConditionalGeneration.from_pretrained("Qwen/Qwen2-Audio-7B-Instruct", device_map="auto") + +conversation = [ + {'role': 'system', 'content': 'You are a helpful assistant.'}, + {"role": "user", "content": [ + {"type": "audio", "audio_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/glass-breaking-151256.mp3"}, + {"type": "text", "text": "What's that sound?"}, + ]}, + {"role": "assistant", "content": "It is the sound of glass shattering."}, + {"role": "user", "content": [ + {"type": "text", "text": "What can you do when you hear that?"}, + ]}, + {"role": "assistant", "content": "Stay alert and cautious, and check if anyone is hurt or if there is any damage to property."}, + {"role": "user", "content": [ + {"type": "audio", "audio_url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2-Audio/audio/1272-128104-0000.flac"}, + {"type": "text", "text": "What does the person say?"}, + ]}, +] +text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False) +audios = [] +for message in conversation: + if isinstance(message["content"], list): + for ele in message["content"]: + if ele["type"] == "audio": + audios.append( + librosa.load( + BytesIO(urlopen(ele['audio_url']).read()), + sr=processor.feature_extractor.sampling_rate)[0] + ) + +inputs = processor(text=text, audios=audios, return_tensors="pt", padding=True) +inputs.input_ids = inputs.input_ids.to("cuda") + +generate_ids = model.generate(**inputs, max_length=256) +generate_ids = generate_ids[:, inputs.input_ids.size(1):] + +response = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] +``` + +## Useful Resources + +If you want to learn more about this concept, here are some useful links: + +### Papers + +- [SpeechGPT](https://huggingface.co/papers/2507.13264) — multimodal dialogue with speech and text. +- [Voxtral](https://huggingface.co/papers/2507.13264) — a state-of-the-art audio-text model. +- [Qwen2-audio-instruct](https://huggingface.co/papers/2407.10759) — large-scale audio-language modeling for instruction following. +- [AudioPaLM](https://huggingface.co/papers/2306.12925) — scaling audio-language models with PaLM. + +### Models, Codes & Demos + +- [Qwen2-audio-instruct](https://github.com/QwenLM/Qwen2-Audio) — open-source implementation with demos. +- [SpeechGPT](https://github.com/0nutation/SpeechGPT) — An end-to-end framework for audio conversational models built on top of large language models. +- [AudioPaLM](https://google-research.github.io/seanet/audiopalm/examples/) — resources and code for AudioPaLM. +- [Audio Flamingo](https://huggingface.co/nvidia/audio-flamingo-3) — unifies speech, sound, and music understanding with long-context reasoning. +- [Ultravox](https://github.com/fixie-ai/ultravox) — a fast multimodal large language model designed for real-time voice interactions. +- [Ichigo](https://github.com/menloresearch/ichigo) — an audio-text-to-text model for audio-related tasks. + +### Datasets + +- [nvidia/AF-Think](https://huggingface.co/datasets/nvidia/AF-Think) +- [nvidia/AudioSkills](https://huggingface.co/datasets/nvidia/AudioSkills) + +### Tools & Extras + +- [Fast-RTC](https://huggingface.co/fastrtc) — turn any Python function into a real-time audio/video stream. +- [PhiCookBook](https://github.com/microsoft/PhiCookBook) — Microsoft’s open-source guide to small language models. +- [Qwen2-audio-instruct](https://qwenlm.github.io/blog/qwen2-audio/) — Blogpost explaining usage and demos of Qwen2-audio-instruct. diff --git a/node_modules/@huggingface/tasks/src/tasks/audio-text-to-text/data.ts b/node_modules/@huggingface/tasks/src/tasks/audio-text-to-text/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..dfc75f0104388ae93a6f5d8a77fcf5fd5c00476b --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/audio-text-to-text/data.ts @@ -0,0 +1,70 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A dataset containing audio conversations with question–answer pairs.", + id: "nvidia/AF-Think", + }, + { + description: "A more advanced and comprehensive dataset that contains characteristics of the audio as well", + id: "tsinghua-ee/QualiSpeech", + }, + ], + demo: { + inputs: [ + { + filename: "audio.wav", + type: "audio", + }, + { + label: "Text Prompt", + content: "What is the gender of the speaker?", + type: "text", + }, + ], + outputs: [ + { + label: "Generated Text", + content: "The gender of the speaker is female.", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: + "A lightweight model that has capabilities of taking both audio and text as inputs and generating responses.", + id: "fixie-ai/ultravox-v0_5-llama-3_2-1b", + }, + { + description: "A multimodal model that supports voice chat and audio analysis.", + id: "Qwen/Qwen2-Audio-7B-Instruct", + }, + { + description: "A model for audio understanding, speech translation, and transcription.", + id: "mistralai/Voxtral-Small-24B-2507", + }, + { + description: "A new model capable of audio question answering and reasoning.", + id: "nvidia/audio-flamingo-3", + }, + ], + spaces: [ + { + description: "A space that takes input as both audio and text and generates answers.", + id: "iamomtiwari/ATTT", + }, + { + description: "A web application that demonstrates chatting with the Qwen2Audio Model.", + id: "freddyaboulton/talk-to-qwen-webrtc", + }, + ], + summary: + "Audio-text-to-text models take both an audio clip and a text prompt as input, and generate natural language text as output. These models can answer questions about spoken content, summarize meetings, analyze music, or interpret speech beyond simple transcription. They are useful for applications that combine speech understanding with reasoning or conversation.", + widgetModels: [], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/audio-to-audio/about.md b/node_modules/@huggingface/tasks/src/tasks/audio-to-audio/about.md new file mode 100644 index 0000000000000000000000000000000000000000..eeda8c16afa49909aa4d2c868f9a87634978908e --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/audio-to-audio/about.md @@ -0,0 +1,56 @@ +## Use Cases + +### Speech Enhancement (Noise removal) + +Speech Enhancement is a bit self explanatory. It improves (or enhances) the quality of an audio by removing noise. There are multiple libraries to solve this task, such as Speechbrain, Asteroid and ESPNet. Here is a simple example using Speechbrain + +```python +from speechbrain.pretrained import SpectralMaskEnhancement +model = SpectralMaskEnhancement.from_hparams( + "speechbrain/mtl-mimic-voicebank" +) +model.enhance_file("file.wav") +``` + +Alternatively, you can use [Inference Endpoints](https://huggingface.co/inference-endpoints) to solve this task + +```python +import json +import requests + +headers = {"Authorization": f"Bearer {API_TOKEN}"} +API_URL = "https://router.huggingface.co/hf-inference/models/speechbrain/mtl-mimic-voicebank" + +def query(filename): + with open(filename, "rb") as f: + data = f.read() + response = requests.request("POST", API_URL, headers=headers, data=data) + return json.loads(response.content.decode("utf-8")) + +data = query("sample1.flac") +``` + +You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer with audio-to-audio models on Hugging Face Hub. + +```javascript +import { InferenceClient } from "@huggingface/inference"; + +const inference = new InferenceClient(HF_TOKEN); +await inference.audioToAudio({ + data: await (await fetch("sample.flac")).blob(), + model: "speechbrain/sepformer-wham", +}); +``` + +### Audio Source Separation + +Audio Source Separation allows you to isolate different sounds from individual sources. For example, if you have an audio file with multiple people speaking, you can get an audio file for each of them. You can then use an Automatic Speech Recognition system to extract the text from each of these sources as an initial step for your system! + +Audio-to-Audio can also be used to remove noise from audio files: you get one audio for the person speaking and another audio for the noise. This can also be useful when you have multi-person audio with some noise: yyou can get one audio for each person and then one audio for the noise. + +## Training a model for your own data + +If you want to learn how to train models for the Audio-to-Audio task, we recommend the following tutorials: + +- [Speech Enhancement](https://speechbrain.github.io/tutorial_enhancement.html) +- [Source Separation](https://speechbrain.github.io/tutorial_separation.html) diff --git a/node_modules/@huggingface/tasks/src/tasks/audio-to-audio/data.ts b/node_modules/@huggingface/tasks/src/tasks/audio-to-audio/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..cea0d5ee0b9b812d7a65ce8a1435ad23fc0dbce5 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/audio-to-audio/data.ts @@ -0,0 +1,66 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "512-element X-vector embeddings of speakers from CMU ARCTIC dataset.", + id: "Matthijs/cmu-arctic-xvectors", + }, + ], + demo: { + inputs: [ + { + filename: "input.wav", + type: "audio", + }, + ], + outputs: [ + { + filename: "label-0.wav", + type: "audio", + }, + { + filename: "label-1.wav", + type: "audio", + }, + ], + }, + metrics: [ + { + description: + "The Signal-to-Noise ratio is the relationship between the target signal level and the background noise level. It is calculated as the logarithm of the target signal divided by the background noise, in decibels.", + id: "snri", + }, + { + description: + "The Signal-to-Distortion ratio is the relationship between the target signal and the sum of noise, interference, and artifact errors", + id: "sdri", + }, + ], + models: [ + { + description: "A speech enhancement model.", + id: "ResembleAI/resemble-enhance", + }, + { + description: "A model that can change the voice in a speech recording.", + id: "microsoft/speecht5_vc", + }, + ], + spaces: [ + { + description: "An application for speech separation.", + id: "younver/speechbrain-speech-separation", + }, + { + description: "An application for audio style transfer.", + id: "nakas/audio-diffusion_style_transfer", + }, + ], + summary: + "Audio-to-Audio is a family of tasks in which the input is an audio and the output is one or multiple generated audios. Some example tasks are speech enhancement and source separation.", + widgetModels: ["speechbrain/sepformer-wham"], + youtubeId: "iohj7nCCYoM", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/about.md b/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/about.md new file mode 100644 index 0000000000000000000000000000000000000000..e1cab59719bfff5746bb9fb8b46213412dd8bd00 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/about.md @@ -0,0 +1,90 @@ +## Use Cases + +### Virtual Speech Assistants + +Many edge devices have an embedded virtual assistant to interact with the end users better. These assistances rely on ASR models to recognize different voice commands to perform various tasks. For instance, you can ask your phone for dialing a phone number, ask a general question, or schedule a meeting. + +### Caption Generation + +A caption generation model takes audio as input from sources to generate automatic captions through transcription, for live-streamed or recorded videos. This can help with content accessibility. For example, an audience watching a video that includes a non-native language, can rely on captions to interpret the content. It can also help with information retention at online-classes environments improving knowledge assimilation while reading and taking notes faster. + +## Task Variants + +### Multilingual ASR + +Multilingual ASR models can convert audio inputs with multiple languages into transcripts. Some multilingual ASR models include [language identification](https://huggingface.co/tasks/audio-classification) blocks to improve the performance. + +The use of Multilingual ASR has become popular, the idea of maintaining just a single model for all language can simplify the production pipeline. Take a look at [Whisper](https://huggingface.co/openai/whisper-large-v2) to get an idea on how 100+ languages can be processed by a single model. + +## Inference + +The Hub contains over [17,000 ASR models](https://huggingface.co/models?pipeline_tag=automatic-speech-recognition&sort=downloads) that you can test right away in your browser using the model page widgets. You can also use any model as a service using the Serverless Inference API. We also support libraries such as [transformers](https://huggingface.co/models?library=transformers&pipeline_tag=automatic-speech-recognition&sort=downloads), [speechbrain](https://huggingface.co/models?library=speechbrain&pipeline_tag=automatic-speech-recognition&sort=downloads), [NeMo](https://huggingface.co/models?pipeline_tag=automatic-speech-recognition&library=nemo&sort=downloads) and [espnet](https://huggingface.co/models?library=espnet&pipeline_tag=automatic-speech-recognition&sort=downloads) via the Serverless Inference API. Here's a simple code snippet to run inference: + +```python +import json +import requests + +headers = {"Authorization": f"Bearer {API_TOKEN}"} +API_URL = "https://router.huggingface.co/hf-inference/models/openai/whisper-large-v3" + +def query(filename): + with open(filename, "rb") as f: + data = f.read() + response = requests.request("POST", API_URL, headers=headers, data=data) + return json.loads(response.content.decode("utf-8")) + +data = query("sample1.flac") +``` + +You can also use [huggingface.js](https://github.com/huggingface/huggingface.js), the JavaScript client, to transcribe audio with the Serverless Inference API. + +```javascript +import { InferenceClient } from "@huggingface/inference"; + +const inference = new InferenceClient(HF_TOKEN); +await inference.automaticSpeechRecognition({ + data: await (await fetch("sample.flac")).blob(), + model: "openai/whisper-large-v3", +}); +``` + +For transformers-compatible models like Whisper, Wav2Vec2, and HuBERT, you can also run inference with the library as follows: + +```python +# pip install --upgrade transformers + +from transformers import pipeline + +pipe = pipeline("automatic-speech-recognition", "openai/whisper-large-v3") + +pipe("sample.flac") +# {'text': "GOING ALONG SLUSHY COUNTRY ROADS AND SPEAKING TO DAMP AUDIENCES IN DRAUGHTY SCHOOL ROOMS DAY AFTER DAY FOR A FORTNIGHT HE'LL HAVE TO PUT IN AN APPEARANCE AT SOME PLACE OF WORSHIP ON SUNDAY MORNING AND HE CAN COME TO US IMMEDIATELY AFTERWARDS"} +``` + +## Solving ASR for your own data + +We have some great news! You can fine-tune (transfer learning) a foundational speech model on a specific language without tonnes of data. Pretrained models such as Whisper, Wav2Vec2-MMS and HuBERT exist. [OpenAI's Whisper model](https://huggingface.co/openai/whisper-large-v3) is a large multilingual model trained on 100+ languages and with 4 Million hours of speech. + +The following detailed [blog post](https://huggingface.co/blog/fine-tune-whisper) shows how to fine-tune a pre-trained Whisper checkpoint on labeled data for ASR. With the right data and strategy you can fine-tune a high-performant model on a free Google Colab instance too. We suggest to read the blog post for more info! + +## Hugging Face Whisper Event + +On December 2022, over 450 participants collaborated, fine-tuned and shared 600+ ASR Whisper models in 100+ different languages. You can compare these models on the event's speech recognition [leaderboard](https://huggingface.co/spaces/whisper-event/leaderboard?dataset=mozilla-foundation%2Fcommon_voice_11_0&config=ar&split=test). + +These events help democratize ASR for all languages, including low-resource languages. In addition to the trained models, the [event](https://github.com/huggingface/community-events/tree/main/whisper-fine-tuning-event) helps to build practical collaborative knowledge. + +## Useful Resources + +- [Hugging Face Audio Course](https://huggingface.co/learn/audio-course/chapter5/introduction) +- [Fine-tuning MetaAI's MMS Adapter Models for Multi-Lingual ASR](https://huggingface.co/blog/mms_adapters) +- [Making automatic speech recognition work on large files with Wav2Vec2 in 🤗 Transformers](https://huggingface.co/blog/asr-chunking) +- [Boosting Wav2Vec2 with n-grams in 🤗 Transformers](https://huggingface.co/blog/wav2vec2-with-ngram) +- [ML for Audio Study Group - Intro to Audio and ASR Deep Dive](https://www.youtube.com/watch?v=D-MH6YjuIlE) +- [Massively Multilingual ASR: 50 Languages, 1 Model, 1 Billion Parameters](https://arxiv.org/pdf/2007.03001.pdf) +- An ASR toolkit made by [NVIDIA: NeMo](https://github.com/NVIDIA/NeMo) with code and pretrained models useful for new ASR models. Watch the [introductory video](https://www.youtube.com/embed/wBgpMf_KQVw) for an overview. +- [An introduction to SpeechT5, a multi-purpose speech recognition and synthesis model](https://huggingface.co/blog/speecht5) +- [Fine-tune Whisper For Multilingual ASR with 🤗Transformers](https://huggingface.co/blog/fine-tune-whisper) +- [Automatic speech recognition task guide](https://huggingface.co/docs/transformers/tasks/asr) +- [Speech Synthesis, Recognition, and More With SpeechT5](https://huggingface.co/blog/speecht5) +- [Fine-Tune W2V2-Bert for low-resource ASR with 🤗 Transformers](https://huggingface.co/blog/fine-tune-w2v2-bert) +- [Speculative Decoding for 2x Faster Whisper Inference](https://huggingface.co/blog/whisper-speculative-decoding) diff --git a/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/data.ts b/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..b177a7f123045e717efe832e24a5d790517ff3cb --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/data.ts @@ -0,0 +1,94 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "31,175 hours of multilingual audio-text dataset in 108 languages.", + id: "mozilla-foundation/common_voice_17_0", + }, + { + description: "Multilingual and diverse audio dataset with 101k hours of audio.", + id: "amphion/Emilia-Dataset", + }, + { + description: "A dataset with 44.6k hours of English speaker data and 6k hours of other language speakers.", + id: "parler-tts/mls_eng", + }, + { + description: "A multilingual audio dataset with 370K hours of audio.", + id: "espnet/yodas", + }, + ], + demo: { + inputs: [ + { + filename: "input.flac", + type: "audio", + }, + ], + outputs: [ + { + /// GOING ALONG SLUSHY COUNTRY ROADS AND SPEAKING TO DAMP AUDIENCES I + label: "Transcript", + content: "Going along slushy country roads and speaking to damp audiences in...", + type: "text", + }, + ], + }, + metrics: [ + { + description: "", + id: "wer", + }, + { + description: "", + id: "cer", + }, + ], + models: [ + { + description: "A powerful ASR model by OpenAI.", + id: "openai/whisper-large-v3", + }, + { + description: "A good generic speech model by MetaAI for fine-tuning.", + id: "facebook/w2v-bert-2.0", + }, + { + description: "An end-to-end model that performs ASR and Speech Translation by MetaAI.", + id: "facebook/seamless-m4t-v2-large", + }, + { + description: "A powerful multilingual ASR and Speech Translation model by Nvidia.", + id: "nvidia/canary-1b", + }, + { + description: "Powerful speaker diarization model.", + id: "pyannote/speaker-diarization-3.1", + }, + ], + spaces: [ + { + description: "A powerful general-purpose speech recognition application.", + id: "hf-audio/whisper-large-v3", + }, + { + description: "Latest ASR model from Useful Sensors.", + id: "mrfakename/Moonshinex", + }, + { + description: "A high quality speech and text translation model by Meta.", + id: "facebook/seamless_m4t", + }, + { + description: "A powerful multilingual ASR and Speech Translation model by Nvidia", + id: "nvidia/canary-1b", + }, + ], + summary: + "Automatic Speech Recognition (ASR), also known as Speech to Text (STT), is the task of transcribing a given audio to text. It has many applications, such as voice user interfaces.", + widgetModels: ["openai/whisper-large-v3"], + youtubeId: "TksaY_FDgnk", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/inference.ts b/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..664a22176769af9d798595b92a820299361b9dc8 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/inference.ts @@ -0,0 +1,150 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Automatic Speech Recognition inference + */ +export interface AutomaticSpeechRecognitionInput { + /** + * The input audio data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the audio data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Automatic Speech Recognition + */ + parameters?: AutomaticSpeechRecognitionParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Automatic Speech Recognition + */ +export interface AutomaticSpeechRecognitionParameters { + /** + * Parametrization of the text generation process + */ + generation_parameters?: GenerationParameters; + /** + * Whether to output corresponding timestamps with the generated text + */ + return_timestamps?: boolean; + [property: string]: unknown; +} +/** + * Parametrization of the text generation process + */ +export interface GenerationParameters { + /** + * Whether to use sampling instead of greedy decoding when generating new tokens. + */ + do_sample?: boolean; + /** + * Controls the stopping condition for beam-based methods. + */ + early_stopping?: EarlyStoppingUnion; + /** + * If set to float strictly between 0 and 1, only tokens with a conditional probability + * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + * Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + */ + epsilon_cutoff?: number; + /** + * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + * float strictly between 0 and 1, a token is only considered if it is greater than either + * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + * for more details. + */ + eta_cutoff?: number; + /** + * The maximum length (in tokens) of the generated text, including the input. + */ + max_length?: number; + /** + * The maximum number of tokens to generate. Takes precedence over max_length. + */ + max_new_tokens?: number; + /** + * The minimum length (in tokens) of the generated text, including the input. + */ + min_length?: number; + /** + * The minimum number of tokens to generate. Takes precedence over min_length. + */ + min_new_tokens?: number; + /** + * Number of groups to divide num_beams into in order to ensure diversity among different + * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + */ + num_beam_groups?: number; + /** + * Number of beams to use for beam search. + */ + num_beams?: number; + /** + * The value balances the model confidence and the degeneration penalty in contrastive + * search decoding. + */ + penalty_alpha?: number; + /** + * The value used to modulate the next token probabilities. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * If set to float < 1, only the smallest set of most probable tokens with probabilities + * that add up to top_p or higher are kept for generation. + */ + top_p?: number; + /** + * Local typicality measures how similar the conditional probability of predicting a target + * token next is to the expected conditional probability of predicting a random token next, + * given the partial text already generated. If set to float < 1, the smallest set of the + * most locally typical tokens with probabilities that add up to typical_p or higher are + * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + */ + typical_p?: number; + /** + * Whether the model should use the past last key/values attentions to speed up decoding + */ + use_cache?: boolean; + [property: string]: unknown; +} +/** + * Controls the stopping condition for beam-based methods. + */ +export type EarlyStoppingUnion = boolean | "never"; +/** + * Outputs of inference for the Automatic Speech Recognition task + */ +export interface AutomaticSpeechRecognitionOutput { + /** + * When returnTimestamps is enabled, chunks contains a list of audio chunks identified by + * the model. + */ + chunks?: AutomaticSpeechRecognitionOutputChunk[]; + /** + * The recognized text. + */ + text: string; + [property: string]: unknown; +} +export interface AutomaticSpeechRecognitionOutputChunk { + /** + * A chunk of text identified by the model + */ + text: string; + /** + * The start and end timestamps corresponding with the text + */ + timestamp: number[]; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..b1e050e75e8a4794c9499c87b2ba9fc1c10b8bb2 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/spec/input.json @@ -0,0 +1,35 @@ +{ + "$id": "/inference/schemas/automatic-speech-recognition/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Automatic Speech Recognition inference", + "title": "AutomaticSpeechRecognitionInput", + "type": "object", + "properties": { + "inputs": { + "description": "The input audio data as a base64-encoded string. If no `parameters` are provided, you can also provide the audio data as a raw bytes payload.", + "type": "string", + "comment": "type=binary" + }, + "parameters": { + "description": "Additional inference parameters for Automatic Speech Recognition", + "$ref": "#/$defs/AutomaticSpeechRecognitionParameters" + } + }, + "$defs": { + "AutomaticSpeechRecognitionParameters": { + "title": "AutomaticSpeechRecognitionParameters", + "type": "object", + "properties": { + "return_timestamps": { + "type": "boolean", + "description": "Whether to output corresponding timestamps with the generated text" + }, + "generation_parameters": { + "description": "Parametrization of the text generation process", + "$ref": "/inference/schemas/common-definitions.json#/definitions/GenerationParameters" + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..fd71699aa4ad069c54522d1e4a65ec9f84f273af --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/automatic-speech-recognition/spec/output.json @@ -0,0 +1,38 @@ +{ + "$id": "/inference/schemas/automatic-speech-recognition/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Automatic Speech Recognition task", + "title": "AutomaticSpeechRecognitionOutput", + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The recognized text." + }, + "chunks": { + "type": "array", + "description": "When returnTimestamps is enabled, chunks contains a list of audio chunks identified by the model.", + "items": { + "type": "object", + "title": "AutomaticSpeechRecognitionOutputChunk", + "properties": { + "text": { + "type": "string", + "description": "A chunk of text identified by the model" + }, + "timestamp": { + "type": "array", + "description": "The start and end timestamps corresponding with the text", + "items": { + "type": "number" + }, + "minLength": 2, + "maxLength": 2 + } + }, + "required": ["text", "timestamp"] + } + } + }, + "required": ["text"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/chat-completion/inference.ts b/node_modules/@huggingface/tasks/src/tasks/chat-completion/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..5777e23e25ab727f48bd02b9e0882cd2ec552a2a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/chat-completion/inference.ts @@ -0,0 +1,333 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Chat Completion Input. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface ChatCompletionInput { + /** + * Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing + * frequency in the text so far, + * decreasing the model's likelihood to repeat the same line verbatim. + */ + frequency_penalty?: number; + /** + * UNUSED + * Modify the likelihood of specified tokens appearing in the completion. Accepts a JSON + * object that maps tokens + * (specified by their token ID in the tokenizer) to an associated bias value from -100 to + * 100. Mathematically, + * the bias is added to the logits generated by the model prior to sampling. The exact + * effect will vary per model, + * but values between -1 and 1 should decrease or increase likelihood of selection; values + * like -100 or 100 should + * result in a ban or exclusive selection of the relevant token. + */ + logit_bias?: number[]; + /** + * Whether to return log probabilities of the output tokens or not. If true, returns the log + * probabilities of each + * output token returned in the content of message. + */ + logprobs?: boolean; + /** + * The maximum number of tokens that can be generated in the chat completion. + */ + max_tokens?: number; + /** + * A list of messages comprising the conversation so far. + */ + messages: ChatCompletionInputMessage[]; + /** + * [UNUSED] ID of the model to use. See the model endpoint compatibility table for details + * on which models work with the Chat API. + */ + model?: string; + /** + * UNUSED + * How many chat completion choices to generate for each input message. Note that you will + * be charged based on the + * number of generated tokens across all of the choices. Keep n as 1 to minimize costs. + */ + n?: number; + /** + * Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they + * appear in the text so far, + * increasing the model's likelihood to talk about new topics + */ + presence_penalty?: number; + /** + * Optional. Constrains effort on reasoning for reasoning models. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning. Common values: none, minimal, low, medium, high, xhigh. Support and defaults are provider and model-dependent. + */ + reasoning_effort?: string; + response_format?: ChatCompletionInputGrammarType; + seed?: number; + /** + * Up to 4 sequences where the API will stop generating further tokens. + */ + stop?: string[]; + stream?: boolean; + stream_options?: ChatCompletionInputStreamOptions; + /** + * What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the + * output more random, while + * lower values like 0.2 will make it more focused and deterministic. + * + * We generally recommend altering this or `top_p` but not both. + */ + temperature?: number; + tool_choice?: ChatCompletionInputToolChoice; + /** + * A prompt to be appended before the tools + */ + tool_prompt?: string; + /** + * A list of tools the model may call. Currently, only functions are supported as a tool. + * Use this to provide a list of + * functions the model may generate JSON inputs for. + */ + tools?: ChatCompletionInputTool[]; + /** + * An integer between 0 and 5 specifying the number of most likely tokens to return at each + * token position, each with + * an associated log probability. logprobs must be set to true if this parameter is used. + */ + top_logprobs?: number; + /** + * An alternative to sampling with temperature, called nucleus sampling, where the model + * considers the results of the + * tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% + * probability mass are considered. + */ + top_p?: number; + [property: string]: unknown; +} +export interface ChatCompletionInputMessage { + content?: ChatCompletionInputMessageContent; + name?: string; + role: string; + tool_calls?: ChatCompletionInputToolCall[]; + [property: string]: unknown; +} +export type ChatCompletionInputMessageContent = ChatCompletionInputMessageChunk[] | string; +export interface ChatCompletionInputMessageChunk { + image_url?: ChatCompletionInputURL; + text?: string; + type: ChatCompletionInputMessageChunkType; + [property: string]: unknown; +} +export interface ChatCompletionInputURL { + url: string; + [property: string]: unknown; +} +export type ChatCompletionInputMessageChunkType = "text" | "image_url"; +export interface ChatCompletionInputToolCall { + function: ChatCompletionInputFunctionDefinition; + id: string; + type: string; + [property: string]: unknown; +} +export interface ChatCompletionInputFunctionDefinition { + description?: string; + name: string; + parameters?: unknown; + [property: string]: unknown; +} +export interface ChatCompletionInputGrammarType { + json_schema?: ChatCompletionInputJSONSchemaConfig; + type: ChatCompletionInputGrammarTypeType; + [property: string]: unknown; +} +export interface ChatCompletionInputJSONSchemaConfig { + /** + * A description of what the response format is for, used by the model to determine how to + * respond in the format. + */ + description?: string; + /** + * The name of the response format. + */ + name: string; + /** + * The schema for the response format, described as a JSON Schema object. Learn how to build + * JSON schemas [here](https://json-schema.org/). + */ + schema?: { + [key: string]: unknown; + }; + /** + * Whether to enable strict schema adherence when generating the output. If set to true, the + * model will always follow the exact schema defined in the `schema` field. + */ + strict?: boolean; + [property: string]: unknown; +} +export type ChatCompletionInputGrammarTypeType = "text" | "json_schema" | "json_object"; +export interface ChatCompletionInputStreamOptions { + /** + * If set, an additional chunk will be streamed before the data: [DONE] message. The usage + * field on this chunk shows the token usage statistics for the entire request, and the + * choices field will always be an empty array. All other chunks will also include a usage + * field, but with a null value. + */ + include_usage?: boolean; + [property: string]: unknown; +} +/** + * + * + */ +export type ChatCompletionInputToolChoice = ChatCompletionInputToolChoiceEnum | ChatCompletionInputToolChoiceObject; +/** + * Means the model can pick between generating a message or calling one or more tools. + * + * Means the model will not call any tool and instead generates a message. + * + * Means the model must call one or more tools. + */ +export type ChatCompletionInputToolChoiceEnum = "auto" | "none" | "required"; +export interface ChatCompletionInputToolChoiceObject { + function: ChatCompletionInputFunctionName; + [property: string]: unknown; +} +export interface ChatCompletionInputFunctionName { + name: string; + [property: string]: unknown; +} +export interface ChatCompletionInputTool { + function: ChatCompletionInputFunctionDefinition; + type: string; + [property: string]: unknown; +} +/** + * Chat Completion Output. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface ChatCompletionOutput { + choices: ChatCompletionOutputComplete[]; + created: number; + id: string; + model: string; + system_fingerprint: string; + usage: ChatCompletionOutputUsage; + [property: string]: unknown; +} +export interface ChatCompletionOutputComplete { + finish_reason: string; + index: number; + logprobs?: ChatCompletionOutputLogprobs; + message: ChatCompletionOutputMessage; + [property: string]: unknown; +} +export interface ChatCompletionOutputLogprobs { + content: ChatCompletionOutputLogprob[]; + [property: string]: unknown; +} +export interface ChatCompletionOutputLogprob { + logprob: number; + token: string; + top_logprobs: ChatCompletionOutputTopLogprob[]; + [property: string]: unknown; +} +export interface ChatCompletionOutputTopLogprob { + logprob: number; + token: string; + [property: string]: unknown; +} +export interface ChatCompletionOutputMessage { + content?: string; + role: string; + tool_call_id?: string; + tool_calls?: ChatCompletionOutputToolCall[]; + [property: string]: unknown; +} +export interface ChatCompletionOutputToolCall { + function: ChatCompletionOutputFunctionDefinition; + id: string; + type: string; + [property: string]: unknown; +} +export interface ChatCompletionOutputFunctionDefinition { + arguments: string; + description?: string; + name: string; + [property: string]: unknown; +} +export interface ChatCompletionOutputUsage { + completion_tokens: number; + prompt_tokens: number; + total_tokens: number; + [property: string]: unknown; +} +/** + * Chat Completion Stream Output. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface ChatCompletionStreamOutput { + choices: ChatCompletionStreamOutputChoice[]; + created: number; + id: string; + model: string; + system_fingerprint: string; + usage?: ChatCompletionStreamOutputUsage; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputChoice { + delta: ChatCompletionStreamOutputDelta; + finish_reason?: string; + index: number; + logprobs?: ChatCompletionStreamOutputLogprobs; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputDelta { + content?: string; + role: string; + tool_call_id?: string; + tool_calls?: ChatCompletionStreamOutputDeltaToolCall[]; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputDeltaToolCall { + function: ChatCompletionStreamOutputFunction; + id: string; + index: number; + type: string; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputFunction { + arguments: string; + name?: string; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputLogprobs { + content: ChatCompletionStreamOutputLogprob[]; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputLogprob { + logprob: number; + token: string; + top_logprobs: ChatCompletionStreamOutputTopLogprob[]; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputTopLogprob { + logprob: number; + token: string; + [property: string]: unknown; +} +export interface ChatCompletionStreamOutputUsage { + completion_tokens: number; + prompt_tokens: number; + total_tokens: number; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/chat-completion/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/chat-completion/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..6402159bda6bc9a5a77b42faf9681ac3de18a9cf --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/chat-completion/spec/input.json @@ -0,0 +1,438 @@ +{ + "$id": "/inference/schemas/chat-completion/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Chat Completion Input.\n\nAuto-generated from TGI specs.\nFor more details, check out https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts.", + "title": "ChatCompletionInput", + "type": "object", + "required": ["messages"], + "properties": { + "frequency_penalty": { + "type": "number", + "format": "float", + "description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far,\ndecreasing the model's likelihood to repeat the same line verbatim.", + "example": "1.0", + "nullable": true + }, + "logit_bias": { + "type": "array", + "items": { + "type": "number", + "format": "float" + }, + "description": "UNUSED\nModify the likelihood of specified tokens appearing in the completion. Accepts a JSON object that maps tokens\n(specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically,\nthe bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model,\nbut values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should\nresult in a ban or exclusive selection of the relevant token.", + "nullable": true + }, + "logprobs": { + "type": "boolean", + "description": "Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each\noutput token returned in the content of message.", + "example": "false", + "nullable": true + }, + "max_tokens": { + "type": "integer", + "format": "int32", + "description": "The maximum number of tokens that can be generated in the chat completion.", + "default": "1024", + "example": "32", + "nullable": true, + "minimum": 0 + }, + "messages": { + "type": "array", + "items": { + "$ref": "#/$defs/ChatCompletionInputMessage" + }, + "description": "A list of messages comprising the conversation so far.", + "example": "[{\"role\": \"user\", \"content\": \"What is Deep Learning?\"}]" + }, + "model": { + "type": "string", + "description": "[UNUSED] ID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.", + "example": "mistralai/Mistral-7B-Instruct-v0.2", + "nullable": true + }, + "n": { + "type": "integer", + "format": "int32", + "description": "UNUSED\nHow many chat completion choices to generate for each input message. Note that you will be charged based on the\nnumber of generated tokens across all of the choices. Keep n as 1 to minimize costs.", + "example": "2", + "nullable": true, + "minimum": 0 + }, + "presence_penalty": { + "type": "number", + "format": "float", + "description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far,\nincreasing the model's likelihood to talk about new topics", + "example": 0.1, + "nullable": true + }, + "reasoning_effort": { + "type": "string", + "description": "Optional. Constrains effort on reasoning for models that support reasoning. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning. Common values: none, minimal, low, medium, high, xhigh. Support and defaults are provider and model-dependent.", + "nullable": true + }, + "response_format": { + "allOf": [ + { + "$ref": "#/$defs/ChatCompletionInputGrammarType" + } + ], + "default": "null", + "nullable": true + }, + "seed": { + "type": "integer", + "format": "int64", + "example": 42, + "nullable": true, + "minimum": 0 + }, + "stop": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Up to 4 sequences where the API will stop generating further tokens.", + "example": "null", + "nullable": true + }, + "stream": { + "type": "boolean" + }, + "stream_options": { + "allOf": [ + { + "$ref": "#/$defs/ChatCompletionInputStreamOptions" + } + ], + "nullable": true + }, + "temperature": { + "type": "number", + "format": "float", + "description": "What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while\nlower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or `top_p` but not both.", + "example": 1, + "nullable": true + }, + "tool_choice": { + "allOf": [ + { + "$ref": "#/$defs/ChatCompletionInputToolChoice" + } + ], + "default": "auto", + "nullable": true + }, + "tool_prompt": { + "type": "string", + "description": "A prompt to be appended before the tools", + "example": "Given the functions available, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. Respond in the format {name: function name, parameters: dictionary of argument name and its value}.Do not use variables.", + "nullable": true + }, + "tools": { + "type": "array", + "items": { + "$ref": "#/$defs/ChatCompletionInputTool" + }, + "description": "A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of\nfunctions the model may generate JSON inputs for.", + "example": "null", + "nullable": true + }, + "top_logprobs": { + "type": "integer", + "format": "int32", + "description": "An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with\nan associated log probability. logprobs must be set to true if this parameter is used.", + "example": "5", + "nullable": true, + "minimum": 0 + }, + "top_p": { + "type": "number", + "format": "float", + "description": "An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the\ntokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.", + "example": 0.95, + "nullable": true + } + }, + "$defs": { + "ChatCompletionInputMessage": { + "allOf": [ + { + "$ref": "#/$defs/ChatCompletionInputMessageBody" + }, + { + "type": "object", + "required": ["role"], + "properties": { + "name": { + "type": "string", + "example": "\"David\"", + "nullable": true + }, + "role": { + "type": "string", + "example": "user" + } + } + } + ], + "title": "ChatCompletionInputMessage" + }, + "ChatCompletionInputMessageBody": { + "oneOf": [ + { + "type": "object", + "required": ["content"], + "properties": { + "content": { + "$ref": "#/$defs/ChatCompletionInputMessageContent" + } + } + }, + { + "type": "object", + "required": ["tool_calls"], + "properties": { + "tool_calls": { + "type": "array", + "items": { + "$ref": "#/$defs/ChatCompletionInputToolCall" + } + } + } + } + ], + "title": "ChatCompletionInputMessageBody" + }, + "ChatCompletionInputMessageContent": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/ChatCompletionInputMessageChunk" + } + } + ], + "title": "ChatCompletionInputMessageContent" + }, + "ChatCompletionInputMessageChunk": { + "oneOf": [ + { + "type": "object", + "required": ["text", "type"], + "properties": { + "text": { + "type": "string" + }, + "type": { + "type": "string", + "enum": ["text"] + } + } + }, + { + "type": "object", + "required": ["image_url", "type"], + "properties": { + "image_url": { + "$ref": "#/$defs/ChatCompletionInputUrl" + }, + "type": { + "type": "string", + "enum": ["image_url"] + } + } + } + ], + "discriminator": { + "propertyName": "type" + }, + "title": "ChatCompletionInputMessageChunk" + }, + "ChatCompletionInputUrl": { + "type": "object", + "required": ["url"], + "properties": { + "url": { + "type": "string" + } + }, + "title": "ChatCompletionInputUrl" + }, + "ChatCompletionInputToolCall": { + "type": "object", + "required": ["id", "type", "function"], + "properties": { + "function": { + "$ref": "#/$defs/ChatCompletionInputFunctionDefinition" + }, + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "title": "ChatCompletionInputToolCall" + }, + "ChatCompletionInputFunctionDefinition": { + "type": "object", + "required": ["name"], + "properties": { + "parameters": {}, + "description": { + "type": "string", + "nullable": true + }, + "name": { + "type": "string" + } + }, + "title": "ChatCompletionInputFunctionDefinition" + }, + "ChatCompletionInputGrammarType": { + "oneOf": [ + { + "$ref": "#/$defs/ChatCompletionInputResponseFormatText" + }, + { + "$ref": "#/$defs/ChatCompletionInputResponseFormatJSONSchema" + }, + { + "$ref": "#/$defs/ChatCompletionInputResponseFormatJSONObject" + } + ], + "title": "ChatCompletionInputGrammarType" + }, + "ChatCompletionInputResponseFormatText": { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["text"] + } + }, + "title": "ChatCompletionInputResponseFormatText" + }, + "ChatCompletionInputResponseFormatJSONSchema": { + "type": "object", + "required": ["type", "json_schema"], + "properties": { + "type": { + "type": "string", + "enum": ["json_schema"] + }, + "json_schema": { + "$ref": "#/$defs/ChatCompletionInputJsonSchemaConfig" + } + }, + "title": "ChatCompletionInputResponseFormatJSONSchema" + }, + "ChatCompletionInputResponseFormatJSONObject": { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": ["json_object"] + } + }, + "title": "ChatCompletionInputResponseFormatJSONObject" + }, + "ChatCompletionInputJsonSchemaConfig": { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string", + "description": "The name of the response format." + }, + "description": { + "type": "string", + "description": "A description of what the response format is for, used by the model to determine how to respond in the format.", + "nullable": true + }, + "schema": { + "type": "object", + "description": "The schema for the response format, described as a JSON Schema object. Learn how to build JSON schemas [here](https://json-schema.org/).", + "nullable": true + }, + "strict": { + "type": "boolean", + "description": "Whether to enable strict schema adherence when generating the output. If set to true, the model will always follow the exact schema defined in the `schema` field.", + "nullable": true + } + }, + "title": "ChatCompletionInputJsonSchemaConfig" + }, + "ChatCompletionInputStreamOptions": { + "type": "object", + "properties": { + "include_usage": { + "type": "boolean", + "description": "If set, an additional chunk will be streamed before the data: [DONE] message. The usage field on this chunk shows the token usage statistics for the entire request, and the choices field will always be an empty array. All other chunks will also include a usage field, but with a null value.", + "example": "true" + } + }, + "title": "ChatCompletionInputStreamOptions" + }, + "ChatCompletionInputToolChoice": { + "oneOf": [ + { + "type": "string", + "description": "Means the model can pick between generating a message or calling one or more tools.", + "enum": ["auto"] + }, + { + "type": "string", + "description": "Means the model will not call any tool and instead generates a message.", + "enum": ["none"] + }, + { + "type": "string", + "description": "Means the model must call one or more tools.", + "enum": ["required"] + }, + { + "type": "object", + "required": ["function"], + "properties": { + "function": { + "$ref": "#/$defs/ChatCompletionInputFunctionName" + } + } + } + ], + "description": "", + "title": "ChatCompletionInputToolChoice" + }, + "ChatCompletionInputFunctionName": { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string" + } + }, + "title": "ChatCompletionInputFunctionName" + }, + "ChatCompletionInputTool": { + "type": "object", + "required": ["type", "function"], + "properties": { + "function": { + "$ref": "#/$defs/ChatCompletionInputFunctionDefinition" + }, + "type": { + "type": "string", + "example": "function" + } + }, + "title": "ChatCompletionInputTool" + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/chat-completion/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/chat-completion/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..759d657badf96b416f794720fd2328472acd7b36 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/chat-completion/spec/output.json @@ -0,0 +1,212 @@ +{ + "$id": "/inference/schemas/chat-completion/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Chat Completion Output.\n\nAuto-generated from TGI specs.\nFor more details, check out https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts.", + "title": "ChatCompletionOutput", + "type": "object", + "required": ["id", "created", "model", "system_fingerprint", "choices", "usage"], + "properties": { + "choices": { + "type": "array", + "items": { + "$ref": "#/$defs/ChatCompletionOutputComplete" + } + }, + "created": { + "type": "integer", + "format": "int64", + "example": "1706270835", + "minimum": 0 + }, + "id": { + "type": "string" + }, + "model": { + "type": "string", + "example": "mistralai/Mistral-7B-Instruct-v0.2" + }, + "system_fingerprint": { + "type": "string" + }, + "usage": { + "$ref": "#/$defs/ChatCompletionOutputUsage" + } + }, + "$defs": { + "ChatCompletionOutputComplete": { + "type": "object", + "required": ["index", "message", "finish_reason"], + "properties": { + "finish_reason": { + "type": "string" + }, + "index": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "logprobs": { + "allOf": [ + { + "$ref": "#/$defs/ChatCompletionOutputLogprobs" + } + ], + "nullable": true + }, + "message": { + "$ref": "#/$defs/ChatCompletionOutputMessage" + } + }, + "title": "ChatCompletionOutputComplete" + }, + "ChatCompletionOutputLogprobs": { + "type": "object", + "required": ["content"], + "properties": { + "content": { + "type": "array", + "items": { + "$ref": "#/$defs/ChatCompletionOutputLogprob" + } + } + }, + "title": "ChatCompletionOutputLogprobs" + }, + "ChatCompletionOutputLogprob": { + "type": "object", + "required": ["token", "logprob", "top_logprobs"], + "properties": { + "logprob": { + "type": "number", + "format": "float" + }, + "token": { + "type": "string" + }, + "top_logprobs": { + "type": "array", + "items": { + "$ref": "#/$defs/ChatCompletionOutputTopLogprob" + } + } + }, + "title": "ChatCompletionOutputLogprob" + }, + "ChatCompletionOutputTopLogprob": { + "type": "object", + "required": ["token", "logprob"], + "properties": { + "logprob": { + "type": "number", + "format": "float" + }, + "token": { + "type": "string" + } + }, + "title": "ChatCompletionOutputTopLogprob" + }, + "ChatCompletionOutputMessage": { + "oneOf": [ + { + "$ref": "#/$defs/ChatCompletionOutputTextMessage" + }, + { + "$ref": "#/$defs/ChatCompletionOutputToolCallMessage" + } + ], + "title": "ChatCompletionOutputMessage" + }, + "ChatCompletionOutputTextMessage": { + "type": "object", + "required": ["role", "content"], + "properties": { + "content": { + "type": "string", + "example": "My name is David and I" + }, + "role": { + "type": "string", + "example": "user" + }, + "tool_call_id": { + "type": "string", + "nullable": true + } + }, + "title": "ChatCompletionOutputTextMessage" + }, + "ChatCompletionOutputToolCallMessage": { + "type": "object", + "required": ["role", "tool_calls"], + "properties": { + "role": { + "type": "string", + "example": "assistant" + }, + "tool_calls": { + "type": "array", + "items": { + "$ref": "#/$defs/ChatCompletionOutputToolCall" + } + } + }, + "title": "ChatCompletionOutputToolCallMessage" + }, + "ChatCompletionOutputToolCall": { + "type": "object", + "required": ["id", "type", "function"], + "properties": { + "function": { + "$ref": "#/$defs/ChatCompletionOutputFunctionDefinition" + }, + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "title": "ChatCompletionOutputToolCall" + }, + "ChatCompletionOutputFunctionDefinition": { + "type": "object", + "required": ["name", "arguments"], + "properties": { + "arguments": { + "type": "string" + }, + "description": { + "type": "string", + "nullable": true + }, + "name": { + "type": "string" + } + }, + "title": "ChatCompletionOutputFunctionDefinition" + }, + "ChatCompletionOutputUsage": { + "type": "object", + "required": ["prompt_tokens", "completion_tokens", "total_tokens"], + "properties": { + "completion_tokens": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "prompt_tokens": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "total_tokens": { + "type": "integer", + "format": "int32", + "minimum": 0 + } + }, + "title": "ChatCompletionOutputUsage" + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/chat-completion/spec/stream_output.json b/node_modules/@huggingface/tasks/src/tasks/chat-completion/spec/stream_output.json new file mode 100644 index 0000000000000000000000000000000000000000..e5b382696b9ad141dd2628cbaa14905c7de2eda8 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/chat-completion/spec/stream_output.json @@ -0,0 +1,220 @@ +{ + "$id": "/inference/schemas/chat-completion/stream_output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Chat Completion Stream Output.\n\nAuto-generated from TGI specs.\nFor more details, check out https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts.", + "title": "ChatCompletionStreamOutput", + "type": "object", + "required": ["id", "created", "model", "system_fingerprint", "choices"], + "properties": { + "choices": { + "type": "array", + "items": { + "$ref": "#/$defs/ChatCompletionStreamOutputChoice" + } + }, + "created": { + "type": "integer", + "format": "int64", + "example": "1706270978", + "minimum": 0 + }, + "id": { + "type": "string" + }, + "model": { + "type": "string", + "example": "mistralai/Mistral-7B-Instruct-v0.2" + }, + "system_fingerprint": { + "type": "string" + }, + "usage": { + "allOf": [ + { + "$ref": "#/$defs/ChatCompletionStreamOutputUsage" + } + ], + "nullable": true + } + }, + "$defs": { + "ChatCompletionStreamOutputChoice": { + "type": "object", + "required": ["index", "delta"], + "properties": { + "delta": { + "$ref": "#/$defs/ChatCompletionStreamOutputDelta" + }, + "finish_reason": { + "type": "string", + "nullable": true + }, + "index": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "logprobs": { + "allOf": [ + { + "$ref": "#/$defs/ChatCompletionStreamOutputLogprobs" + } + ], + "nullable": true + } + }, + "title": "ChatCompletionStreamOutputChoice" + }, + "ChatCompletionStreamOutputDelta": { + "oneOf": [ + { + "$ref": "#/$defs/ChatCompletionStreamOutputTextMessage" + }, + { + "$ref": "#/$defs/ChatCompletionStreamOutputToolCallDelta" + } + ], + "title": "ChatCompletionStreamOutputDelta" + }, + "ChatCompletionStreamOutputTextMessage": { + "type": "object", + "required": ["role", "content"], + "properties": { + "content": { + "type": "string", + "example": "My name is David and I" + }, + "role": { + "type": "string", + "example": "user" + }, + "tool_call_id": { + "type": "string", + "nullable": true + } + }, + "title": "ChatCompletionStreamOutputTextMessage" + }, + "ChatCompletionStreamOutputToolCallDelta": { + "type": "object", + "required": ["role", "tool_calls"], + "properties": { + "role": { + "type": "string", + "example": "assistant" + }, + "tool_calls": { + "type": "array", + "items": { + "$ref": "#/$defs/ChatCompletionStreamOutputDeltaToolCall" + } + } + }, + "title": "ChatCompletionStreamOutputToolCallDelta" + }, + "ChatCompletionStreamOutputDeltaToolCall": { + "type": "object", + "required": ["index", "id", "type", "function"], + "properties": { + "function": { + "$ref": "#/$defs/ChatCompletionStreamOutputFunction" + }, + "id": { + "type": "string" + }, + "index": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "type": { + "type": "string" + } + }, + "title": "ChatCompletionStreamOutputDeltaToolCall" + }, + "ChatCompletionStreamOutputFunction": { + "type": "object", + "required": ["arguments"], + "properties": { + "arguments": { + "type": "string" + }, + "name": { + "type": "string", + "nullable": true + } + }, + "title": "ChatCompletionStreamOutputFunction" + }, + "ChatCompletionStreamOutputLogprobs": { + "type": "object", + "required": ["content"], + "properties": { + "content": { + "type": "array", + "items": { + "$ref": "#/$defs/ChatCompletionStreamOutputLogprob" + } + } + }, + "title": "ChatCompletionStreamOutputLogprobs" + }, + "ChatCompletionStreamOutputLogprob": { + "type": "object", + "required": ["token", "logprob", "top_logprobs"], + "properties": { + "logprob": { + "type": "number", + "format": "float" + }, + "token": { + "type": "string" + }, + "top_logprobs": { + "type": "array", + "items": { + "$ref": "#/$defs/ChatCompletionStreamOutputTopLogprob" + } + } + }, + "title": "ChatCompletionStreamOutputLogprob" + }, + "ChatCompletionStreamOutputTopLogprob": { + "type": "object", + "required": ["token", "logprob"], + "properties": { + "logprob": { + "type": "number", + "format": "float" + }, + "token": { + "type": "string" + } + }, + "title": "ChatCompletionStreamOutputTopLogprob" + }, + "ChatCompletionStreamOutputUsage": { + "type": "object", + "required": ["prompt_tokens", "completion_tokens", "total_tokens"], + "properties": { + "completion_tokens": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "prompt_tokens": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "total_tokens": { + "type": "integer", + "format": "int32", + "minimum": 0 + } + }, + "title": "ChatCompletionStreamOutputUsage" + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/common-definitions.json b/node_modules/@huggingface/tasks/src/tasks/common-definitions.json new file mode 100644 index 0000000000000000000000000000000000000000..a9603fc067fb6623f47f3e2e32f7366c3013bbcc --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/common-definitions.json @@ -0,0 +1,99 @@ +{ + "$id": "/inference/schemas/common-definitions.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "(Incomplete!) Common type definitions shared by several tasks", + "definitions": { + "ClassificationOutputTransform": { + "title": "ClassificationOutputTransform", + "type": "string", + "description": "The function to apply to the model outputs in order to retrieve the scores.", + "enum": ["sigmoid", "softmax", "none"] + }, + "ClassificationOutput": { + "title": "ClassificationOutput", + "type": "object", + "properties": { + "label": { + "type": "string", + "description": "The predicted class label." + }, + "score": { + "type": "number", + "description": "The corresponding probability." + } + }, + "required": ["label", "score"] + }, + "GenerationParameters": { + "title": "GenerationParameters", + "type": "object", + "properties": { + "temperature": { + "type": "number", + "description": "The value used to modulate the next token probabilities." + }, + "top_k": { + "type": "integer", + "description": "The number of highest probability vocabulary tokens to keep for top-k-filtering." + }, + "top_p": { + "type": "number", + "description": "If set to float < 1, only the smallest set of most probable tokens with probabilities that add up to top_p or higher are kept for generation." + }, + "typical_p": { + "type": "number", + "description": " Local typicality measures how similar the conditional probability of predicting a target token next is to the expected conditional probability of predicting a random token next, given the partial text already generated. If set to float < 1, the smallest set of the most locally typical tokens with probabilities that add up to typical_p or higher are kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details." + }, + "epsilon_cutoff": { + "type": "number", + "description": "If set to float strictly between 0 and 1, only tokens with a conditional probability greater than epsilon_cutoff will be sampled. In the paper, suggested values range from 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) for more details." + }, + "eta_cutoff": { + "type": "number", + "description": "Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to float strictly between 0 and 1, a token is only considered if it is greater than either eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) for more details." + }, + "max_length": { + "type": "integer", + "description": "The maximum length (in tokens) of the generated text, including the input." + }, + "max_new_tokens": { + "type": "integer", + "description": "The maximum number of tokens to generate. Takes precedence over max_length." + }, + "min_length": { + "type": "integer", + "description": "The minimum length (in tokens) of the generated text, including the input." + }, + "min_new_tokens": { + "type": "integer", + "description": "The minimum number of tokens to generate. Takes precedence over min_length." + }, + "do_sample": { + "type": "boolean", + "description": "Whether to use sampling instead of greedy decoding when generating new tokens." + }, + "early_stopping": { + "type": ["boolean", "string"], + "description": "Controls the stopping condition for beam-based methods.", + "enum": ["never", true, false] + }, + "num_beams": { + "type": "integer", + "description": "Number of beams to use for beam search." + }, + "num_beam_groups": { + "type": "integer", + "description": "Number of groups to divide num_beams into in order to ensure diversity among different groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details." + }, + "penalty_alpha": { + "type": "number", + "description": "The value balances the model confidence and the degeneration penalty in contrastive search decoding." + }, + "use_cache": { + "type": "boolean", + "description": "Whether the model should use the past last key/values attentions to speed up decoding" + } + } + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/depth-estimation/about.md b/node_modules/@huggingface/tasks/src/tasks/depth-estimation/about.md new file mode 100644 index 0000000000000000000000000000000000000000..37c7d85a0991522e3bc5e6d06986f671cdc874d1 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/depth-estimation/about.md @@ -0,0 +1,45 @@ +## Use Cases + +Depth estimation models can be used to estimate the depth of different objects present in an image. + +### Estimation of Volumetric Information +Depth estimation models are widely used to study volumetric formation of objects present inside an image. This is an important use case in the domain of computer graphics. + +### 3D Representation + +Depth estimation models can also be used to develop a 3D representation from a 2D image. + +## Depth Estimation Subtasks + +There are two depth estimation subtasks. + +- **Absolute depth estimation**: Absolute (or metric) depth estimation aims to provide exact depth measurements from the camera. Absolute depth estimation models output depth maps with real-world distances in meter or feet. + +- **Relative depth estimation**: Relative depth estimation aims to predict the depth order of objects or points in a scene without providing the precise measurements. + +## Inference + +With the `transformers` library, you can use the `depth-estimation` pipeline to infer with image classification models. You can initialize the pipeline with a model id from the Hub. If you do not provide a model id it will initialize with [Intel/dpt-large](https://huggingface.co/Intel/dpt-large) by default. When calling the pipeline you just need to specify a path, http link or an image loaded in PIL. Additionally, you can find a comprehensive list of various depth estimation models at [this link](https://huggingface.co/models?pipeline_tag=depth-estimation). + +```python +from transformers import pipeline + +estimator = pipeline(task="depth-estimation", model="Intel/dpt-large") +result = estimator(images="http://images.cocodataset.org/val2017/000000039769.jpg") +result + +# {'predicted_depth': tensor([[[ 6.3199, 6.3629, 6.4148, ..., 10.4104, 10.5109, 10.3847], +# [ 6.3850, 6.3615, 6.4166, ..., 10.4540, 10.4384, 10.4554], +# [ 6.3519, 6.3176, 6.3575, ..., 10.4247, 10.4618, 10.4257], +# ..., +# [22.3772, 22.4624, 22.4227, ..., 22.5207, 22.5593, 22.5293], +# [22.5073, 22.5148, 22.5114, ..., 22.6604, 22.6344, 22.5871], +# [22.5176, 22.5275, 22.5218, ..., 22.6282, 22.6216, 22.6108]]]), +# 'depth': } + +# You can visualize the result just by calling `result["depth"]`. +``` + +## Useful Resources + +- [Monocular depth estimation task guide](https://huggingface.co/docs/transformers/tasks/monocular_depth_estimation) diff --git a/node_modules/@huggingface/tasks/src/tasks/depth-estimation/data.ts b/node_modules/@huggingface/tasks/src/tasks/depth-estimation/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..a20ec9facdb8eeb18afe0d9601a90b99d0c236f9 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/depth-estimation/data.ts @@ -0,0 +1,70 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "NYU Depth V2 Dataset: Video dataset containing both RGB and depth sensor data.", + id: "sayakpaul/nyu_depth_v2", + }, + { + description: "Monocular depth estimation benchmark based without noise and errors.", + id: "depth-anything/DA-2K", + }, + ], + demo: { + inputs: [ + { + filename: "depth-estimation-input.jpg", + type: "img", + }, + ], + outputs: [ + { + filename: "depth-estimation-output.png", + type: "img", + }, + ], + }, + metrics: [], + models: [ + { + description: "Cutting-edge depth estimation model.", + id: "depth-anything/Depth-Anything-V2-Large", + }, + { + description: "A strong monocular depth estimation model.", + id: "jingheya/lotus-depth-g-v1-0", + }, + { + description: "A depth estimation model that predicts depth in videos.", + id: "tencent/DepthCrafter", + }, + { + description: "A robust depth estimation model.", + id: "apple/DepthPro-hf", + }, + ], + spaces: [ + { + description: "An application that predicts the depth of an image and then reconstruct the 3D model as voxels.", + id: "radames/dpt-depth-estimation-3d-voxels", + }, + { + description: "An application for bleeding-edge depth estimation.", + id: "akhaliq/depth-pro", + }, + { + description: "An application on cutting-edge depth estimation in videos.", + id: "tencent/DepthCrafter", + }, + { + description: "A human-centric depth estimation application.", + id: "facebook/sapiens-depth", + }, + ], + summary: "Depth estimation is the task of predicting depth of the objects present in an image.", + widgetModels: [""], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/depth-estimation/inference.ts b/node_modules/@huggingface/tasks/src/tasks/depth-estimation/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..0e81e8de28c9ce934d9c1a534cc5c52b8a737f7b --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/depth-estimation/inference.ts @@ -0,0 +1,35 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Depth Estimation inference + */ +export interface DepthEstimationInput { + /** + * The input image data + */ + inputs: unknown; + /** + * Additional inference parameters for Depth Estimation + */ + parameters?: { + [key: string]: unknown; + }; + [property: string]: unknown; +} +/** + * Outputs of inference for the Depth Estimation task + */ +export interface DepthEstimationOutput { + /** + * The predicted depth as an image + */ + depth?: unknown; + /** + * The predicted depth as a tensor + */ + predicted_depth?: unknown; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/depth-estimation/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/depth-estimation/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..fb6c70bddab71a3166209fe0506fe37161fd3acb --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/depth-estimation/spec/input.json @@ -0,0 +1,24 @@ +{ + "$id": "/inference/schemas/depth-estimation/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Depth Estimation inference", + "title": "DepthEstimationInput", + "type": "object", + "properties": { + "inputs": { + "description": "The input image data" + }, + "parameters": { + "description": "Additional inference parameters for Depth Estimation", + "$ref": "#/$defs/DepthEstimationParameters" + } + }, + "$defs": { + "DepthEstimationParameters": { + "title": "DepthEstimationParameters", + "type": "object", + "properties": {} + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/depth-estimation/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/depth-estimation/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..85bc6ef103c28dbc13e114df5ddc00d1a32f8406 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/depth-estimation/spec/output.json @@ -0,0 +1,16 @@ +{ + "$id": "/inference/schemas/depth-estimation/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Depth Estimation task", + "title": "DepthEstimationOutput", + + "type": "object", + "properties": { + "predicted_depth": { + "description": "The predicted depth as a tensor" + }, + "depth": { + "description": "The predicted depth as an image" + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/document-question-answering/about.md b/node_modules/@huggingface/tasks/src/tasks/document-question-answering/about.md new file mode 100644 index 0000000000000000000000000000000000000000..528c29ec917ace00387344b09671e9a90fcc6e06 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/document-question-answering/about.md @@ -0,0 +1,53 @@ +## Use Cases + +Document Question Answering models can be used to answer natural language questions about documents. Typically, document QA models consider textual, layout and potentially visual information. This is useful when the question requires some understanding of the visual aspects of the document. +Nevertheless, certain document QA models can work without document images. Hence the task is not limited to visually-rich documents and allows users to ask questions based on spreadsheets, text PDFs, etc! + +### Document Parsing + +One of the most popular use cases of document question answering models is the parsing of structured documents. For example, you can extract the name, address, and other information from a form. You can also use the model to extract information from a table, or even a resume. + +### Invoice Information Extraction + +Another very popular use case is invoice information extraction. For example, you can extract the invoice number, the invoice date, the total amount, the VAT number, and the invoice recipient. + +## Inference + +You can infer with Document QA models with the 🤗 Transformers library using the [`document-question-answering` pipeline](https://huggingface.co/docs/transformers/en/main_classes/pipelines#transformers.DocumentQuestionAnsweringPipeline). If no model checkpoint is given, the pipeline will be initialized with [`impira/layoutlm-document-qa`](https://huggingface.co/impira/layoutlm-document-qa). This pipeline takes question(s) and document(s) as input, and returns the answer. +👉 Note that the question answering task solved here is extractive: the model extracts the answer from a context (the document). + +```python +from transformers import pipeline +from PIL import Image + +pipe = pipeline("document-question-answering", model="naver-clova-ix/donut-base-finetuned-docvqa") + +question = "What is the purchase amount?" +image = Image.open("your-document.png") + +pipe(image=image, question=question) + +## [{'answer': '20,000$'}] +``` + +## Useful Resources + +Would you like to learn more about Document QA? Awesome! Here are some curated resources that you may find helpful! + +- [Document Visual Question Answering (DocVQA) challenge](https://rrc.cvc.uab.es/?ch=17) +- [DocVQA: A Dataset for Document Visual Question Answering](https://arxiv.org/abs/2007.00398) (Dataset paper) +- [ICDAR 2021 Competition on Document Visual Question Answering](https://lilianweng.github.io/lil-log/2020/10/29/open-domain-question-answering.html) (Conference paper) +- [HuggingFace's Document Question Answering pipeline](https://huggingface.co/docs/transformers/en/main_classes/pipelines#transformers.DocumentQuestionAnsweringPipeline) +- [Github repo: DocQuery - Document Query Engine Powered by Large Language Models](https://github.com/impira/docquery) + +### Notebooks + +- [Fine-tuning Donut on DocVQA dataset](https://github.com/NielsRogge/Transformers-Tutorials/tree/0ea77f29d01217587d7e32a848f3691d9c15d6ab/Donut/DocVQA) +- [Fine-tuning LayoutLMv2 on DocVQA dataset](https://github.com/NielsRogge/Transformers-Tutorials/tree/1b4bad710c41017d07a8f63b46a12523bfd2e835/LayoutLMv2/DocVQA) +- [Accelerating Document AI](https://huggingface.co/blog/document-ai) + +### Documentation + +- [Document question answering task guide](https://huggingface.co/docs/transformers/tasks/document_question_answering) + +The contents of this page are contributed by [Eliott Zemour](https://huggingface.co/eliolio) and reviewed by [Kwadwo Agyapon-Ntra](https://huggingface.co/KayO) and [Ankur Goyal](https://huggingface.co/ankrgyl). diff --git a/node_modules/@huggingface/tasks/src/tasks/document-question-answering/data.ts b/node_modules/@huggingface/tasks/src/tasks/document-question-answering/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..d40cfe613e81a87f8de7cb762df1dbc519809c10 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/document-question-answering/data.ts @@ -0,0 +1,85 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "Largest document understanding dataset.", + id: "HuggingFaceM4/Docmatix", + }, + { + description: + "Dataset from the 2020 DocVQA challenge. The documents are taken from the UCSF Industry Documents Library.", + id: "eliolio/docvqa", + }, + ], + demo: { + inputs: [ + { + label: "Question", + content: "What is the idea behind the consumer relations efficiency team?", + type: "text", + }, + { + filename: "document-question-answering-input.png", + type: "img", + }, + ], + outputs: [ + { + label: "Answer", + content: "Balance cost efficiency with quality customer service", + type: "text", + }, + ], + }, + metrics: [ + { + description: + "The evaluation metric for the DocVQA challenge is the Average Normalized Levenshtein Similarity (ANLS). This metric is flexible to character regognition errors and compares the predicted answer with the ground truth answer.", + id: "anls", + }, + { + description: + "Exact Match is a metric based on the strict character match of the predicted answer and the right answer. For answers predicted correctly, the Exact Match will be 1. Even if only one character is different, Exact Match will be 0", + id: "exact-match", + }, + ], + models: [ + { + description: "A robust document question answering model.", + id: "impira/layoutlm-document-qa", + }, + { + description: "A document question answering model specialized in invoices.", + id: "impira/layoutlm-invoices", + }, + { + description: "A special model for OCR-free document question answering.", + id: "microsoft/udop-large", + }, + { + description: "A powerful model for document question answering.", + id: "google/pix2struct-docvqa-large", + }, + ], + spaces: [ + { + description: "A robust document question answering application.", + id: "impira/docquery", + }, + { + description: "An application that can answer questions from invoices.", + id: "impira/invoices", + }, + { + description: "An application to compare different document question answering models.", + id: "merve/compare_docvqa_models", + }, + ], + summary: + "Document Question Answering (also known as Document Visual Question Answering) is the task of answering questions on document images. Document question answering models take a (document, question) pair as input and return an answer in natural language. Models usually rely on multi-modal features, combining text, position of words (bounding-boxes) and image.", + widgetModels: ["impira/layoutlm-invoices"], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/document-question-answering/inference.ts b/node_modules/@huggingface/tasks/src/tasks/document-question-answering/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..8ec6b58a01167ed1fdcc57d25e456ff51cde4ee2 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/document-question-answering/inference.ts @@ -0,0 +1,104 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Document Question Answering inference + */ +export interface DocumentQuestionAnsweringInput { + /** + * One (document, question) pair to answer + */ + inputs: DocumentQuestionAnsweringInputData; + /** + * Additional inference parameters for Document Question Answering + */ + parameters?: DocumentQuestionAnsweringParameters; + [property: string]: unknown; +} +/** + * One (document, question) pair to answer + */ +export interface DocumentQuestionAnsweringInputData { + /** + * The image on which the question is asked + */ + image: unknown; + /** + * A question to ask of the document + */ + question: string; + [property: string]: unknown; +} +/** + * Additional inference parameters for Document Question Answering + */ +export interface DocumentQuestionAnsweringParameters { + /** + * If the words in the document are too long to fit with the question for the model, it will + * be split in several chunks with some overlap. This argument controls the size of that + * overlap. + */ + doc_stride?: number; + /** + * Whether to accept impossible as an answer + */ + handle_impossible_answer?: boolean; + /** + * Language to use while running OCR. Defaults to english. + */ + lang?: string; + /** + * The maximum length of predicted answers (e.g., only answers with a shorter length are + * considered). + */ + max_answer_len?: number; + /** + * The maximum length of the question after tokenization. It will be truncated if needed. + */ + max_question_len?: number; + /** + * The maximum length of the total sentence (context + question) in tokens of each chunk + * passed to the model. The context will be split in several chunks (using doc_stride as + * overlap) if needed. + */ + max_seq_len?: number; + /** + * The number of answers to return (will be chosen by order of likelihood). Can return less + * than top_k answers if there are not enough options available within the context. + */ + top_k?: number; + /** + * A list of words and bounding boxes (normalized 0->1000). If provided, the inference will + * skip the OCR step and use the provided bounding boxes instead. + */ + word_boxes?: WordBox[]; + [property: string]: unknown; +} +export type WordBox = number[] | string; +export type DocumentQuestionAnsweringOutput = DocumentQuestionAnsweringOutputElement[]; +/** + * Outputs of inference for the Document Question Answering task + */ +export interface DocumentQuestionAnsweringOutputElement { + /** + * The answer to the question. + */ + answer: string; + /** + * The end word index of the answer (in the OCR’d version of the input or provided word + * boxes). + */ + end: number; + /** + * The probability associated to the answer. + */ + score: number; + /** + * The start word index of the answer (in the OCR’d version of the input or provided word + * boxes). + */ + start: number; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/document-question-answering/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/document-question-answering/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..3a8035f71e0abb324de3cb29db9003c71a86b6e6 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/document-question-answering/spec/input.json @@ -0,0 +1,85 @@ +{ + "$id": "/inference/schemas/document-question-answering/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Document Question Answering inference", + "title": "DocumentQuestionAnsweringInput", + "type": "object", + "properties": { + "inputs": { + "description": "One (document, question) pair to answer", + "type": "object", + "title": "DocumentQuestionAnsweringInputData", + "properties": { + "image": { + "description": "The image on which the question is asked", + "comment": "type=binary" + }, + "question": { + "type": "string", + "description": "A question to ask of the document" + } + }, + "required": ["image", "question"] + }, + "parameters": { + "description": "Additional inference parameters for Document Question Answering", + "$ref": "#/$defs/DocumentQuestionAnsweringParameters" + } + }, + "$defs": { + "DocumentQuestionAnsweringParameters": { + "title": "DocumentQuestionAnsweringParameters", + "type": "object", + "properties": { + "doc_stride": { + "type": "integer", + "description": "If the words in the document are too long to fit with the question for the model, it will be split in several chunks with some overlap. This argument controls the size of that overlap." + }, + "handle_impossible_answer": { + "type": "boolean", + "description": "Whether to accept impossible as an answer" + }, + "lang": { + "type": "string", + "description": "Language to use while running OCR. Defaults to english." + }, + "max_answer_len": { + "type": "integer", + "description": "The maximum length of predicted answers (e.g., only answers with a shorter length are considered)." + }, + "max_seq_len": { + "type": "integer", + "description": "The maximum length of the total sentence (context + question) in tokens of each chunk passed to the model. The context will be split in several chunks (using doc_stride as overlap) if needed." + }, + "max_question_len": { + "type": "integer", + "description": "The maximum length of the question after tokenization. It will be truncated if needed." + }, + "top_k": { + "type": "integer", + "description": "The number of answers to return (will be chosen by order of likelihood). Can return less than top_k answers if there are not enough options available within the context." + }, + "word_boxes": { + "type": "array", + "description": "A list of words and bounding boxes (normalized 0->1000). If provided, the inference will skip the OCR step and use the provided bounding boxes instead.", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "number" + }, + "maxLength": 4, + "minLength": 4 + } + ] + } + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/document-question-answering/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/document-question-answering/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..f8d0c20fee227973f53321eb97d3c0a0628d2af1 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/document-question-answering/spec/output.json @@ -0,0 +1,29 @@ +{ + "$id": "/inference/schemas/document-question-answering/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Document Question Answering task", + "title": "DocumentQuestionAnsweringOutput", + "type": "array", + "items": { + "type": "object", + "properties": { + "answer": { + "type": "string", + "description": "The answer to the question." + }, + "score": { + "type": "number", + "description": "The probability associated to the answer." + }, + "start": { + "type": "integer", + "description": "The start word index of the answer (in the OCR\u2019d version of the input or provided word boxes)." + }, + "end": { + "type": "integer", + "description": "The end word index of the answer (in the OCR\u2019d version of the input or provided word boxes)." + } + }, + "required": ["answer", "score", "start", "end"] + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/feature-extraction/about.md b/node_modules/@huggingface/tasks/src/tasks/feature-extraction/about.md new file mode 100644 index 0000000000000000000000000000000000000000..1563acc655dd2ea06e42e8d4d3ca7498805c92e3 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/feature-extraction/about.md @@ -0,0 +1,72 @@ +## Use Cases + +### Transfer Learning + +Models trained on a specific dataset can learn features about the data. For instance, a model trained on an English poetry dataset learns English grammar at a very high level. This information can be transferred to a new model that is going to be trained on tweets. This process of extracting features and transferring to another model is called transfer learning. One can pass their dataset through a feature extraction pipeline and feed the result to a classifier. + +### Retrieval and Reranking + +Retrieval is the process of obtaining relevant documents or information based on a user's search query. In the context of NLP, retrieval systems aim to find relevant text passages or documents from a large corpus of data that match the user's query. The goal is to return a set of results that are likely to be useful to the user. On the other hand, reranking is a technique used to improve the quality of retrieval results by reordering them based on their relevance to the query. + +### Retrieval Augmented Generation + +Retrieval-augmented generation (RAG) is a technique in which user inputs to generative models are first queried through a knowledge base, and the most relevant information from the knowledge base is used to augment the prompt to reduce hallucinations during generation. Feature extraction models (primarily retrieval and reranking models) can be used in RAG to reduce model hallucinations and ground the model. + +## Inference + +You can infer feature extraction models using `pipeline` of transformers library. + +```python +from transformers import pipeline +checkpoint = "facebook/bart-base" +feature_extractor = pipeline("feature-extraction", framework="pt", model=checkpoint) +text = "Transformers is an awesome library!" + +#Reducing along the first dimension to get a 768 dimensional array +feature_extractor(text,return_tensors = "pt")[0].numpy().mean(axis=0) + +'''tensor([[[ 2.5834, 2.7571, 0.9024, ..., 1.5036, -0.0435, -0.8603], + [-1.2850, -1.0094, -2.0826, ..., 1.5993, -0.9017, 0.6426], + [ 0.9082, 0.3896, -0.6843, ..., 0.7061, 0.6517, 1.0550], + ..., + [ 0.6919, -1.1946, 0.2438, ..., 1.3646, -1.8661, -0.1642], + [-0.1701, -2.0019, -0.4223, ..., 0.3680, -1.9704, -0.0068], + [ 0.2520, -0.6869, -1.0582, ..., 0.5198, -2.2106, 0.4547]]])''' +``` + +A very popular library for training similarity and search models is called `sentence-transformers`.  To get started, install the library. + +```bash +pip install -U sentence-transformers +``` + +You can infer with `sentence-transformers` models as follows. + +```python +from sentence_transformers import SentenceTransformer + +model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2") +sentences = [ + "The weather is lovely today.", + "It's so sunny outside!", + "He drove to the stadium.", +] + +embeddings = model.encode(sentences) +similarities = model.similarity(embeddings, embeddings) +print(similarities) +# tensor([[1.0000, 0.6660, 0.1046], +# [0.6660, 1.0000, 0.1411], +# [0.1046, 0.1411, 1.0000]]) +``` + +### Text Embedding Inference + +[Text Embeddings Inference (TEI)](https://github.com/huggingface/text-embeddings-inference) is a toolkit to easily serve feature extraction models using few lines of code. + +## Useful resources + +- [Documentation for feature extraction task in 🤗Transformers](https://huggingface.co/docs/transformers/main_classes/feature_extractor) +- [Introduction to MTEB Benchmark](https://huggingface.co/blog/mteb) +- [Cookbook: Simple RAG for GitHub issues using Hugging Face Zephyr and LangChain](https://huggingface.co/learn/cookbook/rag_zephyr_langchain) +- [sentence-transformers organization on Hugging Face Hub](https://huggingface.co/sentence-transformers) diff --git a/node_modules/@huggingface/tasks/src/tasks/feature-extraction/data.ts b/node_modules/@huggingface/tasks/src/tasks/feature-extraction/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..75f5d6f5ed3965e462c4e0205c90a148d336509d --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/feature-extraction/data.ts @@ -0,0 +1,57 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: + "Wikipedia dataset containing cleaned articles of all languages. Can be used to train `feature-extraction` models.", + id: "wikipedia", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "India, officially the Republic of India, is a country in South Asia.", + type: "text", + }, + ], + outputs: [ + { + table: [ + ["Dimension 1", "Dimension 2", "Dimension 3"], + ["2.583383083343506", "2.757075071334839", "0.9023529887199402"], + ["8.29393482208252", "1.1071064472198486", "2.03399395942688"], + ["-0.7754912972450256", "-1.647324562072754", "-0.6113331913948059"], + ["0.07087723910808563", "1.5942802429199219", "1.4610432386398315"], + ], + type: "tabular", + }, + ], + }, + metrics: [], + models: [ + { + description: "A powerful feature extraction model for natural language processing tasks.", + id: "thenlper/gte-large", + }, + { + description: "A strong feature extraction model for retrieval.", + id: "Alibaba-NLP/gte-Qwen1.5-7B-instruct", + }, + ], + spaces: [ + { + description: "A leaderboard to rank text feature extraction models based on a benchmark.", + id: "mteb/leaderboard", + }, + { + description: "A leaderboard to rank best feature extraction models based on human feedback.", + id: "mteb/arena", + }, + ], + summary: "Feature extraction is the task of extracting features learnt in a model.", + widgetModels: ["facebook/bart-base"], +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/feature-extraction/inference.ts b/node_modules/@huggingface/tasks/src/tasks/feature-extraction/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..f60bef611339dc786a7e179f9d80a0434452b4a2 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/feature-extraction/inference.ts @@ -0,0 +1,41 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +export type FeatureExtractionOutput = Array; +/** + * Feature Extraction Input. + * + * Auto-generated from TEI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tei-import.ts. + */ +export interface FeatureExtractionInput { + /** + * The text or list of texts to embed. + */ + inputs: FeatureExtractionInputs; + normalize?: boolean; + /** + * The name of the prompt that should be used by for encoding. If not set, no prompt + * will be applied. + * + * Must be a key in the `sentence-transformers` configuration `prompts` dictionary. + * + * For example if ``prompt_name`` is "query" and the ``prompts`` is {"query": "query: ", + * ...}, + * then the sentence "What is the capital of France?" will be encoded as + * "query: What is the capital of France?" because the prompt text will be prepended before + * any text to encode. + */ + prompt_name?: string; + truncate?: boolean; + truncation_direction?: FeatureExtractionInputTruncationDirection; + [property: string]: unknown; +} +/** + * The text or list of texts to embed. + */ +export type FeatureExtractionInputs = string[] | string; +export type FeatureExtractionInputTruncationDirection = "left" | "right"; diff --git a/node_modules/@huggingface/tasks/src/tasks/feature-extraction/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/feature-extraction/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..1b386746f71ab88db1e1ab1c62343e7b72899d69 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/feature-extraction/spec/input.json @@ -0,0 +1,58 @@ +{ + "$id": "/inference/schemas/feature-extraction/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Feature Extraction Input.\n\nAuto-generated from TEI specs.\nFor more details, check out https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tei-import.ts.", + "title": "FeatureExtractionInput", + "type": "object", + "required": ["inputs"], + "properties": { + "inputs": { + "title": "FeatureExtractionInputs", + "description": "The text or list of texts to embed.", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "normalize": { + "type": "boolean", + "default": "true", + "example": "true" + }, + "prompt_name": { + "type": "string", + "description": "The name of the prompt that should be used by for encoding. If not set, no prompt\nwill be applied.\n\nMust be a key in the `sentence-transformers` configuration `prompts` dictionary.\n\nFor example if ``prompt_name`` is \"query\" and the ``prompts`` is {\"query\": \"query: \", ...},\nthen the sentence \"What is the capital of France?\" will be encoded as\n\"query: What is the capital of France?\" because the prompt text will be prepended before\nany text to encode.", + "default": "null", + "example": "null", + "nullable": true + }, + "truncate": { + "type": "boolean", + "default": "false", + "example": "false", + "nullable": true + }, + "truncation_direction": { + "allOf": [ + { + "$ref": "#/$defs/FeatureExtractionInputTruncationDirection" + } + ], + "default": "right" + } + }, + "$defs": { + "FeatureExtractionInputTruncationDirection": { + "type": "string", + "enum": ["left", "right"], + "title": "FeatureExtractionInputTruncationDirection" + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/feature-extraction/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/feature-extraction/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..591fe9c64828712f67d579b5356a1ede5f23fdb9 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/feature-extraction/spec/output.json @@ -0,0 +1,15 @@ +{ + "$id": "/inference/schemas/feature-extraction/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Feature Extraction Output.\n\nAuto-generated from TEI specs.\nFor more details, check out https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tei-import.ts.", + "title": "FeatureExtractionOutput", + "type": "array", + "$defs": {}, + "items": { + "type": "array", + "items": { + "type": "number", + "format": "float" + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/fill-mask/about.md b/node_modules/@huggingface/tasks/src/tasks/fill-mask/about.md new file mode 100644 index 0000000000000000000000000000000000000000..4fabd3cf6d06d8ba9e676eb1f637c5f688b456fb --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/fill-mask/about.md @@ -0,0 +1,51 @@ +## Use Cases + +### Domain Adaptation 👩‍⚕️ + +Masked language models do not require labelled data! They are trained by masking a couple of words in sentences and the model is expected to guess the masked word. This makes it very practical! + +For example, masked language modeling is used to train large models for domain-specific problems. If you have to work on a domain-specific task, such as retrieving information from medical research papers, you can train a masked language model using those papers. 📄 + +The resulting model has a statistical understanding of the language used in medical research papers, and can be further trained in a process called fine-tuning to solve different tasks, such as [Text Classification](/tasks/text-classification) or [Question Answering](/tasks/question-answering) to build a medical research papers information extraction system. 👩‍⚕️ Pre-training on domain-specific data tends to yield better results (see [this paper](https://arxiv.org/abs/2007.15779) for an example). + +If you don't have the data to train a masked language model, you can also use an existing [domain-specific masked language model](https://huggingface.co/microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext) from the Hub and fine-tune it with your smaller task dataset. That's the magic of Open Source and sharing your work! 🎉 + +## Inference with Fill-Mask Pipeline + +You can use the 🤗 Transformers library `fill-mask` pipeline to do inference with masked language models. If a model name is not provided, the pipeline will be initialized with [distilroberta-base](/distilroberta-base). You can provide masked text and it will return a list of possible mask values ​​ranked according to the score. + +```python +from transformers import pipeline + +classifier = pipeline("fill-mask") +classifier("Paris is the of France.") + +# [{'score': 0.7, 'sequence': 'Paris is the capital of France.'}, +# {'score': 0.2, 'sequence': 'Paris is the birthplace of France.'}, +# {'score': 0.1, 'sequence': 'Paris is the heart of France.'}] +``` + +## Useful Resources + +Would you like to learn more about the topic? Awesome! Here you can find some curated resources that can be helpful to you! + +- [Course Chapter on Fine-tuning a Masked Language Model](https://huggingface.co/course/chapter7/3?fw=pt) +- [Workshop on Pretraining Language Models and CodeParrot](https://www.youtube.com/watch?v=ExUR7w6xe94) +- [BERT 101: State Of The Art NLP Model Explained](https://huggingface.co/blog/bert-101) +- [Nyströmformer: Approximating self-attention in linear time and memory via the Nyström method](https://huggingface.co/blog/nystromformer) + +### Notebooks + +- [Pre-training an MLM for JAX/Flax](https://github.com/huggingface/notebooks/blob/master/examples/masked_language_modeling_flax.ipynb) +- [Masked language modeling in TensorFlow](https://github.com/huggingface/notebooks/blob/master/examples/language_modeling-tf.ipynb) +- [Masked language modeling in PyTorch](https://github.com/huggingface/notebooks/blob/master/examples/language_modeling.ipynb) + +### Scripts for training + +- [PyTorch](https://github.com/huggingface/transformers/tree/main/examples/pytorch/language-modeling) +- [Flax](https://github.com/huggingface/transformers/tree/main/examples/flax/language-modeling) +- [TensorFlow](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/language-modeling) + +### Documentation + +- [Masked language modeling task guide](https://huggingface.co/docs/transformers/tasks/masked_language_modeling) diff --git a/node_modules/@huggingface/tasks/src/tasks/fill-mask/data.ts b/node_modules/@huggingface/tasks/src/tasks/fill-mask/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..1c38bf6be0f8ee8d84a7d2ca986fdbd53e0134bc --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/fill-mask/data.ts @@ -0,0 +1,79 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A common dataset that is used to train models for many languages.", + id: "wikipedia", + }, + { + description: "A large English dataset with text crawled from the web.", + id: "c4", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "The barked at me", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "wolf", + score: 0.487, + }, + { + label: "dog", + score: 0.061, + }, + { + label: "cat", + score: 0.058, + }, + { + label: "fox", + score: 0.047, + }, + { + label: "squirrel", + score: 0.025, + }, + ], + }, + ], + }, + metrics: [ + { + description: + "Cross Entropy is a metric that calculates the difference between two probability distributions. Each probability distribution is the distribution of predicted words", + id: "cross_entropy", + }, + { + description: + "Perplexity is the exponential of the cross-entropy loss. It evaluates the probabilities assigned to the next word by the model. Lower perplexity indicates better performance", + id: "perplexity", + }, + ], + models: [ + { + description: "State-of-the-art masked language model.", + id: "answerdotai/ModernBERT-large", + }, + { + description: "A multilingual model trained on 100 languages.", + id: "FacebookAI/xlm-roberta-base", + }, + ], + spaces: [], + summary: + "Masked language modeling is the task of masking some of the words in a sentence and predicting which words should replace those masks. These models are useful when we want to get a statistical understanding of the language in which the model is trained in.", + widgetModels: ["distilroberta-base"], + youtubeId: "mqElG5QJWUg", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/fill-mask/inference.ts b/node_modules/@huggingface/tasks/src/tasks/fill-mask/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..4f48bb55ee981393d31e5a0d1146db673a6081d6 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/fill-mask/inference.ts @@ -0,0 +1,60 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Fill Mask inference + */ +export interface FillMaskInput { + /** + * The text with masked tokens + */ + inputs: string; + /** + * Additional inference parameters for Fill Mask + */ + parameters?: FillMaskParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Fill Mask + */ +export interface FillMaskParameters { + /** + * When passed, the model will limit the scores to the passed targets instead of looking up + * in the whole vocabulary. If the provided targets are not in the model vocab, they will be + * tokenized and the first resulting token will be used (with a warning, and that might be + * slower). + */ + targets?: string[]; + /** + * When passed, overrides the number of predictions to return. + */ + top_k?: number; + [property: string]: unknown; +} +export type FillMaskOutput = FillMaskOutputElement[]; +/** + * Outputs of inference for the Fill Mask task + */ +export interface FillMaskOutputElement { + /** + * The corresponding probability + */ + score: number; + /** + * The corresponding input with the mask token prediction. + */ + sequence: string; + /** + * The predicted token id (to replace the masked one). + */ + token: number; + tokenStr: unknown; + /** + * The predicted token (to replace the masked one). + */ + token_str?: string; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/fill-mask/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/fill-mask/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..a2ebaf13e16212e29df8ec3c552071fca7d6078c --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/fill-mask/spec/input.json @@ -0,0 +1,37 @@ +{ + "$id": "/inference/schemas/fill-mask/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Fill Mask inference", + "title": "FillMaskInput", + "type": "object", + "properties": { + "inputs": { + "description": "The text with masked tokens", + "type": "string" + }, + "parameters": { + "description": "Additional inference parameters for Fill Mask", + "$ref": "#/$defs/FillMaskParameters" + } + }, + "$defs": { + "FillMaskParameters": { + "title": "FillMaskParameters", + "type": "object", + "properties": { + "top_k": { + "type": "integer", + "description": "When passed, overrides the number of predictions to return." + }, + "targets": { + "description": "When passed, the model will limit the scores to the passed targets instead of looking up in the whole vocabulary. If the provided targets are not in the model vocab, they will be tokenized and the first resulting token will be used (with a warning, and that might be slower).", + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/fill-mask/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/fill-mask/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..0b613382e781cf0405c76df5c1f9f5091da6b196 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/fill-mask/spec/output.json @@ -0,0 +1,29 @@ +{ + "$id": "/inference/schemas/fill-mask/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Fill Mask task", + "title": "FillMaskOutput", + "type": "array", + "items": { + "type": "object", + "properties": { + "sequence": { + "type": "string", + "description": "The corresponding input with the mask token prediction." + }, + "score": { + "type": "number", + "description": "The corresponding probability" + }, + "token": { + "type": "integer", + "description": "The predicted token id (to replace the masked one)." + }, + "token_str": { + "type": "string", + "description": "The predicted token (to replace the masked one)." + } + }, + "required": ["sequence", "score", "token", "tokenStr"] + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-classification/about.md b/node_modules/@huggingface/tasks/src/tasks/image-classification/about.md new file mode 100644 index 0000000000000000000000000000000000000000..6d2b445fa62b8fcd88fb738820e0c18e327460be --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-classification/about.md @@ -0,0 +1,50 @@ +## Use Cases + +Image classification models can be used when we are not interested in specific instances of objects with location information or their shape. + +### Keyword Classification + +Image classification models are used widely in stock photography to assign each image a keyword. + +### Image Search + +Models trained in image classification can improve user experience by organizing and categorizing photo galleries on the phone or in the cloud, on multiple keywords or tags. + +## Inference + +With the `transformers` library, you can use the `image-classification` pipeline to infer with image classification models. You can initialize the pipeline with a model id from the Hub. If you do not provide a model id it will initialize with [google/vit-base-patch16-224](https://huggingface.co/google/vit-base-patch16-224) by default. When calling the pipeline you just need to specify a path, http link or an image loaded in PIL. You can also provide a `top_k` parameter which determines how many results it should return. + +```python +from transformers import pipeline +clf = pipeline("image-classification") +clf("path_to_a_cat_image") + +[{'label': 'tabby cat', 'score': 0.731}, +... +] +``` + +You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to classify images using models on Hugging Face Hub. + +```javascript +import { InferenceClient } from "@huggingface/inference"; + +const inference = new InferenceClient(HF_TOKEN); +await inference.imageClassification({ + data: await (await fetch("https://picsum.photos/300/300")).blob(), + model: "microsoft/resnet-50", +}); +``` + +## Useful Resources + +- [Let's Play Pictionary with Machine Learning!](https://www.youtube.com/watch?v=LS9Y2wDVI0k) +- [Fine-Tune ViT for Image Classification with 🤗Transformers](https://huggingface.co/blog/fine-tune-vit) +- [Walkthrough of Computer Vision Ecosystem in Hugging Face - CV Study Group](https://www.youtube.com/watch?v=oL-xmufhZM8) +- [Computer Vision Study Group: Swin Transformer](https://www.youtube.com/watch?v=Ngikt-K1Ecc) +- [Computer Vision Study Group: Masked Autoencoders Paper Walkthrough](https://www.youtube.com/watch?v=Ngikt-K1Ecc) +- [Image classification task guide](https://huggingface.co/docs/transformers/tasks/image_classification) + +### Creating your own image classifier in just a few minutes + +With [HuggingPics](https://github.com/nateraw/huggingpics), you can fine-tune Vision Transformers for anything using images found on the web. This project downloads images of classes defined by you, trains a model, and pushes it to the Hub. You even get to try out the model directly with a working widget in the browser, ready to be shared with all your friends! diff --git a/node_modules/@huggingface/tasks/src/tasks/image-classification/data.ts b/node_modules/@huggingface/tasks/src/tasks/image-classification/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..acdb201e3286a7aa3a73f1a77010e4f735696353 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-classification/data.ts @@ -0,0 +1,87 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + // TODO write proper description + description: "Benchmark dataset used for image classification with images that belong to 100 classes.", + id: "cifar100", + }, + { + // TODO write proper description + description: "Dataset consisting of images of garments.", + id: "fashion_mnist", + }, + ], + demo: { + inputs: [ + { + filename: "image-classification-input.jpeg", + type: "img", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Egyptian cat", + score: 0.514, + }, + { + label: "Tabby cat", + score: 0.193, + }, + { + label: "Tiger cat", + score: 0.068, + }, + ], + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + description: "A strong image classification model.", + id: "google/vit-base-patch16-224", + }, + { + description: "A robust image classification model.", + id: "facebook/deit-base-distilled-patch16-224", + }, + { + description: "A strong image classification model.", + id: "facebook/convnext-large-224", + }, + ], + spaces: [ + { + description: "A leaderboard to evaluate different image classification models.", + id: "timm/leaderboard", + }, + ], + summary: + "Image classification is the task of assigning a label or class to an entire image. Images are expected to have only one class for each image. Image classification models take an image as input and return a prediction about which class the image belongs to.", + widgetModels: ["google/vit-base-patch16-224"], + youtubeId: "tjAIM7BOYhw", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/image-classification/inference.ts b/node_modules/@huggingface/tasks/src/tasks/image-classification/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..923fbfec0e7568e07918d9c650fc4dec73837b99 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-classification/inference.ts @@ -0,0 +1,53 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image Classification inference + */ +export interface ImageClassificationInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Image Classification + */ + parameters?: ImageClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image Classification + */ +export interface ImageClassificationParameters { + /** + * The function to apply to the model outputs in order to retrieve the scores. + */ + function_to_apply?: ClassificationOutputTransform; + /** + * When specified, limits the output to the top K most probable classes. + */ + top_k?: number; + [property: string]: unknown; +} +/** + * The function to apply to the model outputs in order to retrieve the scores. + */ +export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none"; +export type ImageClassificationOutput = ImageClassificationOutputElement[]; +/** + * Outputs of inference for the Image Classification task + */ +export interface ImageClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-classification/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/image-classification/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..4942226d2963287bb4fad878b8856d37e0dde36f --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-classification/spec/input.json @@ -0,0 +1,36 @@ +{ + "$id": "/inference/schemas/image-classification/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Image Classification inference", + "title": "ImageClassificationInput", + "type": "object", + "properties": { + "inputs": { + "type": "string", + "description": "The input image data as a base64-encoded string. If no `parameters` are provided, you can also provide the image data as a raw bytes payload.", + "comment": "type=binary" + }, + "parameters": { + "description": "Additional inference parameters for Image Classification", + "$ref": "#/$defs/ImageClassificationParameters" + } + }, + "$defs": { + "ImageClassificationParameters": { + "title": "ImageClassificationParameters", + "type": "object", + "properties": { + "function_to_apply": { + "title": "ImageClassificationOutputTransform", + "$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutputTransform", + "description": "The function to apply to the model outputs in order to retrieve the scores." + }, + "top_k": { + "type": "integer", + "description": "When specified, limits the output to the top K most probable classes." + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-classification/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/image-classification/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..3ababaf63d47baf941b1f290891367ce2f43794f --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-classification/spec/output.json @@ -0,0 +1,11 @@ +{ + "$id": "/inference/schemas/image-classification/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Image Classification task", + "title": "ImageClassificationOutput", + "type": "array", + "items": { + "type": "object", + "$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput" + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-feature-extraction/about.md b/node_modules/@huggingface/tasks/src/tasks/image-feature-extraction/about.md new file mode 100644 index 0000000000000000000000000000000000000000..9a968b106d527b16556f907c09e6cf67c610f7eb --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-feature-extraction/about.md @@ -0,0 +1,23 @@ +## Use Cases + +### Transfer Learning + +Models trained on a specific dataset can learn features about the data. For instance, a model trained on a car classification dataset learns to recognize edges and curves on a very high level and car-specific features on a low level. This information can be transferred to a new model that is going to be trained on classifying trucks. This process of extracting features and transferring to another model is called transfer learning. + +### Similarity + +Features extracted from models contain semantically meaningful information about the world. These features can be used to detect the similarity between two images. Assume there are two images: a photo of a stray cat in a street setting and a photo of a cat at home. These images both contain cats, and the features will contain the information that there's a cat in the image. Thus, comparing the features of a stray cat photo to the features of a domestic cat photo will result in higher similarity compared to any other image that doesn't contain any cats. + +## Inference + +```python +import torch +from transformers import pipeline + +pipe = pipeline(task="image-feature-extraction", model_name="google/vit-base-patch16-384", framework="pt", pool=True) +pipe("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/cats.png") + +feature_extractor(text,return_tensors = "pt")[0].numpy().mean(axis=0) + +'[[[0.21236686408519745, 1.0919708013534546, 0.8512550592422485, ...]]]' +``` diff --git a/node_modules/@huggingface/tasks/src/tasks/image-feature-extraction/data.ts b/node_modules/@huggingface/tasks/src/tasks/image-feature-extraction/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..d2565dec61e6b089fe5f62bd97f4abe36e008943 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-feature-extraction/data.ts @@ -0,0 +1,64 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: + "ImageNet-1K is a image classification dataset in which images are used to train image-feature-extraction models.", + id: "imagenet-1k", + }, + ], + demo: { + inputs: [ + { + filename: "mask-generation-input.png", + type: "img", + }, + ], + outputs: [ + { + table: [ + ["Dimension 1", "Dimension 2", "Dimension 3"], + ["0.21236686408519745", "1.0919708013534546", "0.8512550592422485"], + ["0.809657871723175", "-0.18544459342956543", "-0.7851548194885254"], + ["1.3103108406066895", "-0.2479034662246704", "-0.9107287526130676"], + ["1.8536205291748047", "-0.36419737339019775", "0.09717650711536407"], + ], + type: "tabular", + }, + ], + }, + metrics: [], + models: [ + { + description: "A powerful image feature extraction model.", + id: "timm/vit_large_patch14_dinov2.lvd142m", + }, + { + description: "A strong image feature extraction model.", + id: "nvidia/MambaVision-T-1K", + }, + { + description: "A robust image feature extraction model.", + id: "facebook/dino-vitb16", + }, + { + description: "Cutting-edge image feature extraction model.", + id: "apple/aimv2-large-patch14-336-distilled", + }, + { + description: "Strong image feature extraction model that can be used on images and documents.", + id: "OpenGVLab/InternViT-6B-448px-V1-2", + }, + ], + spaces: [ + { + description: "A leaderboard to evaluate different image-feature-extraction models on classification performances", + id: "timm/leaderboard", + }, + ], + summary: "Image feature extraction is the task of extracting features learnt in a computer vision model.", + widgetModels: [], +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/image-segmentation/about.md b/node_modules/@huggingface/tasks/src/tasks/image-segmentation/about.md new file mode 100644 index 0000000000000000000000000000000000000000..18af58ad8628bd12e4c16f9f3f17667fe1821414 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-segmentation/about.md @@ -0,0 +1,63 @@ +## Use Cases + +### Autonomous Driving + +Segmentation models are used to identify road patterns such as lanes and obstacles for safer driving. + +### Background Removal + +Image Segmentation models are used in cameras to erase the background of certain objects and apply filters to them. + +### Medical Imaging + +Image Segmentation models are used to distinguish organs or tissues, improving medical imaging workflows. Models are used to segment dental instances, analyze X-Ray scans or even segment cells for pathological diagnosis. This [dataset](https://github.com/v7labs/covid-19-xray-dataset) contains images of lungs of healthy patients and patients with COVID-19 segmented with masks. Another [segmentation dataset](https://ivdm3seg.weebly.com/data.html) contains segmented MRI data of the lower spine to analyze the effect of spaceflight simulation. + +## Task Variants + +### Semantic Segmentation + +Semantic Segmentation is the task of segmenting parts of an image that belong to the same class. Semantic Segmentation models make predictions for each pixel and return the probabilities of the classes for each pixel. These models are evaluated on Mean Intersection Over Union (Mean IoU). + +### Instance Segmentation + +Instance Segmentation is the variant of Image Segmentation where every distinct object is segmented, instead of one segment per class. + +### Panoptic Segmentation + +Panoptic Segmentation is the Image Segmentation task that segments the image both by instance and by class, assigning each pixel a different instance of the class. + +## Inference + +You can infer with Image Segmentation models using the `image-segmentation` pipeline. You need to install [timm](https://github.com/rwightman/pytorch-image-models) first. + +```python +!pip install timm +model = pipeline("image-segmentation") +model("cat.png") +#[{'label': 'cat', +# 'mask': mask_code, +# 'score': 0.999} +# ...] +``` + +You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer image segmentation models on Hugging Face Hub. + +```javascript +import { InferenceClient } from "@huggingface/inference"; + +const inference = new InferenceClient(HF_TOKEN); +await inference.imageSegmentation({ + data: await (await fetch("https://picsum.photos/300/300")).blob(), + model: "facebook/mask2former-swin-base-coco-panoptic", +}); +``` + +## Useful Resources + +Would you like to learn more about image segmentation? Great! Here you can find some curated resources that you may find helpful! + +- [Fine-Tune a Semantic Segmentation Model with a Custom Dataset](https://huggingface.co/blog/fine-tune-segformer) +- [Walkthrough of Computer Vision Ecosystem in Hugging Face - CV Study Group](https://www.youtube.com/watch?v=oL-xmufhZM8) +- [A Guide on Universal Image Segmentation with Mask2Former and OneFormer](https://huggingface.co/blog/mask2former) +- [Zero-shot image segmentation with CLIPSeg](https://huggingface.co/blog/clipseg-zero-shot) +- [Semantic segmentation task guide](https://huggingface.co/docs/transformers/tasks/semantic_segmentation) diff --git a/node_modules/@huggingface/tasks/src/tasks/image-segmentation/data.ts b/node_modules/@huggingface/tasks/src/tasks/image-segmentation/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..c34481f9acf7c67ecd9cf6700a4858de83340c98 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-segmentation/data.ts @@ -0,0 +1,99 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "Scene segmentation dataset.", + id: "scene_parse_150", + }, + ], + demo: { + inputs: [ + { + filename: "image-segmentation-input.jpeg", + type: "img", + }, + ], + outputs: [ + { + filename: "image-segmentation-output.png", + type: "img", + }, + ], + }, + metrics: [ + { + description: + "Average Precision (AP) is the Area Under the PR Curve (AUC-PR). It is calculated for each semantic class separately", + id: "Average Precision", + }, + { + description: "Mean Average Precision (mAP) is the overall average of the AP values", + id: "Mean Average Precision", + }, + { + description: + "Intersection over Union (IoU) is the overlap of segmentation masks. Mean IoU is the average of the IoU of all semantic classes", + id: "Mean Intersection over Union", + }, + { + description: "APα is the Average Precision at the IoU threshold of a α value, for example, AP50 and AP75", + id: "APα", + }, + ], + models: [ + { + // TO DO: write description + description: "Solid panoptic segmentation model trained on COCO.", + id: "tue-mps/coco_panoptic_eomt_large_640", + }, + { + description: "Background removal model.", + id: "briaai/RMBG-1.4", + }, + { + description: "A multipurpose image segmentation model for high resolution images.", + id: "ZhengPeng7/BiRefNet", + }, + { + description: "Powerful human-centric image segmentation model.", + id: "facebook/sapiens-seg-1b", + }, + { + description: "Panoptic segmentation model trained on the COCO (common objects) dataset.", + id: "facebook/mask2former-swin-large-coco-panoptic", + }, + ], + spaces: [ + { + description: "A semantic segmentation application that can predict unseen instances out of the box.", + id: "facebook/ov-seg", + }, + { + description: "One of the strongest segmentation applications.", + id: "jbrinkma/segment-anything", + }, + { + description: "A human-centric segmentation model.", + id: "facebook/sapiens-pose", + }, + { + description: "An instance segmentation application to predict neuronal cell types from microscopy images.", + id: "rashmi/sartorius-cell-instance-segmentation", + }, + { + description: "An application that segments videos.", + id: "ArtGAN/Segment-Anything-Video", + }, + { + description: "An panoptic segmentation application built for outdoor environments.", + id: "segments/panoptic-segment-anything", + }, + ], + summary: + "Image Segmentation divides an image into segments where each pixel in the image is mapped to an object. This task has multiple variants such as instance segmentation, panoptic segmentation and semantic segmentation.", + widgetModels: ["nvidia/segformer-b0-finetuned-ade-512-512"], + youtubeId: "dKE8SIt9C-w", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/image-segmentation/inference.ts b/node_modules/@huggingface/tasks/src/tasks/image-segmentation/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..72db730745c3ce37b2bfe38f211c5620c9feb8f8 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-segmentation/inference.ts @@ -0,0 +1,67 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image Segmentation inference + */ +export interface ImageSegmentationInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Image Segmentation + */ + parameters?: ImageSegmentationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image Segmentation + */ +export interface ImageSegmentationParameters { + /** + * Threshold to use when turning the predicted masks into binary values. + */ + mask_threshold?: number; + /** + * Mask overlap threshold to eliminate small, disconnected segments. + */ + overlap_mask_area_threshold?: number; + /** + * Segmentation task to be performed, depending on model capabilities. + */ + subtask?: ImageSegmentationSubtask; + /** + * Probability threshold to filter out predicted masks. + */ + threshold?: number; + [property: string]: unknown; +} +/** + * Segmentation task to be performed, depending on model capabilities. + */ +export type ImageSegmentationSubtask = "instance" | "panoptic" | "semantic"; +export type ImageSegmentationOutput = ImageSegmentationOutputElement[]; +/** + * Outputs of inference for the Image Segmentation task + * + * A predicted mask / segment + */ +export interface ImageSegmentationOutputElement { + /** + * The label of the predicted segment. + */ + label: string; + /** + * The corresponding mask as a black-and-white image (base64-encoded). + */ + mask: string; + /** + * The score or confidence degree the model has. + */ + score?: number; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-segmentation/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/image-segmentation/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..afc40ad2db5c70d8234cfa8063be31669e9a4b7b --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-segmentation/spec/input.json @@ -0,0 +1,45 @@ +{ + "$id": "/inference/schemas/image-segmentation/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Image Segmentation inference", + "title": "ImageSegmentationInput", + "type": "object", + "properties": { + "inputs": { + "type": "string", + "description": "The input image data as a base64-encoded string. If no `parameters` are provided, you can also provide the image data as a raw bytes payload.", + "comment": "type=binary" + }, + "parameters": { + "description": "Additional inference parameters for Image Segmentation", + "$ref": "#/$defs/ImageSegmentationParameters" + } + }, + "$defs": { + "ImageSegmentationParameters": { + "title": "ImageSegmentationParameters", + "type": "object", + "properties": { + "mask_threshold": { + "type": "number", + "description": "Threshold to use when turning the predicted masks into binary values." + }, + "overlap_mask_area_threshold": { + "type": "number", + "description": "Mask overlap threshold to eliminate small, disconnected segments." + }, + "subtask": { + "title": "ImageSegmentationSubtask", + "type": "string", + "description": "Segmentation task to be performed, depending on model capabilities.", + "enum": ["instance", "panoptic", "semantic"] + }, + "threshold": { + "type": "number", + "description": "Probability threshold to filter out predicted masks." + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-segmentation/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/image-segmentation/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..6fa5b0d8e83ad97a63f6dc2da6588ccc5bd58097 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-segmentation/spec/output.json @@ -0,0 +1,26 @@ +{ + "$id": "/inference/schemas/image-segmentation/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Image Segmentation task", + "title": "ImageSegmentationOutput", + "type": "array", + "items": { + "description": "A predicted mask / segment", + "type": "object", + "properties": { + "label": { + "type": "string", + "description": "The label of the predicted segment." + }, + "mask": { + "type": "string", + "description": "The corresponding mask as a black-and-white image (base64-encoded)." + }, + "score": { + "type": "number", + "description": "The score or confidence degree the model has." + } + }, + "required": ["label", "mask"] + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/about.md b/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/about.md new file mode 100644 index 0000000000000000000000000000000000000000..c05a81fceaa340b66faf52732d1f4926a4e803e4 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/about.md @@ -0,0 +1,73 @@ +## Use Cases + +### Instruction-based Image Editing + +Image-text-to-image models can be used to edit images based on natural language instructions. For example, you can provide an image of a summer landscape and the instruction "Make it winter, add snow" to generate a winter version of the same scene. + +### Style Transfer + +These models can apply artistic styles or transformations to images based on text descriptions. For instance, you can transform a photo into a painting style by providing prompts like "Make it look like a Van Gogh painting" or "Convert to watercolor style." + +### Image Variations + +Generate variations of an existing image by providing different text prompts. This is useful for creative workflows where you want to explore different versions of the same image with specific modifications. + +### Guided Image Generation + +Use a reference image along with text prompts to guide the generation process. This allows for more controlled image generation compared to text-to-image models alone, as the reference image provides structural guidance. + +### Image Inpainting and Outpainting + +Fill in missing or masked parts of an image based on text descriptions, or extend an image beyond its original boundaries with text-guided generation. + +## Task Variants + +### Instruction-based Editing + +Models that follow natural language instructions to edit images, which can perform complex edits like object removal, color changes, and compositional modifications. + +### Reference-guided Generation + +Models that use a reference image to guide the generation process while incorporating text prompts to control specific attributes or modifications. + +### Conditional Image-to-Image + +Models that perform specific transformations based on text conditions, such as changing weather conditions, time of day, or seasonal variations. + +## Inference + +You can use the Diffusers library to interact with image-text-to-image models. + +```python +import torch +from diffusers import Flux2Pipeline +from diffusers.utils import load_image + +repo_id = "black-forest-labs/FLUX.2-dev" +device = "cuda:0" +torch_dtype = torch.bfloat16 + +pipe = Flux2Pipeline.from_pretrained( + repo_id, torch_dtype=torch_dtype +) +pipe.enable_model_cpu_offload() #no need to do cpu offload for >80G VRAM carts like H200, B200, etc. and do a `pipe.to(device)` instead + +prompt = "Realistic macro photograph of a hermit crab using a soda can as its shell, partially emerging from the can, captured with sharp detail and natural colors, on a sunlit beach with soft shadows and a shallow depth of field, with blurred ocean waves in the background. The can has the text `BFL Diffusers` on it and it has a color gradient that start with #FF5733 at the top and transitions to #33FF57 at the bottom." + +#cat_image = load_image("https://huggingface.co/spaces/zerogpu-aoti/FLUX.1-Kontext-Dev-fp8-dynamic/resolve/main/cat.png") +image = pipe( + prompt=prompt, + #image=[cat_image] #multi-image input + generator=torch.Generator(device=device).manual_seed(42), + num_inference_steps=50, + guidance_scale=4, +).images[0] + +image.save("flux2_output.png") +``` + +## Useful Resources + +- [FLUX.2 Model Card](https://huggingface.co/black-forest-labs/FLUX.2-dev) +- [Diffusers documentation on Image-to-Image](https://huggingface.co/docs/diffusers/using-diffusers/img2img) +- [ControlNet for Conditional Image Generation](https://huggingface.co/docs/diffusers/using-diffusers/controlnet) diff --git a/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/data.ts b/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..f45410e91cddf0bca75166f449d8b4c17317281d --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/data.ts @@ -0,0 +1,54 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [], + demo: { + inputs: [ + { + filename: "image-text-to-image-input.jpeg", + type: "img", + }, + { + label: "Input", + content: "A city above clouds, pastel colors, Victorian style", + type: "text", + }, + ], + outputs: [ + { + filename: "image-text-to-image-output.png", + type: "img", + }, + ], + }, + metrics: [ + { + description: + "The Fréchet Inception Distance (FID) calculates the distance between distributions between synthetic and real samples. A lower FID score indicates better similarity between the distributions of real and generated images.", + id: "FID", + }, + { + description: + "CLIP Score measures the similarity between the generated image and the text prompt using CLIP embeddings. A higher score indicates better alignment with the text prompt.", + id: "CLIP", + }, + ], + models: [ + { + description: "A powerful model for image-text-to-image generation.", + id: "black-forest-labs/FLUX.2-dev", + }, + ], + spaces: [ + { + description: "An application for image-text-to-image generation.", + id: "black-forest-labs/FLUX.2-dev", + }, + ], + summary: + "Image-text-to-image models take an image and a text prompt as input and generate a new image based on the reference image and text instructions. These models are useful for image editing, style transfer, image variations, and guided image generation tasks.", + widgetModels: ["black-forest-labs/FLUX.2-dev"], + youtubeId: undefined, +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/inference.ts b/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..549b0f4118b8c142a22adfdb950d8947c4d3c4be --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/inference.ts @@ -0,0 +1,75 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image Text To Image inference. Either inputs (image) or prompt (in parameters) + * must be provided, or both. + */ +export interface ImageTextToImageInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. Either this or prompt must be + * provided. + */ + inputs?: Blob; + /** + * Additional inference parameters for Image Text To Image + */ + parameters?: ImageTextToImageParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image Text To Image + */ +export interface ImageTextToImageParameters { + /** + * For diffusion models. A higher guidance scale value encourages the model to generate + * images closely linked to the text prompt at the expense of lower image quality. + */ + guidance_scale?: number; + /** + * One prompt to guide what NOT to include in image generation. + */ + negative_prompt?: string; + /** + * For diffusion models. The number of denoising steps. More denoising steps usually lead to + * a higher quality image at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * The text prompt to guide the image generation. Either this or inputs (image) must be + * provided. + */ + prompt?: string; + /** + * Seed for the random number generator. + */ + seed?: number; + /** + * The size in pixels of the output image. This parameter is only supported by some + * providers and for specific models. It will be ignored when unsupported. + */ + target_size?: TargetSize; + [property: string]: unknown; +} +/** + * The size in pixels of the output image. This parameter is only supported by some + * providers and for specific models. It will be ignored when unsupported. + */ +export interface TargetSize { + height: number; + width: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Image Text To Image task + */ +export interface ImageTextToImageOutput { + /** + * The generated image returned as raw bytes in the payload. + */ + image: unknown; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..259353e337d7db9d6b479eb7e5f91f7df8364e2c --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/spec/input.json @@ -0,0 +1,59 @@ +{ + "$id": "/inference/schemas/image-text-to-image/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Image Text To Image inference. Either inputs (image) or prompt (in parameters) must be provided, or both.", + "title": "ImageTextToImageInput", + "type": "object", + "properties": { + "inputs": { + "type": "string", + "description": "The input image data as a base64-encoded string. If no `parameters` are provided, you can also provide the image data as a raw bytes payload. Either this or prompt must be provided.", + "comment": "type=binary" + }, + "parameters": { + "description": "Additional inference parameters for Image Text To Image", + "$ref": "#/$defs/ImageTextToImageParameters" + } + }, + "$defs": { + "ImageTextToImageParameters": { + "title": "ImageTextToImageParameters", + "type": "object", + "properties": { + "prompt": { + "type": "string", + "description": "The text prompt to guide the image generation. Either this or inputs (image) must be provided." + }, + "guidance_scale": { + "type": "number", + "description": "For diffusion models. A higher guidance scale value encourages the model to generate images closely linked to the text prompt at the expense of lower image quality." + }, + "negative_prompt": { + "type": "string", + "description": "One prompt to guide what NOT to include in image generation." + }, + "num_inference_steps": { + "type": "integer", + "description": "For diffusion models. The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference." + }, + "target_size": { + "type": "object", + "description": "The size in pixels of the output image. This parameter is only supported by some providers and for specific models. It will be ignored when unsupported.", + "properties": { + "width": { + "type": "integer" + }, + "height": { + "type": "integer" + } + }, + "required": ["width", "height"] + }, + "seed": { + "type": "integer", + "description": "Seed for the random number generator." + } + } + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..f5b782b6253ee6b9f35f1f4acc2d6513d03ce135 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-text-to-image/spec/output.json @@ -0,0 +1,13 @@ +{ + "$id": "/inference/schemas/image-text-to-image/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Image Text To Image task", + "title": "ImageTextToImageOutput", + "type": "object", + "properties": { + "image": { + "description": "The generated image returned as raw bytes in the payload." + } + }, + "required": ["image"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-text-to-text/about.md b/node_modules/@huggingface/tasks/src/tasks/image-text-to-text/about.md new file mode 100644 index 0000000000000000000000000000000000000000..8595509bb4cc7f85bf3a7d91d668e79e3e41367c --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-text-to-text/about.md @@ -0,0 +1,94 @@ +## Different Types of Vision Language Models + +Vision language models come in three types: + +- **Base:** Pre-trained models that can be fine-tuned. A good example of base models is the [PaliGemma models family](https://huggingface.co/models?sort=trending&search=google%2Fpaligemma-3b-pt) by Google. +- **Instruction:** Base models fine-tuned on instruction datasets. A good example of instruction fine-tuned models is [idefics2-8b](https://huggingface.co/HuggingFaceM4/idefics2-8b). +- **Chatty/Conversational:** Base models fine-tuned on conversation datasets. A good example of chatty models is [deepseek-vl-7b-chat](https://huggingface.co/deepseek-ai/deepseek-vl-7b-chat). + +![VLM uses](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/vlm/visual.jpg) + +## Use Cases + +### Multimodal Dialogue + +Vision language models can be used as multimodal assistants, keeping context about the conversation and keeping the image to have multiple-turn dialogues. + +### Zero-shot Object Detection, Image Segmentation and Localization + +Some vision language models can detect or segment a set of objects or describe the positions or relative positions of the objects. For example, one could prompt such a model to ask if one object is behind another. Such a model can also output bounding box coordination or segmentation masks directly in the text output, unlike the traditional models explicitly trained on only object detection or image segmentation. + +### Visual Question Answering + +Vision language models trained on image-text pairs can be used for visual question answering and generating captions for images. + +### Document Question Answering and Retrieval + +Documents often consist of different layouts, charts, tables, images, and more. Vision language models trained on formatted documents can extract information from them. This is an OCR-free approach; the inputs skip OCR, and documents are directly fed to vision language models. To find the relevant documents to be fed, models like [ColPali](https://huggingface.co/blog/manu/colpali) are used. An example workflow can be found [here](https://github.com/merveenoyan/smol-vision/blob/main/ColPali_%2B_Qwen2_VL.ipynb). + +### Image Recognition with Instructions + +Vision language models can recognize images through descriptions. When given detailed descriptions of specific entities, it can classify the entities in an image. + +### Computer Use + +Image-text-to-text models can be used to control computers with agentic workflows. Models like [ShowUI](https://huggingface.co/showlab/ShowUI-2B) and [OmniParser](https://huggingface.co/microsoft/OmniParser) are used to parse screenshots to later take actions on the computer autonomously. + +## Inference + +You can use the Transformers library to interact with [vision-language models](https://huggingface.co/models?pipeline_tag=image-text-to-text&transformers). Specifically, `pipeline` makes it easy to infer models. + +Initialize the pipeline first. + +```python +from transformers import pipeline + +pipe = pipeline("image-text-to-text", model="llava-hf/llava-interleave-qwen-0.5b-hf") +``` + +The model's built-in chat template will be used to format the conversational input. We can pass the image as an URL in the `content` part of the user message: + +```python +messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "image": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg", + }, + {"type": "text", "text": "Describe this image."}, + ], + } + ] + +``` + +We can now directly pass in the messages to the pipeline to infer. The `return_full_text` flag is used to return the full prompt in the response, including the user input. Here we pass `False` to only return the generated text. + +```python +outputs = pipe(text=messages, max_new_tokens=60, return_full_text=False) + +outputs[0]["generated_text"] +# The image captures a moment of tranquility in nature. At the center of the frame, a pink flower with a yellow center is in full bloom. The flower is surrounded by a cluster of red flowers, their vibrant color contrasting with the pink of the flower. \n\nA black and yellow bee is per +``` + +You can also use the Inference API to test image-text-to-text models. You need to use a [Hugging Face token](https://huggingface.co/settings/tokens) for authentication. + +```bash +curl https://router.huggingface.co/hf-inference/models/meta-llama/Llama-3.2-11B-Vision-Instruct \ + -X POST \ + -d '{"messages": [{"role": "user","content": [{"type": "image"}, {"type": "text", "text": "Can you describe the image?"}]}]}' \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer hf_***" +``` + +## Useful Resources + +- [Vision Language Models (Better, Faster, Stronger)](https://huggingface.co/blog/vlms-2025) +- [Vision Language Models Explained](https://huggingface.co/blog/vlms) +- [Welcome PaliGemma 2 – New vision language models by Google](https://huggingface.co/blog/paligemma2) +- [SmolVLM - small yet mighty Vision Language Model](https://huggingface.co/blog/smolvlm) +- [Multimodal RAG using ColPali and Qwen2-VL](https://github.com/merveenoyan/smol-vision/blob/main/ColPali_%2B_Qwen2_VL.ipynb) +- [Preference Optimization for Vision Language Models with TRL](https://huggingface.co/blog/dpo_vlm) +- [Image-text-to-text task guide](https://huggingface.co/docs/transformers/tasks/image_text_to_text) diff --git a/node_modules/@huggingface/tasks/src/tasks/image-text-to-text/data.ts b/node_modules/@huggingface/tasks/src/tasks/image-text-to-text/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..c1fbaed2dcf9c1fe15621f494e0814370ab281b6 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-text-to-text/data.ts @@ -0,0 +1,86 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "Instructions composed of image and text.", + id: "liuhaotian/LLaVA-Instruct-150K", + }, + { + description: "Collection of image-text pairs on scientific topics.", + id: "DAMO-NLP-SG/multimodal_textbook", + }, + { + description: "A collection of datasets made for model fine-tuning.", + id: "HuggingFaceM4/the_cauldron", + }, + { + description: "Screenshots of websites with their HTML/CSS codes.", + id: "HuggingFaceM4/WebSight", + }, + ], + demo: { + inputs: [ + { + filename: "image-text-to-text-input.png", + type: "img", + }, + { + label: "Text Prompt", + content: "Describe the position of the bee in detail.", + type: "text", + }, + ], + outputs: [ + { + label: "Answer", + content: + "The bee is sitting on a pink flower, surrounded by other flowers. The bee is positioned in the center of the flower, with its head and front legs sticking out.", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "Small and efficient yet powerful vision language model.", + id: "HuggingFaceTB/SmolVLM-Instruct", + }, + { + description: "Cutting-edge reasoning vision language model.", + id: "zai-org/GLM-4.5V", + }, + { + description: "Cutting-edge small vision language model to convert documents to text.", + id: "rednote-hilab/dots.ocr", + }, + { + description: "Small yet powerful model.", + id: "Qwen/Qwen2.5-VL-3B-Instruct", + }, + { + description: "Image-text-to-text model with agentic capabilities.", + id: "microsoft/Magma-8B", + }, + ], + spaces: [ + { + description: "Leaderboard to evaluate vision language models.", + id: "opencompass/open_vlm_leaderboard", + }, + { + description: "An application that compares object detection capabilities of different vision language models.", + id: "sergiopaniego/vlm_object_understanding", + }, + { + description: "An application to compare different OCR models.", + id: "prithivMLmods/Multimodal-OCR", + }, + ], + summary: + "Image-text-to-text models take in an image and text prompt and output text. These models are also called vision-language models, or VLMs. The difference from image-to-text models is that these models take an additional text input, not restricting the model to certain use cases like image captioning, and may also be trained to accept a conversation as input.", + widgetModels: ["zai-org/GLM-4.5V"], + youtubeId: "IoGaGfU1CIg", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/about.md b/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/about.md new file mode 100644 index 0000000000000000000000000000000000000000..0119b6a5171c4b236778b70863ef8df81bf18408 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/about.md @@ -0,0 +1,71 @@ +## Use Cases + +### Image Animation + +Image-text-to-video models can be used to animate still images based on text descriptions. For example, you can provide a landscape photo and the instruction "A camera pan from left to right" to create a video with camera movement. + +### Dynamic Content Creation + +Transform images into video by adding motion, transformations, or effects described in text prompts. This is useful for creating engaging social media content, presentations, or marketing materials. + +### Guided Video Generation + +Use a reference image with text prompts to guide the video generation process. This provides more control over the visual style and composition compared to text-to-video models alone. + +### Story Visualization + +Create video sequences from storyboards or concept art by providing scene descriptions. This can help filmmakers and animators visualize scenes before production. + +### Motion Control + +Generate videos with specific camera movements, object motions, or scene transitions by combining reference images with detailed motion descriptions. + +## Task Variants + +### Image-to-Video with Motion Control + +Models that generate videos from images while following specific motion instructions, such as camera movements, object animations, or scene dynamics. + +### Reference-guided Video Generation + +Models that use a reference image to guide the visual style and composition of the generated video while incorporating text prompts for motion and transformation control. + +### Conditional Video Synthesis + +Models that perform specific video transformations based on text conditions, such as adding weather effects, time-of-day changes, or environmental animations. + +## Inference + +You can use the Diffusers library to interact with image-text-to-video models. Here's example snippet to use `LTXImageToVideoPipeline`. + +```python +import torch +from diffusers import LTXImageToVideoPipeline +from diffusers.utils import export_to_video, load_image + +pipe = LTXImageToVideoPipeline.from_pretrained("Lightricks/LTX-Video", torch_dtype=torch.bfloat16) +pipe.to("cuda") + +image = load_image( + "https://huggingface.co/datasets/a-r-r-o-w/tiny-meme-dataset-captioned/resolve/main/images/8.png" +) +prompt = "A young girl stands calmly in the foreground, looking directly at the camera, as a house fire rages in the background. Flames engulf the structure, with smoke billowing into the air. Firefighters in protective gear rush to the scene, a fire truck labeled '38' visible behind them. The girl's neutral expression contrasts sharply with the chaos of the fire, creating a poignant and emotionally charged scene." +negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted" + +video = pipe( + image=image, + prompt=prompt, + negative_prompt=negative_prompt, + width=704, + height=480, + num_frames=161, + num_inference_steps=50, +).frames[0] +export_to_video(video, "output.mp4", fps=24) +``` + +## Useful Resources + +- [LTX-Video Model Card](https://huggingface.co/Lightricks/LTX-Video) +- [Text-to-Video: The Task, Challenges and the Current State](https://huggingface.co/blog/text-to-video) +- [Diffusers documentation on Video Generation](https://huggingface.co/docs/diffusers/using-diffusers/text-img2vid) diff --git a/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/data.ts b/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..ded013562dad10c8043ee4108d2f5a6ad9c006a4 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/data.ts @@ -0,0 +1,54 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [], + demo: { + inputs: [ + { + filename: "image-text-to-video-input.jpg", + type: "img", + }, + { + label: "Input", + content: "Darth Vader is surfing on the waves.", + type: "text", + }, + ], + outputs: [ + { + filename: "image-text-to-video-output.gif", + type: "img", + }, + ], + }, + metrics: [ + { + description: + "Frechet Video Distance uses a model that captures coherence for changes in frames and the quality of each frame. A smaller score indicates better video generation.", + id: "fvd", + }, + { + description: + "CLIPSIM measures similarity between video frames and text using an image-text similarity model. A higher score indicates better video generation.", + id: "clipsim", + }, + ], + models: [ + { + description: "A powerful model for image-text-to-video generation.", + id: "Lightricks/LTX-Video", + }, + ], + spaces: [ + { + description: "An application for image-text-to-video generation.", + id: "Lightricks/ltx-video-distilled", + }, + ], + summary: + "Image-text-to-video models take an reference image and a text instructions as and generate a video based on them. These models are useful for animating still images, creating dynamic content from static references, and generating videos with specific motion or transformation guidance.", + widgetModels: ["Lightricks/LTX-Video"], + youtubeId: undefined, +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/inference.ts b/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..ad048a7bc955b04fe119fd09461190eb4aa2a3df --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/inference.ts @@ -0,0 +1,77 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image Text To Video inference. Either inputs (image) or prompt (in parameters) + * must be provided, or both. + */ +export interface ImageTextToVideoInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. Either this or prompt must be + * provided. + */ + inputs?: Blob; + /** + * Additional inference parameters for Image Text To Video + */ + parameters?: ImageTextToVideoParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image Text To Video + */ +export interface ImageTextToVideoParameters { + /** + * For diffusion models. A higher guidance scale value encourages the model to generate + * videos closely linked to the text prompt at the expense of lower image quality. + */ + guidance_scale?: number; + /** + * One prompt to guide what NOT to include in video generation. + */ + negative_prompt?: string; + /** + * The num_frames parameter determines how many video frames are generated. + */ + num_frames?: number; + /** + * The number of denoising steps. More denoising steps usually lead to a higher quality + * video at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * The text prompt to guide the video generation. Either this or inputs (image) must be + * provided. + */ + prompt?: string; + /** + * Seed for the random number generator. + */ + seed?: number; + /** + * The size in pixel of the output video frames. + */ + target_size?: TargetSize; + [property: string]: unknown; +} +/** + * The size in pixel of the output video frames. + */ +export interface TargetSize { + height: number; + width: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Image Text To Video task + */ +export interface ImageTextToVideoOutput { + /** + * The generated video returned as raw bytes in the payload. + */ + video: unknown; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..3874073e48c9ac50c1ecd9c45c60fae9ac9a71f9 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/spec/input.json @@ -0,0 +1,63 @@ +{ + "$id": "/inference/schemas/image-text-to-video/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Image Text To Video inference. Either inputs (image) or prompt (in parameters) must be provided, or both.", + "title": "ImageTextToVideoInput", + "type": "object", + "properties": { + "inputs": { + "type": "string", + "description": "The input image data as a base64-encoded string. If no `parameters` are provided, you can also provide the image data as a raw bytes payload. Either this or prompt must be provided.", + "comment": "type=binary" + }, + "parameters": { + "description": "Additional inference parameters for Image Text To Video", + "$ref": "#/$defs/ImageTextToVideoParameters" + } + }, + "$defs": { + "ImageTextToVideoParameters": { + "title": "ImageTextToVideoParameters", + "type": "object", + "properties": { + "prompt": { + "type": "string", + "description": "The text prompt to guide the video generation. Either this or inputs (image) must be provided." + }, + "guidance_scale": { + "type": "number", + "description": "For diffusion models. A higher guidance scale value encourages the model to generate videos closely linked to the text prompt at the expense of lower image quality." + }, + "negative_prompt": { + "type": "string", + "description": "One prompt to guide what NOT to include in video generation." + }, + "num_inference_steps": { + "type": "integer", + "description": "The number of denoising steps. More denoising steps usually lead to a higher quality video at the expense of slower inference." + }, + "num_frames": { + "type": "number", + "description": "The num_frames parameter determines how many video frames are generated." + }, + "target_size": { + "type": "object", + "description": "The size in pixel of the output video frames.", + "properties": { + "width": { + "type": "integer" + }, + "height": { + "type": "integer" + } + }, + "required": ["width", "height"] + }, + "seed": { + "type": "integer", + "description": "Seed for the random number generator." + } + } + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..26a08e9be533d6a0770be2a6d084bc7941cd0f16 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-text-to-video/spec/output.json @@ -0,0 +1,13 @@ +{ + "$id": "/inference/schemas/image-text-to-video/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Image Text To Video task", + "title": "ImageTextToVideoOutput", + "type": "object", + "properties": { + "video": { + "description": "The generated video returned as raw bytes in the payload." + } + }, + "required": ["video"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-3d/about.md b/node_modules/@huggingface/tasks/src/tasks/image-to-3d/about.md new file mode 100644 index 0000000000000000000000000000000000000000..dca8e4708190708275de87ebf0b52aa2dc7f99c3 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-3d/about.md @@ -0,0 +1,62 @@ +## Use Cases + +Image-to-3D models can be used in a wide variety of applications that require 3D, such as games, animation, design, architecture, engineering, marketing, and more. + +![Image-to-3D Thumbnail](https://huggingface.co/datasets/huggingfacejs/tasks/resolve/main/image-to-3d/image-to-3d-thumbnail.png) + +### Generating Meshes + +Meshes are the standard representation of 3D in industry. + +### Generating Gaussian Splats + +[Gaussian Splatting](https://huggingface.co/blog/gaussian-splatting) is a rendering technique that represents scenes as fuzzy points. + +### Inference + +Inference for this task typically leverages the [Diffusers](https://huggingface.co/docs/diffusers/index) library for inference, using [Custom Pipelines](https://huggingface.co/docs/diffusers/v0.6.0/en/using-diffusers/custom_pipelines). + +These are unstandardized and depend on the model. More details can be found in each model repository. + +```python +import torch +import requests +import numpy as np +from io import BytesIO +from diffusers import DiffusionPipeline +from PIL import Image + +pipeline = DiffusionPipeline.from_pretrained( + "dylanebert/LGM-full", + custom_pipeline="dylanebert/LGM-full", + torch_dtype=torch.float16, + trust_remote_code=True, +).to("cuda") + +input_url = "https://huggingface.co/datasets/dylanebert/iso3d/resolve/main/jpg@512/a_cat_statue.jpg" +input_image = Image.open(BytesIO(requests.get(input_url).content)) +input_image = np.array(input_image, dtype=np.float32) / 255.0 +result = pipeline("", input_image) +result_path = "/tmp/output.ply" +pipeline.save_ply(result, result_path) +``` + +In the code above, we: + +1. Import the necessary libraries +2. Load the `LGM-full` model and custom pipeline +3. Load and preprocess the input image +4. Run the pipeline on the input image +5. Save the output to a file + +### Output Formats + +Meshes can be in `.obj`, `.glb`, `.stl`, or `.gltf` format. Other formats are allowed, but won't be rendered in the gradio [Model3D](https://www.gradio.app/docs/gradio/model3d) component. + +Splats can be in `.ply` or `.splat` format. They can be rendered in the gradio [Model3D](https://www.gradio.app/docs/gradio/model3d) component using the [gsplat.js](https://github.com/huggingface/gsplat.js) library. + +## Useful Resources + +- [ML for 3D Course](https://huggingface.co/learn/ml-for-3d-course) +- [3D Arena Leaderboard](https://huggingface.co/spaces/dylanebert/3d-arena) +- [gsplat.js](https://github.com/huggingface/gsplat.js) diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-3d/data.ts b/node_modules/@huggingface/tasks/src/tasks/image-to-3d/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..61e8e50c758ec13b640e989bc355d6fe0f76d542 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-3d/data.ts @@ -0,0 +1,75 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A large dataset of over 10 million 3D objects.", + id: "allenai/objaverse-xl", + }, + { + description: "A dataset of isolated object images for evaluating image-to-3D models.", + id: "dylanebert/iso3d", + }, + ], + demo: { + inputs: [ + { + filename: "image-to-3d-image-input.png", + type: "img", + }, + ], + outputs: [ + { + label: "Result", + content: "image-to-3d-3d-output-filename.glb", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "Fast image-to-3D mesh model by Tencent.", + id: "TencentARC/InstantMesh", + }, + { + description: "3D world generation model.", + id: "tencent/HunyuanWorld-1", + }, + { + description: "A scaled up image-to-3D mesh model derived from TripoSR.", + id: "hwjiang/Real3D", + }, + { + description: "Consistent image-to-3d generation model.", + id: "stabilityai/stable-point-aware-3d", + }, + ], + spaces: [ + { + description: "Leaderboard to evaluate image-to-3D models.", + id: "dylanebert/3d-arena", + }, + { + description: "Image-to-3D demo with mesh outputs.", + id: "TencentARC/InstantMesh", + }, + { + description: "Image-to-3D demo.", + id: "stabilityai/stable-point-aware-3d", + }, + { + description: "Image-to-3D demo with mesh outputs.", + id: "hwjiang/Real3D", + }, + { + description: "Image-to-3D demo with splat outputs.", + id: "dylanebert/LGM-mini", + }, + ], + summary: "Image-to-3D models take in image input and produce 3D output.", + widgetModels: [], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-image/about.md b/node_modules/@huggingface/tasks/src/tasks/image-to-image/about.md new file mode 100644 index 0000000000000000000000000000000000000000..b2e8b69131670c64ed5ea377418c5903dcafcbd7 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-image/about.md @@ -0,0 +1,129 @@ +Image-to-image pipelines can also be used in text-to-image tasks, to provide visual guidance to the text-guided generation process. + +## Use Cases + +### Image inpainting + +Image inpainting is widely used during photography editing to remove unwanted objects, such as poles, wires, or sensor dust. + +### Image colorization + +Old or black and white images can be brought up to life using an image colorization model. + +### Super Resolution + +Super-resolution models increase the resolution of an image, allowing for higher-quality viewing and printing. + +## Inference + +You can use pipelines for image-to-image in 🧨diffusers library to easily use image-to-image models. See an example for `StableDiffusionImg2ImgPipeline` below. + +```python +import torch +from diffusers import AutoPipelineForImage2Image +from diffusers.utils import make_image_grid, load_image + +pipeline = AutoPipelineForImage2Image.from_pretrained( + "stabilityai/stable-diffusion-xl-refiner-1.0", torch_dtype=torch.float16, variant="fp16", use_safetensors=True +) + +# this helps us to reduce memory usage- since SDXL is a bit heavy, this could help by +# offloading the model to CPU w/o hurting performance. +pipeline.enable_model_cpu_offload() + +# prepare image +url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/img2img-sdxl-init.png" +init_image = load_image(url) + +prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" + +# pass prompt and image to pipeline +image = pipeline(prompt, image=init_image, strength=0.5).images[0] +make_image_grid([init_image, image], rows=1, cols=2) +``` + +You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer image-to-image models on Hugging Face Hub. + +```javascript +import { InferenceClient } from "@huggingface/inference"; + +const inference = new InferenceClient(HF_TOKEN); +await inference.imageToImage({ + data: await (await fetch("image")).blob(), + model: "timbrooks/instruct-pix2pix", + parameters: { + prompt: "Deblur this image", + }, +}); +``` + +## Uses Cases for Text Guided Image Generation + +### Style Transfer + +One of the most popular use cases of image-to-image is style transfer. With style transfer models: + +- a regular photo can be transformed into a variety of artistic styles or genres, such as a watercolor painting, a comic book illustration and more. +- new images can be generated using a text prompt, in the style of a reference input image. + +See 🧨diffusers example for style transfer with `AutoPipelineForText2Image` below. + +```python +from diffusers import AutoPipelineForText2Image +from diffusers.utils import load_image +import torch + +# load pipeline +pipeline = AutoPipelineForText2Image.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16).to("cuda") +pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter_sdxl.bin") + +# set the adapter and scales - this is a component that lets us add the style control from an image to the text-to-image model +scale = { + "down": {"block_2": [0.0, 1.0]}, + "up": {"block_0": [0.0, 1.0, 0.0]}, +} +pipeline.set_ip_adapter_scale(scale) + +style_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg") + +generator = torch.Generator(device="cpu").manual_seed(26) +image = pipeline( + prompt="a cat, masterpiece, best quality, high quality", + ip_adapter_image=style_image, + negative_prompt="text, watermark, lowres, low quality, worst quality, deformed, glitch, low contrast, noisy, saturation, blurry", + guidance_scale=5, + num_inference_steps=30, + generator=generator, +).images[0] +image +``` + +### ControlNet + +Controlling the outputs of diffusion models only with a text prompt is a challenging problem. ControlNet is a neural network model that provides image-based control to diffusion models. Control images can be edges or other landmarks extracted from a source image. +![Examples](https://huggingface.co/datasets/optimum/documentation-images/resolve/main/neuron/models/12-sdxl-text2img-controlnet.png) + +## Pix2Pix + +Pix2Pix is a popular model used for image-to-image translation tasks. It is based on a conditional-GAN (generative adversarial network) where instead of a noise vector a 2D image is given as input. More information about Pix2Pix can be retrieved from this [link](https://phillipi.github.io/pix2pix/) where the associated paper and the GitHub repository can be found. + +The images below show some examples extracted from the Pix2Pix paper. This model can be applied to various use cases. It is capable of relatively simpler things, e.g., converting a grayscale image to its colored version. But more importantly, it can generate realistic pictures from rough sketches (can be seen in the purse example) or from painting-like images (can be seen in the street and facade examples below). + +![Examples](https://huggingface.co/datasets/huggingfacejs/tasks/resolve/main/image-to-image/pix2pix_examples.jpg) + +## Useful Resources + +- [Image-to-image guide with diffusers](https://huggingface.co/docs/diffusers/using-diffusers/img2img) +- Image inpainting: [inpainting with 🧨diffusers](https://huggingface.co/docs/diffusers/main/en/using-diffusers/inpaint#inpainting), [demo](https://huggingface.co/spaces/linoyts/Qwen-Image-Edit-Inpaint) +- Colorization: [demo](https://huggingface.co/spaces/modelscope/old_photo_restoration) +- Super resolution: [image upscaling with 🧨diffusers](https://huggingface.co/docs/diffusers/main/en/api/pipelines/stable_diffusion/upscale#super-resolution), [demo](https://huggingface.co/spaces/radames/Enhance-This-HiDiffusion-SDXL) +- [Style transfer and layout control with diffusers 🧨](https://huggingface.co/docs/diffusers/main/en/using-diffusers/ip_adapter#style--layout-control) +- [Train your ControlNet with diffusers 🧨](https://huggingface.co/blog/train-your-controlnet) +- [Ultra fast ControlNet with 🧨 Diffusers](https://huggingface.co/blog/controlnet) +- [List of ControlNets trained in the community JAX Diffusers sprint](https://huggingface.co/spaces/jax-diffusers-event/leaderboard) + +## References + +[1] P. Isola, J. -Y. Zhu, T. Zhou and A. A. Efros, "Image-to-Image Translation with Conditional Adversarial Networks," 2017 IEEE Conference on Computer Vision and Pattern Recognition (CVPR), 2017, pp. 5967-5976, doi: 10.1109/CVPR.2017.632. + +This page was made possible thanks to the efforts of [Paul Gafton](https://github.com/Paul92) and [Osman Alenbey](https://huggingface.co/osman93). diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-image/data.ts b/node_modules/@huggingface/tasks/src/tasks/image-to-image/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..d0da0241e23d8b66032f395b04c9190f5dc2942a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-image/data.ts @@ -0,0 +1,96 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "Synthetic dataset, for image relighting", + id: "VIDIT", + }, + { + description: "Multiple images of celebrities, used for facial expression translation", + id: "huggan/CelebA-faces", + }, + { + description: "12M image-caption pairs.", + id: "Spawning/PD12M", + }, + ], + demo: { + inputs: [ + { + filename: "image-to-image-input.jpeg", + type: "img", + }, + ], + outputs: [ + { + filename: "image-to-image-output.png", + type: "img", + }, + ], + }, + isPlaceholder: false, + metrics: [ + { + description: + "Peak Signal to Noise Ratio (PSNR) is an approximation of the human perception, considering the ratio of the absolute intensity with respect to the variations. Measured in dB, a high value indicates a high fidelity.", + id: "PSNR", + }, + { + description: + "Structural Similarity Index (SSIM) is a perceptual metric which compares the luminance, contrast and structure of two images. The values of SSIM range between -1 and 1, and higher values indicate closer resemblance to the original image.", + id: "SSIM", + }, + { + description: + "Inception Score (IS) is an analysis of the labels predicted by an image classification model when presented with a sample of the generated images.", + id: "IS", + }, + ], + models: [ + { + description: "An image-to-image model to improve image resolution.", + id: "fal/AuraSR-v2", + }, + { + description: "Powerful image editing model.", + id: "black-forest-labs/FLUX.1-Kontext-dev", + }, + { + description: "Virtual try-on model.", + id: "yisol/IDM-VTON", + }, + { + description: "Image re-lighting model.", + id: "kontext-community/relighting-kontext-dev-lora-v3", + }, + { + description: "Strong model for inpainting and outpainting.", + id: "black-forest-labs/FLUX.1-Fill-dev", + }, + { + description: "Strong model for image editing using depth maps.", + id: "black-forest-labs/FLUX.1-Depth-dev-lora", + }, + ], + spaces: [ + { + description: "Image editing application.", + id: "black-forest-labs/FLUX.1-Kontext-Dev", + }, + { + description: "Image relighting application.", + id: "lllyasviel/iclight-v2-vary", + }, + { + description: "An application for image upscaling.", + id: "jasperai/Flux.1-dev-Controlnet-Upscaler", + }, + ], + summary: + "Image-to-image is the task of transforming an input image through a variety of possible manipulations and enhancements, such as super-resolution, image inpainting, colorization, and more.", + widgetModels: ["Qwen/Qwen-Image"], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-image/inference.ts b/node_modules/@huggingface/tasks/src/tasks/image-to-image/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..0a681093ca15521e9c8a51bf1db042f094fff27f --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-image/inference.ts @@ -0,0 +1,68 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image To Image inference + */ +export interface ImageToImageInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Image To Image + */ + parameters?: ImageToImageParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image To Image + */ +export interface ImageToImageParameters { + /** + * For diffusion models. A higher guidance scale value encourages the model to generate + * images closely linked to the text prompt at the expense of lower image quality. + */ + guidance_scale?: number; + /** + * One prompt to guide what NOT to include in image generation. + */ + negative_prompt?: string; + /** + * For diffusion models. The number of denoising steps. More denoising steps usually lead to + * a higher quality image at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * The text prompt to guide the image generation. + */ + prompt?: string; + /** + * The size in pixels of the output image. This parameter is only supported by some + * providers and for specific models. It will be ignored when unsupported. + */ + target_size?: TargetSize; + [property: string]: unknown; +} +/** + * The size in pixels of the output image. This parameter is only supported by some + * providers and for specific models. It will be ignored when unsupported. + */ +export interface TargetSize { + height: number; + width: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Image To Image task + */ +export interface ImageToImageOutput { + /** + * The output image returned as raw bytes in the payload. + */ + image?: unknown; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-image/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/image-to-image/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..49e8695c444db86328688df6df2119bfa1838587 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-image/spec/input.json @@ -0,0 +1,56 @@ +{ + "$id": "/inference/schemas/image-to-image/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Image To Image inference", + "title": "ImageToImageInput", + "type": "object", + "properties": { + "inputs": { + "type": "string", + "description": "The input image data as a base64-encoded string. If no `parameters` are provided, you can also provide the image data as a raw bytes payload.", + "comment": "type=binary" + }, + "parameters": { + "description": "Additional inference parameters for Image To Image", + "$ref": "#/$defs/ImageToImageParameters" + } + }, + "$defs": { + "ImageToImageParameters": { + "title": "ImageToImageParameters", + "type": "object", + "properties": { + "prompt": { + "type": "string", + "description": "The text prompt to guide the image generation." + }, + "guidance_scale": { + "type": "number", + "description": "For diffusion models. A higher guidance scale value encourages the model to generate images closely linked to the text prompt at the expense of lower image quality." + }, + "negative_prompt": { + "type": "string", + "description": "One prompt to guide what NOT to include in image generation." + }, + "num_inference_steps": { + "type": "integer", + "description": "For diffusion models. The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference." + }, + "target_size": { + "type": "object", + "description": "The size in pixels of the output image. This parameter is only supported by some providers and for specific models. It will be ignored when unsupported.", + "properties": { + "width": { + "type": "integer" + }, + "height": { + "type": "integer" + } + }, + "required": ["width", "height"] + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-image/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/image-to-image/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..043544e7516fb38688967c0acf6dff933d3da64a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-image/spec/output.json @@ -0,0 +1,12 @@ +{ + "$id": "/inference/schemas/image-to-image/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Image To Image task", + "title": "ImageToImageOutput", + "type": "object", + "properties": { + "image": { + "description": "The output image returned as raw bytes in the payload." + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-text/about.md b/node_modules/@huggingface/tasks/src/tasks/image-to-text/about.md new file mode 100644 index 0000000000000000000000000000000000000000..403dbdc41de67241bfab8f6535f3977e5a7a570e --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-text/about.md @@ -0,0 +1,61 @@ +## Use Cases + +### Image Captioning + +Image Captioning is the process of generating textual description of an image. +This can help the visually impaired people to understand what's happening in their surroundings. + +### Optical Character Recognition (OCR) + +OCR models convert the text present in an image, e.g. a scanned document, to text. + +## Inference + +### Image Captioning + +You can use the 🤗 Transformers library's `image-to-text` pipeline to generate caption for the Image input. + +```python +from transformers import pipeline + +captioner = pipeline("image-to-text", model="Salesforce/blip-image-captioning-base") +captioner("https://huggingface.co/datasets/Narsil/image_dummy/resolve/main/parrots.png") +## [{'generated_text': 'two birds are standing next to each other '}] +``` + +### OCR + +This code snippet uses Microsoft’s TrOCR, an encoder-decoder model consisting of an image Transformer encoder and a text Transformer decoder for state-of-the-art optical character recognition (OCR) on single-text line images. + +```python +from transformers import TrOCRProcessor, VisionEncoderDecoderModel + +processor = TrOCRProcessor.from_pretrained('microsoft/trocr-base-handwritten') +model = VisionEncoderDecoderModel.from_pretrained('microsoft/trocr-base-handwritten') +pixel_values = processor(images="image.jpeg", return_tensors="pt").pixel_values + +generated_ids = model.generate(pixel_values) +generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] + +``` + +You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer image-to-text models on Hugging Face Hub. + +```javascript +import { InferenceClient } from "@huggingface/inference"; + +const inference = new InferenceClient(HF_TOKEN); +await inference.imageToText({ + data: await (await fetch("https://picsum.photos/300/300")).blob(), + model: "Salesforce/blip-image-captioning-base", +}); +``` + +## Useful Resources + +- [Image Captioning](https://huggingface.co/docs/transformers/main/en/tasks/image_captioning) +- [Image Captioning Use Case](https://blog.google/outreach-initiatives/accessibility/get-image-descriptions/) +- [Train Image Captioning model on your dataset](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/GIT/Fine_tune_GIT_on_an_image_captioning_dataset.ipynb) +- [Train OCR model on your dataset ](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/TrOCR) + +This page was made possible thanks to efforts of [Sukesh Perla](https://huggingface.co/hitchhiker3010) and [Johannes Kolbe](https://huggingface.co/johko). diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-text/data.ts b/node_modules/@huggingface/tasks/src/tasks/image-to-text/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..2cda2e765f97fed36231bc8607febb7e483350bc --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-text/data.ts @@ -0,0 +1,62 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + // TODO write proper description + description: "Dataset from 12M image-text of Reddit", + id: "red_caps", + }, + { + // TODO write proper description + description: "Dataset from 3.3M images of Google", + id: "datasets/conceptual_captions", + }, + ], + demo: { + inputs: [ + { + filename: "savanna.jpg", + type: "img", + }, + ], + outputs: [ + { + label: "Detailed description", + content: "a herd of giraffes and zebras grazing in a field", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "Strong OCR model.", + id: "allenai/olmOCR-7B-0725", + }, + { + description: "Powerful image captioning model.", + id: "fancyfeast/llama-joycaption-beta-one-hf-llava", + }, + ], + spaces: [ + { + description: "SVG generator app from images.", + id: "multimodalart/OmniSVG-3B", + }, + { + description: "An application that converts documents to markdown.", + id: "numind/NuMarkdown-8B-Thinking", + }, + { + description: "An application that can caption images.", + id: "fancyfeast/joy-caption-beta-one", + }, + ], + summary: + "Image to text models output a text from a given image. Image captioning or optical character recognition can be considered as the most common applications of image to text.", + widgetModels: ["Salesforce/blip-image-captioning-large"], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-text/inference.ts b/node_modules/@huggingface/tasks/src/tasks/image-to-text/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..5c6bbe30c13bee2064c1e897c29a5945de8f7738 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-text/inference.ts @@ -0,0 +1,134 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image To Text inference + */ +export interface ImageToTextInput { + /** + * The input image data + */ + inputs: Blob; + /** + * Additional inference parameters for Image To Text + */ + parameters?: ImageToTextParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image To Text + */ +export interface ImageToTextParameters { + /** + * Parametrization of the text generation process + */ + generation_parameters?: GenerationParameters; + /** + * The amount of maximum tokens to generate. + */ + max_new_tokens?: number; + [property: string]: unknown; +} +/** + * Parametrization of the text generation process + */ +export interface GenerationParameters { + /** + * Whether to use sampling instead of greedy decoding when generating new tokens. + */ + do_sample?: boolean; + /** + * Controls the stopping condition for beam-based methods. + */ + early_stopping?: EarlyStoppingUnion; + /** + * If set to float strictly between 0 and 1, only tokens with a conditional probability + * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + * Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + */ + epsilon_cutoff?: number; + /** + * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + * float strictly between 0 and 1, a token is only considered if it is greater than either + * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + * for more details. + */ + eta_cutoff?: number; + /** + * The maximum length (in tokens) of the generated text, including the input. + */ + max_length?: number; + /** + * The maximum number of tokens to generate. Takes precedence over max_length. + */ + max_new_tokens?: number; + /** + * The minimum length (in tokens) of the generated text, including the input. + */ + min_length?: number; + /** + * The minimum number of tokens to generate. Takes precedence over min_length. + */ + min_new_tokens?: number; + /** + * Number of groups to divide num_beams into in order to ensure diversity among different + * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + */ + num_beam_groups?: number; + /** + * Number of beams to use for beam search. + */ + num_beams?: number; + /** + * The value balances the model confidence and the degeneration penalty in contrastive + * search decoding. + */ + penalty_alpha?: number; + /** + * The value used to modulate the next token probabilities. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * If set to float < 1, only the smallest set of most probable tokens with probabilities + * that add up to top_p or higher are kept for generation. + */ + top_p?: number; + /** + * Local typicality measures how similar the conditional probability of predicting a target + * token next is to the expected conditional probability of predicting a random token next, + * given the partial text already generated. If set to float < 1, the smallest set of the + * most locally typical tokens with probabilities that add up to typical_p or higher are + * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + */ + typical_p?: number; + /** + * Whether the model should use the past last key/values attentions to speed up decoding + */ + use_cache?: boolean; + [property: string]: unknown; +} +/** + * Controls the stopping condition for beam-based methods. + */ +export type EarlyStoppingUnion = boolean | "never"; +/** + * Outputs of inference for the Image To Text task + */ +export interface ImageToTextOutput { + generatedText: unknown; + /** + * The generated text. + */ + generated_text?: string; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-text/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/image-to-text/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..26915b34a7511c7a0da959cb83382bd2bbba6efc --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-text/spec/input.json @@ -0,0 +1,34 @@ +{ + "$id": "/inference/schemas/image-to-text/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Image To Text inference", + "title": "ImageToTextInput", + "type": "object", + "properties": { + "inputs": { + "description": "The input image data", + "comment": "type=binary" + }, + "parameters": { + "description": "Additional inference parameters for Image To Text", + "$ref": "#/$defs/ImageToTextParameters" + } + }, + "$defs": { + "ImageToTextParameters": { + "title": "ImageToTextParameters", + "type": "object", + "properties": { + "max_new_tokens": { + "type": "integer", + "description": "The amount of maximum tokens to generate." + }, + "generation_parameters": { + "description": "Parametrization of the text generation process", + "$ref": "/inference/schemas/common-definitions.json#/definitions/GenerationParameters" + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-text/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/image-to-text/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..388c3456f4e7f50b0c7b133725a2d951f152cb01 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-text/spec/output.json @@ -0,0 +1,14 @@ +{ + "$id": "/inference/schemas/image-to-text/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Image To Text task", + "title": "ImageToTextOutput", + "type": "object", + "properties": { + "generated_text": { + "type": "string", + "description": "The generated text." + } + }, + "required": ["generatedText"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-video/about.md b/node_modules/@huggingface/tasks/src/tasks/image-to-video/about.md new file mode 100644 index 0000000000000000000000000000000000000000..1b32dde6e61a0f96fb1372b3eaa31301eef1ade5 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-video/about.md @@ -0,0 +1,51 @@ +## Use Cases + +Image-to-video models transform a static image into a video sequence. This can be used for a variety of creative and practical applications. + +### Animated Images + +Bring still photos to life by adding subtle motion or creating short animated clips. This is great for social media content or dynamic presentations. + +### Storytelling from a Single Frame + +Expand on the narrative of an image by generating a short video that imagines what happened before or after the moment captured in the photo. + +### Video Generation with Visual Consistency + +Use an input image as a strong visual anchor to guide the generation of a video, ensuring that the style, characters, or objects in the video remain consistent with the source image. + +### Controllable Motion + +Image-to-video models can be used to specify the direction or intensity of motion or camera control, giving more fine-grained control over the generated animation. + +## Inference + +Running the model Wan 2.1 T2V 1.3B with diffusers + +```py +import torch +from diffusers import AutoencoderKLWan, WanPipeline +from diffusers.utils import export_to_video + +model_id = "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" +vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", torch_dtype=torch.float32) +pipe = WanPipeline.from_pretrained(model_id, vae=vae, torch_dtype=torch.bfloat16) +pipe.to("cuda") + +prompt = "A cat walks on the grass, realistic" +negative_prompt = "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards" + +output = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + height=480, + width=832, + num_frames=81, + guidance_scale=5.0 +).frames[0] +export_to_video(output, "output.mp4", fps=15) +``` + +## Useful Resources + +To train image-to-video LoRAs check out [finetrainers](https://github.com/a-r-r-o-w/finetrainers) and [musubi trainer](https://github.com/kohya-ss/musubi-tuner). diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-video/data.ts b/node_modules/@huggingface/tasks/src/tasks/image-to-video/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..66ee4c1162ae40ea6caab24ee98eb7844d805414 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-video/data.ts @@ -0,0 +1,126 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A benchmark dataset for reference image controlled video generation.", + id: "ali-vilab/VACE-Benchmark", + }, + { + description: "A dataset of video generation style preferences.", + id: "Rapidata/sora-video-generation-style-likert-scoring", + }, + { + description: "A dataset with videos and captions throughout the videos.", + id: "BestWishYsh/ChronoMagic", + }, + ], + demo: { + inputs: [ + { + filename: "image-to-video-input.jpg", + type: "img", + }, + { + label: "Optional Text Prompt", + content: "This penguin is dancing", + type: "text", + }, + ], + outputs: [ + { + filename: "image-to-video-output.gif", + type: "img", + }, + ], + }, + metrics: [ + { + description: + "Fréchet Video Distance (FVD) measures the perceptual similarity between the distributions of generated videos and a set of real videos, assessing overall visual quality and temporal coherence of the video generated from an input image.", + id: "fvd", + }, + { + description: + "CLIP Score measures the semantic similarity between a textual prompt (if provided alongside the input image) and the generated video frames. It evaluates how well the video's generated content and motion align with the textual description, conditioned on the initial image.", + id: "clip_score", + }, + { + description: + "First Frame Fidelity, often measured using LPIPS (Learned Perceptual Image Patch Similarity), PSNR, or SSIM, quantifies how closely the first frame of the generated video matches the input conditioning image.", + id: "lpips", + }, + { + description: + "Identity Preservation Score measures the consistency of identity (e.g., a person's face or a specific object's characteristics) between the input image and throughout the generated video frames, often calculated using features from specialized models like face recognition (e.g., ArcFace) or re-identification models.", + id: "identity_preservation", + }, + { + description: + "Motion Score evaluates the quality, realism, and temporal consistency of motion in the video generated from a static image. This can be based on optical flow analysis (e.g., smoothness, magnitude), consistency of object trajectories, or specific motion plausibility assessments.", + id: "motion_score", + }, + ], + models: [ + { + description: "LTX-Video, a 13B parameter model for high quality video generation", + id: "Lightricks/LTX-Video-0.9.7-dev", + }, + { + description: "A 14B parameter model for reference image controlled video generation", + id: "Wan-AI/Wan2.1-VACE-14B", + }, + { + description: "An image-to-video generation model using FramePack F1 methodology with Hunyuan-DiT architecture", + id: "lllyasviel/FramePack_F1_I2V_HY_20250503", + }, + { + description: "A distilled version of the LTX-Video-0.9.7-dev model for faster inference", + id: "Lightricks/LTX-Video-0.9.7-distilled", + }, + { + description: "An image-to-video generation model by Skywork AI, 14B parameters, producing 720p videos.", + id: "Skywork/SkyReels-V2-I2V-14B-720P", + }, + { + description: "Image-to-video variant of Tencent's HunyuanVideo.", + id: "tencent/HunyuanVideo-I2V", + }, + { + description: "A 14B parameter model for 720p image-to-video generation by Wan-AI.", + id: "Wan-AI/Wan2.1-I2V-14B-720P", + }, + { + description: "A Diffusers version of the Wan2.1-I2V-14B-720P model for 720p image-to-video generation.", + id: "Wan-AI/Wan2.1-I2V-14B-720P-Diffusers", + }, + ], + spaces: [ + { + description: "An application to generate videos fast.", + id: "Lightricks/ltx-video-distilled", + }, + { + description: "Generate videos with the FramePack-F1", + id: "linoyts/FramePack-F1", + }, + { + description: "Generate videos with the FramePack", + id: "lisonallen/framepack-i2v", + }, + { + description: "Wan2.1 with CausVid LoRA", + id: "multimodalart/wan2-1-fast", + }, + { + description: "A demo for Stable Video Diffusion", + id: "multimodalart/stable-video-diffusion", + }, + ], + summary: + "Image-to-video models take a still image as input and generate a video. These models can be guided by text prompts to influence the content and style of the output video.", + widgetModels: [], + youtubeId: undefined, +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-video/inference.ts b/node_modules/@huggingface/tasks/src/tasks/image-to-video/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..6b218eb05854db516320eabe19d7c33d70c0ec2c --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-video/inference.ts @@ -0,0 +1,74 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Image To Video inference + */ +export interface ImageToVideoInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Image To Video + */ + parameters?: ImageToVideoParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Image To Video + */ +export interface ImageToVideoParameters { + /** + * For diffusion models. A higher guidance scale value encourages the model to generate + * videos closely linked to the text prompt at the expense of lower image quality. + */ + guidance_scale?: number; + /** + * One prompt to guide what NOT to include in video generation. + */ + negative_prompt?: string; + /** + * The num_frames parameter determines how many video frames are generated. + */ + num_frames?: number; + /** + * The number of denoising steps. More denoising steps usually lead to a higher quality + * video at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * The text prompt to guide the video generation. + */ + prompt?: string; + /** + * Seed for the random number generator. + */ + seed?: number; + /** + * The size in pixel of the output video frames. + */ + target_size?: TargetSize; + [property: string]: unknown; +} +/** + * The size in pixel of the output video frames. + */ +export interface TargetSize { + height: number; + width: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Image To Video task + */ +export interface ImageToVideoOutput { + /** + * The generated video returned as raw bytes in the payload. + */ + video: unknown; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-video/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/image-to-video/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..05178a700883087bda21ded761744f60c3d0c4d0 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-video/spec/input.json @@ -0,0 +1,64 @@ +{ + "$id": "/inference/schemas/image-to-video/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Image To Video inference", + "title": "ImageToVideoInput", + "type": "object", + "properties": { + "inputs": { + "type": "string", + "description": "The input image data as a base64-encoded string. If no `parameters` are provided, you can also provide the image data as a raw bytes payload.", + "comment": "type=binary" + }, + "parameters": { + "description": "Additional inference parameters for Image To Video", + "$ref": "#/$defs/ImageToVideoParameters" + } + }, + "$defs": { + "ImageToVideoParameters": { + "title": "ImageToVideoParameters", + "type": "object", + "properties": { + "prompt": { + "type": "string", + "description": "The text prompt to guide the video generation." + }, + "guidance_scale": { + "type": "number", + "description": "For diffusion models. A higher guidance scale value encourages the model to generate videos closely linked to the text prompt at the expense of lower image quality." + }, + "negative_prompt": { + "type": "string", + "description": "One prompt to guide what NOT to include in video generation." + }, + "num_inference_steps": { + "type": "integer", + "description": "The number of denoising steps. More denoising steps usually lead to a higher quality video at the expense of slower inference." + }, + "num_frames": { + "type": "number", + "description": "The num_frames parameter determines how many video frames are generated." + }, + "target_size": { + "type": "object", + "description": "The size in pixel of the output video frames.", + "properties": { + "width": { + "type": "integer" + }, + "height": { + "type": "integer" + } + }, + "required": ["width", "height"] + }, + "seed": { + "type": "integer", + "description": "Seed for the random number generator." + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/image-to-video/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/image-to-video/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..ecd27522c3cdc4313fbc1b1a47239f6a9cd4f0f0 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/image-to-video/spec/output.json @@ -0,0 +1,13 @@ +{ + "$id": "/inference/schemas/image-to-video/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Image To Video task", + "title": "ImageToVideoOutput", + "type": "object", + "properties": { + "video": { + "description": "The generated video returned as raw bytes in the payload." + } + }, + "required": ["video"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/index.ts b/node_modules/@huggingface/tasks/src/tasks/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..aa62295e6b21dba20c05cc9c9c3d45c07ec16417 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/index.ts @@ -0,0 +1,342 @@ +import type { PipelineType } from "../pipelines.js"; +import { PIPELINE_DATA } from "../pipelines.js"; + +import anyToAny from "./any-to-any/data.js"; +import audioClassification from "./audio-classification/data.js"; +import audioTextToText from "./audio-text-to-text/data.js"; +import audioToAudio from "./audio-to-audio/data.js"; +import automaticSpeechRecognition from "./automatic-speech-recognition/data.js"; +import documentQuestionAnswering from "./document-question-answering/data.js"; +import featureExtraction from "./feature-extraction/data.js"; +import fillMask from "./fill-mask/data.js"; +import imageClassification from "./image-classification/data.js"; +import imageFeatureExtraction from "./image-feature-extraction/data.js"; +import imageToImage from "./image-to-image/data.js"; +import imageToText from "./image-to-text/data.js"; +import imageTextToText from "./image-text-to-text/data.js"; +import imageTextToImage from "./image-text-to-image/data.js"; +import imageTextToVideo from "./image-text-to-video/data.js"; +import imageSegmentation from "./image-segmentation/data.js"; +import imageToVideo from "./image-to-video/data.js"; +import maskGeneration from "./mask-generation/data.js"; +import objectDetection from "./object-detection/data.js"; +import depthEstimation from "./depth-estimation/data.js"; +import placeholder from "./placeholder/data.js"; +import reinforcementLearning from "./reinforcement-learning/data.js"; +import questionAnswering from "./question-answering/data.js"; +import sentenceSimilarity from "./sentence-similarity/data.js"; +import summarization from "./summarization/data.js"; +import tableQuestionAnswering from "./table-question-answering/data.js"; +import tabularClassification from "./tabular-classification/data.js"; +import tabularRegression from "./tabular-regression/data.js"; +import textToImage from "./text-to-image/data.js"; +import textToSpeech from "./text-to-speech/data.js"; +import tokenClassification from "./token-classification/data.js"; +import translation from "./translation/data.js"; +import textClassification from "./text-classification/data.js"; +import textGeneration from "./text-generation/data.js"; +import textRanking from "./text-ranking/data.js"; +import textToVideo from "./text-to-video/data.js"; +import unconditionalImageGeneration from "./unconditional-image-generation/data.js"; +import videoClassification from "./video-classification/data.js"; +import visualDocumentRetrieval from "./visual-document-retrieval/data.js"; +import visualQuestionAnswering from "./visual-question-answering/data.js"; +import zeroShotClassification from "./zero-shot-classification/data.js"; +import zeroShotImageClassification from "./zero-shot-image-classification/data.js"; +import zeroShotObjectDetection from "./zero-shot-object-detection/data.js"; +import imageTo3D from "./image-to-3d/data.js"; +import textTo3D from "./text-to-3d/data.js"; +import keypointDetection from "./keypoint-detection/data.js"; +import videoTextToText from "./video-text-to-text/data.js"; +import videoToVideo from "./video-to-video/data.js"; + +export type * from "./audio-classification/inference.js"; +export type * from "./automatic-speech-recognition/inference.js"; +export type { + ChatCompletionInput, + ChatCompletionInputMessage, + ChatCompletionInputMessageChunkType, + ChatCompletionOutput, + ChatCompletionOutputComplete, + ChatCompletionOutputMessage, + ChatCompletionStreamOutput, + ChatCompletionStreamOutputChoice, + ChatCompletionStreamOutputDelta, +} from "./chat-completion/inference.js"; +export type * from "./document-question-answering/inference.js"; +export type * from "./feature-extraction/inference.js"; +export type * from "./fill-mask/inference.js"; +export type { + ImageClassificationInput, + ImageClassificationOutput, + ImageClassificationOutputElement, + ImageClassificationParameters, +} from "./image-classification/inference.js"; +export type * from "./image-to-image/inference.js"; +export type { ImageToTextInput, ImageToTextOutput, ImageToTextParameters } from "./image-to-text/inference.js"; +export type * from "./image-segmentation/inference.js"; +export type { ImageToVideoInput, ImageToVideoOutput, ImageToVideoParameters } from "./image-to-video/inference.js"; +export type { + ImageTextToImageInput, + ImageTextToImageOutput, + ImageTextToImageParameters, +} from "./image-text-to-image/inference.js"; +export type { + ImageTextToVideoInput, + ImageTextToVideoOutput, + ImageTextToVideoParameters, +} from "./image-text-to-video/inference.js"; +export type * from "./object-detection/inference.js"; +export type * from "./depth-estimation/inference.js"; +export type * from "./question-answering/inference.js"; +export type * from "./sentence-similarity/inference.js"; +export type * from "./summarization/inference.js"; +export type * from "./table-question-answering/inference.js"; +export type { TextToImageInput, TextToImageOutput, TextToImageParameters } from "./text-to-image/inference.js"; +export type { TextToVideoParameters, TextToVideoOutput, TextToVideoInput } from "./text-to-video/inference.js"; +export type { TextToSpeechParameters, TextToSpeechInput, TextToSpeechOutput } from "./text-to-speech/inference.js"; +export type { TextToAudioInput, TextToAudioOutput, TextToAudioParameters } from "./text-to-audio/inference.js"; +export type * from "./token-classification/inference.js"; +export type { TranslationInput, TranslationOutput } from "./translation/inference.js"; +export type { + ClassificationOutputTransform, + TextClassificationInput, + TextClassificationOutput, + TextClassificationOutputElement, + TextClassificationParameters, +} from "./text-classification/inference.js"; +export type { + TextGenerationOutputFinishReason, + TextGenerationOutputPrefillToken, + TextGenerationInput, + TextGenerationOutput, + TextGenerationOutputDetails, + TextGenerationInputGenerateParameters, + TextGenerationOutputBestOfSequence, + TextGenerationOutputToken, + TextGenerationStreamOutputStreamDetails, + TextGenerationStreamOutput, +} from "./text-generation/inference.js"; +export type * from "./video-classification/inference.js"; +export type * from "./visual-question-answering/inference.js"; +export type * from "./zero-shot-classification/inference.js"; +export type * from "./zero-shot-image-classification/inference.js"; +export type { + BoundingBox, + ZeroShotObjectDetectionInput, + ZeroShotObjectDetectionOutput, + ZeroShotObjectDetectionOutputElement, +} from "./zero-shot-object-detection/inference.js"; + +import type { ModelLibraryKey } from "../model-libraries.js"; +/** + * Model libraries compatible with each ML task + */ +export const TASKS_MODEL_LIBRARIES: Record = { + "audio-classification": ["speechbrain", "transformers", "transformers.js"], + "audio-to-audio": ["asteroid", "fairseq", "speechbrain"], + "automatic-speech-recognition": ["espnet", "nemo", "speechbrain", "transformers", "transformers.js"], + "audio-text-to-text": ["transformers"], + "depth-estimation": ["transformers", "transformers.js"], + "document-question-answering": ["transformers", "transformers.js"], + "feature-extraction": ["sentence-transformers", "transformers", "transformers.js"], + "fill-mask": ["transformers", "transformers.js"], + "graph-ml": ["transformers"], + "image-classification": ["keras", "timm", "transformers", "transformers.js"], + "image-feature-extraction": ["timm", "transformers"], + "image-segmentation": ["transformers", "transformers.js"], + "image-text-to-text": ["transformers"], + "image-text-to-image": ["diffusers"], + "image-text-to-video": ["diffusers"], + "image-to-image": ["diffusers", "transformers", "transformers.js"], + "image-to-text": ["transformers", "transformers.js"], + "image-to-video": ["diffusers"], + "keypoint-detection": ["transformers"], + "video-classification": ["transformers"], + "mask-generation": ["transformers"], + "multiple-choice": ["transformers"], + "object-detection": ["transformers", "transformers.js", "ultralytics"], + other: [], + "question-answering": ["adapter-transformers", "allennlp", "transformers", "transformers.js"], + robotics: [], + "reinforcement-learning": ["transformers", "stable-baselines3", "ml-agents", "sample-factory"], + "sentence-similarity": ["sentence-transformers", "spacy", "transformers.js"], + summarization: ["transformers", "transformers.js"], + "table-question-answering": ["transformers"], + "table-to-text": ["transformers"], + "tabular-classification": ["sklearn"], + "tabular-regression": ["sklearn"], + "tabular-to-text": ["transformers"], + "text-classification": ["adapter-transformers", "setfit", "spacy", "transformers", "transformers.js"], + "text-generation": ["transformers", "transformers.js"], + "text-ranking": ["sentence-transformers", "transformers"], + "text-retrieval": [], + "text-to-image": ["diffusers"], + "text-to-speech": ["espnet", "tensorflowtts", "transformers", "transformers.js"], + "text-to-audio": ["transformers", "transformers.js"], + "text-to-video": ["diffusers"], + "time-series-forecasting": [], + "token-classification": [ + "adapter-transformers", + "flair", + "spacy", + "span-marker", + "stanza", + "transformers", + "transformers.js", + ], + translation: ["transformers", "transformers.js"], + "unconditional-image-generation": ["diffusers"], + "video-text-to-text": ["transformers"], + "visual-question-answering": ["transformers", "transformers.js"], + "voice-activity-detection": [], + "zero-shot-classification": ["transformers", "transformers.js"], + "zero-shot-image-classification": ["transformers", "transformers.js"], + "zero-shot-object-detection": ["transformers", "transformers.js"], + "text-to-3d": ["diffusers"], + "image-to-3d": ["diffusers"], + "any-to-any": ["transformers"], + "visual-document-retrieval": ["transformers"], + "video-to-video": ["diffusers"], +}; + +/** + * Return the whole TaskData object for a certain task. + * If the partialTaskData argument is left undefined, + * the default placeholder data will be used. + */ +function getData(type: PipelineType, partialTaskData: TaskDataCustom = placeholder): TaskData { + return { + ...partialTaskData, + id: type, + label: PIPELINE_DATA[type].name, + libraries: TASKS_MODEL_LIBRARIES[type], + }; +} + +// To make comparisons easier, task order is the same as in const.ts +// Tasks set to undefined won't have an associated task page. +// Tasks that call getData() without the second argument will +// have a "placeholder" page. +export const TASKS_DATA: Record = { + "any-to-any": getData("any-to-any", anyToAny), + "audio-classification": getData("audio-classification", audioClassification), + "audio-to-audio": getData("audio-to-audio", audioToAudio), + "audio-text-to-text": getData("audio-text-to-text", audioTextToText), + "automatic-speech-recognition": getData("automatic-speech-recognition", automaticSpeechRecognition), + "depth-estimation": getData("depth-estimation", depthEstimation), + "document-question-answering": getData("document-question-answering", documentQuestionAnswering), + "visual-document-retrieval": getData("visual-document-retrieval", visualDocumentRetrieval), + "feature-extraction": getData("feature-extraction", featureExtraction), + "fill-mask": getData("fill-mask", fillMask), + "graph-ml": undefined, + "image-classification": getData("image-classification", imageClassification), + "image-feature-extraction": getData("image-feature-extraction", imageFeatureExtraction), + "image-segmentation": getData("image-segmentation", imageSegmentation), + "image-to-image": getData("image-to-image", imageToImage), + "image-text-to-text": getData("image-text-to-text", imageTextToText), + "image-text-to-image": getData("image-text-to-image", imageTextToImage), + "image-text-to-video": getData("image-text-to-video", imageTextToVideo), + "image-to-text": getData("image-to-text", imageToText), + "image-to-video": getData("image-to-video", imageToVideo), + "keypoint-detection": getData("keypoint-detection", keypointDetection), + "mask-generation": getData("mask-generation", maskGeneration), + "multiple-choice": undefined, + "object-detection": getData("object-detection", objectDetection), + "video-classification": getData("video-classification", videoClassification), + other: undefined, + "question-answering": getData("question-answering", questionAnswering), + "reinforcement-learning": getData("reinforcement-learning", reinforcementLearning), + robotics: undefined, + "sentence-similarity": getData("sentence-similarity", sentenceSimilarity), + summarization: getData("summarization", summarization), + "table-question-answering": getData("table-question-answering", tableQuestionAnswering), + "table-to-text": undefined, + "tabular-classification": getData("tabular-classification", tabularClassification), + "tabular-regression": getData("tabular-regression", tabularRegression), + "tabular-to-text": undefined, + "text-classification": getData("text-classification", textClassification), + "text-generation": getData("text-generation", textGeneration), + "text-ranking": getData("text-ranking", textRanking), + "text-retrieval": undefined, + "text-to-image": getData("text-to-image", textToImage), + "text-to-speech": getData("text-to-speech", textToSpeech), + "text-to-audio": undefined, + "text-to-video": getData("text-to-video", textToVideo), + "time-series-forecasting": undefined, + "token-classification": getData("token-classification", tokenClassification), + translation: getData("translation", translation), + "unconditional-image-generation": getData("unconditional-image-generation", unconditionalImageGeneration), + "video-text-to-text": getData("video-text-to-text", videoTextToText), + "video-to-video": getData("video-to-video", videoToVideo), + "visual-question-answering": getData("visual-question-answering", visualQuestionAnswering), + "voice-activity-detection": undefined, + "zero-shot-classification": getData("zero-shot-classification", zeroShotClassification), + "zero-shot-image-classification": getData("zero-shot-image-classification", zeroShotImageClassification), + "zero-shot-object-detection": getData("zero-shot-object-detection", zeroShotObjectDetection), + "text-to-3d": getData("text-to-3d", textTo3D), + "image-to-3d": getData("image-to-3d", imageTo3D), +} as const; + +export interface ExampleRepo { + description: string; + id: string; +} + +export type TaskDemoEntry = + | { + filename: string; + type: "audio"; + } + | { + data: Array<{ + label: string; + score: number; + }>; + type: "chart"; + } + | { + filename: string; + type: "img"; + } + | { + table: string[][]; + type: "tabular"; + } + | { + content: string; + label: string; + type: "text"; + } + | { + text: string; + tokens: Array<{ + end: number; + start: number; + type: string; + }>; + type: "text-with-tokens"; + }; + +export interface TaskDemo { + inputs: TaskDemoEntry[]; + outputs: TaskDemoEntry[]; +} + +export interface TaskData { + datasets: ExampleRepo[]; + demo: TaskDemo; + id: PipelineType; + canonicalId?: PipelineType; + isPlaceholder?: boolean; + label: string; + libraries: ModelLibraryKey[]; + metrics: ExampleRepo[]; + models: ExampleRepo[]; + spaces: ExampleRepo[]; + summary: string; + widgetModels: string[]; + youtubeId?: string; +} + +export type TaskDataCustom = Omit; diff --git a/node_modules/@huggingface/tasks/src/tasks/keypoint-detection/about.md b/node_modules/@huggingface/tasks/src/tasks/keypoint-detection/about.md new file mode 100644 index 0000000000000000000000000000000000000000..3758fdd831d23163ea1974842faf1330c7748c0a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/keypoint-detection/about.md @@ -0,0 +1,57 @@ +## Task Variants + +### Pose Estimation + +Pose estimation is the process of determining the position and orientation of an object or a camera in a 3D space. It is a fundamental task in computer vision and is widely used in various applications such as robotics, augmented reality, and 3D reconstruction. + +## Use Cases for Keypoint Detection + +### Facial Landmark Estimation + +Keypoint detection models can be used to estimate the position of facial landmarks. Facial landmarks are points on the face such as the corners of the mouth, the outer corners of the eyes, and the tip of the nose. These landmarks can be used for a variety of applications, such as facial expression recognition, 3D face reconstruction, and cinematic animation. + +### Fitness Tracking + +Keypoint detection models can be used to track the movement of the human body, e.g. position of the joints in a 3D space. This can be used for a variety of applications, such as fitness tracking, sports analysis or virtual reality applications. + +## Inference Code + +Below you can find an example of how to use a keypoint detection model and how to visualize the results. + +```python +from transformers import AutoImageProcessor, SuperPointForKeypointDetection +import torch +import matplotlib.pyplot as plt +from PIL import Image +import requests + +url_image = "http://images.cocodataset.org/val2017/000000039769.jpg" +image = Image.open(requests.get(url_image_1, stream=True).raw) + +# initialize the model and processor +processor = AutoImageProcessor.from_pretrained("magic-leap-community/superpoint") +model = SuperPointForKeypointDetection.from_pretrained("magic-leap-community/superpoint") + +# infer +inputs = processor(image, return_tensors="pt").to(model.device, model.dtype) +outputs = model(**inputs) + +# postprocess +image_sizes = [(image.size[1], image.size[0])] +outputs = processor.post_process_keypoint_detection(model_outputs, image_sizes) +keypoints = outputs[0]["keypoints"].detach().numpy() +scores = outputs[0]["scores"].detach().numpy() +image_width, image_height = image.size + +# plot +plt.axis('off') +plt.imshow(image) +plt.scatter( + keypoints[:, 0], + keypoints[:, 1], + s=scores * 100, + c='cyan', + alpha=0.4 +) +plt.show() +``` diff --git a/node_modules/@huggingface/tasks/src/tasks/keypoint-detection/data.ts b/node_modules/@huggingface/tasks/src/tasks/keypoint-detection/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..5d6e3d6c25825ecf75ac60a9fe681940cc30a13a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/keypoint-detection/data.ts @@ -0,0 +1,58 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A dataset of hand keypoints of over 500k examples.", + id: "Vincent-luo/hagrid-mediapipe-hands", + }, + ], + demo: { + inputs: [ + { + filename: "keypoint-detection-input.png", + type: "img", + }, + ], + outputs: [ + { + filename: "keypoint-detection-output.png", + type: "img", + }, + ], + }, + metrics: [], + models: [ + { + description: "A robust keypoint detection model.", + id: "magic-leap-community/superpoint", + }, + { + description: "A robust keypoint matching model.", + id: "magic-leap-community/superglue_outdoor", + }, + { + description: "Strong keypoint detection model used to detect human pose.", + id: "qualcomm/RTMPose-Body2d", + }, + { + description: "Powerful keypoint matching model.", + id: "ETH-CVG/lightglue_disk", + }, + ], + spaces: [ + { + description: "An application that detects hand keypoints in real-time.", + id: "datasciencedojo/Hand-Keypoint-Detection-Realtime", + }, + { + description: "An application for keypoint detection and matching.", + id: "ETH-CVG/LightGlue", + }, + ], + summary: "Keypoint detection is the task of identifying meaningful distinctive points or features in an image.", + widgetModels: [], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/mask-generation/about.md b/node_modules/@huggingface/tasks/src/tasks/mask-generation/about.md new file mode 100644 index 0000000000000000000000000000000000000000..2fa3a65d22a4be467172062f324248023bc7360a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/mask-generation/about.md @@ -0,0 +1,75 @@ +## Use Cases + +### Filtering an Image + +When filtering for an image, the generated masks might serve as an initial filter to eliminate irrelevant information. For instance, when monitoring vegetation in satellite imaging, mask generation models identify green spots, highlighting the relevant region of the image. + +### Masked Image Modelling + +Generating masks can facilitate learning, especially in semi or unsupervised learning. For example, the [BEiT model](https://huggingface.co/docs/transformers/model_doc/beit) uses image-mask patches in the pre-training. + +### Human-in-the-loop Computer Vision Applications + +For applications where humans are in the loop, masks highlight certain regions of images for humans to validate. + +### Medical Imaging + +Mask generation models are used in medical imaging to aid in segmenting and analyzing specific regions. + +### Autonomous Vehicles + +Mask generation models are used to create segments and masks for obstacles and other objects in view. + +This page was made possible thanks to the efforts of [Raj Aryan](https://huggingface.co/thatrajaryan) and other contributors. + +## Task Variants + +### Segmentation + +Image Segmentation divides an image into segments where each pixel is mapped to an object. This task has multiple variants, such as instance segmentation, panoptic segmentation, and semantic segmentation. You can learn more about segmentation on its [task page](https://huggingface.co/tasks/image-segmentation). + +## Inference + +Mask generation models often work in two modes: segment everything or prompt mode. +The example below works in segment-everything-mode, where many masks will be returned. + +```python +from transformers import pipeline + +generator = pipeline("mask-generation", model="Zigeng/SlimSAM-uniform-50", points_per_batch=64, device="cuda") +image_url = "https://huggingface.co/ybelkada/segment-anything/resolve/main/assets/car.png" +outputs = generator(image_url) +outputs["masks"] +# array of multiple binary masks returned for each generated mask +``` + +Prompt mode takes in three types of prompts: + +- **Point prompt:** The user can select a point on the image, and a meaningful segment around the point will be returned. +- **Box prompt:** The user can draw a box on the image, and a meaningful segment within the box will be returned. +- **Text prompt:** The user can input a text, and the objects of that type will be segmented. Note that this capability has not yet been released and has only been explored in research. + +Below you can see how to use an input-point prompt. It also demonstrates direct model inference without the `pipeline` abstraction. The input prompt here is a nested list where the outermost list is the batch size (`1`), then the number of points (also `1` in this example), and the innermost list contains the actual coordinates of the point (`[450, 600]`). + +```python +from transformers import SamModel, SamProcessor +from PIL import Image +import requests + +model = SamModel.from_pretrained("Zigeng/SlimSAM-uniform-50").to("cuda") +processor = SamProcessor.from_pretrained("Zigeng/SlimSAM-uniform-50") + +raw_image = Image.open(requests.get(image_url, stream=True).raw).convert("RGB") +# pointing to the car window +input_points = [[[450, 600]]] +inputs = processor(raw_image, input_points=input_points, return_tensors="pt").to("cuda") +outputs = model(**inputs) +masks = processor.post_process_masks(outputs.pred_masks.cpu(), inputs["original_sizes"].cpu(), inputs["reshaped_input_sizes"].cpu()) +scores = outputs.iou_scores +``` + +## Useful Resources + +Would you like to learn more about mask generation? Great! Here you can find some curated resources that you may find helpful! + +- [Segment anything model](https://huggingface.co/docs/transformers/main/model_doc/sam) diff --git a/node_modules/@huggingface/tasks/src/tasks/mask-generation/data.ts b/node_modules/@huggingface/tasks/src/tasks/mask-generation/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..cc18380dbe2f37b4ad6d2587dd7b102d6fbef5f3 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/mask-generation/data.ts @@ -0,0 +1,69 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "Widely used benchmark dataset for multiple Vision tasks.", + id: "merve/coco2017", + }, + { + description: "Medical Imaging dataset of the Human Brain for segmentation and mask generating tasks", + id: "rocky93/BraTS_segmentation", + }, + ], + demo: { + inputs: [ + { + filename: "mask-generation-input.png", + type: "img", + }, + ], + outputs: [ + { + filename: "mask-generation-output.png", + type: "img", + }, + ], + }, + metrics: [ + { + description: "IoU is used to measure the overlap between predicted mask and the ground truth mask.", + id: "Intersection over Union (IoU)", + }, + ], + models: [ + { + description: "Small yet powerful mask generation model.", + id: "Zigeng/SlimSAM-uniform-50", + }, + { + description: "Very strong mask generation model.", + id: "facebook/sam2-hiera-large", + }, + ], + spaces: [ + { + description: + "An application that combines a mask generation model with a zero-shot object detection model for text-guided image segmentation.", + id: "merve/OWLSAM2", + }, + { + description: "An application that compares the performance of a large and a small mask generation model.", + id: "merve/slimsam", + }, + { + description: "An application based on an improved mask generation model.", + id: "SkalskiP/segment-anything-model-2", + }, + { + description: "An application to remove objects from videos using mask generation models.", + id: "SkalskiP/SAM_and_ProPainter", + }, + ], + summary: + "Mask generation is the task of generating masks that identify a specific object or region of interest in a given image. Masks are often used in segmentation tasks, where they provide a precise way to isolate the object of interest for further processing or analysis.", + widgetModels: [], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/object-detection/about.md b/node_modules/@huggingface/tasks/src/tasks/object-detection/about.md new file mode 100644 index 0000000000000000000000000000000000000000..4dda21224f937a27a5f56d40ee877ff03eaf1d09 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/object-detection/about.md @@ -0,0 +1,37 @@ +## Use Cases + +### Autonomous Driving + +Object Detection is widely used in computer vision for autonomous driving. Self-driving cars use Object Detection models to detect pedestrians, bicycles, traffic lights and road signs to decide which step to take. + +### Object Tracking in Matches + +Object Detection models are widely used in sports where the ball or a player is tracked for monitoring and refereeing during matches. + +### Image Search + +Object Detection models are widely used in image search. Smartphones use Object Detection models to detect entities (such as specific places or objects) and allow the user to search for the entity on the Internet. + +### Object Counting + +Object Detection models are used to count instances of objects in a given image, this can include counting the objects in warehouses or stores, or counting the number of visitors in a store. They are also used to manage crowds at events to prevent disasters. + +## Inference + +You can infer with Object Detection models through the `object-detection` pipeline. When calling the pipeline you just need to specify a path or http link to an image. + +```python +model = pipeline("object-detection") + +model("path_to_cat_image") + +# [{'label': 'blanket', +# 'mask': mask_string, +# 'score': 0.917}, +#...] +``` + +# Useful Resources + +- [Walkthrough of Computer Vision Ecosystem in Hugging Face - CV Study Group](https://www.youtube.com/watch?v=oL-xmufhZM8) +- [Object detection task guide](https://huggingface.co/docs/transformers/tasks/object_detection) diff --git a/node_modules/@huggingface/tasks/src/tasks/object-detection/data.ts b/node_modules/@huggingface/tasks/src/tasks/object-detection/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..3b962813340e659b93a3574844bcb3c11f070846 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/object-detection/data.ts @@ -0,0 +1,90 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "Widely used benchmark dataset for multiple vision tasks.", + id: "merve/coco2017", + }, + { + description: "Multi-task computer vision benchmark.", + id: "merve/pascal-voc", + }, + ], + demo: { + inputs: [ + { + filename: "object-detection-input.jpg", + type: "img", + }, + ], + outputs: [ + { + filename: "object-detection-output.jpg", + type: "img", + }, + ], + }, + metrics: [ + { + description: + "The Average Precision (AP) metric is the Area Under the PR Curve (AUC-PR). It is calculated for each class separately", + id: "Average Precision", + }, + { + description: "The Mean Average Precision (mAP) metric is the overall average of the AP values", + id: "Mean Average Precision", + }, + { + description: + "The APα metric is the Average Precision at the IoU threshold of a α value, for example, AP50 and AP75", + id: "APα", + }, + ], + models: [ + { + description: "Solid object detection model pre-trained on the COCO 2017 dataset.", + id: "facebook/detr-resnet-50", + }, + { + description: "Accurate object detection model.", + id: "IDEA-Research/dab-detr-resnet-50", + }, + { + description: "Fast and accurate object detection model.", + id: "PekingU/rtdetr_v2_r50vd", + }, + { + description: "Object detection model for low-lying objects.", + id: "StephanST/WALDO30", + }, + ], + spaces: [ + { + description: "Real-time object detection demo.", + id: "Roboflow/RF-DETR", + }, + { + description: "An application that contains various object detection models to try from.", + id: "Gradio-Blocks/Object-Detection-With-DETR-and-YOLOS", + }, + { + description: "A cutting-edge object detection application.", + id: "sunsmarterjieleaf/yolov12", + }, + { + description: "An object tracking, segmentation and inpainting application.", + id: "VIPLab/Track-Anything", + }, + { + description: "Very fast object tracking application based on object detection.", + id: "merve/RT-DETR-tracking-coco", + }, + ], + summary: + "Object Detection models allow users to identify objects of certain defined classes. Object detection models receive an image as input and output the images with bounding boxes and labels on detected objects.", + widgetModels: ["facebook/detr-resnet-50"], + youtubeId: "WdAeKSOpxhw", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/object-detection/inference.ts b/node_modules/@huggingface/tasks/src/tasks/object-detection/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..58f330e681686e98f84dab3bb6cccffde197c79b --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/object-detection/inference.ts @@ -0,0 +1,73 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Object Detection inference + */ +export interface ObjectDetectionInput { + /** + * The input image data as a base64-encoded string. If no `parameters` are provided, you can + * also provide the image data as a raw bytes payload. + */ + inputs: Blob; + /** + * Additional inference parameters for Object Detection + */ + parameters?: ObjectDetectionParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Object Detection + */ +export interface ObjectDetectionParameters { + /** + * The probability necessary to make a prediction. + */ + threshold?: number; + [property: string]: unknown; +} +/** + * The predicted bounding box. Coordinates are relative to the top left corner of the input + * image. + */ +export interface BoundingBox { + /** + * The x-coordinate of the bottom-right corner of the bounding box. + */ + xmax: number; + /** + * The x-coordinate of the top-left corner of the bounding box. + */ + xmin: number; + /** + * The y-coordinate of the bottom-right corner of the bounding box. + */ + ymax: number; + /** + * The y-coordinate of the top-left corner of the bounding box. + */ + ymin: number; + [property: string]: unknown; +} +export type ObjectDetectionOutput = ObjectDetectionOutputElement[]; +/** + * Outputs of inference for the Object Detection task + */ +export interface ObjectDetectionOutputElement { + /** + * The predicted bounding box. Coordinates are relative to the top left corner of the input + * image. + */ + box: BoundingBox; + /** + * The predicted label for the bounding box. + */ + label: string; + /** + * The associated score / probability. + */ + score: number; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/object-detection/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/object-detection/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..55df78f56bc0130af7b952de88be400ae62d9d98 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/object-detection/spec/input.json @@ -0,0 +1,31 @@ +{ + "$id": "/inference/schemas/object-detection/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Object Detection inference", + "title": "ObjectDetectionInput", + "type": "object", + "properties": { + "inputs": { + "type": "string", + "description": "The input image data as a base64-encoded string. If no `parameters` are provided, you can also provide the image data as a raw bytes payload.", + "comment": "type=binary" + }, + "parameters": { + "description": "Additional inference parameters for Object Detection", + "$ref": "#/$defs/ObjectDetectionParameters" + } + }, + "$defs": { + "ObjectDetectionParameters": { + "title": "ObjectDetectionParameters", + "type": "object", + "properties": { + "threshold": { + "type": "number", + "description": "The probability necessary to make a prediction." + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/object-detection/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/object-detection/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..8d91f1078f664fff39eff78d0ff4bba00d9a0a79 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/object-detection/spec/output.json @@ -0,0 +1,50 @@ +{ + "$id": "/inference/schemas/object-detection/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Object Detection task", + "title": "ObjectDetectionOutput", + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string", + "description": "The predicted label for the bounding box." + }, + "score": { + "type": "number", + "description": "The associated score / probability." + }, + "box": { + "$ref": "#/$defs/BoundingBox", + "description": "The predicted bounding box. Coordinates are relative to the top left corner of the input image." + } + }, + "required": ["box", "label", "score"] + }, + "$defs": { + "BoundingBox": { + "type": "object", + "title": "BoundingBox", + "properties": { + "xmin": { + "type": "integer", + "description": "The x-coordinate of the top-left corner of the bounding box." + }, + "xmax": { + "type": "integer", + "description": "The x-coordinate of the bottom-right corner of the bounding box." + }, + "ymin": { + "type": "integer", + "description": "The y-coordinate of the top-left corner of the bounding box." + }, + "ymax": { + "type": "integer", + "description": "The y-coordinate of the bottom-right corner of the bounding box." + } + }, + "required": ["xmin", "xmax", "ymin", "ymax"] + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/placeholder/about.md b/node_modules/@huggingface/tasks/src/tasks/placeholder/about.md new file mode 100644 index 0000000000000000000000000000000000000000..fdb45584410dcd07e530607d469140038ede6b25 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/placeholder/about.md @@ -0,0 +1,15 @@ +## Use Cases + +You can contribute this area with common use cases of the task! + +## Task Variants + +This place can be filled with variants of this task if there's any. + +## Inference + +This section should have useful information about how to pull a model from Hugging Face Hub that is a part of a library specialized in a task and use it. + +## Useful Resources + +In this area, you can insert useful resources about how to train or use a model for this task. diff --git a/node_modules/@huggingface/tasks/src/tasks/placeholder/data.ts b/node_modules/@huggingface/tasks/src/tasks/placeholder/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..c103e16a889e42b08a4987fb91a3c962cb1afb02 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/placeholder/data.ts @@ -0,0 +1,21 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [], + demo: { + inputs: [], + outputs: [], + }, + isPlaceholder: true, + metrics: [], + models: [], + spaces: [], + summary: "", + widgetModels: [], + youtubeId: undefined, + /// If this is a subtask, link to the most general task ID + /// (eg, text-generation is the canonical ID of text-simplification) + canonicalId: undefined, +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/placeholder/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/placeholder/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..aab9dd1525b387db8aaaf2ef07d75ab355143d8d --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/placeholder/spec/input.json @@ -0,0 +1,34 @@ +{ + "$id": "/inference/schemas//input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for inference", + "title": "PlaceholderInput", + "type": "object", + "properties": { + "inputs": { + "description": "TODO: describe the input here. This must be model & framework agnostic.", + "type": "string" + }, + "parameters": { + "description": "TODO: describe additional parameters here.", + "$ref": "#/$defs/Parameters" + } + }, + "$defs": { + "Parameters": { + "title": "Parameters", + "type": "object", + "properties": { + "dummy_parameter_name": { + "type": "boolean", + "description": "TODO: describe the parameter here" + }, + "dummy_parameter_name2": { + "type": "integer", + "description": "TODO: describe the parameter here" + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/placeholder/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/placeholder/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..a71a1ae149704cb6317fd35a539be343c454ef81 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/placeholder/spec/output.json @@ -0,0 +1,17 @@ +{ + "$id": "/inference/schemas//output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs for inference", + "title": "PlaceholderOutput", + "type": "array", + "items": { + "type": "object", + "properties": { + "meaningful_output_name": { + "type": "string", + "description": "TODO: Describe what is outputted by the inference here" + } + }, + "required": ["meaningfulOutputName"] + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/question-answering/about.md b/node_modules/@huggingface/tasks/src/tasks/question-answering/about.md new file mode 100644 index 0000000000000000000000000000000000000000..d5934ee80c7ca32c53726ef5b9e715af54a0b5d6 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/question-answering/about.md @@ -0,0 +1,56 @@ +## Use Cases + +### Frequently Asked Questions + +You can use Question Answering (QA) models to automate the response to frequently asked questions by using a knowledge base (documents) as context. Answers to customer questions can be drawn from those documents. + +⚡⚡ If you’d like to save inference time, you can first use [passage ranking models](/tasks/sentence-similarity) to see which document might contain the answer to the question and iterate over that document with the QA model instead. + +## Task Variants +There are different QA variants based on the inputs and outputs: + +- **Extractive QA:** The model **extracts** the answer from a context. The context here could be a provided text, a table or even HTML! This is usually solved with BERT-like models. +- **Open Generative QA:** The model **generates** free text directly based on the context. You can learn more about the Text Generation task in [its page](/tasks/text-generation). +- **Closed Generative QA:** In this case, no context is provided. The answer is completely generated by a model. + +The schema above illustrates extractive, open book QA. The model takes a context and the question and extracts the answer from the given context. + +You can also differentiate QA models depending on whether they are open-domain or closed-domain. Open-domain models are not restricted to a specific domain, while closed-domain models are restricted to a specific domain (e.g. legal, medical documents). + +## Inference + +You can infer with QA models with the 🤗 Transformers library using the `question-answering` pipeline. If no model checkpoint is given, the pipeline will be initialized with `distilbert-base-cased-distilled-squad`. This pipeline takes a question and a context from which the answer will be extracted and returned. + +```python +from transformers import pipeline + +qa_model = pipeline("question-answering") +question = "Where do I live?" +context = "My name is Merve and I live in İstanbul." +qa_model(question = question, context = context) +## {'answer': 'İstanbul', 'end': 39, 'score': 0.953, 'start': 31} +``` + +## Useful Resources + +Would you like to learn more about QA? Awesome! Here are some curated resources that you may find helpful! + +- [Course Chapter on Question Answering](https://huggingface.co/course/chapter7/7?fw=pt) +- [Question Answering Workshop](https://www.youtube.com/watch?v=Ihgk8kGLpIE&ab_channel=HuggingFace) +- [How to Build an Open-Domain Question Answering System?](https://lilianweng.github.io/lil-log/2020/10/29/open-domain-question-answering.html) +- [Blog Post: ELI5 A Model for Open Domain Long Form Question Answering](https://yjernite.github.io/lfqa.html) + +### Notebooks + +- [PyTorch](https://github.com/huggingface/notebooks/blob/master/examples/question_answering.ipynb) +- [TensorFlow](https://github.com/huggingface/notebooks/blob/main/examples/question_answering-tf.ipynb) + +### Scripts for training + +- [PyTorch](https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering) +- [TensorFlow](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/question-answering) +- [Flax](https://github.com/huggingface/transformers/tree/main/examples/flax/question-answering) + +### Documentation + +- [Question answering task guide](https://huggingface.co/docs/transformers/tasks/question_answering) diff --git a/node_modules/@huggingface/tasks/src/tasks/question-answering/data.ts b/node_modules/@huggingface/tasks/src/tasks/question-answering/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..cf75dd3d7b9748e6670fef59497dd0feea6d4a6b --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/question-answering/data.ts @@ -0,0 +1,75 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + // TODO write proper description + description: "A famous question answering dataset based on English articles from Wikipedia.", + id: "squad_v2", + }, + { + // TODO write proper description + description: "A dataset of aggregated anonymized actual queries issued to the Google search engine.", + id: "natural_questions", + }, + ], + demo: { + inputs: [ + { + label: "Question", + content: "Which name is also used to describe the Amazon rainforest in English?", + type: "text", + }, + { + label: "Context", + content: "The Amazon rainforest, also known in English as Amazonia or the Amazon Jungle", + type: "text", + }, + ], + outputs: [ + { + label: "Answer", + content: "Amazonia", + type: "text", + }, + ], + }, + metrics: [ + { + description: + "Exact Match is a metric based on the strict character match of the predicted answer and the right answer. For answers predicted correctly, the Exact Match will be 1. Even if only one character is different, Exact Match will be 0", + id: "exact-match", + }, + { + description: + " The F1-Score metric is useful if we value both false positives and false negatives equally. The F1-Score is calculated on each word in the predicted sequence against the correct answer", + id: "f1", + }, + ], + models: [ + { + description: "A robust baseline model for most question answering domains.", + id: "deepset/roberta-base-squad2", + }, + { + description: "Small yet robust model that can answer questions.", + id: "distilbert/distilbert-base-cased-distilled-squad", + }, + { + description: "A special model that can answer questions from tables.", + id: "google/tapas-base-finetuned-wtq", + }, + ], + spaces: [ + { + description: "An application that can answer a long question from Wikipedia.", + id: "deepset/wikipedia-assistant", + }, + ], + summary: + "Question Answering models can retrieve the answer to a question from a given text, which is useful for searching for an answer in a document. Some question answering models can generate answers without context!", + widgetModels: ["deepset/roberta-base-squad2"], + youtubeId: "ajPx5LwJD-I", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/question-answering/inference.ts b/node_modules/@huggingface/tasks/src/tasks/question-answering/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..bd478276bdd8e9c8473aa73d562a32845a93a968 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/question-answering/inference.ts @@ -0,0 +1,97 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Question Answering inference + */ +export interface QuestionAnsweringInput { + /** + * One (context, question) pair to answer + */ + inputs: QuestionAnsweringInputData; + /** + * Additional inference parameters for Question Answering + */ + parameters?: QuestionAnsweringParameters; + [property: string]: unknown; +} +/** + * One (context, question) pair to answer + */ +export interface QuestionAnsweringInputData { + /** + * The context to be used for answering the question + */ + context: string; + /** + * The question to be answered + */ + question: string; + [property: string]: unknown; +} +/** + * Additional inference parameters for Question Answering + */ +export interface QuestionAnsweringParameters { + /** + * Attempts to align the answer to real words. Improves quality on space separated + * languages. Might hurt on non-space-separated languages (like Japanese or Chinese) + */ + align_to_words?: boolean; + /** + * If the context is too long to fit with the question for the model, it will be split in + * several chunks with some overlap. This argument controls the size of that overlap. + */ + doc_stride?: number; + /** + * Whether to accept impossible as an answer. + */ + handle_impossible_answer?: boolean; + /** + * The maximum length of predicted answers (e.g., only answers with a shorter length are + * considered). + */ + max_answer_len?: number; + /** + * The maximum length of the question after tokenization. It will be truncated if needed. + */ + max_question_len?: number; + /** + * The maximum length of the total sentence (context + question) in tokens of each chunk + * passed to the model. The context will be split in several chunks (using docStride as + * overlap) if needed. + */ + max_seq_len?: number; + /** + * The number of answers to return (will be chosen by order of likelihood). Note that we + * return less than topk answers if there are not enough options available within the + * context. + */ + top_k?: number; + [property: string]: unknown; +} +export type QuestionAnsweringOutput = QuestionAnsweringOutputElement[]; +/** + * Outputs of inference for the Question Answering task + */ +export interface QuestionAnsweringOutputElement { + /** + * The answer to the question. + */ + answer: string; + /** + * The character position in the input where the answer ends. + */ + end: number; + /** + * The probability associated to the answer. + */ + score: number; + /** + * The character position in the input where the answer begins. + */ + start: number; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/question-answering/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/question-answering/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..266b329df3c3d27637bab119a70cf9394b77a16f --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/question-answering/spec/input.json @@ -0,0 +1,66 @@ +{ + "$id": "/inference/schemas/question-answering/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Question Answering inference", + "title": "QuestionAnsweringInput", + "type": "object", + "properties": { + "inputs": { + "title": "QuestionAnsweringInputData", + "description": "One (context, question) pair to answer", + "type": "object", + "properties": { + "context": { + "type": "string", + "description": "The context to be used for answering the question" + }, + "question": { + "type": "string", + "description": "The question to be answered" + } + }, + "required": ["question", "context"] + }, + "parameters": { + "description": "Additional inference parameters for Question Answering", + "$ref": "#/$defs/QuestionAnsweringParameters" + } + }, + "$defs": { + "QuestionAnsweringParameters": { + "title": "QuestionAnsweringParameters", + "type": "object", + "properties": { + "top_k": { + "type": "integer", + "description": "The number of answers to return (will be chosen by order of likelihood). Note that we return less than topk answers if there are not enough options available within the context." + }, + "doc_stride": { + "type": "integer", + "description": "If the context is too long to fit with the question for the model, it will be split in several chunks with some overlap. This argument controls the size of that overlap." + }, + "max_answer_len": { + "type": "integer", + "description": "The maximum length of predicted answers (e.g., only answers with a shorter length are considered)." + }, + "max_seq_len": { + "type": "integer", + "description": "The maximum length of the total sentence (context + question) in tokens of each chunk passed to the model. The context will be split in several chunks (using docStride as overlap) if needed." + }, + "max_question_len": { + "type": "integer", + "description": "The maximum length of the question after tokenization. It will be truncated if needed." + }, + "handle_impossible_answer": { + "type": "boolean", + "description": "Whether to accept impossible as an answer." + }, + "align_to_words": { + "type": "boolean", + "description": "Attempts to align the answer to real words. Improves quality on space separated languages. Might hurt on non-space-separated languages (like Japanese or Chinese)" + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/question-answering/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/question-answering/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..9da8f988ad21668db65d18e22cb105a0da96a63d --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/question-answering/spec/output.json @@ -0,0 +1,29 @@ +{ + "$id": "/inference/schemas/question-answering/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "title": "QuestionAnsweringOutput", + "description": "Outputs of inference for the Question Answering task", + "type": "array", + "items": { + "type": "object", + "properties": { + "answer": { + "type": "string", + "description": "The answer to the question." + }, + "score": { + "type": "number", + "description": "The probability associated to the answer." + }, + "start": { + "type": "integer", + "description": "The character position in the input where the answer begins." + }, + "end": { + "type": "integer", + "description": "The character position in the input where the answer ends." + } + }, + "required": ["answer", "score", "start", "end"] + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/reinforcement-learning/about.md b/node_modules/@huggingface/tasks/src/tasks/reinforcement-learning/about.md new file mode 100644 index 0000000000000000000000000000000000000000..286fba9296b04b6389e3bcbb20bf7770f1bc982c --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/reinforcement-learning/about.md @@ -0,0 +1,167 @@ +## Use Cases + +### Gaming + +Reinforcement learning is known for its application to video games. Since the games provide a safe environment for the agent to be trained in the sense that it is perfectly defined and controllable, this makes them perfect candidates for experimentation and will help a lot to learn about the capabilities and limitations of various RL algorithms. + +There are many videos on the Internet where a game-playing reinforcement learning agent starts with a terrible gaming strategy due to random initialization of its settings, but over iterations, the agent gets better and better with each episode of the training. This [paper](https://arxiv.org/abs/1912.10944) mainly investigates the performance of RL in popular games such as Minecraft or Dota2. The agent's performance can exceed a human player's, although there are still some challenges mainly related to efficiency in constructing the gaming policy of the reinforcement learning agent. + +### Trading and Finance + +Reinforcement learning is the science to train computers to make decisions and thus has a novel use in trading and finance. All time-series models are helpful in predicting prices, volume and future sales of a product or a stock. Reinforcement based automated agents can decide to sell, buy or hold a stock. It shifts the impact of AI in this field to real time decision making rather than just prediction of prices. The glossary given below will clear some parameters to as to how we can train a model to take these decisions. + +## Task Variants + +### Model Based RL + +In model based reinforcement learning techniques intend to create a model of the environment, learn the state transition probabilities and the reward function, to find the optimal action. Some typical examples for model based reinforcement learning algorithms are dynamic programming, value iteration and policy iteration. + +### Model Free RL + +In model free reinforcement learning, agent decides on optimal actions based on its experience in the environment and the reward it collects from it. This is one of the most commonly used algorithms beneficial in complex environments, where modeling of state transition probabilities and reward functions are difficult. Some of the examples of model free reinforcement learning are SARSA, Q-Learning, actor-critic and proximal policy optimization (PPO) algorithms. + +## Glossary + + + +**Agent:** The learner and the decision maker. + +**Environment:** The part of the world the agent interacts, comprising everything outside the agent. + +Observations and states are the information our agent gets from the environment. In the case of a video game, it can be a frame (a screenshot). In the case of the trading agent, it can be the value of a certain stock. + +**State:** Complete description of the state of the environment with no hidden information. + +**Observation:** Partial description of the state, in a partially observed environment. + +**Action:** The decision taken by the agent. + +**Reward:** The numerical feedback signal that the agent receives from the environment based on the chosen action. + +**Return:** Cumulative Reward. In the simplest case, the return is the sum of the rewards. + +**Episode:** For some applications there is a natural notion of final time step. In this case, there is a starting point and an ending point (a terminal state). This creates an episode: a list of States, Actions, Rewards, and new States. For instance, think about Chess: an episode begins at the initial board position and ends when the game is over. + +**Policy:** The Policy is the brain of the Agent, it’s the function that tells what action to take given the state. So it defines the agent’s behavior at a given time. Reinforcement learning methods specify how the agent’s policy is changed as a result of its experience. + +## Inference + +Inference in reinforcement learning differs from other modalities, in which there's a model and test data. In reinforcement learning, once you have trained an agent in an environment, you try to run the trained agent for additional steps to get the average reward. + +A typical training cycle consists of gathering experience from the environment, training the agent, and running the agent on a test environment to obtain average reward. Below there's a snippet on how you can interact with the environment using the `gymnasium` library, train an agent using `stable-baselines3`, evaluate the agent on test environment and infer actions from the trained agent. + +```python +# Here we are running 20 episodes of CartPole-v1 environment, taking random actions +import gymnasium as gym + +env = gym.make("CartPole-v1") +observation, info = env.reset() + +for _ in range(20): + action = env.action_space.sample() # samples random action from action sample space + + # the agent takes the action + observation, reward, terminated, truncated, info = env.step(action) + + +# if the agent reaches terminal state, we reset the environment +if terminated or truncated: + + print("Environment is reset") + observation = env.reset() + +env.close() +``` + +Below snippet shows how to train a PPO model on LunarLander-v2 environment using `stable-baselines3` library and saving the model + +```python +from stable_baselines3 import PPO + +# initialize the environment + +env = gym.make("LunarLander-v2") + +# initialize the model + +model = PPO(policy = "MlpPolicy", + env = env, + n_steps = 1024, + batch_size = 64, + n_epochs = 4, + verbose = 1) + +# train the model for 1000 time steps +model.learn(total_timesteps = 1000) + +# Saving the model in desired directory +model_name = "PPO-LunarLander-v2" +model.save(model_name) +``` + +Below code shows how to evaluate an agent trained using `stable-baselines3` + +```python +# Loading a saved model and evaluating the model for 10 episodes +from stable_baselines3.common.evaluation import evaluate_policy +from stable_baselines3 import PPO + + +env = gym.make("LunarLander-v2") +# Loading the saved model +model = PPO.load("PPO-LunarLander-v2",env=env) + +# Initializing the evaluation environment +eval_env = gym.make("LunarLander-v2") + +# Running the trained agent on eval_env for 10 time steps and getting the mean reward +mean_reward, std_reward = evaluate_policy(model, eval_env, n_eval_episodes = 10, + deterministic=True) + +print(f"mean_reward={mean_reward:.2f} +/- {std_reward}") +``` + +Below code snippet shows how to infer actions from an agent trained using `stable-baselines3` + +```python +from stable_baselines3.common.evaluation import evaluate_policy +from stable_baselines3 import PPO + +# Loading the saved model +model = PPO.load("PPO-LunarLander-v2",env=env) + +# Getting the environment from the trained agent +env = model.get_env() + +obs = env.reset() +for i in range(1000): + # getting action predictions from the trained agent + action, _states = model.predict(obs, deterministic=True) + + # taking the predicted action in the environment to observe next state and rewards + obs, rewards, dones, info = env.step(action) +``` + +For more information, you can check out the documentations of the respective libraries. + +[Gymnasium Documentation](https://gymnasium.farama.org/) +[Stable Baselines Documentation](https://stable-baselines3.readthedocs.io/en/master/) + +## Useful Resources + +Would you like to learn more about the topic? Awesome! Here you can find some curated resources that you may find helpful! + +- [HuggingFace Deep Reinforcement Learning Class](https://github.com/huggingface/deep-rl-class) +- [Introduction to Deep Reinforcement Learning](https://huggingface.co/blog/deep-rl-intro) +- [Stable Baselines Integration with HuggingFace](https://huggingface.co/blog/sb3) +- Learn how reinforcement learning is used in conversational agents in this blog: [Illustrating Reinforcement Learning from Human Feedback (RLHF)](https://huggingface.co/blog/rlhf) +- [Reinforcement Learning from Human Feedback From Zero to ChatGPT](https://www.youtube.com/watch?v=EAd4oQtEJOM) +- [Guide on Multi-Agent Competition Systems](https://huggingface.co/blog/aivsai) + +### Notebooks + +- [Train a Deep Reinforcement Learning lander agent to land correctly on the Moon 🌕 using Stable-Baselines3](https://github.com/huggingface/deep-rl-class/blob/main/notebooks/unit1/unit1.ipynb) +- [Introduction to Unity MLAgents](https://github.com/huggingface/deep-rl-class/blob/main/notebooks/unit5/unit5.ipynb) +- [Training Decision Transformers with 🤗 transformers](https://github.com/huggingface/blog/blob/main/notebooks/101_train-decision-transformers.ipynb) + +This page was made possible thanks to the efforts of [Ram Ananth](https://huggingface.co/RamAnanth1), [Emilio Lehoucq](https://huggingface.co/emiliol), [Sagar Mathpal](https://huggingface.co/sagarmathpal) and [Osman Alenbey](https://huggingface.co/osman93). diff --git a/node_modules/@huggingface/tasks/src/tasks/reinforcement-learning/data.ts b/node_modules/@huggingface/tasks/src/tasks/reinforcement-learning/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..231ecabda2c346cc1a681ac6f99f585086d202b7 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/reinforcement-learning/data.ts @@ -0,0 +1,75 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A curation of widely used datasets for Data Driven Deep Reinforcement Learning (D4RL)", + id: "edbeeching/decision_transformer_gym_replay", + }, + ], + demo: { + inputs: [ + { + label: "State", + content: "Red traffic light, pedestrians are about to pass.", + type: "text", + }, + ], + outputs: [ + { + label: "Action", + content: "Stop the car.", + type: "text", + }, + { + label: "Next State", + content: "Yellow light, pedestrians have crossed.", + type: "text", + }, + ], + }, + metrics: [ + { + description: + "Accumulated reward across all time steps discounted by a factor that ranges between 0 and 1 and determines how much the agent optimizes for future relative to immediate rewards. Measures how good is the policy ultimately found by a given algorithm considering uncertainty over the future.", + id: "Discounted Total Reward", + }, + { + description: + "Average return obtained after running the policy for a certain number of evaluation episodes. As opposed to total reward, mean reward considers how much reward a given algorithm receives while learning.", + id: "Mean Reward", + }, + { + description: + "Measures how good a given algorithm is after a predefined time. Some algorithms may be guaranteed to converge to optimal behavior across many time steps. However, an agent that reaches an acceptable level of optimality after a given time horizon may be preferable to one that ultimately reaches optimality but takes a long time.", + id: "Level of Performance After Some Time", + }, + ], + models: [ + { + description: "A Reinforcement Learning model trained on expert data from the Gym Hopper environment", + + id: "edbeeching/decision-transformer-gym-hopper-expert", + }, + { + description: "A PPO agent playing seals/CartPole-v0 using the stable-baselines3 library and the RL Zoo.", + id: "HumanCompatibleAI/ppo-seals-CartPole-v0", + }, + ], + spaces: [ + { + description: "An application for a cute puppy agent learning to catch a stick.", + id: "ThomasSimonini/Huggy", + }, + { + description: "An application to play Snowball Fight with a reinforcement learning agent.", + id: "ThomasSimonini/SnowballFight", + }, + ], + summary: + "Reinforcement learning is the computational approach of learning from action by interacting with an environment through trial and error and receiving rewards (negative or positive) as feedback", + widgetModels: [], + youtubeId: "q0BiUn5LiBc", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/about.md b/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/about.md new file mode 100644 index 0000000000000000000000000000000000000000..160d35d8185f68ab0da5f09093495cb0896ea7d6 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/about.md @@ -0,0 +1,97 @@ +## Use Cases 🔍 + +### Information Retrieval + +You can extract information from documents using Sentence Similarity models. The first step is to rank documents using Passage Ranking models. You can then get to the top ranked document and search it with Sentence Similarity models by selecting the sentence that has the most similarity to the input query. + +## The Sentence Transformers library + +The [Sentence Transformers](https://www.sbert.net/) library is very powerful for calculating embeddings of sentences, paragraphs, and entire documents. An embedding is just a vector representation of a text and is useful for finding how similar two texts are. + +You can find and use [thousands of Sentence Transformers](https://huggingface.co/models?library=sentence-transformers&sort=downloads) models from the Hub by directly using the library, playing with the widgets in the browser or using Inference Endpoints. + +## Task Variants + +### Passage Ranking + +Passage Ranking is the task of ranking documents based on their relevance to a given query. The task is evaluated on Mean Reciprocal Rank. These models take one query and multiple documents and return ranked documents according to the relevancy to the query. 📄 + +You can infer with Passage Ranking models using [Inference Endpoints](https://huggingface.co/inference-endpoints). The Passage Ranking model inputs are a query for which we look for relevancy in the documents and the documents we want to search. The model will return scores according to the relevancy of these documents for the query. + +```python +import json +import requests + +API_URL = "https://router.huggingface.co/hf-inference/models/sentence-transformers/msmarco-distilbert-base-tas-b" +headers = {"Authorization": f"Bearer {api_token}"} + +def query(payload): + response = requests.post(API_URL, headers=headers, json=payload) + return response.json() + +data = query( + { + "inputs": { + "source_sentence": "That is a happy person", + "sentences": [ + "That is a happy dog", + "That is a very happy person", + "Today is a sunny day" + ] + } + } +## [0.853, 0.981, 0.655] +``` + +### Semantic Textual Similarity + +Semantic Textual Similarity is the task of evaluating how similar two texts are in terms of meaning. These models take a source sentence and a list of sentences in which we will look for similarities and will return a list of similarity scores. The benchmark dataset is the [Semantic Textual Similarity Benchmark](http://ixa2.si.ehu.eus/stswiki/index.php/STSbenchmark). The task is evaluated on Pearson’s Rank Correlation. + +```python +import json +import requests + +API_URL = "https://router.huggingface.co/hf-inference/models/sentence-transformers/all-MiniLM-L6-v2" +headers = {"Authorization": f"Bearer {api_token}"} + +def query(payload): + response = requests.post(API_URL, headers=headers, json=payload) + return response.json() + +data = query( + { + "inputs": { + "source_sentence": "I'm very happy", + "sentences":["I'm filled with happiness", "I'm happy"] + } + }) + +## [0.605, 0.894] +``` + +You can also infer with the models in the Hub using Sentence Transformer models. + +```python +pip install -U sentence-transformers + +from sentence_transformers import SentenceTransformer, util +sentences = ["I'm happy", "I'm full of happiness"] + +model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2') + +# Compute embedding for both lists +embedding_1 = model.encode(sentences[0], convert_to_tensor=True) +embedding_2 = model.encode(sentences[1], convert_to_tensor=True) + +util.pytorch_cos_sim(embedding_1, embedding_2) +## tensor([[0.6003]]) +``` + +## Useful Resources + +Would you like to learn more about Sentence Transformers and Sentence Similarity? Awesome! Here you can find some curated resources that you may find helpful! + +- [Sentence Transformers Documentation](https://www.sbert.net/) +- [Sentence Transformers in the Hub](https://huggingface.co/blog/sentence-transformers-in-the-hub) +- [Building a Playlist Generator with Sentence Transformers](https://huggingface.co/blog/playlist-generator) +- [Getting Started With Embeddings](https://huggingface.co/blog/getting-started-with-embeddings) diff --git a/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/data.ts b/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..4cad5c918cac065bb6c73ff29e30ce1f1fec02c3 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/data.ts @@ -0,0 +1,105 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "Bing queries with relevant passages from various web sources.", + id: "microsoft/ms_marco", + }, + ], + demo: { + inputs: [ + { + label: "Source sentence", + content: "Machine learning is so easy.", + type: "text", + }, + { + label: "Sentences to compare to", + content: "Deep learning is so straightforward.", + type: "text", + }, + { + label: "", + content: "This is so difficult, like rocket science.", + type: "text", + }, + { + label: "", + content: "I can't believe how much I struggled with this.", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Deep learning is so straightforward.", + score: 0.623, + }, + { + label: "This is so difficult, like rocket science.", + score: 0.413, + }, + { + label: "I can't believe how much I struggled with this.", + score: 0.256, + }, + ], + }, + ], + }, + metrics: [ + { + description: + "Reciprocal Rank is a measure used to rank the relevancy of documents given a set of documents. Reciprocal Rank is the reciprocal of the rank of the document retrieved, meaning, if the rank is 3, the Reciprocal Rank is 0.33. If the rank is 1, the Reciprocal Rank is 1", + id: "Mean Reciprocal Rank", + }, + { + description: + "The similarity of the embeddings is evaluated mainly on cosine similarity. It is calculated as the cosine of the angle between two vectors. It is particularly useful when your texts are not the same length", + id: "Cosine Similarity", + }, + ], + models: [ + { + description: + "This model works well for sentences and paragraphs and can be used for clustering/grouping and semantic searches.", + id: "sentence-transformers/all-mpnet-base-v2", + }, + { + description: "A multilingual robust sentence similarity model.", + id: "BAAI/bge-m3", + }, + { + description: "A robust sentence similarity model.", + id: "HIT-TMG/KaLM-embedding-multilingual-mini-instruct-v1.5", + }, + ], + spaces: [ + { + description: "An application that leverages sentence similarity to answer questions from YouTube videos.", + id: "Gradio-Blocks/Ask_Questions_To_YouTube_Videos", + }, + { + description: + "An application that retrieves relevant PubMed abstracts for a given online article which can be used as further references.", + id: "Gradio-Blocks/pubmed-abstract-retriever", + }, + { + description: "An application that leverages sentence similarity to summarize text.", + id: "nickmuchi/article-text-summarizer", + }, + { + description: "A guide that explains how Sentence Transformers can be used for semantic search.", + id: "sentence-transformers/Sentence_Transformers_for_semantic_search", + }, + ], + summary: + "Sentence Similarity is the task of determining how similar two texts are. Sentence similarity models convert input texts into vectors (embeddings) that capture semantic information and calculate how close (similar) they are between them. This task is particularly useful for information retrieval and clustering/grouping.", + widgetModels: ["sentence-transformers/all-MiniLM-L6-v2"], + youtubeId: "VCZq5AkbNEU", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/inference.ts b/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..c589124bdfa372c8974d177032485a3054d3cb94 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/inference.ts @@ -0,0 +1,31 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +export type SentenceSimilarityOutput = number[]; +/** + * Inputs for Sentence similarity inference + */ +export interface SentenceSimilarityInput { + inputs: SentenceSimilarityInputData; + /** + * Additional inference parameters for Sentence Similarity + */ + parameters?: { + [key: string]: unknown; + }; + [property: string]: unknown; +} +export interface SentenceSimilarityInputData { + /** + * A list of strings which will be compared against the source_sentence. + */ + sentences: string[]; + /** + * The string that you wish to compare the other strings with. This can be a phrase, + * sentence, or longer passage, depending on the model being used. + */ + source_sentence: string; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..d724af588d3f57d9535e8f7e81a5e24d34e750d3 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/spec/input.json @@ -0,0 +1,39 @@ +{ + "$id": "/inference/schemas/sentence-similarity/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Sentence similarity inference", + "title": "SentenceSimilarityInput", + "type": "object", + "properties": { + "inputs": { + "title": "SentenceSimilarityInputData", + "type": "object", + "properties": { + "source_sentence": { + "description": "The string that you wish to compare the other strings with. This can be a phrase, sentence, or longer passage, depending on the model being used.", + "type": "string" + }, + "sentences": { + "type": "array", + "description": "A list of strings which will be compared against the source_sentence.", + "items": { + "type": "string" + } + } + }, + "required": ["source_sentence", "sentences"] + }, + "parameters": { + "description": "Additional inference parameters for Sentence Similarity", + "$ref": "#/$defs/SentenceSimilarityParameters" + } + }, + "$defs": { + "SentenceSimilarityParameters": { + "title": "SentenceSimilarityParameters", + "type": "object", + "properties": {} + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..ca13d98bd5f55bd581e99c8cc4d970b9b7735512 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/sentence-similarity/spec/output.json @@ -0,0 +1,12 @@ +{ + "$id": "/inference/schemas/sentence-similarity/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "title": "SentenceSimilarityOutput", + "description": "Outputs of inference for the Sentence Similarity task", + "type": "array", + "items": { + "description": "The associated similarity score for each of the given sentences", + "type": "number", + "title": "SentenceSimilarityScore" + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/summarization/about.md b/node_modules/@huggingface/tasks/src/tasks/summarization/about.md new file mode 100644 index 0000000000000000000000000000000000000000..44c8d2a4a0e12aaf91562977f447edf214d894c6 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/summarization/about.md @@ -0,0 +1,58 @@ +## Use Cases + +### Research Paper Summarization 🧐 + +Research papers can be summarized to allow researchers to spend less time selecting which articles to read. There are several approaches you can take for a task like this: + +1. Use an existing extractive summarization model on the Hub to do inference. +2. Pick an existing language model trained for academic papers. This model can then be trained in a process called fine-tuning so it can solve the summarization task. +3. Use a sequence-to-sequence model like [T5](https://huggingface.co/docs/transformers/model_doc/t5) for abstractive text summarization. + +## Inference + +You can use the 🤗 Transformers library `summarization` pipeline to infer with existing Summarization models. If no model name is provided the pipeline will be initialized with [sshleifer/distilbart-cnn-12-6](https://huggingface.co/sshleifer/distilbart-cnn-12-6). + +```python +from transformers import pipeline + +classifier = pipeline("summarization") +classifier("Paris is the capital and most populous city of France, with an estimated population of 2,175,601 residents as of 2018, in an area of more than 105 square kilometres (41 square miles). The City of Paris is the centre and seat of government of the region and province of Île-de-France, or Paris Region, which has an estimated population of 12,174,880, or about 18 percent of the population of France as of 2017.") +## [{ "summary_text": " Paris is the capital and most populous city of France..." }] +``` + +You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer summarization models on Hugging Face Hub. + +```javascript +import { InferenceClient } from "@huggingface/inference"; + +const inference = new InferenceClient(HF_TOKEN); +const inputs = + "Paris is the capital and most populous city of France, with an estimated population of 2,175,601 residents as of 2018, in an area of more than 105 square kilometres (41 square miles). The City of Paris is the centre and seat of government of the region and province of Île-de-France, or Paris Region, which has an estimated population of 12,174,880, or about 18 percent of the population of France as of 2017."; + +await inference.summarization({ + model: "sshleifer/distilbart-cnn-12-6", + inputs, +}); +``` + +## Useful Resources + +Would you like to learn more about the topic? Awesome! Here you can find some curated resources that you may find helpful! + +- [Course Chapter on Summarization](https://huggingface.co/course/chapter7/5?fw=pt) +- [Distributed Training: Train BART/T5 for Summarization using 🤗 Transformers and Amazon SageMaker](https://huggingface.co/blog/sagemaker-distributed-training-seq2seq) + +### Notebooks + +- [PyTorch](https://github.com/huggingface/notebooks/blob/master/examples/summarization.ipynb) +- [TensorFlow](https://github.com/huggingface/notebooks/blob/master/examples/summarization-tf.ipynb) + +### Scripts for training + +- [PyTorch](https://github.com/huggingface/transformers/tree/main/examples/pytorch/summarization) +- [TensorFlow](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/summarization) +- [Flax](https://github.com/huggingface/transformers/tree/main/examples/flax/summarization) + +### Documentation + +- [Summarization task guide](https://huggingface.co/docs/transformers/tasks/summarization) diff --git a/node_modules/@huggingface/tasks/src/tasks/summarization/data.ts b/node_modules/@huggingface/tasks/src/tasks/summarization/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..744503ece07e487fb1463bb6f764cb64a6633a05 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/summarization/data.ts @@ -0,0 +1,76 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + canonicalId: "text-generation", + datasets: [ + { + description: + "News articles in five different languages along with their summaries. Widely used for benchmarking multilingual summarization models.", + id: "mlsum", + }, + { + description: "English conversations and their summaries. Useful for benchmarking conversational agents.", + id: "samsum", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: + "The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. It was the first structure to reach a height of 300 metres. Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct.", + type: "text", + }, + ], + outputs: [ + { + label: "Output", + content: + "The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building. It was the first structure to reach a height of 300 metres.", + type: "text", + }, + ], + }, + metrics: [ + { + description: + "The generated sequence is compared against its summary, and the overlap of tokens are counted. ROUGE-N refers to overlap of N subsequent tokens, ROUGE-1 refers to overlap of single tokens and ROUGE-2 is the overlap of two subsequent tokens.", + id: "rouge", + }, + ], + models: [ + { + description: + "A strong summarization model trained on English news articles. Excels at generating factual summaries.", + id: "facebook/bart-large-cnn", + }, + { + description: "A summarization model trained on medical articles.", + id: "Falconsai/medical_summarization", + }, + ], + spaces: [ + { + description: "An application that can summarize long paragraphs.", + id: "pszemraj/summarize-long-text", + }, + { + description: "A much needed summarization application for terms and conditions.", + id: "ml6team/distilbart-tos-summarizer-tosdr", + }, + { + description: "An application that summarizes long documents.", + id: "pszemraj/document-summarization", + }, + { + description: "An application that can detect errors in abstractive summarization.", + id: "ml6team/post-processing-summarization", + }, + ], + summary: + "Summarization is the task of producing a shorter version of a document while preserving its important information. Some models can extract text from the original input, while other models can generate entirely new text.", + widgetModels: ["facebook/bart-large-cnn"], + youtubeId: "yHnr5Dk2zCI", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/summarization/inference.ts b/node_modules/@huggingface/tasks/src/tasks/summarization/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..ed28c5632f713eda38f6c4b669c5843c5845e073 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/summarization/inference.ts @@ -0,0 +1,53 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Summarization inference + */ +export interface SummarizationInput { + /** + * The input text to summarize. + */ + inputs: string; + /** + * Additional inference parameters for summarization. + */ + parameters?: SummarizationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for summarization. + */ +export interface SummarizationParameters { + /** + * Whether to clean up the potential extra spaces in the text output. + */ + clean_up_tokenization_spaces?: boolean; + /** + * Additional parametrization of the text generation algorithm. + */ + generate_parameters?: { + [key: string]: unknown; + }; + /** + * The truncation strategy to use. + */ + truncation?: SummarizationTruncationStrategy; + [property: string]: unknown; +} +/** + * The truncation strategy to use. + */ +export type SummarizationTruncationStrategy = "do_not_truncate" | "longest_first" | "only_first" | "only_second"; +/** + * Outputs of inference for the Summarization task + */ +export interface SummarizationOutput { + /** + * The summarized text. + */ + summary_text: string; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/summarization/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/summarization/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..c2d7aebaf578bab13eff197aea6d51dff2bb5d4a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/summarization/spec/input.json @@ -0,0 +1,41 @@ +{ + "$id": "/inference/schemas/summarization/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Summarization inference", + "title": "SummarizationInput", + "type": "object", + "properties": { + "inputs": { + "description": "The input text to summarize.", + "type": "string" + }, + "parameters": { + "description": "Additional inference parameters for summarization.", + "$ref": "#/$defs/SummarizationParameters" + } + }, + "$defs": { + "SummarizationParameters": { + "title": "SummarizationParameters", + "type": "object", + "properties": { + "clean_up_tokenization_spaces": { + "type": "boolean", + "description": "Whether to clean up the potential extra spaces in the text output." + }, + "truncation": { + "title": "SummarizationTruncationStrategy", + "type": "string", + "description": "The truncation strategy to use.", + "enum": ["do_not_truncate", "longest_first", "only_first", "only_second"] + }, + "generate_parameters": { + "title": "generateParameters", + "type": "object", + "description": "Additional parametrization of the text generation algorithm." + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/summarization/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/summarization/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..dfa307b642213451b0aa7e8b88d1bfdaf6f77598 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/summarization/spec/output.json @@ -0,0 +1,14 @@ +{ + "$id": "/inference/schemas/summarization/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Summarization task", + "title": "SummarizationOutput", + "type": "object", + "properties": { + "summary_text": { + "type": "string", + "description": "The summarized text." + } + }, + "required": ["summary_text"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/table-question-answering/about.md b/node_modules/@huggingface/tasks/src/tasks/table-question-answering/about.md new file mode 100644 index 0000000000000000000000000000000000000000..62cdb44ad25d94249ea6c4953a054b60c4021057 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/table-question-answering/about.md @@ -0,0 +1,43 @@ +## Use Cases + +### SQL execution + +You can use the Table Question Answering models to simulate SQL execution by inputting a table. + +### Table Question Answering + +Table Question Answering models are capable of answering questions based on a table. + +## Task Variants + +This place can be filled with variants of this task if there's any. + +## Inference + +You can infer with TableQA models using the 🤗 Transformers library. + +```python +from transformers import pipeline +import pandas as pd + +# prepare table + question +data = {"Actors": ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], "Number of movies": ["87", "53", "69"]} +table = pd.DataFrame.from_dict(data) +question = "how many movies does Leonardo Di Caprio have?" + +# pipeline model +# Note: you must to install torch-scatter first. +tqa = pipeline(task="table-question-answering", model="google/tapas-large-finetuned-wtq") + +# result + +print(tqa(table=table, query=question)['cells'][0]) +#53 + +``` + +## Useful Resources + +In this area, you can insert useful resources about how to train or use a model for this task. + +This task page is complete thanks to the efforts of [Hao Kim Tieu](https://huggingface.co/haotieu). 🦸 diff --git a/node_modules/@huggingface/tasks/src/tasks/table-question-answering/data.ts b/node_modules/@huggingface/tasks/src/tasks/table-question-answering/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..b5f161a4ca80310e591d20d96ea53d33ffe49d10 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/table-question-answering/data.ts @@ -0,0 +1,59 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: + "The WikiTableQuestions dataset is a large-scale dataset for the task of question answering on semi-structured tables.", + id: "wikitablequestions", + }, + { + description: + "WikiSQL is a dataset of 80654 hand-annotated examples of questions and SQL queries distributed across 24241 tables from Wikipedia.", + id: "wikisql", + }, + ], + demo: { + inputs: [ + { + table: [ + ["Rank", "Name", "No.of reigns", "Combined days"], + ["1", "lou Thesz", "3", "3749"], + ["2", "Ric Flair", "8", "3103"], + ["3", "Harley Race", "7", "1799"], + ], + type: "tabular", + }, + + { label: "Question", content: "What is the number of reigns for Harley Race?", type: "text" }, + ], + outputs: [{ label: "Result", content: "7", type: "text" }], + }, + metrics: [ + { + description: "Checks whether the predicted answer(s) is the same as the ground-truth answer(s).", + id: "Denotation Accuracy", + }, + ], + models: [ + { + description: + "A table question answering model that is capable of neural SQL execution, i.e., employ TAPEX to execute a SQL query on a given table.", + id: "microsoft/tapex-base", + }, + { + description: "A robust table question answering model.", + id: "google/tapas-base-finetuned-wtq", + }, + ], + spaces: [ + { + description: "An application that answers questions based on table CSV files.", + id: "katanaml/table-query", + }, + ], + summary: "Table Question Answering (Table QA) is the answering a question about an information on a given table.", + widgetModels: ["google/tapas-base-finetuned-wtq"], +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/table-question-answering/inference.ts b/node_modules/@huggingface/tasks/src/tasks/table-question-answering/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..18339e033180ad689aae33ccc671cc14c157f0b8 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/table-question-answering/inference.ts @@ -0,0 +1,83 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Table Question Answering inference + */ +export interface TableQuestionAnsweringInput { + /** + * One (table, question) pair to answer + */ + inputs: TableQuestionAnsweringInputData; + /** + * Additional inference parameters for Table Question Answering + */ + parameters?: TableQuestionAnsweringParameters; + [property: string]: unknown; +} +/** + * One (table, question) pair to answer + */ +export interface TableQuestionAnsweringInputData { + /** + * The question to be answered about the table + */ + question: string; + /** + * The table to serve as context for the questions + */ + table: { + [key: string]: string[]; + }; + [property: string]: unknown; +} +/** + * Additional inference parameters for Table Question Answering + */ +export interface TableQuestionAnsweringParameters { + /** + * Activates and controls padding. + */ + padding?: Padding; + /** + * Whether to do inference sequentially or as a batch. Batching is faster, but models like + * SQA require the inference to be done sequentially to extract relations within sequences, + * given their conversational nature. + */ + sequential?: boolean; + /** + * Activates and controls truncation. + */ + truncation?: boolean; + [property: string]: unknown; +} +/** + * Activates and controls padding. + */ +export type Padding = "do_not_pad" | "longest" | "max_length"; +export type TableQuestionAnsweringOutput = TableQuestionAnsweringOutputElement[]; +/** + * Outputs of inference for the Table Question Answering task + */ +export interface TableQuestionAnsweringOutputElement { + /** + * If the model has an aggregator, this returns the aggregator. + */ + aggregator?: string; + /** + * The answer of the question given the table. If there is an aggregator, the answer will be + * preceded by `AGGREGATOR >`. + */ + answer: string; + /** + * List of strings made up of the answer cell values. + */ + cells: string[]; + /** + * Coordinates of the cells of the answers. + */ + coordinates: Array; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/table-question-answering/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/table-question-answering/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..67cc18eae986d8d392c4b5842cf9ecd7aec4309c --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/table-question-answering/spec/input.json @@ -0,0 +1,60 @@ +{ + "$id": "/inference/schemas/table-question-answering/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Table Question Answering inference", + "title": "TableQuestionAnsweringInput", + "type": "object", + "properties": { + "inputs": { + "description": "One (table, question) pair to answer", + "title": "TableQuestionAnsweringInputData", + "type": "object", + "properties": { + "table": { + "description": "The table to serve as context for the questions", + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "question": { + "description": "The question to be answered about the table", + "type": "string" + } + }, + "required": ["table", "question"] + }, + "parameters": { + "description": "Additional inference parameters for Table Question Answering", + "$ref": "#/$defs/TableQuestionAnsweringParameters" + } + }, + "$defs": { + "TableQuestionAnsweringParameters": { + "title": "TableQuestionAnsweringParameters", + "type": "object", + "properties": { + "padding": { + "type": "string", + "default": "do_not_pad", + "description": "Activates and controls padding.", + "enum": ["do_not_pad", "longest", "max_length"] + }, + "sequential": { + "type": "boolean", + "default": "false", + "description": "Whether to do inference sequentially or as a batch. Batching is faster, but models like SQA require the inference to be done sequentially to extract relations within sequences, given their conversational nature." + }, + "truncation": { + "type": "boolean", + "default": "false", + "description": "Activates and controls truncation." + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/table-question-answering/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/table-question-answering/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..9b43026ea12299dc83110c99d3983841a8d30c6e --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/table-question-answering/spec/output.json @@ -0,0 +1,40 @@ +{ + "$id": "/inference/schemas/table-question-answering/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Table Question Answering task", + "title": "TableQuestionAnsweringOutput", + "type": "array", + "items": { + "type": "object", + "properties": { + "answer": { + "type": "string", + "description": "The answer of the question given the table. If there is an aggregator, the answer will be preceded by `AGGREGATOR >`." + }, + "coordinates": { + "type": "array", + "description": "Coordinates of the cells of the answers.", + "items": { + "type": "array", + "items": { + "type": "integer" + }, + "minLength": 2, + "maxLength": 2 + } + }, + "cells": { + "type": "array", + "description": "List of strings made up of the answer cell values.", + "items": { + "type": "string" + } + }, + "aggregator": { + "type": "string", + "description": "If the model has an aggregator, this returns the aggregator." + } + }, + "required": ["answer", "cells", "coordinates"] + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/tabular-classification/about.md b/node_modules/@huggingface/tasks/src/tasks/tabular-classification/about.md new file mode 100644 index 0000000000000000000000000000000000000000..d46a48976c894b8a5d99dda14a893261c49e6d41 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/tabular-classification/about.md @@ -0,0 +1,65 @@ +## About the Task + +Tabular classification is the task of assigning a label or class given a limited number of attributes. For example, the input can be data related to a customer (balance of the customer, the time being a customer, or more) and the output can be whether the customer will churn from the service or not. +There are three types of categorical variables: + +- Binary variables: Variables that can take two values, like yes or no, open or closed. The task of predicting binary variables is called binary classification. +- Ordinal variables: Variables with a ranking relationship, e.g., good, insignificant, and bad product reviews. The task of predicting ordinal variables is called ordinal classification. +- Nominal variables: Variables with no ranking relationship among them, e.g., predicting an animal from their weight and height, where categories are cat, dog, or bird. The task of predicting nominal variables is called multinomial classification. + +## Use Cases + +### Fraud Detection +Tabular classification models can be used in detecting fraudulent credit card transactions, where the features could be the amount of the transaction and the account balance, and the target to predict could be whether the transaction is fraudulent or not. This is an example of binary classification. + +### Churn Prediction +Tabular classification models can be used in predicting customer churn in telecommunication. An example dataset for the task is hosted [here](https://huggingface.co/datasets/scikit-learn/churn-prediction). + +# Model Hosting and Inference + +You can use [skops](https://skops.readthedocs.io/) for model hosting and inference on the Hugging Face Hub. This library is built to improve production workflows of various libraries that are used to train tabular models, including [sklearn](https://scikit-learn.org/stable/) and [xgboost](https://xgboost.readthedocs.io/en/stable/). Using `skops` you can: + +- Easily use Inference Endpoints +- Build neat UIs with one line of code, +- Programmatically create model cards, +- Securely serialize your scikit-learn model. (See limitations of using pickle [here](https://huggingface.co/docs/hub/security-pickle).) + +You can push your model as follows: + +```python +from skops import hub_utils +# initialize a repository with a trained model +local_repo = "/path_to_new_repo" +hub_utils.init(model, dst=local_repo) +# push to Hub! +hub_utils.push("username/my-awesome-model", source=local_repo) +``` + +Once the model is pushed, you can infer easily. + +```python +import skops.hub_utils as hub_utils +import pandas as pd +data = pd.DataFrame(your_data) +# Load the model from the Hub +res = hub_utils.get_model_output("username/my-awesome-model", data) +``` + +You can launch a UI for your model with only one line of code! + +```python +import gradio as gr +gr.Interface.load("huggingface/username/my-awesome-model").launch() +``` + +## Useful Resources + +- Check out the [scikit-learn organization](https://huggingface.co/scikit-learn) to learn more about different algorithms used for this task. +- [Skops documentation](https://skops.readthedocs.io/en/latest/) +- [Skops announcement blog](https://huggingface.co/blog/skops) +- [Notebook: Persisting your scikit-learn model using skops](https://www.kaggle.com/code/unofficialmerve/persisting-your-scikit-learn-model-using-skops) +- Check out [interactive sklearn examples](https://huggingface.co/sklearn-docs) built with ❤️ using Gradio. + +### Training your own model in just a few seconds + +We have built a [baseline trainer](https://huggingface.co/spaces/scikit-learn/baseline-trainer) application to which you can drag and drop your dataset. It will train a baseline and push it to your Hugging Face Hub profile with a model card containing information about the model. diff --git a/node_modules/@huggingface/tasks/src/tasks/tabular-classification/data.ts b/node_modules/@huggingface/tasks/src/tasks/tabular-classification/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..80dbe57d20a127d16e3ddecaeae5afe5e888cde1 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/tabular-classification/data.ts @@ -0,0 +1,68 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A comprehensive curation of datasets covering all benchmarks.", + id: "inria-soda/tabular-benchmark", + }, + ], + demo: { + inputs: [ + { + table: [ + ["Glucose", "Blood Pressure ", "Skin Thickness", "Insulin", "BMI"], + ["148", "72", "35", "0", "33.6"], + ["150", "50", "30", "0", "35.1"], + ["141", "60", "29", "1", "39.2"], + ], + type: "tabular", + }, + ], + outputs: [ + { + table: [["Diabetes"], ["1"], ["1"], ["0"]], + type: "tabular", + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + description: "Breast cancer prediction model based on decision trees.", + id: "scikit-learn/cancer-prediction-trees", + }, + ], + spaces: [ + { + description: "An application that can predict defective products on a production line.", + id: "scikit-learn/tabular-playground", + }, + { + description: "An application that compares various tabular classification techniques on different datasets.", + id: "scikit-learn/classification", + }, + ], + summary: "Tabular classification is the task of classifying a target category (a group) based on set of attributes.", + widgetModels: ["scikit-learn/tabular-playground"], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/tabular-regression/about.md b/node_modules/@huggingface/tasks/src/tasks/tabular-regression/about.md new file mode 100644 index 0000000000000000000000000000000000000000..4a6d9d302cc484c9b8b642d8bdba4e4f626a2e13 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/tabular-regression/about.md @@ -0,0 +1,86 @@ +## About the Task + +Tabular regression is the task of predicting a numerical value given a set of attributes/features. _Tabular_ meaning that data is stored in a table (like an excel sheet), and each sample is contained in its own row. The features used to predict our target can be both numerical and categorical. However, including categorical features often requires additional preprocessing/feature engineering (a few models do accept categorical features directly, like [CatBoost](https://catboost.ai/)). An example of tabular regression would be predicting the weight of a fish given its' species and length. + +## Use Cases + +### Sales Prediction: a Use Case for Predicting a Continuous Target Variable + +Here the objective is to predict a continuous variable based on a set of input variable(s). For example, predicting `sales` of an ice cream shop based on `temperature` of weather and `duration of hours` shop was open. Here we can build a regression model with `temperature` and `duration of hours` as input variable and `sales` as target variable. + +### Missing Value Imputation for Other Tabular Tasks +In real-world applications, due to human error or other reasons, some of the input values can be missing or there might not be any recorded data. Considering the example above, say the shopkeeper's watch was broken and they forgot to calculate the `hours` for which the shop was open. This will lead to a missing value in their dataset. In this case, missing values could be replaced it with zero, or average hours for which the shop is kept open. Another approach we can try is to use `temperature` and `sales` variables to predict the `hours` variable here. + +## Model Training + +A simple regression model can be created using `sklearn` as follows: + +```python +#set the input features +X = data[["Feature 1", "Feature 2", "Feature 3"]] +#set the target variable +y = data["Target Variable"] +#initialize the model +model = LinearRegression() +#Fit the model +model.fit(X, y) +``` + +# Model Hosting and Inference + +You can use [skops](https://skops.readthedocs.io/) for model hosting and inference on the Hugging Face Hub. This library is built to improve production workflows of various libraries that are used to train tabular models, including [sklearn](https://scikit-learn.org/stable/) and [xgboost](https://xgboost.readthedocs.io/en/stable/). Using `skops` you can: + +- Easily use Inference Endpoints, +- Build neat UIs with one line of code, +- Programmatically create model cards, +- Securely serialize your models. (See limitations of using pickle [here](https://huggingface.co/docs/hub/security-pickle).) + +You can push your model as follows: + +```python +from skops import hub_utils +# initialize a repository with a trained model +local_repo = "/path_to_new_repo" +hub_utils.init(model, dst=local_repo) +# push to Hub! +hub_utils.push("username/my-awesome-model", source=local_repo) +``` + +Once the model is pushed, you can infer easily. + +```python +import skops.hub_utils as hub_utils +import pandas as pd +data = pd.DataFrame(your_data) +# Load the model from the Hub +res = hub_utils.get_model_output("username/my-awesome-model", data) +``` + +You can launch a UI for your model with only one line of code! + +```python +import gradio as gr +gr.Interface.load("huggingface/username/my-awesome-model").launch() +``` + +## Useful Resources + +- [Skops documentation](https://skops.readthedocs.io/en/stable/index.html) + +- Check out [interactive sklearn examples](https://huggingface.co/sklearn-docs) built with ❤️ using Gradio. +- [Notebook: Persisting your scikit-learn model using skops](https://www.kaggle.com/code/unofficialmerve/persisting-your-scikit-learn-model-using-skops) + +- For starting with tabular regression: + - Doing [Exploratory Data Analysis](https://neptune.ai/blog/exploratory-data-analysis-for-tabular-data) for tabular data. + - The data considered here consists of details of Olympic athletes and medal results from Athens 1896 to Rio 2016. + - Here you can learn more about how to explore and analyse the data and visualize them in order to get a better understanding of dataset. + - Building your [first ML model](https://www.kaggle.com/code/dansbecker/your-first-machine-learning-model). + +- Intermediate level tutorials on tabular regression: + - [A Short Chronology of Deep Learning for Tabular Data](https://sebastianraschka.com/blog/2022/deep-learning-for-tabular-data.html) by Sebastian Raschka. + +### Training your own model in just a few seconds + +We have built a [baseline trainer](https://huggingface.co/spaces/scikit-learn/baseline-trainer) application to which you can drag and drop your dataset. It will train a baseline and push it to your Hugging Face Hub profile with a model card containing information about the model. + +This page was made possible thanks to efforts of [Brenden Connors](https://huggingface.co/brendenc) and [Ayush Bihani](https://huggingface.co/hsuyab). diff --git a/node_modules/@huggingface/tasks/src/tasks/tabular-regression/data.ts b/node_modules/@huggingface/tasks/src/tasks/tabular-regression/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..c4c0b6d018687d90a46654c90e8c3e5c9773816a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/tabular-regression/data.ts @@ -0,0 +1,57 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A comprehensive curation of datasets covering all benchmarks.", + id: "inria-soda/tabular-benchmark", + }, + ], + demo: { + inputs: [ + { + table: [ + ["Car Name", "Horsepower", "Weight"], + ["ford torino", "140", "3,449"], + ["amc hornet", "97", "2,774"], + ["toyota corolla", "65", "1,773"], + ], + type: "tabular", + }, + ], + outputs: [ + { + table: [["MPG (miles per gallon)"], ["17"], ["18"], ["31"]], + type: "tabular", + }, + ], + }, + metrics: [ + { + description: "", + id: "mse", + }, + { + description: + "Coefficient of determination (or R-squared) is a measure of how well the model fits the data. Higher R-squared is considered a better fit.", + id: "r-squared", + }, + ], + models: [ + { + description: "Fish weight prediction based on length measurements and species.", + id: "scikit-learn/Fish-Weight", + }, + ], + spaces: [ + { + description: "An application that can predict weight of a fish based on set of attributes.", + id: "scikit-learn/fish-weight-prediction", + }, + ], + summary: "Tabular regression is the task of predicting a numerical value given a set of attributes.", + widgetModels: ["scikit-learn/Fish-Weight"], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/text-classification/about.md b/node_modules/@huggingface/tasks/src/tasks/text-classification/about.md new file mode 100644 index 0000000000000000000000000000000000000000..dc0bfdf4709bb7a6b06529b2a6e18a1099738d14 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-classification/about.md @@ -0,0 +1,173 @@ +## Use Cases + +### Sentiment Analysis on Customer Reviews + +You can track the sentiments of your customers from the product reviews using sentiment analysis models. This can help understand churn and retention by grouping reviews by sentiment, to later analyze the text and make strategic decisions based on this knowledge. + +## Task Variants + +### Natural Language Inference (NLI) + +In NLI the model determines the relationship between two given texts. Concretely, the model takes a premise and a hypothesis and returns a class that can either be: + +- **entailment**, which means the hypothesis is true. +- **contraction**, which means the hypothesis is false. +- **neutral**, which means there's no relation between the hypothesis and the premise. + +The benchmark dataset for this task is GLUE (General Language Understanding Evaluation). NLI models have different variants, such as Multi-Genre NLI, Question NLI and Winograd NLI. + +### Multi-Genre NLI (MNLI) + +MNLI is used for general NLI. Here are som examples: + +``` +Example 1: + Premise: A man inspects the uniform of a figure in some East Asian country. + Hypothesis: The man is sleeping. + Label: Contradiction + +Example 2: + Premise: Soccer game with multiple males playing. + Hypothesis: Some men are playing a sport. + Label: Entailment +``` + +#### Inference + +You can use the 🤗 Transformers library `text-classification` pipeline to infer with NLI models. + +```python +from transformers import pipeline + +classifier = pipeline("text-classification", model = "roberta-large-mnli") +classifier("A soccer game with multiple males playing. Some men are playing a sport.") +## [{'label': 'ENTAILMENT', 'score': 0.98}] +``` + +### Question Natural Language Inference (QNLI) + +QNLI is the task of determining if the answer to a certain question can be found in a given document. If the answer can be found the label is “entailment”. If the answer cannot be found the label is “not entailment". + +``` +Question: What percentage of marine life died during the extinction? +Sentence: It is also known as the “Great Dying” because it is considered the largest mass extinction in the Earth’s history. +Label: not entailment + +Question: Who was the London Weekend Television’s Managing Director? +Sentence: The managing director of London Weekend Television (LWT), Greg Dyke, met with the representatives of the "big five" football clubs in England in 1990. +Label: entailment +``` + +#### Inference + +You can use the 🤗 Transformers library `text-classification` pipeline to infer with QNLI models. The model returns the label and the confidence. + +```python +from transformers import pipeline + +classifier = pipeline("text-classification", model = "cross-encoder/qnli-electra-base") +classifier("Where is the capital of France?, Paris is the capital of France.") +## [{'label': 'entailment', 'score': 0.997}] +``` + +### Sentiment Analysis + +In Sentiment Analysis, the classes can be polarities like positive, negative, neutral, or sentiments such as happiness or anger. + +#### Inference + +You can use the 🤗 Transformers library with the `sentiment-analysis` pipeline to infer with Sentiment Analysis models. The model returns the label with the score. + +```python +from transformers import pipeline + +classifier = pipeline("sentiment-analysis") +classifier("I loved Star Wars so much!") +## [{'label': 'POSITIVE', 'score': 0.99} +``` + +### Quora Question Pairs + +Quora Question Pairs models assess whether two provided questions are paraphrases of each other. The model takes two questions and returns a binary value, with 0 being mapped to “not paraphrase” and 1 to “paraphrase". The benchmark dataset is [Quora Question Pairs](https://huggingface.co/datasets/glue/viewer/qqp/test) inside the [GLUE benchmark](https://huggingface.co/datasets/glue). The dataset consists of question pairs and their labels. + +``` +Question1: “How can I increase the speed of my internet connection while using a VPN?” +Question2: How can Internet speed be increased by hacking through DNS? +Label: Not paraphrase + +Question1: “What can make Physics easy to learn?” +Question2: “How can you make physics easy to learn?” +Label: Paraphrase +``` + +#### Inference + +You can use the 🤗 Transformers library `text-classification` pipeline to infer with QQPI models. + +```python +from transformers import pipeline + +classifier = pipeline("text-classification", model = "textattack/bert-base-uncased-QQP") +classifier("Which city is the capital of France?, Where is the capital of France?") +## [{'label': 'paraphrase', 'score': 0.998}] +``` + +You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer text classification models on Hugging Face Hub. + +```javascript +import { InferenceClient } from "@huggingface/inference"; + +const inference = new InferenceClient(HF_TOKEN); +await inference.conversational({ + model: "distilbert-base-uncased-finetuned-sst-2-english", + inputs: "I love this movie!", +}); +``` + +### Grammatical Correctness + +Linguistic Acceptability is the task of assessing the grammatical acceptability of a sentence. The classes in this task are “acceptable” and “unacceptable”. The benchmark dataset used for this task is [Corpus of Linguistic Acceptability (CoLA)](https://huggingface.co/datasets/glue/viewer/cola/test). The dataset consists of texts and their labels. + +``` +Example: Books were sent to each other by the students. +Label: Unacceptable + +Example: She voted for herself. +Label: Acceptable. +``` + +#### Inference + +```python +from transformers import pipeline + +classifier = pipeline("text-classification", model = "textattack/distilbert-base-uncased-CoLA") +classifier("I will walk to home when I went through the bus.") +## [{'label': 'unacceptable', 'score': 0.95}] +``` + +## Useful Resources + +Would you like to learn more about the topic? Awesome! Here you can find some curated resources that you may find helpful! + +- [SetFitABSA: Few-Shot Aspect Based Sentiment Analysis using SetFit](https://huggingface.co/blog/setfit-absa) +- [Course Chapter on Fine-tuning a Text Classification Model](https://huggingface.co/course/chapter3/1?fw=pt) +- [Getting Started with Sentiment Analysis using Python](https://huggingface.co/blog/sentiment-analysis-python) +- [Sentiment Analysis on Encrypted Data with Homomorphic Encryption](https://huggingface.co/blog/sentiment-analysis-fhe) +- [Leveraging Hugging Face for complex text classification use cases](https://huggingface.co/blog/classification-use-cases) + +### Notebooks + +- [PyTorch](https://github.com/huggingface/notebooks/blob/master/examples/text_classification.ipynb) +- [TensorFlow](https://github.com/huggingface/notebooks/blob/master/examples/text_classification-tf.ipynb) +- [Flax](https://github.com/huggingface/notebooks/blob/master/examples/text_classification_flax.ipynb) + +### Scripts for training + +- [PyTorch](https://github.com/huggingface/transformers/tree/main/examples/pytorch/text-classification) +- [TensorFlow](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/text-classification) +- [Flax](https://github.com/huggingface/transformers/tree/main/examples/flax/text-classification) + +### Documentation + +- [Text classification task guide](https://huggingface.co/docs/transformers/tasks/sequence_classification) diff --git a/node_modules/@huggingface/tasks/src/tasks/text-classification/data.ts b/node_modules/@huggingface/tasks/src/tasks/text-classification/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..5ba1506e9bc6379d2226a64f5cdd24e2c629d881 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-classification/data.ts @@ -0,0 +1,103 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A widely used dataset used to benchmark multiple variants of text classification.", + id: "nyu-mll/glue", + }, + { + description: "A text classification dataset used to benchmark natural language inference models", + id: "stanfordnlp/snli", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "I love Hugging Face!", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "POSITIVE", + score: 0.9, + }, + { + label: "NEUTRAL", + score: 0.1, + }, + { + label: "NEGATIVE", + score: 0.0, + }, + ], + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: + "The F1 metric is the harmonic mean of the precision and recall. It can be calculated as: F1 = 2 * (precision * recall) / (precision + recall)", + id: "f1", + }, + ], + models: [ + { + description: "A robust model trained for sentiment analysis.", + id: "distilbert/distilbert-base-uncased-finetuned-sst-2-english", + }, + { + description: "A sentiment analysis model specialized in financial sentiment.", + id: "ProsusAI/finbert", + }, + { + description: "A sentiment analysis model specialized in analyzing tweets.", + id: "cardiffnlp/twitter-roberta-base-sentiment-latest", + }, + { + description: "A model that can classify languages.", + id: "papluca/xlm-roberta-base-language-detection", + }, + { + description: "A model that can classify text generation attacks.", + id: "meta-llama/Prompt-Guard-86M", + }, + ], + spaces: [ + { + description: "An application that can classify financial sentiment.", + id: "IoannisTr/Tech_Stocks_Trading_Assistant", + }, + { + description: "A dashboard that contains various text classification tasks.", + id: "miesnerjacob/Multi-task-NLP", + }, + { + description: "An application that analyzes user reviews in healthcare.", + id: "spacy/healthsea-demo", + }, + ], + summary: + "Text Classification is the task of assigning a label or class to a given text. Some use cases are sentiment analysis, natural language inference, and assessing grammatical correctness.", + widgetModels: ["distilbert/distilbert-base-uncased-finetuned-sst-2-english"], + youtubeId: "leNG9fN9FQU", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/text-classification/inference.ts b/node_modules/@huggingface/tasks/src/tasks/text-classification/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..84d6a80fe35f1257534d89aff2ed66101cdfc50a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-classification/inference.ts @@ -0,0 +1,52 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text Classification inference + */ +export interface TextClassificationInput { + /** + * The text to classify + */ + inputs: string; + /** + * Additional inference parameters for Text Classification + */ + parameters?: TextClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text Classification + */ +export interface TextClassificationParameters { + /** + * The function to apply to the model outputs in order to retrieve the scores. + */ + function_to_apply?: ClassificationOutputTransform; + /** + * When specified, limits the output to the top K most probable classes. + */ + top_k?: number; + [property: string]: unknown; +} +/** + * The function to apply to the model outputs in order to retrieve the scores. + */ +export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none"; +export type TextClassificationOutput = TextClassificationOutputElement[]; +/** + * Outputs of inference for the Text Classification task + */ +export interface TextClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-classification/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/text-classification/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..468f865dd598af5a8ee581f7a1b8245770d65662 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-classification/spec/input.json @@ -0,0 +1,35 @@ +{ + "$id": "/inference/schemas/text-classification/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Text Classification inference", + "title": "TextClassificationInput", + "type": "object", + "properties": { + "inputs": { + "description": "The text to classify", + "type": "string" + }, + "parameters": { + "description": "Additional inference parameters for Text Classification", + "$ref": "#/$defs/TextClassificationParameters" + } + }, + "$defs": { + "TextClassificationParameters": { + "title": "TextClassificationParameters", + "type": "object", + "properties": { + "function_to_apply": { + "title": "TextClassificationOutputTransform", + "$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutputTransform", + "description": "The function to apply to the model outputs in order to retrieve the scores." + }, + "top_k": { + "type": "integer", + "description": "When specified, limits the output to the top K most probable classes." + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-classification/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/text-classification/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..2bf3def357d3cfd65e9ca7f4f2fdc191ded633db --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-classification/spec/output.json @@ -0,0 +1,11 @@ +{ + "$id": "/inference/schemas/text-classification/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Text Classification task", + "title": "TextClassificationOutput", + "type": "array", + "items": { + "type": "object", + "$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput" + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-generation/about.md b/node_modules/@huggingface/tasks/src/tasks/text-generation/about.md new file mode 100644 index 0000000000000000000000000000000000000000..ee8cdeadcff207f6c1b7e5c1faa8d6cbf3ab4023 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-generation/about.md @@ -0,0 +1,154 @@ +This task covers guides on both [text-generation](https://huggingface.co/models?pipeline_tag=text-generation&sort=downloads) and [text-to-text generation](https://huggingface.co/models?other=text2text-generation&sort=downloads) models. Popular large language models that are used for chats or following instructions are also covered in this task. You can find the list of selected open-source large language models [here](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard), ranked by their performance scores. + +## Use Cases + +### Instruction Models + +A model trained for text generation can be later adapted to follow instructions. You can try some of the most powerful instruction-tuned open-access models like Mixtral 8x7B, Cohere Command R+, and Meta Llama3 70B [at Hugging Chat](https://huggingface.co/chat). + +### Code Generation + +A Text Generation model, also known as a causal language model, can be trained on code from scratch to help the programmers in their repetitive coding tasks. One of the most popular open-source models for code generation is StarCoder, which can generate code in 80+ languages. You can try it [here](https://huggingface.co/spaces/bigcode/bigcode-playground). + +### Stories Generation + +A story generation model can receive an input like "Once upon a time" and proceed to create a story-like text based on those first words. You can try [this application](https://huggingface.co/spaces/mosaicml/mpt-7b-storywriter) which contains a model trained on story generation, by MosaicML. + +If your generative model training data is different than your use case, you can train a causal language model from scratch. Learn how to do it in the free transformers [course](https://huggingface.co/course/chapter7/6?fw=pt)! + +## Task Variants + +### Completion Generation Models + +A popular variant of Text Generation models predicts the next word given a bunch of words. Word by word a longer text is formed that results in for example: + +- Given an incomplete sentence, complete it. +- Continue a story given the first sentences. +- Provided a code description, generate the code. + +The most popular models for this task are GPT-based models, [Mistral](mistralai/Mistral-7B-v0.1) or [Llama series](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf). These models are trained on data that has no labels, so you just need plain text to train your own model. You can train text generation models to generate a wide variety of documents, from code to stories. + +### Text-to-Text Generation Models + +These models are trained to learn the mapping between a pair of texts (e.g. translation from one language to another). The most popular variants of these models are [NLLB](facebook/nllb-200-distilled-600M), [FLAN-T5](https://huggingface.co/google/flan-t5-xxl), and [BART](https://huggingface.co/docs/transformers/model_doc/bart). Text-to-Text models are trained with multi-tasking capabilities, they can accomplish a wide range of tasks, including summarization, translation, and text classification. + +## Language Model Variants + +When it comes to text generation, the underlying language model can come in several types: + +- **Base models:** refers to plain language models like [Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.3) and [Meta Llama-3-70b](https://huggingface.co/meta-llama/Meta-Llama-3-70B). These models are good for fine-tuning and few-shot prompting. + +- **Instruction-trained models:** these models are trained in a multi-task manner to follow a broad range of instructions like "Write me a recipe for chocolate cake". Models like [Qwen 2 7B](https://huggingface.co/Qwen/Qwen2-7B-Instruct), [Yi 1.5 34B Chat](https://huggingface.co/01-ai/Yi-1.5-34B-Chat), and [Meta Llama 70B Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct) are examples of instruction-trained models. In general, instruction-trained models will produce better responses to instructions than base models. + +- **Human feedback models:** these models extend base and instruction-trained models by incorporating human feedback that rates the quality of the generated text according to criteria like [helpfulness, honesty, and harmlessness](https://arxiv.org/abs/2112.00861). The human feedback is then combined with an optimization technique like reinforcement learning to align the original model to be closer with human preferences. The overall methodology is often called [Reinforcement Learning from Human Feedback](https://huggingface.co/blog/rlhf), or RLHF for short. [Zephyr ORPO 141B A35B](https://huggingface.co/HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1) is an open-source model aligned through human feedback. + +## Text Generation from Image and Text + +There are language models that can input both text and image and output text, called vision language models. [IDEFICS 2](https://huggingface.co/HuggingFaceM4/idefics2-8b) and [MiniCPM Llama3 V](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5) are good examples. They accept the same generation parameters as other language models. However, since they also take images as input, you have to use them with the `image-to-text` pipeline. You can find more information about this in the [image-to-text task page](https://huggingface.co/tasks/image-to-text). + +## Inference + +You can use the 🤗 Transformers library `text-generation` pipeline to do inference with Text Generation models. It takes an incomplete text and returns multiple outputs with which the text can be completed. + +```python +from transformers import pipeline +generator = pipeline('text-generation', model = 'HuggingFaceH4/zephyr-7b-beta') +generator("Hello, I'm a language model", max_length = 30, num_return_sequences=3) +## [{'generated_text': "Hello, I'm a language modeler. So while writing this, when I went out to meet my wife or come home she told me that my"}, +## {'generated_text': "Hello, I'm a language modeler. I write and maintain software in Python. I love to code, and that includes coding things that require writing"}, ... +``` + +[Text-to-Text generation models](https://huggingface.co/models?other=text2text-generation&sort=downloads) have a separate pipeline called `text2text-generation`. This pipeline takes an input containing the sentence including the task and returns the output of the accomplished task. + +```python +from transformers import pipeline + +text2text_generator = pipeline("text2text-generation") +text2text_generator("question: What is 42 ? context: 42 is the answer to life, the universe and everything") +[{'generated_text': 'the answer to life, the universe and everything'}] + +text2text_generator("translate from English to French: I'm very happy") +[{'generated_text': 'Je suis très heureux'}] +``` + +You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer text classification models on Hugging Face Hub. + +```javascript +import { InferenceClient } from "@huggingface/inference"; + +const inference = new InferenceClient(HF_TOKEN); +await inference.conversational({ + model: "distilbert-base-uncased-finetuned-sst-2-english", + inputs: "I love this movie!", +}); +``` + +## Text Generation Inference + +[Text Generation Inference (TGI)](https://github.com/huggingface/text-generation-inference) is an open-source toolkit for serving LLMs tackling challenges such as response time. TGI powers inference solutions like [Inference Endpoints](https://huggingface.co/inference-endpoints) and [Hugging Chat](https://huggingface.co/chat/), as well as multiple community projects. You can use it to deploy any supported open-source large language model of your choice. + +## ChatUI Spaces + +Hugging Face Spaces includes templates to easily deploy your own instance of a specific application. [ChatUI](https://github.com/huggingface/chat-ui) is an open-source interface that enables serving conversational interface for large language models and can be deployed with few clicks at Spaces. TGI powers these Spaces under the hood for faster inference. Thanks to the template, you can deploy your own instance based on a large language model with only a few clicks and customize it. Learn more about it [here](https://huggingface.co/docs/hub/spaces-sdks-docker-chatui) and create your large language model instance [here](https://huggingface.co/new-space?template=huggingchat/chat-ui-template). + +![ChatUI](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/os_llms/docker_chat.png) + +## Useful Resources + +Would you like to learn more about the topic? Awesome! Here you can find some curated resources that you may find helpful! + +### Tools within Hugging Face Ecosystem + +- You can use [PEFT](https://github.com/huggingface/peft) to adapt large language models in efficient way. +- [ChatUI](https://github.com/huggingface/chat-ui) is the open-source interface to conversate with Large Language Models. +- [text-generation-inference](https://github.com/huggingface/text-generation-inference) +- [HuggingChat](https://huggingface.co/chat/) is a chat interface powered by Hugging Face to chat with powerful models like Meta Llama 3 70B, Mixtral 8x7B, etc. + +### Documentation + +- [PEFT documentation](https://huggingface.co/docs/peft/index) +- [ChatUI Docker Spaces](https://huggingface.co/docs/hub/spaces-sdks-docker-chatui) +- [Causal language modeling task guide](https://huggingface.co/docs/transformers/tasks/language_modeling) +- [Text generation strategies](https://huggingface.co/docs/transformers/generation_strategies) +- [Course chapter on training a causal language model from scratch](https://huggingface.co/course/chapter7/6?fw=pt) + +### Model Inference & Deployment + +- [Optimizing your LLM in production](https://huggingface.co/blog/optimize-llm) +- [Open-Source Text Generation & LLM Ecosystem at Hugging Face](https://huggingface.co/blog/os-llms) +- [Introducing RWKV - An RNN with the advantages of a transformer](https://huggingface.co/blog/rwkv) +- [Llama 2 is at Hugging Face](https://huggingface.co/blog/llama2) +- [Guiding Text Generation with Constrained Beam Search in 🤗 Transformers](https://huggingface.co/blog/constrained-beam-search) +- [Code generation with Hugging Face](https://huggingface.co/spaces/codeparrot/code-generation-models) +- [Assisted Generation: a new direction toward low-latency text generation](https://huggingface.co/blog/assisted-generation) +- [How to generate text: using different decoding methods for language generation with Transformers](https://huggingface.co/blog/how-to-generate) +- [Faster Text Generation with TensorFlow and XLA](https://huggingface.co/blog/tf-xla-generate) + +### Model Fine-tuning/Training + +- [Non-engineers guide: Train a LLaMA 2 chatbot](https://huggingface.co/blog/Llama2-for-non-engineers) +- [Training CodeParrot 🦜 from Scratch](https://huggingface.co/blog/codeparrot) +- [Creating a Coding Assistant with StarCoder](https://huggingface.co/blog/starchat-alpha) + +### Advanced Concepts Explained Simply + +- [Mixture of Experts Explained](https://huggingface.co/blog/moe) + +### Advanced Fine-tuning/Training Recipes + +- [Fine-tuning Llama 2 70B using PyTorch FSDP](https://huggingface.co/blog/ram-efficient-pytorch-fsdp) +- [The N Implementation Details of RLHF with PPO](https://huggingface.co/blog/the_n_implementation_details_of_rlhf_with_ppo) +- [Preference Tuning LLMs with Direct Preference Optimization Methods](https://huggingface.co/blog/pref-tuning) +- [Fine-tune Llama 2 with DPO](https://huggingface.co/blog/dpo-trl) + +### Notebooks + +- [Training a CLM in Flax](https://github.com/huggingface/notebooks/blob/master/examples/causal_language_modeling_flax.ipynb) +- [Training a CLM in TensorFlow](https://github.com/huggingface/notebooks/blob/master/examples/language_modeling_from_scratch-tf.ipynb) +- [Training a CLM in PyTorch](https://github.com/huggingface/notebooks/blob/master/examples/language_modeling_from_scratch.ipynb) + +### Scripts for training + +- [Training a CLM in PyTorch](https://github.com/huggingface/transformers/tree/main/examples/pytorch/language-modeling) +- [Training a CLM in TensorFlow](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/language-modeling) +- [Text Generation in PyTorch](https://github.com/huggingface/transformers/tree/main/examples/pytorch/text-generation) diff --git a/node_modules/@huggingface/tasks/src/tasks/text-generation/data.ts b/node_modules/@huggingface/tasks/src/tasks/text-generation/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..a43fb437027db6b2b73e985342945ee17d10f058 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-generation/data.ts @@ -0,0 +1,130 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "Multilingual dataset used to evaluate text generation models.", + id: "CohereForAI/Global-MMLU", + }, + { + description: "High quality multilingual data used to train text-generation models.", + id: "HuggingFaceFW/fineweb-2", + }, + { + description: "Truly open-source, curated and cleaned dialogue dataset.", + id: "HuggingFaceH4/ultrachat_200k", + }, + { + description: "A reasoning dataset.", + id: "open-r1/OpenThoughts-114k-math", + }, + { + description: "A multilingual instruction dataset with preference ratings on responses.", + id: "allenai/tulu-3-sft-mixture", + }, + { + description: "A large synthetic dataset for alignment of text generation models.", + id: "HuggingFaceTB/smoltalk", + }, + { + description: "A dataset made for training text generation models solving math questions.", + id: "HuggingFaceTB/finemath", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "Once upon a time,", + type: "text", + }, + ], + outputs: [ + { + label: "Output", + content: + "Once upon a time, we knew that our ancestors were on the verge of extinction. The great explorers and poets of the Old World, from Alexander the Great to Chaucer, are dead and gone. A good many of our ancient explorers and poets have", + type: "text", + }, + ], + }, + metrics: [ + { + description: + "Cross Entropy is a metric that calculates the difference between two probability distributions. Each probability distribution is the distribution of predicted words", + id: "Cross Entropy", + }, + { + description: + "The Perplexity metric is the exponential of the cross-entropy loss. It evaluates the probabilities assigned to the next word by the model. Lower perplexity indicates better performance", + id: "Perplexity", + }, + ], + models: [ + { description: "A text-generation model trained to follow instructions.", id: "google/gemma-2-2b-it" }, + { + description: "Powerful text generation model for coding.", + id: "Qwen/Qwen3-Coder-480B-A35B-Instruct", + }, + { + description: "Great text generation model with top-notch tool calling capabilities.", + id: "openai/gpt-oss-120b", + }, + { + description: "Powerful text generation model.", + id: "zai-org/GLM-4.5", + }, + { + description: "A powerful small model with reasoning capabilities.", + id: "Qwen/Qwen3-4B-Thinking-2507", + }, + { + description: "Strong conversational model that supports very long instructions.", + id: "Qwen/Qwen2.5-7B-Instruct-1M", + }, + { + description: "Text generation model used to write code.", + id: "Qwen/Qwen2.5-Coder-32B-Instruct", + }, + { + description: "Powerful reasoning based open large language model.", + id: "deepseek-ai/DeepSeek-R1", + }, + ], + spaces: [ + { + description: "An application that writes and executes code from text instructions and supports many models.", + id: "akhaliq/anycoder", + }, + { + description: "An application that builds websites from natural language prompts.", + id: "enzostvs/deepsite", + }, + { + description: "A leaderboard for comparing chain-of-thought performance of models.", + id: "logikon/open_cot_leaderboard", + }, + { + description: "An text generation based application based on a very powerful LLaMA2 model.", + id: "ysharma/Explore_llamav2_with_TGI", + }, + { + description: "An text generation based application to converse with Zephyr model.", + id: "HuggingFaceH4/zephyr-chat", + }, + { + description: "A leaderboard that ranks text generation models based on blind votes from people.", + id: "lmsys/chatbot-arena-leaderboard", + }, + { + description: "An chatbot to converse with a very powerful text generation model.", + id: "mlabonne/phixtral-chat", + }, + ], + summary: + "Generating text is the task of generating new text given another text. These models can, for example, fill in incomplete text or paraphrase.", + widgetModels: ["mistralai/Mistral-Nemo-Instruct-2407"], + youtubeId: "e9gNEAlsOvU", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/text-generation/inference.ts b/node_modules/@huggingface/tasks/src/tasks/text-generation/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..f77c1a60505f1a1a961b36d9d9aaea2e39454b7c --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-generation/inference.ts @@ -0,0 +1,187 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Text Generation Input. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface TextGenerationInput { + inputs: string; + parameters?: TextGenerationInputGenerateParameters; + stream?: boolean; + [property: string]: unknown; +} +export interface TextGenerationInputGenerateParameters { + /** + * Lora adapter id + */ + adapter_id?: string; + /** + * Generate best_of sequences and return the one if the highest token logprobs. + */ + best_of?: number; + /** + * Whether to return decoder input token logprobs and ids. + */ + decoder_input_details?: boolean; + /** + * Whether to return generation details. + */ + details?: boolean; + /** + * Activate logits sampling. + */ + do_sample?: boolean; + /** + * The parameter for frequency penalty. 1.0 means no penalty + * Penalize new tokens based on their existing frequency in the text so far, + * decreasing the model's likelihood to repeat the same line verbatim. + */ + frequency_penalty?: number; + grammar?: TextGenerationInputGrammarType; + /** + * Maximum number of tokens to generate. + */ + max_new_tokens?: number; + /** + * The parameter for repetition penalty. 1.0 means no penalty. + * See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details. + */ + repetition_penalty?: number; + /** + * Whether to prepend the prompt to the generated text + */ + return_full_text?: boolean; + /** + * Random sampling seed. + */ + seed?: number; + /** + * Stop generating tokens if a member of `stop` is generated. + */ + stop?: string[]; + /** + * The value used to module the logits distribution. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-n-filtering. + */ + top_n_tokens?: number; + /** + * Top-p value for nucleus sampling. + */ + top_p?: number; + /** + * Truncate inputs tokens to the given size. + */ + truncate?: number; + /** + * Typical Decoding mass + * See [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) + * for more information. + */ + typical_p?: number; + /** + * Watermarking with [A Watermark for Large Language + * Models](https://arxiv.org/abs/2301.10226). + */ + watermark?: boolean; + [property: string]: unknown; +} +export interface TextGenerationInputGrammarType { + type: Type; + /** + * A string that represents a [JSON Schema](https://json-schema.org/). + * + * JSON Schema is a declarative language that allows to annotate JSON documents + * with types and descriptions. + */ + value: unknown; + [property: string]: unknown; +} +export type Type = "json" | "regex" | "json_schema"; +/** + * Text Generation Output. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface TextGenerationOutput { + details?: TextGenerationOutputDetails; + generated_text: string; + [property: string]: unknown; +} +export interface TextGenerationOutputDetails { + best_of_sequences?: TextGenerationOutputBestOfSequence[]; + finish_reason: TextGenerationOutputFinishReason; + generated_tokens: number; + prefill: TextGenerationOutputPrefillToken[]; + seed?: number; + tokens: TextGenerationOutputToken[]; + top_tokens?: Array; + [property: string]: unknown; +} +export interface TextGenerationOutputBestOfSequence { + finish_reason: TextGenerationOutputFinishReason; + generated_text: string; + generated_tokens: number; + prefill: TextGenerationOutputPrefillToken[]; + seed?: number; + tokens: TextGenerationOutputToken[]; + top_tokens?: Array; + [property: string]: unknown; +} +export type TextGenerationOutputFinishReason = "length" | "eos_token" | "stop_sequence"; +export interface TextGenerationOutputPrefillToken { + id: number; + logprob: number; + text: string; + [property: string]: unknown; +} +export interface TextGenerationOutputToken { + id: number; + logprob: number; + special: boolean; + text: string; + [property: string]: unknown; +} +/** + * Text Generation Stream Output. + * + * Auto-generated from TGI specs. + * For more details, check out + * https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts. + */ +export interface TextGenerationStreamOutput { + details?: TextGenerationStreamOutputStreamDetails; + generated_text?: string; + index: number; + token: TextGenerationStreamOutputToken; + top_tokens?: TextGenerationStreamOutputToken[]; + [property: string]: unknown; +} +export interface TextGenerationStreamOutputStreamDetails { + finish_reason: TextGenerationOutputFinishReason; + generated_tokens: number; + input_length: number; + seed?: number; + [property: string]: unknown; +} +export interface TextGenerationStreamOutputToken { + id: number; + logprob: number; + special: boolean; + text: string; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-generation/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/text-generation/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..e734be1d5074605ba8525b1216ac214a259ca7cf --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-generation/spec/input.json @@ -0,0 +1,247 @@ +{ + "$id": "/inference/schemas/text-generation/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Text Generation Input.\n\nAuto-generated from TGI specs.\nFor more details, check out https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts.", + "title": "TextGenerationInput", + "type": "object", + "required": ["inputs"], + "properties": { + "inputs": { + "type": "string", + "example": "My name is Olivier and I" + }, + "parameters": { + "$ref": "#/$defs/TextGenerationInputGenerateParameters" + }, + "stream": { + "type": "boolean", + "default": "false" + } + }, + "$defs": { + "TextGenerationInputGenerateParameters": { + "type": "object", + "properties": { + "adapter_id": { + "type": "string", + "description": "Lora adapter id", + "default": "null", + "example": "null", + "nullable": true + }, + "best_of": { + "type": "integer", + "description": "Generate best_of sequences and return the one if the highest token logprobs.", + "default": "null", + "example": 1, + "nullable": true, + "minimum": 0, + "exclusiveMinimum": 0 + }, + "decoder_input_details": { + "type": "boolean", + "description": "Whether to return decoder input token logprobs and ids.", + "default": "false" + }, + "details": { + "type": "boolean", + "description": "Whether to return generation details.", + "default": "true" + }, + "do_sample": { + "type": "boolean", + "description": "Activate logits sampling.", + "default": "false", + "example": true + }, + "frequency_penalty": { + "type": "number", + "format": "float", + "description": "The parameter for frequency penalty. 1.0 means no penalty\nPenalize new tokens based on their existing frequency in the text so far,\ndecreasing the model's likelihood to repeat the same line verbatim.", + "default": "null", + "example": 0.1, + "nullable": true, + "exclusiveMinimum": -2 + }, + "grammar": { + "allOf": [ + { + "$ref": "#/$defs/TextGenerationInputGrammarType" + } + ], + "default": "null", + "nullable": true + }, + "max_new_tokens": { + "type": "integer", + "format": "int32", + "description": "Maximum number of tokens to generate.", + "default": "1024", + "example": "20", + "nullable": true, + "minimum": 0 + }, + "repetition_penalty": { + "type": "number", + "format": "float", + "description": "The parameter for repetition penalty. 1.0 means no penalty.\nSee [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details.", + "default": "null", + "example": 1.03, + "nullable": true, + "exclusiveMinimum": 0 + }, + "return_full_text": { + "type": "boolean", + "description": "Whether to prepend the prompt to the generated text", + "default": "null", + "example": false, + "nullable": true + }, + "seed": { + "type": "integer", + "format": "int64", + "description": "Random sampling seed.", + "default": "null", + "example": "null", + "nullable": true, + "minimum": 0, + "exclusiveMinimum": 0 + }, + "stop": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Stop generating tokens if a member of `stop` is generated.", + "example": ["photographer"], + "maxItems": 4 + }, + "temperature": { + "type": "number", + "format": "float", + "description": "The value used to module the logits distribution.", + "default": "null", + "example": 0.5, + "nullable": true, + "exclusiveMinimum": 0 + }, + "top_k": { + "type": "integer", + "format": "int32", + "description": "The number of highest probability vocabulary tokens to keep for top-k-filtering.", + "default": "null", + "example": 10, + "nullable": true, + "exclusiveMinimum": 0 + }, + "top_n_tokens": { + "type": "integer", + "format": "int32", + "description": "The number of highest probability vocabulary tokens to keep for top-n-filtering.", + "default": "null", + "example": 5, + "nullable": true, + "minimum": 0, + "exclusiveMinimum": 0 + }, + "top_p": { + "type": "number", + "format": "float", + "description": "Top-p value for nucleus sampling.", + "default": "null", + "example": 0.95, + "nullable": true, + "maximum": 1, + "exclusiveMinimum": 0 + }, + "truncate": { + "type": "integer", + "description": "Truncate inputs tokens to the given size.", + "default": "null", + "example": "null", + "nullable": true, + "minimum": 0 + }, + "typical_p": { + "type": "number", + "format": "float", + "description": "Typical Decoding mass\nSee [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information.", + "default": "null", + "example": 0.95, + "nullable": true, + "maximum": 1, + "exclusiveMinimum": 0 + }, + "watermark": { + "type": "boolean", + "description": "Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226).", + "default": "false", + "example": true + } + }, + "title": "TextGenerationInputGenerateParameters" + }, + "TextGenerationInputGrammarType": { + "oneOf": [ + { + "type": "object", + "required": ["type", "value"], + "properties": { + "type": { + "type": "string", + "enum": ["json"] + }, + "value": { + "description": "A string that represents a [JSON Schema](https://json-schema.org/).\n\nJSON Schema is a declarative language that allows to annotate JSON documents\nwith types and descriptions." + } + } + }, + { + "type": "object", + "required": ["type", "value"], + "properties": { + "type": { + "type": "string", + "enum": ["regex"] + }, + "value": { + "type": "string" + } + } + }, + { + "type": "object", + "required": ["type", "value"], + "properties": { + "type": { + "type": "string", + "enum": ["json_schema"] + }, + "value": { + "$ref": "#/$defs/TextGenerationInputJsonSchemaConfig" + } + } + } + ], + "discriminator": { + "propertyName": "type" + }, + "title": "TextGenerationInputGrammarType" + }, + "TextGenerationInputJsonSchemaConfig": { + "type": "object", + "required": ["schema"], + "properties": { + "name": { + "type": "string", + "description": "Optional name identifier for the schema", + "nullable": true + }, + "schema": { + "description": "The actual JSON schema definition" + } + }, + "title": "TextGenerationInputJsonSchemaConfig" + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-generation/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/text-generation/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..cb6ef3f99694ae4ed5eb6a05b8f3ea18e1da8a60 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-generation/spec/output.json @@ -0,0 +1,179 @@ +{ + "$id": "/inference/schemas/text-generation/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Text Generation Output.\n\nAuto-generated from TGI specs.\nFor more details, check out https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts.", + "title": "TextGenerationOutput", + "type": "object", + "required": ["generated_text"], + "properties": { + "details": { + "allOf": [ + { + "$ref": "#/$defs/TextGenerationOutputDetails" + } + ], + "nullable": true + }, + "generated_text": { + "type": "string", + "example": "test" + } + }, + "$defs": { + "TextGenerationOutputDetails": { + "type": "object", + "required": ["finish_reason", "generated_tokens", "prefill", "tokens"], + "properties": { + "best_of_sequences": { + "type": "array", + "items": { + "$ref": "#/$defs/TextGenerationOutputBestOfSequence" + }, + "nullable": true + }, + "finish_reason": { + "$ref": "#/$defs/TextGenerationOutputFinishReason" + }, + "generated_tokens": { + "type": "integer", + "format": "int32", + "example": 1, + "minimum": 0 + }, + "prefill": { + "type": "array", + "items": { + "$ref": "#/$defs/TextGenerationOutputPrefillToken" + } + }, + "seed": { + "type": "integer", + "format": "int64", + "example": 42, + "nullable": true, + "minimum": 0 + }, + "tokens": { + "type": "array", + "items": { + "$ref": "#/$defs/TextGenerationOutputToken" + } + }, + "top_tokens": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/TextGenerationOutputToken" + } + } + } + }, + "title": "TextGenerationOutputDetails" + }, + "TextGenerationOutputBestOfSequence": { + "type": "object", + "required": ["generated_text", "finish_reason", "generated_tokens", "prefill", "tokens"], + "properties": { + "finish_reason": { + "$ref": "#/$defs/TextGenerationOutputFinishReason" + }, + "generated_text": { + "type": "string", + "example": "test" + }, + "generated_tokens": { + "type": "integer", + "format": "int32", + "example": 1, + "minimum": 0 + }, + "prefill": { + "type": "array", + "items": { + "$ref": "#/$defs/TextGenerationOutputPrefillToken" + } + }, + "seed": { + "type": "integer", + "format": "int64", + "example": 42, + "nullable": true, + "minimum": 0 + }, + "tokens": { + "type": "array", + "items": { + "$ref": "#/$defs/TextGenerationOutputToken" + } + }, + "top_tokens": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/TextGenerationOutputToken" + } + } + } + }, + "title": "TextGenerationOutputBestOfSequence" + }, + "TextGenerationOutputFinishReason": { + "type": "string", + "enum": ["length", "eos_token", "stop_sequence"], + "example": "Length", + "title": "TextGenerationOutputFinishReason" + }, + "TextGenerationOutputPrefillToken": { + "type": "object", + "required": ["id", "text", "logprob"], + "properties": { + "id": { + "type": "integer", + "format": "int32", + "example": 0, + "minimum": 0 + }, + "logprob": { + "type": "number", + "format": "float", + "example": -0.34, + "nullable": true + }, + "text": { + "type": "string", + "example": "test" + } + }, + "title": "TextGenerationOutputPrefillToken" + }, + "TextGenerationOutputToken": { + "type": "object", + "required": ["id", "text", "logprob", "special"], + "properties": { + "id": { + "type": "integer", + "format": "int32", + "example": 0, + "minimum": 0 + }, + "logprob": { + "type": "number", + "format": "float", + "example": -0.34, + "nullable": true + }, + "special": { + "type": "boolean", + "example": "false" + }, + "text": { + "type": "string", + "example": "test" + } + }, + "title": "TextGenerationOutputToken" + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-generation/spec/stream_output.json b/node_modules/@huggingface/tasks/src/tasks/text-generation/spec/stream_output.json new file mode 100644 index 0000000000000000000000000000000000000000..fc566c6fe003eb445269af3668a941b403eb3549 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-generation/spec/stream_output.json @@ -0,0 +1,103 @@ +{ + "$id": "/inference/schemas/text-generation/stream_output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Text Generation Stream Output.\n\nAuto-generated from TGI specs.\nFor more details, check out https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/scripts/inference-tgi-import.ts.", + "title": "TextGenerationStreamOutput", + "type": "object", + "required": ["index", "token"], + "properties": { + "details": { + "allOf": [ + { + "$ref": "#/$defs/TextGenerationStreamOutputStreamDetails" + } + ], + "default": "null", + "nullable": true + }, + "generated_text": { + "type": "string", + "default": "null", + "example": "test", + "nullable": true + }, + "index": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "token": { + "$ref": "#/$defs/TextGenerationStreamOutputToken" + }, + "top_tokens": { + "type": "array", + "items": { + "$ref": "#/$defs/TextGenerationStreamOutputToken" + } + } + }, + "$defs": { + "TextGenerationStreamOutputStreamDetails": { + "type": "object", + "required": ["finish_reason", "generated_tokens", "input_length"], + "properties": { + "finish_reason": { + "$ref": "#/$defs/TextGenerationStreamOutputFinishReason" + }, + "generated_tokens": { + "type": "integer", + "format": "int32", + "example": 1, + "minimum": 0 + }, + "input_length": { + "type": "integer", + "format": "int32", + "example": 1, + "minimum": 0 + }, + "seed": { + "type": "integer", + "format": "int64", + "example": 42, + "nullable": true, + "minimum": 0 + } + }, + "title": "TextGenerationStreamOutputStreamDetails" + }, + "TextGenerationStreamOutputFinishReason": { + "type": "string", + "enum": ["length", "eos_token", "stop_sequence"], + "example": "Length", + "title": "TextGenerationStreamOutputFinishReason" + }, + "TextGenerationStreamOutputToken": { + "type": "object", + "required": ["id", "text", "logprob", "special"], + "properties": { + "id": { + "type": "integer", + "format": "int32", + "example": 0, + "minimum": 0 + }, + "logprob": { + "type": "number", + "format": "float", + "example": -0.34, + "nullable": true + }, + "special": { + "type": "boolean", + "example": "false" + }, + "text": { + "type": "string", + "example": "test" + } + }, + "title": "TextGenerationStreamOutputToken" + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-ranking/about.md b/node_modules/@huggingface/tasks/src/tasks/text-ranking/about.md new file mode 100644 index 0000000000000000000000000000000000000000..cd5986c5de759601815701bdb2cfb507d62e40ae --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-ranking/about.md @@ -0,0 +1,74 @@ +## Use Cases 🔍 + +### Information Retrieval + +You can improve Information Retrieval search stacks by applying a Text Ranking model as a Reranker in the common "[Retrieve and Rerank pipeline](https://sbert.net/examples/applications/retrieve_rerank/README.html)". First, you can use a [Sentence Similarity](https://huggingface.co/tasks/sentence-similarity) or [Feature Extraction](https://huggingface.co/tasks/feature-extraction) model as a Retriever to find the (for example) 100 most relevant documents for a query. Afterwards, you can rerank each of these 100 documents with a Text Ranking model to select an updated top 10. Often times, this results in improved retrieval performance than only using a Retriever model. + +## The Sentence Transformers library + +The [Sentence Transformers](https://www.sbert.net/) library is very powerful for using and training both Sentence Transformer (a.k.a. embedding or retriever) models as well as Cross Encoder (a.k.a. reranker) models. + +You can find and use [Sentence Transformers](https://huggingface.co/models?library=sentence-transformers&sort=downloads) models from the Hub by directly using the library, playing with the widgets in the browser or using Inference Endpoints. + +## Task Variants + +### Passage Ranking + +Passage Ranking is the task of ranking documents based on their relevance to a given query. The task is evaluated on Normalized Discounted Cumulative Gain, Mean Reciprocal Rank, or Mean Average Precision. These models take one query and multiple documents and return ranked documents according to the relevancy to the query. 📄 + +You can use it via the [Sentence Transformers library](https://sbert.net/docs/cross_encoder/usage/usage.html) like so: + +```python +from sentence_transformers import CrossEncoder + +# 1. Load a pre-trained CrossEncoder model +model = CrossEncoder("cross-encoder/ms-marco-MiniLM-L6-v2") + +query = "How many people live in Berlin?" +passages = [ + "Berlin had a population of 3,520,031 registered inhabitants in an area of 891.82 square kilometers.", + "Berlin is well known for its museums.", + "In 2014, the city state Berlin had 37,368 live births (+6.6%), a record number since 1991.", + "The urban area of Berlin comprised about 4.1 million people in 2014, making it the seventh most populous urban area in the European Union.", + "The city of Paris had a population of 2,165,423 people within its administrative city limits as of January 1, 2019", + "An estimated 300,000-420,000 Muslims reside in Berlin, making up about 8-11 percent of the population.", + "Berlin is subdivided into 12 boroughs or districts (Bezirke).", + "In 2015, the total labour force in Berlin was 1.85 million.", + "In 2013 around 600,000 Berliners were registered in one of the more than 2,300 sport and fitness clubs.", + "Berlin has a yearly total of about 135 million day visitors, which puts it in third place among the most-visited city destinations in the European Union.", +] + +# 2a. Either: predict scores for all pairs of sentences involved in the query +scores = model.predict([(query, passage) for passage in passages]) +# => [ 8.607138 -4.320077 7.5978117 8.915804 -4.237982 8.2359 0.33119553 3.4510403 6.352979 5.416662 ] + +# 2b. Or rank a list of passages for a query +ranks = model.rank(query, passages, return_documents=True) + +# Print the reranked passages +print("Query:", query) +for rank in ranks: + print(f"- #{rank['corpus_id']} ({rank['score']:.2f}): {rank['text']}") +""" +Query: How many people live in Berlin? +- #3 (8.92): The urban area of Berlin comprised about 4.1 million people in 2014, making it the seventh most populous urban area in the European Union. +- #0 (8.61): Berlin had a population of 3,520,031 registered inhabitants in an area of 891.82 square kilometers. +- #5 (8.24): An estimated 300,000-420,000 Muslims reside in Berlin, making up about 8-11 percent of the population. +- #2 (7.60): In 2014, the city state Berlin had 37,368 live births (+6.6%), a record number since 1991. +- #8 (6.35): In 2013 around 600,000 Berliners were registered in one of the more than 2,300 sport and fitness clubs. +- #9 (5.42): Berlin has a yearly total of about 135 million day visitors, which puts it in third place among the most-visited city destinations in the European Union. +- #7 (3.45): In 2015, the total labour force in Berlin was 1.85 million. +- #6 (0.33): Berlin is subdivided into 12 boroughs or districts (Bezirke). +- #4 (-4.24): The city of Paris had a population of 2,165,423 people within its administrative city limits as of January 1, 2019 +- #1 (-4.32): Berlin is well known for its museums. +""" +``` + +Rerankers often outperform [Sentence Similarity](https://huggingface.co/tasks/sentence-similarity) or [Feature Extraction](https://huggingface.co/tasks/feature-extraction) models, but they're too slow to rank a query against all documents. This is why they're commonly used to perform a final reranking of the top documents from a retriever: you can get the efficiency of a retriever model with the performance of a reranker. + +## Useful Resources + +Would you like to learn more about Text Ranking? Here is a curated resource that you may find helpful! + +- [Sentence Transformers > Cross Encoder Documentation](https://www.sbert.net/docs/cross_encoder/usage/usage.html) +- [Sentence Transformers > Usage > Retrieve & Re-Rank](https://www.sbert.net/examples/applications/retrieve_rerank/README.html) diff --git a/node_modules/@huggingface/tasks/src/tasks/text-ranking/data.ts b/node_modules/@huggingface/tasks/src/tasks/text-ranking/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..265adc88fe6e78763410f79289440d872e036dfd --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-ranking/data.ts @@ -0,0 +1,91 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "Bing queries with relevant passages from various web sources.", + id: "microsoft/ms_marco", + }, + ], + demo: { + inputs: [ + { + label: "Source sentence", + content: "Machine learning is so easy.", + type: "text", + }, + { + label: "Sentences to compare to", + content: "Deep learning is so straightforward.", + type: "text", + }, + { + label: "", + content: "This is so difficult, like rocket science.", + type: "text", + }, + { + label: "", + content: "I can't believe how much I struggled with this.", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Deep learning is so straightforward.", + score: 2.2006407, + }, + { + label: "This is so difficult, like rocket science.", + score: -6.2634873, + }, + { + label: "I can't believe how much I struggled with this.", + score: -10.251488, + }, + ], + }, + ], + }, + metrics: [ + { + description: + "Discounted Cumulative Gain (DCG) measures the gain, or usefulness, of search results discounted by their position. The normalization is done by dividing the DCG by the ideal DCG, which is the DCG of the perfect ranking.", + id: "Normalized Discounted Cumulative Gain", + }, + { + description: + "Reciprocal Rank is a measure used to rank the relevancy of documents given a set of documents. Reciprocal Rank is the reciprocal of the rank of the document retrieved, meaning, if the rank is 3, the Reciprocal Rank is 0.33. If the rank is 1, the Reciprocal Rank is 1", + id: "Mean Reciprocal Rank", + }, + { + description: + "Mean Average Precision (mAP) is the overall average of the Average Precision (AP) values, where AP is the Area Under the PR Curve (AUC-PR)", + id: "Mean Average Precision", + }, + ], + models: [ + { + description: "An extremely efficient text ranking model trained on a web search dataset.", + id: "cross-encoder/ms-marco-MiniLM-L6-v2", + }, + { + description: "A strong multilingual text reranker model.", + id: "Alibaba-NLP/gte-multilingual-reranker-base", + }, + { + description: "An efficient text ranking model that punches above its weight.", + id: "Alibaba-NLP/gte-reranker-modernbert-base", + }, + ], + spaces: [], + summary: + "Text Ranking is the task of ranking a set of texts based on their relevance to a query. Text ranking models are trained on large datasets of queries and relevant documents to learn how to rank documents based on their relevance to the query. This task is particularly useful for search engines and information retrieval systems.", + widgetModels: ["cross-encoder/ms-marco-MiniLM-L6-v2"], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-3d/about.md b/node_modules/@huggingface/tasks/src/tasks/text-to-3d/about.md new file mode 100644 index 0000000000000000000000000000000000000000..9f76ba3e4ddee504215bf676dc9bc28926c98fef --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-3d/about.md @@ -0,0 +1,62 @@ +## Use Cases + +Text-to-3D models can be used in a wide variety of applications that require 3D, such as games, animation, design, architecture, engineering, marketing, and more. + +![Text-to-3D Thumbnail](https://huggingface.co/datasets/huggingfacejs/tasks/resolve/main/text-to-3d/text-to-3d-thumbnail.png) + +This task is similar to the [image-to-3d](https://huggingface.co/tasks/image-to-3d) task, but takes text input instead of image input. In practice, this is often equivalent to a combination of [text-to-image](https://huggingface.co/tasks/text-to-image) and [image-to-3d](https://huggingface.co/tasks/image-to-3d). That is, the text is first converted to an image, then the image is converted to 3D. + +### Generating Meshes + +Meshes are the standard representation of 3D in industry. + +### Generating Gaussian Splats + +[Gaussian Splatting](https://huggingface.co/blog/gaussian-splatting) is a rendering technique that represents scenes as fuzzy points. + +### Inference + +Inference for this task typically leverages the [Diffusers](https://huggingface.co/docs/diffusers/index) library for inference, using [Custom Pipelines](https://huggingface.co/docs/diffusers/v0.6.0/en/using-diffusers/custom_pipelines). + +These are unstandardized and depend on the model. More details can be found in each model repository. + +```python +import torch +import requests +import numpy as np +from io import BytesIO +from diffusers import DiffusionPipeline +from PIL import Image + +pipeline = DiffusionPipeline.from_pretrained( + "dylanebert/LGM-full", + custom_pipeline="dylanebert/LGM-full", + torch_dtype=torch.float16, + trust_remote_code=True, +).to("cuda") + +input_prompt = "a cat statue" +result = pipeline(input_prompt, None) +result_path = "/tmp/output.ply" +pipeline.save_ply(result, result_path) +``` + +In the code above, we: + +1. Import the necessary libraries +2. Load the `LGM-full` model and custom pipeline +3. Define the input prompt +4. Run the pipeline on the input prompt +5. Save the output to a file + +### Output Formats + +Meshes can be in `.obj`, `.glb`, `.stl`, or `.gltf` format. Other formats are allowed, but won't be rendered in the gradio [Model3D](https://www.gradio.app/docs/gradio/model3d) component. + +Splats can be in `.ply` or `.splat` format. They can be rendered in the gradio [Model3D](https://www.gradio.app/docs/gradio/model3d) component using the [gsplat.js](https://github.com/huggingface/gsplat.js) library. + +## Useful Resources + +- [ML for 3D Course](https://huggingface.co/learn/ml-for-3d-course) +- [3D Arena Leaderboard](https://huggingface.co/spaces/dylanebert/3d-arena) +- [gsplat.js](https://github.com/huggingface/gsplat.js) diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-3d/data.ts b/node_modules/@huggingface/tasks/src/tasks/text-to-3d/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..30f3aefc49bc00b9ec37b6ee18bc520d82b88b20 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-3d/data.ts @@ -0,0 +1,56 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A large dataset of over 10 million 3D objects.", + id: "allenai/objaverse-xl", + }, + { + description: "Descriptive captions for 3D objects in Objaverse.", + id: "tiange/Cap3D", + }, + ], + demo: { + inputs: [ + { + label: "Prompt", + content: "a cat statue", + type: "text", + }, + ], + outputs: [ + { + label: "Result", + content: "text-to-3d-3d-output-filename.glb", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "Text-to-3D mesh model by OpenAI", + id: "openai/shap-e", + }, + { + description: "Generative 3D gaussian splatting model.", + id: "ashawkey/LGM", + }, + ], + spaces: [ + { + description: "Text-to-3D demo with mesh outputs.", + id: "hysts/Shap-E", + }, + { + description: "Text/image-to-3D demo with splat outputs.", + id: "ashawkey/LGM", + }, + ], + summary: "Text-to-3D models take in text input and produce 3D output.", + widgetModels: [], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-audio/inference.ts b/node_modules/@huggingface/tasks/src/tasks/text-to-audio/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..4a4101944814a62d082edbd1e1aaa14cac389e09 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-audio/inference.ts @@ -0,0 +1,133 @@ +/** + * Outputs of inference for the Text To Audio task + */ +export interface TextToAudioOutput { + /** + * The generated audio waveform. + */ + audio: Blob; + /** + * The sampling rate of the generated audio waveform. + */ + sampling_rate: number; + [property: string]: unknown; +} +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text To Audio inference + */ +export interface TextToAudioInput { + /** + * The input text data + */ + inputs: string; + /** + * Additional inference parameters for Text To Audio + */ + parameters?: TextToAudioParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text To Audio + */ +export interface TextToAudioParameters { + /** + * Parametrization of the text generation process + */ + generation_parameters?: GenerationParameters; + [property: string]: unknown; +} +/** + * Parametrization of the text generation process + */ +export interface GenerationParameters { + /** + * Whether to use sampling instead of greedy decoding when generating new tokens. + */ + do_sample?: boolean; + /** + * Controls the stopping condition for beam-based methods. + */ + early_stopping?: EarlyStoppingUnion; + /** + * If set to float strictly between 0 and 1, only tokens with a conditional probability + * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + * Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + */ + epsilon_cutoff?: number; + /** + * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + * float strictly between 0 and 1, a token is only considered if it is greater than either + * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + * for more details. + */ + eta_cutoff?: number; + /** + * The maximum length (in tokens) of the generated text, including the input. + */ + max_length?: number; + /** + * The maximum number of tokens to generate. Takes precedence over max_length. + */ + max_new_tokens?: number; + /** + * The minimum length (in tokens) of the generated text, including the input. + */ + min_length?: number; + /** + * The minimum number of tokens to generate. Takes precedence over min_length. + */ + min_new_tokens?: number; + /** + * Number of groups to divide num_beams into in order to ensure diversity among different + * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + */ + num_beam_groups?: number; + /** + * Number of beams to use for beam search. + */ + num_beams?: number; + /** + * The value balances the model confidence and the degeneration penalty in contrastive + * search decoding. + */ + penalty_alpha?: number; + /** + * The value used to modulate the next token probabilities. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * If set to float < 1, only the smallest set of most probable tokens with probabilities + * that add up to top_p or higher are kept for generation. + */ + top_p?: number; + /** + * Local typicality measures how similar the conditional probability of predicting a target + * token next is to the expected conditional probability of predicting a random token next, + * given the partial text already generated. If set to float < 1, the smallest set of the + * most locally typical tokens with probabilities that add up to typical_p or higher are + * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + */ + typical_p?: number; + /** + * Whether the model should use the past last key/values attentions to speed up decoding + */ + use_cache?: boolean; + [property: string]: unknown; +} +/** + * Controls the stopping condition for beam-based methods. + */ +export type EarlyStoppingUnion = boolean | "never"; diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-audio/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/text-to-audio/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..a836c53d56e71e2117ed9e6909744461e23cfe17 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-audio/spec/input.json @@ -0,0 +1,30 @@ +{ + "$id": "/inference/schemas/text-to-audio/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Text To Audio inference", + "title": "TextToAudioInput", + "type": "object", + "properties": { + "inputs": { + "description": "The input text data", + "type": "string" + }, + "parameters": { + "description": "Additional inference parameters for Text To Audio", + "$ref": "#/$defs/TextToAudioParameters" + } + }, + "$defs": { + "TextToAudioParameters": { + "title": "TextToAudioParameters", + "type": "object", + "properties": { + "generation_parameters": { + "description": "Parametrization of the text generation process", + "$ref": "/inference/schemas/common-definitions.json#/definitions/GenerationParameters" + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-audio/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/text-to-audio/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..8d39db79c1b8bdf1ee4ab42c2a977fd9bde03eb7 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-audio/spec/output.json @@ -0,0 +1,18 @@ +{ + "$id": "/inference/schemas/text-to-audio/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Text To Audio task", + "title": "TextToAudioOutput", + "type": "object", + "properties": { + "audio": { + "description": "The generated audio waveform.", + "comment": "type=binary" + }, + "sampling_rate": { + "type": "number", + "description": "The sampling rate of the generated audio waveform." + } + }, + "required": ["audio", "sampling_rate"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-image/about.md b/node_modules/@huggingface/tasks/src/tasks/text-to-image/about.md new file mode 100644 index 0000000000000000000000000000000000000000..9cb11798ece00cde3707664cfcc27d8a9c04c530 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-image/about.md @@ -0,0 +1,96 @@ +## Use Cases + +### Data Generation + +Businesses can generate data for their use cases by inputting text and getting image outputs. + +### Immersive Conversational Chatbots + +Chatbots can be made more immersive if they provide contextual images based on the input provided by the user. + +### Creative Ideas for Fashion Industry + +Different patterns can be generated to obtain unique pieces of fashion. Text-to-image models make creations easier for designers to conceptualize their design before actually implementing it. + +### Architecture Industry + +Architects can utilise the models to construct an environment based out on the requirements of the floor plan. This can also include the furniture that has to be placed in that environment. + +## Task Variants + +### Image Editing + +Image editing with text-to-image models involves modifying an image following edit instructions provided in a text prompt. + +- **Synthetic image editing**: Adjusting images that were initially created using an input prompt while preserving the overall meaning or context of the original image. + + ![Examples](https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/edit_p2p.png) + _Figure taken from ["InstructPix2Pix: Learning to Follow Image Editing Instructions"](https://www.timothybrooks.com/instruct-pix2pix)_ + +- **Real image editing**: Similar to synthetic image editing, except we're using real photos/images. This task is usually more complex. + + ![Examples](https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/pix2pix.jpeg) + _Figure taken from ["Prompt-to-Prompt Image Editing with Cross-Attention Control"](https://prompt-to-prompt.github.io)_ + +### Personalization + +Personalization refers to techniques used to customize text-to-image models. We introduce new subjects or concepts to the model, which the model can then generate when we refer to them with a text prompt. + +For example, you can use these techniques to generate images of your dog in imaginary settings, after you have taught the model using a few reference images of the subject (or just one in some cases). Teaching the model a new concept can be achieved through fine-tuning, or by using training-free techniques. + +## Inference + +You can use diffusers pipelines to infer with `text-to-image` models. + +```python +from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler + +model_id = "stabilityai/stable-diffusion-2" +scheduler = EulerDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler") +pipe = StableDiffusionPipeline.from_pretrained(model_id, scheduler=scheduler, torch_dtype=torch.float16) +pipe = pipe.to("cuda") + +prompt = "a photo of an astronaut riding a horse on mars" +image = pipe(prompt).images[0] +``` + +You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer text-to-image models on Hugging Face Hub. + +```javascript +import { InferenceClient } from "@huggingface/inference"; + +const inference = new InferenceClient(HF_TOKEN); +await inference.textToImage({ + model: "stabilityai/stable-diffusion-2", + inputs: "award winning high resolution photo of a giant tortoise/((ladybird)) hybrid, [trending on artstation]", + parameters: { + negative_prompt: "blurry", + }, +}); +``` + +## Useful Resources + +### Model Inference + +- [Hugging Face Diffusion Models Course](https://github.com/huggingface/diffusion-models-class) +- [Getting Started with Diffusers](https://huggingface.co/docs/diffusers/index) +- [Text-to-Image Generation](https://huggingface.co/docs/diffusers/using-diffusers/conditional_image_generation) +- [Using Stable Diffusion with Core ML on Apple Silicon](https://huggingface.co/blog/diffusers-coreml) +- [A guide on Vector Quantized Diffusion](https://huggingface.co/blog/vq-diffusion) +- [🧨 Stable Diffusion in JAX/Flax](https://huggingface.co/blog/stable_diffusion_jax) +- [Running IF with 🧨 diffusers on a Free Tier Google Colab](https://huggingface.co/blog/if) +- [Introducing Würstchen: Fast Diffusion for Image Generation](https://huggingface.co/blog/wuerstchen) +- [Efficient Controllable Generation for SDXL with T2I-Adapters](https://huggingface.co/blog/t2i-sdxl-adapters) +- [Welcome aMUSEd: Efficient Text-to-Image Generation](https://huggingface.co/blog/amused) +- Image Editing Demos: [LEDITS++](https://huggingface.co/spaces/editing-images/leditsplusplus), [Turbo Edit](https://huggingface.co/spaces/turboedit/turbo_edit), [InstructPix2Pix](https://huggingface.co/spaces/timbrooks/instruct-pix2pix), [CosXL](https://huggingface.co/spaces/multimodalart/cosxl) +- Training free Personalization Demos: [Face-to-All](https://huggingface.co/spaces/multimodalart/face-to-all), [InstantStyle](https://huggingface.co/spaces/InstantX/InstantStyle), [RB-modulation](https://huggingface.co/spaces/fffiloni/RB-Modulation), [Photomaker v2](https://huggingface.co/spaces/TencentARC/PhotoMaker-V2) + +### Model Fine-tuning + +- [Finetune Stable Diffusion Models with DDPO via TRL](https://huggingface.co/blog/pref-tuning) +- [LoRA training scripts of the world, unite!](https://huggingface.co/blog/sdxl_lora_advanced_script) +- [Using LoRA for Efficient Stable Diffusion Fine-Tuning](https://huggingface.co/blog/lora) +- LoRA fine tuning Spaces: [FLUX.1 finetuning](https://huggingface.co/spaces/autotrain-projects/train-flux-lora-ease), [SDXL finetuning](https://huggingface.co/spaces/multimodalart/lora-ease) + +This page was made possible thanks to the efforts of [Ishan Dutta](https://huggingface.co/ishandutta), [Enrique Elias Ubaldo](https://huggingface.co/herrius) and [Oğuz Akif](https://huggingface.co/oguzakif). diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-image/data.ts b/node_modules/@huggingface/tasks/src/tasks/text-to-image/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..2a1fcc80992c86ff61a9f7736ec0155554ab8ebf --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-image/data.ts @@ -0,0 +1,104 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "RedCaps is a large-scale dataset of 12M image-text pairs collected from Reddit.", + id: "red_caps", + }, + { + description: "Conceptual Captions is a dataset consisting of ~3.3M images annotated with captions.", + id: "conceptual_captions", + }, + { + description: "12M image-caption pairs.", + id: "Spawning/PD12M", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "A city above clouds, pastel colors, Victorian style", + type: "text", + }, + ], + outputs: [ + { + filename: "image.jpeg", + type: "img", + }, + ], + }, + metrics: [ + { + description: + "The Inception Score (IS) measure assesses diversity and meaningfulness. It uses a generated image sample to predict its label. A higher score signifies more diverse and meaningful images.", + id: "IS", + }, + { + description: + "The Fréchet Inception Distance (FID) calculates the distance between distributions between synthetic and real samples. A lower FID score indicates better similarity between the distributions of real and generated images.", + id: "FID", + }, + { + description: + "R-precision assesses how the generated image aligns with the provided text description. It uses the generated images as queries to retrieve relevant text descriptions. The top 'r' relevant descriptions are selected and used to calculate R-precision as r/R, where 'R' is the number of ground truth descriptions associated with the generated images. A higher R-precision value indicates a better model.", + id: "R-Precision", + }, + ], + models: [ + { + description: "One of the most powerful image generation models that can generate realistic outputs.", + id: "black-forest-labs/FLUX.1-Krea-dev", + }, + { + description: "A powerful image generation model.", + id: "Qwen/Qwen-Image", + }, + { + description: "Powerful and fast image generation model.", + id: "ByteDance/SDXL-Lightning", + }, + { + description: "A powerful text-to-image model.", + id: "ByteDance/Hyper-SD", + }, + ], + spaces: [ + { + description: "A powerful text-to-image application.", + id: "stabilityai/stable-diffusion-3-medium", + }, + { + description: "A text-to-image application to generate comics.", + id: "jbilcke-hf/ai-comic-factory", + }, + { + description: "An application to match multiple custom image generation models.", + id: "multimodalart/flux-lora-lab", + }, + { + description: "A powerful yet very fast image generation application.", + id: "latent-consistency/lcm-lora-for-sdxl", + }, + { + description: "A gallery to explore various text-to-image models.", + id: "multimodalart/LoraTheExplorer", + }, + { + description: "An application for `text-to-image`, `image-to-image` and image inpainting.", + id: "ArtGAN/Stable-Diffusion-ControlNet-WebUI", + }, + { + description: "An application to generate realistic images given photos of a person and a prompt.", + id: "InstantX/InstantID", + }, + ], + summary: + "Text-to-image is the task of generating images from input text. These pipelines can also be used to modify and edit images based on text prompts.", + widgetModels: ["black-forest-labs/FLUX.1-dev"], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-image/inference.ts b/node_modules/@huggingface/tasks/src/tasks/text-to-image/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..5bdc8a34036010c2d1764570cd2b81e9999205c6 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-image/inference.ts @@ -0,0 +1,65 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text To Image inference + */ +export interface TextToImageInput { + /** + * The input text data (sometimes called "prompt") + */ + inputs: string; + /** + * Additional inference parameters for Text To Image + */ + parameters?: TextToImageParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text To Image + */ +export interface TextToImageParameters { + /** + * A higher guidance scale value encourages the model to generate images closely linked to + * the text prompt, but values too high may cause saturation and other artifacts. + */ + guidance_scale?: number; + /** + * The height in pixels of the output image + */ + height?: number; + /** + * One prompt to guide what NOT to include in image generation. + */ + negative_prompt?: string; + /** + * The number of denoising steps. More denoising steps usually lead to a higher quality + * image at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * Override the scheduler with a compatible one. + */ + scheduler?: string; + /** + * Seed for the random number generator. + */ + seed?: number; + /** + * The width in pixels of the output image + */ + width?: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Text To Image task + */ +export interface TextToImageOutput { + /** + * The generated image returned as raw bytes in the payload. + */ + image: unknown; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-image/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/text-to-image/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..f94e0bbc3ea7fa723bfaab85be1c4d65385e2c7b --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-image/spec/input.json @@ -0,0 +1,54 @@ +{ + "$id": "/inference/schemas/text-to-image/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Text To Image inference", + "title": "TextToImageInput", + "type": "object", + "properties": { + "inputs": { + "description": "The input text data (sometimes called \"prompt\")", + "type": "string" + }, + "parameters": { + "description": "Additional inference parameters for Text To Image", + "$ref": "#/$defs/TextToImageParameters" + } + }, + "$defs": { + "TextToImageParameters": { + "title": "TextToImageParameters", + "type": "object", + "properties": { + "guidance_scale": { + "type": "number", + "description": "A higher guidance scale value encourages the model to generate images closely linked to the text prompt, but values too high may cause saturation and other artifacts." + }, + "negative_prompt": { + "type": "string", + "description": "One prompt to guide what NOT to include in image generation." + }, + "num_inference_steps": { + "type": "integer", + "description": "The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference." + }, + "width": { + "type": "integer", + "description": "The width in pixels of the output image" + }, + "height": { + "type": "integer", + "description": "The height in pixels of the output image" + }, + "scheduler": { + "type": "string", + "description": "Override the scheduler with a compatible one." + }, + "seed": { + "type": "integer", + "description": "Seed for the random number generator." + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-image/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/text-to-image/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..f90a1eee18655cc5febb99cba38a3d784994261d --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-image/spec/output.json @@ -0,0 +1,13 @@ +{ + "$id": "/inference/schemas/text-to-image/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Text To Image task", + "title": "TextToImageOutput", + "type": "object", + "properties": { + "image": { + "description": "The generated image returned as raw bytes in the payload." + } + }, + "required": ["image"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-speech/about.md b/node_modules/@huggingface/tasks/src/tasks/text-to-speech/about.md new file mode 100644 index 0000000000000000000000000000000000000000..c241d63df680c9c8e8cf9ae12763f611a58e3789 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-speech/about.md @@ -0,0 +1,63 @@ +## Use Cases + +Text-to-Speech (TTS) models can be used in any speech-enabled application that requires converting text to speech imitating human voice. + +### Voice Assistants + +TTS models are used to create voice assistants on smart devices. These models are a better alternative compared to concatenative methods where the assistant is built by recording sounds and mapping them, since the outputs in TTS models contain elements in natural speech such as emphasis. + +### Announcement Systems + +TTS models are widely used in airport and public transportation announcement systems to convert the announcement of a given text into speech. + +## Inference Endpoints + +The Hub contains over [1500 TTS models](https://huggingface.co/models?pipeline_tag=text-to-speech&sort=downloads) that you can use right away by trying out the widgets directly in the browser or calling the models as a service using Inference Endpoints. Here is a simple code snippet to get you started: + +```python +import json +import requests + +headers = {"Authorization": f"Bearer {API_TOKEN}"} +API_URL = "https://router.huggingface.co/hf-inference/models/microsoft/speecht5_tts" + +def query(payload): + response = requests.post(API_URL, headers=headers, json=payload) + return response + +output = query({"text_inputs": "Max is the best doggo."}) +``` + +You can also use libraries such as [espnet](https://huggingface.co/models?library=espnet&pipeline_tag=text-to-speech&sort=downloads) or [transformers](https://huggingface.co/models?pipeline_tag=text-to-speech&library=transformers&sort=trending) if you want to handle the Inference directly. + +## Direct Inference + +Now, you can also use the Text-to-Speech pipeline in Transformers to synthesise high quality voice. + +```python +from transformers import pipeline + +synthesizer = pipeline("text-to-speech", "suno/bark") + +synthesizer("Look I am generating speech in three lines of code!") +``` + +You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer summarization models on Hugging Face Hub. + +```javascript +import { InferenceClient } from "@huggingface/inference"; + +const inference = new InferenceClient(HF_TOKEN); +await inference.textToSpeech({ + model: "facebook/mms-tts", + inputs: "text to generate speech from", +}); +``` + +## Useful Resources + +- [Hugging Face Audio Course](https://huggingface.co/learn/audio-course/chapter6/introduction) +- [ML for Audio Study Group - Text to Speech Deep Dive](https://www.youtube.com/watch?v=aLBedWj-5CQ) +- [Speech Synthesis, Recognition, and More With SpeechT5](https://huggingface.co/blog/speecht5) +- [Optimizing a Text-To-Speech model using 🤗 Transformers](https://huggingface.co/blog/optimizing-bark) +- [Train your own TTS models with Parler-TTS](https://github.com/huggingface/parler-tts) diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-speech/data.ts b/node_modules/@huggingface/tasks/src/tasks/text-to-speech/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..d85cd1e24adf3484c43273850a9ca2431b755032 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-speech/data.ts @@ -0,0 +1,86 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + canonicalId: "text-to-audio", + datasets: [ + { + description: "10K hours of multi-speaker English dataset.", + id: "parler-tts/mls_eng_10k", + }, + { + description: "Multi-speaker English dataset.", + id: "mythicinfinity/libritts_r", + }, + { + description: "Multi-lingual dataset.", + id: "facebook/multilingual_librispeech", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "I love audio models on the Hub!", + type: "text", + }, + ], + outputs: [ + { + filename: "audio.wav", + type: "audio", + }, + ], + }, + metrics: [ + { + description: "The Mel Cepstral Distortion (MCD) metric is used to calculate the quality of generated speech.", + id: "mel cepstral distortion", + }, + ], + models: [ + { + description: "Small yet powerful TTS model.", + id: "KittenML/kitten-tts-nano-0.1", + }, + { + description: "Bleeding edge TTS model.", + id: "ResembleAI/chatterbox", + }, + { + description: "A massively multi-lingual TTS model.", + id: "fishaudio/fish-speech-1.5", + }, + { + description: "A text-to-dialogue model.", + id: "nari-labs/Dia-1.6B-0626", + }, + ], + spaces: [ + { + description: "An application for generate high quality speech in different languages.", + id: "hexgrad/Kokoro-TTS", + }, + { + description: "A multilingual text-to-speech application.", + id: "fishaudio/fish-speech-1", + }, + { + description: "Performant TTS application.", + id: "ResembleAI/Chatterbox", + }, + { + description: "An application to compare different TTS models.", + id: "TTS-AGI/TTS-Arena-V2", + }, + { + description: "An application that generates podcast episodes.", + id: "ngxson/kokoro-podcast-generator", + }, + ], + summary: + "Text-to-Speech (TTS) is the task of generating natural sounding speech given text input. TTS models can be extended to have a single model that generates speech for multiple speakers and multiple languages.", + widgetModels: ["suno/bark"], + youtubeId: "NW62DpzJ274", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-speech/inference.ts b/node_modules/@huggingface/tasks/src/tasks/text-to-speech/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..4515de301a8b4dc044b0d7a7f45f3729e4f4662f --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-speech/inference.ts @@ -0,0 +1,133 @@ +/** + * Outputs of inference for the Text To Speech task + */ +export interface TextToSpeechOutput { + /** + * The generated audio + */ + audio: Blob; + /** + * The sampling rate of the generated audio waveform. + */ + sampling_rate?: number; + [property: string]: unknown; +} +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text To Speech inference + */ +export interface TextToSpeechInput { + /** + * The input text data + */ + inputs: string; + /** + * Additional inference parameters for Text To Speech + */ + parameters?: TextToSpeechParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text To Speech + */ +export interface TextToSpeechParameters { + /** + * Parametrization of the text generation process + */ + generation_parameters?: GenerationParameters; + [property: string]: unknown; +} +/** + * Parametrization of the text generation process + */ +export interface GenerationParameters { + /** + * Whether to use sampling instead of greedy decoding when generating new tokens. + */ + do_sample?: boolean; + /** + * Controls the stopping condition for beam-based methods. + */ + early_stopping?: EarlyStoppingUnion; + /** + * If set to float strictly between 0 and 1, only tokens with a conditional probability + * greater than epsilon_cutoff will be sampled. In the paper, suggested values range from + * 3e-4 to 9e-4, depending on the size of the model. See [Truncation Sampling as Language + * Model Desmoothing](https://hf.co/papers/2210.15191) for more details. + */ + epsilon_cutoff?: number; + /** + * Eta sampling is a hybrid of locally typical sampling and epsilon sampling. If set to + * float strictly between 0 and 1, a token is only considered if it is greater than either + * eta_cutoff or sqrt(eta_cutoff) * exp(-entropy(softmax(next_token_logits))). The latter + * term is intuitively the expected next token probability, scaled by sqrt(eta_cutoff). In + * the paper, suggested values range from 3e-4 to 2e-3, depending on the size of the model. + * See [Truncation Sampling as Language Model Desmoothing](https://hf.co/papers/2210.15191) + * for more details. + */ + eta_cutoff?: number; + /** + * The maximum length (in tokens) of the generated text, including the input. + */ + max_length?: number; + /** + * The maximum number of tokens to generate. Takes precedence over max_length. + */ + max_new_tokens?: number; + /** + * The minimum length (in tokens) of the generated text, including the input. + */ + min_length?: number; + /** + * The minimum number of tokens to generate. Takes precedence over min_length. + */ + min_new_tokens?: number; + /** + * Number of groups to divide num_beams into in order to ensure diversity among different + * groups of beams. See [this paper](https://hf.co/papers/1610.02424) for more details. + */ + num_beam_groups?: number; + /** + * Number of beams to use for beam search. + */ + num_beams?: number; + /** + * The value balances the model confidence and the degeneration penalty in contrastive + * search decoding. + */ + penalty_alpha?: number; + /** + * The value used to modulate the next token probabilities. + */ + temperature?: number; + /** + * The number of highest probability vocabulary tokens to keep for top-k-filtering. + */ + top_k?: number; + /** + * If set to float < 1, only the smallest set of most probable tokens with probabilities + * that add up to top_p or higher are kept for generation. + */ + top_p?: number; + /** + * Local typicality measures how similar the conditional probability of predicting a target + * token next is to the expected conditional probability of predicting a random token next, + * given the partial text already generated. If set to float < 1, the smallest set of the + * most locally typical tokens with probabilities that add up to typical_p or higher are + * kept for generation. See [this paper](https://hf.co/papers/2202.00666) for more details. + */ + typical_p?: number; + /** + * Whether the model should use the past last key/values attentions to speed up decoding + */ + use_cache?: boolean; + [property: string]: unknown; +} +/** + * Controls the stopping condition for beam-based methods. + */ +export type EarlyStoppingUnion = boolean | "never"; diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-speech/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/text-to-speech/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..a643f2d7c7e659ff79e02f3c77334031b38a36d1 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-speech/spec/input.json @@ -0,0 +1,30 @@ +{ + "$id": "/inference/schemas/text-to-speech/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Text To Speech inference", + "title": "TextToSpeechInput", + "type": "object", + "properties": { + "inputs": { + "description": "The input text data", + "type": "string" + }, + "parameters": { + "description": "Additional inference parameters for Text To Speech", + "$ref": "#/$defs/TextToSpeechParameters" + } + }, + "$defs": { + "TextToSpeechParameters": { + "title": "TextToSpeechParameters", + "type": "object", + "properties": { + "generation_parameters": { + "description": "Parametrization of the text generation process", + "$ref": "/inference/schemas/common-definitions.json#/definitions/GenerationParameters" + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-speech/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/text-to-speech/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..4836ed246fa23f7f83b64f6522695f1adb399c83 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-speech/spec/output.json @@ -0,0 +1,18 @@ +{ + "$id": "/inference/schemas/text-to-speech/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Text To Speech task", + "title": "TextToSpeechOutput", + "type": "object", + "properties": { + "audio": { + "description": "The generated audio", + "comment": "type=binary" + }, + "sampling_rate": { + "type": "number", + "description": "The sampling rate of the generated audio waveform." + } + }, + "required": ["audio"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-video/about.md b/node_modules/@huggingface/tasks/src/tasks/text-to-video/about.md new file mode 100644 index 0000000000000000000000000000000000000000..898d638c264aa8219cdc3a71d1a4562de0d084b8 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-video/about.md @@ -0,0 +1,41 @@ +## Use Cases + +### Script-based Video Generation + +Text-to-video models can be used to create short-form video content from a provided text script. These models can be used to create engaging and informative marketing videos. For example, a company could use a text-to-video model to create a video that explains how their product works. + +### Content format conversion + +Text-to-video models can be used to generate videos from long-form text, including blog posts, articles, and text files. Text-to-video models can be used to create educational videos that are more engaging and interactive. An example of this is creating a video that explains a complex concept from an article. + +### Voice-overs and Speech + +Text-to-video models can be used to create an AI newscaster to deliver daily news, or for a film-maker to create a short film or a music video. + +## Task Variants +Text-to-video models have different variants based on inputs and outputs. + +### Text-to-video Editing + +One text-to-video task is generating text-based video style and local attribute editing. Text-to-video editing models can make it easier to perform tasks like cropping, stabilization, color correction, resizing and audio editing consistently. + +### Text-to-video Search + +Text-to-video search is the task of retrieving videos that are relevant to a given text query. This can be challenging, as videos are a complex medium that can contain a lot of information. By using semantic analysis to extract the meaning of the text query, visual analysis to extract features from the videos, such as the objects and actions that are present in the video, and temporal analysis to categorize relationships between the objects and actions in the video, we can determine which videos are most likely to be relevant to the text query. + +### Text-driven Video Prediction + +Text-driven video prediction is the task of generating a video sequence from a text description. Text description can be anything from a simple sentence to a detailed story. The goal of this task is to generate a video that is both visually realistic and semantically consistent with the text description. + +### Video Translation + +Text-to-video translation models can translate videos from one language to another or allow to query the multilingual text-video model with non-English sentences. This can be useful for people who want to watch videos in a language that they don't understand, especially when multi-lingual captions are available for training. + +## Inference +Contribute an inference snippet for text-to-video here! + +## Useful Resources + +In this area, you can insert useful resources about how to train or use a model for this task. + +- [Text-to-Video: The Task, Challenges and the Current State](https://huggingface.co/blog/text-to-video) diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-video/data.ts b/node_modules/@huggingface/tasks/src/tasks/text-to-video/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..334146ba81e64ff378de65a09da14fddd0ae5fd4 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-video/data.ts @@ -0,0 +1,106 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "Microsoft Research Video to Text is a large-scale dataset for open domain video captioning", + id: "iejMac/CLIP-MSR-VTT", + }, + { + description: "UCF101 Human Actions dataset consists of 13,320 video clips from YouTube, with 101 classes.", + id: "quchenyuan/UCF101-ZIP", + }, + { + description: "A high-quality dataset for human action recognition in YouTube videos.", + id: "nateraw/kinetics", + }, + { + description: "A dataset of video clips of humans performing pre-defined basic actions with everyday objects.", + id: "HuggingFaceM4/something_something_v2", + }, + { + description: + "This dataset consists of text-video pairs and contains noisy samples with irrelevant video descriptions", + id: "HuggingFaceM4/webvid", + }, + { + description: "A dataset of short Flickr videos for the temporal localization of events with descriptions.", + id: "iejMac/CLIP-DiDeMo", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "Darth Vader is surfing on the waves.", + type: "text", + }, + ], + outputs: [ + { + filename: "text-to-video-output.gif", + type: "img", + }, + ], + }, + metrics: [ + { + description: + "Inception Score uses an image classification model that predicts class labels and evaluates how distinct and diverse the images are. A higher score indicates better video generation.", + id: "is", + }, + { + description: + "Frechet Inception Distance uses an image classification model to obtain image embeddings. The metric compares mean and standard deviation of the embeddings of real and generated images. A smaller score indicates better video generation.", + id: "fid", + }, + { + description: + "Frechet Video Distance uses a model that captures coherence for changes in frames and the quality of each frame. A smaller score indicates better video generation.", + id: "fvd", + }, + { + description: + "CLIPSIM measures similarity between video frames and text using an image-text similarity model. A higher score indicates better video generation.", + id: "clipsim", + }, + ], + models: [ + { + description: "A strong model for consistent video generation.", + id: "tencent/HunyuanVideo", + }, + { + description: "A text-to-video model with high fidelity motion and strong prompt adherence.", + id: "Lightricks/LTX-Video", + }, + { + description: "A text-to-video model focusing on physics-aware applications like robotics.", + id: "nvidia/Cosmos-1.0-Diffusion-7B-Text2World", + }, + { + description: "Very fast model for video generation.", + id: "Lightricks/LTX-Video-0.9.8-13B-distilled", + }, + ], + spaces: [ + { + description: "An application that generates video from text.", + id: "VideoCrafter/VideoCrafter", + }, + { + description: "Consistent video generation application.", + id: "Wan-AI/Wan2.1", + }, + { + description: "A cutting edge video generation application.", + id: "Pyramid-Flow/pyramid-flow", + }, + ], + summary: + "Text-to-video models can be used in any application that requires generating consistent sequence of images from text. ", + widgetModels: ["Wan-AI/Wan2.2-TI2V-5B"], + youtubeId: undefined, +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-video/inference.ts b/node_modules/@huggingface/tasks/src/tasks/text-to-video/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..0859ba524fb18b54bad8fc660190552f6a02cee0 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-video/inference.ts @@ -0,0 +1,57 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Text To Video inference + */ +export interface TextToVideoInput { + /** + * The input text data (sometimes called "prompt") + */ + inputs: string; + /** + * Additional inference parameters for Text To Video + */ + parameters?: TextToVideoParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Text To Video + */ +export interface TextToVideoParameters { + /** + * A higher guidance scale value encourages the model to generate videos closely linked to + * the text prompt, but values too high may cause saturation and other artifacts. + */ + guidance_scale?: number; + /** + * One or several prompt to guide what NOT to include in video generation. + */ + negative_prompt?: string[]; + /** + * The num_frames parameter determines how many video frames are generated. + */ + num_frames?: number; + /** + * The number of denoising steps. More denoising steps usually lead to a higher quality + * video at the expense of slower inference. + */ + num_inference_steps?: number; + /** + * Seed for the random number generator. + */ + seed?: number; + [property: string]: unknown; +} +/** + * Outputs of inference for the Text To Video task + */ +export interface TextToVideoOutput { + /** + * The generated video returned as raw bytes in the payload. + */ + video: unknown; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-video/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/text-to-video/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..077c988b60715c97657f9db2674e85fac4ba0272 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-video/spec/input.json @@ -0,0 +1,49 @@ +{ + "$id": "/inference/schemas/text-to-video/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Text To Video inference", + "title": "TextToVideoInput", + "type": "object", + "properties": { + "inputs": { + "description": "The input text data (sometimes called \"prompt\")", + "type": "string" + }, + "parameters": { + "description": "Additional inference parameters for Text To Video", + "$ref": "#/$defs/TextToVideoParameters" + } + }, + "$defs": { + "TextToVideoParameters": { + "title": "TextToVideoParameters", + "type": "object", + "properties": { + "num_frames": { + "type": "number", + "description": "The num_frames parameter determines how many video frames are generated." + }, + "guidance_scale": { + "type": "number", + "description": "A higher guidance scale value encourages the model to generate videos closely linked to the text prompt, but values too high may cause saturation and other artifacts." + }, + "negative_prompt": { + "type": "array", + "items": { + "type": "string" + }, + "description": "One or several prompt to guide what NOT to include in video generation." + }, + "num_inference_steps": { + "type": "integer", + "description": "The number of denoising steps. More denoising steps usually lead to a higher quality video at the expense of slower inference." + }, + "seed": { + "type": "integer", + "description": "Seed for the random number generator." + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/text-to-video/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/text-to-video/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..1a2c006dbc60d8f9d1e0db0afd3ef01e78d9221a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/text-to-video/spec/output.json @@ -0,0 +1,13 @@ +{ + "$id": "/inference/schemas/text-to-video/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Text To Video task", + "title": "TextToVideoOutput", + "type": "object", + "properties": { + "video": { + "description": "The generated video returned as raw bytes in the payload." + } + }, + "required": ["video"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/token-classification/about.md b/node_modules/@huggingface/tasks/src/tasks/token-classification/about.md new file mode 100644 index 0000000000000000000000000000000000000000..9b0701385b5793f32bdafd890c476a4efb99b509 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/token-classification/about.md @@ -0,0 +1,76 @@ +## Use Cases + +### Information Extraction from Invoices + +You can extract entities of interest from invoices automatically using Named Entity Recognition (NER) models. Invoices can be read with Optical Character Recognition models and the output can be used to do inference with NER models. In this way, important information such as date, company name, and other named entities can be extracted. + +## Task Variants + +### Named Entity Recognition (NER) + +NER is the task of recognizing named entities in a text. These entities can be the names of people, locations, or organizations. The task is formulated as labeling each token with a class for each named entity and a class named "0" for tokens that do not contain any entities. The input for this task is text and the output is the annotated text with named entities. + +#### Inference + +You can use the 🤗 Transformers library `ner` pipeline to infer with NER models. + +```python +from transformers import pipeline + +classifier = pipeline("ner") +classifier("Hello I'm Omar and I live in Zürich.") +``` + +### Part-of-Speech (PoS) Tagging +In PoS tagging, the model recognizes parts of speech, such as nouns, pronouns, adjectives, or verbs, in a given text. The task is formulated as labeling each word with a part of the speech. + +#### Inference + +You can use the 🤗 Transformers library `token-classification` pipeline with a POS tagging model of your choice. The model will return a json with PoS tags for each token. + +```python +from transformers import pipeline + +classifier = pipeline("token-classification", model = "vblagoje/bert-english-uncased-finetuned-pos") +classifier("Hello I'm Omar and I live in Zürich.") +``` + +This is not limited to transformers! You can also use other libraries such as Stanza, spaCy, and Flair to do inference! Here is an example using a canonical [spaCy](https://hf.co/blog/spacy) model. + +```python +!pip install https://huggingface.co/spacy/en_core_web_sm/resolve/main/en_core_web_sm-any-py3-none-any.whl + +import en_core_web_sm + +nlp = en_core_web_sm.load() +doc = nlp("I'm Omar and I live in Zürich.") +for token in doc: + print(token.text, token.pos_, token.dep_, token.ent_type_) + +## I PRON nsubj +## 'm AUX ROOT +## Omar PROPN attr PERSON +### ... +``` + +## Useful Resources + +Would you like to learn more about token classification? Great! Here you can find some curated resources that you may find helpful! + +- [Course Chapter on Token Classification](https://huggingface.co/course/chapter7/2?fw=pt) +- [Blog post: Welcome spaCy to the Hugging Face Hub](https://huggingface.co/blog/spacy) + +### Notebooks + +- [PyTorch](https://github.com/huggingface/notebooks/blob/master/examples/token_classification.ipynb) +- [TensorFlow](https://github.com/huggingface/notebooks/blob/master/examples/token_classification-tf.ipynb) + +### Scripts for training + +- [PyTorch](https://github.com/huggingface/transformers/tree/main/examples/pytorch/token-classification) +- [TensorFlow](https://github.com/huggingface/transformers/tree/main/examples/tensorflow) +- [Flax](https://github.com/huggingface/transformers/tree/main/examples/flax/token-classification) + +### Documentation + +- [Token classification task guide](https://huggingface.co/docs/transformers/tasks/token_classification) diff --git a/node_modules/@huggingface/tasks/src/tasks/token-classification/data.ts b/node_modules/@huggingface/tasks/src/tasks/token-classification/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..2ecce1bcc9e5537f092066ad9488fbd525fe1286 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/token-classification/data.ts @@ -0,0 +1,92 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A widely used dataset useful to benchmark named entity recognition models.", + id: "eriktks/conll2003", + }, + { + description: + "A multilingual dataset of Wikipedia articles annotated for named entity recognition in over 150 different languages.", + id: "unimelb-nlp/wikiann", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "My name is Omar and I live in Zürich.", + type: "text", + }, + ], + outputs: [ + { + text: "My name is Omar and I live in Zürich.", + tokens: [ + { + type: "PERSON", + start: 11, + end: 15, + }, + { + type: "GPE", + start: 30, + end: 36, + }, + ], + type: "text-with-tokens", + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + description: + "A robust performance model to identify people, locations, organizations and names of miscellaneous entities.", + id: "dslim/bert-base-NER", + }, + { + description: "A strong model to identify people, locations, organizations and names in multiple languages.", + id: "FacebookAI/xlm-roberta-large-finetuned-conll03-english", + }, + { + description: "A token classification model specialized on medical entity recognition.", + id: "blaze999/Medical-NER", + }, + { + description: "Flair models are typically the state of the art in named entity recognition tasks.", + id: "flair/ner-english", + }, + ], + spaces: [ + { + description: + "An application that can recognizes entities, extracts noun chunks and recognizes various linguistic features of each token.", + id: "spacy/gradio_pipeline_visualizer", + }, + ], + summary: + "Token classification is a natural language understanding task in which a label is assigned to some tokens in a text. Some popular token classification subtasks are Named Entity Recognition (NER) and Part-of-Speech (PoS) tagging. NER models could be trained to identify specific entities in a text, such as dates, individuals and places; and PoS tagging would identify, for example, which words in a text are verbs, nouns, and punctuation marks.", + widgetModels: ["FacebookAI/xlm-roberta-large-finetuned-conll03-english"], + youtubeId: "wVHdVlPScxA", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/token-classification/inference.ts b/node_modules/@huggingface/tasks/src/tasks/token-classification/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..09f899c6b039a8f78f313c02e25b55a791e7e3c6 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/token-classification/inference.ts @@ -0,0 +1,83 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Token Classification inference + */ +export interface TokenClassificationInput { + /** + * The input text data + */ + inputs: string; + /** + * Additional inference parameters for Token Classification + */ + parameters?: TokenClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Token Classification + */ +export interface TokenClassificationParameters { + /** + * The strategy used to fuse tokens based on model predictions + */ + aggregation_strategy?: TokenClassificationAggregationStrategy; + /** + * A list of labels to ignore + */ + ignore_labels?: string[]; + /** + * The number of overlapping tokens between chunks when splitting the input text. + */ + stride?: number; + [property: string]: unknown; +} +/** + * Do not aggregate tokens + * + * Group consecutive tokens with the same label in a single entity. + * + * Similar to "simple", also preserves word integrity (use the label predicted for the first + * token in a word). + * + * Similar to "simple", also preserves word integrity (uses the label with the highest + * score, averaged across the word's tokens). + * + * Similar to "simple", also preserves word integrity (uses the label with the highest score + * across the word's tokens). + */ +export type TokenClassificationAggregationStrategy = "none" | "simple" | "first" | "average" | "max"; +export type TokenClassificationOutput = TokenClassificationOutputElement[]; +/** + * Outputs of inference for the Token Classification task + */ +export interface TokenClassificationOutputElement { + /** + * The character position in the input where this group ends. + */ + end: number; + /** + * The predicted label for a single token + */ + entity?: string; + /** + * The predicted label for a group of one or more tokens + */ + entity_group?: string; + /** + * The associated score / probability + */ + score: number; + /** + * The character position in the input where this group begins. + */ + start: number; + /** + * The corresponding text + */ + word: string; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/token-classification/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/token-classification/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..1176f3dd9a4a903dd41737bbd6dc088483285407 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/token-classification/spec/input.json @@ -0,0 +1,64 @@ +{ + "$id": "/inference/schemas/token-classification/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Token Classification inference", + "title": "TokenClassificationInput", + "type": "object", + "properties": { + "inputs": { + "description": "The input text data", + "type": "string" + }, + "parameters": { + "description": "Additional inference parameters for Token Classification", + "$ref": "#/$defs/TokenClassificationParameters" + } + }, + "$defs": { + "TokenClassificationParameters": { + "title": "TokenClassificationParameters", + "type": "object", + "properties": { + "ignore_labels": { + "type": "array", + "items": { + "type": "string" + }, + "description": "A list of labels to ignore" + }, + "stride": { + "type": "integer", + "description": "The number of overlapping tokens between chunks when splitting the input text." + }, + "aggregation_strategy": { + "title": "TokenClassificationAggregationStrategy", + "type": "string", + "description": "The strategy used to fuse tokens based on model predictions", + "oneOf": [ + { + "const": "none", + "description": "Do not aggregate tokens" + }, + { + "const": "simple", + "description": "Group consecutive tokens with the same label in a single entity." + }, + { + "const": "first", + "description": "Similar to \"simple\", also preserves word integrity (use the label predicted for the first token in a word)." + }, + { + "const": "average", + "description": "Similar to \"simple\", also preserves word integrity (uses the label with the highest score, averaged across the word's tokens)." + }, + { + "const": "max", + "description": "Similar to \"simple\", also preserves word integrity (uses the label with the highest score across the word's tokens)." + } + ] + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/token-classification/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/token-classification/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..649a871b7b42b527b6a7f1c28fe86f5bfc085b25 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/token-classification/spec/output.json @@ -0,0 +1,37 @@ +{ + "$id": "/inference/schemas/token-classification/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Token Classification task", + "title": "TokenClassificationOutput", + "type": "array", + "items": { + "type": "object", + "properties": { + "entity_group": { + "type": "string", + "description": "The predicted label for a group of one or more tokens" + }, + "entity": { + "type": "string", + "description": "The predicted label for a single token" + }, + "score": { + "type": "number", + "description": "The associated score / probability" + }, + "word": { + "type": "string", + "description": "The corresponding text" + }, + "start": { + "type": "integer", + "description": "The character position in the input where this group begins." + }, + "end": { + "type": "integer", + "description": "The character position in the input where this group ends." + } + }, + "required": ["score", "word", "start", "end"] + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/translation/about.md b/node_modules/@huggingface/tasks/src/tasks/translation/about.md new file mode 100644 index 0000000000000000000000000000000000000000..f4806687a5cbebba9dce19f0053b33bd349e3a82 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/translation/about.md @@ -0,0 +1,65 @@ +## Use Cases + +You can find over a thousand Translation models on the Hub, but sometimes you might not find a model for the language pair you are interested in. When this happen, you can use a pretrained multilingual Translation model like [mBART](https://huggingface.co/facebook/mbart-large-cc25) and further train it on your own data in a process called fine-tuning. + +### Multilingual conversational agents + +Translation models can be used to build conversational agents across different languages. This can be done in two ways. + +- **Translate the dataset to a new language.** You can translate a dataset of intents (inputs) and responses to the target language. You can then train a new intent classification model with this new dataset. This allows you to proofread responses in the target language and have better control of the chatbot's outputs. + +* **Translate the input and output of the agent.** You can use a Translation model in user inputs so that the chatbot can process it. You can then translate the output of the chatbot into the language of the user. This approach might be less reliable as the chatbot will generate responses that were not defined before. + +## Inference + +You can use the 🤗 Transformers library with the `translation_xx_to_yy` pattern where xx is the source language code and yy is the target language code. The default model for the pipeline is [t5-base](https://huggingface.co/t5-base) which under the hood adds a task prefix indicating the task itself, e.g. “translate: English to French”. + +```python +from transformers import pipeline +en_fr_translator = pipeline("translation_en_to_fr") +en_fr_translator("How old are you?") +## [{'translation_text': ' quel âge êtes-vous?'}] +``` + +If you’d like to use a specific model checkpoint that is from one specific language to another, you can also directly use the `translation` pipeline. + +```python +from transformers import pipeline + +model_checkpoint = "Helsinki-NLP/opus-mt-en-fr" +translator = pipeline("translation", model=model_checkpoint) +translator("How are you?") +# [{'translation_text': 'Comment allez-vous ?'}] +``` + +You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer translation models on Hugging Face Hub. + +```javascript +import { InferenceClient } from "@huggingface/inference"; + +const inference = new InferenceClient(HF_TOKEN); +await inference.translation({ + model: "t5-base", + inputs: "My name is Wolfgang and I live in Berlin", +}); +``` + +## Useful Resources + +Would you like to learn more about Translation? Great! Here you can find some curated resources that you may find helpful! + +- [Course Chapter on Translation](https://huggingface.co/course/chapter7/4?fw=pt) + +### Notebooks + +- [PyTorch](https://github.com/huggingface/notebooks/blob/master/examples/translation.ipynb) +- [TensorFlow](https://github.com/huggingface/notebooks/blob/master/examples/translation-tf.ipynb) + +### Scripts for training + +- [PyTorch](https://github.com/huggingface/transformers/tree/main/examples/pytorch/translation) +- [TensorFlow](https://github.com/huggingface/transformers/tree/main/examples/tensorflow/translation) + +### Documentation + +- [Translation task guide](https://huggingface.co/docs/transformers/tasks/translation) diff --git a/node_modules/@huggingface/tasks/src/tasks/translation/data.ts b/node_modules/@huggingface/tasks/src/tasks/translation/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..1f7cb112c2a694464765858cd92bdc89280a0451 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/translation/data.ts @@ -0,0 +1,70 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + canonicalId: "text-generation", + datasets: [ + { + description: "A dataset of copyright-free books translated into 16 different languages.", + id: "Helsinki-NLP/opus_books", + }, + { + description: + "An example of translation between programming languages. This dataset consists of functions in Java and C#.", + id: "google/code_x_glue_cc_code_to_code_trans", + }, + ], + demo: { + inputs: [ + { + label: "Input", + content: "My name is Omar and I live in Zürich.", + type: "text", + }, + ], + outputs: [ + { + label: "Output", + content: "Mein Name ist Omar und ich wohne in Zürich.", + type: "text", + }, + ], + }, + metrics: [ + { + description: + "BLEU score is calculated by counting the number of shared single or subsequent tokens between the generated sequence and the reference. Subsequent n tokens are called “n-grams”. Unigram refers to a single token while bi-gram refers to token pairs and n-grams refer to n subsequent tokens. The score ranges from 0 to 1, where 1 means the translation perfectly matched and 0 did not match at all", + id: "bleu", + }, + { + description: "", + id: "sacrebleu", + }, + ], + models: [ + { + description: + "Very powerful model that can translate many languages between each other, especially low-resource languages.", + id: "facebook/nllb-200-1.3B", + }, + { + description: + "A general-purpose Transformer that can be used to translate from English to German, French, or Romanian.", + id: "google-t5/t5-base", + }, + ], + spaces: [ + { + description: "An application that can translate between 100 languages.", + id: "Iker/Translate-100-languages", + }, + { + description: "An application that can translate between many languages.", + id: "Geonmo/nllb-translation-demo", + }, + ], + summary: "Translation is the task of converting text from one language to another.", + widgetModels: ["facebook/mbart-large-50-many-to-many-mmt"], + youtubeId: "1JvfrvZgi6c", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/translation/inference.ts b/node_modules/@huggingface/tasks/src/tasks/translation/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..caa875033056ebed2ecbfeb6382e28d5bb8211f4 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/translation/inference.ts @@ -0,0 +1,63 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Translation inference + */ +export interface TranslationInput { + /** + * The text to translate. + */ + inputs: string; + /** + * Additional inference parameters for Translation + */ + parameters?: TranslationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Translation + */ +export interface TranslationParameters { + /** + * Whether to clean up the potential extra spaces in the text output. + */ + clean_up_tokenization_spaces?: boolean; + /** + * Additional parametrization of the text generation algorithm. + */ + generate_parameters?: { + [key: string]: unknown; + }; + /** + * The source language of the text. Required for models that can translate from multiple + * languages. + */ + src_lang?: string; + /** + * Target language to translate to. Required for models that can translate to multiple + * languages. + */ + tgt_lang?: string; + /** + * The truncation strategy to use. + */ + truncation?: TranslationTruncationStrategy; + [property: string]: unknown; +} +/** + * The truncation strategy to use. + */ +export type TranslationTruncationStrategy = "do_not_truncate" | "longest_first" | "only_first" | "only_second"; +/** + * Outputs of inference for the Translation task + */ +export interface TranslationOutput { + /** + * The translated text. + */ + translation_text: string; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/translation/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/translation/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..8f0a6fc6f936d37f129dfb7998e6abd9671930e0 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/translation/spec/input.json @@ -0,0 +1,49 @@ +{ + "$id": "/inference/schemas/translation/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Translation inference", + "title": "TranslationInput", + "type": "object", + "properties": { + "inputs": { + "description": "The text to translate.", + "type": "string" + }, + "parameters": { + "description": "Additional inference parameters for Translation", + "$ref": "#/$defs/TranslationParameters" + } + }, + "$defs": { + "TranslationParameters": { + "title": "TranslationParameters", + "type": "object", + "properties": { + "src_lang": { + "type": "string", + "description": "The source language of the text. Required for models that can translate from multiple languages." + }, + "tgt_lang": { + "type": "string", + "description": "Target language to translate to. Required for models that can translate to multiple languages." + }, + "clean_up_tokenization_spaces": { + "type": "boolean", + "description": "Whether to clean up the potential extra spaces in the text output." + }, + "truncation": { + "title": "TranslationTruncationStrategy", + "type": "string", + "description": "The truncation strategy to use.", + "enum": ["do_not_truncate", "longest_first", "only_first", "only_second"] + }, + "generate_parameters": { + "title": "generateParameters", + "type": "object", + "description": "Additional parametrization of the text generation algorithm." + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/translation/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/translation/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..976c3641eb32a07865d2efc921242ade895da3c0 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/translation/spec/output.json @@ -0,0 +1,14 @@ +{ + "$id": "/inference/schemas/translation/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Translation task", + "title": "TranslationOutput", + "type": "object", + "properties": { + "translation_text": { + "type": "string", + "description": "The translated text." + } + }, + "required": ["translation_text"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/unconditional-image-generation/about.md b/node_modules/@huggingface/tasks/src/tasks/unconditional-image-generation/about.md new file mode 100644 index 0000000000000000000000000000000000000000..e5a9585528ae0afd0bc779d0d8628ceca167376e --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/unconditional-image-generation/about.md @@ -0,0 +1,50 @@ +## About the Task + +Unconditional image generation is the task of generating new images without any specific input. The main goal of this is to create novel, original images that are not based on existing images. +This can be used for a variety of applications, such as creating new artistic images, improving image recognition algorithms, or generating photorealistic images for virtual reality environments. + +Unconditional image generation models usually start with a _seed_ that generates a _random noise vector_. The model will then use this vector to create an output image similar to the images used for training the model. + +An example of unconditional image generation would be generating the image of a face on a model trained with the [CelebA dataset](https://huggingface.co/datasets/huggan/CelebA-HQ) or [generating a butterfly](https://huggingface.co/spaces/huggan/butterfly-gan) on a model trained with the [Smithsonian Butterflies dataset](https://huggingface.co/datasets/ceyda/smithsonian_butterflies). + +[Generative adversarial networks](https://en.wikipedia.org/wiki/Generative_adversarial_network) and [Diffusion](https://huggingface.co/docs/diffusers/index) are common architectures for this task. + +## Use Cases + +Unconditional image generation can be used for a variety of applications. + +### Artistic Expression + +Unconditional image generation can be used to create novel, original artwork that is not based on any existing images. This can be used to explore new creative possibilities and produce unique, imaginative images. + +### Data Augmentation + +Unconditional image generation models can be used to generate new images to improve the performance of image recognition algorithms. This makes algorithms more robust and able to handle a broader range of images. + +### Virtual Reality + +Unconditional image generation models can be used to create photorealistic images that can be used in virtual reality environments. This makes the VR experience more immersive and realistic. + +### Medical Imaging + +Unconditional image generation models can generate new medical images, such as CT or MRI scans, that can be used to train and evaluate medical imaging algorithms. This can improve the accuracy and reliability of these algorithms. + +### Industrial Design + +Unconditional image generation models can generate new designs for products, such as clothing or furniture, that are not based on any existing designs. This way, designers can explore new creative possibilities and produce unique, innovative designs. + +## Model Hosting and Inference + +This section should have useful information about Model Hosting and Inference + +## Useful Resources + +- [Hugging Face Diffusion Models Course](https://github.com/huggingface/diffusion-models-class) +- [Getting Started with Diffusers](https://huggingface.co/docs/diffusers/index) +- [Unconditional Image Generation Training](https://huggingface.co/docs/diffusers/training/unconditional_training) + +### Training your own model in just a few seconds + +In this area, you can insert useful information about training the model + +This page was made possible thanks to the efforts of [Someet Sahoo](https://huggingface.co/Someet24) and [Juan Carlos Piñeros](https://huggingface.co/juancopi81). diff --git a/node_modules/@huggingface/tasks/src/tasks/unconditional-image-generation/data.ts b/node_modules/@huggingface/tasks/src/tasks/unconditional-image-generation/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..fcd66648eff68ecd77ab1c1a93528c7f305dab1b --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/unconditional-image-generation/data.ts @@ -0,0 +1,72 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: + "The CIFAR-100 dataset consists of 60000 32x32 colour images in 100 classes, with 600 images per class.", + id: "cifar100", + }, + { + description: "Multiple images of celebrities, used for facial expression translation.", + id: "CelebA", + }, + ], + demo: { + inputs: [ + { + label: "Seed", + content: "42", + type: "text", + }, + { + label: "Number of images to generate:", + content: "4", + type: "text", + }, + ], + outputs: [ + { + filename: "unconditional-image-generation-output.jpeg", + type: "img", + }, + ], + }, + metrics: [ + { + description: + "The inception score (IS) evaluates the quality of generated images. It measures the diversity of the generated images (the model predictions are evenly distributed across all possible labels) and their 'distinction' or 'sharpness' (the model confidently predicts a single label for each image).", + id: "Inception score (IS)", + }, + { + description: + "The Fréchet Inception Distance (FID) evaluates the quality of images created by a generative model by calculating the distance between feature vectors for real and generated images.", + id: "Frećhet Inception Distance (FID)", + }, + ], + models: [ + { + description: + "High-quality image generation model trained on the CIFAR-10 dataset. It synthesizes images of the ten classes presented in the dataset using diffusion probabilistic models, a class of latent variable models inspired by considerations from nonequilibrium thermodynamics.", + id: "google/ddpm-cifar10-32", + }, + { + description: + "High-quality image generation model trained on the 256x256 CelebA-HQ dataset. It synthesizes images of faces using diffusion probabilistic models, a class of latent variable models inspired by considerations from nonequilibrium thermodynamics.", + id: "google/ddpm-celebahq-256", + }, + ], + spaces: [ + { + description: "An application that can generate realistic faces.", + id: "CompVis/celeba-latent-diffusion", + }, + ], + summary: + "Unconditional image generation is the task of generating images with no condition in any context (like a prompt text or another image). Once trained, the model will create images that resemble its training data distribution.", + widgetModels: [""], + // TODO: Add related video + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/video-classification/about.md b/node_modules/@huggingface/tasks/src/tasks/video-classification/about.md new file mode 100644 index 0000000000000000000000000000000000000000..0436a873d3989f5c54fbff555fbfc573d3bd43b9 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/video-classification/about.md @@ -0,0 +1,37 @@ +## Use Cases + +Video classification models can be used to categorize what a video is all about. + +### Activity Recognition + +Video classification models are used to perform activity recognition which is useful for fitness applications. Activity recognition is also helpful for vision-impaired individuals especially when they're commuting. + +### Video Search + +Models trained in video classification can improve user experience by organizing and categorizing video galleries on the phone or in the cloud, on multiple keywords or tags. + +## Inference + +Below you can find code for inferring with a pre-trained video classification model. + +```python +from transformers import pipeline + +pipe = pipeline(task = "video-classification", model="nateraw/videomae-base-finetuned-ucf101-subset") +pipe("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/basketball.avi?download=true") + +#[{'score': 0.90, 'label': 'BasketballDunk'}, +# {'score': 0.02, 'label': 'BalanceBeam'}, +# ... ] +``` + +## Useful Resources + +- [Developing a simple video classification model](https://keras.io/examples/vision/video_classification) +- [Video classification with Transformers](https://keras.io/examples/vision/video_transformers) +- [Building a video archive](https://www.youtube.com/watch?v=_IeS1m8r6SY) +- [Video classification task guide](https://huggingface.co/docs/transformers/tasks/video_classification) + +### Creating your own video classifier in minutes + +- [Fine-tuning tutorial notebook (PyTorch)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/video_classification.ipynb) diff --git a/node_modules/@huggingface/tasks/src/tasks/video-classification/data.ts b/node_modules/@huggingface/tasks/src/tasks/video-classification/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..1e5abbce9cb324cd1f1a573e7b8a5a4184fd93ff --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/video-classification/data.ts @@ -0,0 +1,84 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + // TODO write proper description + description: "Benchmark dataset used for video classification with videos that belong to 400 classes.", + id: "kinetics400", + }, + ], + demo: { + inputs: [ + { + filename: "video-classification-input.gif", + type: "img", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Playing Guitar", + score: 0.514, + }, + { + label: "Playing Tennis", + score: 0.193, + }, + { + label: "Cooking", + score: 0.068, + }, + ], + }, + ], + }, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: "", + id: "recall", + }, + { + description: "", + id: "precision", + }, + { + description: "", + id: "f1", + }, + ], + models: [ + { + // TO DO: write description + description: "Strong Video Classification model trained on the Kinetics 400 dataset.", + id: "google/vivit-b-16x2-kinetics400", + }, + { + // TO DO: write description + description: "Strong Video Classification model trained on the Kinetics 400 dataset.", + id: "microsoft/xclip-base-patch32", + }, + ], + spaces: [ + { + description: "An application that classifies video at different timestamps.", + id: "nateraw/lavila", + }, + { + description: "An application that classifies video.", + id: "fcakyon/video-classification", + }, + ], + summary: + "Video classification is the task of assigning a label or class to an entire video. Videos are expected to have only one class for each video. Video classification models take a video as input and return a prediction about which class the video belongs to.", + widgetModels: [], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/video-classification/inference.ts b/node_modules/@huggingface/tasks/src/tasks/video-classification/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..5c937d9c180d5022695b1f3eb00568bb45b417e9 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/video-classification/inference.ts @@ -0,0 +1,60 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Video Classification inference + */ +export interface VideoClassificationInput { + /** + * The input video data + */ + inputs: unknown; + /** + * Additional inference parameters for Video Classification + */ + parameters?: VideoClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Video Classification + */ +export interface VideoClassificationParameters { + /** + * The sampling rate used to select frames from the video. + */ + frame_sampling_rate?: number; + /** + * The function to apply to the model outputs in order to retrieve the scores. + */ + function_to_apply?: ClassificationOutputTransform; + /** + * The number of sampled frames to consider for classification. + */ + num_frames?: number; + /** + * When specified, limits the output to the top K most probable classes. + */ + top_k?: number; + [property: string]: unknown; +} +/** + * The function to apply to the model outputs in order to retrieve the scores. + */ +export type ClassificationOutputTransform = "sigmoid" | "softmax" | "none"; +export type VideoClassificationOutput = VideoClassificationOutputElement[]; +/** + * Outputs of inference for the Video Classification task + */ +export interface VideoClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/video-classification/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/video-classification/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..277950e59d0834c788935ba0e5ab1b4dbc6a2525 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/video-classification/spec/input.json @@ -0,0 +1,42 @@ +{ + "$id": "/inference/schemas/video-classification/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Video Classification inference", + "title": "VideoClassificationInput", + "type": "object", + "properties": { + "inputs": { + "description": "The input video data" + }, + "parameters": { + "description": "Additional inference parameters for Video Classification", + "$ref": "#/$defs/VideoClassificationParameters" + } + }, + "$defs": { + "VideoClassificationParameters": { + "title": "VideoClassificationParameters", + "type": "object", + "properties": { + "function_to_apply": { + "title": "TextClassificationOutputTransform", + "$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutputTransform", + "description": "The function to apply to the model outputs in order to retrieve the scores." + }, + "num_frames": { + "type": "integer", + "description": "The number of sampled frames to consider for classification." + }, + "frame_sampling_rate": { + "type": "integer", + "description": "The sampling rate used to select frames from the video." + }, + "top_k": { + "type": "integer", + "description": "When specified, limits the output to the top K most probable classes." + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/video-classification/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/video-classification/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..4c24f5d577717994e0b4a8e329a7e063a967cb10 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/video-classification/spec/output.json @@ -0,0 +1,10 @@ +{ + "$id": "/inference/schemas/video-classification/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Video Classification task", + "title": "VideoClassificationOutput", + "type": "array", + "items": { + "$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput" + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/video-text-to-text/about.md b/node_modules/@huggingface/tasks/src/tasks/video-text-to-text/about.md new file mode 100644 index 0000000000000000000000000000000000000000..2d9af88d36a9e15b64e11623e25ecb2f3b20649f --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/video-text-to-text/about.md @@ -0,0 +1,98 @@ +Most of the video language models can take in videos, multiple videos, images and multiple images. Some of these models can also take interleaved inputs, which can have images and videos inside the text, where you can refer to the input images and input videos within the text prompt. + +## Different Types of Video Language Models + +Video language models come in three types: + +- **Base:** Pre-trained models that can be fine-tuned. +- **Instruction:** Base models fine-tuned on video-instruction pairs and answers. +- **Chatty/Conversational:** Base models fine-tuned on video conversation datasets. + +## Use Cases + +### Video Question Answering + +Video language models trained on video-question-answer pairs can be used for video question answering and generating captions for videos. + +### Video Chat + +Video language models can be used to have a dialogue about a video. + +### Video Recognition with Instructions + +Video language models can recognize images through descriptions. When given detailed descriptions of specific entities, they can classify the entities in a video. + +## Inference + +You can use the Transformers library to interact with video-language models. +Below we load [a video language model](https://huggingface.co/llava-hf/LLaVA-NeXT-Video-7B-hf), write a simple utility to sample videos, use chat template to format the text prompt, process the video and the text prompt and infer. To run the snippet below, please install [OpenCV](https://pypi.org/project/opencv-python/) by running `pip install opencv-python`. + +```python +import uuid +import requests +import cv2 +import torch +from transformers import LlavaNextVideoProcessor, LlavaNextVideoForConditionalGeneration + +device = "cuda" if torch.cuda.is_available() else "cpu" +model_id = "llava-hf/LLaVA-NeXT-Video-7B-hf" + +model = LlavaNextVideoForConditionalGeneration.from_pretrained( + model_id, + torch_dtype=torch.float16, + low_cpu_mem_usage=True, +).to(device) + +processor = LlavaNextVideoProcessor.from_pretrained(model_id) + +def sample_frames(url, num_frames): + response = requests.get(url) + path_id = str(uuid.uuid4()) + + path = f"./{path_id}.mp4" + + with open(path, "wb") as f: + f.write(response.content) + + video = cv2.VideoCapture(path) + total_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) + interval = total_frames // num_frames + frames = [] + for i in range(total_frames): + ret, frame = video.read() + if not ret: + continue + if i % interval == 0: + pil_img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + frames.append(pil_img) + video.release() + return frames + +conversation = [ + { + + "role": "user", + "content": [ + {"type": "text", "text": "Why is this video funny?"}, + {"type": "video"}, + ], + }, +] + +prompt = processor.apply_chat_template(conversation, add_generation_prompt=True) + +video_url = "https://huggingface.co/spaces/merve/llava-interleave/resolve/main/cats_1.mp4" +video = sample_frames(video, 8) + +inputs = processor(text=prompt, videos=video, padding=True, return_tensors="pt").to(model.device) + +output = model.generate(**inputs, max_new_tokens=100, do_sample=False) +print(processor.decode(output[0][2:], skip_special_tokens=True)) + +# Why is this video funny? ASSISTANT: The humor in this video comes from the cat's facial expression and body language. The cat appears to be making a funny face, with its eyes squinted and mouth open, which can be interpreted as a playful or mischievous expression. Cats often make such faces when they are in a good mood or are playful, and this can be amusing to people who are familiar with their behavior. The combination of the cat's expression and the close- + +``` + +## Useful Resources + +- [Transformers task guide on video-text-to-text](https://huggingface.co/docs/transformers/tasks/video_text_to_text) diff --git a/node_modules/@huggingface/tasks/src/tasks/video-text-to-text/data.ts b/node_modules/@huggingface/tasks/src/tasks/video-text-to-text/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..75e6908efe3b103345e5ffb2aaf647a2e9ccc880 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/video-text-to-text/data.ts @@ -0,0 +1,74 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "Multiple-choice questions and answers about videos.", + id: "lmms-lab/Video-MME", + }, + { + description: "A dataset of instructions and question-answer pairs about videos.", + id: "lmms-lab/VideoChatGPT", + }, + { + description: "Large video understanding dataset.", + id: "HuggingFaceFV/finevideo", + }, + ], + demo: { + inputs: [ + { + filename: "video-text-to-text-input.gif", + type: "img", + }, + { + label: "Text Prompt", + content: "What is happening in this video?", + type: "text", + }, + ], + outputs: [ + { + label: "Answer", + content: + "The video shows a series of images showing a fountain with water jets and a variety of colorful flowers and butterflies in the background.", + type: "text", + }, + ], + }, + metrics: [], + models: [ + { + description: "A robust video-text-to-text model.", + id: "Vision-CAIR/LongVU_Qwen2_7B", + }, + { + description: "Strong video-text-to-text model with reasoning capabilities.", + id: "GoodiesHere/Apollo-LMMs-Apollo-7B-t32", + }, + { + description: "Strong video-text-to-text model.", + id: "HuggingFaceTB/SmolVLM2-2.2B-Instruct", + }, + ], + spaces: [ + { + description: "An application to chat with a video-text-to-text model.", + id: "llava-hf/video-llava", + }, + { + description: "A leaderboard for various video-text-to-text models.", + id: "opencompass/openvlm_video_leaderboard", + }, + { + description: "An application to generate highlights from a video.", + id: "HuggingFaceTB/SmolVLM2-HighlightGenerator", + }, + ], + summary: + "Video-text-to-text models take in a video and a text prompt and output text. These models are also called video-language models.", + widgetModels: [""], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/video-to-video/about.md b/node_modules/@huggingface/tasks/src/tasks/video-to-video/about.md new file mode 100644 index 0000000000000000000000000000000000000000..9846262a8d7edfebe9d5ab69802a05b68e33c125 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/video-to-video/about.md @@ -0,0 +1,86 @@ +## Use Cases + +### Video Style Transfer + +Apply artistic or cinematic styles to a video while preserving motion and structure. For example, convert real footage into anime, painting, or film-like visuals. + +### Frame Interpolation + +Generate intermediate frames to make videos smoother or convert 30 FPS videos to 60 FPS. This improves motion flow and enables realistic slow-motion playback. + +### Video Super-Resolution + +Enhance low-resolution videos into high-definition outputs with preserved detail and sharpness. Ideal for restoring old footage or improving video quality. + +### Motion Transfer + +Transfer the motion from a source video to another subject while maintaining identity and environment. This enables realistic animation or gesture replication. + +### Video Editing & Synthesis + +Add, remove, or modify objects in videos while keeping lighting and motion consistent. Perfect for visual effects, object replacement, and content-aware editing. + +### Temporal Modification + +Change a video’s overall time or environmental conditions, such as day to night or summer to winter. These models preserve motion dynamics and lighting continuity. + +### Virtual Try-on + +Simulate clothing changes or outfit fitting in videos while keeping the person’s motion and identity intact. Useful for digital fashion and e-commerce applications. + +## Inference + +Below is an example demonstrating how to use [Lucy-Edit-Dev](https://huggingface.co/decart-ai/Lucy-Edit-Dev) to perform video costume editing, changing a character’s clothing while maintaining identity and motion consistency. Lucy-Edit-Dev is trained on paired video edits, captioned videos, and extended image–text datasets. + +```python +!pip install torch diffusers + +import torch +from PIL import Image + +from diffusers import AutoencoderKLWan, LucyEditPipeline +from diffusers.utils import export_to_video, load_video + + +url = "https://d2drjpuinn46lb.cloudfront.net/painter_original_edit.mp4" +prompt = "Change the apron and blouse to a classic clown costume: satin polka-dot jumpsuit in bright primary colors, ruffled white collar, oversized pom-pom buttons, white gloves, oversized red shoes, red foam nose; soft window light from left, eye-level medium shot, natural folds and fabric highlights." +negative_prompt = "" +num_frames = 81 +height = 480 +width = 832 + +def convert_video(video: List[Image.Image]) -> List[Image.Image]: + video = load_video(url)[:num_frames] + video = [video[i].resize((width, height)) for i in range(num_frames)] + return video + +video = load_video(url, convert_method=convert_video) + +model_id = "decart-ai/Lucy-Edit-Dev" +vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", torch_dtype=torch.float32) +pipe = LucyEditPipeline.from_pretrained(model_id, vae=vae, torch_dtype=torch.bfloat16) +pipe.to("cuda") + +output = pipe( + prompt=prompt, + video=video, + negative_prompt=negative_prompt, + height=480, + width=832, + num_frames=81, + guidance_scale=5.0 +).frames[0] + +export_to_video(output, "output.mp4", fps=24) +``` + +For more inference examples, check out the model cards on Hugging Face, where you can try the provided example code. + +## Useful Resources + +You can read more about the datasets, model architectures, and open-source implementations in the following repositories: + +- [Lumen](https://github.com/Kunbyte-AI/Lumen) - Official implementation of Lumen for text-guided video editing. +- [VIRES](https://github.com/suimuc/VIRES) - Implementation for sketch- and text-guided video instance repainting. +- [ECCV2022-RIFE: Video Frame Interpolation](https://github.com/hzwer/ECCV2022-RIFE) - Real-time video frame interpolation via intermediate flow estimation. +- [StableVSR: Enhancing Perceptual Quality in Video](https://github.com/claudiom4sir/StableVSR) - Super-resolution method to enhance perceptual video quality. diff --git a/node_modules/@huggingface/tasks/src/tasks/video-to-video/data.ts b/node_modules/@huggingface/tasks/src/tasks/video-to-video/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..3bd172b2d1c59247ad61380601f84a69f745c46f --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/video-to-video/data.ts @@ -0,0 +1,67 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "Dataset with detailed annotations for training and benchmarking video instance editing.", + id: "suimu/VIRESET", + }, + { + description: "Dataset to evaluate models on long video generation and understanding.", + id: "zhangsh2001/LongV-EVAL", + }, + { + description: "Collection of 104 demo videos from the SeedVR/SeedVR2 series showcasing model outputs.", + id: "Iceclear/SeedVR_VideoDemos", + }, + ], + demo: { + inputs: [ + { + filename: "input.gif", + type: "img", + }, + ], + outputs: [ + { + filename: "output.gif", + type: "img", + }, + ], + }, + metrics: [], + models: [ + { + description: "Model for editing outfits, character, and scenery in videos.", + id: "decart-ai/Lucy-Edit-Dev", + }, + { + description: "Framework that uses 3D mesh proxies for precise, consistent video editing.", + id: "LeoLau/Shape-for-Motion", + }, + { + description: "Model for generating physics-aware videos from input videos and control conditions.", + id: "nvidia/Cosmos-Transfer2.5-2B", + }, + { + description: "A model to upscale videos at input, designed for seamless use with ComfyUI.", + id: "numz/SeedVR2_comfyUI", + }, + ], + spaces: [ + { + description: "Interactive demo space for Lucy-Edit-Dev video editing.", + id: "decart-ai/lucy-edit-dev", + }, + { + description: "Demo space for SeedVR2-3B showcasing video upscaling and restoration.", + id: "ByteDance-Seed/SeedVR2-3B", + }, + ], + summary: + "Video-to-video models take one or more videos as input and generate new videos as output. They can enhance quality, interpolate frames, modify styles, or create new motion dynamics, enabling creative applications, video production, and research.", + widgetModels: [], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/visual-document-retrieval/about.md b/node_modules/@huggingface/tasks/src/tasks/visual-document-retrieval/about.md new file mode 100644 index 0000000000000000000000000000000000000000..97635b411d28583b306eb6cb53bb46e72a814655 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/visual-document-retrieval/about.md @@ -0,0 +1,54 @@ +## Use Cases + +### Multimodal Document Retrieval + +Visual document retrieval models can be used to retrieve relevant documents when given a text query. One needs to index the documents first, which is a one-time operation. After indexing is done, the retrieval model takes in a text query (question) and number `k` of documents to return, and the model returns the top-k most relevant documents for the query. The index can be used repetitively for inference. + +### Multimodal Retrieval Augmented Generation (RAG) + +Multimodal RAG is the task of generating answers from documents (texts or images) when given a text query and a bunch of documents. These documents and the text query can be fed to [a vision language model](https://huggingface.co/tasks/image-text-to-text) to get the actual answer. + +## Inference + +You can use transformers to infer visual document retrieval models. To calculate similarity between images and text, simply process both separately and pass each processed input through the model. The model outputs can then be passed to calculate similarity scores. + +```python +import torch +from PIL import Image +from transformers import ColPaliForRetrieval, ColPaliProcessor + +device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + +model = ColPaliForRetrieval.from_pretrained( + "vidore/colpali-v1.2-hf", + torch_dtype=torch.bfloat16, +).to(device) + +processor = ColPaliProcessor.from_pretrained("vidore/colpali-v1.2-hf") + +# Your inputs (replace dummy images with screenshots of your documents) +images = [ + Image.new("RGB", (32, 32), color="white"), + Image.new("RGB", (16, 16), color="black"), +] +queries = [ + "What is the organizational structure for our R&D department?", + "Can you provide a breakdown of last year’s financial performance?", +] + +# Process the image and text +batch_images = processor(images=images).to(device) +batch_queries = processor(text=queries).to(device) + +with torch.no_grad(): + image_embeddings = model(**batch_images).embeddings + query_embeddings = model(**batch_queries).embeddings + +# Score the queries against the images +scores = processor.score_retrieval(query_embeddings, image_embeddings) +``` + +## Useful Resources + +- [Multimodal Retrieval Augmented Generation using ColPali and Qwen2VL](https://github.com/merveenoyan/smol-vision/blob/main/ColPali_%2B_Qwen2_VL.ipynb) +- [Fine-tuning ColPali for Multimodal Retrieval Augmented Generation](https://github.com/merveenoyan/smol-vision/blob/main/Finetune_ColPali.ipynb) diff --git a/node_modules/@huggingface/tasks/src/tasks/visual-document-retrieval/data.ts b/node_modules/@huggingface/tasks/src/tasks/visual-document-retrieval/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..a6a8aceb5255c49b7c7b2ec74b391030ec244b4e --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/visual-document-retrieval/data.ts @@ -0,0 +1,76 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A large dataset used to train visual document retrieval models.", + id: "vidore/colpali_train_set", + }, + ], + demo: { + inputs: [ + { + filename: "input.png", + type: "img", + }, + { + label: "Question", + content: "Is the model in this paper the fastest for inference?", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Page 10", + score: 0.7, + }, + { + label: "Page 11", + score: 0.06, + }, + { + label: "Page 9", + score: 0.003, + }, + ], + }, + ], + }, + isPlaceholder: false, + metrics: [ + { + description: "NDCG@k scores ranked recommendation lists for top-k results. 0 is the worst, 1 is the best.", + id: "Normalized Discounted Cumulative Gain at K", + }, + ], + models: [ + { + description: "Very accurate visual document retrieval model for multilingual queries and documents.", + id: "vidore/colqwen2-v1.0", + }, + { + description: + "Very fast and efficient visual document retrieval model that can also take in other modalities like audio.", + id: "Tevatron/OmniEmbed-v0.1", + }, + ], + spaces: [ + { + description: "A leaderboard of visual document retrieval models.", + id: "vidore/vidore-leaderboard", + }, + { + description: "Visual retrieval augmented generation demo based on ColQwen2 model.", + id: "vidore/visual-rag-tool", + }, + ], + summary: + "Visual document retrieval is the task of searching for relevant image-based documents, such as PDFs. These models take a text query and multiple documents as input and return the top-most relevant documents and relevancy scores as output.", + widgetModels: [""], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/about.md b/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/about.md new file mode 100644 index 0000000000000000000000000000000000000000..7f96e1679b8a5b46042f5c6e2eb533e80749160f --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/about.md @@ -0,0 +1,48 @@ +## Use Cases + +### Aid the Visually Impaired Persons + +VQA models can be used to reduce visual barriers for visually impaired individuals by allowing them to get information about images from the web and the real world. + +### Education + +VQA models can be used to improve experiences at museums by allowing observers to directly ask questions they interested in. + +### Improved Image Retrieval + +Visual question answering models can be used to retrieve images with specific characteristics. For example, the user can ask "Is there a dog?" to find all images with dogs from a set of images. + +### Video Search + +Specific snippets/timestamps of a video can be retrieved based on search queries. For example, the user can ask "At which part of the video does the guitar appear?" and get a specific timestamp range from the whole video. + +## Task Variants + +### Video Question Answering + +Video Question Answering aims to answer questions asked about the content of a video. + +## Inference + +You can infer with Visual Question Answering models using the `vqa` (or `visual-question-answering`) pipeline. This pipeline requires [the Python Image Library (PIL)](https://pillow.readthedocs.io/en/stable/) to process images. You can install it with (`pip install pillow`). + +```python +from PIL import Image +from transformers import pipeline + +vqa_pipeline = pipeline("visual-question-answering") + +image = Image.open("elephant.jpeg") +question = "Is there an elephant?" + +vqa_pipeline(image, question, top_k=1) +#[{'score': 0.9998154044151306, 'answer': 'yes'}] +``` + +## Useful Resources + +- [An introduction to Visual Question Answering - AllenAI](https://blog.allenai.org/vanilla-vqa-adcaaaa94336) +- [Multi Modal Framework (MMF) - Meta Research](https://mmf.sh/docs/getting_started/video_overview/) + +The contents of this page are contributed by [ +Bharat Raghunathan](https://huggingface.co/bharat-raghunathan) and [Jose Londono Botero](https://huggingface.co/jlondonobo). diff --git a/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/data.ts b/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..51de32a80d53a733265e3ebb40358d3efeb02d82 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/data.ts @@ -0,0 +1,97 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A widely used dataset containing questions (with answers) about images.", + id: "Graphcore/vqa", + }, + { + description: "A dataset to benchmark visual reasoning based on text in images.", + id: "facebook/textvqa", + }, + ], + demo: { + inputs: [ + { + filename: "elephant.jpeg", + type: "img", + }, + { + label: "Question", + content: "What is in this image?", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "elephant", + score: 0.97, + }, + { + label: "elephants", + score: 0.06, + }, + { + label: "animal", + score: 0.003, + }, + ], + }, + ], + }, + isPlaceholder: false, + metrics: [ + { + description: "", + id: "accuracy", + }, + { + description: + "Measures how much a predicted answer differs from the ground truth based on the difference in their semantic meaning.", + id: "wu-palmer similarity", + }, + ], + models: [ + { + description: "A visual question answering model trained to convert charts and plots to text.", + id: "google/deplot", + }, + { + description: + "A visual question answering model trained for mathematical reasoning and chart derendering from images.", + id: "google/matcha-base", + }, + { + description: "A strong visual question answering that answers questions from book covers.", + id: "google/pix2struct-ocrvqa-large", + }, + ], + spaces: [ + { + description: "An application that compares visual question answering models across different tasks.", + id: "merve/pix2struct", + }, + { + description: "An application that can answer questions based on images.", + id: "nielsr/vilt-vqa", + }, + { + description: "An application that can caption images and answer questions about a given image. ", + id: "Salesforce/BLIP", + }, + { + description: "An application that can caption images and answer questions about a given image. ", + id: "vumichien/Img2Prompt", + }, + ], + summary: + "Visual Question Answering is the task of answering open-ended questions based on an image. They output natural language responses to natural language questions.", + widgetModels: ["dandelin/vilt-b32-finetuned-vqa"], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/inference.ts b/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..3eb292c62d8c7aa03e540af836e7a65d0c21fe0d --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/inference.ts @@ -0,0 +1,60 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Visual Question Answering inference + */ +export interface VisualQuestionAnsweringInput { + /** + * One (image, question) pair to answer + */ + inputs: VisualQuestionAnsweringInputData; + /** + * Additional inference parameters for Visual Question Answering + */ + parameters?: VisualQuestionAnsweringParameters; + [property: string]: unknown; +} +/** + * One (image, question) pair to answer + */ +export interface VisualQuestionAnsweringInputData { + /** + * The image. + */ + image: unknown; + /** + * The question to answer based on the image. + */ + question: string; + [property: string]: unknown; +} +/** + * Additional inference parameters for Visual Question Answering + */ +export interface VisualQuestionAnsweringParameters { + /** + * The number of answers to return (will be chosen by order of likelihood). Note that we + * return less than topk answers if there are not enough options available within the + * context. + */ + top_k?: number; + [property: string]: unknown; +} +export type VisualQuestionAnsweringOutput = VisualQuestionAnsweringOutputElement[]; +/** + * Outputs of inference for the Visual Question Answering task + */ +export interface VisualQuestionAnsweringOutputElement { + /** + * The answer to the question + */ + answer?: string; + /** + * The associated score / probability + */ + score: number; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..d51f35c37992cef2147b35a263f2281cab1b3ea0 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/spec/input.json @@ -0,0 +1,42 @@ +{ + "$id": "/inference/schemas/visual-question-answering/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Visual Question Answering inference", + "title": "VisualQuestionAnsweringInput", + "type": "object", + "properties": { + "inputs": { + "description": "One (image, question) pair to answer", + "type": "object", + "title": "VisualQuestionAnsweringInputData", + "properties": { + "image": { + "description": "The image.", + "comment": "type=binary" + }, + "question": { + "description": "The question to answer based on the image.", + "type": "string" + } + }, + "required": ["question", "image"] + }, + "parameters": { + "description": "Additional inference parameters for Visual Question Answering", + "$ref": "#/$defs/VisualQuestionAnsweringParameters" + } + }, + "$defs": { + "VisualQuestionAnsweringParameters": { + "title": "VisualQuestionAnsweringParameters", + "type": "object", + "properties": { + "top_k": { + "type": "integer", + "description": "The number of answers to return (will be chosen by order of likelihood). Note that we return less than topk answers if there are not enough options available within the context." + } + } + } + }, + "required": ["inputs"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..2233951410bc63e7a6ee1b827c4ee385690e3877 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/visual-question-answering/spec/output.json @@ -0,0 +1,21 @@ +{ + "$id": "/inference/schemas/visual-question-answering/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Visual Question Answering task", + "title": "VisualQuestionAnsweringOutput", + "type": "array", + "items": { + "type": "object", + "properties": { + "answer": { + "type": "string", + "description": "The answer to the question" + }, + "score": { + "type": "number", + "description": "The associated score / probability" + } + }, + "required": ["score"] + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/about.md b/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/about.md new file mode 100644 index 0000000000000000000000000000000000000000..9b7ff3c48c931d3355c76aed20b891fe8f57c54b --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/about.md @@ -0,0 +1,40 @@ +## About the Task + +Zero Shot Classification is the task of predicting a class that wasn't seen by the model during training. This method, which leverages a pre-trained language model, can be thought of as an instance of [transfer learning](https://www.youtube.com/watch?v=BqqfQnyjmgg) which generally refers to using a model trained for one task in a different application than what it was originally trained for. This is particularly useful for situations where the amount of labeled data is small. + +In zero shot classification, we provide the model with a prompt and a sequence of text that describes what we want our model to do, in natural language. Zero-shot classification excludes any examples of the desired task being completed. This differs from single or few-shot classification, as these tasks include a single or a few examples of the selected task. + +Zero, single and few-shot classification seem to be an emergent feature of large language models. This feature seems to come about around model sizes of +100M parameters. The effectiveness of a model at a zero, single or few-shot task seems to scale with model size, meaning that larger models (models with more trainable parameters or layers) generally do better at this task. + +Here is an example of a zero-shot prompt for classifying the sentiment of a sequence of text: + +``` +Classify the following input text into one of the following three categories: [positive, negative, neutral] + +Input Text: Hugging Face is awesome for making all of these +state of the art models available! +Sentiment: positive + +``` + +One great example of this task with a nice off-the-shelf model is available at the widget of this page, where the user can input a sequence of text and candidate labels to the model. This is a _word level_ example of zero shot classification, more elaborate and lengthy generations are available with larger models. Testing these models out and getting a feel for prompt engineering is the best way to learn how to use them. + +## Inference + +You can use the 🤗 Transformers library zero-shot-classification pipeline to infer with zero shot text classification models. + +```python +from transformers import pipeline + +pipe = pipeline(model="facebook/bart-large-mnli") +pipe("I have a problem with my iphone that needs to be resolved asap!", + candidate_labels=["urgent", "not urgent", "phone", "tablet", "computer"], +) +# output +>>> {'sequence': 'I have a problem with my iphone that needs to be resolved asap!!', 'labels': ['urgent', 'phone', 'computer', 'not urgent', 'tablet'], 'scores': [0.504, 0.479, 0.013, 0.003, 0.002]} +``` + +## Useful Resources + +- [Zero Shot Learning](https://joeddav.github.io/blog/2020/05/29/ZSL.html) +- [Hugging Face on Transfer Learning](https://huggingface.co/course/en/chapter1/4?fw=pt#transfer-learning) diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/data.ts b/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..cbd0ceda5f77e2e880bb41aa7602f8f15820ad0a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/data.ts @@ -0,0 +1,74 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A widely used dataset used to benchmark multiple variants of text classification.", + id: "nyu-mll/glue", + }, + { + description: + "The Multi-Genre Natural Language Inference (MultiNLI) corpus is a crowd-sourced collection of 433k sentence pairs annotated with textual entailment information.", + id: "nyu-mll/multi_nli", + }, + { + description: + "FEVER is a publicly available dataset for fact extraction and verification against textual sources.", + id: "fever/fever", + }, + ], + demo: { + inputs: [ + { + label: "Text Input", + content: "Dune is the best movie ever.", + type: "text", + }, + { + label: "Candidate Labels", + content: "CINEMA, ART, MUSIC", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "CINEMA", + score: 0.9, + }, + { + label: "ART", + score: 0.1, + }, + { + label: "MUSIC", + score: 0.0, + }, + ], + }, + ], + }, + metrics: [], + models: [ + { + description: "Powerful zero-shot text classification model.", + id: "facebook/bart-large-mnli", + }, + { + description: "Cutting-edge zero-shot multilingual text classification model.", + id: "MoritzLaurer/ModernBERT-large-zeroshot-v2.0", + }, + { + description: "Zero-shot text classification model that can be used for topic and sentiment classification.", + id: "knowledgator/gliclass-modern-base-v2.0-init", + }, + ], + spaces: [], + summary: + "Zero-shot text classification is a task in natural language processing where a model is trained on a set of labeled examples but is then able to classify new examples from previously unseen classes.", + widgetModels: ["facebook/bart-large-mnli"], +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/inference.ts b/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..aaae36f62661d9fd51dd4f6d4956aacc31d3a03a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/inference.ts @@ -0,0 +1,55 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Zero Shot Classification inference + */ +export interface ZeroShotClassificationInput { + /** + * The text to classify + */ + inputs: string; + /** + * Additional inference parameters for Zero Shot Classification + */ + parameters: ZeroShotClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Zero Shot Classification + */ +export interface ZeroShotClassificationParameters { + /** + * The set of possible class labels to classify the text into. + */ + candidate_labels: string[]; + /** + * The sentence used in conjunction with `candidate_labels` to attempt the text + * classification by replacing the placeholder with the candidate labels. + */ + hypothesis_template?: string; + /** + * Whether multiple candidate labels can be true. If false, the scores are normalized such + * that the sum of the label likelihoods for each sequence is 1. If true, the labels are + * considered independent and probabilities are normalized for each candidate. + */ + multi_label?: boolean; + [property: string]: unknown; +} +export type ZeroShotClassificationOutput = ZeroShotClassificationOutputElement[]; +/** + * Outputs of inference for the Zero Shot Classification task + */ +export interface ZeroShotClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..c919eac7ed262d6b744dbf06acab124c3989aae2 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/spec/input.json @@ -0,0 +1,42 @@ +{ + "$id": "/inference/schemas/zero-shot-classification/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Zero Shot Classification inference", + "title": "ZeroShotClassificationInput", + "type": "object", + "properties": { + "inputs": { + "description": "The text to classify", + "type": "string" + }, + "parameters": { + "description": "Additional inference parameters for Zero Shot Classification", + "$ref": "#/$defs/ZeroShotClassificationParameters" + } + }, + "$defs": { + "ZeroShotClassificationParameters": { + "title": "ZeroShotClassificationParameters", + "type": "object", + "properties": { + "candidate_labels": { + "type": "array", + "description": "The set of possible class labels to classify the text into.", + "items": { + "type": "string" + } + }, + "hypothesis_template": { + "type": "string", + "description": "The sentence used in conjunction with `candidate_labels` to attempt the text classification by replacing the placeholder with the candidate labels." + }, + "multi_label": { + "type": "boolean", + "description": "Whether multiple candidate labels can be true. If false, the scores are normalized such that the sum of the label likelihoods for each sequence is 1. If true, the labels are considered independent and probabilities are normalized for each candidate." + } + }, + "required": ["candidate_labels"] + } + }, + "required": ["inputs", "parameters"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..1b5ac0cc378da256645aef340709d0543482c522 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-classification/spec/output.json @@ -0,0 +1,11 @@ +{ + "$id": "/inference/schemas/zero-shot-classification/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Zero Shot Classification task", + "title": "ZeroShotClassificationOutput", + "type": "array", + "items": { + "type": "object", + "$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput" + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/about.md b/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/about.md new file mode 100644 index 0000000000000000000000000000000000000000..9cf273b299be7828ea2c75fd0da6f50d65c50029 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/about.md @@ -0,0 +1,75 @@ +## About the Task + +Zero-shot image classification is a computer vision task to classify images into one of several classes, without any prior training or knowledge of the classes. + +Zero shot image classification works by transferring knowledge learnt during training of one model, to classify novel classes that was not present in the training data. So this is a variation of [transfer learning](https://www.youtube.com/watch?v=BqqfQnyjmgg). For instance, a model trained to differentiate cars from airplanes can be used to classify images of ships. + +The data in this learning paradigm consists of + +- Seen data - images and their corresponding labels +- Unseen data - only labels and no images +- Auxiliary information - additional information given to the model during training connecting the unseen and seen data. This can be in the form of textual description or word embeddings. + +## Use Cases + +### Image Retrieval + +Zero-shot learning resolves several challenges in image retrieval systems. For example, with the rapid growth of categories on the web, it is challenging to index images based on unseen categories. With zero-shot learning we can associate unseen categories to images by exploiting attributes to model the relationships among visual features and labels. + +### Action Recognition + +Action recognition is the task of identifying when a person in an image/video is performing a given action from a set of actions. If all the possible actions are not known beforehand, conventional deep learning models fail. With zero-shot learning, for a given domain of a set of actions, we can create a mapping connecting low-level features and a semantic description of auxiliary data to classify unknown classes of actions. + +## Task Variants + +You can contribute variants of this task [here](https://github.com/huggingface/hub-docs/blob/main/tasks/src/zero-shot-image-classification/about.md). + +## Inference + +The model can be loaded with the zero-shot-image-classification pipeline like so: + +```python +from transformers import pipeline +# More models in the model hub. +model_name = "openai/clip-vit-large-patch14-336" +classifier = pipeline("zero-shot-image-classification", model = model_name) +``` + +You can then use this pipeline to classify images into any of the class names you specify. You can specify more than two class labels too. + +```python +image_to_classify = "path_to_cat_and_dog_image.jpeg" +labels_for_classification = ["cat and dog", + "lion and cheetah", + "rabbit and lion"] +scores = classifier(image_to_classify, + candidate_labels = labels_for_classification) +``` + +The classifier would return a list of dictionaries after the inference which is stored in the variable `scores` in the code snippet above. Variable `scores` would look as follows: + +```python +[{'score': 0.9950482249259949, 'label': 'cat and dog'}, +{'score': 0.004863627254962921, 'label': 'rabbit and lion'}, +{'score': 8.816882473183796e-05, 'label': 'lion and cheetah'}] +``` + +The dictionary at the zeroth index of the list will contain the label with the highest score. + +```python +print(f"The highest score is {scores[0]['score']:.3f} for the label {scores[0]['label']}") +``` + +The output from the print statement above would look as follows: + +``` +The highest probability is 0.995 for the label cat and dog +``` + +## Useful Resources + +- [Zero-shot image classification task guide](https://huggingface.co/docs/transformers/tasks/zero_shot_image_classification). +- [Image-text Similarity Search](https://huggingface.co/learn/cookbook/faiss_with_hf_datasets_and_clip) + +This page was made possible thanks to the efforts of [Shamima Hossain](https://huggingface.co/Shamima), [Haider Zaidi +](https://huggingface.co/chefhaider) and [Paarth Bhatnagar](https://huggingface.co/Paarth). diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/data.ts b/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..ae87eb4fa0fae8d42fca404ff15531f3417ea62c --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/data.ts @@ -0,0 +1,88 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [ + { + // TODO write proper description + description: "", + id: "", + }, + ], + demo: { + inputs: [ + { + filename: "image-classification-input.jpeg", + type: "img", + }, + { + label: "Classes", + content: "cat, dog, bird", + type: "text", + }, + ], + outputs: [ + { + type: "chart", + data: [ + { + label: "Cat", + score: 0.664, + }, + { + label: "Dog", + score: 0.329, + }, + { + label: "Bird", + score: 0.008, + }, + ], + }, + ], + }, + metrics: [ + { + description: "Computes the number of times the correct label appears in top K labels predicted", + id: "top-K accuracy", + }, + ], + models: [ + { + description: "Multilingual image classification model for 80 languages.", + id: "visheratin/mexma-siglip", + }, + { + description: "Strong zero-shot image classification model.", + id: "google/siglip2-base-patch16-224", + }, + { + description: "Robust zero-shot image classification model.", + id: "intfloat/mmE5-mllama-11b-instruct", + }, + { + description: "Powerful zero-shot image classification model supporting 94 languages.", + id: "jinaai/jina-clip-v2", + }, + { + description: "Strong image classification model for biomedical domain.", + id: "microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224", + }, + ], + spaces: [ + { + description: + "An application that leverages zero-shot image classification to find best captions to generate an image. ", + id: "pharma/CLIP-Interrogator", + }, + { + description: "An application to compare different zero-shot image classification models. ", + id: "merve/compare_clip_siglip", + }, + ], + summary: + "Zero-shot image classification is the task of classifying previously unseen classes during training of a model.", + widgetModels: ["google/siglip-so400m-patch14-224"], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/inference.ts b/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..a700bb4f30eb515c393aea91a2808c82e7e54388 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/inference.ts @@ -0,0 +1,49 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Zero Shot Image Classification inference + */ +export interface ZeroShotImageClassificationInput { + /** + * The input image data to classify as a base64-encoded string. + */ + inputs: Blob; + /** + * Additional inference parameters for Zero Shot Image Classification + */ + parameters: ZeroShotImageClassificationParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Zero Shot Image Classification + */ +export interface ZeroShotImageClassificationParameters { + /** + * The candidate labels for this image + */ + candidate_labels: string[]; + /** + * The sentence used in conjunction with `candidate_labels` to attempt the image + * classification by replacing the placeholder with the candidate labels. + */ + hypothesis_template?: string; + [property: string]: unknown; +} +export type ZeroShotImageClassificationOutput = ZeroShotImageClassificationOutputElement[]; +/** + * Outputs of inference for the Zero Shot Image Classification task + */ +export interface ZeroShotImageClassificationOutputElement { + /** + * The predicted class label. + */ + label: string; + /** + * The corresponding probability. + */ + score: number; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..7c3c413436c6354cca1ddd2bd6b516309ac3fa72 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/spec/input.json @@ -0,0 +1,39 @@ +{ + "$id": "/inference/schemas/zero-shot-image-classification/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Zero Shot Image Classification inference", + "title": "ZeroShotImageClassificationInput", + "type": "object", + "properties": { + "inputs": { + "type": "string", + "description": "The input image data to classify as a base64-encoded string.", + "comment": "type=binary" + }, + "parameters": { + "description": "Additional inference parameters for Zero Shot Image Classification", + "$ref": "#/$defs/ZeroShotImageClassificationParameters" + } + }, + "$defs": { + "ZeroShotImageClassificationParameters": { + "title": "ZeroShotImageClassificationParameters", + "type": "object", + "properties": { + "candidate_labels": { + "description": "The candidate labels for this image", + "type": "array", + "items": { + "type": "string" + } + }, + "hypothesis_template": { + "type": "string", + "description": "The sentence used in conjunction with `candidate_labels` to attempt the image classification by replacing the placeholder with the candidate labels." + } + }, + "required": ["candidate_labels"] + } + }, + "required": ["inputs", "parameters"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..6b795fbdbae8b566845fb424f30a7d7908609358 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-image-classification/spec/output.json @@ -0,0 +1,10 @@ +{ + "$id": "/inference/schemas/zero-shot-image-classification/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Zero Shot Image Classification task", + "title": "ZeroShotImageClassificationOutput", + "type": "array", + "items": { + "$ref": "/inference/schemas/common-definitions.json#/definitions/ClassificationOutput" + } +} diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/about.md b/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/about.md new file mode 100644 index 0000000000000000000000000000000000000000..46c4bf7c169fc3664b89d6b1243b47d9b4cdc7aa --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/about.md @@ -0,0 +1,45 @@ +## Use Cases + +Zero-shot object detection models can be used in any object detection application where the detection involves text queries for objects of interest. + +### Object Search + +Zero-shot object detection models can be used in image search. Smartphones, for example, use zero-shot object detection models to detect entities (such as specific places or objects) and allow the user to search for the entity on the internet. + +### Object Counting + +Zero-shot object detection models are used to count instances of objects in a given image. This can include counting the objects in warehouses or stores or the number of visitors in a store. They are also used to manage crowds at events to prevent disasters. + +### Object Tracking + +Zero-shot object detectors can track objects in videos. + +## Inference + +You can infer with zero-shot object detection models through the `zero-shot-object-detection` pipeline. When calling the pipeline, you just need to specify a path or HTTP link to an image and the candidate labels. + +```python +from transformers import pipeline +from PIL import Image + +image = Image.open("my-image.png").convert("RGB") + +detector = pipeline(model="google/owlvit-base-patch32", task="zero-shot-object-detection") + +predictions = detector( + image, + candidate_labels=["a photo of a cat", "a photo of a dog"], +) + +# [{'score': 0.95, +# 'label': 'a photo of a cat', +# 'box': {'xmin': 180, 'ymin': 71, 'xmax': 271, 'ymax': 178}}, +# ... +# ] +``` + +# Useful Resources + +- [Zero-shot object detection task guide](https://huggingface.co/docs/transformers/tasks/zero_shot_object_detection) + +This page was made possible thanks to the efforts of [Victor Guichard](https://huggingface.co/VictorGuichard) diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/data.ts b/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/data.ts new file mode 100644 index 0000000000000000000000000000000000000000..705bbdeceaa10a04dd1f9232881b0a263ad0219a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/data.ts @@ -0,0 +1,67 @@ +import type { TaskDataCustom } from "../index.js"; + +const taskData: TaskDataCustom = { + datasets: [], + demo: { + inputs: [ + { + filename: "zero-shot-object-detection-input.jpg", + type: "img", + }, + { + label: "Classes", + content: "cat, dog, bird", + type: "text", + }, + ], + outputs: [ + { + filename: "zero-shot-object-detection-output.jpg", + type: "img", + }, + ], + }, + metrics: [ + { + description: + "The Average Precision (AP) metric is the Area Under the PR Curve (AUC-PR). It is calculated for each class separately", + id: "Average Precision", + }, + { + description: "The Mean Average Precision (mAP) metric is the overall average of the AP values", + id: "Mean Average Precision", + }, + { + description: + "The APα metric is the Average Precision at the IoU threshold of a α value, for example, AP50 and AP75", + id: "APα", + }, + ], + models: [ + { + description: "Solid zero-shot object detection model.", + id: "openmmlab-community/mm_grounding_dino_large_all", + }, + { + description: "Cutting-edge zero-shot object detection model.", + id: "fushh7/LLMDet", + }, + ], + spaces: [ + { + description: "A demo to compare different zero-shot object detection models per output and latency.", + id: "ariG23498/zero-shot-od", + }, + { + description: + "A demo that combines a zero-shot object detection and mask generation model for zero-shot segmentation.", + id: "merve/OWLSAM", + }, + ], + summary: + "Zero-shot object detection is a computer vision task to detect objects and their classes in images, without any prior training or knowledge of the classes. Zero-shot object detection models receive an image as input, as well as a list of candidate classes, and output the bounding boxes and labels where the objects have been detected.", + widgetModels: [], + youtubeId: "", +}; + +export default taskData; diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/inference.ts b/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/inference.ts new file mode 100644 index 0000000000000000000000000000000000000000..e5ae3eb1ac8a5f6832ed098d9e80ecb2f21b41eb --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/inference.ts @@ -0,0 +1,60 @@ +/** + * Inference code generated from the JSON schema spec in ./spec + * + * Using src/scripts/inference-codegen + */ +/** + * Inputs for Zero Shot Object Detection inference + */ +export interface ZeroShotObjectDetectionInput { + /** + * The input image data as a base64-encoded string. + */ + inputs: Blob; + /** + * Additional inference parameters for Zero Shot Object Detection + */ + parameters: ZeroShotObjectDetectionParameters; + [property: string]: unknown; +} +/** + * Additional inference parameters for Zero Shot Object Detection + */ +export interface ZeroShotObjectDetectionParameters { + /** + * The candidate labels for this image + */ + candidate_labels: string[]; + [property: string]: unknown; +} +/** + * The predicted bounding box. Coordinates are relative to the top left corner of the input + * image. + */ +export interface BoundingBox { + xmax: number; + xmin: number; + ymax: number; + ymin: number; + [property: string]: unknown; +} +export type ZeroShotObjectDetectionOutput = ZeroShotObjectDetectionOutputElement[]; +/** + * Outputs of inference for the Zero Shot Object Detection task + */ +export interface ZeroShotObjectDetectionOutputElement { + /** + * The predicted bounding box. Coordinates are relative to the top left corner of the input + * image. + */ + box: BoundingBox; + /** + * A candidate label + */ + label: string; + /** + * The associated score / probability + */ + score: number; + [property: string]: unknown; +} diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/spec/input.json b/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/spec/input.json new file mode 100644 index 0000000000000000000000000000000000000000..7e6264dee50a44b8e93a261ed96a3bb4c4f0a37d --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/spec/input.json @@ -0,0 +1,35 @@ +{ + "$id": "/inference/schemas/zero-shot-object-detection/input.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Inputs for Zero Shot Object Detection inference", + "title": "ZeroShotObjectDetectionInput", + "type": "object", + "properties": { + "inputs": { + "description": "The input image data as a base64-encoded string.", + "type": "string", + "comment": "type=binary" + }, + "parameters": { + "description": "Additional inference parameters for Zero Shot Object Detection", + "$ref": "#/$defs/ZeroShotObjectDetectionParameters" + } + }, + "$defs": { + "ZeroShotObjectDetectionParameters": { + "title": "ZeroShotObjectDetectionParameters", + "type": "object", + "properties": { + "candidate_labels": { + "description": "The candidate labels for this image", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["candidate_labels"] + } + }, + "required": ["inputs", "parameters"] +} diff --git a/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/spec/output.json b/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/spec/output.json new file mode 100644 index 0000000000000000000000000000000000000000..8afa6052769f617ae365348d4d560ee43095ae4a --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tasks/zero-shot-object-detection/spec/output.json @@ -0,0 +1,47 @@ +{ + "$id": "/inference/schemas/zero-shot-object-detection/output.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "description": "Outputs of inference for the Zero Shot Object Detection task", + "title": "ZeroShotObjectDetectionOutput", + "type": "array", + "items": { + "type": "object", + "title": "ZeroShotObjectDetectionOutputElement", + "properties": { + "label": { + "type": "string", + "description": "A candidate label" + }, + "score": { + "type": "number", + "description": "The associated score / probability" + }, + "box": { + "$ref": "#/$defs/BoundingBox", + "description": "The predicted bounding box. Coordinates are relative to the top left corner of the input image." + } + }, + "required": ["box", "label", "score"] + }, + "$defs": { + "BoundingBox": { + "title": "BoundingBox", + "type": "object", + "properties": { + "xmin": { + "type": "integer" + }, + "xmax": { + "type": "integer" + }, + "ymin": { + "type": "integer" + }, + "ymax": { + "type": "integer" + } + }, + "required": ["xmin", "xmax", "ymin", "ymax"] + } + } +} diff --git a/node_modules/@huggingface/tasks/src/tokenizer-data.ts b/node_modules/@huggingface/tasks/src/tokenizer-data.ts new file mode 100644 index 0000000000000000000000000000000000000000..6be41e8f60df763bf854feea8ca8ed689794a609 --- /dev/null +++ b/node_modules/@huggingface/tasks/src/tokenizer-data.ts @@ -0,0 +1,32 @@ +export const SPECIAL_TOKENS_ATTRIBUTES = [ + "bos_token", + "eos_token", + "unk_token", + "sep_token", + "pad_token", + "cls_token", + "mask_token", + // additional_special_tokens (TODO) +] as const; + +/** + * Public interface for a tokenizer's special tokens mapping + */ +export interface AddedToken { + __type: "AddedToken"; + content?: string; + lstrip?: boolean; + normalized?: boolean; + rstrip?: boolean; + single_word?: boolean; +} +export type SpecialTokensMap = { + [key in (typeof SPECIAL_TOKENS_ATTRIBUTES)[number]]?: string | AddedToken | null; +}; +/** + * Public interface for tokenizer config + */ +export interface TokenizerConfig extends SpecialTokensMap { + use_default_system_prompt?: boolean; + chat_template?: string | Array<{ name: string; template: string }>; +} diff --git a/node_modules/@huggingface/tasks/src/widget-example.ts b/node_modules/@huggingface/tasks/src/widget-example.ts new file mode 100644 index 0000000000000000000000000000000000000000..2f60cd76bc9e314f037acca747af2c5c1d3cf35c --- /dev/null +++ b/node_modules/@huggingface/tasks/src/widget-example.ts @@ -0,0 +1,128 @@ +/** + * See default-widget-inputs.ts for the default widget inputs, this files only contains the types + */ + +import type { ChatCompletionInputMessage } from "./tasks/index.js"; + +type TableData = Record; + +//#region outputs +export type WidgetExampleOutputLabels = Array<{ label: string; score: number }>; +export interface WidgetExampleOutputAnswerScore { + answer: string; + score: number; +} +export interface WidgetExampleOutputText { + text: string; +} +export interface WidgetExampleOutputUrl { + url: string; +} + +export type WidgetExampleOutput = + | WidgetExampleOutputLabels + | WidgetExampleOutputAnswerScore + | WidgetExampleOutputText + | WidgetExampleOutputUrl; +//#endregion + +export interface WidgetExampleBase { + example_title?: string; + group?: string; + /** + * Potential overrides to API parameters for this specific example + * (takes precedences over the model card metadata's inference.parameters) + */ + parameters?: { + /// token-classification + aggregation_strategy?: string; + /// text-generation + top_k?: number; + top_p?: number; + temperature?: number; + max_new_tokens?: number; + do_sample?: boolean; + /// text-to-image + negative_prompt?: string; + guidance_scale?: number; + num_inference_steps?: number; + }; + /** + * Optional output + */ + output?: TOutput; +} + +export interface WidgetExampleChatInput extends WidgetExampleBase { + messages: ChatCompletionInputMessage[]; +} + +export interface WidgetExampleTextInput extends WidgetExampleBase { + text: string; +} + +export interface WidgetExampleTextAndContextInput< + TOutput = WidgetExampleOutput, +> extends WidgetExampleTextInput { + context: string; +} + +export interface WidgetExampleTextAndTableInput extends WidgetExampleTextInput { + table: TableData; +} + +export interface WidgetExampleAssetInput extends WidgetExampleBase { + src: string; +} +export interface WidgetExampleAssetAndPromptInput< + TOutput = WidgetExampleOutput, +> extends WidgetExampleAssetInput { + prompt: string; +} + +export type WidgetExampleAssetAndTextInput = WidgetExampleAssetInput & + WidgetExampleTextInput; + +export type WidgetExampleAssetAndZeroShotInput = WidgetExampleAssetInput & + WidgetExampleZeroShotTextInput; + +export interface WidgetExampleStructuredDataInput extends WidgetExampleBase { + structured_data: TableData; +} + +export interface WidgetExampleTableDataInput extends WidgetExampleBase { + table: TableData; +} + +export interface WidgetExampleZeroShotTextInput extends WidgetExampleTextInput { + text: string; + candidate_labels: string; + multi_class: boolean; +} + +export interface WidgetExampleSentenceSimilarityInput< + TOutput = WidgetExampleOutput, +> extends WidgetExampleBase { + source_sentence: string; + sentences: string[]; +} + +//#endregion + +export type WidgetExample = + | WidgetExampleChatInput + | WidgetExampleTextInput + | WidgetExampleTextAndContextInput + | WidgetExampleTextAndTableInput + | WidgetExampleAssetInput + | WidgetExampleAssetAndPromptInput + | WidgetExampleAssetAndTextInput + | WidgetExampleAssetAndZeroShotInput + | WidgetExampleStructuredDataInput + | WidgetExampleTableDataInput + | WidgetExampleZeroShotTextInput + | WidgetExampleSentenceSimilarityInput; + +type KeysOfUnion = T extends unknown ? keyof T : never; + +export type WidgetExampleAttribute = KeysOfUnion; diff --git a/node_modules/@huggingface/tasks/tsconfig.json b/node_modules/@huggingface/tasks/tsconfig.json new file mode 100644 index 0000000000000000000000000000000000000000..3e3c571e5e55e7e4df25af4ea1e4565e19969231 --- /dev/null +++ b/node_modules/@huggingface/tasks/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "allowSyntheticDefaultImports": true, + "lib": ["ES2022", "DOM"], + "module": "NodeNext", + "target": "ESNext", + "moduleResolution": "nodenext", + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "skipLibCheck": true, + "noImplicitOverride": true, + "outDir": "./dist", + "declaration": true, + "declarationMap": true + }, + "include": ["src"], + "exclude": ["dist"] +} diff --git a/node_modules/@huggingface/xetchunk-wasm/LICENSE b/node_modules/@huggingface/xetchunk-wasm/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..a3ce68eb3a1d6900bebd200d107c47769642a3c1 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Hugging Face + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/node_modules/@huggingface/xetchunk-wasm/README.md b/node_modules/@huggingface/xetchunk-wasm/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b3eccd6b1e0078355381225376283d90117707f0 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/README.md @@ -0,0 +1,62 @@ +# @huggingface/xetchunk-wasm + +Content-defined chunking and hashing for Hugging Face [Xet storage](https://huggingface.co/docs/hub/storage-regions), matching the [Rust reference implementation](https://github.com/huggingface/xet-core/blob/main/deduplication/src/chunking.rs). + +Uses [`gearhash-jit`](https://www.npmjs.com/package/gearhash-jit) for fast GEAR rolling hash boundary detection and [`@huggingface/blake3-jit`](https://www.npmjs.com/package/@huggingface/blake3-jit) for BLAKE3 chunk hashing. + +## Usage + +```typescript +import { createChunker, nextBlock, finalize, getChunks, hashToHex, xorbHash, fileHash } from '@huggingface/xetchunk-wasm'; + +// One-shot: chunk all data at once +const data = new Uint8Array(1_000_000); +const chunks = getChunks(data); + +console.log(`${chunks.length} chunks`); +console.log('xorb hash:', hashToHex(xorbHash(chunks))); +console.log('file hash:', hashToHex(fileHash(chunks))); + +// Streaming: process data incrementally +const chunker = createChunker(); + +for await (const buf of source) { + const chunks = nextBlock(chunker, buf); + for (const chunk of chunks) { + console.log(hashToHex(chunk.hash), chunk.length); + } +} + +const lastChunk = finalize(chunker); +``` + +## API + +### Chunking + +- **`createChunker(targetChunkSize?: number)`** — Create a chunker (default 64KB target). +- **`nextBlock(chunker, data: Uint8Array): Chunk[]`** — Feed data, get complete chunks. +- **`finalize(chunker): Chunk | null`** — Flush remaining data as a final chunk. +- **`getChunks(data: Uint8Array, targetChunkSize?: number): Chunk[]`** — One-shot convenience. + +### Hash functions + +All hash functions return `Uint8Array` (32 bytes). Use `hashToHex()` to convert to hex strings. + +- **`xorbHash(chunks: Chunk[]): Uint8Array`** — Merkle tree hash over chunks (matches Rust `xorb_hash`). +- **`fileHash(chunks: Chunk[]): Uint8Array`** — File-level hash (matches Rust `file_hash`). +- **`hmac(hash: Uint8Array, key: Uint8Array): Uint8Array`** — BLAKE3 keyed hash (matches Rust `DataHash::hmac`). +- **`verificationHash(chunkHashes: Uint8Array[]): Uint8Array`** — Range verification hash (matches Rust `range_hash_from_chunks`). + +### Utilities + +- **`hashToHex(hash: Uint8Array): string`** — Convert 32-byte hash to hex string. +- **`hexToBytes(hex: string): Uint8Array`** — Convert 64-char hex string to 32 bytes. + +## Benchmarking + +```shell +pnpm --filter @huggingface/xetchunk-wasm bench +# or with a specific file: +pnpm --filter @huggingface/xetchunk-wasm bench path/to/large-file +``` diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/hash-utils.d.ts b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/hash-utils.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..bb4409de47f224792dbfc4768d4c68b506ef4616 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/hash-utils.d.ts @@ -0,0 +1,26 @@ +import type { Chunk } from "./xet-chunker.js"; +/** + * file_hash = hmac(xorb_hash(chunks), zero_key) + * + * Matches Rust's `merklehash::file_hash` which calls + * `file_hash_with_salt(chunks, &[0; 32])`. + */ +export declare function fileHash(chunks: Chunk[]): Uint8Array; +/** + * HMAC: blake3_keyed_hash(key_bytes, hash_bytes) + * + * Both inputs are 32-byte Uint8Arrays. + * Matches Rust's `DataHash::hmac`. + * + * Uses a fresh hasher per call since the key varies. + */ +export declare function hmac(hash: Uint8Array, key: Uint8Array): Uint8Array; +/** + * Verification hash for a range of chunk hashes. + * Concatenates all 32-byte hashes and applies blake3_keyed_hash + * with VERIFICATION_KEY. + * + * Matches Rust's `chunk_verification::range_hash_from_chunks`. + */ +export declare function verificationHash(chunkHashes: Uint8Array[]): Uint8Array; +//# sourceMappingURL=hash-utils.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/hash-utils.d.ts.map b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/hash-utils.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6414997936746db4abcc55a05fbf3953e8682b9f --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/hash-utils.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"hash-utils.d.ts","sourceRoot":"","sources":["../../src/hash-utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAa9C;;;;;GAKG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,UAAU,CAUpD;AAED;;;;;;;GAOG;AACH,wBAAgB,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,GAAG,UAAU,CAElE;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,UAAU,CAMtE"} \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/hash-utils.js b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/hash-utils.js new file mode 100644 index 0000000000000000000000000000000000000000..d114c0144e032ff4d942c17fca8b8ba7f020d980 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/hash-utils.js @@ -0,0 +1,56 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.fileHash = fileHash; +exports.hmac = hmac; +exports.verificationHash = verificationHash; +const blake3_jit_1 = require("@huggingface/blake3-jit"); +const xorb_hash_js_1 = require("./xorb-hash.js"); +const ZERO_KEY = new Uint8Array(32); +const VERIFICATION_KEY = new Uint8Array([ + 127, 24, 87, 214, 206, 86, 237, 102, 18, 127, 249, 19, 231, 165, 195, 243, 164, 205, 38, 213, 181, 219, 73, 230, + 65, 36, 152, 127, 40, 251, 148, 195, +]); +const fileHasher = blake3_jit_1.Hasher.newKeyed(ZERO_KEY); +const verificationHasher = blake3_jit_1.Hasher.newKeyed(VERIFICATION_KEY); +/** + * file_hash = hmac(xorb_hash(chunks), zero_key) + * + * Matches Rust's `merklehash::file_hash` which calls + * `file_hash_with_salt(chunks, &[0; 32])`. + */ +function fileHash(chunks) { + // Empty input short-circuits to the all-zero MerkleHash, matching Rust's + // `file_hash_with_salt` (`if chunks.is_empty() { return MerkleHash::default(); }`). + // Without this we'd return `hmac(0, zero_key)`, which the CAS shard validation rejects + // for empty files with "file reconstruction does not produce this hash". + if (chunks.length === 0) { + return new Uint8Array(32); + } + const xorb = (0, xorb_hash_js_1.xorbHash)(chunks); + return fileHasher.reset().update(xorb).finalize(32); +} +/** + * HMAC: blake3_keyed_hash(key_bytes, hash_bytes) + * + * Both inputs are 32-byte Uint8Arrays. + * Matches Rust's `DataHash::hmac`. + * + * Uses a fresh hasher per call since the key varies. + */ +function hmac(hash, key) { + return blake3_jit_1.Hasher.newKeyed(key).update(hash).finalize(32); +} +/** + * Verification hash for a range of chunk hashes. + * Concatenates all 32-byte hashes and applies blake3_keyed_hash + * with VERIFICATION_KEY. + * + * Matches Rust's `chunk_verification::range_hash_from_chunks`. + */ +function verificationHash(chunkHashes) { + const combined = new Uint8Array(chunkHashes.length * 32); + for (let i = 0; i < chunkHashes.length; i++) { + combined.set(chunkHashes[i], i * 32); + } + return verificationHasher.reset().update(combined).finalize(32); +} diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/index.d.ts b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2458e126ab905c8c90a1e68b09ca766044783132 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/index.d.ts @@ -0,0 +1,4 @@ +export { createChunker, finalize, nextBlock, getChunks, hashToHex, hexToBytes, type Chunk } from "./xet-chunker.js"; +export { xorbHash } from "./xorb-hash.js"; +export { fileHash, hmac, verificationHash } from "./hash-utils.js"; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/index.d.ts.map b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..401ec0280d0ca1a309e82e7b57f4dd30c2a48c1e --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACpH,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/index.js b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/index.js new file mode 100644 index 0000000000000000000000000000000000000000..43e3f25e4fc83ce87b90e44e65126517f3af055e --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/index.js @@ -0,0 +1,16 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.verificationHash = exports.hmac = exports.fileHash = exports.xorbHash = exports.hexToBytes = exports.hashToHex = exports.getChunks = exports.nextBlock = exports.finalize = exports.createChunker = void 0; +var xet_chunker_js_1 = require("./xet-chunker.js"); +Object.defineProperty(exports, "createChunker", { enumerable: true, get: function () { return xet_chunker_js_1.createChunker; } }); +Object.defineProperty(exports, "finalize", { enumerable: true, get: function () { return xet_chunker_js_1.finalize; } }); +Object.defineProperty(exports, "nextBlock", { enumerable: true, get: function () { return xet_chunker_js_1.nextBlock; } }); +Object.defineProperty(exports, "getChunks", { enumerable: true, get: function () { return xet_chunker_js_1.getChunks; } }); +Object.defineProperty(exports, "hashToHex", { enumerable: true, get: function () { return xet_chunker_js_1.hashToHex; } }); +Object.defineProperty(exports, "hexToBytes", { enumerable: true, get: function () { return xet_chunker_js_1.hexToBytes; } }); +var xorb_hash_js_1 = require("./xorb-hash.js"); +Object.defineProperty(exports, "xorbHash", { enumerable: true, get: function () { return xorb_hash_js_1.xorbHash; } }); +var hash_utils_js_1 = require("./hash-utils.js"); +Object.defineProperty(exports, "fileHash", { enumerable: true, get: function () { return hash_utils_js_1.fileHash; } }); +Object.defineProperty(exports, "hmac", { enumerable: true, get: function () { return hash_utils_js_1.hmac; } }); +Object.defineProperty(exports, "verificationHash", { enumerable: true, get: function () { return hash_utils_js_1.verificationHash; } }); diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/package.json b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/package.json new file mode 100644 index 0000000000000000000000000000000000000000..5bbefffbabee392d1855491b84dc0a716b6a3bf2 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/package.json @@ -0,0 +1,3 @@ +{ + "type": "commonjs" +} diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xet-chunker.d.ts b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xet-chunker.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b0d053845aa3b33a04abf7caed5d7752b0d6add7 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xet-chunker.d.ts @@ -0,0 +1,38 @@ +export interface Chunk { + hash: Uint8Array; + length: number; +} +interface NextResult { + chunk: Chunk | null; + bytesConsumed: number; +} +declare class XetChunker { + private minimumChunk; + private maximumChunk; + private chunkBuf; + private curChunkLen; + private gear; + private blake3; + constructor(targetChunkSize?: number); + /** + * Streaming entry point: accepts an arbitrary slice of data, accumulates + * it, and emits a chunk when a boundary (or max size) is reached. + * Data is copied into an internal buffer because it may span calls. + */ + next(data: Uint8Array, isFinal: boolean): NextResult; + /** + * Batch entry point: processes a large contiguous buffer and returns all + * complete chunks. Hashes directly from `data` — no intermediate copy + * to chunkBuf — for every chunk whose bytes are fully within `data`. + */ + nextBlock(data: Uint8Array, isFinal: boolean): Chunk[]; + finish(): Chunk | null; +} +export declare function createChunker(targetChunkSize?: number): XetChunker; +export declare function nextBlock(chunker: XetChunker, data: Uint8Array): Chunk[]; +export declare function finalize(chunker: XetChunker): Chunk | null; +export declare function getChunks(data: Uint8Array, targetChunkSize?: number): Chunk[]; +export declare function hashToHex(hash: Uint8Array): string; +export declare function hexToBytes(hex: string): Uint8Array; +export {}; +//# sourceMappingURL=xet-chunker.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xet-chunker.d.ts.map b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xet-chunker.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..b7f92e53661b9b2b3d44a0328ab8228ea3d6dc4a --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xet-chunker.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"xet-chunker.d.ts","sourceRoot":"","sources":["../../src/xet-chunker.ts"],"names":[],"mappings":"AAaA,MAAM,WAAW,KAAK;IACrB,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CACf;AAED,UAAU,UAAU;IACnB,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;CACtB;AAED,cAAM,UAAU;IACf,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,MAAM,CAAe;gBAEjB,eAAe,GAAE,MAA0B;IAkCvD;;;;OAIG;IACH,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,GAAG,UAAU;IAwDpD;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,GAAG,KAAK,EAAE;IA2DtD,MAAM,IAAI,KAAK,GAAG,IAAI;CAWtB;AAED,wBAAgB,aAAa,CAAC,eAAe,GAAE,MAA0B,GAAG,UAAU,CAErF;AAED,wBAAgB,SAAS,CAAC,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,GAAG,KAAK,EAAE,CAExE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,UAAU,GAAG,KAAK,GAAG,IAAI,CAE1D;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,eAAe,GAAE,MAA0B,GAAG,KAAK,EAAE,CAGhG;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAalD;AAED,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,CAQlD"} \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xet-chunker.js b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xet-chunker.js new file mode 100644 index 0000000000000000000000000000000000000000..75f2f007ba53bb0db1bfa831e003074e7c587204 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xet-chunker.js @@ -0,0 +1,212 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.createChunker = createChunker; +exports.nextBlock = nextBlock; +exports.finalize = finalize; +exports.getChunks = getChunks; +exports.hashToHex = hashToHex; +exports.hexToBytes = hexToBytes; +const gearhash_jit_1 = require("gearhash-jit"); +const blake3_jit_1 = require("@huggingface/blake3-jit"); +const TARGET_CHUNK_SIZE = 64 * 1024; // 64KB +const MINIMUM_CHUNK_DIVISOR = 8; +const MAXIMUM_CHUNK_MULTIPLIER = 2; +const HASH_WINDOW_SIZE = 64; +const BLAKE3_DATA_KEY = new Uint8Array([ + 102, 151, 245, 119, 91, 149, 80, 222, 49, 53, 203, 172, 165, 151, 24, 28, 157, 228, 33, 16, 155, 235, 43, 88, 180, + 208, 176, 75, 147, 173, 242, 41, +]); +class XetChunker { + minimumChunk; + maximumChunk; + chunkBuf; + curChunkLen; + gear; + blake3; + constructor(targetChunkSize = TARGET_CHUNK_SIZE) { + if (targetChunkSize <= 0) { + throw new Error("Target chunk size must be greater than 0"); + } + if ((targetChunkSize & (targetChunkSize - 1)) !== 0) { + throw new Error("Target chunk size must be a power of 2"); + } + if (targetChunkSize <= HASH_WINDOW_SIZE) { + throw new Error("Target chunk size must be greater than hash window size"); + } + if (targetChunkSize >= Number.MAX_SAFE_INTEGER) { + throw new Error("Target chunk size must be less than Number.MAX_SAFE_INTEGER"); + } + let mask = BigInt(targetChunkSize - 1); + let leadingZeros = 0; + for (let i = 63; i >= 0; i--) { + if ((mask & (1n << BigInt(i))) !== 0n) { + break; + } + leadingZeros++; + } + mask = mask << BigInt(leadingZeros); + const maximumChunk = targetChunkSize * MAXIMUM_CHUNK_MULTIPLIER; + this.minimumChunk = targetChunkSize / MINIMUM_CHUNK_DIVISOR; + this.maximumChunk = maximumChunk; + this.chunkBuf = new Uint8Array(maximumChunk); + this.curChunkLen = 0; + this.gear = new gearhash_jit_1.Hasher(mask); + this.blake3 = blake3_jit_1.Hasher.newKeyed(BLAKE3_DATA_KEY); + } + /** + * Streaming entry point: accepts an arbitrary slice of data, accumulates + * it, and emits a chunk when a boundary (or max size) is reached. + * Data is copied into an internal buffer because it may span calls. + */ + next(data, isFinal) { + const nBytes = data.length; + let createChunk = false; + let consumeLen = 0; + if (nBytes !== 0) { + if (this.curChunkLen + HASH_WINDOW_SIZE < this.minimumChunk) { + const maxAdvance = Math.min(this.minimumChunk - this.curChunkLen - HASH_WINDOW_SIZE - 1, nBytes - consumeLen); + consumeLen += maxAdvance; + this.curChunkLen += maxAdvance; + } + const readEnd = Math.min(nBytes, consumeLen + this.maximumChunk - this.curChunkLen); + let bytesToNextBoundary; + const position = this.gear.nextMatch(data.subarray(consumeLen, readEnd)); + if (position !== -1) { + bytesToNextBoundary = position; + createChunk = true; + } + else { + bytesToNextBoundary = readEnd - consumeLen; + } + if (bytesToNextBoundary + this.curChunkLen >= this.maximumChunk) { + bytesToNextBoundary = this.maximumChunk - this.curChunkLen; + createChunk = true; + } + this.curChunkLen += bytesToNextBoundary; + consumeLen += bytesToNextBoundary; + this.chunkBuf.set(data.subarray(0, consumeLen), this.curChunkLen - consumeLen); + } + if (createChunk || (isFinal && this.curChunkLen > 0)) { + const chunkData = this.chunkBuf.subarray(0, this.curChunkLen); + const hash = this.blake3.reset().update(chunkData).finalize(32); + const chunk = { + length: chunkData.length, + hash: hash, + }; + this.curChunkLen = 0; + this.gear.resetHash(); + return { + chunk, + bytesConsumed: consumeLen, + }; + } + return { + chunk: null, + bytesConsumed: consumeLen, + }; + } + /** + * Batch entry point: processes a large contiguous buffer and returns all + * complete chunks. Hashes directly from `data` — no intermediate copy + * to chunkBuf — for every chunk whose bytes are fully within `data`. + */ + nextBlock(data, isFinal) { + const chunks = []; + let pos = 0; + // Drain any leftover from a previous nextBlock / next call. + while (pos < data.length && this.curChunkLen > 0) { + const result = this.next(data.subarray(pos), false); + if (result.chunk) + chunks.push(result.chunk); + pos += result.bytesConsumed; + } + const minSkip = this.minimumChunk > HASH_WINDOW_SIZE + ? this.minimumChunk - HASH_WINDOW_SIZE - 1 + : 0; + while (pos < data.length) { + const chunkStart = pos; + const scanStart = Math.min(pos + minSkip, data.length); + const scanEnd = Math.min(data.length, pos + this.maximumChunk); + const position = this.gear.nextMatch(data.subarray(scanStart, scanEnd)); + let chunkEnd; + let foundBoundary; + if (position !== -1 && scanStart + position - chunkStart <= this.maximumChunk) { + chunkEnd = scanStart + position; + foundBoundary = true; + } + else if (scanEnd - chunkStart >= this.maximumChunk) { + chunkEnd = chunkStart + this.maximumChunk; + foundBoundary = true; + } + else { + foundBoundary = false; + chunkEnd = scanEnd; + } + if (foundBoundary) { + const hash = this.blake3.reset() + .update(data.subarray(chunkStart, chunkEnd)) + .finalize(32); + chunks.push({ length: chunkEnd - chunkStart, hash }); + pos = chunkEnd; + this.gear.resetHash(); + } + else if (isFinal) { + const hash = this.blake3.reset() + .update(data.subarray(chunkStart)) + .finalize(32); + chunks.push({ length: data.length - chunkStart, hash }); + pos = data.length; + } + else { + this.chunkBuf.set(data.subarray(chunkStart), 0); + this.curChunkLen = data.length - chunkStart; + pos = data.length; + } + } + return chunks; + } + finish() { + if (this.curChunkLen > 0) { + const chunkData = this.chunkBuf.subarray(0, this.curChunkLen); + const hash = this.blake3.reset().update(chunkData).finalize(32); + const chunk = { length: this.curChunkLen, hash }; + this.curChunkLen = 0; + this.gear.resetHash(); + return chunk; + } + return null; + } +} +function createChunker(targetChunkSize = TARGET_CHUNK_SIZE) { + return new XetChunker(targetChunkSize); +} +function nextBlock(chunker, data) { + return chunker.nextBlock(data, false); +} +function finalize(chunker) { + return chunker.finish(); +} +function getChunks(data, targetChunkSize = TARGET_CHUNK_SIZE) { + const chunker = createChunker(targetChunkSize); + return chunker.nextBlock(data, true); +} +function hashToHex(hash) { + const view = new DataView(hash.buffer, hash.byteOffset, hash.byteLength); + const u64 = view.getBigUint64(0, true); + const u64_2 = view.getBigUint64(8, true); + const u64_3 = view.getBigUint64(16, true); + const u64_4 = view.getBigUint64(24, true); + return (u64.toString(16).padStart(16, "0") + + u64_2.toString(16).padStart(16, "0") + + u64_3.toString(16).padStart(16, "0") + + u64_4.toString(16).padStart(16, "0")); +} +function hexToBytes(hex) { + const bytes = new Uint8Array(32); + const view = new DataView(bytes.buffer); + view.setBigUint64(0, BigInt("0x" + hex.slice(0, 16)), true); + view.setBigUint64(8, BigInt("0x" + hex.slice(16, 32)), true); + view.setBigUint64(16, BigInt("0x" + hex.slice(32, 48)), true); + view.setBigUint64(24, BigInt("0x" + hex.slice(48, 64)), true); + return bytes; +} diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xorb-hash.d.ts b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xorb-hash.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..c555da94197ac63629ff2ebc87737a6f6f5f4eeb --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xorb-hash.d.ts @@ -0,0 +1,3 @@ +import type { Chunk } from "./xet-chunker.js"; +export declare function xorbHash(chunks: Chunk[]): Uint8Array; +//# sourceMappingURL=xorb-hash.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xorb-hash.d.ts.map b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xorb-hash.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7d618ac97868a6562f4a97f7c086eb2ca28f38e0 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xorb-hash.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"xorb-hash.d.ts","sourceRoot":"","sources":["../../src/xorb-hash.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAc9C,wBAAgB,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,UAAU,CA8BpD"} \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xorb-hash.js b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xorb-hash.js new file mode 100644 index 0000000000000000000000000000000000000000..e7bb34c4f5192a0881c9709d6ce2356afd4944cb --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/commonjs/xorb-hash.js @@ -0,0 +1,56 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.xorbHash = xorbHash; +const blake3_jit_1 = require("@huggingface/blake3-jit"); +const xet_chunker_js_1 = require("./xet-chunker.js"); +const MEAN_CHUNK_PER_NODE = 4; +const BLAKE3_NODE_KEY = new Uint8Array([ + 1, 126, 197, 199, 165, 71, 41, 150, 253, 148, 102, 102, 180, 138, 2, 230, 93, 221, 83, 111, 55, 199, 109, 210, 248, + 99, 82, 230, 74, 83, 113, 63, +]); +const INDEX_OF_LAST_BYTE_OF_LAST_U64_IN_CHUNK_HASH = 3 * 8; +const nodeHasher = blake3_jit_1.Hasher.newKeyed(BLAKE3_NODE_KEY); +function xorbHash(chunks) { + if (chunks.length === 0) { + return new Uint8Array(32); + } + let currentChunks = chunks; + while (currentChunks.length > 1) { + const nodes = []; + let currentIndex = 0; + let numOfChildrenSoFar = 0; + for (let i = 0; i < currentChunks.length; i++) { + if (i === currentChunks.length - 1 || + numOfChildrenSoFar === 2 * MEAN_CHUNK_PER_NODE || + (numOfChildrenSoFar >= 2 && + currentChunks[i].hash[INDEX_OF_LAST_BYTE_OF_LAST_U64_IN_CHUNK_HASH] % MEAN_CHUNK_PER_NODE === 0)) { + nodes.push(mergedHashOfSequence(currentChunks.slice(currentIndex, i + 1))); + currentIndex = i + 1; + numOfChildrenSoFar = 0; + } + else { + numOfChildrenSoFar++; + } + } + currentChunks = nodes; + } + return currentChunks[0].hash; +} +/** + * Matches Rust's `merged_hash_of_sequence`: serializes each entry as + * "{hash_hex} : {length_decimal}\n" then hashes with BLAKE3_NODE_KEY. + */ +function mergedHashOfSequence(chunks) { + let text = ""; + let totalLength = 0; + for (const chunk of chunks) { + text += (0, xet_chunker_js_1.hashToHex)(chunk.hash) + " : " + chunk.length + "\n"; + totalLength += chunk.length; + } + const bytes = new Uint8Array(text.length); + for (let i = 0; i < text.length; i++) { + bytes[i] = text.charCodeAt(i); + } + const hash = nodeHasher.reset().update(bytes).finalize(32); + return { hash, length: totalLength }; +} diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/esm/hash-utils.d.ts b/node_modules/@huggingface/xetchunk-wasm/dist/esm/hash-utils.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..bb4409de47f224792dbfc4768d4c68b506ef4616 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/esm/hash-utils.d.ts @@ -0,0 +1,26 @@ +import type { Chunk } from "./xet-chunker.js"; +/** + * file_hash = hmac(xorb_hash(chunks), zero_key) + * + * Matches Rust's `merklehash::file_hash` which calls + * `file_hash_with_salt(chunks, &[0; 32])`. + */ +export declare function fileHash(chunks: Chunk[]): Uint8Array; +/** + * HMAC: blake3_keyed_hash(key_bytes, hash_bytes) + * + * Both inputs are 32-byte Uint8Arrays. + * Matches Rust's `DataHash::hmac`. + * + * Uses a fresh hasher per call since the key varies. + */ +export declare function hmac(hash: Uint8Array, key: Uint8Array): Uint8Array; +/** + * Verification hash for a range of chunk hashes. + * Concatenates all 32-byte hashes and applies blake3_keyed_hash + * with VERIFICATION_KEY. + * + * Matches Rust's `chunk_verification::range_hash_from_chunks`. + */ +export declare function verificationHash(chunkHashes: Uint8Array[]): Uint8Array; +//# sourceMappingURL=hash-utils.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/esm/hash-utils.d.ts.map b/node_modules/@huggingface/xetchunk-wasm/dist/esm/hash-utils.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..6414997936746db4abcc55a05fbf3953e8682b9f --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/esm/hash-utils.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"hash-utils.d.ts","sourceRoot":"","sources":["../../src/hash-utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAa9C;;;;;GAKG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,UAAU,CAUpD;AAED;;;;;;;GAOG;AACH,wBAAgB,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,UAAU,GAAG,UAAU,CAElE;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,UAAU,CAMtE"} \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/esm/hash-utils.js b/node_modules/@huggingface/xetchunk-wasm/dist/esm/hash-utils.js new file mode 100644 index 0000000000000000000000000000000000000000..9167be4d12f5e4114235a466cee7b9fbe6983163 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/esm/hash-utils.js @@ -0,0 +1,51 @@ +import { Hasher } from "@huggingface/blake3-jit"; +import { xorbHash } from "./xorb-hash.js"; +const ZERO_KEY = new Uint8Array(32); +const VERIFICATION_KEY = new Uint8Array([ + 127, 24, 87, 214, 206, 86, 237, 102, 18, 127, 249, 19, 231, 165, 195, 243, 164, 205, 38, 213, 181, 219, 73, 230, + 65, 36, 152, 127, 40, 251, 148, 195, +]); +const fileHasher = Hasher.newKeyed(ZERO_KEY); +const verificationHasher = Hasher.newKeyed(VERIFICATION_KEY); +/** + * file_hash = hmac(xorb_hash(chunks), zero_key) + * + * Matches Rust's `merklehash::file_hash` which calls + * `file_hash_with_salt(chunks, &[0; 32])`. + */ +export function fileHash(chunks) { + // Empty input short-circuits to the all-zero MerkleHash, matching Rust's + // `file_hash_with_salt` (`if chunks.is_empty() { return MerkleHash::default(); }`). + // Without this we'd return `hmac(0, zero_key)`, which the CAS shard validation rejects + // for empty files with "file reconstruction does not produce this hash". + if (chunks.length === 0) { + return new Uint8Array(32); + } + const xorb = xorbHash(chunks); + return fileHasher.reset().update(xorb).finalize(32); +} +/** + * HMAC: blake3_keyed_hash(key_bytes, hash_bytes) + * + * Both inputs are 32-byte Uint8Arrays. + * Matches Rust's `DataHash::hmac`. + * + * Uses a fresh hasher per call since the key varies. + */ +export function hmac(hash, key) { + return Hasher.newKeyed(key).update(hash).finalize(32); +} +/** + * Verification hash for a range of chunk hashes. + * Concatenates all 32-byte hashes and applies blake3_keyed_hash + * with VERIFICATION_KEY. + * + * Matches Rust's `chunk_verification::range_hash_from_chunks`. + */ +export function verificationHash(chunkHashes) { + const combined = new Uint8Array(chunkHashes.length * 32); + for (let i = 0; i < chunkHashes.length; i++) { + combined.set(chunkHashes[i], i * 32); + } + return verificationHasher.reset().update(combined).finalize(32); +} diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/esm/index.d.ts b/node_modules/@huggingface/xetchunk-wasm/dist/esm/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2458e126ab905c8c90a1e68b09ca766044783132 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/esm/index.d.ts @@ -0,0 +1,4 @@ +export { createChunker, finalize, nextBlock, getChunks, hashToHex, hexToBytes, type Chunk } from "./xet-chunker.js"; +export { xorbHash } from "./xorb-hash.js"; +export { fileHash, hmac, verificationHash } from "./hash-utils.js"; +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/esm/index.d.ts.map b/node_modules/@huggingface/xetchunk-wasm/dist/esm/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..401ec0280d0ca1a309e82e7b57f4dd30c2a48c1e --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/esm/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACpH,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC"} \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/esm/index.js b/node_modules/@huggingface/xetchunk-wasm/dist/esm/index.js new file mode 100644 index 0000000000000000000000000000000000000000..1bf78501fc0c9013786014536b32681dd2bd62c4 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/esm/index.js @@ -0,0 +1,3 @@ +export { createChunker, finalize, nextBlock, getChunks, hashToHex, hexToBytes } from "./xet-chunker.js"; +export { xorbHash } from "./xorb-hash.js"; +export { fileHash, hmac, verificationHash } from "./hash-utils.js"; diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/esm/package.json b/node_modules/@huggingface/xetchunk-wasm/dist/esm/package.json new file mode 100644 index 0000000000000000000000000000000000000000..3dbc1ca591c0557e35b6004aeba250e6a70b56e3 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/esm/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/esm/xet-chunker.d.ts b/node_modules/@huggingface/xetchunk-wasm/dist/esm/xet-chunker.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b0d053845aa3b33a04abf7caed5d7752b0d6add7 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/esm/xet-chunker.d.ts @@ -0,0 +1,38 @@ +export interface Chunk { + hash: Uint8Array; + length: number; +} +interface NextResult { + chunk: Chunk | null; + bytesConsumed: number; +} +declare class XetChunker { + private minimumChunk; + private maximumChunk; + private chunkBuf; + private curChunkLen; + private gear; + private blake3; + constructor(targetChunkSize?: number); + /** + * Streaming entry point: accepts an arbitrary slice of data, accumulates + * it, and emits a chunk when a boundary (or max size) is reached. + * Data is copied into an internal buffer because it may span calls. + */ + next(data: Uint8Array, isFinal: boolean): NextResult; + /** + * Batch entry point: processes a large contiguous buffer and returns all + * complete chunks. Hashes directly from `data` — no intermediate copy + * to chunkBuf — for every chunk whose bytes are fully within `data`. + */ + nextBlock(data: Uint8Array, isFinal: boolean): Chunk[]; + finish(): Chunk | null; +} +export declare function createChunker(targetChunkSize?: number): XetChunker; +export declare function nextBlock(chunker: XetChunker, data: Uint8Array): Chunk[]; +export declare function finalize(chunker: XetChunker): Chunk | null; +export declare function getChunks(data: Uint8Array, targetChunkSize?: number): Chunk[]; +export declare function hashToHex(hash: Uint8Array): string; +export declare function hexToBytes(hex: string): Uint8Array; +export {}; +//# sourceMappingURL=xet-chunker.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/esm/xet-chunker.d.ts.map b/node_modules/@huggingface/xetchunk-wasm/dist/esm/xet-chunker.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..b7f92e53661b9b2b3d44a0328ab8228ea3d6dc4a --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/esm/xet-chunker.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"xet-chunker.d.ts","sourceRoot":"","sources":["../../src/xet-chunker.ts"],"names":[],"mappings":"AAaA,MAAM,WAAW,KAAK;IACrB,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CACf;AAED,UAAU,UAAU;IACnB,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;CACtB;AAED,cAAM,UAAU;IACf,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,MAAM,CAAe;gBAEjB,eAAe,GAAE,MAA0B;IAkCvD;;;;OAIG;IACH,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,GAAG,UAAU;IAwDpD;;;;OAIG;IACH,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,GAAG,KAAK,EAAE;IA2DtD,MAAM,IAAI,KAAK,GAAG,IAAI;CAWtB;AAED,wBAAgB,aAAa,CAAC,eAAe,GAAE,MAA0B,GAAG,UAAU,CAErF;AAED,wBAAgB,SAAS,CAAC,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,GAAG,KAAK,EAAE,CAExE;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,UAAU,GAAG,KAAK,GAAG,IAAI,CAE1D;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,eAAe,GAAE,MAA0B,GAAG,KAAK,EAAE,CAGhG;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAalD;AAED,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,CAQlD"} \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/esm/xet-chunker.js b/node_modules/@huggingface/xetchunk-wasm/dist/esm/xet-chunker.js new file mode 100644 index 0000000000000000000000000000000000000000..97a31477ec8d37f544abc9025a61277b36218c9f --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/esm/xet-chunker.js @@ -0,0 +1,204 @@ +import { Hasher } from "gearhash-jit"; +import { Hasher as Blake3Hasher } from "@huggingface/blake3-jit"; +const TARGET_CHUNK_SIZE = 64 * 1024; // 64KB +const MINIMUM_CHUNK_DIVISOR = 8; +const MAXIMUM_CHUNK_MULTIPLIER = 2; +const HASH_WINDOW_SIZE = 64; +const BLAKE3_DATA_KEY = new Uint8Array([ + 102, 151, 245, 119, 91, 149, 80, 222, 49, 53, 203, 172, 165, 151, 24, 28, 157, 228, 33, 16, 155, 235, 43, 88, 180, + 208, 176, 75, 147, 173, 242, 41, +]); +class XetChunker { + minimumChunk; + maximumChunk; + chunkBuf; + curChunkLen; + gear; + blake3; + constructor(targetChunkSize = TARGET_CHUNK_SIZE) { + if (targetChunkSize <= 0) { + throw new Error("Target chunk size must be greater than 0"); + } + if ((targetChunkSize & (targetChunkSize - 1)) !== 0) { + throw new Error("Target chunk size must be a power of 2"); + } + if (targetChunkSize <= HASH_WINDOW_SIZE) { + throw new Error("Target chunk size must be greater than hash window size"); + } + if (targetChunkSize >= Number.MAX_SAFE_INTEGER) { + throw new Error("Target chunk size must be less than Number.MAX_SAFE_INTEGER"); + } + let mask = BigInt(targetChunkSize - 1); + let leadingZeros = 0; + for (let i = 63; i >= 0; i--) { + if ((mask & (1n << BigInt(i))) !== 0n) { + break; + } + leadingZeros++; + } + mask = mask << BigInt(leadingZeros); + const maximumChunk = targetChunkSize * MAXIMUM_CHUNK_MULTIPLIER; + this.minimumChunk = targetChunkSize / MINIMUM_CHUNK_DIVISOR; + this.maximumChunk = maximumChunk; + this.chunkBuf = new Uint8Array(maximumChunk); + this.curChunkLen = 0; + this.gear = new Hasher(mask); + this.blake3 = Blake3Hasher.newKeyed(BLAKE3_DATA_KEY); + } + /** + * Streaming entry point: accepts an arbitrary slice of data, accumulates + * it, and emits a chunk when a boundary (or max size) is reached. + * Data is copied into an internal buffer because it may span calls. + */ + next(data, isFinal) { + const nBytes = data.length; + let createChunk = false; + let consumeLen = 0; + if (nBytes !== 0) { + if (this.curChunkLen + HASH_WINDOW_SIZE < this.minimumChunk) { + const maxAdvance = Math.min(this.minimumChunk - this.curChunkLen - HASH_WINDOW_SIZE - 1, nBytes - consumeLen); + consumeLen += maxAdvance; + this.curChunkLen += maxAdvance; + } + const readEnd = Math.min(nBytes, consumeLen + this.maximumChunk - this.curChunkLen); + let bytesToNextBoundary; + const position = this.gear.nextMatch(data.subarray(consumeLen, readEnd)); + if (position !== -1) { + bytesToNextBoundary = position; + createChunk = true; + } + else { + bytesToNextBoundary = readEnd - consumeLen; + } + if (bytesToNextBoundary + this.curChunkLen >= this.maximumChunk) { + bytesToNextBoundary = this.maximumChunk - this.curChunkLen; + createChunk = true; + } + this.curChunkLen += bytesToNextBoundary; + consumeLen += bytesToNextBoundary; + this.chunkBuf.set(data.subarray(0, consumeLen), this.curChunkLen - consumeLen); + } + if (createChunk || (isFinal && this.curChunkLen > 0)) { + const chunkData = this.chunkBuf.subarray(0, this.curChunkLen); + const hash = this.blake3.reset().update(chunkData).finalize(32); + const chunk = { + length: chunkData.length, + hash: hash, + }; + this.curChunkLen = 0; + this.gear.resetHash(); + return { + chunk, + bytesConsumed: consumeLen, + }; + } + return { + chunk: null, + bytesConsumed: consumeLen, + }; + } + /** + * Batch entry point: processes a large contiguous buffer and returns all + * complete chunks. Hashes directly from `data` — no intermediate copy + * to chunkBuf — for every chunk whose bytes are fully within `data`. + */ + nextBlock(data, isFinal) { + const chunks = []; + let pos = 0; + // Drain any leftover from a previous nextBlock / next call. + while (pos < data.length && this.curChunkLen > 0) { + const result = this.next(data.subarray(pos), false); + if (result.chunk) + chunks.push(result.chunk); + pos += result.bytesConsumed; + } + const minSkip = this.minimumChunk > HASH_WINDOW_SIZE + ? this.minimumChunk - HASH_WINDOW_SIZE - 1 + : 0; + while (pos < data.length) { + const chunkStart = pos; + const scanStart = Math.min(pos + minSkip, data.length); + const scanEnd = Math.min(data.length, pos + this.maximumChunk); + const position = this.gear.nextMatch(data.subarray(scanStart, scanEnd)); + let chunkEnd; + let foundBoundary; + if (position !== -1 && scanStart + position - chunkStart <= this.maximumChunk) { + chunkEnd = scanStart + position; + foundBoundary = true; + } + else if (scanEnd - chunkStart >= this.maximumChunk) { + chunkEnd = chunkStart + this.maximumChunk; + foundBoundary = true; + } + else { + foundBoundary = false; + chunkEnd = scanEnd; + } + if (foundBoundary) { + const hash = this.blake3.reset() + .update(data.subarray(chunkStart, chunkEnd)) + .finalize(32); + chunks.push({ length: chunkEnd - chunkStart, hash }); + pos = chunkEnd; + this.gear.resetHash(); + } + else if (isFinal) { + const hash = this.blake3.reset() + .update(data.subarray(chunkStart)) + .finalize(32); + chunks.push({ length: data.length - chunkStart, hash }); + pos = data.length; + } + else { + this.chunkBuf.set(data.subarray(chunkStart), 0); + this.curChunkLen = data.length - chunkStart; + pos = data.length; + } + } + return chunks; + } + finish() { + if (this.curChunkLen > 0) { + const chunkData = this.chunkBuf.subarray(0, this.curChunkLen); + const hash = this.blake3.reset().update(chunkData).finalize(32); + const chunk = { length: this.curChunkLen, hash }; + this.curChunkLen = 0; + this.gear.resetHash(); + return chunk; + } + return null; + } +} +export function createChunker(targetChunkSize = TARGET_CHUNK_SIZE) { + return new XetChunker(targetChunkSize); +} +export function nextBlock(chunker, data) { + return chunker.nextBlock(data, false); +} +export function finalize(chunker) { + return chunker.finish(); +} +export function getChunks(data, targetChunkSize = TARGET_CHUNK_SIZE) { + const chunker = createChunker(targetChunkSize); + return chunker.nextBlock(data, true); +} +export function hashToHex(hash) { + const view = new DataView(hash.buffer, hash.byteOffset, hash.byteLength); + const u64 = view.getBigUint64(0, true); + const u64_2 = view.getBigUint64(8, true); + const u64_3 = view.getBigUint64(16, true); + const u64_4 = view.getBigUint64(24, true); + return (u64.toString(16).padStart(16, "0") + + u64_2.toString(16).padStart(16, "0") + + u64_3.toString(16).padStart(16, "0") + + u64_4.toString(16).padStart(16, "0")); +} +export function hexToBytes(hex) { + const bytes = new Uint8Array(32); + const view = new DataView(bytes.buffer); + view.setBigUint64(0, BigInt("0x" + hex.slice(0, 16)), true); + view.setBigUint64(8, BigInt("0x" + hex.slice(16, 32)), true); + view.setBigUint64(16, BigInt("0x" + hex.slice(32, 48)), true); + view.setBigUint64(24, BigInt("0x" + hex.slice(48, 64)), true); + return bytes; +} diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/esm/xorb-hash.d.ts b/node_modules/@huggingface/xetchunk-wasm/dist/esm/xorb-hash.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..c555da94197ac63629ff2ebc87737a6f6f5f4eeb --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/esm/xorb-hash.d.ts @@ -0,0 +1,3 @@ +import type { Chunk } from "./xet-chunker.js"; +export declare function xorbHash(chunks: Chunk[]): Uint8Array; +//# sourceMappingURL=xorb-hash.d.ts.map \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/esm/xorb-hash.d.ts.map b/node_modules/@huggingface/xetchunk-wasm/dist/esm/xorb-hash.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..7d618ac97868a6562f4a97f7c086eb2ca28f38e0 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/esm/xorb-hash.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"xorb-hash.d.ts","sourceRoot":"","sources":["../../src/xorb-hash.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAc9C,wBAAgB,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,UAAU,CA8BpD"} \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/dist/esm/xorb-hash.js b/node_modules/@huggingface/xetchunk-wasm/dist/esm/xorb-hash.js new file mode 100644 index 0000000000000000000000000000000000000000..3ef5fbe3f46daeb21c1fb9d1082978e270cd1da0 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/dist/esm/xorb-hash.js @@ -0,0 +1,53 @@ +import { Hasher } from "@huggingface/blake3-jit"; +import { hashToHex } from "./xet-chunker.js"; +const MEAN_CHUNK_PER_NODE = 4; +const BLAKE3_NODE_KEY = new Uint8Array([ + 1, 126, 197, 199, 165, 71, 41, 150, 253, 148, 102, 102, 180, 138, 2, 230, 93, 221, 83, 111, 55, 199, 109, 210, 248, + 99, 82, 230, 74, 83, 113, 63, +]); +const INDEX_OF_LAST_BYTE_OF_LAST_U64_IN_CHUNK_HASH = 3 * 8; +const nodeHasher = Hasher.newKeyed(BLAKE3_NODE_KEY); +export function xorbHash(chunks) { + if (chunks.length === 0) { + return new Uint8Array(32); + } + let currentChunks = chunks; + while (currentChunks.length > 1) { + const nodes = []; + let currentIndex = 0; + let numOfChildrenSoFar = 0; + for (let i = 0; i < currentChunks.length; i++) { + if (i === currentChunks.length - 1 || + numOfChildrenSoFar === 2 * MEAN_CHUNK_PER_NODE || + (numOfChildrenSoFar >= 2 && + currentChunks[i].hash[INDEX_OF_LAST_BYTE_OF_LAST_U64_IN_CHUNK_HASH] % MEAN_CHUNK_PER_NODE === 0)) { + nodes.push(mergedHashOfSequence(currentChunks.slice(currentIndex, i + 1))); + currentIndex = i + 1; + numOfChildrenSoFar = 0; + } + else { + numOfChildrenSoFar++; + } + } + currentChunks = nodes; + } + return currentChunks[0].hash; +} +/** + * Matches Rust's `merged_hash_of_sequence`: serializes each entry as + * "{hash_hex} : {length_decimal}\n" then hashes with BLAKE3_NODE_KEY. + */ +function mergedHashOfSequence(chunks) { + let text = ""; + let totalLength = 0; + for (const chunk of chunks) { + text += hashToHex(chunk.hash) + " : " + chunk.length + "\n"; + totalLength += chunk.length; + } + const bytes = new Uint8Array(text.length); + for (let i = 0; i < text.length; i++) { + bytes[i] = text.charCodeAt(i); + } + const hash = nodeHasher.reset().update(bytes).finalize(32); + return { hash, length: totalLength }; +} diff --git a/node_modules/@huggingface/xetchunk-wasm/package.json b/node_modules/@huggingface/xetchunk-wasm/package.json new file mode 100644 index 0000000000000000000000000000000000000000..863c8c0410f8c23526b4d09b6b3f5f29c03102f7 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/package.json @@ -0,0 +1,55 @@ +{ + "name": "@huggingface/xetchunk-wasm", + "version": "0.1.0", + "description": "Content-defined chunking and hashing for Hugging Face Xet storage", + "keywords": [ + "xet", + "chunk", + "chunking" + ], + "license": "MIT", + "author": "Hugging Face", + "repository": "https://github.com/huggingface/huggingface.js.git", + "publishConfig": { + "access": "public" + }, + "files": [ + "dist", + "src", + "README.md" + ], + "tshy": { + "exports": { + ".": "./src/index.ts", + "./package.json": "./package.json" + } + }, + "dependencies": { + "gearhash-jit": "1.0.2", + "@huggingface/blake3-jit": "0.0.2" + }, + "devDependencies": { + "@huggingface/splitmix64-wasm": "0.0.1" + }, + "type": "module", + "exports": { + ".": { + "import": { + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/index.js" + }, + "require": { + "types": "./dist/commonjs/index.d.ts", + "default": "./dist/commonjs/index.js" + } + }, + "./package.json": "./package.json" + }, + "main": "./dist/commonjs/index.js", + "types": "./dist/commonjs/index.d.ts", + "module": "./dist/esm/index.js", + "scripts": { + "test": "vitest run", + "bench": "node tests/bench.js" + } +} \ No newline at end of file diff --git a/node_modules/@huggingface/xetchunk-wasm/src/hash-utils.ts b/node_modules/@huggingface/xetchunk-wasm/src/hash-utils.ts new file mode 100644 index 0000000000000000000000000000000000000000..e017189d97c5244486dca0f17a183523929e4f8b --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/src/hash-utils.ts @@ -0,0 +1,58 @@ +import { Hasher } from "@huggingface/blake3-jit"; +import type { Chunk } from "./xet-chunker.js"; +import { xorbHash } from "./xorb-hash.js"; + +const ZERO_KEY = new Uint8Array(32); + +const VERIFICATION_KEY = new Uint8Array([ + 127, 24, 87, 214, 206, 86, 237, 102, 18, 127, 249, 19, 231, 165, 195, 243, 164, 205, 38, 213, 181, 219, 73, 230, + 65, 36, 152, 127, 40, 251, 148, 195, +]); + +const fileHasher = Hasher.newKeyed(ZERO_KEY); +const verificationHasher = Hasher.newKeyed(VERIFICATION_KEY); + +/** + * file_hash = hmac(xorb_hash(chunks), zero_key) + * + * Matches Rust's `merklehash::file_hash` which calls + * `file_hash_with_salt(chunks, &[0; 32])`. + */ +export function fileHash(chunks: Chunk[]): Uint8Array { + // Empty input short-circuits to the all-zero MerkleHash, matching Rust's + // `file_hash_with_salt` (`if chunks.is_empty() { return MerkleHash::default(); }`). + // Without this we'd return `hmac(0, zero_key)`, which the CAS shard validation rejects + // for empty files with "file reconstruction does not produce this hash". + if (chunks.length === 0) { + return new Uint8Array(32); + } + const xorb = xorbHash(chunks); + return fileHasher.reset().update(xorb).finalize(32); +} + +/** + * HMAC: blake3_keyed_hash(key_bytes, hash_bytes) + * + * Both inputs are 32-byte Uint8Arrays. + * Matches Rust's `DataHash::hmac`. + * + * Uses a fresh hasher per call since the key varies. + */ +export function hmac(hash: Uint8Array, key: Uint8Array): Uint8Array { + return Hasher.newKeyed(key).update(hash).finalize(32); +} + +/** + * Verification hash for a range of chunk hashes. + * Concatenates all 32-byte hashes and applies blake3_keyed_hash + * with VERIFICATION_KEY. + * + * Matches Rust's `chunk_verification::range_hash_from_chunks`. + */ +export function verificationHash(chunkHashes: Uint8Array[]): Uint8Array { + const combined = new Uint8Array(chunkHashes.length * 32); + for (let i = 0; i < chunkHashes.length; i++) { + combined.set(chunkHashes[i], i * 32); + } + return verificationHasher.reset().update(combined).finalize(32); +} diff --git a/node_modules/@huggingface/xetchunk-wasm/src/index.ts b/node_modules/@huggingface/xetchunk-wasm/src/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..a62ffe22e1f3635180f2a655f00776af3a73107a --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/src/index.ts @@ -0,0 +1,3 @@ +export { createChunker, finalize, nextBlock, getChunks, hashToHex, hexToBytes, type Chunk } from "./xet-chunker.js"; +export { xorbHash } from "./xorb-hash.js"; +export { fileHash, hmac, verificationHash } from "./hash-utils.js"; diff --git a/node_modules/@huggingface/xetchunk-wasm/src/xet-chunker.ts b/node_modules/@huggingface/xetchunk-wasm/src/xet-chunker.ts new file mode 100644 index 0000000000000000000000000000000000000000..9c97e009049975fd417348014076c4107e6c5e11 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/src/xet-chunker.ts @@ -0,0 +1,244 @@ +import { Hasher } from "gearhash-jit"; +import { createKeyed, Hasher as Blake3Hasher } from "@huggingface/blake3-jit"; + +const TARGET_CHUNK_SIZE = 64 * 1024; // 64KB +const MINIMUM_CHUNK_DIVISOR = 8; +const MAXIMUM_CHUNK_MULTIPLIER = 2; +const HASH_WINDOW_SIZE = 64; + +const BLAKE3_DATA_KEY = new Uint8Array([ + 102, 151, 245, 119, 91, 149, 80, 222, 49, 53, 203, 172, 165, 151, 24, 28, 157, 228, 33, 16, 155, 235, 43, 88, 180, + 208, 176, 75, 147, 173, 242, 41, +]); + +export interface Chunk { + hash: Uint8Array; + length: number; +} + +interface NextResult { + chunk: Chunk | null; + bytesConsumed: number; +} + +class XetChunker { + private minimumChunk: number; + private maximumChunk: number; + private chunkBuf: Uint8Array; + private curChunkLen: number; + private gear: Hasher; + private blake3: Blake3Hasher; + + constructor(targetChunkSize: number = TARGET_CHUNK_SIZE) { + if (targetChunkSize <= 0) { + throw new Error("Target chunk size must be greater than 0"); + } + if ((targetChunkSize & (targetChunkSize - 1)) !== 0) { + throw new Error("Target chunk size must be a power of 2"); + } + if (targetChunkSize <= HASH_WINDOW_SIZE) { + throw new Error("Target chunk size must be greater than hash window size"); + } + if (targetChunkSize >= Number.MAX_SAFE_INTEGER) { + throw new Error("Target chunk size must be less than Number.MAX_SAFE_INTEGER"); + } + + let mask = BigInt(targetChunkSize - 1); + let leadingZeros = 0; + for (let i = 63; i >= 0; i--) { + if ((mask & (1n << BigInt(i))) !== 0n) { + break; + } + leadingZeros++; + } + mask = mask << BigInt(leadingZeros); + + const maximumChunk = targetChunkSize * MAXIMUM_CHUNK_MULTIPLIER; + + this.minimumChunk = targetChunkSize / MINIMUM_CHUNK_DIVISOR; + this.maximumChunk = maximumChunk; + this.chunkBuf = new Uint8Array(maximumChunk); + this.curChunkLen = 0; + this.gear = new Hasher(mask); + this.blake3 = Blake3Hasher.newKeyed(BLAKE3_DATA_KEY); + } + + /** + * Streaming entry point: accepts an arbitrary slice of data, accumulates + * it, and emits a chunk when a boundary (or max size) is reached. + * Data is copied into an internal buffer because it may span calls. + */ + next(data: Uint8Array, isFinal: boolean): NextResult { + const nBytes = data.length; + let createChunk = false; + let consumeLen = 0; + + if (nBytes !== 0) { + if (this.curChunkLen + HASH_WINDOW_SIZE < this.minimumChunk) { + const maxAdvance = Math.min(this.minimumChunk - this.curChunkLen - HASH_WINDOW_SIZE - 1, nBytes - consumeLen); + consumeLen += maxAdvance; + this.curChunkLen += maxAdvance; + } + + const readEnd = Math.min(nBytes, consumeLen + this.maximumChunk - this.curChunkLen); + + let bytesToNextBoundary: number; + const position = this.gear.nextMatch(data.subarray(consumeLen, readEnd)); + + if (position !== -1) { + bytesToNextBoundary = position; + createChunk = true; + } else { + bytesToNextBoundary = readEnd - consumeLen; + } + + if (bytesToNextBoundary + this.curChunkLen >= this.maximumChunk) { + bytesToNextBoundary = this.maximumChunk - this.curChunkLen; + createChunk = true; + } + + this.curChunkLen += bytesToNextBoundary; + consumeLen += bytesToNextBoundary; + + this.chunkBuf.set(data.subarray(0, consumeLen), this.curChunkLen - consumeLen); + } + + if (createChunk || (isFinal && this.curChunkLen > 0)) { + const chunkData = this.chunkBuf.subarray(0, this.curChunkLen); + const hash = this.blake3.reset().update(chunkData).finalize(32); + const chunk: Chunk = { + length: chunkData.length, + hash: hash, + }; + this.curChunkLen = 0; + this.gear.resetHash(); + return { + chunk, + bytesConsumed: consumeLen, + }; + } + + return { + chunk: null, + bytesConsumed: consumeLen, + }; + } + + /** + * Batch entry point: processes a large contiguous buffer and returns all + * complete chunks. Hashes directly from `data` — no intermediate copy + * to chunkBuf — for every chunk whose bytes are fully within `data`. + */ + nextBlock(data: Uint8Array, isFinal: boolean): Chunk[] { + const chunks: Chunk[] = []; + let pos = 0; + + // Drain any leftover from a previous nextBlock / next call. + while (pos < data.length && this.curChunkLen > 0) { + const result = this.next(data.subarray(pos), false); + if (result.chunk) chunks.push(result.chunk); + pos += result.bytesConsumed; + } + + const minSkip = this.minimumChunk > HASH_WINDOW_SIZE + ? this.minimumChunk - HASH_WINDOW_SIZE - 1 + : 0; + + while (pos < data.length) { + const chunkStart = pos; + const scanStart = Math.min(pos + minSkip, data.length); + const scanEnd = Math.min(data.length, pos + this.maximumChunk); + + const position = this.gear.nextMatch(data.subarray(scanStart, scanEnd)); + + let chunkEnd: number; + let foundBoundary: boolean; + + if (position !== -1 && scanStart + position - chunkStart <= this.maximumChunk) { + chunkEnd = scanStart + position; + foundBoundary = true; + } else if (scanEnd - chunkStart >= this.maximumChunk) { + chunkEnd = chunkStart + this.maximumChunk; + foundBoundary = true; + } else { + foundBoundary = false; + chunkEnd = scanEnd; + } + + if (foundBoundary) { + const hash = this.blake3.reset() + .update(data.subarray(chunkStart, chunkEnd)) + .finalize(32); + chunks.push({ length: chunkEnd - chunkStart, hash }); + pos = chunkEnd; + this.gear.resetHash(); + } else if (isFinal) { + const hash = this.blake3.reset() + .update(data.subarray(chunkStart)) + .finalize(32); + chunks.push({ length: data.length - chunkStart, hash }); + pos = data.length; + } else { + this.chunkBuf.set(data.subarray(chunkStart), 0); + this.curChunkLen = data.length - chunkStart; + pos = data.length; + } + } + + return chunks; + } + + finish(): Chunk | null { + if (this.curChunkLen > 0) { + const chunkData = this.chunkBuf.subarray(0, this.curChunkLen); + const hash = this.blake3.reset().update(chunkData).finalize(32); + const chunk: Chunk = { length: this.curChunkLen, hash }; + this.curChunkLen = 0; + this.gear.resetHash(); + return chunk; + } + return null; + } +} + +export function createChunker(targetChunkSize: number = TARGET_CHUNK_SIZE): XetChunker { + return new XetChunker(targetChunkSize); +} + +export function nextBlock(chunker: XetChunker, data: Uint8Array): Chunk[] { + return chunker.nextBlock(data, false); +} + +export function finalize(chunker: XetChunker): Chunk | null { + return chunker.finish(); +} + +export function getChunks(data: Uint8Array, targetChunkSize: number = TARGET_CHUNK_SIZE): Chunk[] { + const chunker = createChunker(targetChunkSize); + return chunker.nextBlock(data, true); +} + +export function hashToHex(hash: Uint8Array): string { + const view = new DataView(hash.buffer, hash.byteOffset, hash.byteLength); + const u64 = view.getBigUint64(0, true); + const u64_2 = view.getBigUint64(8, true); + const u64_3 = view.getBigUint64(16, true); + const u64_4 = view.getBigUint64(24, true); + + return ( + u64.toString(16).padStart(16, "0") + + u64_2.toString(16).padStart(16, "0") + + u64_3.toString(16).padStart(16, "0") + + u64_4.toString(16).padStart(16, "0") + ); +} + +export function hexToBytes(hex: string): Uint8Array { + const bytes = new Uint8Array(32); + const view = new DataView(bytes.buffer); + view.setBigUint64(0, BigInt("0x" + hex.slice(0, 16)), true); + view.setBigUint64(8, BigInt("0x" + hex.slice(16, 32)), true); + view.setBigUint64(16, BigInt("0x" + hex.slice(32, 48)), true); + view.setBigUint64(24, BigInt("0x" + hex.slice(48, 64)), true); + return bytes; +} diff --git a/node_modules/@huggingface/xetchunk-wasm/src/xorb-hash.ts b/node_modules/@huggingface/xetchunk-wasm/src/xorb-hash.ts new file mode 100644 index 0000000000000000000000000000000000000000..ecd251d3accc1e64a40bf8827f86da91c92405b2 --- /dev/null +++ b/node_modules/@huggingface/xetchunk-wasm/src/xorb-hash.ts @@ -0,0 +1,65 @@ +import { Hasher } from "@huggingface/blake3-jit"; +import type { Chunk } from "./xet-chunker.js"; +import { hashToHex } from "./xet-chunker.js"; + +const MEAN_CHUNK_PER_NODE = 4; + +const BLAKE3_NODE_KEY = new Uint8Array([ + 1, 126, 197, 199, 165, 71, 41, 150, 253, 148, 102, 102, 180, 138, 2, 230, 93, 221, 83, 111, 55, 199, 109, 210, 248, + 99, 82, 230, 74, 83, 113, 63, +]); + +const INDEX_OF_LAST_BYTE_OF_LAST_U64_IN_CHUNK_HASH = 3 * 8; + +const nodeHasher = Hasher.newKeyed(BLAKE3_NODE_KEY); + +export function xorbHash(chunks: Chunk[]): Uint8Array { + if (chunks.length === 0) { + return new Uint8Array(32); + } + + let currentChunks = chunks; + + while (currentChunks.length > 1) { + const nodes: Chunk[] = []; + let currentIndex = 0; + let numOfChildrenSoFar = 0; + + for (let i = 0; i < currentChunks.length; i++) { + if ( + i === currentChunks.length - 1 || + numOfChildrenSoFar === 2 * MEAN_CHUNK_PER_NODE || + (numOfChildrenSoFar >= 2 && + currentChunks[i].hash[INDEX_OF_LAST_BYTE_OF_LAST_U64_IN_CHUNK_HASH] % MEAN_CHUNK_PER_NODE === 0) + ) { + nodes.push(mergedHashOfSequence(currentChunks.slice(currentIndex, i + 1))); + currentIndex = i + 1; + numOfChildrenSoFar = 0; + } else { + numOfChildrenSoFar++; + } + } + currentChunks = nodes; + } + + return currentChunks[0].hash; +} + +/** + * Matches Rust's `merged_hash_of_sequence`: serializes each entry as + * "{hash_hex} : {length_decimal}\n" then hashes with BLAKE3_NODE_KEY. + */ +function mergedHashOfSequence(chunks: Chunk[]): Chunk { + let text = ""; + let totalLength = 0; + for (const chunk of chunks) { + text += hashToHex(chunk.hash) + " : " + chunk.length + "\n"; + totalLength += chunk.length; + } + const bytes = new Uint8Array(text.length); + for (let i = 0; i < text.length; i++) { + bytes[i] = text.charCodeAt(i); + } + const hash = nodeHasher.reset().update(bytes).finalize(32); + return { hash, length: totalLength }; +} diff --git a/node_modules/ansi-regex/index.d.ts b/node_modules/ansi-regex/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..2dbf6af2b6f3b5701c83caaea7ed2210192023a8 --- /dev/null +++ b/node_modules/ansi-regex/index.d.ts @@ -0,0 +1,37 @@ +declare namespace ansiRegex { + interface Options { + /** + Match only the first ANSI escape. + + @default false + */ + onlyFirst: boolean; + } +} + +/** +Regular expression for matching ANSI escape codes. + +@example +``` +import ansiRegex = require('ansi-regex'); + +ansiRegex().test('\u001B[4mcake\u001B[0m'); +//=> true + +ansiRegex().test('cake'); +//=> false + +'\u001B[4mcake\u001B[0m'.match(ansiRegex()); +//=> ['\u001B[4m', '\u001B[0m'] + +'\u001B[4mcake\u001B[0m'.match(ansiRegex({onlyFirst: true})); +//=> ['\u001B[4m'] + +'\u001B]8;;https://github.com\u0007click\u001B]8;;\u0007'.match(ansiRegex()); +//=> ['\u001B]8;;https://github.com\u0007', '\u001B]8;;\u0007'] +``` +*/ +declare function ansiRegex(options?: ansiRegex.Options): RegExp; + +export = ansiRegex; diff --git a/node_modules/ansi-regex/index.js b/node_modules/ansi-regex/index.js new file mode 100644 index 0000000000000000000000000000000000000000..616ff837d3ff01e028c208062fc699c7f1c8d418 --- /dev/null +++ b/node_modules/ansi-regex/index.js @@ -0,0 +1,10 @@ +'use strict'; + +module.exports = ({onlyFirst = false} = {}) => { + const pattern = [ + '[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)', + '(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))' + ].join('|'); + + return new RegExp(pattern, onlyFirst ? undefined : 'g'); +}; diff --git a/node_modules/ansi-regex/license b/node_modules/ansi-regex/license new file mode 100644 index 0000000000000000000000000000000000000000..e7af2f77107d73046421ef56c4684cbfdd3c1e89 --- /dev/null +++ b/node_modules/ansi-regex/license @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/node_modules/ansi-regex/package.json b/node_modules/ansi-regex/package.json new file mode 100644 index 0000000000000000000000000000000000000000..017f53116a9e2805ccb12efc3c7fcd7e4384735d --- /dev/null +++ b/node_modules/ansi-regex/package.json @@ -0,0 +1,55 @@ +{ + "name": "ansi-regex", + "version": "5.0.1", + "description": "Regular expression for matching ANSI escape codes", + "license": "MIT", + "repository": "chalk/ansi-regex", + "author": { + "name": "Sindre Sorhus", + "email": "sindresorhus@gmail.com", + "url": "sindresorhus.com" + }, + "engines": { + "node": ">=8" + }, + "scripts": { + "test": "xo && ava && tsd", + "view-supported": "node fixtures/view-codes.js" + }, + "files": [ + "index.js", + "index.d.ts" + ], + "keywords": [ + "ansi", + "styles", + "color", + "colour", + "colors", + "terminal", + "console", + "cli", + "string", + "tty", + "escape", + "formatting", + "rgb", + "256", + "shell", + "xterm", + "command-line", + "text", + "regex", + "regexp", + "re", + "match", + "test", + "find", + "pattern" + ], + "devDependencies": { + "ava": "^2.4.0", + "tsd": "^0.9.0", + "xo": "^0.25.3" + } +} diff --git a/node_modules/ansi-regex/readme.md b/node_modules/ansi-regex/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..4d848bc36f6b862e5db3effa59ebed515730284e --- /dev/null +++ b/node_modules/ansi-regex/readme.md @@ -0,0 +1,78 @@ +# ansi-regex + +> Regular expression for matching [ANSI escape codes](https://en.wikipedia.org/wiki/ANSI_escape_code) + + +## Install + +``` +$ npm install ansi-regex +``` + + +## Usage + +```js +const ansiRegex = require('ansi-regex'); + +ansiRegex().test('\u001B[4mcake\u001B[0m'); +//=> true + +ansiRegex().test('cake'); +//=> false + +'\u001B[4mcake\u001B[0m'.match(ansiRegex()); +//=> ['\u001B[4m', '\u001B[0m'] + +'\u001B[4mcake\u001B[0m'.match(ansiRegex({onlyFirst: true})); +//=> ['\u001B[4m'] + +'\u001B]8;;https://github.com\u0007click\u001B]8;;\u0007'.match(ansiRegex()); +//=> ['\u001B]8;;https://github.com\u0007', '\u001B]8;;\u0007'] +``` + + +## API + +### ansiRegex(options?) + +Returns a regex for matching ANSI escape codes. + +#### options + +Type: `object` + +##### onlyFirst + +Type: `boolean`
+Default: `false` *(Matches any ANSI escape codes in a string)* + +Match only the first ANSI escape. + + +## FAQ + +### Why do you test for codes not in the ECMA 48 standard? + +Some of the codes we run as a test are codes that we acquired finding various lists of non-standard or manufacturer specific codes. We test for both standard and non-standard codes, as most of them follow the same or similar format and can be safely matched in strings without the risk of removing actual string content. There are a few non-standard control codes that do not follow the traditional format (i.e. they end in numbers) thus forcing us to exclude them from the test because we cannot reliably match them. + +On the historical side, those ECMA standards were established in the early 90's whereas the VT100, for example, was designed in the mid/late 70's. At that point in time, control codes were still pretty ungoverned and engineers used them for a multitude of things, namely to activate hardware ports that may have been proprietary. Somewhere else you see a similar 'anarchy' of codes is in the x86 architecture for processors; there are a ton of "interrupts" that can mean different things on certain brands of processors, most of which have been phased out. + + +## Maintainers + +- [Sindre Sorhus](https://github.com/sindresorhus) +- [Josh Junon](https://github.com/qix-) + + +--- + +
+ + Get professional support for this package with a Tidelift subscription + +
+ + Tidelift helps make open source sustainable for maintainers while giving companies
assurances about security, maintenance, and licensing for their dependencies. + +
diff --git a/node_modules/cli-progress/CHANGES.md b/node_modules/cli-progress/CHANGES.md new file mode 100755 index 0000000000000000000000000000000000000000..bda0b88dee91f7db35464007828424616e9541da --- /dev/null +++ b/node_modules/cli-progress/CHANGES.md @@ -0,0 +1,201 @@ +## Branch 3.x ## + +### 3.12.0 ### + +* Added: option to override bar characters via instance options on `multibar.create()` - thanks to [Araxeus on GitHub](https://github.com/npkgz/cli-progress/pull/136) +* Added: example howto use multibars with different bar styles +* Bugfix: global terminal instance was not used for multibar elements which forces hard string trimming to terminal width - caused by default `linewrap=true` state of the terminal - thanks to [emmercm on GitHub](https://github.com/npkgz/cli-progress/issues/135) + +### 3.11.2 ### + +* Bugfix: disabled `gracefulExit` by default, because the default SIGINT/SIGTERM handlers of nodejs are removed + +### 3.11.1 ### + +* Bugfix: `MaxListenersExceededWarning` was triggered by `gracefulExit` handlers added in `v3.11.0` - thanks to [TychoTheTaco on GitHub](https://github.com/npkgz/cli-progress/pull/125) + +### 3.11.0 ### + +* Added: `log()` convenience method the multibar to enable custom logging output on top of the progress bars during operation +* Added: `gracefulExit` option (enabled by default) to stop the bars in case of `SIGINT` or `SIGTERM` - this restores most cursor settings before exiting +* Added: `progressCalculationRelative` option (disabled by default) to use the `startValue` as offset for the progress calculation and calculate the absolute progress from the difference given by `total-startValue` #121 +* Added: ability to pass bar options (overrides the global options) to `multibar.create` +* Bugfix: within a non-tty environment (e.g. CI/CD taskrunners) `multibar.create()` returns an undefined value in case `noTTYOutput` is not enabled #117 + +### 3.10.0 ### + +* Changed: foreground color of `preset.shades-grey` is set directly by ANSI codes +* Changed: example snippets are using `ansi-colors` library +* Bugfix: removed `colors` dependency due to some issues with the maintainer... see [Zalgo bomb](https://github.com/Marak/colors.js/issues/285#issuecomment-1008212640) + +### 3.9.1 ### + +* Bugfix: duration calculation doesn't work for bar restart scenarios - thanks to [autlaw on GitHub](https://github.com/npkgz/cli-progress/pull/101) + +### 3.9.0 ### + +* Added: exported standard formatter and format helper +* Added: example howto use multibars in synchronous context +* Changed: upper eta display limit to `1e7` (115days) #92 + +### 3.8.2 ### + +* Bugfix: bar duration not stopped until all bars have finished - thanks to [omjadas on GitHub](https://github.com/npkgz/cli-progress/issues/71) + +### 3.8.1 ### + +* Bugfix: percentage calculation used `Math.round` which caused incorrect values for edge cases - thanks to [OxCom on GitHub](https://github.com/npkgz/cli-progress/issues/70) + +### 3.8.0 ### + +* Changed: allow to pass payload as first argument to `increment()` with implicit delta of 1 - thanks to [ecdeveloper on GitHub](https://github.com/npkgz/cli-progress/pull/67) +* Changed: allow to pass payload as first argument to `update()` without updating bar value +* Bugfix: `formatTime` option ignored due to type - thanks to [omjadas on GitHub](https://github.com/npkgz/cli-progress/issues/68) + +### 3.7.0 ### + +* Added: asynchronous eta update for long running processes (optional) - feature [requested on GitHub](https://github.com/npkgz/cli-progress/issues/65) +* Added: method to trigger eta calculation without progress update + +### 3.6.1 ### + +* Bugfix: bar initialization overrides options within all instances - thanks to [BigBrainAFK on GitHub](https://github.com/npkgz/cli-progress/issues/64) + +### 3.6.0 ### + +* Added: support for custom time-format function +* Added: support for custom bar-format function +* Added: support for custom value-format function +* Added: auto-padding option to enforce fixed size of values - feature [requested on GitHub](https://github.com/npkgz/cli-progress/issues/60) +* Added: `barGlue` option to insert ascii escape sequences (e.g. for colorization) between the bar complete/incomplete elements - feature [requested on GitHub](https://github.com/npkgz/cli-progress/issues/53) +* Bugfix: `eta` value can be negative for multibars in case the bar is alredy completed + +### 3.5.0 ### + +* Added: support for events via [EventEmitter](https://nodejs.org/api/events.html) - feature [requested on GitHub](https://github.com/npkgz/cli-progress/pull/58) + +### 3.4.0 ### + +* Added: testsuites based on mocha - thanks to [on GitHub](https://github.com/npkgz/cli-progress/pull/49) +* Added: automatic tests via [Travis CI](https://travis-ci.org/) +* Bugfix: Fixing issues with falsy values in format which causes remdering artifacts - thanks to [on GitHub](https://github.com/npkgz/cli-progress/pull/49) +* Bugfix: documentation of the `stream` options was wrong - thanks to [ehmicky on GitHub](https://github.com/npkgz/cli-progress/pull/51) +* Changed: updated examples/syntax of `README.md` - thanks to [justsml on GitHub](https://github.com/npkgz/cli-progress/pull/50) + +### 3.3.1 ### + +* Bugifx: synchronous update may cause unexpected behaviour on multibars - limited to single bars +* Changed: renamed internal eta `push()` method to `update()` +* Changed: moved internal eta calculation call into `update()` + +### 3.3.0 ### + +* Added: option to pass custom formatters as callback via `options.format` +* Changed: replaced static placeholder code with generic regex (performance enhancement) + +### 3.2.0 ### + +* Added: `emptyOnZero` option to display total:0 bars as empty, not full - thanks to [nickcmaynard on GitHub](https://github.com/npkgz/cli-progress/pull/42) +* Bugfix: removed cursor save/restore calls for multibars - clearOnComplete might not work on all environments - thanks to [sayem314 onGitHub](https://github.com/npkgz/cli-progress/issues/40) + +### 3.1.0 ### + +* Added: notty support (interval/schedule based output) - feature requested [on GitHub](https://github.com/npkgz/cli-progress/issues/25) +* Added: `stopOnComplete` support within `MultiBar` - thanks to [Nox-404 on GitHub](https://github.com/npkgz/cli-progress/pull/35) +* Changed: initial throttel time of `MultiBar` is controlled by `fps` option instead of static `500ms` value +* Bugfix: provided option didn't take precedence over the preset as in v2 - thanks to [AxelTerizaki on GitHub](https://github.com/npkgz/cli-progress/issues/37) #37 + +### 3.0.0 ### + +* Added: multi-progressbar support - feature requested [on GitHub](https://github.com/npkgz/cli-progress/issues/26) +* Added: option `synchronousUpdate` to control the synchronized redraw during `update()` call (default=`true`) +* Changed: project split into multiple classes +* Changed: default cli progress output is written to `stdout` instead of `stderr` + +## Branch 2.x ## + +### 2.1.1 ### + +* Bugifx: preset object got altered by options - thanks to [rvalitov on GitHub](https://github.com/npkgz/cli-progress/issues/27) #27 + +### 2.1.0 ### + +* Added: `align` option to change the position of the progress bar (left, center, right) - thanks to [sidneys on GitHub](https://github.com/npkgz/cli-progress/pull/22) #22 +* Changed: ETA value of type `Infinity` is displayed as **INF**, `NaN` as **NULL** - feature requested by [AxelTerizaki on GitHub](https://github.com/npkgz/cli-progress/issues/21) #21 +* Changed: Limited the maximum ETA value to `100000s` (**INF** is displayed in this case) +* Changed: ETA calculation moved to own scope +* Bugfix: example `example-notty.php` was broken + +### 2.0.0 ### + +Upgrade is possible without any code modifications! requires **node.js 4** + +* Added: option `linewrap` to disable terminal line wrapping (default) +* Changed: requires **node.js >= 4** +* Changed: Native ES2015 class syntax +* Changed: renamed application entry file to `cli-progress.js` +* Changed: low-level terminal interactions are encapsulated within `Terminal` class +* Changed: terminal/cursor settings are restored after progress bar stopped +* Bugfix: used hex ascii escape sequences instaed of octals to avoid javascript errors in recent nodejs version +* Bugfix: disabled line wrapping by default to avoid multiple line breaks on small terminals (cut on the right) - reported by [puppeteer701 on GitHub](https://github.com/npkgz/cli-progress/issues/20) #20 + +## Branch 1.x ## + +### 1.8.0 ### +* Added: method `setTotal()` to manipulate the total value within running progress-bar - feature requested by [ReggaePanda on GitHub](https://github.com/npkgz/cli-progress/issues/19) #19 +* Changed: moved example file to `examples/` directory + +### 1.7.0 ### +* Added: payload argument to `increment()` - feature requested by [dsego on GitHub](https://github.com/npkgz/cli-progress/issues/18) #18 + +### 1.6.1 ### +* Bugfix: `roundTo` parameter was not set for `elapsedTime` calculation which caused raw float values within formatted time strings - thanks to [rekinyz on GitHub](https://github.com/npkgz/cli-progress/pull/16) #16 + +### 1.6.0 ### +* Added: Additional payload data which can be used as **custom-tokens** within the bar - thanks to [tobiasps on GitHub](https://github.com/npkgz/cli-progress/pull/15) #15 + +### 1.5.1 ### +* Bugfix: Progressbar cannot be initialized to 0% - thanks to [erikkallen on GitHub](https://github.com/npkgz/cli-progress/pull/14) #13 +* Bugfix: ETA was **NULL** in case the progress bar is initialized with (0/0) + +### 1.5.0 ### +* Added: **0** values for total/progress initialization are allowed - feature requested by [jfmmm on GitHub](https://github.com/npkgz/cli-progress/issues/11) #11 + +### 1.4.0 ### +* Added: **Preset/Theme support**. Different bar-styles can be loaded from internal library (in addition to full customization) +* Added: Dependency **colors** for colorized progress bars +* Added: Preset `legacy` +* Added: Preset `shades-classic` +* Added: Preset `shades-grey` +* Added: Preset `rect` + +### 1.3.1 ### +* Added: `example-notty` to test the behaviour of progress bar in non-interactive environments (input streams closed) +* Bugfix: `update()` throws an error in **non-tty** environments - reported by [Ognian on GitHub](https://github.com/npkgz/cli-progress/issues/9) #9 + +### 1.3.0 ### +* Added: `stopOnComplete` option to automatically call `stop()` when the value reaches the total - thanks to [lennym on GitHub](https://github.com/lennym) #7 + +### 1.2.0 ### +* Added: `increment()` method to increase the current progress relatively - thanks to [lennym on GitHub](https://github.com/lennym) #6 +* Added: ETA time formatting options (mm:ss, hh:mm, ss) - thanks to [lennym on GitHub](https://github.com/lennym) #5 +* Improvement: More accurate ETA calculation using linear estimation of last N values - thanks to [lennym on GitHub](https://github.com/lennym) #4 +* Bugfix: FPS calculation error which caused performance issues - thanks to [lennym on GitHub](https://github.com/lennym) #7 + +### 1.1.2 ### +* Bugfix: stdout.cursorTo/stdout.clearLine is not a function; replaced by `readline` - thanks to [remcoder on GitHub](https://github.com/npkgz/cli-progress/pull/2) + +### 1.1.1 ### +* Bugfix: Hide cursor options was enabled by default + +### 1.1.0 ### +* Added: Support for synchronous operations (interval has been replaced by timeout and throttle time) - feature requested [GitHub](https://github.com/npkgz/cli-progress/issues/1) +* Added: Synchronous Operation Example `example-synchronous.js` +* Added: Option to hide the cursor `options.hideCursor` - default set to false +* Changed: Improved ETA calculation + +### 1.0.1 ### +* Bugfix: the bar-size is limited to `options.barsize` - in some (numerical) situations it can be too long (n+1) + +### 1.0.0 ### +* Initial public release \ No newline at end of file diff --git a/node_modules/cli-progress/LICENSE.md b/node_modules/cli-progress/LICENSE.md new file mode 100755 index 0000000000000000000000000000000000000000..94842c596b40d8f8f1aa0f4c85420f2822f429a7 --- /dev/null +++ b/node_modules/cli-progress/LICENSE.md @@ -0,0 +1,24 @@ +The MIT License (X11 License) + +Copyright (c) 2015-2022 Andi Dittrich + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/node_modules/cli-progress/README.md b/node_modules/cli-progress/README.md new file mode 100755 index 0000000000000000000000000000000000000000..7cc676827d300afcf51f8a37615b1abbe9ca4918 --- /dev/null +++ b/node_modules/cli-progress/README.md @@ -0,0 +1,468 @@ +[![Build Status](https://travis-ci.org/npkgz/cli-progress.svg?branch=master)](https://travis-ci.org/npkgz/cli-progress) + +[Single Bar](#single-bar-mode) | [Multi Bar](#multi-bar-mode) | [Options](#options-1) | [Examples](examples/) | [Presets](presets/) | [Events](docs/events.md) + +CLI-Progress +============ +easy to use progress-bar for command-line/terminal applications + +![Demo](assets/cli-progress.gif) + +![Demo](assets/presets.png) + +Install +-------- + +```bash +$ yarn add cli-progress +$ npm install cli-progress --save +``` + +Features +-------- + +* **Simple**, **Robust** and **Easy** to use +* Full customizable output format (various placeholders are available) +* Single progressbar mode +* Multi progessbar mode +* Custom Bar Characters +* FPS limiter +* ETA calculation based on elapsed time +* Custom Tokens to display additional data (payload) within the bar +* TTY and NOTTY mode +* No callbacks required - designed as pure, external controlled UI widget +* Works in Asynchronous and Synchronous tasks +* Preset/Theme support +* Custom bar formatters (via callback) +* Logging during multibar operation + +Usage +------------ + +Multiple examples are available e.g. [example.js](https://github.com/npkgz/cli-progress/blob/master/examples/example.js) - just try it `$ node example.js` + +```js +const cliProgress = require('cli-progress'); + +// create a new progress bar instance and use shades_classic theme +const bar1 = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic); + +// start the progress bar with a total value of 200 and start value of 0 +bar1.start(200, 0); + +// update the current value in your application.. +bar1.update(100); + +// stop the progress bar +bar1.stop(); +``` + +Single Bar Mode +----------------------------------- + +![Demo](assets/presets.png) + +### Example ### + +```js +const cliProgress = require('cli-progress'); + +// note: you have to install this dependency manually since it's not required by cli-progress +const colors = require('ansi-colors'); + +// create new progress bar +const b1 = new cliProgress.SingleBar({ + format: 'CLI Progress |' + colors.cyan('{bar}') + '| {percentage}% || {value}/{total} Chunks || Speed: {speed}', + barCompleteChar: '\u2588', + barIncompleteChar: '\u2591', + hideCursor: true +}); + +// initialize the bar - defining payload token "speed" with the default value "N/A" +b1.start(200, 0, { + speed: "N/A" +}); + +// update values +b1.increment(); +b1.update(20); + +// stop the bar +b1.stop(); +``` + +### Constructor ### + +Initialize a new Progress bar. An instance can be used **multiple** times! it's not required to re-create it! + +```js +const cliProgress = require('cli-progress'); + +const = new cliProgress.SingleBar(options:object [, preset:object]); +``` + +#### Options #### + + +### ::start() ### + +Starts the progress bar and set the total and initial value + +```js +.start(totalValue:int, startValue:int [, payload:object = {}]); +``` + +### ::update() ### + +Sets the current progress value and optionally the payload with values of custom tokens as a second parameter. To update payload only, set currentValue to `null`. + +```js +.update([currentValue:int [, payload:object = {}]]); + +// update progress without altering value +.update([payload:object = {}]); +``` + +### ::increment() ### + +Increases the current progress value by a specified amount (default +1). Update payload optionally + +```js +.increment([delta:int [, payload:object = {}]]); + +// delta=1 assumed +.increment(payload:object = {}]); +``` + +### ::setTotal() ### + +Sets the total progress value while progressbar is active. Especially useful handling dynamic tasks. + +```js +.setTotal(totalValue:int); +``` + +### ::stop() ### + +Stops the progress bar and go to next line + +```js +.stop(); +``` + +### ::updateETA() ### + +Force eta calculation update (long running processes) without altering the progress values. + +Note: you may want to increase `etaBuffer` size - otherwise it can cause `INF` eta values in case the value didn't changed within the time series. + +```js +.updateETA(); +``` + + +Multi Bar Mode +----------------------------------- + +![Demo](assets/multibar.png) + +### Example ### + +```js +const cliProgress = require('cli-progress'); + +// create new container +const multibar = new cliProgress.MultiBar({ + clearOnComplete: false, + hideCursor: true, + format: ' {bar} | {filename} | {value}/{total}', +}, cliProgress.Presets.shades_grey); + +// add bars +const b1 = multibar.create(200, 0); +const b2 = multibar.create(1000, 0); + +// control bars +b1.increment(); +b2.update(20, {filename: "test1.txt"}); +b1.update(20, {filename: "helloworld.txt"}); + +// stop all bars +multibar.stop(); +``` + +### Constructor ### + +Initialize a new multiprogress container. Bars need to be added. The options/presets are used for each single bar! + +```js +const cliProgress = require('cli-progress'); + +const = new cliProgress.MultiBar(options:object [, preset:object]); +``` + +### ::create() ### + +Adds a new progress bar to the container and starts the bar. Returns regular `SingleBar` object which can be individually controlled. + +Additional `barOptions` can be passed directly to the [generic-bar](lib/generic-bar.js) to override the global options for a single bar instance. This can be useful to change the appearance of a single bar object. But be patient: this should only be used to override formats - DON'T try to set other global options like the terminal, synchronous flags, etc.. + +```js +const = .create(totalValue:int, startValue:int [, payload:object = {} [, barOptions:object = {}]]); +``` + +### ::remove() ### + +Removes an existing bar from the multi progress container. + +```js +.remove(:object); +``` + +### ::stop() ### + +Stops the all progress bars + +```js +.stop(); +``` + +### ::log() ### + +Outputs (buffered) content on top of the multibars during operation. + +**Notice: newline at the end is required** + +Example: [example-logging.js](examples/example-logging.js) + +```js +.log("Hello World\n"); +``` + +Options +----------------------------------- + +The following options can be changed + +- `format` (type:string|function) - progress bar output format @see format section +- `fps` (type:float) - the maximum update rate (default: 10) +- `stream` (type:stream) - output stream to use (default: `process.stderr`) +- `stopOnComplete` (type:boolean) - automatically call `stop()` when the value reaches the total (default: false) +- `clearOnComplete` (type:boolean) - clear the progress bar on complete / `stop()` call (default: false) +- `barsize` (type:int) - the length of the progress bar in chars (default: 40) +- `align` (type:char) - position of the progress bar - 'left' (default), 'right' or 'center' +- `barCompleteChar` (type:char) - character to use as "complete" indicator in the bar (default: "=") +- `barIncompleteChar` (type:char) - character to use as "incomplete" indicator in the bar (default: "-") +- `hideCursor` (type:boolean) - hide the cursor during progress operation; restored on complete (default: false) - pass `null` to keep terminal settings +- `linewrap` (type:boolean) - disable line wrapping (default: false) - pass `null` to keep terminal settings; pass `true` to add linebreaks automatically (not recommended) +- `gracefulExit` (type:boolean) - stop the bars in case of `SIGINT` or `SIGTERM` - this restores most cursor settings before exiting (default: `true`) +- `etaBuffer` (type:int) - number of updates with which to calculate the eta; higher numbers give a more stable eta (default: 10) +- `etaAsynchronousUpdate` (type:boolean) - trigger an eta calculation update during asynchronous rendering trigger using the current value - should only be used for long running processes in conjunction with lof `fps` values and large `etaBuffer` (default: false) +- `progressCalculationRelative` (type:boolean) - progress calculation uses `startValue` as zero-offset (default: false) +- `synchronousUpdate` (type:boolean) - trigger redraw during `update()` in case threshold time x2 is exceeded (default: true) - limited to single bar usage +- `noTTYOutput` (type:boolean) - enable scheduled output to notty streams - e.g. redirect to files (default: false) +- `notTTYSchedule` (type:int) - set the output schedule/interval for notty output in `ms` (default: 2000ms) +- `emptyOnZero` (type:boolean) - display progress bars with 'total' of zero(0) as empty, not full (default: false) +- `forceRedraw` (type:boolean) - trigger redraw on every frame even if progress remains the same; can be useful if progress bar gets overwritten by other concurrent writes to the terminal (default: false) +- `barGlue` (type:string) - a "glue" string between the complete and incomplete bar elements used to insert ascii control sequences for colorization (default: empty) - Note: in case you add visible "glue" characters the barsize will be increased by the length of the glue! +- `autopadding` (type: boolean) - add padding chars to formatted time and percentage to force fixed width (default: false) - Note: handled standard format functions! +- `autopaddingChar` (type: string) - the character sequence used for autopadding (default: " ") - Note: due to performance optimizations this value requires a length of 3 identical chars +- `formatBar` (type: function) - a custom bar formatter function which renders the bar-element (default: [format-bar.js](lib/format-bar.js)) +- `formatTime` (type: function) - a custom timer formatter function which renders the formatted time elements like `eta_formatted` and `duration-formatted` (default: [format-time.js](lib/format-time.js)) +- `formatValue` (type: function) - a custom value formatter function which renders all other values (default: [format-value.js](lib/format-value.js)) + +Events +----------------------------------- + +The classes extends [EventEmitter](https://nodejs.org/api/events.html) which allows you to hook into different events. + +See [event docs](docs/events.md) for detailed information + examples. + +Bar Formatting +----------------------------------- + +The progressbar can be customized by using the following build-in placeholders. They can be combined in any order. + +- `{bar}` - the progress bar, customizable by the options **barsize**, **barCompleteString** and **barIncompleteString** +- `{percentage}` - the current progress in percent (0-100) +- `{total}` - the end value +- `{value}` - the current value set by last `update()` call +- `{eta}` - expected time of accomplishment in seconds (limmited to 115days, otherwise INF is displayed) +- `{duration}` - elapsed time in seconds +- `{eta_formatted}` - expected time of accomplishment formatted into appropriate units +- `{duration_formatted}` - elapsed time formatted into appropriate units +- `{}` - the payload value identified by its key + +### Example ### + +```js +const opt = { + format: 'progress [{bar}] {percentage}% | ETA: {eta}s | {value}/{total}' +} +``` + +is rendered as + +``` +progress [========================================] 100% | ETA: 0s | 200/200 +``` + +Custom formatters +----------------------------------- + +Instead of a "static" format string it is also possible to pass a custom callback function as formatter. +For a full example (including params) take a look on `lib/formatter.js` + +### Example 1 ### + +```js +function formatter(options, params, payload){ + + // bar grows dynamically by current progress - no whitespaces are added + const bar = options.barCompleteString.substr(0, Math.round(params.progress*options.barsize)); + + // end value reached ? + // change color to green when finished + if (params.value >= params.total){ + return '# ' + colors.grey(payload.task) + ' ' + colors.green(params.value + '/' + params.total) + ' --[' + bar + ']-- '; + }else{ + return '# ' + payload.task + ' ' + colors.yellow(params.value + '/' + params.total) + ' --[' + bar + ']-- '; + } +} + +const opt = { + format: formatter +} +``` + +is rendered as + +``` +# Task 1 0/200 --[]-- +# Task 1 98/200 --[████████████████████]-- +# Task 1 200/200 --[████████████████████████████████████████]-- +``` + + +### Example 2 ### + +You can also access the default format functions to use them within your formatter: + +```js +const {TimeFormat, ValueFormat, BarFormat, Formatter} = require('cli-progess').Format; +... +``` + +Examples +--------------------------------------------- + +### Example 1 - Set Options ### + +```js +// change the progress characters +// set fps limit to 5 +// change the output stream and barsize +const bar = new _progress.Bar({ + barCompleteChar: '#', + barIncompleteChar: '.', + fps: 5, + stream: process.stdout, + barsize: 65, + position: 'center' +}); +``` + +### Example 2 - Change Styles defined by Preset ### + +```js +// uee shades preset +// change the barsize +const bar = new _progress.Bar({ + barsize: 65, + position: 'right' +}, _progress.Presets.shades_grey); +``` + +### Example 3 - Custom Payload ### + +The payload object keys should only contain keys matching standard `\w+` regex! + +```js +// create new progress bar with custom token "speed" +const bar = new _progress.Bar({ + format: 'progress [{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | Speed: {speed} kbit' +}); + +// initialize the bar - set payload token "speed" with the default value "N/A" +bar.start(200, 0, { + speed: "N/A" +}); + +// some code/update loop +// ... + +// update bar value. set custom token "speed" to 125 +bar.update(5, { + speed: '125' +}); + +// process finished +bar.stop(); +``` + +### Example 4 - Custom Presets ### + +**File** `myPreset.js` + +```js +const colors = require('ansi-colors'); + +module.exports = { + format: colors.red(' {bar}') + ' {percentage}% | ETA: {eta}s | {value}/{total} | Speed: {speed} kbit', + barCompleteChar: '\u2588', + barIncompleteChar: '\u2591' +}; +``` + +**Application** + +```js +const myPreset = require('./myPreset.js'); + +const bar = new _progress.Bar({ + barsize: 65 +}, myPreset); +``` + + +Presets/Themes +--------------------------------------------- + +Need a more modern appearance ? **cli-progress** supports predefined themes via presets. You are welcome to add your custom one :) + +But keep in mind that a lot of the "special-chars" rely on Unicode - it might not work as expected on legacy systems. + +### Default Presets ### + +The following presets are included by default + +* **legacy** - Styles as of cli-progress v1.3.0 +* **shades-classic** - Unicode background shades are used for the bar +* **shades-grey** - Unicode background shades with grey bar +* **rect** - Unicode Rectangles + + +Compatibility +--------------------------------------------- + +**cli-progress** is designed for linux/macOS/container applications which mostly providing standard compliant tty terminals/shells. In non-tty mode it is suitable to be used with logging daemons (cyclic output). + +It also works with PowerShell on Windows 10 - the legacy command prompt on outdated Windows versions won't work as expected and is not supported! + +Any Questions ? Report a Bug ? Enhancements ? +--------------------------------------------- +Please open a new issue on [GitHub](https://github.com/npkgz/cli-progress/issues) + +License +------- +CLI-Progress is OpenSource and licensed under the Terms of [The MIT License (X11)](http://opensource.org/licenses/MIT). You're welcome to [contribute](https://github.com/npkgz/cli-progress/blob/master/CONTRIBUTE.md)! diff --git a/node_modules/cli-progress/cli-progress.js b/node_modules/cli-progress/cli-progress.js new file mode 100755 index 0000000000000000000000000000000000000000..54ab3137bdcd1f4f7dd234e8b7c5181e78894b7c --- /dev/null +++ b/node_modules/cli-progress/cli-progress.js @@ -0,0 +1,21 @@ +const _SingleBar = require('./lib/single-bar'); +const _MultiBar = require('./lib/multi-bar'); +const _Presets = require('./presets/index'); +const _Formatter = require('./lib/formatter'); +const _defaultFormatValue = require('./lib/format-value'); +const _defaultFormatBar = require('./lib/format-bar'); +const _defaultFormatTime = require('./lib/format-time'); + +// sub-module access +module.exports = { + Bar: _SingleBar, + SingleBar: _SingleBar, + MultiBar: _MultiBar, + Presets: _Presets, + Format: { + Formatter: _Formatter, + BarFormat: _defaultFormatBar, + ValueFormat: _defaultFormatValue, + TimeFormat: _defaultFormatTime + } +}; \ No newline at end of file diff --git a/node_modules/cli-progress/lib/eta.js b/node_modules/cli-progress/lib/eta.js new file mode 100644 index 0000000000000000000000000000000000000000..4474f9936f55181f9baf328e9e9f1d992e656ee0 --- /dev/null +++ b/node_modules/cli-progress/lib/eta.js @@ -0,0 +1,73 @@ + +// ETA calculation +class ETA{ + + constructor(length, initTime, initValue){ + // size of eta buffer + this.etaBufferLength = length || 100; + + // eta buffer with initial values + this.valueBuffer = [initValue]; + this.timeBuffer = [initTime]; + + // eta time value + this.eta = '0'; + } + + // add new values to calculation buffer + update(time, value, total){ + this.valueBuffer.push(value); + this.timeBuffer.push(time); + + // trigger recalculation + this.calculate(total-value); + } + + // fetch estimated time + getTime(){ + return this.eta; + } + + // eta calculation - request number of remaining events + calculate(remaining){ + // get number of samples in eta buffer + const currentBufferSize = this.valueBuffer.length; + const buffer = Math.min(this.etaBufferLength, currentBufferSize); + + const v_diff = this.valueBuffer[currentBufferSize - 1] - this.valueBuffer[currentBufferSize - buffer]; + const t_diff = this.timeBuffer[currentBufferSize - 1] - this.timeBuffer[currentBufferSize - buffer]; + + // get progress per ms + const vt_rate = v_diff/t_diff; + + // strip past elements + this.valueBuffer = this.valueBuffer.slice(-this.etaBufferLength); + this.timeBuffer = this.timeBuffer.slice(-this.etaBufferLength); + + // eq: vt_rate *x = total + const eta = Math.ceil(remaining/vt_rate/1000); + + // check values + if (isNaN(eta)){ + this.eta = 'NULL'; + + // +/- Infinity --- NaN already handled + }else if (!isFinite(eta)){ + this.eta = 'INF'; + + // > 10M s ? - set upper display limit ~115days (1e7/60/60/24) + }else if (eta > 1e7){ + this.eta = 'INF'; + + // negative ? + }else if (eta < 0){ + this.eta = 0; + + }else{ + // assign + this.eta = eta; + } + } +} + +module.exports = ETA; \ No newline at end of file diff --git a/node_modules/cli-progress/lib/format-bar.js b/node_modules/cli-progress/lib/format-bar.js new file mode 100644 index 0000000000000000000000000000000000000000..ec7b82f624f4c8e284e46beefe3d3bba6704b3ec --- /dev/null +++ b/node_modules/cli-progress/lib/format-bar.js @@ -0,0 +1,11 @@ +// format bar +module.exports = function formatBar(progress, options){ + // calculate barsize + const completeSize = Math.round(progress*options.barsize); + const incompleteSize = options.barsize-completeSize; + + // generate bar string by stripping the pre-rendered strings + return options.barCompleteString.substr(0, completeSize) + + options.barGlue + + options.barIncompleteString.substr(0, incompleteSize); +} \ No newline at end of file diff --git a/node_modules/cli-progress/lib/format-time.js b/node_modules/cli-progress/lib/format-time.js new file mode 100644 index 0000000000000000000000000000000000000000..e419c67d218391fc5a62524185239b363e8a2aeb --- /dev/null +++ b/node_modules/cli-progress/lib/format-time.js @@ -0,0 +1,34 @@ +// default time format + +// format a number of seconds into hours and minutes as appropriate +module.exports = function formatTime(t, options, roundToMultipleOf){ + function round(input) { + if (roundToMultipleOf) { + return roundToMultipleOf * Math.round(input / roundToMultipleOf); + } else { + return input + } + } + + // leading zero padding + function autopadding(v){ + return (options.autopaddingChar + v).slice(-2); + } + + // > 1h ? + if (t > 3600) { + return autopadding(Math.floor(t / 3600)) + 'h' + autopadding(round((t % 3600) / 60)) + 'm'; + + // > 60s ? + } else if (t > 60) { + return autopadding(Math.floor(t / 60)) + 'm' + autopadding(round((t % 60))) + 's'; + + // > 10s ? + } else if (t > 10) { + return autopadding(round(t)) + 's'; + + // default: don't apply round to multiple + }else{ + return autopadding(t) + 's'; + } +} \ No newline at end of file diff --git a/node_modules/cli-progress/lib/format-value.js b/node_modules/cli-progress/lib/format-value.js new file mode 100644 index 0000000000000000000000000000000000000000..3b27607a97239245907eff7d55c3a307bf323c21 --- /dev/null +++ b/node_modules/cli-progress/lib/format-value.js @@ -0,0 +1,22 @@ +// default value format (apply autopadding) + +// format valueset +module.exports = function formatValue(v, options, type){ + // no autopadding ? passthrough + if (options.autopadding !== true){ + return v; + } + + // padding + function autopadding(value, length){ + return (options.autopaddingChar + value).slice(-length); + } + + switch (type){ + case 'percentage': + return autopadding(v, 3); + + default: + return v; + } +} \ No newline at end of file diff --git a/node_modules/cli-progress/lib/formatter.js b/node_modules/cli-progress/lib/formatter.js new file mode 100644 index 0000000000000000000000000000000000000000..8ca662de1e264b5ae4cc5906e47f010ba6a857d8 --- /dev/null +++ b/node_modules/cli-progress/lib/formatter.js @@ -0,0 +1,80 @@ +const _stringWidth = require('string-width'); +const _defaultFormatValue = require('./format-value'); +const _defaultFormatBar = require('./format-bar'); +const _defaultFormatTime = require('./format-time'); + +// generic formatter +module.exports = function defaultFormatter(options, params, payload){ + + // copy format string + let s = options.format; + + // custom time format set ? + const formatTime = options.formatTime || _defaultFormatTime; + + // custom value format set ? + const formatValue = options.formatValue || _defaultFormatValue; + + // custom bar format set ? + const formatBar = options.formatBar || _defaultFormatBar; + + // calculate progress in percent + const percentage = Math.floor(params.progress*100) + ''; + + // bar stopped and stopTime set ? + const stopTime = params.stopTime || Date.now(); + + // calculate elapsed time + const elapsedTime = Math.round((stopTime - params.startTime)/1000); + + // merges data from payload and calculated + const context = Object.assign({}, payload, { + bar: formatBar(params.progress, options), + + percentage: formatValue(percentage, options, 'percentage'), + total: formatValue(params.total, options, 'total'), + value: formatValue(params.value, options, 'value'), + + eta: formatValue(params.eta, options, 'eta'), + eta_formatted: formatTime(params.eta, options, 5), + + duration: formatValue(elapsedTime, options, 'duration'), + duration_formatted: formatTime(elapsedTime, options, 1) + }); + + // assign placeholder tokens + s = s.replace(/\{(\w+)\}/g, function(match, key){ + // key exists within payload/context + if (typeof context[key] !== 'undefined') { + return context[key]; + } + + // no changes to unknown values + return match; + }); + + // calculate available whitespace (2 characters margin of error) + const fullMargin = Math.max(0, params.maxWidth - _stringWidth(s) -2); + const halfMargin = Math.floor(fullMargin / 2); + + // distribute available whitespace according to position + switch (options.align) { + + // fill start-of-line with whitespaces + case 'right': + s = (fullMargin > 0) ? ' '.repeat(fullMargin) + s : s; + break; + + // distribute whitespaces to left+right + case 'center': + s = (halfMargin > 0) ? ' '.repeat(halfMargin) + s : s; + break; + + // default: left align, no additional whitespaces + case 'left': + default: + break; + } + + return s; +} diff --git a/node_modules/cli-progress/lib/generic-bar.js b/node_modules/cli-progress/lib/generic-bar.js new file mode 100755 index 0000000000000000000000000000000000000000..2e5d11ee83a860791fac30f72bda3a203337ba35 --- /dev/null +++ b/node_modules/cli-progress/lib/generic-bar.js @@ -0,0 +1,234 @@ +const _ETA = require('./eta'); +const _Terminal = require('./terminal'); +const _formatter = require('./formatter'); +const _options = require('./options'); +const _EventEmitter = require('events'); + +// Progress-Bar constructor +module.exports = class GenericBar extends _EventEmitter{ + + constructor(options){ + super(); + + // store options and assign derived ones (instance specific) + this.options = _options.assignDerivedOptions(options); + + // store terminal instance + this.terminal = (this.options.terminal) ? this.options.terminal : new _Terminal(this.options.stream); + + // the current bar value + this.value = 0; + + // bar start value (used for progress calculation) + this.startValue = 0; + + // the end value of the bar + this.total = 100; + + // last drawn string - only render on change! + this.lastDrawnString = null; + + // start time (used for eta calculation) + this.startTime = null; + + // stop time (used for duration calculation) + this.stopTime = null; + + // last update time + this.lastRedraw = Date.now(); + + // default eta calculator (will be re-create on start) + this.eta = new _ETA(this.options.etaBufferLength, 0, 0); + + // payload data + this.payload = {}; + + // progress bar active ? + this.isActive = false; + + // use default formatter or custom one ? + this.formatter = (typeof this.options.format === 'function') ? this.options.format : _formatter; + } + + // internal render function + render(forceRendering=false){ + + // formatter params + const params = { + progress: this.getProgress(), + eta: this.eta.getTime(), + startTime: this.startTime, + stopTime: this.stopTime, + total: this.total, + value: this.value, + maxWidth: this.terminal.getWidth() + }; + + // automatic eta update ? (long running processes) + if (this.options.etaAsynchronousUpdate){ + this.updateETA(); + } + + // format string + const s = this.formatter(this.options, params, this.payload); + + const forceRedraw = forceRendering || this.options.forceRedraw + // force redraw in notty-mode! + || (this.options.noTTYOutput && !this.terminal.isTTY()); + + // string changed ? only trigger redraw on change! + if (forceRedraw || this.lastDrawnString != s){ + // trigger event + this.emit('redraw-pre'); + + // set cursor to start of line + this.terminal.cursorTo(0, null); + + // write output + this.terminal.write(s); + + // clear to the right from cursor + this.terminal.clearRight(); + + // store string + this.lastDrawnString = s; + + // set last redraw time + this.lastRedraw = Date.now(); + + // trigger event + this.emit('redraw-post'); + } + } + + // start the progress bar + start(total, startValue, payload){ + // set initial values + this.value = startValue || 0; + this.total = (typeof total !== 'undefined' && total >= 0) ? total : 100; + + // set start value for progress calculation + this.startValue = (startValue || 0); + + // store payload (optional) + this.payload = payload || {}; + + // store start time for duration+eta calculation + this.startTime = Date.now(); + + // reset stop time for 're-start' scenario (used for duration calculation) + this.stopTime = null; + + // reset string line buffer (redraw detection) + this.lastDrawnString = ''; + + // initialize eta buffer + this.eta = new _ETA(this.options.etaBufferLength, this.startTime, this.value); + + // set flag + this.isActive = true; + + // start event + this.emit('start', total, startValue); + } + + // stop the bar + stop(){ + // set flag + this.isActive = false; + + // store stop timestamp to get total duration + this.stopTime = Date.now(); + + // stop event + this.emit('stop', this.total, this.value); + } + + // update the bar value + // update(value, payload) + // update(payload) + update(arg0, arg1 = {}){ + // value set ? + // update(value, [payload]); + if (typeof arg0 === 'number') { + // update value + this.value = arg0; + + // add new value; recalculate eta + this.eta.update(Date.now(), arg0, this.total); + } + + // extract payload + // update(value, payload) + // update(payload) + const payloadData = ((typeof arg0 === 'object') ? arg0 : arg1) || {}; + + // update event (before stop() is called) + this.emit('update', this.total, this.value); + + // merge payload + for (const key in payloadData){ + this.payload[key] = payloadData[key]; + } + + // limit reached ? autostop set ? + if (this.value >= this.getTotal() && this.options.stopOnComplete) { + this.stop(); + } + } + + // calculate the actual progress value + getProgress(){ + // calculate the normalized current progress + let progress = (this.value/this.total); + + // use relative progress calculation ? range between startValue and total is then used as 100% + // startValue (offset) is ignored for calculations + if (this.options.progressCalculationRelative){ + progress = (this.value-this.startValue)/(this.total-this.startValue); + } + + // handle NaN Errors caused by total=0. Set to complete in this case + if (isNaN(progress)){ + progress = (this.options && this.options.emptyOnZero) ? 0.0 : 1.0; + } + + // limiter + progress = Math.min(Math.max(progress, 0.0), 1.0); + + return progress; + } + + // update the bar value + // increment(delta, payload) + // increment(payload) + increment(arg0 = 1, arg1 = {}){ + // increment([payload]) => step=1 + // handle the use case when `step` is omitted but payload is passed + if (typeof arg0 === 'object') { + this.update(this.value + 1, arg0); + + // increment([step=1], [payload={}]) + }else{ + this.update(this.value + arg0, arg1); + } + } + + // get the total (limit) value + getTotal(){ + return this.total; + } + + // set the total (limit) value + setTotal(total){ + if (typeof total !== 'undefined' && total >= 0){ + this.total = total; + } + } + + // force eta calculation update (long running processes) + updateETA(){ + // add new value; recalculate eta + this.eta.update(Date.now(), this.value, this.total); + } +} diff --git a/node_modules/cli-progress/lib/multi-bar.js b/node_modules/cli-progress/lib/multi-bar.js new file mode 100644 index 0000000000000000000000000000000000000000..d40ccfc4032960ef42064d7952a2b7fe0edf6dcb --- /dev/null +++ b/node_modules/cli-progress/lib/multi-bar.js @@ -0,0 +1,250 @@ +const _Terminal = require('./terminal'); +const _BarElement = require('./generic-bar'); +const _options = require('./options'); +const _EventEmitter = require('events'); + +// Progress-Bar constructor +module.exports = class MultiBar extends _EventEmitter{ + + constructor(options, preset){ + super(); + + // list of bars + this.bars = []; + + // parse+store options + this.options = _options.parse(options, preset); + + // disable synchronous updates + this.options.synchronousUpdate = false; + + // store terminal instance + this.terminal = (this.options.terminal) ? this.options.terminal : new _Terminal(this.options.stream); + + // the update timer + this.timer = null; + + // progress bar active ? + this.isActive = false; + + // update interval + this.schedulingRate = (this.terminal.isTTY() ? this.options.throttleTime : this.options.notTTYSchedule); + + // logging output buffer + this.loggingBuffer = []; + + // callback used for gracefulExit + this.sigintCallback = null; + } + + // add a new bar to the stack + create(total, startValue, payload, barOptions={}){ + // create new bar element and merge global options + overrides + // use the same global terminal instance for all instances + const bar = new _BarElement(Object.assign( + {}, + + // global options + this.options, + + // terminal instance + { + terminal: this.terminal + }, + + // overrides + barOptions, + )); + + // store bar + this.bars.push(bar); + + // progress updates are only visible in TTY mode! + if (this.options.noTTYOutput === false && this.terminal.isTTY() === false){ + return bar; + } + + // add handler to restore cursor settings (stop the bar) on SIGINT/SIGTERM ? + if (this.sigintCallback === null && this.options.gracefulExit){ + this.sigintCallback = this.stop.bind(this); + process.once('SIGINT', this.sigintCallback); + process.once('SIGTERM', this.sigintCallback); + } + + // multiprogress already active ? + if (!this.isActive){ + // hide the cursor ? + if (this.options.hideCursor === true){ + this.terminal.cursor(false); + } + + // disable line wrapping ? + if (this.options.linewrap === false){ + this.terminal.lineWrapping(false); + } + + // initialize update timer + this.timer = setTimeout(this.update.bind(this), this.schedulingRate); + } + + // set flag + this.isActive = true; + + // start progress bar + bar.start(total, startValue, payload); + + // trigger event + this.emit('start'); + + // return new instance + return bar; + } + + // remove a bar from the stack + remove(bar){ + // find element + const index = this.bars.indexOf(bar); + + // element found ? + if (index < 0){ + return false; + } + + // remove element + this.bars.splice(index, 1); + + // force update + this.update(); + + // clear bottom + this.terminal.newline(); + this.terminal.clearBottom(); + + return true; + } + + // internal update routine + update(){ + // stop timer + if (this.timer){ + clearTimeout(this.timer); + this.timer = null; + } + + // trigger event + this.emit('update-pre'); + + // reset cursor + this.terminal.cursorRelativeReset(); + + // trigger event + this.emit('redraw-pre'); + + // content within logging buffer ? + if (this.loggingBuffer.length > 0){ + this.terminal.clearLine(); + + // flush logging buffer and write content to terminal + while (this.loggingBuffer.length > 0){ + this.terminal.write(this.loggingBuffer.shift(), true); + } + } + + // update each bar + for (let i=0; i< this.bars.length; i++){ + // add new line ? + if (i > 0){ + this.terminal.newline(); + } + + // render + this.bars[i].render(); + } + + // trigger event + this.emit('redraw-post'); + + // add new line in notty mode! + if (this.options.noTTYOutput && this.terminal.isTTY() === false){ + this.terminal.newline(); + this.terminal.newline(); + } + + // next update + this.timer = setTimeout(this.update.bind(this), this.schedulingRate); + + // trigger event + this.emit('update-post'); + + // stop if stopOnComplete and all bars stopped + if (this.options.stopOnComplete && !this.bars.find(bar => bar.isActive)) { + this.stop(); + } + } + + stop(){ + + // stop timer + clearTimeout(this.timer); + this.timer = null; + + // remove sigint listener + if (this.sigintCallback){ + process.removeListener('SIGINT', this.sigintCallback); + process.removeListener('SIGTERM', this.sigintCallback); + this.sigintCallback = null; + } + + // set flag + this.isActive = false; + + // cursor hidden ? + if (this.options.hideCursor === true){ + this.terminal.cursor(true); + } + + // re-enable line wrpaping ? + if (this.options.linewrap === false){ + this.terminal.lineWrapping(true); + } + + // reset cursor + this.terminal.cursorRelativeReset(); + + // trigger event + this.emit('stop-pre-clear'); + + // clear line on complete ? + if (this.options.clearOnComplete){ + // clear all bars + this.terminal.clearBottom(); + + // or show final progress ? + }else{ + // update each bar + for (let i=0; i< this.bars.length; i++){ + // add new line ? + if (i > 0){ + this.terminal.newline(); + } + + // trigger final rendering + this.bars[i].render(); + + // stop + this.bars[i].stop(); + } + + // new line on complete + this.terminal.newline(); + } + + // trigger event + this.emit('stop'); + } + + log(s){ + // push content into logging buffer + this.loggingBuffer.push(s); + } +} diff --git a/node_modules/cli-progress/lib/options.js b/node_modules/cli-progress/lib/options.js new file mode 100644 index 0000000000000000000000000000000000000000..4671d4ad76eca59557266076d3a19d2f66c0d3a6 --- /dev/null +++ b/node_modules/cli-progress/lib/options.js @@ -0,0 +1,110 @@ +// utility to merge defaults +function mergeOption(v, defaultValue){ + if (typeof v === 'undefined' || v === null){ + return defaultValue; + }else{ + return v; + } +} + +module.exports = { + // set global options + parse: function parse(rawOptions, preset){ + + // options storage + const options = {}; + + // merge preset + const opt = Object.assign({}, preset, rawOptions); + + // the max update rate in fps (redraw will only triggered on value change) + options.throttleTime = 1000 / (mergeOption(opt.fps, 10)); + + // the output stream to write on + options.stream = mergeOption(opt.stream, process.stderr); + + // external terminal provided ? + options.terminal = mergeOption(opt.terminal, null); + + // clear on finish ? + options.clearOnComplete = mergeOption(opt.clearOnComplete, false); + + // stop on finish ? + options.stopOnComplete = mergeOption(opt.stopOnComplete, false); + + // size of the progressbar in chars + options.barsize = mergeOption(opt.barsize, 40); + + // position of the progress bar - 'left' (default), 'right' or 'center' + options.align = mergeOption(opt.align, 'left'); + + // hide the cursor ? + options.hideCursor = mergeOption(opt.hideCursor, false); + + // disable linewrapping ? + options.linewrap = mergeOption(opt.linewrap, false); + + // glue sequence (control chars) between bar elements ? + options.barGlue = mergeOption(opt.barGlue, ''); + + // bar chars + options.barCompleteChar = mergeOption(opt.barCompleteChar, '='); + options.barIncompleteChar = mergeOption(opt.barIncompleteChar, '-'); + + // the bar format + options.format = mergeOption(opt.format, 'progress [{bar}] {percentage}% | ETA: {eta}s | {value}/{total}'); + + // external time-format provided ? + options.formatTime = mergeOption(opt.formatTime, null); + + // external value-format provided ? + options.formatValue = mergeOption(opt.formatValue, null); + + // external bar-format provided ? + options.formatBar = mergeOption(opt.formatBar, null); + + // the number of results to average ETA over + options.etaBufferLength = mergeOption(opt.etaBuffer, 10); + + // automatic eta updates based on fps + options.etaAsynchronousUpdate = mergeOption(opt.etaAsynchronousUpdate, false); + + // progress calculation relative to start value ? default start at 0 + options.progressCalculationRelative = mergeOption(opt.progressCalculationRelative, false); + + // allow synchronous updates ? + options.synchronousUpdate = mergeOption(opt.synchronousUpdate, true); + + // notty mode + options.noTTYOutput = mergeOption(opt.noTTYOutput, false); + + // schedule - 2s + options.notTTYSchedule = mergeOption(opt.notTTYSchedule, 2000); + + // emptyOnZero - false + options.emptyOnZero = mergeOption(opt.emptyOnZero, false); + + // force bar redraw even if progress did not change + options.forceRedraw = mergeOption(opt.forceRedraw, false); + + // automated padding to fixed width ? + options.autopadding = mergeOption(opt.autopadding, false); + + // stop bar on SIGINT/SIGTERM to restore cursor settings ? + options.gracefulExit = mergeOption(opt.gracefulExit, false); + + return options; + }, + + // derived options: instance specific, has to be created for every bar element + assignDerivedOptions: function assignDerivedOptions(options){ + // pre-render bar strings (performance) + options.barCompleteString = options.barCompleteChar.repeat(options.barsize + 1); + options.barIncompleteString = options.barIncompleteChar.repeat(options.barsize + 1); + + // autopadding character - empty in case autopadding is disabled + options.autopaddingChar = options.autopadding ? mergeOption(options.autopaddingChar, ' ') : ''; + + return options; + } +}; \ No newline at end of file diff --git a/node_modules/cli-progress/lib/single-bar.js b/node_modules/cli-progress/lib/single-bar.js new file mode 100644 index 0000000000000000000000000000000000000000..e8235e42d1c59503d4da251090e557fdb21a4b67 --- /dev/null +++ b/node_modules/cli-progress/lib/single-bar.js @@ -0,0 +1,141 @@ +const _GenericBar = require('./generic-bar'); +const _options = require('./options'); + +// Progress-Bar constructor +module.exports = class SingleBar extends _GenericBar{ + + constructor(options, preset){ + super(_options.parse(options, preset)); + + // the update timer + this.timer = null; + + // disable synchronous updates in notty mode + if (this.options.noTTYOutput && this.terminal.isTTY() === false){ + this.options.synchronousUpdate = false; + } + + // update interval + this.schedulingRate = (this.terminal.isTTY() ? this.options.throttleTime : this.options.notTTYSchedule); + + // callback used for gracefulExit + this.sigintCallback = null; + } + + // internal render function + render(){ + // stop timer + if (this.timer){ + clearTimeout(this.timer); + this.timer = null; + } + + // run internal rendering + super.render(); + + // add new line in notty mode! + if (this.options.noTTYOutput && this.terminal.isTTY() === false){ + this.terminal.newline(); + } + + // next update + this.timer = setTimeout(this.render.bind(this), this.schedulingRate); + } + + update(current, payload){ + // timer inactive ? + if (!this.timer) { + return; + } + + super.update(current, payload); + + // trigger synchronous update ? + // check for throttle time + if (this.options.synchronousUpdate && (this.lastRedraw + this.options.throttleTime*2) < Date.now()){ + // force update + this.render(); + } + } + + // start the progress bar + start(total, startValue, payload){ + // progress updates are only visible in TTY mode! + if (this.options.noTTYOutput === false && this.terminal.isTTY() === false){ + return; + } + + // add handler to restore cursor settings (stop the bar) on SIGINT/SIGTERM ? + if (this.sigintCallback === null && this.options.gracefulExit){ + this.sigintCallback = this.stop.bind(this); + process.once('SIGINT', this.sigintCallback); + process.once('SIGTERM', this.sigintCallback); + } + + // save current cursor settings + this.terminal.cursorSave(); + + // hide the cursor ? + if (this.options.hideCursor === true){ + this.terminal.cursor(false); + } + + // disable line wrapping ? + if (this.options.linewrap === false){ + this.terminal.lineWrapping(false); + } + + // initialize bar + super.start(total, startValue, payload); + + // redraw on start! + this.render(); + } + + // stop the bar + stop(){ + // timer inactive ? + if (!this.timer) { + return; + } + + // remove sigint listener + if (this.sigintCallback){ + process.removeListener('SIGINT', this.sigintCallback); + process.removeListener('SIGTERM', this.sigintCallback); + this.sigintCallback = null; + } + + // trigger final rendering + this.render(); + + // restore state + super.stop(); + + // stop timer + clearTimeout(this.timer); + this.timer = null; + + // cursor hidden ? + if (this.options.hideCursor === true){ + this.terminal.cursor(true); + } + + // re-enable line wrapping ? + if (this.options.linewrap === false){ + this.terminal.lineWrapping(true); + } + + // restore cursor on complete (position + settings) + this.terminal.cursorRestore(); + + // clear line on complete ? + if (this.options.clearOnComplete){ + this.terminal.cursorTo(0, null); + this.terminal.clearLine(); + }else{ + // new line on complete + this.terminal.newline(); + } + } +} \ No newline at end of file diff --git a/node_modules/cli-progress/lib/terminal.js b/node_modules/cli-progress/lib/terminal.js new file mode 100644 index 0000000000000000000000000000000000000000..7445563b7a56a1a8e5b109be8f8c1bb8b2faebeb --- /dev/null +++ b/node_modules/cli-progress/lib/terminal.js @@ -0,0 +1,162 @@ +const _readline = require('readline'); + +// low-level terminal interactions +class Terminal{ + + constructor(outputStream){ + this.stream = outputStream; + + // default: line wrapping enabled + this.linewrap = true; + + // current, relative y position + this.dy = 0; + } + + // save cursor position + settings + cursorSave(){ + if (!this.stream.isTTY){ + return; + } + + // save position + this.stream.write('\x1B7'); + } + + // restore last cursor position + settings + cursorRestore(){ + if (!this.stream.isTTY){ + return; + } + + // restore cursor + this.stream.write('\x1B8'); + } + + // show/hide cursor + cursor(enabled){ + if (!this.stream.isTTY){ + return; + } + + if (enabled){ + this.stream.write('\x1B[?25h'); + }else{ + this.stream.write('\x1B[?25l'); + } + } + + // change cursor positionn + cursorTo(x=null, y=null){ + if (!this.stream.isTTY){ + return; + } + + // move cursor absolute + _readline.cursorTo(this.stream, x, y); + } + + // change relative cursor position + cursorRelative(dx=null, dy=null){ + if (!this.stream.isTTY){ + return; + } + + // store current position + this.dy = this.dy + dy; + + // move cursor relative + _readline.moveCursor(this.stream, dx, dy); + } + + // relative reset + cursorRelativeReset(){ + if (!this.stream.isTTY){ + return; + } + + // move cursor to initial line + _readline.moveCursor(this.stream, 0, -this.dy); + + // first char + _readline.cursorTo(this.stream, 0, null); + + // reset counter + this.dy = 0; + } + + // clear to the right from cursor + clearRight(){ + if (!this.stream.isTTY){ + return; + } + + _readline.clearLine(this.stream, 1); + } + + // clear the full line + clearLine(){ + if (!this.stream.isTTY){ + return; + } + + _readline.clearLine(this.stream, 0); + } + + // clear everyting beyond the current line + clearBottom(){ + if (!this.stream.isTTY){ + return; + } + + _readline.clearScreenDown(this.stream); + } + + // add new line; increment counter + newline(){ + this.stream.write('\n'); + this.dy++; + } + + // write content to output stream + // @TODO use string-width to strip length + write(s, rawWrite=false){ + // line wrapping enabled ? trim output + // this is just a fallback mechanism in case user enabled line-wrapping via options or set it to auto + if (this.linewrap === true && rawWrite === false){ + this.stream.write(s.substr(0, this.getWidth())); + + // standard behaviour with disabled linewrapping + }else{ + this.stream.write(s); + } + } + + // control line wrapping + lineWrapping(enabled){ + if (!this.stream.isTTY){ + return; + } + + // store state + this.linewrap = enabled; + if (enabled){ + this.stream.write('\x1B[?7h'); + }else{ + this.stream.write('\x1B[?7l'); + } + } + + // tty environment ? + isTTY(){ + return (this.stream.isTTY === true); + } + + // get terminal width + getWidth(){ + // set max width to 80 in tty-mode and 200 in notty-mode + return this.stream.columns || (this.stream.isTTY ? 80 : 200); + } +} + +module.exports = Terminal; diff --git a/node_modules/cli-progress/package.json b/node_modules/cli-progress/package.json new file mode 100755 index 0000000000000000000000000000000000000000..2073db817b0fe347d8f9872e8e407aa85ce00b8d --- /dev/null +++ b/node_modules/cli-progress/package.json @@ -0,0 +1,45 @@ +{ + "name": "cli-progress", + "version": "3.12.0", + "description": "easy to use progress-bar for command-line/terminal applications", + "keywords": [ + "cli", + "tty", + "terminal", + "progress", + "progressbar", + "multibar", + "bar", + "status", + "statusbar", + "utility", + "widget" + ], + "homepage": "https://github.com/npkgz/cli-progress", + "bugs": "https://github.com/npkgz/cli-progress/issues", + "repository": "npkgz/cli-progress", + "files": [ + "cli-progress.js", + "lib/", + "presets/" + ], + "scripts": { + "lint": "eslint lib/**.js", + "pretest": "npm run lint", + "test": "mocha test/**/*.test.js" + }, + "engines": { + "node": ">=4" + }, + "main": "./cli-progress.js", + "author": "Andi Dittrich (https://andidittrich.com)", + "license": "MIT", + "dependencies": { + "string-width": "^4.2.3" + }, + "devDependencies": { + "eslint": "^8.14.0", + "eslint-config-aenondynamics": "^0.2.0", + "mocha": "^9.2.2" + } +} diff --git a/node_modules/cli-progress/presets/index.js b/node_modules/cli-progress/presets/index.js new file mode 100644 index 0000000000000000000000000000000000000000..471831d32b18d17a614fb921355337428887f1a2 --- /dev/null +++ b/node_modules/cli-progress/presets/index.js @@ -0,0 +1,11 @@ +const _legacy = require('./legacy'); +const _shades_classic = require('./shades-classic'); +const _shades_grey = require('./shades-grey'); +const _rect = require('./rect'); + +module.exports = { + legacy: _legacy, + shades_classic: _shades_classic, + shades_grey: _shades_grey, + rect: _rect +}; \ No newline at end of file diff --git a/node_modules/cli-progress/presets/legacy.js b/node_modules/cli-progress/presets/legacy.js new file mode 100755 index 0000000000000000000000000000000000000000..86832db86fa3ab9fade0ff7da9083f543e29c34a --- /dev/null +++ b/node_modules/cli-progress/presets/legacy.js @@ -0,0 +1,6 @@ +// cli-progress legacy style as of 1.x +module.exports = { + format: 'progress [{bar}] {percentage}% | ETA: {eta}s | {value}/{total}', + barCompleteChar: '=', + barIncompleteChar: '-' +}; \ No newline at end of file diff --git a/node_modules/cli-progress/presets/rect.js b/node_modules/cli-progress/presets/rect.js new file mode 100755 index 0000000000000000000000000000000000000000..44b1233c9cf92ae24a16be79059986af13d6f041 --- /dev/null +++ b/node_modules/cli-progress/presets/rect.js @@ -0,0 +1,5 @@ +module.exports = { + format: ' {bar}\u25A0 {percentage}% | ETA: {eta}s | {value}/{total}', + barCompleteChar: '\u25A0', + barIncompleteChar: ' ' +}; \ No newline at end of file diff --git a/node_modules/cli-progress/presets/shades-classic.js b/node_modules/cli-progress/presets/shades-classic.js new file mode 100755 index 0000000000000000000000000000000000000000..4cfc4423e43a8af8010f83648c8544fabaf40a31 --- /dev/null +++ b/node_modules/cli-progress/presets/shades-classic.js @@ -0,0 +1,6 @@ +// cli-progress legacy style as of 1.x +module.exports = { + format: ' {bar} {percentage}% | ETA: {eta}s | {value}/{total}', + barCompleteChar: '\u2588', + barIncompleteChar: '\u2591' +}; \ No newline at end of file diff --git a/node_modules/cli-progress/presets/shades-grey.js b/node_modules/cli-progress/presets/shades-grey.js new file mode 100755 index 0000000000000000000000000000000000000000..cd618abca474c07376ccd1f283c95df50a6e4984 --- /dev/null +++ b/node_modules/cli-progress/presets/shades-grey.js @@ -0,0 +1,7 @@ + +// cli-progress legacy style as of 1.x +module.exports = { + format: ' \u001b[90m{bar}\u001b[0m {percentage}% | ETA: {eta}s | {value}/{total}', + barCompleteChar: '\u2588', + barIncompleteChar: '\u2591' +}; \ No newline at end of file diff --git a/node_modules/emoji-regex/LICENSE-MIT.txt b/node_modules/emoji-regex/LICENSE-MIT.txt new file mode 100644 index 0000000000000000000000000000000000000000..a41e0a7ef970ecdd83d82cd99bda97b22077bc62 --- /dev/null +++ b/node_modules/emoji-regex/LICENSE-MIT.txt @@ -0,0 +1,20 @@ +Copyright Mathias Bynens + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/node_modules/emoji-regex/README.md b/node_modules/emoji-regex/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f10e1733350471c85b8d85d6370978b03e5fabfb --- /dev/null +++ b/node_modules/emoji-regex/README.md @@ -0,0 +1,73 @@ +# emoji-regex [![Build status](https://travis-ci.org/mathiasbynens/emoji-regex.svg?branch=master)](https://travis-ci.org/mathiasbynens/emoji-regex) + +_emoji-regex_ offers a regular expression to match all emoji symbols (including textual representations of emoji) as per the Unicode Standard. + +This repository contains a script that generates this regular expression based on [the data from Unicode v12](https://github.com/mathiasbynens/unicode-12.0.0). Because of this, the regular expression can easily be updated whenever new emoji are added to the Unicode standard. + +## Installation + +Via [npm](https://www.npmjs.com/): + +```bash +npm install emoji-regex +``` + +In [Node.js](https://nodejs.org/): + +```js +const emojiRegex = require('emoji-regex'); +// Note: because the regular expression has the global flag set, this module +// exports a function that returns the regex rather than exporting the regular +// expression itself, to make it impossible to (accidentally) mutate the +// original regular expression. + +const text = ` +\u{231A}: ⌚ default emoji presentation character (Emoji_Presentation) +\u{2194}\u{FE0F}: ↔️ default text presentation character rendered as emoji +\u{1F469}: 👩 emoji modifier base (Emoji_Modifier_Base) +\u{1F469}\u{1F3FF}: 👩🏿 emoji modifier base followed by a modifier +`; + +const regex = emojiRegex(); +let match; +while (match = regex.exec(text)) { + const emoji = match[0]; + console.log(`Matched sequence ${ emoji } — code points: ${ [...emoji].length }`); +} +``` + +Console output: + +``` +Matched sequence ⌚ — code points: 1 +Matched sequence ⌚ — code points: 1 +Matched sequence ↔️ — code points: 2 +Matched sequence ↔️ — code points: 2 +Matched sequence 👩 — code points: 1 +Matched sequence 👩 — code points: 1 +Matched sequence 👩🏿 — code points: 2 +Matched sequence 👩🏿 — code points: 2 +``` + +To match emoji in their textual representation as well (i.e. emoji that are not `Emoji_Presentation` symbols and that aren’t forced to render as emoji by a variation selector), `require` the other regex: + +```js +const emojiRegex = require('emoji-regex/text.js'); +``` + +Additionally, in environments which support ES2015 Unicode escapes, you may `require` ES2015-style versions of the regexes: + +```js +const emojiRegex = require('emoji-regex/es2015/index.js'); +const emojiRegexText = require('emoji-regex/es2015/text.js'); +``` + +## Author + +| [![twitter/mathias](https://gravatar.com/avatar/24e08a9ea84deb17ae121074d0f17125?s=70)](https://twitter.com/mathias "Follow @mathias on Twitter") | +|---| +| [Mathias Bynens](https://mathiasbynens.be/) | + +## License + +_emoji-regex_ is available under the [MIT](https://mths.be/mit) license. diff --git a/node_modules/emoji-regex/es2015/index.js b/node_modules/emoji-regex/es2015/index.js new file mode 100644 index 0000000000000000000000000000000000000000..b4cf3dcd389935061d82cd4f2d9da78c77a5b088 --- /dev/null +++ b/node_modules/emoji-regex/es2015/index.js @@ -0,0 +1,6 @@ +"use strict"; + +module.exports = () => { + // https://mths.be/emoji + return /\u{1F3F4}\u{E0067}\u{E0062}(?:\u{E0065}\u{E006E}\u{E0067}|\u{E0073}\u{E0063}\u{E0074}|\u{E0077}\u{E006C}\u{E0073})\u{E007F}|\u{1F468}(?:\u{1F3FC}\u200D(?:\u{1F91D}\u200D\u{1F468}\u{1F3FB}|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F3FF}\u200D(?:\u{1F91D}\u200D\u{1F468}[\u{1F3FB}-\u{1F3FE}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F3FE}\u200D(?:\u{1F91D}\u200D\u{1F468}[\u{1F3FB}-\u{1F3FD}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F3FD}\u200D(?:\u{1F91D}\u200D\u{1F468}[\u{1F3FB}\u{1F3FC}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u200D(?:\u2764\uFE0F\u200D(?:\u{1F48B}\u200D)?\u{1F468}|[\u{1F468}\u{1F469}]\u200D(?:\u{1F466}\u200D\u{1F466}|\u{1F467}\u200D[\u{1F466}\u{1F467}])|\u{1F466}\u200D\u{1F466}|\u{1F467}\u200D[\u{1F466}\u{1F467}]|[\u{1F468}\u{1F469}]\u200D[\u{1F466}\u{1F467}]|[\u2695\u2696\u2708]\uFE0F|[\u{1F466}\u{1F467}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|(?:\u{1F3FB}\u200D[\u2695\u2696\u2708]|\u{1F3FF}\u200D[\u2695\u2696\u2708]|\u{1F3FE}\u200D[\u2695\u2696\u2708]|\u{1F3FD}\u200D[\u2695\u2696\u2708]|\u{1F3FC}\u200D[\u2695\u2696\u2708])\uFE0F|\u{1F3FB}\u200D[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}]|[\u{1F3FB}-\u{1F3FF}])|(?:\u{1F9D1}\u{1F3FB}\u200D\u{1F91D}\u200D\u{1F9D1}|\u{1F469}\u{1F3FC}\u200D\u{1F91D}\u200D\u{1F469})\u{1F3FB}|\u{1F9D1}(?:\u{1F3FF}\u200D\u{1F91D}\u200D\u{1F9D1}[\u{1F3FB}-\u{1F3FF}]|\u200D\u{1F91D}\u200D\u{1F9D1})|(?:\u{1F9D1}\u{1F3FE}\u200D\u{1F91D}\u200D\u{1F9D1}|\u{1F469}\u{1F3FF}\u200D\u{1F91D}\u200D[\u{1F468}\u{1F469}])[\u{1F3FB}-\u{1F3FE}]|(?:\u{1F9D1}\u{1F3FC}\u200D\u{1F91D}\u200D\u{1F9D1}|\u{1F469}\u{1F3FD}\u200D\u{1F91D}\u200D\u{1F469})[\u{1F3FB}\u{1F3FC}]|\u{1F469}(?:\u{1F3FE}\u200D(?:\u{1F91D}\u200D\u{1F468}[\u{1F3FB}-\u{1F3FD}\u{1F3FF}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F3FC}\u200D(?:\u{1F91D}\u200D\u{1F468}[\u{1F3FB}\u{1F3FD}-\u{1F3FF}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F3FB}\u200D(?:\u{1F91D}\u200D\u{1F468}[\u{1F3FC}-\u{1F3FF}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F3FD}\u200D(?:\u{1F91D}\u200D\u{1F468}[\u{1F3FB}\u{1F3FC}\u{1F3FE}\u{1F3FF}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u200D(?:\u2764\uFE0F\u200D(?:\u{1F48B}\u200D[\u{1F468}\u{1F469}]|[\u{1F468}\u{1F469}])|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F3FF}\u200D[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F469}\u200D\u{1F469}\u200D(?:\u{1F466}\u200D\u{1F466}|\u{1F467}\u200D[\u{1F466}\u{1F467}])|(?:\u{1F9D1}\u{1F3FD}\u200D\u{1F91D}\u200D\u{1F9D1}|\u{1F469}\u{1F3FE}\u200D\u{1F91D}\u200D\u{1F469})[\u{1F3FB}-\u{1F3FD}]|\u{1F469}\u200D\u{1F466}\u200D\u{1F466}|\u{1F469}\u200D\u{1F469}\u200D[\u{1F466}\u{1F467}]|(?:\u{1F441}\uFE0F\u200D\u{1F5E8}|\u{1F469}(?:\u{1F3FF}\u200D[\u2695\u2696\u2708]|\u{1F3FE}\u200D[\u2695\u2696\u2708]|\u{1F3FC}\u200D[\u2695\u2696\u2708]|\u{1F3FB}\u200D[\u2695\u2696\u2708]|\u{1F3FD}\u200D[\u2695\u2696\u2708]|\u200D[\u2695\u2696\u2708])|(?:[\u26F9\u{1F3CB}\u{1F3CC}\u{1F575}]\uFE0F|[\u{1F46F}\u{1F93C}\u{1F9DE}\u{1F9DF}])\u200D[\u2640\u2642]|[\u26F9\u{1F3CB}\u{1F3CC}\u{1F575}][\u{1F3FB}-\u{1F3FF}]\u200D[\u2640\u2642]|[\u{1F3C3}\u{1F3C4}\u{1F3CA}\u{1F46E}\u{1F471}\u{1F473}\u{1F477}\u{1F481}\u{1F482}\u{1F486}\u{1F487}\u{1F645}-\u{1F647}\u{1F64B}\u{1F64D}\u{1F64E}\u{1F6A3}\u{1F6B4}-\u{1F6B6}\u{1F926}\u{1F937}-\u{1F939}\u{1F93D}\u{1F93E}\u{1F9B8}\u{1F9B9}\u{1F9CD}-\u{1F9CF}\u{1F9D6}-\u{1F9DD}](?:[\u{1F3FB}-\u{1F3FF}]\u200D[\u2640\u2642]|\u200D[\u2640\u2642])|\u{1F3F4}\u200D\u2620)\uFE0F|\u{1F469}\u200D\u{1F467}\u200D[\u{1F466}\u{1F467}]|\u{1F3F3}\uFE0F\u200D\u{1F308}|\u{1F415}\u200D\u{1F9BA}|\u{1F469}\u200D\u{1F466}|\u{1F469}\u200D\u{1F467}|\u{1F1FD}\u{1F1F0}|\u{1F1F4}\u{1F1F2}|\u{1F1F6}\u{1F1E6}|[#\*0-9]\uFE0F\u20E3|\u{1F1E7}[\u{1F1E6}\u{1F1E7}\u{1F1E9}-\u{1F1EF}\u{1F1F1}-\u{1F1F4}\u{1F1F6}-\u{1F1F9}\u{1F1FB}\u{1F1FC}\u{1F1FE}\u{1F1FF}]|\u{1F1F9}[\u{1F1E6}\u{1F1E8}\u{1F1E9}\u{1F1EB}-\u{1F1ED}\u{1F1EF}-\u{1F1F4}\u{1F1F7}\u{1F1F9}\u{1F1FB}\u{1F1FC}\u{1F1FF}]|\u{1F1EA}[\u{1F1E6}\u{1F1E8}\u{1F1EA}\u{1F1EC}\u{1F1ED}\u{1F1F7}-\u{1F1FA}]|\u{1F9D1}[\u{1F3FB}-\u{1F3FF}]|\u{1F1F7}[\u{1F1EA}\u{1F1F4}\u{1F1F8}\u{1F1FA}\u{1F1FC}]|\u{1F469}[\u{1F3FB}-\u{1F3FF}]|\u{1F1F2}[\u{1F1E6}\u{1F1E8}-\u{1F1ED}\u{1F1F0}-\u{1F1FF}]|\u{1F1E6}[\u{1F1E8}-\u{1F1EC}\u{1F1EE}\u{1F1F1}\u{1F1F2}\u{1F1F4}\u{1F1F6}-\u{1F1FA}\u{1F1FC}\u{1F1FD}\u{1F1FF}]|\u{1F1F0}[\u{1F1EA}\u{1F1EC}-\u{1F1EE}\u{1F1F2}\u{1F1F3}\u{1F1F5}\u{1F1F7}\u{1F1FC}\u{1F1FE}\u{1F1FF}]|\u{1F1ED}[\u{1F1F0}\u{1F1F2}\u{1F1F3}\u{1F1F7}\u{1F1F9}\u{1F1FA}]|\u{1F1E9}[\u{1F1EA}\u{1F1EC}\u{1F1EF}\u{1F1F0}\u{1F1F2}\u{1F1F4}\u{1F1FF}]|\u{1F1FE}[\u{1F1EA}\u{1F1F9}]|\u{1F1EC}[\u{1F1E6}\u{1F1E7}\u{1F1E9}-\u{1F1EE}\u{1F1F1}-\u{1F1F3}\u{1F1F5}-\u{1F1FA}\u{1F1FC}\u{1F1FE}]|\u{1F1F8}[\u{1F1E6}-\u{1F1EA}\u{1F1EC}-\u{1F1F4}\u{1F1F7}-\u{1F1F9}\u{1F1FB}\u{1F1FD}-\u{1F1FF}]|\u{1F1EB}[\u{1F1EE}-\u{1F1F0}\u{1F1F2}\u{1F1F4}\u{1F1F7}]|\u{1F1F5}[\u{1F1E6}\u{1F1EA}-\u{1F1ED}\u{1F1F0}-\u{1F1F3}\u{1F1F7}-\u{1F1F9}\u{1F1FC}\u{1F1FE}]|\u{1F1FB}[\u{1F1E6}\u{1F1E8}\u{1F1EA}\u{1F1EC}\u{1F1EE}\u{1F1F3}\u{1F1FA}]|\u{1F1F3}[\u{1F1E6}\u{1F1E8}\u{1F1EA}-\u{1F1EC}\u{1F1EE}\u{1F1F1}\u{1F1F4}\u{1F1F5}\u{1F1F7}\u{1F1FA}\u{1F1FF}]|\u{1F1E8}[\u{1F1E6}\u{1F1E8}\u{1F1E9}\u{1F1EB}-\u{1F1EE}\u{1F1F0}-\u{1F1F5}\u{1F1F7}\u{1F1FA}-\u{1F1FF}]|\u{1F1F1}[\u{1F1E6}-\u{1F1E8}\u{1F1EE}\u{1F1F0}\u{1F1F7}-\u{1F1FB}\u{1F1FE}]|\u{1F1FF}[\u{1F1E6}\u{1F1F2}\u{1F1FC}]|\u{1F1FC}[\u{1F1EB}\u{1F1F8}]|\u{1F1FA}[\u{1F1E6}\u{1F1EC}\u{1F1F2}\u{1F1F3}\u{1F1F8}\u{1F1FE}\u{1F1FF}]|\u{1F1EE}[\u{1F1E8}-\u{1F1EA}\u{1F1F1}-\u{1F1F4}\u{1F1F6}-\u{1F1F9}]|\u{1F1EF}[\u{1F1EA}\u{1F1F2}\u{1F1F4}\u{1F1F5}]|[\u{1F3C3}\u{1F3C4}\u{1F3CA}\u{1F46E}\u{1F471}\u{1F473}\u{1F477}\u{1F481}\u{1F482}\u{1F486}\u{1F487}\u{1F645}-\u{1F647}\u{1F64B}\u{1F64D}\u{1F64E}\u{1F6A3}\u{1F6B4}-\u{1F6B6}\u{1F926}\u{1F937}-\u{1F939}\u{1F93D}\u{1F93E}\u{1F9B8}\u{1F9B9}\u{1F9CD}-\u{1F9CF}\u{1F9D6}-\u{1F9DD}][\u{1F3FB}-\u{1F3FF}]|[\u26F9\u{1F3CB}\u{1F3CC}\u{1F575}][\u{1F3FB}-\u{1F3FF}]|[\u261D\u270A-\u270D\u{1F385}\u{1F3C2}\u{1F3C7}\u{1F442}\u{1F443}\u{1F446}-\u{1F450}\u{1F466}\u{1F467}\u{1F46B}-\u{1F46D}\u{1F470}\u{1F472}\u{1F474}-\u{1F476}\u{1F478}\u{1F47C}\u{1F483}\u{1F485}\u{1F4AA}\u{1F574}\u{1F57A}\u{1F590}\u{1F595}\u{1F596}\u{1F64C}\u{1F64F}\u{1F6C0}\u{1F6CC}\u{1F90F}\u{1F918}-\u{1F91C}\u{1F91E}\u{1F91F}\u{1F930}-\u{1F936}\u{1F9B5}\u{1F9B6}\u{1F9BB}\u{1F9D2}-\u{1F9D5}][\u{1F3FB}-\u{1F3FF}]|[\u231A\u231B\u23E9-\u23EC\u23F0\u23F3\u25FD\u25FE\u2614\u2615\u2648-\u2653\u267F\u2693\u26A1\u26AA\u26AB\u26BD\u26BE\u26C4\u26C5\u26CE\u26D4\u26EA\u26F2\u26F3\u26F5\u26FA\u26FD\u2705\u270A\u270B\u2728\u274C\u274E\u2753-\u2755\u2757\u2795-\u2797\u27B0\u27BF\u2B1B\u2B1C\u2B50\u2B55\u{1F004}\u{1F0CF}\u{1F18E}\u{1F191}-\u{1F19A}\u{1F1E6}-\u{1F1FF}\u{1F201}\u{1F21A}\u{1F22F}\u{1F232}-\u{1F236}\u{1F238}-\u{1F23A}\u{1F250}\u{1F251}\u{1F300}-\u{1F320}\u{1F32D}-\u{1F335}\u{1F337}-\u{1F37C}\u{1F37E}-\u{1F393}\u{1F3A0}-\u{1F3CA}\u{1F3CF}-\u{1F3D3}\u{1F3E0}-\u{1F3F0}\u{1F3F4}\u{1F3F8}-\u{1F43E}\u{1F440}\u{1F442}-\u{1F4FC}\u{1F4FF}-\u{1F53D}\u{1F54B}-\u{1F54E}\u{1F550}-\u{1F567}\u{1F57A}\u{1F595}\u{1F596}\u{1F5A4}\u{1F5FB}-\u{1F64F}\u{1F680}-\u{1F6C5}\u{1F6CC}\u{1F6D0}-\u{1F6D2}\u{1F6D5}\u{1F6EB}\u{1F6EC}\u{1F6F4}-\u{1F6FA}\u{1F7E0}-\u{1F7EB}\u{1F90D}-\u{1F93A}\u{1F93C}-\u{1F945}\u{1F947}-\u{1F971}\u{1F973}-\u{1F976}\u{1F97A}-\u{1F9A2}\u{1F9A5}-\u{1F9AA}\u{1F9AE}-\u{1F9CA}\u{1F9CD}-\u{1F9FF}\u{1FA70}-\u{1FA73}\u{1FA78}-\u{1FA7A}\u{1FA80}-\u{1FA82}\u{1FA90}-\u{1FA95}]|[#\*0-9\xA9\xAE\u203C\u2049\u2122\u2139\u2194-\u2199\u21A9\u21AA\u231A\u231B\u2328\u23CF\u23E9-\u23F3\u23F8-\u23FA\u24C2\u25AA\u25AB\u25B6\u25C0\u25FB-\u25FE\u2600-\u2604\u260E\u2611\u2614\u2615\u2618\u261D\u2620\u2622\u2623\u2626\u262A\u262E\u262F\u2638-\u263A\u2640\u2642\u2648-\u2653\u265F\u2660\u2663\u2665\u2666\u2668\u267B\u267E\u267F\u2692-\u2697\u2699\u269B\u269C\u26A0\u26A1\u26AA\u26AB\u26B0\u26B1\u26BD\u26BE\u26C4\u26C5\u26C8\u26CE\u26CF\u26D1\u26D3\u26D4\u26E9\u26EA\u26F0-\u26F5\u26F7-\u26FA\u26FD\u2702\u2705\u2708-\u270D\u270F\u2712\u2714\u2716\u271D\u2721\u2728\u2733\u2734\u2744\u2747\u274C\u274E\u2753-\u2755\u2757\u2763\u2764\u2795-\u2797\u27A1\u27B0\u27BF\u2934\u2935\u2B05-\u2B07\u2B1B\u2B1C\u2B50\u2B55\u3030\u303D\u3297\u3299\u{1F004}\u{1F0CF}\u{1F170}\u{1F171}\u{1F17E}\u{1F17F}\u{1F18E}\u{1F191}-\u{1F19A}\u{1F1E6}-\u{1F1FF}\u{1F201}\u{1F202}\u{1F21A}\u{1F22F}\u{1F232}-\u{1F23A}\u{1F250}\u{1F251}\u{1F300}-\u{1F321}\u{1F324}-\u{1F393}\u{1F396}\u{1F397}\u{1F399}-\u{1F39B}\u{1F39E}-\u{1F3F0}\u{1F3F3}-\u{1F3F5}\u{1F3F7}-\u{1F4FD}\u{1F4FF}-\u{1F53D}\u{1F549}-\u{1F54E}\u{1F550}-\u{1F567}\u{1F56F}\u{1F570}\u{1F573}-\u{1F57A}\u{1F587}\u{1F58A}-\u{1F58D}\u{1F590}\u{1F595}\u{1F596}\u{1F5A4}\u{1F5A5}\u{1F5A8}\u{1F5B1}\u{1F5B2}\u{1F5BC}\u{1F5C2}-\u{1F5C4}\u{1F5D1}-\u{1F5D3}\u{1F5DC}-\u{1F5DE}\u{1F5E1}\u{1F5E3}\u{1F5E8}\u{1F5EF}\u{1F5F3}\u{1F5FA}-\u{1F64F}\u{1F680}-\u{1F6C5}\u{1F6CB}-\u{1F6D2}\u{1F6D5}\u{1F6E0}-\u{1F6E5}\u{1F6E9}\u{1F6EB}\u{1F6EC}\u{1F6F0}\u{1F6F3}-\u{1F6FA}\u{1F7E0}-\u{1F7EB}\u{1F90D}-\u{1F93A}\u{1F93C}-\u{1F945}\u{1F947}-\u{1F971}\u{1F973}-\u{1F976}\u{1F97A}-\u{1F9A2}\u{1F9A5}-\u{1F9AA}\u{1F9AE}-\u{1F9CA}\u{1F9CD}-\u{1F9FF}\u{1FA70}-\u{1FA73}\u{1FA78}-\u{1FA7A}\u{1FA80}-\u{1FA82}\u{1FA90}-\u{1FA95}]\uFE0F|[\u261D\u26F9\u270A-\u270D\u{1F385}\u{1F3C2}-\u{1F3C4}\u{1F3C7}\u{1F3CA}-\u{1F3CC}\u{1F442}\u{1F443}\u{1F446}-\u{1F450}\u{1F466}-\u{1F478}\u{1F47C}\u{1F481}-\u{1F483}\u{1F485}-\u{1F487}\u{1F48F}\u{1F491}\u{1F4AA}\u{1F574}\u{1F575}\u{1F57A}\u{1F590}\u{1F595}\u{1F596}\u{1F645}-\u{1F647}\u{1F64B}-\u{1F64F}\u{1F6A3}\u{1F6B4}-\u{1F6B6}\u{1F6C0}\u{1F6CC}\u{1F90F}\u{1F918}-\u{1F91F}\u{1F926}\u{1F930}-\u{1F939}\u{1F93C}-\u{1F93E}\u{1F9B5}\u{1F9B6}\u{1F9B8}\u{1F9B9}\u{1F9BB}\u{1F9CD}-\u{1F9CF}\u{1F9D1}-\u{1F9DD}]/gu; +}; diff --git a/node_modules/emoji-regex/es2015/text.js b/node_modules/emoji-regex/es2015/text.js new file mode 100644 index 0000000000000000000000000000000000000000..780309df58f1a20d32e6fc14e564369951516d3b --- /dev/null +++ b/node_modules/emoji-regex/es2015/text.js @@ -0,0 +1,6 @@ +"use strict"; + +module.exports = () => { + // https://mths.be/emoji + return /\u{1F3F4}\u{E0067}\u{E0062}(?:\u{E0065}\u{E006E}\u{E0067}|\u{E0073}\u{E0063}\u{E0074}|\u{E0077}\u{E006C}\u{E0073})\u{E007F}|\u{1F468}(?:\u{1F3FC}\u200D(?:\u{1F91D}\u200D\u{1F468}\u{1F3FB}|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F3FF}\u200D(?:\u{1F91D}\u200D\u{1F468}[\u{1F3FB}-\u{1F3FE}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F3FE}\u200D(?:\u{1F91D}\u200D\u{1F468}[\u{1F3FB}-\u{1F3FD}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F3FD}\u200D(?:\u{1F91D}\u200D\u{1F468}[\u{1F3FB}\u{1F3FC}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u200D(?:\u2764\uFE0F\u200D(?:\u{1F48B}\u200D)?\u{1F468}|[\u{1F468}\u{1F469}]\u200D(?:\u{1F466}\u200D\u{1F466}|\u{1F467}\u200D[\u{1F466}\u{1F467}])|\u{1F466}\u200D\u{1F466}|\u{1F467}\u200D[\u{1F466}\u{1F467}]|[\u{1F468}\u{1F469}]\u200D[\u{1F466}\u{1F467}]|[\u2695\u2696\u2708]\uFE0F|[\u{1F466}\u{1F467}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|(?:\u{1F3FB}\u200D[\u2695\u2696\u2708]|\u{1F3FF}\u200D[\u2695\u2696\u2708]|\u{1F3FE}\u200D[\u2695\u2696\u2708]|\u{1F3FD}\u200D[\u2695\u2696\u2708]|\u{1F3FC}\u200D[\u2695\u2696\u2708])\uFE0F|\u{1F3FB}\u200D[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}]|[\u{1F3FB}-\u{1F3FF}])|(?:\u{1F9D1}\u{1F3FB}\u200D\u{1F91D}\u200D\u{1F9D1}|\u{1F469}\u{1F3FC}\u200D\u{1F91D}\u200D\u{1F469})\u{1F3FB}|\u{1F9D1}(?:\u{1F3FF}\u200D\u{1F91D}\u200D\u{1F9D1}[\u{1F3FB}-\u{1F3FF}]|\u200D\u{1F91D}\u200D\u{1F9D1})|(?:\u{1F9D1}\u{1F3FE}\u200D\u{1F91D}\u200D\u{1F9D1}|\u{1F469}\u{1F3FF}\u200D\u{1F91D}\u200D[\u{1F468}\u{1F469}])[\u{1F3FB}-\u{1F3FE}]|(?:\u{1F9D1}\u{1F3FC}\u200D\u{1F91D}\u200D\u{1F9D1}|\u{1F469}\u{1F3FD}\u200D\u{1F91D}\u200D\u{1F469})[\u{1F3FB}\u{1F3FC}]|\u{1F469}(?:\u{1F3FE}\u200D(?:\u{1F91D}\u200D\u{1F468}[\u{1F3FB}-\u{1F3FD}\u{1F3FF}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F3FC}\u200D(?:\u{1F91D}\u200D\u{1F468}[\u{1F3FB}\u{1F3FD}-\u{1F3FF}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F3FB}\u200D(?:\u{1F91D}\u200D\u{1F468}[\u{1F3FC}-\u{1F3FF}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F3FD}\u200D(?:\u{1F91D}\u200D\u{1F468}[\u{1F3FB}\u{1F3FC}\u{1F3FE}\u{1F3FF}]|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u200D(?:\u2764\uFE0F\u200D(?:\u{1F48B}\u200D[\u{1F468}\u{1F469}]|[\u{1F468}\u{1F469}])|[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F3FF}\u200D[\u{1F33E}\u{1F373}\u{1F393}\u{1F3A4}\u{1F3A8}\u{1F3EB}\u{1F3ED}\u{1F4BB}\u{1F4BC}\u{1F527}\u{1F52C}\u{1F680}\u{1F692}\u{1F9AF}-\u{1F9B3}\u{1F9BC}\u{1F9BD}])|\u{1F469}\u200D\u{1F469}\u200D(?:\u{1F466}\u200D\u{1F466}|\u{1F467}\u200D[\u{1F466}\u{1F467}])|(?:\u{1F9D1}\u{1F3FD}\u200D\u{1F91D}\u200D\u{1F9D1}|\u{1F469}\u{1F3FE}\u200D\u{1F91D}\u200D\u{1F469})[\u{1F3FB}-\u{1F3FD}]|\u{1F469}\u200D\u{1F466}\u200D\u{1F466}|\u{1F469}\u200D\u{1F469}\u200D[\u{1F466}\u{1F467}]|(?:\u{1F441}\uFE0F\u200D\u{1F5E8}|\u{1F469}(?:\u{1F3FF}\u200D[\u2695\u2696\u2708]|\u{1F3FE}\u200D[\u2695\u2696\u2708]|\u{1F3FC}\u200D[\u2695\u2696\u2708]|\u{1F3FB}\u200D[\u2695\u2696\u2708]|\u{1F3FD}\u200D[\u2695\u2696\u2708]|\u200D[\u2695\u2696\u2708])|(?:[\u26F9\u{1F3CB}\u{1F3CC}\u{1F575}]\uFE0F|[\u{1F46F}\u{1F93C}\u{1F9DE}\u{1F9DF}])\u200D[\u2640\u2642]|[\u26F9\u{1F3CB}\u{1F3CC}\u{1F575}][\u{1F3FB}-\u{1F3FF}]\u200D[\u2640\u2642]|[\u{1F3C3}\u{1F3C4}\u{1F3CA}\u{1F46E}\u{1F471}\u{1F473}\u{1F477}\u{1F481}\u{1F482}\u{1F486}\u{1F487}\u{1F645}-\u{1F647}\u{1F64B}\u{1F64D}\u{1F64E}\u{1F6A3}\u{1F6B4}-\u{1F6B6}\u{1F926}\u{1F937}-\u{1F939}\u{1F93D}\u{1F93E}\u{1F9B8}\u{1F9B9}\u{1F9CD}-\u{1F9CF}\u{1F9D6}-\u{1F9DD}](?:[\u{1F3FB}-\u{1F3FF}]\u200D[\u2640\u2642]|\u200D[\u2640\u2642])|\u{1F3F4}\u200D\u2620)\uFE0F|\u{1F469}\u200D\u{1F467}\u200D[\u{1F466}\u{1F467}]|\u{1F3F3}\uFE0F\u200D\u{1F308}|\u{1F415}\u200D\u{1F9BA}|\u{1F469}\u200D\u{1F466}|\u{1F469}\u200D\u{1F467}|\u{1F1FD}\u{1F1F0}|\u{1F1F4}\u{1F1F2}|\u{1F1F6}\u{1F1E6}|[#\*0-9]\uFE0F\u20E3|\u{1F1E7}[\u{1F1E6}\u{1F1E7}\u{1F1E9}-\u{1F1EF}\u{1F1F1}-\u{1F1F4}\u{1F1F6}-\u{1F1F9}\u{1F1FB}\u{1F1FC}\u{1F1FE}\u{1F1FF}]|\u{1F1F9}[\u{1F1E6}\u{1F1E8}\u{1F1E9}\u{1F1EB}-\u{1F1ED}\u{1F1EF}-\u{1F1F4}\u{1F1F7}\u{1F1F9}\u{1F1FB}\u{1F1FC}\u{1F1FF}]|\u{1F1EA}[\u{1F1E6}\u{1F1E8}\u{1F1EA}\u{1F1EC}\u{1F1ED}\u{1F1F7}-\u{1F1FA}]|\u{1F9D1}[\u{1F3FB}-\u{1F3FF}]|\u{1F1F7}[\u{1F1EA}\u{1F1F4}\u{1F1F8}\u{1F1FA}\u{1F1FC}]|\u{1F469}[\u{1F3FB}-\u{1F3FF}]|\u{1F1F2}[\u{1F1E6}\u{1F1E8}-\u{1F1ED}\u{1F1F0}-\u{1F1FF}]|\u{1F1E6}[\u{1F1E8}-\u{1F1EC}\u{1F1EE}\u{1F1F1}\u{1F1F2}\u{1F1F4}\u{1F1F6}-\u{1F1FA}\u{1F1FC}\u{1F1FD}\u{1F1FF}]|\u{1F1F0}[\u{1F1EA}\u{1F1EC}-\u{1F1EE}\u{1F1F2}\u{1F1F3}\u{1F1F5}\u{1F1F7}\u{1F1FC}\u{1F1FE}\u{1F1FF}]|\u{1F1ED}[\u{1F1F0}\u{1F1F2}\u{1F1F3}\u{1F1F7}\u{1F1F9}\u{1F1FA}]|\u{1F1E9}[\u{1F1EA}\u{1F1EC}\u{1F1EF}\u{1F1F0}\u{1F1F2}\u{1F1F4}\u{1F1FF}]|\u{1F1FE}[\u{1F1EA}\u{1F1F9}]|\u{1F1EC}[\u{1F1E6}\u{1F1E7}\u{1F1E9}-\u{1F1EE}\u{1F1F1}-\u{1F1F3}\u{1F1F5}-\u{1F1FA}\u{1F1FC}\u{1F1FE}]|\u{1F1F8}[\u{1F1E6}-\u{1F1EA}\u{1F1EC}-\u{1F1F4}\u{1F1F7}-\u{1F1F9}\u{1F1FB}\u{1F1FD}-\u{1F1FF}]|\u{1F1EB}[\u{1F1EE}-\u{1F1F0}\u{1F1F2}\u{1F1F4}\u{1F1F7}]|\u{1F1F5}[\u{1F1E6}\u{1F1EA}-\u{1F1ED}\u{1F1F0}-\u{1F1F3}\u{1F1F7}-\u{1F1F9}\u{1F1FC}\u{1F1FE}]|\u{1F1FB}[\u{1F1E6}\u{1F1E8}\u{1F1EA}\u{1F1EC}\u{1F1EE}\u{1F1F3}\u{1F1FA}]|\u{1F1F3}[\u{1F1E6}\u{1F1E8}\u{1F1EA}-\u{1F1EC}\u{1F1EE}\u{1F1F1}\u{1F1F4}\u{1F1F5}\u{1F1F7}\u{1F1FA}\u{1F1FF}]|\u{1F1E8}[\u{1F1E6}\u{1F1E8}\u{1F1E9}\u{1F1EB}-\u{1F1EE}\u{1F1F0}-\u{1F1F5}\u{1F1F7}\u{1F1FA}-\u{1F1FF}]|\u{1F1F1}[\u{1F1E6}-\u{1F1E8}\u{1F1EE}\u{1F1F0}\u{1F1F7}-\u{1F1FB}\u{1F1FE}]|\u{1F1FF}[\u{1F1E6}\u{1F1F2}\u{1F1FC}]|\u{1F1FC}[\u{1F1EB}\u{1F1F8}]|\u{1F1FA}[\u{1F1E6}\u{1F1EC}\u{1F1F2}\u{1F1F3}\u{1F1F8}\u{1F1FE}\u{1F1FF}]|\u{1F1EE}[\u{1F1E8}-\u{1F1EA}\u{1F1F1}-\u{1F1F4}\u{1F1F6}-\u{1F1F9}]|\u{1F1EF}[\u{1F1EA}\u{1F1F2}\u{1F1F4}\u{1F1F5}]|[\u{1F3C3}\u{1F3C4}\u{1F3CA}\u{1F46E}\u{1F471}\u{1F473}\u{1F477}\u{1F481}\u{1F482}\u{1F486}\u{1F487}\u{1F645}-\u{1F647}\u{1F64B}\u{1F64D}\u{1F64E}\u{1F6A3}\u{1F6B4}-\u{1F6B6}\u{1F926}\u{1F937}-\u{1F939}\u{1F93D}\u{1F93E}\u{1F9B8}\u{1F9B9}\u{1F9CD}-\u{1F9CF}\u{1F9D6}-\u{1F9DD}][\u{1F3FB}-\u{1F3FF}]|[\u26F9\u{1F3CB}\u{1F3CC}\u{1F575}][\u{1F3FB}-\u{1F3FF}]|[\u261D\u270A-\u270D\u{1F385}\u{1F3C2}\u{1F3C7}\u{1F442}\u{1F443}\u{1F446}-\u{1F450}\u{1F466}\u{1F467}\u{1F46B}-\u{1F46D}\u{1F470}\u{1F472}\u{1F474}-\u{1F476}\u{1F478}\u{1F47C}\u{1F483}\u{1F485}\u{1F4AA}\u{1F574}\u{1F57A}\u{1F590}\u{1F595}\u{1F596}\u{1F64C}\u{1F64F}\u{1F6C0}\u{1F6CC}\u{1F90F}\u{1F918}-\u{1F91C}\u{1F91E}\u{1F91F}\u{1F930}-\u{1F936}\u{1F9B5}\u{1F9B6}\u{1F9BB}\u{1F9D2}-\u{1F9D5}][\u{1F3FB}-\u{1F3FF}]|[\u231A\u231B\u23E9-\u23EC\u23F0\u23F3\u25FD\u25FE\u2614\u2615\u2648-\u2653\u267F\u2693\u26A1\u26AA\u26AB\u26BD\u26BE\u26C4\u26C5\u26CE\u26D4\u26EA\u26F2\u26F3\u26F5\u26FA\u26FD\u2705\u270A\u270B\u2728\u274C\u274E\u2753-\u2755\u2757\u2795-\u2797\u27B0\u27BF\u2B1B\u2B1C\u2B50\u2B55\u{1F004}\u{1F0CF}\u{1F18E}\u{1F191}-\u{1F19A}\u{1F1E6}-\u{1F1FF}\u{1F201}\u{1F21A}\u{1F22F}\u{1F232}-\u{1F236}\u{1F238}-\u{1F23A}\u{1F250}\u{1F251}\u{1F300}-\u{1F320}\u{1F32D}-\u{1F335}\u{1F337}-\u{1F37C}\u{1F37E}-\u{1F393}\u{1F3A0}-\u{1F3CA}\u{1F3CF}-\u{1F3D3}\u{1F3E0}-\u{1F3F0}\u{1F3F4}\u{1F3F8}-\u{1F43E}\u{1F440}\u{1F442}-\u{1F4FC}\u{1F4FF}-\u{1F53D}\u{1F54B}-\u{1F54E}\u{1F550}-\u{1F567}\u{1F57A}\u{1F595}\u{1F596}\u{1F5A4}\u{1F5FB}-\u{1F64F}\u{1F680}-\u{1F6C5}\u{1F6CC}\u{1F6D0}-\u{1F6D2}\u{1F6D5}\u{1F6EB}\u{1F6EC}\u{1F6F4}-\u{1F6FA}\u{1F7E0}-\u{1F7EB}\u{1F90D}-\u{1F93A}\u{1F93C}-\u{1F945}\u{1F947}-\u{1F971}\u{1F973}-\u{1F976}\u{1F97A}-\u{1F9A2}\u{1F9A5}-\u{1F9AA}\u{1F9AE}-\u{1F9CA}\u{1F9CD}-\u{1F9FF}\u{1FA70}-\u{1FA73}\u{1FA78}-\u{1FA7A}\u{1FA80}-\u{1FA82}\u{1FA90}-\u{1FA95}]|[#\*0-9\xA9\xAE\u203C\u2049\u2122\u2139\u2194-\u2199\u21A9\u21AA\u231A\u231B\u2328\u23CF\u23E9-\u23F3\u23F8-\u23FA\u24C2\u25AA\u25AB\u25B6\u25C0\u25FB-\u25FE\u2600-\u2604\u260E\u2611\u2614\u2615\u2618\u261D\u2620\u2622\u2623\u2626\u262A\u262E\u262F\u2638-\u263A\u2640\u2642\u2648-\u2653\u265F\u2660\u2663\u2665\u2666\u2668\u267B\u267E\u267F\u2692-\u2697\u2699\u269B\u269C\u26A0\u26A1\u26AA\u26AB\u26B0\u26B1\u26BD\u26BE\u26C4\u26C5\u26C8\u26CE\u26CF\u26D1\u26D3\u26D4\u26E9\u26EA\u26F0-\u26F5\u26F7-\u26FA\u26FD\u2702\u2705\u2708-\u270D\u270F\u2712\u2714\u2716\u271D\u2721\u2728\u2733\u2734\u2744\u2747\u274C\u274E\u2753-\u2755\u2757\u2763\u2764\u2795-\u2797\u27A1\u27B0\u27BF\u2934\u2935\u2B05-\u2B07\u2B1B\u2B1C\u2B50\u2B55\u3030\u303D\u3297\u3299\u{1F004}\u{1F0CF}\u{1F170}\u{1F171}\u{1F17E}\u{1F17F}\u{1F18E}\u{1F191}-\u{1F19A}\u{1F1E6}-\u{1F1FF}\u{1F201}\u{1F202}\u{1F21A}\u{1F22F}\u{1F232}-\u{1F23A}\u{1F250}\u{1F251}\u{1F300}-\u{1F321}\u{1F324}-\u{1F393}\u{1F396}\u{1F397}\u{1F399}-\u{1F39B}\u{1F39E}-\u{1F3F0}\u{1F3F3}-\u{1F3F5}\u{1F3F7}-\u{1F4FD}\u{1F4FF}-\u{1F53D}\u{1F549}-\u{1F54E}\u{1F550}-\u{1F567}\u{1F56F}\u{1F570}\u{1F573}-\u{1F57A}\u{1F587}\u{1F58A}-\u{1F58D}\u{1F590}\u{1F595}\u{1F596}\u{1F5A4}\u{1F5A5}\u{1F5A8}\u{1F5B1}\u{1F5B2}\u{1F5BC}\u{1F5C2}-\u{1F5C4}\u{1F5D1}-\u{1F5D3}\u{1F5DC}-\u{1F5DE}\u{1F5E1}\u{1F5E3}\u{1F5E8}\u{1F5EF}\u{1F5F3}\u{1F5FA}-\u{1F64F}\u{1F680}-\u{1F6C5}\u{1F6CB}-\u{1F6D2}\u{1F6D5}\u{1F6E0}-\u{1F6E5}\u{1F6E9}\u{1F6EB}\u{1F6EC}\u{1F6F0}\u{1F6F3}-\u{1F6FA}\u{1F7E0}-\u{1F7EB}\u{1F90D}-\u{1F93A}\u{1F93C}-\u{1F945}\u{1F947}-\u{1F971}\u{1F973}-\u{1F976}\u{1F97A}-\u{1F9A2}\u{1F9A5}-\u{1F9AA}\u{1F9AE}-\u{1F9CA}\u{1F9CD}-\u{1F9FF}\u{1FA70}-\u{1FA73}\u{1FA78}-\u{1FA7A}\u{1FA80}-\u{1FA82}\u{1FA90}-\u{1FA95}]\uFE0F?|[\u261D\u26F9\u270A-\u270D\u{1F385}\u{1F3C2}-\u{1F3C4}\u{1F3C7}\u{1F3CA}-\u{1F3CC}\u{1F442}\u{1F443}\u{1F446}-\u{1F450}\u{1F466}-\u{1F478}\u{1F47C}\u{1F481}-\u{1F483}\u{1F485}-\u{1F487}\u{1F48F}\u{1F491}\u{1F4AA}\u{1F574}\u{1F575}\u{1F57A}\u{1F590}\u{1F595}\u{1F596}\u{1F645}-\u{1F647}\u{1F64B}-\u{1F64F}\u{1F6A3}\u{1F6B4}-\u{1F6B6}\u{1F6C0}\u{1F6CC}\u{1F90F}\u{1F918}-\u{1F91F}\u{1F926}\u{1F930}-\u{1F939}\u{1F93C}-\u{1F93E}\u{1F9B5}\u{1F9B6}\u{1F9B8}\u{1F9B9}\u{1F9BB}\u{1F9CD}-\u{1F9CF}\u{1F9D1}-\u{1F9DD}]/gu; +}; diff --git a/node_modules/emoji-regex/index.d.ts b/node_modules/emoji-regex/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..1955b4704ecfc9693da9689e01ad99432c8cae9d --- /dev/null +++ b/node_modules/emoji-regex/index.d.ts @@ -0,0 +1,23 @@ +declare module 'emoji-regex' { + function emojiRegex(): RegExp; + + export default emojiRegex; +} + +declare module 'emoji-regex/text' { + function emojiRegex(): RegExp; + + export default emojiRegex; +} + +declare module 'emoji-regex/es2015' { + function emojiRegex(): RegExp; + + export default emojiRegex; +} + +declare module 'emoji-regex/es2015/text' { + function emojiRegex(): RegExp; + + export default emojiRegex; +} diff --git a/node_modules/emoji-regex/index.js b/node_modules/emoji-regex/index.js new file mode 100644 index 0000000000000000000000000000000000000000..d993a3a99cb95ab89a103eb576389b4861c0b52a --- /dev/null +++ b/node_modules/emoji-regex/index.js @@ -0,0 +1,6 @@ +"use strict"; + +module.exports = function () { + // https://mths.be/emoji + return /\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62(?:\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67|\uDB40\uDC73\uDB40\uDC63\uDB40\uDC74|\uDB40\uDC77\uDB40\uDC6C\uDB40\uDC73)\uDB40\uDC7F|\uD83D\uDC68(?:\uD83C\uDFFC\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68\uD83C\uDFFB|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFF\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFE])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFE\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFD])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFD\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB\uDFFC])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\u200D(?:\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D)?\uD83D\uDC68|(?:\uD83D[\uDC68\uDC69])\u200D(?:\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67]))|\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67])|(?:\uD83D[\uDC68\uDC69])\u200D(?:\uD83D[\uDC66\uDC67])|[\u2695\u2696\u2708]\uFE0F|\uD83D[\uDC66\uDC67]|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|(?:\uD83C\uDFFB\u200D[\u2695\u2696\u2708]|\uD83C\uDFFF\u200D[\u2695\u2696\u2708]|\uD83C\uDFFE\u200D[\u2695\u2696\u2708]|\uD83C\uDFFD\u200D[\u2695\u2696\u2708]|\uD83C\uDFFC\u200D[\u2695\u2696\u2708])\uFE0F|\uD83C\uDFFB\u200D(?:\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C[\uDFFB-\uDFFF])|(?:\uD83E\uDDD1\uD83C\uDFFB\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFC\u200D\uD83E\uDD1D\u200D\uD83D\uDC69)\uD83C\uDFFB|\uD83E\uDDD1(?:\uD83C\uDFFF\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1(?:\uD83C[\uDFFB-\uDFFF])|\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1)|(?:\uD83E\uDDD1\uD83C\uDFFE\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFF\u200D\uD83E\uDD1D\u200D(?:\uD83D[\uDC68\uDC69]))(?:\uD83C[\uDFFB-\uDFFE])|(?:\uD83E\uDDD1\uD83C\uDFFC\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFD\u200D\uD83E\uDD1D\u200D\uD83D\uDC69)(?:\uD83C[\uDFFB\uDFFC])|\uD83D\uDC69(?:\uD83C\uDFFE\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFD\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFC\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB\uDFFD-\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFB\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFC-\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFD\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB\uDFFC\uDFFE\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\u200D(?:\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D(?:\uD83D[\uDC68\uDC69])|\uD83D[\uDC68\uDC69])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFF\u200D(?:\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD]))|\uD83D\uDC69\u200D\uD83D\uDC69\u200D(?:\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67]))|(?:\uD83E\uDDD1\uD83C\uDFFD\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFE\u200D\uD83E\uDD1D\u200D\uD83D\uDC69)(?:\uD83C[\uDFFB-\uDFFD])|\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC69\u200D\uD83D\uDC69\u200D(?:\uD83D[\uDC66\uDC67])|(?:\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8|\uD83D\uDC69(?:\uD83C\uDFFF\u200D[\u2695\u2696\u2708]|\uD83C\uDFFE\u200D[\u2695\u2696\u2708]|\uD83C\uDFFC\u200D[\u2695\u2696\u2708]|\uD83C\uDFFB\u200D[\u2695\u2696\u2708]|\uD83C\uDFFD\u200D[\u2695\u2696\u2708]|\u200D[\u2695\u2696\u2708])|(?:(?:\u26F9|\uD83C[\uDFCB\uDFCC]|\uD83D\uDD75)\uFE0F|\uD83D\uDC6F|\uD83E[\uDD3C\uDDDE\uDDDF])\u200D[\u2640\u2642]|(?:\u26F9|\uD83C[\uDFCB\uDFCC]|\uD83D\uDD75)(?:\uD83C[\uDFFB-\uDFFF])\u200D[\u2640\u2642]|(?:\uD83C[\uDFC3\uDFC4\uDFCA]|\uD83D[\uDC6E\uDC71\uDC73\uDC77\uDC81\uDC82\uDC86\uDC87\uDE45-\uDE47\uDE4B\uDE4D\uDE4E\uDEA3\uDEB4-\uDEB6]|\uD83E[\uDD26\uDD37-\uDD39\uDD3D\uDD3E\uDDB8\uDDB9\uDDCD-\uDDCF\uDDD6-\uDDDD])(?:(?:\uD83C[\uDFFB-\uDFFF])\u200D[\u2640\u2642]|\u200D[\u2640\u2642])|\uD83C\uDFF4\u200D\u2620)\uFE0F|\uD83D\uDC69\u200D\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67])|\uD83C\uDFF3\uFE0F\u200D\uD83C\uDF08|\uD83D\uDC15\u200D\uD83E\uDDBA|\uD83D\uDC69\u200D\uD83D\uDC66|\uD83D\uDC69\u200D\uD83D\uDC67|\uD83C\uDDFD\uD83C\uDDF0|\uD83C\uDDF4\uD83C\uDDF2|\uD83C\uDDF6\uD83C\uDDE6|[#\*0-9]\uFE0F\u20E3|\uD83C\uDDE7(?:\uD83C[\uDDE6\uDDE7\uDDE9-\uDDEF\uDDF1-\uDDF4\uDDF6-\uDDF9\uDDFB\uDDFC\uDDFE\uDDFF])|\uD83C\uDDF9(?:\uD83C[\uDDE6\uDDE8\uDDE9\uDDEB-\uDDED\uDDEF-\uDDF4\uDDF7\uDDF9\uDDFB\uDDFC\uDDFF])|\uD83C\uDDEA(?:\uD83C[\uDDE6\uDDE8\uDDEA\uDDEC\uDDED\uDDF7-\uDDFA])|\uD83E\uDDD1(?:\uD83C[\uDFFB-\uDFFF])|\uD83C\uDDF7(?:\uD83C[\uDDEA\uDDF4\uDDF8\uDDFA\uDDFC])|\uD83D\uDC69(?:\uD83C[\uDFFB-\uDFFF])|\uD83C\uDDF2(?:\uD83C[\uDDE6\uDDE8-\uDDED\uDDF0-\uDDFF])|\uD83C\uDDE6(?:\uD83C[\uDDE8-\uDDEC\uDDEE\uDDF1\uDDF2\uDDF4\uDDF6-\uDDFA\uDDFC\uDDFD\uDDFF])|\uD83C\uDDF0(?:\uD83C[\uDDEA\uDDEC-\uDDEE\uDDF2\uDDF3\uDDF5\uDDF7\uDDFC\uDDFE\uDDFF])|\uD83C\uDDED(?:\uD83C[\uDDF0\uDDF2\uDDF3\uDDF7\uDDF9\uDDFA])|\uD83C\uDDE9(?:\uD83C[\uDDEA\uDDEC\uDDEF\uDDF0\uDDF2\uDDF4\uDDFF])|\uD83C\uDDFE(?:\uD83C[\uDDEA\uDDF9])|\uD83C\uDDEC(?:\uD83C[\uDDE6\uDDE7\uDDE9-\uDDEE\uDDF1-\uDDF3\uDDF5-\uDDFA\uDDFC\uDDFE])|\uD83C\uDDF8(?:\uD83C[\uDDE6-\uDDEA\uDDEC-\uDDF4\uDDF7-\uDDF9\uDDFB\uDDFD-\uDDFF])|\uD83C\uDDEB(?:\uD83C[\uDDEE-\uDDF0\uDDF2\uDDF4\uDDF7])|\uD83C\uDDF5(?:\uD83C[\uDDE6\uDDEA-\uDDED\uDDF0-\uDDF3\uDDF7-\uDDF9\uDDFC\uDDFE])|\uD83C\uDDFB(?:\uD83C[\uDDE6\uDDE8\uDDEA\uDDEC\uDDEE\uDDF3\uDDFA])|\uD83C\uDDF3(?:\uD83C[\uDDE6\uDDE8\uDDEA-\uDDEC\uDDEE\uDDF1\uDDF4\uDDF5\uDDF7\uDDFA\uDDFF])|\uD83C\uDDE8(?:\uD83C[\uDDE6\uDDE8\uDDE9\uDDEB-\uDDEE\uDDF0-\uDDF5\uDDF7\uDDFA-\uDDFF])|\uD83C\uDDF1(?:\uD83C[\uDDE6-\uDDE8\uDDEE\uDDF0\uDDF7-\uDDFB\uDDFE])|\uD83C\uDDFF(?:\uD83C[\uDDE6\uDDF2\uDDFC])|\uD83C\uDDFC(?:\uD83C[\uDDEB\uDDF8])|\uD83C\uDDFA(?:\uD83C[\uDDE6\uDDEC\uDDF2\uDDF3\uDDF8\uDDFE\uDDFF])|\uD83C\uDDEE(?:\uD83C[\uDDE8-\uDDEA\uDDF1-\uDDF4\uDDF6-\uDDF9])|\uD83C\uDDEF(?:\uD83C[\uDDEA\uDDF2\uDDF4\uDDF5])|(?:\uD83C[\uDFC3\uDFC4\uDFCA]|\uD83D[\uDC6E\uDC71\uDC73\uDC77\uDC81\uDC82\uDC86\uDC87\uDE45-\uDE47\uDE4B\uDE4D\uDE4E\uDEA3\uDEB4-\uDEB6]|\uD83E[\uDD26\uDD37-\uDD39\uDD3D\uDD3E\uDDB8\uDDB9\uDDCD-\uDDCF\uDDD6-\uDDDD])(?:\uD83C[\uDFFB-\uDFFF])|(?:\u26F9|\uD83C[\uDFCB\uDFCC]|\uD83D\uDD75)(?:\uD83C[\uDFFB-\uDFFF])|(?:[\u261D\u270A-\u270D]|\uD83C[\uDF85\uDFC2\uDFC7]|\uD83D[\uDC42\uDC43\uDC46-\uDC50\uDC66\uDC67\uDC6B-\uDC6D\uDC70\uDC72\uDC74-\uDC76\uDC78\uDC7C\uDC83\uDC85\uDCAA\uDD74\uDD7A\uDD90\uDD95\uDD96\uDE4C\uDE4F\uDEC0\uDECC]|\uD83E[\uDD0F\uDD18-\uDD1C\uDD1E\uDD1F\uDD30-\uDD36\uDDB5\uDDB6\uDDBB\uDDD2-\uDDD5])(?:\uD83C[\uDFFB-\uDFFF])|(?:[\u231A\u231B\u23E9-\u23EC\u23F0\u23F3\u25FD\u25FE\u2614\u2615\u2648-\u2653\u267F\u2693\u26A1\u26AA\u26AB\u26BD\u26BE\u26C4\u26C5\u26CE\u26D4\u26EA\u26F2\u26F3\u26F5\u26FA\u26FD\u2705\u270A\u270B\u2728\u274C\u274E\u2753-\u2755\u2757\u2795-\u2797\u27B0\u27BF\u2B1B\u2B1C\u2B50\u2B55]|\uD83C[\uDC04\uDCCF\uDD8E\uDD91-\uDD9A\uDDE6-\uDDFF\uDE01\uDE1A\uDE2F\uDE32-\uDE36\uDE38-\uDE3A\uDE50\uDE51\uDF00-\uDF20\uDF2D-\uDF35\uDF37-\uDF7C\uDF7E-\uDF93\uDFA0-\uDFCA\uDFCF-\uDFD3\uDFE0-\uDFF0\uDFF4\uDFF8-\uDFFF]|\uD83D[\uDC00-\uDC3E\uDC40\uDC42-\uDCFC\uDCFF-\uDD3D\uDD4B-\uDD4E\uDD50-\uDD67\uDD7A\uDD95\uDD96\uDDA4\uDDFB-\uDE4F\uDE80-\uDEC5\uDECC\uDED0-\uDED2\uDED5\uDEEB\uDEEC\uDEF4-\uDEFA\uDFE0-\uDFEB]|\uD83E[\uDD0D-\uDD3A\uDD3C-\uDD45\uDD47-\uDD71\uDD73-\uDD76\uDD7A-\uDDA2\uDDA5-\uDDAA\uDDAE-\uDDCA\uDDCD-\uDDFF\uDE70-\uDE73\uDE78-\uDE7A\uDE80-\uDE82\uDE90-\uDE95])|(?:[#\*0-9\xA9\xAE\u203C\u2049\u2122\u2139\u2194-\u2199\u21A9\u21AA\u231A\u231B\u2328\u23CF\u23E9-\u23F3\u23F8-\u23FA\u24C2\u25AA\u25AB\u25B6\u25C0\u25FB-\u25FE\u2600-\u2604\u260E\u2611\u2614\u2615\u2618\u261D\u2620\u2622\u2623\u2626\u262A\u262E\u262F\u2638-\u263A\u2640\u2642\u2648-\u2653\u265F\u2660\u2663\u2665\u2666\u2668\u267B\u267E\u267F\u2692-\u2697\u2699\u269B\u269C\u26A0\u26A1\u26AA\u26AB\u26B0\u26B1\u26BD\u26BE\u26C4\u26C5\u26C8\u26CE\u26CF\u26D1\u26D3\u26D4\u26E9\u26EA\u26F0-\u26F5\u26F7-\u26FA\u26FD\u2702\u2705\u2708-\u270D\u270F\u2712\u2714\u2716\u271D\u2721\u2728\u2733\u2734\u2744\u2747\u274C\u274E\u2753-\u2755\u2757\u2763\u2764\u2795-\u2797\u27A1\u27B0\u27BF\u2934\u2935\u2B05-\u2B07\u2B1B\u2B1C\u2B50\u2B55\u3030\u303D\u3297\u3299]|\uD83C[\uDC04\uDCCF\uDD70\uDD71\uDD7E\uDD7F\uDD8E\uDD91-\uDD9A\uDDE6-\uDDFF\uDE01\uDE02\uDE1A\uDE2F\uDE32-\uDE3A\uDE50\uDE51\uDF00-\uDF21\uDF24-\uDF93\uDF96\uDF97\uDF99-\uDF9B\uDF9E-\uDFF0\uDFF3-\uDFF5\uDFF7-\uDFFF]|\uD83D[\uDC00-\uDCFD\uDCFF-\uDD3D\uDD49-\uDD4E\uDD50-\uDD67\uDD6F\uDD70\uDD73-\uDD7A\uDD87\uDD8A-\uDD8D\uDD90\uDD95\uDD96\uDDA4\uDDA5\uDDA8\uDDB1\uDDB2\uDDBC\uDDC2-\uDDC4\uDDD1-\uDDD3\uDDDC-\uDDDE\uDDE1\uDDE3\uDDE8\uDDEF\uDDF3\uDDFA-\uDE4F\uDE80-\uDEC5\uDECB-\uDED2\uDED5\uDEE0-\uDEE5\uDEE9\uDEEB\uDEEC\uDEF0\uDEF3-\uDEFA\uDFE0-\uDFEB]|\uD83E[\uDD0D-\uDD3A\uDD3C-\uDD45\uDD47-\uDD71\uDD73-\uDD76\uDD7A-\uDDA2\uDDA5-\uDDAA\uDDAE-\uDDCA\uDDCD-\uDDFF\uDE70-\uDE73\uDE78-\uDE7A\uDE80-\uDE82\uDE90-\uDE95])\uFE0F|(?:[\u261D\u26F9\u270A-\u270D]|\uD83C[\uDF85\uDFC2-\uDFC4\uDFC7\uDFCA-\uDFCC]|\uD83D[\uDC42\uDC43\uDC46-\uDC50\uDC66-\uDC78\uDC7C\uDC81-\uDC83\uDC85-\uDC87\uDC8F\uDC91\uDCAA\uDD74\uDD75\uDD7A\uDD90\uDD95\uDD96\uDE45-\uDE47\uDE4B-\uDE4F\uDEA3\uDEB4-\uDEB6\uDEC0\uDECC]|\uD83E[\uDD0F\uDD18-\uDD1F\uDD26\uDD30-\uDD39\uDD3C-\uDD3E\uDDB5\uDDB6\uDDB8\uDDB9\uDDBB\uDDCD-\uDDCF\uDDD1-\uDDDD])/g; +}; diff --git a/node_modules/emoji-regex/package.json b/node_modules/emoji-regex/package.json new file mode 100644 index 0000000000000000000000000000000000000000..6d323528292b00502f47fc568556bed091ec003f --- /dev/null +++ b/node_modules/emoji-regex/package.json @@ -0,0 +1,50 @@ +{ + "name": "emoji-regex", + "version": "8.0.0", + "description": "A regular expression to match all Emoji-only symbols as per the Unicode Standard.", + "homepage": "https://mths.be/emoji-regex", + "main": "index.js", + "types": "index.d.ts", + "keywords": [ + "unicode", + "regex", + "regexp", + "regular expressions", + "code points", + "symbols", + "characters", + "emoji" + ], + "license": "MIT", + "author": { + "name": "Mathias Bynens", + "url": "https://mathiasbynens.be/" + }, + "repository": { + "type": "git", + "url": "https://github.com/mathiasbynens/emoji-regex.git" + }, + "bugs": "https://github.com/mathiasbynens/emoji-regex/issues", + "files": [ + "LICENSE-MIT.txt", + "index.js", + "index.d.ts", + "text.js", + "es2015/index.js", + "es2015/text.js" + ], + "scripts": { + "build": "rm -rf -- es2015; babel src -d .; NODE_ENV=es2015 babel src -d ./es2015; node script/inject-sequences.js", + "test": "mocha", + "test:watch": "npm run test -- --watch" + }, + "devDependencies": { + "@babel/cli": "^7.2.3", + "@babel/core": "^7.3.4", + "@babel/plugin-proposal-unicode-property-regex": "^7.2.0", + "@babel/preset-env": "^7.3.4", + "mocha": "^6.0.2", + "regexgen": "^1.3.0", + "unicode-12.0.0": "^0.7.9" + } +} diff --git a/node_modules/emoji-regex/text.js b/node_modules/emoji-regex/text.js new file mode 100644 index 0000000000000000000000000000000000000000..0a55ce2f2308adf2788f93ad799d9d513697f48f --- /dev/null +++ b/node_modules/emoji-regex/text.js @@ -0,0 +1,6 @@ +"use strict"; + +module.exports = function () { + // https://mths.be/emoji + return /\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62(?:\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67|\uDB40\uDC73\uDB40\uDC63\uDB40\uDC74|\uDB40\uDC77\uDB40\uDC6C\uDB40\uDC73)\uDB40\uDC7F|\uD83D\uDC68(?:\uD83C\uDFFC\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68\uD83C\uDFFB|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFF\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFE])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFE\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFD])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFD\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB\uDFFC])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\u200D(?:\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D)?\uD83D\uDC68|(?:\uD83D[\uDC68\uDC69])\u200D(?:\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67]))|\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67])|(?:\uD83D[\uDC68\uDC69])\u200D(?:\uD83D[\uDC66\uDC67])|[\u2695\u2696\u2708]\uFE0F|\uD83D[\uDC66\uDC67]|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|(?:\uD83C\uDFFB\u200D[\u2695\u2696\u2708]|\uD83C\uDFFF\u200D[\u2695\u2696\u2708]|\uD83C\uDFFE\u200D[\u2695\u2696\u2708]|\uD83C\uDFFD\u200D[\u2695\u2696\u2708]|\uD83C\uDFFC\u200D[\u2695\u2696\u2708])\uFE0F|\uD83C\uDFFB\u200D(?:\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C[\uDFFB-\uDFFF])|(?:\uD83E\uDDD1\uD83C\uDFFB\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFC\u200D\uD83E\uDD1D\u200D\uD83D\uDC69)\uD83C\uDFFB|\uD83E\uDDD1(?:\uD83C\uDFFF\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1(?:\uD83C[\uDFFB-\uDFFF])|\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1)|(?:\uD83E\uDDD1\uD83C\uDFFE\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFF\u200D\uD83E\uDD1D\u200D(?:\uD83D[\uDC68\uDC69]))(?:\uD83C[\uDFFB-\uDFFE])|(?:\uD83E\uDDD1\uD83C\uDFFC\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFD\u200D\uD83E\uDD1D\u200D\uD83D\uDC69)(?:\uD83C[\uDFFB\uDFFC])|\uD83D\uDC69(?:\uD83C\uDFFE\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFD\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFC\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB\uDFFD-\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFB\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFC-\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFD\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB\uDFFC\uDFFE\uDFFF])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\u200D(?:\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D(?:\uD83D[\uDC68\uDC69])|\uD83D[\uDC68\uDC69])|\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFF\u200D(?:\uD83C[\uDF3E\uDF73\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD]))|\uD83D\uDC69\u200D\uD83D\uDC69\u200D(?:\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67]))|(?:\uD83E\uDDD1\uD83C\uDFFD\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFE\u200D\uD83E\uDD1D\u200D\uD83D\uDC69)(?:\uD83C[\uDFFB-\uDFFD])|\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC69\u200D\uD83D\uDC69\u200D(?:\uD83D[\uDC66\uDC67])|(?:\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8|\uD83D\uDC69(?:\uD83C\uDFFF\u200D[\u2695\u2696\u2708]|\uD83C\uDFFE\u200D[\u2695\u2696\u2708]|\uD83C\uDFFC\u200D[\u2695\u2696\u2708]|\uD83C\uDFFB\u200D[\u2695\u2696\u2708]|\uD83C\uDFFD\u200D[\u2695\u2696\u2708]|\u200D[\u2695\u2696\u2708])|(?:(?:\u26F9|\uD83C[\uDFCB\uDFCC]|\uD83D\uDD75)\uFE0F|\uD83D\uDC6F|\uD83E[\uDD3C\uDDDE\uDDDF])\u200D[\u2640\u2642]|(?:\u26F9|\uD83C[\uDFCB\uDFCC]|\uD83D\uDD75)(?:\uD83C[\uDFFB-\uDFFF])\u200D[\u2640\u2642]|(?:\uD83C[\uDFC3\uDFC4\uDFCA]|\uD83D[\uDC6E\uDC71\uDC73\uDC77\uDC81\uDC82\uDC86\uDC87\uDE45-\uDE47\uDE4B\uDE4D\uDE4E\uDEA3\uDEB4-\uDEB6]|\uD83E[\uDD26\uDD37-\uDD39\uDD3D\uDD3E\uDDB8\uDDB9\uDDCD-\uDDCF\uDDD6-\uDDDD])(?:(?:\uD83C[\uDFFB-\uDFFF])\u200D[\u2640\u2642]|\u200D[\u2640\u2642])|\uD83C\uDFF4\u200D\u2620)\uFE0F|\uD83D\uDC69\u200D\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67])|\uD83C\uDFF3\uFE0F\u200D\uD83C\uDF08|\uD83D\uDC15\u200D\uD83E\uDDBA|\uD83D\uDC69\u200D\uD83D\uDC66|\uD83D\uDC69\u200D\uD83D\uDC67|\uD83C\uDDFD\uD83C\uDDF0|\uD83C\uDDF4\uD83C\uDDF2|\uD83C\uDDF6\uD83C\uDDE6|[#\*0-9]\uFE0F\u20E3|\uD83C\uDDE7(?:\uD83C[\uDDE6\uDDE7\uDDE9-\uDDEF\uDDF1-\uDDF4\uDDF6-\uDDF9\uDDFB\uDDFC\uDDFE\uDDFF])|\uD83C\uDDF9(?:\uD83C[\uDDE6\uDDE8\uDDE9\uDDEB-\uDDED\uDDEF-\uDDF4\uDDF7\uDDF9\uDDFB\uDDFC\uDDFF])|\uD83C\uDDEA(?:\uD83C[\uDDE6\uDDE8\uDDEA\uDDEC\uDDED\uDDF7-\uDDFA])|\uD83E\uDDD1(?:\uD83C[\uDFFB-\uDFFF])|\uD83C\uDDF7(?:\uD83C[\uDDEA\uDDF4\uDDF8\uDDFA\uDDFC])|\uD83D\uDC69(?:\uD83C[\uDFFB-\uDFFF])|\uD83C\uDDF2(?:\uD83C[\uDDE6\uDDE8-\uDDED\uDDF0-\uDDFF])|\uD83C\uDDE6(?:\uD83C[\uDDE8-\uDDEC\uDDEE\uDDF1\uDDF2\uDDF4\uDDF6-\uDDFA\uDDFC\uDDFD\uDDFF])|\uD83C\uDDF0(?:\uD83C[\uDDEA\uDDEC-\uDDEE\uDDF2\uDDF3\uDDF5\uDDF7\uDDFC\uDDFE\uDDFF])|\uD83C\uDDED(?:\uD83C[\uDDF0\uDDF2\uDDF3\uDDF7\uDDF9\uDDFA])|\uD83C\uDDE9(?:\uD83C[\uDDEA\uDDEC\uDDEF\uDDF0\uDDF2\uDDF4\uDDFF])|\uD83C\uDDFE(?:\uD83C[\uDDEA\uDDF9])|\uD83C\uDDEC(?:\uD83C[\uDDE6\uDDE7\uDDE9-\uDDEE\uDDF1-\uDDF3\uDDF5-\uDDFA\uDDFC\uDDFE])|\uD83C\uDDF8(?:\uD83C[\uDDE6-\uDDEA\uDDEC-\uDDF4\uDDF7-\uDDF9\uDDFB\uDDFD-\uDDFF])|\uD83C\uDDEB(?:\uD83C[\uDDEE-\uDDF0\uDDF2\uDDF4\uDDF7])|\uD83C\uDDF5(?:\uD83C[\uDDE6\uDDEA-\uDDED\uDDF0-\uDDF3\uDDF7-\uDDF9\uDDFC\uDDFE])|\uD83C\uDDFB(?:\uD83C[\uDDE6\uDDE8\uDDEA\uDDEC\uDDEE\uDDF3\uDDFA])|\uD83C\uDDF3(?:\uD83C[\uDDE6\uDDE8\uDDEA-\uDDEC\uDDEE\uDDF1\uDDF4\uDDF5\uDDF7\uDDFA\uDDFF])|\uD83C\uDDE8(?:\uD83C[\uDDE6\uDDE8\uDDE9\uDDEB-\uDDEE\uDDF0-\uDDF5\uDDF7\uDDFA-\uDDFF])|\uD83C\uDDF1(?:\uD83C[\uDDE6-\uDDE8\uDDEE\uDDF0\uDDF7-\uDDFB\uDDFE])|\uD83C\uDDFF(?:\uD83C[\uDDE6\uDDF2\uDDFC])|\uD83C\uDDFC(?:\uD83C[\uDDEB\uDDF8])|\uD83C\uDDFA(?:\uD83C[\uDDE6\uDDEC\uDDF2\uDDF3\uDDF8\uDDFE\uDDFF])|\uD83C\uDDEE(?:\uD83C[\uDDE8-\uDDEA\uDDF1-\uDDF4\uDDF6-\uDDF9])|\uD83C\uDDEF(?:\uD83C[\uDDEA\uDDF2\uDDF4\uDDF5])|(?:\uD83C[\uDFC3\uDFC4\uDFCA]|\uD83D[\uDC6E\uDC71\uDC73\uDC77\uDC81\uDC82\uDC86\uDC87\uDE45-\uDE47\uDE4B\uDE4D\uDE4E\uDEA3\uDEB4-\uDEB6]|\uD83E[\uDD26\uDD37-\uDD39\uDD3D\uDD3E\uDDB8\uDDB9\uDDCD-\uDDCF\uDDD6-\uDDDD])(?:\uD83C[\uDFFB-\uDFFF])|(?:\u26F9|\uD83C[\uDFCB\uDFCC]|\uD83D\uDD75)(?:\uD83C[\uDFFB-\uDFFF])|(?:[\u261D\u270A-\u270D]|\uD83C[\uDF85\uDFC2\uDFC7]|\uD83D[\uDC42\uDC43\uDC46-\uDC50\uDC66\uDC67\uDC6B-\uDC6D\uDC70\uDC72\uDC74-\uDC76\uDC78\uDC7C\uDC83\uDC85\uDCAA\uDD74\uDD7A\uDD90\uDD95\uDD96\uDE4C\uDE4F\uDEC0\uDECC]|\uD83E[\uDD0F\uDD18-\uDD1C\uDD1E\uDD1F\uDD30-\uDD36\uDDB5\uDDB6\uDDBB\uDDD2-\uDDD5])(?:\uD83C[\uDFFB-\uDFFF])|(?:[\u231A\u231B\u23E9-\u23EC\u23F0\u23F3\u25FD\u25FE\u2614\u2615\u2648-\u2653\u267F\u2693\u26A1\u26AA\u26AB\u26BD\u26BE\u26C4\u26C5\u26CE\u26D4\u26EA\u26F2\u26F3\u26F5\u26FA\u26FD\u2705\u270A\u270B\u2728\u274C\u274E\u2753-\u2755\u2757\u2795-\u2797\u27B0\u27BF\u2B1B\u2B1C\u2B50\u2B55]|\uD83C[\uDC04\uDCCF\uDD8E\uDD91-\uDD9A\uDDE6-\uDDFF\uDE01\uDE1A\uDE2F\uDE32-\uDE36\uDE38-\uDE3A\uDE50\uDE51\uDF00-\uDF20\uDF2D-\uDF35\uDF37-\uDF7C\uDF7E-\uDF93\uDFA0-\uDFCA\uDFCF-\uDFD3\uDFE0-\uDFF0\uDFF4\uDFF8-\uDFFF]|\uD83D[\uDC00-\uDC3E\uDC40\uDC42-\uDCFC\uDCFF-\uDD3D\uDD4B-\uDD4E\uDD50-\uDD67\uDD7A\uDD95\uDD96\uDDA4\uDDFB-\uDE4F\uDE80-\uDEC5\uDECC\uDED0-\uDED2\uDED5\uDEEB\uDEEC\uDEF4-\uDEFA\uDFE0-\uDFEB]|\uD83E[\uDD0D-\uDD3A\uDD3C-\uDD45\uDD47-\uDD71\uDD73-\uDD76\uDD7A-\uDDA2\uDDA5-\uDDAA\uDDAE-\uDDCA\uDDCD-\uDDFF\uDE70-\uDE73\uDE78-\uDE7A\uDE80-\uDE82\uDE90-\uDE95])|(?:[#\*0-9\xA9\xAE\u203C\u2049\u2122\u2139\u2194-\u2199\u21A9\u21AA\u231A\u231B\u2328\u23CF\u23E9-\u23F3\u23F8-\u23FA\u24C2\u25AA\u25AB\u25B6\u25C0\u25FB-\u25FE\u2600-\u2604\u260E\u2611\u2614\u2615\u2618\u261D\u2620\u2622\u2623\u2626\u262A\u262E\u262F\u2638-\u263A\u2640\u2642\u2648-\u2653\u265F\u2660\u2663\u2665\u2666\u2668\u267B\u267E\u267F\u2692-\u2697\u2699\u269B\u269C\u26A0\u26A1\u26AA\u26AB\u26B0\u26B1\u26BD\u26BE\u26C4\u26C5\u26C8\u26CE\u26CF\u26D1\u26D3\u26D4\u26E9\u26EA\u26F0-\u26F5\u26F7-\u26FA\u26FD\u2702\u2705\u2708-\u270D\u270F\u2712\u2714\u2716\u271D\u2721\u2728\u2733\u2734\u2744\u2747\u274C\u274E\u2753-\u2755\u2757\u2763\u2764\u2795-\u2797\u27A1\u27B0\u27BF\u2934\u2935\u2B05-\u2B07\u2B1B\u2B1C\u2B50\u2B55\u3030\u303D\u3297\u3299]|\uD83C[\uDC04\uDCCF\uDD70\uDD71\uDD7E\uDD7F\uDD8E\uDD91-\uDD9A\uDDE6-\uDDFF\uDE01\uDE02\uDE1A\uDE2F\uDE32-\uDE3A\uDE50\uDE51\uDF00-\uDF21\uDF24-\uDF93\uDF96\uDF97\uDF99-\uDF9B\uDF9E-\uDFF0\uDFF3-\uDFF5\uDFF7-\uDFFF]|\uD83D[\uDC00-\uDCFD\uDCFF-\uDD3D\uDD49-\uDD4E\uDD50-\uDD67\uDD6F\uDD70\uDD73-\uDD7A\uDD87\uDD8A-\uDD8D\uDD90\uDD95\uDD96\uDDA4\uDDA5\uDDA8\uDDB1\uDDB2\uDDBC\uDDC2-\uDDC4\uDDD1-\uDDD3\uDDDC-\uDDDE\uDDE1\uDDE3\uDDE8\uDDEF\uDDF3\uDDFA-\uDE4F\uDE80-\uDEC5\uDECB-\uDED2\uDED5\uDEE0-\uDEE5\uDEE9\uDEEB\uDEEC\uDEF0\uDEF3-\uDEFA\uDFE0-\uDFEB]|\uD83E[\uDD0D-\uDD3A\uDD3C-\uDD45\uDD47-\uDD71\uDD73-\uDD76\uDD7A-\uDDA2\uDDA5-\uDDAA\uDDAE-\uDDCA\uDDCD-\uDDFF\uDE70-\uDE73\uDE78-\uDE7A\uDE80-\uDE82\uDE90-\uDE95])\uFE0F?|(?:[\u261D\u26F9\u270A-\u270D]|\uD83C[\uDF85\uDFC2-\uDFC4\uDFC7\uDFCA-\uDFCC]|\uD83D[\uDC42\uDC43\uDC46-\uDC50\uDC66-\uDC78\uDC7C\uDC81-\uDC83\uDC85-\uDC87\uDC8F\uDC91\uDCAA\uDD74\uDD75\uDD7A\uDD90\uDD95\uDD96\uDE45-\uDE47\uDE4B-\uDE4F\uDEA3\uDEB4-\uDEB6\uDEC0\uDECC]|\uD83E[\uDD0F\uDD18-\uDD1F\uDD26\uDD30-\uDD39\uDD3C-\uDD3E\uDDB5\uDDB6\uDDB8\uDDB9\uDDBB\uDDCD-\uDDCF\uDDD1-\uDDDD])/g; +}; diff --git a/node_modules/gearhash-jit/README.md b/node_modules/gearhash-jit/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f0cb87d3dde66a2e7cab9496e8b87348b925e977 --- /dev/null +++ b/node_modules/gearhash-jit/README.md @@ -0,0 +1,64 @@ +# gearhash-jit + +Fast [GEAR rolling hash](https://en.wikipedia.org/wiki/Rolling_hash) for content-defined chunking (CDC), using hand-written WebAssembly with native `i64` arithmetic. + +Replaces the deprecated `@huggingface/gearhash-wasm` (AssemblyScript) package. + +## How it works + +At init time, a tiny WASM module (~120 bytes of bytecode) is generated and compiled synchronously. The inner loop uses native 64-bit integer operations (`i64.shl`, `i64.add`, `i64.and`) — single-cycle instructions that avoid the overhead of JavaScript `BigInt`. + +## Usage + +```typescript +import { Hasher } from 'gearhash-jit'; + +const mask = 0x0000d90003530000n; // CDC target mask +const hasher = new Hasher(mask); + +// Scan for a chunk boundary +const pos = hasher.nextMatch(buffer); +if (pos !== -1) { + // Boundary found at byte `pos` (1-based) +} + +// Read the rolling hash state (8 LE bytes, zero-copy) +console.log(hasher.hash); + +// Reset for the next chunk +hasher.resetHash(); +``` + +### Streaming + +The hash state carries over between `nextMatch` calls, so you can scan data in pieces: + +```typescript +const hasher = new Hasher(mask); + +for (const chunk of dataSource) { + const pos = hasher.nextMatch(chunk); + if (pos !== -1) { + // Found boundary at `pos` within this chunk + hasher.resetHash(); + } +} +``` + +## API + +### `new Hasher(mask: bigint)` + +Create a hasher with the given 64-bit CDC mask. + +### `hasher.nextMatch(buf: Uint8Array): number` + +Scan `buf` for the next match. Returns a 1-based byte position, or `-1` if no match. + +### `hasher.hash: Uint8Array` + +The current 64-bit rolling hash state as 8 little-endian bytes. Updated after every `nextMatch` call. + +### `hasher.resetHash(): void` + +Reset the rolling hash to zero (call when starting a new chunk). diff --git a/node_modules/gearhash-jit/dist/commonjs/index.d.ts b/node_modules/gearhash-jit/dist/commonjs/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d054edc02abc2d8ff61b5ac2ddd59d9f7e3eee73 --- /dev/null +++ b/node_modules/gearhash-jit/dist/commonjs/index.d.ts @@ -0,0 +1,27 @@ +/** + * gearhash-jit — Fast GEAR rolling hash for content-defined chunking. + * + * Uses a tiny hand-written WASM module with native i64 arithmetic. + * The hash state is kept as raw bytes in JS (avoiding BigInt in the hot path) + * and written to WASM memory only for the `nextMatch` call. + */ +export { GEAR_TABLE } from "./table.js"; +export declare class Hasher { + private readonly maskBytes; + /** + * The current 64-bit rolling hash state as 8 little-endian bytes. + * Updated after every `nextMatch` call. Zeroed by `resetHash()`. + */ + readonly hash: Uint8Array; + constructor(mask: bigint); + /** + * Scan `buf` for the next gear-hash match. The internal hash state + * carries over between calls (for split-buffer scanning). + * + * @returns 1-based byte position of the match, or -1 if none found. + */ + nextMatch(buf: Uint8Array): number; + /** Reset rolling hash to zero (call when starting a new chunk). */ + resetHash(): void; +} +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/gearhash-jit/dist/commonjs/index.d.ts.map b/node_modules/gearhash-jit/dist/commonjs/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5aed9755fe0d803d1a33788e532a7901da282c3c --- /dev/null +++ b/node_modules/gearhash-jit/dist/commonjs/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAYH,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,qBAAa,MAAM;IACjB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IAEvC;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;gBAEd,IAAI,EAAE,MAAM;IAOxB;;;;;OAKG;IACH,SAAS,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM;IAkBlC,mEAAmE;IACnE,SAAS,IAAI,IAAI;CAGlB"} \ No newline at end of file diff --git a/node_modules/gearhash-jit/dist/commonjs/index.js b/node_modules/gearhash-jit/dist/commonjs/index.js new file mode 100644 index 0000000000000000000000000000000000000000..62a751e36350328e64bec596c1db382445f1e82a --- /dev/null +++ b/node_modules/gearhash-jit/dist/commonjs/index.js @@ -0,0 +1,53 @@ +"use strict"; +/** + * gearhash-jit — Fast GEAR rolling hash for content-defined chunking. + * + * Uses a tiny hand-written WASM module with native i64 arithmetic. + * The hash state is kept as raw bytes in JS (avoiding BigInt in the hot path) + * and written to WASM memory only for the `nextMatch` call. + */ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Hasher = exports.GEAR_TABLE = void 0; +const wasm_js_1 = require("./wasm.js"); +var table_js_1 = require("./table.js"); +Object.defineProperty(exports, "GEAR_TABLE", { enumerable: true, get: function () { return table_js_1.GEAR_TABLE; } }); +class Hasher { + maskBytes; + /** + * The current 64-bit rolling hash state as 8 little-endian bytes. + * Updated after every `nextMatch` call. Zeroed by `resetHash()`. + */ + hash; + constructor(mask) { + (0, wasm_js_1.initWasm)(); + this.maskBytes = new Uint8Array(8); + this.hash = new Uint8Array(8); + new DataView(this.maskBytes.buffer).setBigUint64(0, mask, true); + } + /** + * Scan `buf` for the next gear-hash match. The internal hash state + * carries over between calls (for split-buffer scanning). + * + * @returns 1-based byte position of the match, or -1 if none found. + */ + nextMatch(buf) { + const len = buf.length; + if (len === 0) + return -1; + if (len > wasm_js_1.MAX_INPUT_SIZE) { + throw new RangeError(`Input too large: ${len} > ${wasm_js_1.MAX_INPUT_SIZE}`); + } + const view = (0, wasm_js_1.getView)(); + view.set(this.hash, wasm_js_1.HASH_OFFSET); + view.set(this.maskBytes, wasm_js_1.MASK_OFFSET); + view.set(buf, wasm_js_1.INPUT_OFFSET); + const pos = (0, wasm_js_1.wasmNextMatch)(wasm_js_1.INPUT_OFFSET, len); + this.hash.set(view.subarray(wasm_js_1.HASH_OFFSET, wasm_js_1.HASH_OFFSET + 8)); + return pos; + } + /** Reset rolling hash to zero (call when starting a new chunk). */ + resetHash() { + this.hash.fill(0); + } +} +exports.Hasher = Hasher; diff --git a/node_modules/gearhash-jit/dist/commonjs/package.json b/node_modules/gearhash-jit/dist/commonjs/package.json new file mode 100644 index 0000000000000000000000000000000000000000..5bbefffbabee392d1855491b84dc0a716b6a3bf2 --- /dev/null +++ b/node_modules/gearhash-jit/dist/commonjs/package.json @@ -0,0 +1,3 @@ +{ + "type": "commonjs" +} diff --git a/node_modules/gearhash-jit/dist/commonjs/table.d.ts b/node_modules/gearhash-jit/dist/commonjs/table.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc230b86793ef7d36a95296f21901c96a72fbb93 --- /dev/null +++ b/node_modules/gearhash-jit/dist/commonjs/table.d.ts @@ -0,0 +1,2 @@ +export declare const GEAR_TABLE: bigint[]; +//# sourceMappingURL=table.d.ts.map \ No newline at end of file diff --git a/node_modules/gearhash-jit/dist/commonjs/table.d.ts.map b/node_modules/gearhash-jit/dist/commonjs/table.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f0579443bfbbb7d2e059295be6d424add9923c43 --- /dev/null +++ b/node_modules/gearhash-jit/dist/commonjs/table.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"table.d.ts","sourceRoot":"","sources":["../../src/table.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,UAAU,EAAE,MAAM,EAiE9B,CAAC"} \ No newline at end of file diff --git a/node_modules/gearhash-jit/dist/commonjs/table.js b/node_modules/gearhash-jit/dist/commonjs/table.js new file mode 100644 index 0000000000000000000000000000000000000000..7cbf87cfa0fd84a823d8b563e805d5d2a3ef2f4d --- /dev/null +++ b/node_modules/gearhash-jit/dist/commonjs/table.js @@ -0,0 +1,71 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GEAR_TABLE = void 0; +/* eslint-disable */ +// prettier-ignore +exports.GEAR_TABLE = [ + 0xb088d3a9e840f559n, 0x5652c7f739ed20d6n, 0x45b28969898972abn, 0x6b0a89d5b68ec777n, + 0x368f573e8b7a31b7n, 0x1dc636dce936d94bn, 0x207a4c4e5554d5b6n, 0xa474b34628239acbn, + 0x3b06a83e1ca3b912n, 0x90e78d6c2f02baf7n, 0xe1c92df7150d9a8an, 0x8e95053a1086d3adn, + 0x5a2ef4f1b83a0722n, 0xa50fac949f807faen, 0x0e7303eb80d8d681n, 0x99b07edc1570ad0fn, + 0x689d2fb555fd3076n, 0x00005082119ea468n, 0xc4b08306a88fcc28n, 0x3eb0678af6374afdn, + 0xf19f87ab86ad7436n, 0xf2129fbfbe6bc736n, 0x481149575c98a4edn, 0x0000010695477bc5n, + 0x1fba37801a9ceaccn, 0x3bf06fd663a49b6dn, 0x99687e9782e3874bn, 0x79a10673aa50d8e3n, + 0xe4accf9e6211f420n, 0x2520e71f87579071n, 0x2bd5d3fd781a8a9bn, 0x00de4dcddd11c873n, + 0xeaa9311c5a87392fn, 0xdb748eb617bc40ffn, 0xaf579a8df620bf6fn, 0x86a6e5da1b09c2b1n, + 0xcc2fc30ac322a12en, 0x355e2afec1f74267n, 0x2d99c8f4c021a47bn, 0xbade4b4a9404cfc3n, + 0xf7b518721d707d69n, 0x3286b6587bf32c20n, 0x0000b68886af270cn, 0xa115d6e4db8a9079n, + 0x484f7e9c97b2e199n, 0xccca7bb75713e301n, 0xbf2584a62bb0f160n, 0xade7e813625dbcc8n, + 0x000070940d87955an, 0x8ae69108139e626fn, 0xbd776ad72fde38a2n, 0xfb6b001fc2fcc0cfn, + 0xc7a474b8e67bc427n, 0xbaf6f11610eb5d58n, 0x09cb1f5b6de770d1n, 0xb0b219e6977d4c47n, + 0x00ccbc386ea7ad4an, 0xcc849d0adf973f01n, 0x73a3ef7d016af770n, 0xc807d2d386bdbdfen, + 0x7f2ac9966c791730n, 0xd037a86bc6c504dan, 0xf3f17c661eaa609dn, 0xaca626b04daae687n, + 0x755a99374f4a5b07n, 0x90837ee65b2caeden, 0x6ee8ad93fd560785n, 0x0000d9e11053edd8n, + 0x9e063bb2d21cdbd7n, 0x07ab77f12a01d2b2n, 0xec550255e6641b44n, 0x78fb94a8449c14c6n, + 0xc7510e1bc6c0f5f5n, 0x0000320b36e4cae3n, 0x827c33262c8b1a2dn, 0x14675f0b48ea4144n, + 0x267bd3a6498decebn, 0xf1916ff982f5035en, 0x86221b7ff434fb88n, 0x9dbecee7386f49d8n, + 0xea58f8cac80f8f4an, 0x008d198692fc64d8n, 0x6d38704fbabf9a36n, 0xe032cb07d1e7be4cn, + 0x228d21f6ad450890n, 0x635cb1bfc02589a5n, 0x4620a1739ca2ce71n, 0xa7e7dfe3aae5fb58n, + 0x0c10ca932b3c0debn, 0x2727fee884afed7bn, 0xa2df1c6df9e2ab1fn, 0x4dcdd1ac0774f523n, + 0x000070ffad33e24en, 0xa2ace87bc5977816n, 0x9892275ab4286049n, 0xc2861181ddf18959n, + 0xbb9972a042483e19n, 0xef70cd3766513078n, 0x00000513abfc9864n, 0xc058b61858c94083n, + 0x09e850859725e0den, 0x9197fb3bf83e7d94n, 0x7e1e626d12b64bcen, 0x520c54507f7b57d1n, + 0xbee1797174e22416n, 0x6fd9ac3222e95587n, 0x0023957c9adfbf3en, 0xa01c7d7e234bbe15n, + 0xaba2c758b8a38cbbn, 0x0d1fa0ceec3e2b30n, 0x0bb6a58b7e60b991n, 0x4333dd5b9fa26635n, + 0xc2fd3b7d4001c1a3n, 0xfb41802454731127n, 0x65a56185a50d18cbn, 0xf67a02bd8784b54fn, + 0x696f11dd67e65063n, 0x00002022fca814abn, 0x8cd6be912db9d852n, 0x695189b6e9ae8a57n, + 0xee9453b50ada0c28n, 0xd8fc5ea91a78845en, 0xab86bf191a4aa767n, 0x0000c6b5c86415e5n, + 0x267310178e08a22en, 0xed2d101b078bca25n, 0x3b41ed84b226a8fbn, 0x13e622120f28dc06n, + 0xa315f5ebfb706d26n, 0x8816c34e3301bacen, 0xe9395b9cbb71fdaen, 0x002ce9202e721648n, + 0x4283db1d2bb3c91cn, 0xd77d461ad2b1a6a5n, 0xe2ec17e46eeb866bn, 0xb8e0be4039fbc47cn, + 0xdea160c4d5299d04n, 0x7eec86c8d28c3634n, 0x2119ad129f98a399n, 0xa6ccf46b61a283efn, + 0x2c52cedef658c617n, 0x2db4871169acdd83n, 0x0000f0d6f39ecbe9n, 0x3dd5d8c98d2f9489n, + 0x8a1872a22b01f584n, 0xf282a4c40e7b3cf2n, 0x8020ec2ccb1ba196n, 0x6693b6e09e59e313n, + 0x0000ce19cc7c83ebn, 0x20cb5735f6479c3bn, 0x762ebf3759d75a5bn, 0x207bfe823d693975n, + 0xd77dc112339cd9d5n, 0x9ba7834284627d03n, 0x217dc513e95f51e9n, 0xb27b1a29fc5e7816n, + 0x00d5cd9831bb662dn, 0x71e39b806d75734cn, 0x7e572af006fb1a23n, 0xa2734f2f6ae91f85n, + 0xbf82c6b5022cddf2n, 0x5c3beac60761a0den, 0xcdc893bb47416998n, 0x6d1085615c187e01n, + 0x77f8ae30ac277c5dn, 0x917c6b81122a2c91n, 0x5b75b699add16967n, 0x0000cf6ae79a069bn, + 0xf3c40afa60de1104n, 0x2063127aa59167c3n, 0x621de62269d1894dn, 0xd188ac1de62b4726n, + 0x107036e2154b673cn, 0x0000b85f28553a1dn, 0xf2ef4e4c18236f3dn, 0xd9d6de6611b9f602n, + 0xa1fc7955fb47911cn, 0xeb85fd032f298dbdn, 0xbe27502fb3befae1n, 0xe3034251c4cd661en, + 0x441364d354071836n, 0x0082b36c75f2983en, 0xb145910316fa66f0n, 0x021c069c9847caf7n, + 0x2910dfc75a4b5221n, 0x735b353e1c57a8b5n, 0xce44312ce98ed96cn, 0xbc942e4506bdfa65n, + 0xf05086a71257941bn, 0xfec3b215d351ceadn, 0x00ae1055e0144202n, 0xf54b40846f42e454n, + 0x00007fd9c8bcbcc8n, 0xbfbd9ef317de9bfen, 0xa804302ff2854e12n, 0x39ce4957a5e5d8d4n, + 0xffb9e2a45637ba84n, 0x55b9ad1d9ea0818bn, 0x00008acbf319178an, 0x48e2bfc8d0fbfb38n, + 0x8be39841e848b5e8n, 0x0e2712160696a08bn, 0xd51096e84b44242an, 0x1101ba176792e13an, + 0xc22e770f4531689dn, 0x1689eff272bbc56cn, 0x00a92a197f5650ecn, 0xbc765990bda1784en, + 0xc61441e392fcb8aen, 0x07e13a2ced31e4a0n, 0x92cbe984234e9d4dn, 0x8f4ff572bb7d8ac5n, + 0x0b9670c00b963bd0n, 0x62955a581a03eb01n, 0x645f83e5ea000254n, 0x41fce516cd88f299n, + 0xbbda9748da7a98cfn, 0x0000aab2fe4845fan, 0x19761b069bf56555n, 0x8b8f5e8343b6ad56n, + 0x3e5d1cfd144821d9n, 0xec5c1e2ca2b0cd8fn, 0xfaf7e0fea7fbb57fn, 0x000000d3ba12961bn, + 0xda3f90178401b18en, 0x70ff906de33a5febn, 0x0527d5a7c06970e7n, 0x22d8e773607c13e9n, + 0xc9ab70df643c3bacn, 0xeda4c6dc8abe12e3n, 0xecef1f410033e78an, 0x0024c2b274ac72cbn, + 0x06740d954fa900b4n, 0x1d7a299b323d6304n, 0xb3c37cb298cbead5n, 0xc986e3c76178739bn, + 0x9fabea364b46f58an, 0x6da214c5af85cc56n, 0x17a43ed8b7a38f84n, 0x6eccec511d9adbebn, + 0xf9cab30913335afbn, 0x4a5e60c5f415eed2n, 0x00006967503672b4n, 0x9da51d121454bb87n, + 0x84321e13b9bbc816n, 0xfb3d6fb6ab2fdd8dn, 0x60305eed8e160a8dn, 0xcbbf4b14e9946ce8n, + 0x00004f63381b10c3n, 0x07d5b7816fcc4e10n, 0xe5a536726a6a8155n, 0x57afb23447a07fddn, + 0x18f346f7abc9d394n, 0x636dc655d61ad33dn, 0xcc8bab4939f7f3f6n, 0x63c7a906c1dd187bn, +]; diff --git a/node_modules/gearhash-jit/dist/commonjs/wasm.d.ts b/node_modules/gearhash-jit/dist/commonjs/wasm.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d0e001b5856bf8252085af5defbbbaadeb170d0a --- /dev/null +++ b/node_modules/gearhash-jit/dist/commonjs/wasm.d.ts @@ -0,0 +1,21 @@ +/** + * GEAR hash WASM - Runtime bytecode generation + * + * Generates a tiny WebAssembly module with a single `nextMatch` function + * that performs the gear hash rolling scan using native i64 arithmetic. + * + * Memory layout (all little-endian): + * 0-2047: Gear lookup table (256 × 8 bytes) + * 2048-2055: Hash state (u64, persists across calls) + * 2056-2063: Mask (u64, set per-hasher before each call) + * 4096+: Input buffer + */ +export declare const TABLE_OFFSET = 0; +export declare const HASH_OFFSET = 2048; +export declare const MASK_OFFSET = 2056; +export declare const INPUT_OFFSET = 4096; +export declare const MAX_INPUT_SIZE: number; +export declare function initWasm(): void; +export declare function wasmNextMatch(inputStart: number, inputLen: number): number; +export declare function getView(): Uint8Array; +//# sourceMappingURL=wasm.d.ts.map \ No newline at end of file diff --git a/node_modules/gearhash-jit/dist/commonjs/wasm.d.ts.map b/node_modules/gearhash-jit/dist/commonjs/wasm.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..25ad405e574804b5c6c7b1b054d0c84c4c5989a9 --- /dev/null +++ b/node_modules/gearhash-jit/dist/commonjs/wasm.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"wasm.d.ts","sourceRoot":"","sources":["../../src/wasm.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,eAAO,MAAM,YAAY,IAAI,CAAC;AAC9B,eAAO,MAAM,WAAW,OAAO,CAAC;AAChC,eAAO,MAAM,WAAW,OAAO,CAAC;AAChC,eAAO,MAAM,YAAY,OAAO,CAAC;AAEjC,eAAO,MAAM,cAAc,QAA+B,CAAC;AA8K3D,wBAAgB,QAAQ,IAAI,IAAI,CAc/B;AAED,wBAAgB,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAE1E;AAED,wBAAgB,OAAO,IAAI,UAAU,CAEpC"} \ No newline at end of file diff --git a/node_modules/gearhash-jit/dist/commonjs/wasm.js b/node_modules/gearhash-jit/dist/commonjs/wasm.js new file mode 100644 index 0000000000000000000000000000000000000000..ce461d3399f292b58ca0cffee1bcb9f7f5ccfc13 --- /dev/null +++ b/node_modules/gearhash-jit/dist/commonjs/wasm.js @@ -0,0 +1,186 @@ +"use strict"; +/** + * GEAR hash WASM - Runtime bytecode generation + * + * Generates a tiny WebAssembly module with a single `nextMatch` function + * that performs the gear hash rolling scan using native i64 arithmetic. + * + * Memory layout (all little-endian): + * 0-2047: Gear lookup table (256 × 8 bytes) + * 2048-2055: Hash state (u64, persists across calls) + * 2056-2063: Mask (u64, set per-hasher before each call) + * 4096+: Input buffer + */ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.MAX_INPUT_SIZE = exports.INPUT_OFFSET = exports.MASK_OFFSET = exports.HASH_OFFSET = exports.TABLE_OFFSET = void 0; +exports.initWasm = initWasm; +exports.wasmNextMatch = wasmNextMatch; +exports.getView = getView; +const table_js_1 = require("./table.js"); +exports.TABLE_OFFSET = 0; +exports.HASH_OFFSET = 2048; +exports.MASK_OFFSET = 2056; +exports.INPUT_OFFSET = 4096; +const PAGES = 8; // 512 KB +exports.MAX_INPUT_SIZE = PAGES * 65536 - exports.INPUT_OFFSET; +let wasmMemory = null; +let wasmView = null; +let wasmFn = null; +function toSignedLeb128(n) { + const bytes = []; + let value = n | 0; + for (;;) { + const byte = value & 0x7f; + value >>= 7; + if ((value === 0 && (byte & 0x40) === 0) || (value === -1 && (byte & 0x40) !== 0)) { + bytes.push(byte); + return bytes; + } + bytes.push(byte | 0x80); + } +} +function toLebU32Padded5(n) { + return [ + (n & 0x7f) | 0x80, + ((n >>> 7) & 0x7f) | 0x80, + ((n >>> 14) & 0x7f) | 0x80, + ((n >>> 21) & 0x7f) | 0x80, + (n >>> 28) & 0x0f, + ]; +} +/** + * Generate the WASM module bytecode. + * + * Exports one function: + * nextMatch(inputStart: i32, inputLen: i32) -> i32 + * + * Reads hash/mask from fixed memory offsets, scans from `inputStart` + * for `inputLen` bytes, writes updated hash back. + * Returns 1-based match position within the scanned range, or -1. + */ +function generateWasmBytes() { + const code = []; + function emit(...bytes) { + code.push(...bytes); + } + // ── Module header ── + emit(0x00, 0x61, 0x73, 0x6d); // magic + emit(0x01, 0x00, 0x00, 0x00); // version 1 + // ── Type section: (i32, i32) -> (i32) ── + emit(0x01, 0x07, 0x01, 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f); + // ── Import section: memory "js"."mem" min=PAGES ── + emit(0x02, 0x0b, 0x01, 0x02, 0x6a, 0x73, 0x03, 0x6d, 0x65, 0x6d, 0x02, 0x00, PAGES); + // ── Function section: 1 function, type 0 ── + emit(0x03, 0x02, 0x01, 0x00); + // ── Export section: "nextMatch" -> func 0 ── + emit(0x07, 0x0d, 0x01, 0x09, 0x6e, 0x65, 0x78, 0x74, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x00, 0x00); + // ── Code section ── + emit(0x0a); + const sectionSizeOff = code.length; + emit(0x00, 0x00, 0x00, 0x00, 0x00); + emit(0x01); // 1 function body + const funcSizeOff = code.length; + emit(0x00, 0x00, 0x00, 0x00, 0x00); + const bodyStart = code.length; + // Locals: $0 = inputStart (param), $1 = inputLen (param) + // $2 = hash (i64), $3 = mask (i64) + // $4 = ptr (i32), $5 = end (i32) + emit(0x02, 0x02, 0x7e, 0x02, 0x7f); + // Load hash from memory[HASH_OFFSET] + emit(0x41, ...toSignedLeb128(exports.HASH_OFFSET)); + emit(0x29, 0x03, 0x00); + emit(0x21, 0x02); + // Load mask from memory[MASK_OFFSET] + emit(0x41, ...toSignedLeb128(exports.MASK_OFFSET)); + emit(0x29, 0x03, 0x00); + emit(0x21, 0x03); + // ptr = inputStart + emit(0x20, 0x00); + emit(0x21, 0x04); + // end = inputStart + inputLen + emit(0x20, 0x00); + emit(0x20, 0x01); + emit(0x6a); + emit(0x21, 0x05); + // block $done + emit(0x02, 0x40); + // loop $loop + emit(0x03, 0x40); + // if ptr >= end → break + emit(0x20, 0x04); + emit(0x20, 0x05); + emit(0x4e); + emit(0x0d, 0x01); + // hash = (hash << 1) + table[mem[ptr] * 8] + emit(0x20, 0x02); + emit(0x42, 0x01); + emit(0x86); + emit(0x20, 0x04); + emit(0x2d, 0x00, 0x00); + emit(0x41, 0x03); + emit(0x74); + emit(0x29, 0x03, 0x00); + emit(0x7c); + emit(0x22, 0x02); + // if (hash & mask) == 0 → match + emit(0x20, 0x03); + emit(0x83); + emit(0x50); + emit(0x04, 0x40); + // Store updated hash + emit(0x41, ...toSignedLeb128(exports.HASH_OFFSET)); + emit(0x20, 0x02); + emit(0x37, 0x03, 0x00); + // Return: ptr - inputStart + 1 + emit(0x20, 0x04); + emit(0x20, 0x00); + emit(0x6b); + emit(0x41, 0x01); + emit(0x6a); + emit(0x0f); + emit(0x0b); // end if + // ptr++ + emit(0x20, 0x04); + emit(0x41, 0x01); + emit(0x6a); + emit(0x21, 0x04); + emit(0x0c, 0x00); // br $loop + emit(0x0b); // end loop + emit(0x0b); // end block + // no match: store hash, return -1 + emit(0x41, ...toSignedLeb128(exports.HASH_OFFSET)); + emit(0x20, 0x02); + emit(0x37, 0x03, 0x00); + emit(0x41, 0x7f); + emit(0x0b); // end function + // Backpatch sizes + const bodySize = code.length - bodyStart; + const bsPatch = toLebU32Padded5(bodySize); + for (let i = 0; i < 5; i++) + code[funcSizeOff + i] = bsPatch[i]; + const secSize = code.length - sectionSizeOff - 5; + const ssPatch = toLebU32Padded5(secSize); + for (let i = 0; i < 5; i++) + code[sectionSizeOff + i] = ssPatch[i]; + return new Uint8Array(code); +} +function initWasm() { + if (wasmFn) + return; + const bytes = generateWasmBytes(); + wasmMemory = new WebAssembly.Memory({ initial: PAGES }); + const module = new WebAssembly.Module(bytes); + const instance = new WebAssembly.Instance(module, { js: { mem: wasmMemory } }); + wasmFn = instance.exports.nextMatch; + wasmView = new Uint8Array(wasmMemory.buffer); + const dv = new DataView(wasmMemory.buffer); + for (let i = 0; i < 256; i++) { + dv.setBigUint64(exports.TABLE_OFFSET + i * 8, table_js_1.GEAR_TABLE[i], true); + } +} +function wasmNextMatch(inputStart, inputLen) { + return wasmFn(inputStart, inputLen); +} +function getView() { + return wasmView; +} diff --git a/node_modules/gearhash-jit/dist/esm/index.d.ts b/node_modules/gearhash-jit/dist/esm/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d054edc02abc2d8ff61b5ac2ddd59d9f7e3eee73 --- /dev/null +++ b/node_modules/gearhash-jit/dist/esm/index.d.ts @@ -0,0 +1,27 @@ +/** + * gearhash-jit — Fast GEAR rolling hash for content-defined chunking. + * + * Uses a tiny hand-written WASM module with native i64 arithmetic. + * The hash state is kept as raw bytes in JS (avoiding BigInt in the hot path) + * and written to WASM memory only for the `nextMatch` call. + */ +export { GEAR_TABLE } from "./table.js"; +export declare class Hasher { + private readonly maskBytes; + /** + * The current 64-bit rolling hash state as 8 little-endian bytes. + * Updated after every `nextMatch` call. Zeroed by `resetHash()`. + */ + readonly hash: Uint8Array; + constructor(mask: bigint); + /** + * Scan `buf` for the next gear-hash match. The internal hash state + * carries over between calls (for split-buffer scanning). + * + * @returns 1-based byte position of the match, or -1 if none found. + */ + nextMatch(buf: Uint8Array): number; + /** Reset rolling hash to zero (call when starting a new chunk). */ + resetHash(): void; +} +//# sourceMappingURL=index.d.ts.map \ No newline at end of file diff --git a/node_modules/gearhash-jit/dist/esm/index.d.ts.map b/node_modules/gearhash-jit/dist/esm/index.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..5aed9755fe0d803d1a33788e532a7901da282c3c --- /dev/null +++ b/node_modules/gearhash-jit/dist/esm/index.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAYH,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,qBAAa,MAAM;IACjB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IAEvC;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;gBAEd,IAAI,EAAE,MAAM;IAOxB;;;;;OAKG;IACH,SAAS,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM;IAkBlC,mEAAmE;IACnE,SAAS,IAAI,IAAI;CAGlB"} \ No newline at end of file diff --git a/node_modules/gearhash-jit/dist/esm/index.js b/node_modules/gearhash-jit/dist/esm/index.js new file mode 100644 index 0000000000000000000000000000000000000000..f71036d15fa1ce34f16a0ec70e4962234c48f870 --- /dev/null +++ b/node_modules/gearhash-jit/dist/esm/index.js @@ -0,0 +1,48 @@ +/** + * gearhash-jit — Fast GEAR rolling hash for content-defined chunking. + * + * Uses a tiny hand-written WASM module with native i64 arithmetic. + * The hash state is kept as raw bytes in JS (avoiding BigInt in the hot path) + * and written to WASM memory only for the `nextMatch` call. + */ +import { initWasm, wasmNextMatch, getView, HASH_OFFSET, MASK_OFFSET, INPUT_OFFSET, MAX_INPUT_SIZE, } from "./wasm.js"; +export { GEAR_TABLE } from "./table.js"; +export class Hasher { + maskBytes; + /** + * The current 64-bit rolling hash state as 8 little-endian bytes. + * Updated after every `nextMatch` call. Zeroed by `resetHash()`. + */ + hash; + constructor(mask) { + initWasm(); + this.maskBytes = new Uint8Array(8); + this.hash = new Uint8Array(8); + new DataView(this.maskBytes.buffer).setBigUint64(0, mask, true); + } + /** + * Scan `buf` for the next gear-hash match. The internal hash state + * carries over between calls (for split-buffer scanning). + * + * @returns 1-based byte position of the match, or -1 if none found. + */ + nextMatch(buf) { + const len = buf.length; + if (len === 0) + return -1; + if (len > MAX_INPUT_SIZE) { + throw new RangeError(`Input too large: ${len} > ${MAX_INPUT_SIZE}`); + } + const view = getView(); + view.set(this.hash, HASH_OFFSET); + view.set(this.maskBytes, MASK_OFFSET); + view.set(buf, INPUT_OFFSET); + const pos = wasmNextMatch(INPUT_OFFSET, len); + this.hash.set(view.subarray(HASH_OFFSET, HASH_OFFSET + 8)); + return pos; + } + /** Reset rolling hash to zero (call when starting a new chunk). */ + resetHash() { + this.hash.fill(0); + } +} diff --git a/node_modules/gearhash-jit/dist/esm/package.json b/node_modules/gearhash-jit/dist/esm/package.json new file mode 100644 index 0000000000000000000000000000000000000000..3dbc1ca591c0557e35b6004aeba250e6a70b56e3 --- /dev/null +++ b/node_modules/gearhash-jit/dist/esm/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/node_modules/gearhash-jit/dist/esm/table.d.ts b/node_modules/gearhash-jit/dist/esm/table.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..fc230b86793ef7d36a95296f21901c96a72fbb93 --- /dev/null +++ b/node_modules/gearhash-jit/dist/esm/table.d.ts @@ -0,0 +1,2 @@ +export declare const GEAR_TABLE: bigint[]; +//# sourceMappingURL=table.d.ts.map \ No newline at end of file diff --git a/node_modules/gearhash-jit/dist/esm/table.d.ts.map b/node_modules/gearhash-jit/dist/esm/table.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..f0579443bfbbb7d2e059295be6d424add9923c43 --- /dev/null +++ b/node_modules/gearhash-jit/dist/esm/table.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"table.d.ts","sourceRoot":"","sources":["../../src/table.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,UAAU,EAAE,MAAM,EAiE9B,CAAC"} \ No newline at end of file diff --git a/node_modules/gearhash-jit/dist/esm/table.js b/node_modules/gearhash-jit/dist/esm/table.js new file mode 100644 index 0000000000000000000000000000000000000000..93aafe69f545c4aa2e93099494162ec98f2b2e7f --- /dev/null +++ b/node_modules/gearhash-jit/dist/esm/table.js @@ -0,0 +1,68 @@ +/* eslint-disable */ +// prettier-ignore +export const GEAR_TABLE = [ + 0xb088d3a9e840f559n, 0x5652c7f739ed20d6n, 0x45b28969898972abn, 0x6b0a89d5b68ec777n, + 0x368f573e8b7a31b7n, 0x1dc636dce936d94bn, 0x207a4c4e5554d5b6n, 0xa474b34628239acbn, + 0x3b06a83e1ca3b912n, 0x90e78d6c2f02baf7n, 0xe1c92df7150d9a8an, 0x8e95053a1086d3adn, + 0x5a2ef4f1b83a0722n, 0xa50fac949f807faen, 0x0e7303eb80d8d681n, 0x99b07edc1570ad0fn, + 0x689d2fb555fd3076n, 0x00005082119ea468n, 0xc4b08306a88fcc28n, 0x3eb0678af6374afdn, + 0xf19f87ab86ad7436n, 0xf2129fbfbe6bc736n, 0x481149575c98a4edn, 0x0000010695477bc5n, + 0x1fba37801a9ceaccn, 0x3bf06fd663a49b6dn, 0x99687e9782e3874bn, 0x79a10673aa50d8e3n, + 0xe4accf9e6211f420n, 0x2520e71f87579071n, 0x2bd5d3fd781a8a9bn, 0x00de4dcddd11c873n, + 0xeaa9311c5a87392fn, 0xdb748eb617bc40ffn, 0xaf579a8df620bf6fn, 0x86a6e5da1b09c2b1n, + 0xcc2fc30ac322a12en, 0x355e2afec1f74267n, 0x2d99c8f4c021a47bn, 0xbade4b4a9404cfc3n, + 0xf7b518721d707d69n, 0x3286b6587bf32c20n, 0x0000b68886af270cn, 0xa115d6e4db8a9079n, + 0x484f7e9c97b2e199n, 0xccca7bb75713e301n, 0xbf2584a62bb0f160n, 0xade7e813625dbcc8n, + 0x000070940d87955an, 0x8ae69108139e626fn, 0xbd776ad72fde38a2n, 0xfb6b001fc2fcc0cfn, + 0xc7a474b8e67bc427n, 0xbaf6f11610eb5d58n, 0x09cb1f5b6de770d1n, 0xb0b219e6977d4c47n, + 0x00ccbc386ea7ad4an, 0xcc849d0adf973f01n, 0x73a3ef7d016af770n, 0xc807d2d386bdbdfen, + 0x7f2ac9966c791730n, 0xd037a86bc6c504dan, 0xf3f17c661eaa609dn, 0xaca626b04daae687n, + 0x755a99374f4a5b07n, 0x90837ee65b2caeden, 0x6ee8ad93fd560785n, 0x0000d9e11053edd8n, + 0x9e063bb2d21cdbd7n, 0x07ab77f12a01d2b2n, 0xec550255e6641b44n, 0x78fb94a8449c14c6n, + 0xc7510e1bc6c0f5f5n, 0x0000320b36e4cae3n, 0x827c33262c8b1a2dn, 0x14675f0b48ea4144n, + 0x267bd3a6498decebn, 0xf1916ff982f5035en, 0x86221b7ff434fb88n, 0x9dbecee7386f49d8n, + 0xea58f8cac80f8f4an, 0x008d198692fc64d8n, 0x6d38704fbabf9a36n, 0xe032cb07d1e7be4cn, + 0x228d21f6ad450890n, 0x635cb1bfc02589a5n, 0x4620a1739ca2ce71n, 0xa7e7dfe3aae5fb58n, + 0x0c10ca932b3c0debn, 0x2727fee884afed7bn, 0xa2df1c6df9e2ab1fn, 0x4dcdd1ac0774f523n, + 0x000070ffad33e24en, 0xa2ace87bc5977816n, 0x9892275ab4286049n, 0xc2861181ddf18959n, + 0xbb9972a042483e19n, 0xef70cd3766513078n, 0x00000513abfc9864n, 0xc058b61858c94083n, + 0x09e850859725e0den, 0x9197fb3bf83e7d94n, 0x7e1e626d12b64bcen, 0x520c54507f7b57d1n, + 0xbee1797174e22416n, 0x6fd9ac3222e95587n, 0x0023957c9adfbf3en, 0xa01c7d7e234bbe15n, + 0xaba2c758b8a38cbbn, 0x0d1fa0ceec3e2b30n, 0x0bb6a58b7e60b991n, 0x4333dd5b9fa26635n, + 0xc2fd3b7d4001c1a3n, 0xfb41802454731127n, 0x65a56185a50d18cbn, 0xf67a02bd8784b54fn, + 0x696f11dd67e65063n, 0x00002022fca814abn, 0x8cd6be912db9d852n, 0x695189b6e9ae8a57n, + 0xee9453b50ada0c28n, 0xd8fc5ea91a78845en, 0xab86bf191a4aa767n, 0x0000c6b5c86415e5n, + 0x267310178e08a22en, 0xed2d101b078bca25n, 0x3b41ed84b226a8fbn, 0x13e622120f28dc06n, + 0xa315f5ebfb706d26n, 0x8816c34e3301bacen, 0xe9395b9cbb71fdaen, 0x002ce9202e721648n, + 0x4283db1d2bb3c91cn, 0xd77d461ad2b1a6a5n, 0xe2ec17e46eeb866bn, 0xb8e0be4039fbc47cn, + 0xdea160c4d5299d04n, 0x7eec86c8d28c3634n, 0x2119ad129f98a399n, 0xa6ccf46b61a283efn, + 0x2c52cedef658c617n, 0x2db4871169acdd83n, 0x0000f0d6f39ecbe9n, 0x3dd5d8c98d2f9489n, + 0x8a1872a22b01f584n, 0xf282a4c40e7b3cf2n, 0x8020ec2ccb1ba196n, 0x6693b6e09e59e313n, + 0x0000ce19cc7c83ebn, 0x20cb5735f6479c3bn, 0x762ebf3759d75a5bn, 0x207bfe823d693975n, + 0xd77dc112339cd9d5n, 0x9ba7834284627d03n, 0x217dc513e95f51e9n, 0xb27b1a29fc5e7816n, + 0x00d5cd9831bb662dn, 0x71e39b806d75734cn, 0x7e572af006fb1a23n, 0xa2734f2f6ae91f85n, + 0xbf82c6b5022cddf2n, 0x5c3beac60761a0den, 0xcdc893bb47416998n, 0x6d1085615c187e01n, + 0x77f8ae30ac277c5dn, 0x917c6b81122a2c91n, 0x5b75b699add16967n, 0x0000cf6ae79a069bn, + 0xf3c40afa60de1104n, 0x2063127aa59167c3n, 0x621de62269d1894dn, 0xd188ac1de62b4726n, + 0x107036e2154b673cn, 0x0000b85f28553a1dn, 0xf2ef4e4c18236f3dn, 0xd9d6de6611b9f602n, + 0xa1fc7955fb47911cn, 0xeb85fd032f298dbdn, 0xbe27502fb3befae1n, 0xe3034251c4cd661en, + 0x441364d354071836n, 0x0082b36c75f2983en, 0xb145910316fa66f0n, 0x021c069c9847caf7n, + 0x2910dfc75a4b5221n, 0x735b353e1c57a8b5n, 0xce44312ce98ed96cn, 0xbc942e4506bdfa65n, + 0xf05086a71257941bn, 0xfec3b215d351ceadn, 0x00ae1055e0144202n, 0xf54b40846f42e454n, + 0x00007fd9c8bcbcc8n, 0xbfbd9ef317de9bfen, 0xa804302ff2854e12n, 0x39ce4957a5e5d8d4n, + 0xffb9e2a45637ba84n, 0x55b9ad1d9ea0818bn, 0x00008acbf319178an, 0x48e2bfc8d0fbfb38n, + 0x8be39841e848b5e8n, 0x0e2712160696a08bn, 0xd51096e84b44242an, 0x1101ba176792e13an, + 0xc22e770f4531689dn, 0x1689eff272bbc56cn, 0x00a92a197f5650ecn, 0xbc765990bda1784en, + 0xc61441e392fcb8aen, 0x07e13a2ced31e4a0n, 0x92cbe984234e9d4dn, 0x8f4ff572bb7d8ac5n, + 0x0b9670c00b963bd0n, 0x62955a581a03eb01n, 0x645f83e5ea000254n, 0x41fce516cd88f299n, + 0xbbda9748da7a98cfn, 0x0000aab2fe4845fan, 0x19761b069bf56555n, 0x8b8f5e8343b6ad56n, + 0x3e5d1cfd144821d9n, 0xec5c1e2ca2b0cd8fn, 0xfaf7e0fea7fbb57fn, 0x000000d3ba12961bn, + 0xda3f90178401b18en, 0x70ff906de33a5febn, 0x0527d5a7c06970e7n, 0x22d8e773607c13e9n, + 0xc9ab70df643c3bacn, 0xeda4c6dc8abe12e3n, 0xecef1f410033e78an, 0x0024c2b274ac72cbn, + 0x06740d954fa900b4n, 0x1d7a299b323d6304n, 0xb3c37cb298cbead5n, 0xc986e3c76178739bn, + 0x9fabea364b46f58an, 0x6da214c5af85cc56n, 0x17a43ed8b7a38f84n, 0x6eccec511d9adbebn, + 0xf9cab30913335afbn, 0x4a5e60c5f415eed2n, 0x00006967503672b4n, 0x9da51d121454bb87n, + 0x84321e13b9bbc816n, 0xfb3d6fb6ab2fdd8dn, 0x60305eed8e160a8dn, 0xcbbf4b14e9946ce8n, + 0x00004f63381b10c3n, 0x07d5b7816fcc4e10n, 0xe5a536726a6a8155n, 0x57afb23447a07fddn, + 0x18f346f7abc9d394n, 0x636dc655d61ad33dn, 0xcc8bab4939f7f3f6n, 0x63c7a906c1dd187bn, +]; diff --git a/node_modules/gearhash-jit/dist/esm/wasm.d.ts b/node_modules/gearhash-jit/dist/esm/wasm.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..d0e001b5856bf8252085af5defbbbaadeb170d0a --- /dev/null +++ b/node_modules/gearhash-jit/dist/esm/wasm.d.ts @@ -0,0 +1,21 @@ +/** + * GEAR hash WASM - Runtime bytecode generation + * + * Generates a tiny WebAssembly module with a single `nextMatch` function + * that performs the gear hash rolling scan using native i64 arithmetic. + * + * Memory layout (all little-endian): + * 0-2047: Gear lookup table (256 × 8 bytes) + * 2048-2055: Hash state (u64, persists across calls) + * 2056-2063: Mask (u64, set per-hasher before each call) + * 4096+: Input buffer + */ +export declare const TABLE_OFFSET = 0; +export declare const HASH_OFFSET = 2048; +export declare const MASK_OFFSET = 2056; +export declare const INPUT_OFFSET = 4096; +export declare const MAX_INPUT_SIZE: number; +export declare function initWasm(): void; +export declare function wasmNextMatch(inputStart: number, inputLen: number): number; +export declare function getView(): Uint8Array; +//# sourceMappingURL=wasm.d.ts.map \ No newline at end of file diff --git a/node_modules/gearhash-jit/dist/esm/wasm.d.ts.map b/node_modules/gearhash-jit/dist/esm/wasm.d.ts.map new file mode 100644 index 0000000000000000000000000000000000000000..25ad405e574804b5c6c7b1b054d0c84c4c5989a9 --- /dev/null +++ b/node_modules/gearhash-jit/dist/esm/wasm.d.ts.map @@ -0,0 +1 @@ +{"version":3,"file":"wasm.d.ts","sourceRoot":"","sources":["../../src/wasm.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,eAAO,MAAM,YAAY,IAAI,CAAC;AAC9B,eAAO,MAAM,WAAW,OAAO,CAAC;AAChC,eAAO,MAAM,WAAW,OAAO,CAAC;AAChC,eAAO,MAAM,YAAY,OAAO,CAAC;AAEjC,eAAO,MAAM,cAAc,QAA+B,CAAC;AA8K3D,wBAAgB,QAAQ,IAAI,IAAI,CAc/B;AAED,wBAAgB,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAE1E;AAED,wBAAgB,OAAO,IAAI,UAAU,CAEpC"} \ No newline at end of file diff --git a/node_modules/gearhash-jit/dist/esm/wasm.js b/node_modules/gearhash-jit/dist/esm/wasm.js new file mode 100644 index 0000000000000000000000000000000000000000..7c4761ac185a9b69e8fb09f3485bdb40ee2a6555 --- /dev/null +++ b/node_modules/gearhash-jit/dist/esm/wasm.js @@ -0,0 +1,180 @@ +/** + * GEAR hash WASM - Runtime bytecode generation + * + * Generates a tiny WebAssembly module with a single `nextMatch` function + * that performs the gear hash rolling scan using native i64 arithmetic. + * + * Memory layout (all little-endian): + * 0-2047: Gear lookup table (256 × 8 bytes) + * 2048-2055: Hash state (u64, persists across calls) + * 2056-2063: Mask (u64, set per-hasher before each call) + * 4096+: Input buffer + */ +import { GEAR_TABLE } from "./table.js"; +export const TABLE_OFFSET = 0; +export const HASH_OFFSET = 2048; +export const MASK_OFFSET = 2056; +export const INPUT_OFFSET = 4096; +const PAGES = 8; // 512 KB +export const MAX_INPUT_SIZE = PAGES * 65536 - INPUT_OFFSET; +let wasmMemory = null; +let wasmView = null; +let wasmFn = null; +function toSignedLeb128(n) { + const bytes = []; + let value = n | 0; + for (;;) { + const byte = value & 0x7f; + value >>= 7; + if ((value === 0 && (byte & 0x40) === 0) || (value === -1 && (byte & 0x40) !== 0)) { + bytes.push(byte); + return bytes; + } + bytes.push(byte | 0x80); + } +} +function toLebU32Padded5(n) { + return [ + (n & 0x7f) | 0x80, + ((n >>> 7) & 0x7f) | 0x80, + ((n >>> 14) & 0x7f) | 0x80, + ((n >>> 21) & 0x7f) | 0x80, + (n >>> 28) & 0x0f, + ]; +} +/** + * Generate the WASM module bytecode. + * + * Exports one function: + * nextMatch(inputStart: i32, inputLen: i32) -> i32 + * + * Reads hash/mask from fixed memory offsets, scans from `inputStart` + * for `inputLen` bytes, writes updated hash back. + * Returns 1-based match position within the scanned range, or -1. + */ +function generateWasmBytes() { + const code = []; + function emit(...bytes) { + code.push(...bytes); + } + // ── Module header ── + emit(0x00, 0x61, 0x73, 0x6d); // magic + emit(0x01, 0x00, 0x00, 0x00); // version 1 + // ── Type section: (i32, i32) -> (i32) ── + emit(0x01, 0x07, 0x01, 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f); + // ── Import section: memory "js"."mem" min=PAGES ── + emit(0x02, 0x0b, 0x01, 0x02, 0x6a, 0x73, 0x03, 0x6d, 0x65, 0x6d, 0x02, 0x00, PAGES); + // ── Function section: 1 function, type 0 ── + emit(0x03, 0x02, 0x01, 0x00); + // ── Export section: "nextMatch" -> func 0 ── + emit(0x07, 0x0d, 0x01, 0x09, 0x6e, 0x65, 0x78, 0x74, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x00, 0x00); + // ── Code section ── + emit(0x0a); + const sectionSizeOff = code.length; + emit(0x00, 0x00, 0x00, 0x00, 0x00); + emit(0x01); // 1 function body + const funcSizeOff = code.length; + emit(0x00, 0x00, 0x00, 0x00, 0x00); + const bodyStart = code.length; + // Locals: $0 = inputStart (param), $1 = inputLen (param) + // $2 = hash (i64), $3 = mask (i64) + // $4 = ptr (i32), $5 = end (i32) + emit(0x02, 0x02, 0x7e, 0x02, 0x7f); + // Load hash from memory[HASH_OFFSET] + emit(0x41, ...toSignedLeb128(HASH_OFFSET)); + emit(0x29, 0x03, 0x00); + emit(0x21, 0x02); + // Load mask from memory[MASK_OFFSET] + emit(0x41, ...toSignedLeb128(MASK_OFFSET)); + emit(0x29, 0x03, 0x00); + emit(0x21, 0x03); + // ptr = inputStart + emit(0x20, 0x00); + emit(0x21, 0x04); + // end = inputStart + inputLen + emit(0x20, 0x00); + emit(0x20, 0x01); + emit(0x6a); + emit(0x21, 0x05); + // block $done + emit(0x02, 0x40); + // loop $loop + emit(0x03, 0x40); + // if ptr >= end → break + emit(0x20, 0x04); + emit(0x20, 0x05); + emit(0x4e); + emit(0x0d, 0x01); + // hash = (hash << 1) + table[mem[ptr] * 8] + emit(0x20, 0x02); + emit(0x42, 0x01); + emit(0x86); + emit(0x20, 0x04); + emit(0x2d, 0x00, 0x00); + emit(0x41, 0x03); + emit(0x74); + emit(0x29, 0x03, 0x00); + emit(0x7c); + emit(0x22, 0x02); + // if (hash & mask) == 0 → match + emit(0x20, 0x03); + emit(0x83); + emit(0x50); + emit(0x04, 0x40); + // Store updated hash + emit(0x41, ...toSignedLeb128(HASH_OFFSET)); + emit(0x20, 0x02); + emit(0x37, 0x03, 0x00); + // Return: ptr - inputStart + 1 + emit(0x20, 0x04); + emit(0x20, 0x00); + emit(0x6b); + emit(0x41, 0x01); + emit(0x6a); + emit(0x0f); + emit(0x0b); // end if + // ptr++ + emit(0x20, 0x04); + emit(0x41, 0x01); + emit(0x6a); + emit(0x21, 0x04); + emit(0x0c, 0x00); // br $loop + emit(0x0b); // end loop + emit(0x0b); // end block + // no match: store hash, return -1 + emit(0x41, ...toSignedLeb128(HASH_OFFSET)); + emit(0x20, 0x02); + emit(0x37, 0x03, 0x00); + emit(0x41, 0x7f); + emit(0x0b); // end function + // Backpatch sizes + const bodySize = code.length - bodyStart; + const bsPatch = toLebU32Padded5(bodySize); + for (let i = 0; i < 5; i++) + code[funcSizeOff + i] = bsPatch[i]; + const secSize = code.length - sectionSizeOff - 5; + const ssPatch = toLebU32Padded5(secSize); + for (let i = 0; i < 5; i++) + code[sectionSizeOff + i] = ssPatch[i]; + return new Uint8Array(code); +} +export function initWasm() { + if (wasmFn) + return; + const bytes = generateWasmBytes(); + wasmMemory = new WebAssembly.Memory({ initial: PAGES }); + const module = new WebAssembly.Module(bytes); + const instance = new WebAssembly.Instance(module, { js: { mem: wasmMemory } }); + wasmFn = instance.exports.nextMatch; + wasmView = new Uint8Array(wasmMemory.buffer); + const dv = new DataView(wasmMemory.buffer); + for (let i = 0; i < 256; i++) { + dv.setBigUint64(TABLE_OFFSET + i * 8, GEAR_TABLE[i], true); + } +} +export function wasmNextMatch(inputStart, inputLen) { + return wasmFn(inputStart, inputLen); +} +export function getView() { + return wasmView; +} diff --git a/node_modules/gearhash-jit/package.json b/node_modules/gearhash-jit/package.json new file mode 100644 index 0000000000000000000000000000000000000000..a511a3c021b0733cbdd248d2f8cd35cc7507b4f1 --- /dev/null +++ b/node_modules/gearhash-jit/package.json @@ -0,0 +1,45 @@ +{ + "name": "gearhash-jit", + "version": "1.0.2", + "description": "Fast GEAR hash with hand-written WASM i64 bytecode for content-defined chunking", + "keywords": [ + "gearhash", + "cdc", + "chunking", + "wasm" + ], + "license": "MIT", + "sideEffects": false, + "files": [ + "dist", + "README.md" + ], + "scripts": { + "prepare": "tshy", + "test": "vitest run" + }, + "tshy": { + "exports": { + ".": "./src/index.ts", + "./package.json": "./package.json" + } + }, + "devDependencies": {}, + "type": "module", + "exports": { + ".": { + "import": { + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/index.js" + }, + "require": { + "types": "./dist/commonjs/index.d.ts", + "default": "./dist/commonjs/index.js" + } + }, + "./package.json": "./package.json" + }, + "main": "./dist/commonjs/index.js", + "types": "./dist/commonjs/index.d.ts", + "module": "./dist/esm/index.js" +} diff --git a/node_modules/is-fullwidth-code-point/index.d.ts b/node_modules/is-fullwidth-code-point/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..729d2020516f0b64dbcb8cb3443b7777b3c04769 --- /dev/null +++ b/node_modules/is-fullwidth-code-point/index.d.ts @@ -0,0 +1,17 @@ +/** +Check if the character represented by a given [Unicode code point](https://en.wikipedia.org/wiki/Code_point) is [fullwidth](https://en.wikipedia.org/wiki/Halfwidth_and_fullwidth_forms). + +@param codePoint - The [code point](https://en.wikipedia.org/wiki/Code_point) of a character. + +@example +``` +import isFullwidthCodePoint from 'is-fullwidth-code-point'; + +isFullwidthCodePoint('谢'.codePointAt(0)); +//=> true + +isFullwidthCodePoint('a'.codePointAt(0)); +//=> false +``` +*/ +export default function isFullwidthCodePoint(codePoint: number): boolean; diff --git a/node_modules/is-fullwidth-code-point/index.js b/node_modules/is-fullwidth-code-point/index.js new file mode 100644 index 0000000000000000000000000000000000000000..671f97f760779075aa362ec41063e7a3a528b0e8 --- /dev/null +++ b/node_modules/is-fullwidth-code-point/index.js @@ -0,0 +1,50 @@ +/* eslint-disable yoda */ +'use strict'; + +const isFullwidthCodePoint = codePoint => { + if (Number.isNaN(codePoint)) { + return false; + } + + // Code points are derived from: + // http://www.unix.org/Public/UNIDATA/EastAsianWidth.txt + if ( + codePoint >= 0x1100 && ( + codePoint <= 0x115F || // Hangul Jamo + codePoint === 0x2329 || // LEFT-POINTING ANGLE BRACKET + codePoint === 0x232A || // RIGHT-POINTING ANGLE BRACKET + // CJK Radicals Supplement .. Enclosed CJK Letters and Months + (0x2E80 <= codePoint && codePoint <= 0x3247 && codePoint !== 0x303F) || + // Enclosed CJK Letters and Months .. CJK Unified Ideographs Extension A + (0x3250 <= codePoint && codePoint <= 0x4DBF) || + // CJK Unified Ideographs .. Yi Radicals + (0x4E00 <= codePoint && codePoint <= 0xA4C6) || + // Hangul Jamo Extended-A + (0xA960 <= codePoint && codePoint <= 0xA97C) || + // Hangul Syllables + (0xAC00 <= codePoint && codePoint <= 0xD7A3) || + // CJK Compatibility Ideographs + (0xF900 <= codePoint && codePoint <= 0xFAFF) || + // Vertical Forms + (0xFE10 <= codePoint && codePoint <= 0xFE19) || + // CJK Compatibility Forms .. Small Form Variants + (0xFE30 <= codePoint && codePoint <= 0xFE6B) || + // Halfwidth and Fullwidth Forms + (0xFF01 <= codePoint && codePoint <= 0xFF60) || + (0xFFE0 <= codePoint && codePoint <= 0xFFE6) || + // Kana Supplement + (0x1B000 <= codePoint && codePoint <= 0x1B001) || + // Enclosed Ideographic Supplement + (0x1F200 <= codePoint && codePoint <= 0x1F251) || + // CJK Unified Ideographs Extension B .. Tertiary Ideographic Plane + (0x20000 <= codePoint && codePoint <= 0x3FFFD) + ) + ) { + return true; + } + + return false; +}; + +module.exports = isFullwidthCodePoint; +module.exports.default = isFullwidthCodePoint; diff --git a/node_modules/is-fullwidth-code-point/license b/node_modules/is-fullwidth-code-point/license new file mode 100644 index 0000000000000000000000000000000000000000..e7af2f77107d73046421ef56c4684cbfdd3c1e89 --- /dev/null +++ b/node_modules/is-fullwidth-code-point/license @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/node_modules/is-fullwidth-code-point/package.json b/node_modules/is-fullwidth-code-point/package.json new file mode 100644 index 0000000000000000000000000000000000000000..2137e888fa503dadf920e306c1cc12851a9de011 --- /dev/null +++ b/node_modules/is-fullwidth-code-point/package.json @@ -0,0 +1,42 @@ +{ + "name": "is-fullwidth-code-point", + "version": "3.0.0", + "description": "Check if the character represented by a given Unicode code point is fullwidth", + "license": "MIT", + "repository": "sindresorhus/is-fullwidth-code-point", + "author": { + "name": "Sindre Sorhus", + "email": "sindresorhus@gmail.com", + "url": "sindresorhus.com" + }, + "engines": { + "node": ">=8" + }, + "scripts": { + "test": "xo && ava && tsd-check" + }, + "files": [ + "index.js", + "index.d.ts" + ], + "keywords": [ + "fullwidth", + "full-width", + "full", + "width", + "unicode", + "character", + "string", + "codepoint", + "code", + "point", + "is", + "detect", + "check" + ], + "devDependencies": { + "ava": "^1.3.1", + "tsd-check": "^0.5.0", + "xo": "^0.24.0" + } +} diff --git a/node_modules/is-fullwidth-code-point/readme.md b/node_modules/is-fullwidth-code-point/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..4236bba980d8fea1486c883c16417ca8d5a7d5aa --- /dev/null +++ b/node_modules/is-fullwidth-code-point/readme.md @@ -0,0 +1,39 @@ +# is-fullwidth-code-point [![Build Status](https://travis-ci.org/sindresorhus/is-fullwidth-code-point.svg?branch=master)](https://travis-ci.org/sindresorhus/is-fullwidth-code-point) + +> Check if the character represented by a given [Unicode code point](https://en.wikipedia.org/wiki/Code_point) is [fullwidth](https://en.wikipedia.org/wiki/Halfwidth_and_fullwidth_forms) + + +## Install + +``` +$ npm install is-fullwidth-code-point +``` + + +## Usage + +```js +const isFullwidthCodePoint = require('is-fullwidth-code-point'); + +isFullwidthCodePoint('谢'.codePointAt(0)); +//=> true + +isFullwidthCodePoint('a'.codePointAt(0)); +//=> false +``` + + +## API + +### isFullwidthCodePoint(codePoint) + +#### codePoint + +Type: `number` + +The [code point](https://en.wikipedia.org/wiki/Code_point) of a character. + + +## License + +MIT © [Sindre Sorhus](https://sindresorhus.com) diff --git a/node_modules/string-width/index.d.ts b/node_modules/string-width/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..12b5309751dd50aeef72b96d10a9f81207d27066 --- /dev/null +++ b/node_modules/string-width/index.d.ts @@ -0,0 +1,29 @@ +declare const stringWidth: { + /** + Get the visual width of a string - the number of columns required to display it. + + Some Unicode characters are [fullwidth](https://en.wikipedia.org/wiki/Halfwidth_and_fullwidth_forms) and use double the normal width. [ANSI escape codes](https://en.wikipedia.org/wiki/ANSI_escape_code) are stripped and doesn't affect the width. + + @example + ``` + import stringWidth = require('string-width'); + + stringWidth('a'); + //=> 1 + + stringWidth('古'); + //=> 2 + + stringWidth('\u001B[1m古\u001B[22m'); + //=> 2 + ``` + */ + (string: string): number; + + // TODO: remove this in the next major version, refactor the whole definition to: + // declare function stringWidth(string: string): number; + // export = stringWidth; + default: typeof stringWidth; +} + +export = stringWidth; diff --git a/node_modules/string-width/index.js b/node_modules/string-width/index.js new file mode 100644 index 0000000000000000000000000000000000000000..f4d261a96a099ee6b9dcfbf7b050d6e2f42075a2 --- /dev/null +++ b/node_modules/string-width/index.js @@ -0,0 +1,47 @@ +'use strict'; +const stripAnsi = require('strip-ansi'); +const isFullwidthCodePoint = require('is-fullwidth-code-point'); +const emojiRegex = require('emoji-regex'); + +const stringWidth = string => { + if (typeof string !== 'string' || string.length === 0) { + return 0; + } + + string = stripAnsi(string); + + if (string.length === 0) { + return 0; + } + + string = string.replace(emojiRegex(), ' '); + + let width = 0; + + for (let i = 0; i < string.length; i++) { + const code = string.codePointAt(i); + + // Ignore control characters + if (code <= 0x1F || (code >= 0x7F && code <= 0x9F)) { + continue; + } + + // Ignore combining characters + if (code >= 0x300 && code <= 0x36F) { + continue; + } + + // Surrogates + if (code > 0xFFFF) { + i++; + } + + width += isFullwidthCodePoint(code) ? 2 : 1; + } + + return width; +}; + +module.exports = stringWidth; +// TODO: remove this in the next major version +module.exports.default = stringWidth; diff --git a/node_modules/string-width/license b/node_modules/string-width/license new file mode 100644 index 0000000000000000000000000000000000000000..e7af2f77107d73046421ef56c4684cbfdd3c1e89 --- /dev/null +++ b/node_modules/string-width/license @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/node_modules/string-width/package.json b/node_modules/string-width/package.json new file mode 100644 index 0000000000000000000000000000000000000000..28ba7b4cae9bf9ec20d518c3f9b5ea99833aa94b --- /dev/null +++ b/node_modules/string-width/package.json @@ -0,0 +1,56 @@ +{ + "name": "string-width", + "version": "4.2.3", + "description": "Get the visual width of a string - the number of columns required to display it", + "license": "MIT", + "repository": "sindresorhus/string-width", + "author": { + "name": "Sindre Sorhus", + "email": "sindresorhus@gmail.com", + "url": "sindresorhus.com" + }, + "engines": { + "node": ">=8" + }, + "scripts": { + "test": "xo && ava && tsd" + }, + "files": [ + "index.js", + "index.d.ts" + ], + "keywords": [ + "string", + "character", + "unicode", + "width", + "visual", + "column", + "columns", + "fullwidth", + "full-width", + "full", + "ansi", + "escape", + "codes", + "cli", + "command-line", + "terminal", + "console", + "cjk", + "chinese", + "japanese", + "korean", + "fixed-width" + ], + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "devDependencies": { + "ava": "^1.4.1", + "tsd": "^0.7.1", + "xo": "^0.24.0" + } +} diff --git a/node_modules/string-width/readme.md b/node_modules/string-width/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..bdd314129ca7471d65d942148049d0ccfdbc0cae --- /dev/null +++ b/node_modules/string-width/readme.md @@ -0,0 +1,50 @@ +# string-width + +> Get the visual width of a string - the number of columns required to display it + +Some Unicode characters are [fullwidth](https://en.wikipedia.org/wiki/Halfwidth_and_fullwidth_forms) and use double the normal width. [ANSI escape codes](https://en.wikipedia.org/wiki/ANSI_escape_code) are stripped and doesn't affect the width. + +Useful to be able to measure the actual width of command-line output. + + +## Install + +``` +$ npm install string-width +``` + + +## Usage + +```js +const stringWidth = require('string-width'); + +stringWidth('a'); +//=> 1 + +stringWidth('古'); +//=> 2 + +stringWidth('\u001B[1m古\u001B[22m'); +//=> 2 +``` + + +## Related + +- [string-width-cli](https://github.com/sindresorhus/string-width-cli) - CLI for this module +- [string-length](https://github.com/sindresorhus/string-length) - Get the real length of a string +- [widest-line](https://github.com/sindresorhus/widest-line) - Get the visual width of the widest line in a string + + +--- + +
+ + Get professional support for this package with a Tidelift subscription + +
+ + Tidelift helps make open source sustainable for maintainers while giving companies
assurances about security, maintenance, and licensing for their dependencies. + +
diff --git a/node_modules/strip-ansi/index.d.ts b/node_modules/strip-ansi/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..907fccc29269ebcd49141fa8a4513be3b6a8d270 --- /dev/null +++ b/node_modules/strip-ansi/index.d.ts @@ -0,0 +1,17 @@ +/** +Strip [ANSI escape codes](https://en.wikipedia.org/wiki/ANSI_escape_code) from a string. + +@example +``` +import stripAnsi = require('strip-ansi'); + +stripAnsi('\u001B[4mUnicorn\u001B[0m'); +//=> 'Unicorn' + +stripAnsi('\u001B]8;;https://github.com\u0007Click\u001B]8;;\u0007'); +//=> 'Click' +``` +*/ +declare function stripAnsi(string: string): string; + +export = stripAnsi; diff --git a/node_modules/strip-ansi/index.js b/node_modules/strip-ansi/index.js new file mode 100644 index 0000000000000000000000000000000000000000..9a593dfcd1fd5c9e627d9ff2ed85a88df9a41a99 --- /dev/null +++ b/node_modules/strip-ansi/index.js @@ -0,0 +1,4 @@ +'use strict'; +const ansiRegex = require('ansi-regex'); + +module.exports = string => typeof string === 'string' ? string.replace(ansiRegex(), '') : string; diff --git a/node_modules/strip-ansi/license b/node_modules/strip-ansi/license new file mode 100644 index 0000000000000000000000000000000000000000..e7af2f77107d73046421ef56c4684cbfdd3c1e89 --- /dev/null +++ b/node_modules/strip-ansi/license @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) Sindre Sorhus (sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/node_modules/strip-ansi/package.json b/node_modules/strip-ansi/package.json new file mode 100644 index 0000000000000000000000000000000000000000..1a41108d42831c9c156fdcc4e6cd8c2539983ccd --- /dev/null +++ b/node_modules/strip-ansi/package.json @@ -0,0 +1,54 @@ +{ + "name": "strip-ansi", + "version": "6.0.1", + "description": "Strip ANSI escape codes from a string", + "license": "MIT", + "repository": "chalk/strip-ansi", + "author": { + "name": "Sindre Sorhus", + "email": "sindresorhus@gmail.com", + "url": "sindresorhus.com" + }, + "engines": { + "node": ">=8" + }, + "scripts": { + "test": "xo && ava && tsd" + }, + "files": [ + "index.js", + "index.d.ts" + ], + "keywords": [ + "strip", + "trim", + "remove", + "ansi", + "styles", + "color", + "colour", + "colors", + "terminal", + "console", + "string", + "tty", + "escape", + "formatting", + "rgb", + "256", + "shell", + "xterm", + "log", + "logging", + "command-line", + "text" + ], + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "devDependencies": { + "ava": "^2.4.0", + "tsd": "^0.10.0", + "xo": "^0.25.3" + } +} diff --git a/node_modules/strip-ansi/readme.md b/node_modules/strip-ansi/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..7c4b56d46ddc72a3a627a5fd8f3dfcad8c9b599e --- /dev/null +++ b/node_modules/strip-ansi/readme.md @@ -0,0 +1,46 @@ +# strip-ansi [![Build Status](https://travis-ci.org/chalk/strip-ansi.svg?branch=master)](https://travis-ci.org/chalk/strip-ansi) + +> Strip [ANSI escape codes](https://en.wikipedia.org/wiki/ANSI_escape_code) from a string + + +## Install + +``` +$ npm install strip-ansi +``` + + +## Usage + +```js +const stripAnsi = require('strip-ansi'); + +stripAnsi('\u001B[4mUnicorn\u001B[0m'); +//=> 'Unicorn' + +stripAnsi('\u001B]8;;https://github.com\u0007Click\u001B]8;;\u0007'); +//=> 'Click' +``` + + +## strip-ansi for enterprise + +Available as part of the Tidelift Subscription. + +The maintainers of strip-ansi and thousands of other packages are working with Tidelift to deliver commercial support and maintenance for the open source dependencies you use to build your applications. Save time, reduce risk, and improve code health, while paying the maintainers of the exact dependencies you use. [Learn more.](https://tidelift.com/subscription/pkg/npm-strip-ansi?utm_source=npm-strip-ansi&utm_medium=referral&utm_campaign=enterprise&utm_term=repo) + + +## Related + +- [strip-ansi-cli](https://github.com/chalk/strip-ansi-cli) - CLI for this module +- [strip-ansi-stream](https://github.com/chalk/strip-ansi-stream) - Streaming version of this module +- [has-ansi](https://github.com/chalk/has-ansi) - Check if a string has ANSI escape codes +- [ansi-regex](https://github.com/chalk/ansi-regex) - Regular expression for matching ANSI escape codes +- [chalk](https://github.com/chalk/chalk) - Terminal string styling done right + + +## Maintainers + +- [Sindre Sorhus](https://github.com/sindresorhus) +- [Josh Junon](https://github.com/qix-) + diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000000000000000000000000000000000000..ec02b7bc4897c54f566258edd2ef248518a7d87f --- /dev/null +++ b/package-lock.json @@ -0,0 +1,131 @@ +{ + "name": "opencode-space", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "opencode-space", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@huggingface/hub": "^2.13.2" + } + }, + "node_modules/@huggingface/blake3-jit": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/@huggingface/blake3-jit/-/blake3-jit-0.0.2.tgz", + "integrity": "sha512-Bq7B5qabyjrJfhBsl85Jd2QBtf+HzRD7h7A9GfN2lzrrsABhOa5evVPgzoCTxR7Ub0QFj7YDK1YkYRWBU25+2w==", + "license": "MIT" + }, + "node_modules/@huggingface/hub": { + "version": "2.13.2", + "resolved": "https://registry.npmjs.org/@huggingface/hub/-/hub-2.13.2.tgz", + "integrity": "sha512-I1Pbn9UcBHCt0M/SbZkKzYr3JVnbJp4YIFv74eVRH2Y22yT9Jpm6DtfM74tcjNLbC7SyFZ433znUw0szdFd4Fg==", + "license": "MIT", + "dependencies": { + "@huggingface/tasks": "^0.21.13", + "@huggingface/xetchunk-wasm": "^0.1.0" + }, + "bin": { + "hfjs": "dist/cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "cli-progress": "^3.12.0" + } + }, + "node_modules/@huggingface/tasks": { + "version": "0.21.20", + "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.21.20.tgz", + "integrity": "sha512-pGw7pQI8TYSbrOVfNRdrYeISDEYOvQzOCEBceYjEKOH2wOmooc6eGfbYRhAALDgNKZCSISPYYFszRxHta3FaNw==", + "license": "MIT" + }, + "node_modules/@huggingface/xetchunk-wasm": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@huggingface/xetchunk-wasm/-/xetchunk-wasm-0.1.0.tgz", + "integrity": "sha512-wWpp2qwPgf9kv1KLJjcDUk/OrpDOsFoQ3Qpz0U5LGn20csoymBf8eneOv6wm/GzPBzlWac1OYiR0aa1vT6aM2Q==", + "license": "MIT", + "dependencies": { + "@huggingface/blake3-jit": "0.0.2", + "gearhash-jit": "1.0.2" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-progress": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz", + "integrity": "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==", + "license": "MIT", + "optional": true, + "dependencies": { + "string-width": "^4.2.3" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT", + "optional": true + }, + "node_modules/gearhash-jit": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/gearhash-jit/-/gearhash-jit-1.0.2.tgz", + "integrity": "sha512-UhzJL4KXSdqAKepy/tZwmi2Rcy0YMmtiC4DQS4SURCuIWdh8ECZtnXK2ePRMLigfB61hRKdLK/Vgg2bSw73izQ==", + "license": "MIT" + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "optional": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "optional": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000000000000000000000000000000000000..8db384894e37f24dcaa63347207c680f3d87cada --- /dev/null +++ b/package.json @@ -0,0 +1,15 @@ +{ + "name": "opencode-space", + "version": "1.0.0", + "description": "An AI coding agent running in your browser via Hugging Face Spaces, powered by NVIDIA NIM.", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@huggingface/hub": "^2.13.2" + } +}