"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { newObj[key] = obj[key]; } } } newObj.default = obj; return newObj; } } function _nullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return rhsFn(); } } async function _asyncNullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return await rhsFn(); } } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; } var _class; var _class2; var _class3;require('./chunk-UICA3PK6.js'); // src/consts.ts var HUB_URL = "https://huggingface.co"; // src/error.ts async function createApiError(response, opts) { const error = new HubApiError(response.url, response.status, _nullishCoalesce(response.headers.get("X-Request-Id"), () => ( _optionalChain([opts, 'optionalAccess', _2 => _2.requestId])))); error.message = `Api error with status ${error.statusCode}${_optionalChain([opts, 'optionalAccess', _3 => _3.message]) ? `. ${opts.message}` : ""}`; const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : void 0].filter(Boolean).join(". "); if (_optionalChain([response, 'access', _4 => _4.headers, 'access', _5 => _5.get, 'call', _6 => _6("Content-Type"), 'optionalAccess', _7 => _7.startsWith, 'call', _8 => _8("application/json")])) { const json = await response.json(); error.message = json.error || json.message || error.message; if (json.error_description) { error.message = error.message ? error.message + `: ${json.error_description}` : json.error_description; } error.data = json; } else { error.data = { message: await response.text() }; } error.message += `. ${trailer}`; throw error; } var HubApiError = class extends Error { constructor(url, statusCode, requestId, message) { super(message); this.statusCode = statusCode; this.requestId = requestId; this.url = url; } }; var InvalidApiResponseFormatError = class extends Error { }; // src/utils/checkCredentials.ts function checkAccessToken(accessToken) { if (!accessToken.startsWith("hf_")) { throw new TypeError("Your access token must start with 'hf_'"); } } function checkCredentials(params) { if (params.accessToken) { checkAccessToken(params.accessToken); return params.accessToken; } if (_optionalChain([params, 'access', _9 => _9.credentials, 'optionalAccess', _10 => _10.accessToken])) { checkAccessToken(params.credentials.accessToken); return params.credentials.accessToken; } } // src/utils/toRepoId.ts function toRepoId(repo) { if (typeof repo !== "string") { return repo; } if (repo.startsWith("model/") || repo.startsWith("models/")) { throw new TypeError( "A repo designation for a model should not start with 'models/', directly specify the model namespace / name" ); } if (repo.startsWith("space/")) { throw new TypeError("Spaces should start with 'spaces/', plural, not 'space/'"); } if (repo.startsWith("dataset/")) { throw new TypeError("Datasets should start with 'datasets/', plural, not 'dataset/'"); } if (repo.startsWith("bucket/")) { throw new TypeError("Buckets should start with 'buckets/', plural, not 'bucket/'"); } const slashes = repo.split("/").length - 1; if (repo.startsWith("spaces/")) { if (slashes !== 2) { throw new TypeError("Space Id must include namespace and name of the space"); } return { type: "space", name: repo.slice("spaces/".length) }; } if (repo.startsWith("datasets/")) { if (slashes > 2) { throw new TypeError("Too many slashes in repo designation: " + repo); } return { type: "dataset", name: repo.slice("datasets/".length) }; } if (repo.startsWith("buckets/")) { if (slashes !== 2) { throw new TypeError("Bucket Id must include namespace and name of the bucket"); } return { type: "bucket", name: repo.slice("buckets/".length) }; } if (slashes > 1) { throw new TypeError("Too many slashes in repo designation: " + repo); } return { type: "model", name: repo }; } // src/lib/check-repo-access.ts async function checkRepoAccess(params) { const accessToken = params && checkCredentials(params); const repoId = toRepoId(params.repo); const response = await (params.fetch || fetch)(`${_optionalChain([params, 'optionalAccess', _11 => _11.hubUrl]) || HUB_URL}/api/${repoId.type}s/${repoId.name}`, { headers: { ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } }); if (!response.ok) { throw await createApiError(response); } } // src/utils/range.ts function range(n, b) { return b ? Array(b - n).fill(0).map((_, i) => n + i) : Array(n).fill(0).map((_, i) => i); } // src/utils/chunk.ts function chunk(arr, chunkSize) { if (isNaN(chunkSize) || chunkSize < 1) { throw new RangeError("Invalid chunk size: " + chunkSize); } if (!arr.length) { return []; } if (arr.length <= chunkSize) { return [arr]; } return range(Math.ceil(arr.length / chunkSize)).map((i) => { return arr.slice(i * chunkSize, (i + 1) * chunkSize); }); } // src/utils/promisesQueue.ts async function promisesQueue(factories, concurrency) { const results = []; const executing = /* @__PURE__ */ new Set(); let index = 0; for (const factory of factories) { const closureIndex = index++; const e = factory().then((r) => { results[closureIndex] = r; executing.delete(e); }); executing.add(e); if (executing.size >= concurrency) { await Promise.race(executing); } } await Promise.all(executing); return results; } // src/utils/promisesQueueStreaming.ts async function promisesQueueStreaming(factories, concurrency) { const executing = []; for await (const factory of factories) { const e = factory().then(() => { executing.splice(executing.indexOf(e), 1); }); executing.push(e); if (executing.length >= concurrency) { await Promise.race(executing); } } await Promise.all(executing); } // src/utils/eventToGenerator.ts async function* eventToGenerator(cb) { const promises = []; function addPromise() { let resolve2; let reject; const p = new Promise((res, rej) => { resolve2 = res; reject = rej; }); promises.push({ p, resolve: resolve2, reject }); } addPromise(); const callbackRes = Promise.resolve().then( () => cb( (y) => { addPromise(); _optionalChain([promises, 'access', _12 => _12.at, 'call', _13 => _13(-2), 'optionalAccess', _14 => _14.resolve, 'call', _15 => _15({ done: false, value: y })]); }, (r) => { addPromise(); _optionalChain([promises, 'access', _16 => _16.at, 'call', _17 => _17(-2), 'optionalAccess', _18 => _18.resolve, 'call', _19 => _19({ done: true, value: r })]); }, (err) => _optionalChain([promises, 'access', _20 => _20.shift, 'call', _21 => _21(), 'optionalAccess', _22 => _22.reject, 'call', _23 => _23(err)]) ) ).catch((err) => _optionalChain([promises, 'access', _24 => _24.shift, 'call', _25 => _25(), 'optionalAccess', _26 => _26.reject, 'call', _27 => _27(err)])); while (1) { const p = promises[0]; if (!p) { throw new Error("Logic error in eventGenerator, promises should never be empty"); } const result = await p.p; promises.shift(); if (result.done) { await callbackRes; return result.value; } yield result.value; } throw new Error("Unreachable"); } // src/utils/hexFromBytes.ts function hexFromBytes(arr) { if (globalThis.Buffer) { return globalThis.Buffer.from(arr).toString("hex"); } else { const bin = []; arr.forEach((byte) => { bin.push(byte.toString(16).padStart(2, "0")); }); return bin.join(""); } } // src/utils/isBackend.ts var isBrowser = typeof window !== "undefined" && typeof window.document !== "undefined"; var isWebWorker = typeof self === "object" && self.constructor && self.constructor.name === "DedicatedWorkerGlobalScope"; var isBackend = !isBrowser && !isWebWorker; // src/utils/isFrontend.ts var isFrontend = !isBackend; // src/utils/sha256.ts async function getWebWorkerCode() { const sha256Module = await Promise.resolve().then(() => _interopRequireWildcard(require("./sha256-wrapper-DHTT2DPH.js"))); return URL.createObjectURL(new Blob([sha256Module.createSHA256WorkerCode()])); } var pendingWorkers = []; var runningWorkers = /* @__PURE__ */ new Set(); var resolve; var waitPromise = new Promise((r) => { resolve = r; }); async function getWorker(poolSize) { { const worker2 = pendingWorkers.pop(); if (worker2) { runningWorkers.add(worker2); return worker2; } } if (!poolSize) { const worker2 = new Worker(await getWebWorkerCode()); runningWorkers.add(worker2); return worker2; } if (poolSize <= 0) { throw new TypeError("Invalid webworker pool size: " + poolSize); } while (runningWorkers.size >= poolSize) { await waitPromise; } const worker = new Worker(await getWebWorkerCode()); runningWorkers.add(worker); return worker; } async function freeWorker(worker, poolSize) { if (!poolSize) { return destroyWorker(worker); } runningWorkers.delete(worker); pendingWorkers.push(worker); const r = resolve; waitPromise = new Promise((r2) => { resolve = r2; }); r(); } function destroyWorker(worker) { runningWorkers.delete(worker); worker.terminate(); const r = resolve; waitPromise = new Promise((r2) => { resolve = r2; }); r(); } async function* sha256(buffer, opts) { yield 0; const maxCryptoSize = typeof _optionalChain([opts, 'optionalAccess', _28 => _28.useWebWorker]) === "object" && _optionalChain([opts, 'optionalAccess', _29 => _29.useWebWorker, 'access', _30 => _30.minSize]) !== void 0 ? opts.useWebWorker.minSize : 1e7; if (buffer.size < maxCryptoSize && _optionalChain([globalThis, 'access', _31 => _31.crypto, 'optionalAccess', _32 => _32.subtle])) { const res = hexFromBytes( new Uint8Array( await globalThis.crypto.subtle.digest("SHA-256", buffer instanceof Blob ? await buffer.arrayBuffer() : buffer) ) ); yield 1; return res; } if (isFrontend) { if (_optionalChain([opts, 'optionalAccess', _33 => _33.useWebWorker])) { try { const poolSize = typeof _optionalChain([opts, 'optionalAccess', _34 => _34.useWebWorker]) === "object" ? opts.useWebWorker.poolSize : void 0; const worker = await getWorker(poolSize); let messageHandler; let errorHandler; const cleanup = () => { worker.removeEventListener("message", messageHandler); worker.removeEventListener("error", errorHandler); }; return yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { messageHandler = (event) => { if (event.data.sha256) { cleanup(); freeWorker(worker, poolSize); returnCallback(event.data.sha256); } else if (event.data.progress) { yieldCallback(event.data.progress); try { _optionalChain([opts, 'access', _35 => _35.abortSignal, 'optionalAccess', _36 => _36.throwIfAborted, 'call', _37 => _37()]); } catch (err) { cleanup(); destroyWorker(worker); rejectCallback(err); } } else { cleanup(); destroyWorker(worker); rejectCallback(event); } }; errorHandler = (event) => { cleanup(); destroyWorker(worker); rejectCallback(event.error); }; if (_optionalChain([opts, 'optionalAccess', _38 => _38.abortSignal])) { try { _optionalChain([opts, 'access', _39 => _39.abortSignal, 'optionalAccess', _40 => _40.throwIfAborted, 'call', _41 => _41()]); } catch (err) { cleanup(); destroyWorker(worker); rejectCallback(_nullishCoalesce(opts.abortSignal.reason, () => ( new DOMException("Aborted", "AbortError")))); return; } const abortListener = () => { cleanup(); destroyWorker(worker); rejectCallback(_nullishCoalesce(_optionalChain([opts, 'access', _42 => _42.abortSignal, 'optionalAccess', _43 => _43.reason]), () => ( new DOMException("Aborted", "AbortError")))); _optionalChain([opts, 'access', _44 => _44.abortSignal, 'optionalAccess', _45 => _45.removeEventListener, 'call', _46 => _46("abort", abortListener)]); }; opts.abortSignal.addEventListener("abort", abortListener); } worker.addEventListener("message", messageHandler); worker.addEventListener("error", errorHandler); worker.postMessage({ file: buffer }); }); } catch (err) { console.warn("Failed to use web worker for sha256", err); } } if (!wasmModule) { wasmModule = await Promise.resolve().then(() => _interopRequireWildcard(require("./sha256-wrapper-DHTT2DPH.js"))); } const sha2562 = await wasmModule.createSHA256(); sha2562.init(); const reader = buffer.stream().getReader(); const total = buffer.size; let bytesDone = 0; while (true) { const { done, value } = await reader.read(); if (done) { break; } sha2562.update(value); bytesDone += value.length; yield bytesDone / total; _optionalChain([opts, 'optionalAccess', _47 => _47.abortSignal, 'optionalAccess', _48 => _48.throwIfAborted, 'call', _49 => _49()]); } return sha2562.digest("hex"); } if (!cryptoModule) { cryptoModule = await Promise.resolve().then(() => _interopRequireWildcard(require("./sha256-node-FT2Y3VXD.js"))); } return yield* cryptoModule.sha256Node(buffer, { abortSignal: _optionalChain([opts, 'optionalAccess', _50 => _50.abortSignal]) }); } var cryptoModule; var wasmModule; // src/utils/WebBlob.ts var WebBlob = class extends Blob { static async create(url, opts) { const customFetch = _nullishCoalesce(_optionalChain([opts, 'optionalAccess', _51 => _51.fetch]), () => ( fetch)); const response = await customFetch(url, { method: "HEAD", ..._optionalChain([opts, 'optionalAccess', _52 => _52.accessToken]) && { headers: { Authorization: `Bearer ${opts.accessToken}` } } }); const size = Number(response.headers.get("content-length")); const contentType = response.headers.get("content-type") || ""; const supportRange = response.headers.get("accept-ranges") === "bytes"; if (!supportRange || size < (_nullishCoalesce(_optionalChain([opts, 'optionalAccess', _53 => _53.cacheBelow]), () => ( 1e6)))) { return await (await customFetch(url)).blob(); } return new WebBlob(url, 0, size, contentType, true, customFetch, _optionalChain([opts, 'optionalAccess', _54 => _54.accessToken])); } constructor(url, start, end, contentType, full, customFetch, accessToken) { super([]); this.url = url; this.start = start; this.end = end; this.contentType = contentType; this.full = full; this.fetch = customFetch; this.accessToken = accessToken; } get size() { return this.end - this.start; } get type() { return this.contentType; } slice(start = 0, end = this.size) { if (start < 0 || end < 0) { new TypeError("Unsupported negative start/end on WebBlob.slice"); } const slice = new WebBlob( this.url, this.start + start, Math.min(this.start + end, this.end), this.contentType, start === 0 && end === this.size ? this.full : false, this.fetch, this.accessToken ); return slice; } async arrayBuffer() { const result = await this.fetchRange(); return result.arrayBuffer(); } async text() { const result = await this.fetchRange(); return result.text(); } stream() { const stream = new TransformStream(); this.fetchRange().then((response) => _optionalChain([response, 'access', _55 => _55.body, 'optionalAccess', _56 => _56.pipeThrough, 'call', _57 => _57(stream)])).catch((error) => stream.writable.abort(error.message)); return stream.readable; } fetchRange() { const fetch2 = this.fetch; if (this.full) { return fetch2(this.url, { ...this.accessToken && { headers: { Authorization: `Bearer ${this.accessToken}` } } }).then((resp) => resp.ok ? resp : createApiError(resp)); } return fetch2(this.url, { headers: { Range: `bytes=${this.start}-${this.end - 1}`, ...this.accessToken && { Authorization: `Bearer ${this.accessToken}` } } }).then((resp) => resp.ok ? resp : createApiError(resp)); } }; // src/utils/base64FromBytes.ts function base64FromBytes(arr) { if (globalThis.Buffer) { return globalThis.Buffer.from(arr).toString("base64"); } else { const bin = []; arr.forEach((byte) => { bin.push(String.fromCharCode(byte)); }); return globalThis.btoa(bin.join("")); } } // src/utils/createBlobs.ts async function createBlobs(url, destPath, opts) { if (url.protocol === "http:" || url.protocol === "https:") { const blob = await WebBlob.create(url, { fetch: _optionalChain([opts, 'optionalAccess', _58 => _58.fetch]), accessToken: _optionalChain([opts, 'optionalAccess', _59 => _59.accessToken]) }); return [{ path: destPath, blob }]; } if (isFrontend) { throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); } if (url.protocol === "file:") { const { FileBlob } = await Promise.resolve().then(() => _interopRequireWildcard(require("./FileBlob-YC2EPDW4.js"))); const { subPaths } = await Promise.resolve().then(() => _interopRequireWildcard(require("./sub-paths-RH3O65LG.js"))); const paths = await subPaths(url, _optionalChain([opts, 'optionalAccess', _60 => _60.maxFolderDepth])); if (paths.length === 1 && paths[0].relativePath === ".") { const blob = await FileBlob.create(url); return [{ path: destPath, blob }]; } return Promise.all( paths.map(async (path) => ({ path: `${destPath}/${path.relativePath}`.replace(/\/[.]$/, "").replaceAll("//", "/").replace(/^[.]?\//, ""), blob: await FileBlob.create(new URL(path.path)) })) ); } throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); } // src/utils/combineUint8Arrays.ts function combineUint8Arrays(a, b) { const aLength = a.length; const combinedBytes = new Uint8Array(aLength + b.length); combinedBytes.set(a); combinedBytes.set(b, aLength); return combinedBytes; } // src/vendor/lz4js/util.ts function hashU32(a) { a = a | 0; a = a + 2127912214 + (a << 12) | 0; a = a ^ -949894596 ^ a >>> 19; a = a + 374761393 + (a << 5) | 0; a = a + -744332180 ^ a << 9; a = a + -42973499 + (a << 3) | 0; return a ^ -1252372727 ^ a >>> 16 | 0; } function readU64(b, n) { let x = 0; x |= b[n++] << 0; x |= b[n++] << 8; x |= b[n++] << 16; x |= b[n++] << 24; x |= b[n++] << 32; x |= b[n++] << 40; x |= b[n++] << 48; x |= b[n++] << 56; return x; } function readU32(b, n) { let x = 0; x |= b[n++] << 0; x |= b[n++] << 8; x |= b[n++] << 16; x |= b[n++] << 24; return x; } function writeU32(b, n, x) { b[n++] = x >> 0 & 255; b[n++] = x >> 8 & 255; b[n++] = x >> 16 & 255; b[n++] = x >> 24 & 255; } function imul(a, b) { const ah = a >>> 16; const al = a & 65535; const bh = b >>> 16; const bl = b & 65535; return al * bl + (ah * bl + al * bh << 16) | 0; } // src/vendor/lz4js/xxh32.ts var prime1 = 2654435761; var prime2 = 2246822519; var prime3 = 3266489917; var prime4 = 668265263; var prime5 = 374761393; function rotl32(x, r) { x = x | 0; r = r | 0; return x >>> (32 - r | 0) | x << r | 0; } function rotmul32(h, r, m) { h = h | 0; r = r | 0; m = m | 0; return imul(h >>> (32 - r | 0) | h << r, m) | 0; } function shiftxor32(h, s) { h = h | 0; s = s | 0; return h >>> s ^ h | 0; } function xxhapply(h, src, m0, s, m1) { return rotmul32(imul(src, m0) + h, s, m1); } function xxh1(h, src, index) { return rotmul32(h + imul(src[index], prime5), 11, prime1); } function xxh4(h, src, index) { return xxhapply(h, readU32(src, index), prime3, 17, prime4); } function xxh16(h, src, index) { return [ xxhapply(h[0], readU32(src, index + 0), prime2, 13, prime1), xxhapply(h[1], readU32(src, index + 4), prime2, 13, prime1), xxhapply(h[2], readU32(src, index + 8), prime2, 13, prime1), xxhapply(h[3], readU32(src, index + 12), prime2, 13, prime1) ]; } function xxh32(seed, src, index, len) { let h; const l = len; if (len >= 16) { h = [seed + prime1 + prime2, seed + prime2, seed, seed - prime1]; while (len >= 16) { h = xxh16(h, src, index); index += 16; len -= 16; } h = rotl32(h[0], 1) + rotl32(h[1], 7) + rotl32(h[2], 12) + rotl32(h[3], 18) + l; } else { h = seed + prime5 + len >>> 0; } while (len >= 4) { h = xxh4(h, src, index); index += 4; len -= 4; } while (len > 0) { h = xxh1(h, src, index); index++; len--; } h = shiftxor32(imul(shiftxor32(imul(shiftxor32(h, 15), prime2), 13), prime3), 16); return h >>> 0; } var hash = xxh32; // src/vendor/lz4js/index.ts var minMatch = 4; var matchSearchLimit = 12; var minTrailingLitterals = 5; var skipTrigger = 6; var hashSize = 1 << 16; var mlBits = 4; var mlMask = (1 << mlBits) - 1; var runBits = 4; var runMask = (1 << runBits) - 1; var blockBuf = makeBuffer(5 << 20); var hashTable = makeHashTable(); var magicNum = 407708164; var fdContentChksum = 4; var fdContentSize = 8; var fdBlockChksum = 16; var fdVersion = 64; var fdVersionMask = 192; var bsUncompressed = 2147483648; var bsDefault = 7; var bsShift = 4; var bsMask = 7; var bsMap = { 4: 65536, 5: 262144, 6: 1048576, 7: 4194304 }; function makeHashTable() { try { return new Uint32Array(hashSize); } catch (error) { const hashTable2 = new Array(hashSize); for (let i = 0; i < hashSize; i++) { hashTable2[i] = 0; } return hashTable2; } } function clearHashTable(table) { for (let i = 0; i < hashSize; i++) { table[i] = 0; } } function makeBuffer(size) { return new Uint8Array(size); } function sliceArray(array, start, end) { return array.slice(start, end); } function compressBound(n) { return n + n / 255 + 16 | 0; } function decompressBound(src) { let sIndex = 0; if (readU32(src, sIndex) !== magicNum) { throw new Error("invalid magic number"); } sIndex += 4; const descriptor = src[sIndex++]; if ((descriptor & fdVersionMask) !== fdVersion) { throw new Error("incompatible descriptor version " + (descriptor & fdVersionMask)); } const useBlockSum = (descriptor & fdBlockChksum) !== 0; const useContentSize = (descriptor & fdContentSize) !== 0; const bsIdx = src[sIndex++] >> bsShift & bsMask; if (bsMap[bsIdx] === void 0) { throw new Error("invalid block size " + bsIdx); } const maxBlockSize = bsMap[bsIdx]; if (useContentSize) { return readU64(src, sIndex); } sIndex++; let maxSize = 0; while (true) { let blockSize = readU32(src, sIndex); sIndex += 4; if (blockSize & bsUncompressed) { blockSize &= ~bsUncompressed; maxSize += blockSize; } else if (blockSize > 0) { maxSize += maxBlockSize; } if (blockSize === 0) { return maxSize; } if (useBlockSum) { sIndex += 4; } sIndex += blockSize; } } function decompressBlock(src, dst, sIndex, sLength, dIndex) { let mLength, mOffset, sEnd, n, i; const hasCopyWithin = dst.copyWithin !== void 0 && dst.fill !== void 0; sEnd = sIndex + sLength; while (sIndex < sEnd) { const token = src[sIndex++]; let literalCount = token >> 4; if (literalCount > 0) { if (literalCount === 15) { while (true) { literalCount += src[sIndex]; if (src[sIndex++] !== 255) { break; } } } for (n = sIndex + literalCount; sIndex < n; ) { dst[dIndex++] = src[sIndex++]; } } if (sIndex >= sEnd) { break; } mLength = token & 15; mOffset = src[sIndex++] | src[sIndex++] << 8; if (mLength === 15) { while (true) { mLength += src[sIndex]; if (src[sIndex++] !== 255) { break; } } } mLength += minMatch; if (hasCopyWithin && mOffset === 1) { dst.fill(dst[dIndex - 1] | 0, dIndex, dIndex + mLength); dIndex += mLength; } else if (hasCopyWithin && mOffset > mLength && mLength > 31) { dst.copyWithin(dIndex, dIndex - mOffset, dIndex - mOffset + mLength); dIndex += mLength; } else { for (i = dIndex - mOffset, n = i + mLength; i < n; ) { dst[dIndex++] = dst[i++] | 0; } } } return dIndex; } function compressBlock(src, dst, sIndex, sLength, hashTable2) { let mIndex, mAnchor, mLength, mOffset, mStep; let literalCount, dIndex, sEnd, n; dIndex = 0; sEnd = sLength + sIndex; mAnchor = sIndex; let searchMatchCount = (1 << skipTrigger) + 3; while (sIndex <= sEnd - matchSearchLimit) { const seq = readU32(src, sIndex); let hash2 = hashU32(seq) >>> 0; hash2 = (hash2 >> 16 ^ hash2) >>> 0 & 65535; mIndex = hashTable2[hash2] - 1; hashTable2[hash2] = sIndex + 1; if (mIndex < 0 || sIndex - mIndex >>> 16 > 0 || readU32(src, mIndex) !== seq) { mStep = searchMatchCount++ >> skipTrigger; sIndex += mStep; continue; } searchMatchCount = (1 << skipTrigger) + 3; literalCount = sIndex - mAnchor; mOffset = sIndex - mIndex; sIndex += minMatch; mIndex += minMatch; mLength = sIndex; while (sIndex < sEnd - minTrailingLitterals && src[sIndex] === src[mIndex]) { sIndex++; mIndex++; } mLength = sIndex - mLength; const token = mLength < mlMask ? mLength : mlMask; if (literalCount >= runMask) { dst[dIndex++] = (runMask << mlBits) + token; for (n = literalCount - runMask; n >= 255; n -= 255) { dst[dIndex++] = 255; } dst[dIndex++] = n; } else { dst[dIndex++] = (literalCount << mlBits) + token; } for (let i = 0; i < literalCount; i++) { dst[dIndex++] = src[mAnchor + i]; } dst[dIndex++] = mOffset; dst[dIndex++] = mOffset >> 8; if (mLength >= mlMask) { for (n = mLength - mlMask; n >= 255; n -= 255) { dst[dIndex++] = 255; } dst[dIndex++] = n; } mAnchor = sIndex; } if (mAnchor === 0) { return 0; } literalCount = sEnd - mAnchor; if (literalCount >= runMask) { dst[dIndex++] = runMask << mlBits; for (n = literalCount - runMask; n >= 255; n -= 255) { dst[dIndex++] = 255; } dst[dIndex++] = n; } else { dst[dIndex++] = literalCount << mlBits; } sIndex = mAnchor; while (sIndex < sEnd) { dst[dIndex++] = src[sIndex++]; } return dIndex; } function decompressFrame(src, dst) { let useBlockSum, useContentSum, useContentSize, descriptor; let sIndex = 0; let dIndex = 0; if (readU32(src, sIndex) !== magicNum) { throw new Error("invalid magic number"); } sIndex += 4; descriptor = src[sIndex++]; if ((descriptor & fdVersionMask) !== fdVersion) { throw new Error("incompatible descriptor version"); } useBlockSum = (descriptor & fdBlockChksum) !== 0; useContentSum = (descriptor & fdContentChksum) !== 0; useContentSize = (descriptor & fdContentSize) !== 0; const bsIdx = src[sIndex++] >> bsShift & bsMask; if (bsMap[bsIdx] === void 0) { throw new Error("invalid block size"); } if (useContentSize) { sIndex += 8; } sIndex++; while (true) { var compSize; compSize = readU32(src, sIndex); sIndex += 4; if (compSize === 0) { break; } if (useBlockSum) { sIndex += 4; } if ((compSize & bsUncompressed) !== 0) { compSize &= ~bsUncompressed; for (let j = 0; j < compSize; j++) { dst[dIndex++] = src[sIndex++]; } } else { dIndex = decompressBlock(src, dst, sIndex, compSize, dIndex); sIndex += compSize; } } if (useContentSum) { sIndex += 4; } return dIndex; } function compressFrame(src, dst) { let dIndex = 0; writeU32(dst, dIndex, magicNum); dIndex += 4; dst[dIndex++] = fdVersion; dst[dIndex++] = bsDefault << bsShift; dst[dIndex] = hash(0, dst, 4, dIndex - 4) >> 8; dIndex++; const maxBlockSize = bsMap[bsDefault]; let remaining = src.length; let sIndex = 0; clearHashTable(hashTable); while (remaining > 0) { let compSize = 0; const blockSize = remaining > maxBlockSize ? maxBlockSize : remaining; compSize = compressBlock(src, blockBuf, sIndex, blockSize, hashTable); if (compSize > blockSize || compSize === 0) { writeU32(dst, dIndex, 2147483648 | blockSize); dIndex += 4; for (let z = sIndex + blockSize; sIndex < z; ) { dst[dIndex++] = src[sIndex++]; } remaining -= blockSize; } else { writeU32(dst, dIndex, compSize); dIndex += 4; for (let j = 0; j < compSize; ) { dst[dIndex++] = blockBuf[j++]; } sIndex += blockSize; remaining -= blockSize; } } writeU32(dst, dIndex, 0); dIndex += 4; return dIndex; } function decompress(src, maxSize) { let dst, size; if (maxSize === void 0) { maxSize = decompressBound(src); } dst = makeBuffer(maxSize); size = decompressFrame(src, dst); if (size !== maxSize) { dst = sliceArray(dst, 0, size); } return dst; } function compress(src, maxSize) { let dst, size; if (maxSize === void 0) { maxSize = compressBound(src.length); } dst = makeBuffer(maxSize); size = compressFrame(src, dst); if (size !== maxSize) { dst = sliceArray(dst, 0, size); } return dst; } // src/utils/RangeList.ts var RangeList = (_class = class {constructor() { _class.prototype.__init.call(this); } __init() {this.ranges = []} /** * Add a range to the list. If it overlaps with existing ranges, * it will split them and increment reference counts accordingly. */ add(start, end) { if (end <= start) { throw new TypeError("End must be greater than start"); } const overlappingRanges = []; for (let i = 0; i < this.ranges.length; i++) { const range2 = this.ranges[i]; if (start < range2.end && end > range2.start) { overlappingRanges.push({ index: i, range: range2 }); } if (range2.data !== null) { throw new Error("Overlapping range already has data"); } } if (overlappingRanges.length === 0) { this.ranges.push({ start, end, refCount: 1, data: null }); this.ranges.sort((a, b) => a.start - b.start); return; } const newRanges = []; let currentPos = start; for (let i = 0; i < overlappingRanges.length; i++) { const { range: range2 } = overlappingRanges[i]; if (currentPos < range2.start) { newRanges.push({ start: currentPos, end: range2.start, refCount: 1, data: null }); } else if (range2.start < currentPos) { newRanges.push({ start: range2.start, end: currentPos, refCount: range2.refCount, data: null }); } newRanges.push({ start: Math.max(currentPos, range2.start), end: Math.min(end, range2.end), refCount: range2.refCount + 1, data: null }); if (range2.end > end) { newRanges.push({ start: end, end: range2.end, refCount: range2.refCount, data: null }); } currentPos = Math.max(currentPos, range2.end); } if (currentPos < end) { newRanges.push({ start: currentPos, end, refCount: 1, data: null }); } const firstIndex = overlappingRanges[0].index; const lastIndex = overlappingRanges[overlappingRanges.length - 1].index; this.ranges.splice(firstIndex, lastIndex - firstIndex + 1, ...newRanges); this.ranges.sort((a, b) => a.start - b.start); } /** * Remove a range from the list. The range must start and end at existing boundaries. */ remove(start, end) { if (end <= start) { throw new TypeError("End must be greater than start"); } const affectedRanges = []; for (let i = 0; i < this.ranges.length; i++) { const range2 = this.ranges[i]; if (start < range2.end && end > range2.start) { affectedRanges.push({ index: i, range: range2 }); } } if (affectedRanges.length === 0) { throw new Error("No ranges found to remove"); } if (start !== affectedRanges[0].range.start || end !== affectedRanges[affectedRanges.length - 1].range.end) { throw new Error("Range boundaries must match existing boundaries"); } for (let i = 0; i < affectedRanges.length; i++) { const { range: range2 } = affectedRanges[i]; range2.refCount--; } this.ranges = this.ranges.filter((range2) => range2.refCount > 0); } /** * Get all ranges within the specified boundaries. */ getRanges(start, end) { if (end <= start) { throw new TypeError("End must be greater than start"); } return this.ranges.filter((range2) => start < range2.end && end > range2.start); } /** * Get all ranges in the list */ getAllRanges() { return [...this.ranges]; } }, _class); // src/utils/XetBlob.ts var JWT_SAFETY_PERIOD = 6e4; var JWT_CACHE_SIZE = 1e3; var compressionSchemeLabels = { [0 /* None */]: "None", [1 /* LZ4 */]: "LZ4", [2 /* ByteGroupingLZ4 */]: "ByteGroupingLZ4" }; var XET_CHUNK_HEADER_BYTES = 8; var XetBlob = (_class2 = class extends Blob { __init2() {this.start = 0} __init3() {this.end = 0} __init4() {this.internalLogging = false} constructor(params) { super([]);_class2.prototype.__init2.call(this);_class2.prototype.__init3.call(this);_class2.prototype.__init4.call(this);; this.fetch = _nullishCoalesce(params.fetch, () => ( fetch.bind(globalThis))); this.accessToken = checkCredentials(params); this.refreshUrl = params.refreshUrl; this.end = params.size; this.reconstructionUrl = params.reconstructionUrl; this.hash = params.hash; this.listener = params.listener; this.internalLogging = _nullishCoalesce(params.internalLogging, () => ( false)); if (params.readToken) { const key = cacheKey({ refreshUrl: this.refreshUrl, initialAccessToken: this.accessToken }); jwts.set(key, { accessToken: params.readToken.accessToken, expiresAt: new Date(params.readToken.exp * 1e3), casUrl: params.readToken.casUrl }); } } get size() { return this.end - this.start; } #clone() { const blob = new XetBlob({ fetch: this.fetch, hash: this.hash, refreshUrl: this.refreshUrl, // eslint-disable-next-line @typescript-eslint/no-non-null-assertion reconstructionUrl: this.reconstructionUrl, size: this.size }); blob.accessToken = this.accessToken; blob.start = this.start; blob.end = this.end; blob.reconstructionInfo = this.reconstructionInfo; blob.listener = this.listener; blob.internalLogging = this.internalLogging; return blob; } slice(start = 0, end = this.size) { if (start < 0 || end < 0) { new TypeError("Unsupported negative start/end on XetBlob.slice"); } const slice = this.#clone(); slice.start = this.start + start; slice.end = Math.min(this.start + end, this.end); if (slice.start !== this.start || slice.end !== this.end) { slice.reconstructionInfo = void 0; } return slice; } #reconstructionInfoPromise; #loadReconstructionInfo() { if (this.#reconstructionInfoPromise) { return this.#reconstructionInfoPromise; } this.#reconstructionInfoPromise = (async () => { const connParams = await getAccessToken(this.accessToken, this.fetch, this.refreshUrl); const resp = await this.fetch(_nullishCoalesce(this.reconstructionUrl, () => ( `${connParams.casUrl}/v1/reconstructions/${this.hash}`)), { headers: { Authorization: `Bearer ${connParams.accessToken}`, Range: `bytes=${this.start}-${this.end - 1}` } }); if (!resp.ok) { throw await createApiError(resp); } this.reconstructionInfo = await resp.json(); return this.reconstructionInfo; })().finally(() => this.#reconstructionInfoPromise = void 0); return this.#reconstructionInfoPromise; } async #fetch() { if (!this.reconstructionInfo) { await this.#loadReconstructionInfo(); } const rangeLists = /* @__PURE__ */ new Map(); if (!this.reconstructionInfo) { throw new Error("Failed to load reconstruction info"); } for (const term of this.reconstructionInfo.terms) { let rangeList = rangeLists.get(term.hash); if (!rangeList) { rangeList = new RangeList(); rangeLists.set(term.hash, rangeList); } rangeList.add(term.range.start, term.range.end); } const listener = this.listener; const log = this.internalLogging ? (...args) => console.log(...args) : () => { }; async function* readData(reconstructionInfo, customFetch, maxBytes, reloadReconstructionInfo) { let totalBytesRead = 0; let readBytesToSkip = reconstructionInfo.offset_into_first_range; for (const term of reconstructionInfo.terms) { if (totalBytesRead >= maxBytes) { break; } const rangeList = rangeLists.get(term.hash); if (!rangeList) { throw new Error(`Failed to find range list for term ${term.hash}`); } { const termRanges = rangeList.getRanges(term.range.start, term.range.end); if (termRanges.every((range2) => range2.data)) { log("all data available for term", term.hash, readBytesToSkip); rangeLoop: for (const range2 of termRanges) { for (let chunk2 of range2.data) { if (readBytesToSkip) { const skipped = Math.min(readBytesToSkip, chunk2.byteLength); chunk2 = chunk2.slice(skipped); readBytesToSkip -= skipped; if (!chunk2.byteLength) { continue; } } if (chunk2.byteLength > maxBytes - totalBytesRead) { chunk2 = chunk2.slice(0, maxBytes - totalBytesRead); } totalBytesRead += chunk2.byteLength; yield range2.refCount > 1 ? chunk2.slice() : chunk2; _optionalChain([listener, 'optionalCall', _61 => _61({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } })]); if (totalBytesRead >= maxBytes) { break rangeLoop; } } } rangeList.remove(term.range.start, term.range.end); continue; } } const fetchInfo = reconstructionInfo.fetch_info[term.hash].find( (info) => info.range.start <= term.range.start && info.range.end >= term.range.end ); if (!fetchInfo) { throw new Error( `Failed to find fetch info for term ${term.hash} and range ${term.range.start}-${term.range.end}` ); } log("term", term); log("fetchinfo", fetchInfo); log("readBytesToSkip", readBytesToSkip); let resp = await customFetch(fetchInfo.url, { headers: { Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}` } }); if (resp.status === 403) { reconstructionInfo = await reloadReconstructionInfo(); resp = await customFetch(fetchInfo.url, { headers: { Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}` } }); } if (!resp.ok) { throw await createApiError(resp); } log( "expected content length", resp.headers.get("content-length"), "range", fetchInfo.url_range, resp.headers.get("content-range") ); const reader = _optionalChain([resp, 'access', _62 => _62.body, 'optionalAccess', _63 => _63.getReader, 'call', _64 => _64()]); if (!reader) { throw new Error("Failed to get reader from response body"); } let done = false; let chunkIndex = fetchInfo.range.start; const ranges = rangeList.getRanges(fetchInfo.range.start, fetchInfo.range.end); let leftoverBytes = void 0; let totalFetchBytes = 0; fetchData: while (!done && totalBytesRead < maxBytes) { const result = await reader.read(); _optionalChain([listener, 'optionalCall', _65 => _65({ event: "read" })]); done = result.done; log("read", _optionalChain([result, 'access', _66 => _66.value, 'optionalAccess', _67 => _67.byteLength]), "bytes", "total read", totalBytesRead, "toSkip", readBytesToSkip); if (!result.value) { log("no data in result, cancelled", result); continue; } totalFetchBytes += result.value.byteLength; if (leftoverBytes) { result.value = combineUint8Arrays(leftoverBytes, result.value); leftoverBytes = void 0; } while (totalBytesRead < maxBytes && _optionalChain([result, 'access', _68 => _68.value, 'optionalAccess', _69 => _69.byteLength])) { if (result.value.byteLength < 8) { leftoverBytes = result.value; continue fetchData; } const header = new DataView(result.value.buffer, result.value.byteOffset, XET_CHUNK_HEADER_BYTES); const chunkHeader = { version: header.getUint8(0), compressed_length: header.getUint8(1) | header.getUint8(2) << 8 | header.getUint8(3) << 16, compression_scheme: header.getUint8(4), uncompressed_length: header.getUint8(5) | header.getUint8(6) << 8 | header.getUint8(7) << 16 }; log("chunk header", chunkHeader, "to skip", readBytesToSkip); if (chunkHeader.version !== 0) { throw new Error(`Unsupported chunk version ${chunkHeader.version}`); } if (chunkHeader.compression_scheme !== 0 /* None */ && chunkHeader.compression_scheme !== 1 /* LZ4 */ && chunkHeader.compression_scheme !== 2 /* ByteGroupingLZ4 */) { throw new Error( `Unsupported compression scheme ${_nullishCoalesce(compressionSchemeLabels[chunkHeader.compression_scheme], () => ( chunkHeader.compression_scheme))}` ); } if (result.value.byteLength < chunkHeader.compressed_length + XET_CHUNK_HEADER_BYTES) { leftoverBytes = result.value; continue fetchData; } result.value = result.value.slice(XET_CHUNK_HEADER_BYTES); let uncompressed = chunkHeader.compression_scheme === 1 /* LZ4 */ ? decompress(result.value.slice(0, chunkHeader.compressed_length), chunkHeader.uncompressed_length) : chunkHeader.compression_scheme === 2 /* ByteGroupingLZ4 */ ? bg4_regroup_bytes( decompress( result.value.slice(0, chunkHeader.compressed_length), chunkHeader.uncompressed_length ) ) : result.value.slice(0, chunkHeader.compressed_length); const range2 = ranges.find((range3) => chunkIndex >= range3.start && chunkIndex < range3.end); const shouldYield = chunkIndex >= term.range.start && chunkIndex < term.range.end; const minRefCountToStore = shouldYield ? 2 : 1; let stored = false; if (range2 && range2.refCount >= minRefCountToStore) { range2.data ??= []; range2.data.push(uncompressed); stored = true; } if (shouldYield) { if (readBytesToSkip) { const skipped = Math.min(readBytesToSkip, uncompressed.byteLength); uncompressed = uncompressed.slice(readBytesToSkip); readBytesToSkip -= skipped; } if (uncompressed.byteLength > maxBytes - totalBytesRead) { uncompressed = uncompressed.slice(0, maxBytes - totalBytesRead); } if (uncompressed.byteLength) { log( "yield", uncompressed.byteLength, "bytes", result.value.byteLength, "total read", totalBytesRead, stored ); totalBytesRead += uncompressed.byteLength; yield stored ? uncompressed.slice() : uncompressed; _optionalChain([listener, 'optionalCall', _70 => _70({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } })]); } } chunkIndex++; result.value = result.value.slice(chunkHeader.compressed_length); } } if (done && totalBytesRead < maxBytes && totalFetchBytes < fetchInfo.url_range.end - fetchInfo.url_range.start + 1) { log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes); log("failed to fetch all data for term", term.hash); throw new Error( `Failed to fetch all data for term ${term.hash}, fetched ${totalFetchBytes} bytes out of ${fetchInfo.url_range.end - fetchInfo.url_range.start + 1}` ); } log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes); log("cancel reader"); await reader.cancel(); } } const iterator = readData( this.reconstructionInfo, this.fetch, this.end - this.start, this.#loadReconstructionInfo.bind(this) ); return new ReadableStream( { // todo: when Safari supports it, type controller as ReadableByteStreamController async pull(controller) { const result = await iterator.next(); if (result.value) { controller.enqueue(result.value); } if (result.done) { controller.close(); } }, type: "bytes" // todo: when Safari supports it, add autoAllocateChunkSize param }, // todo : use ByteLengthQueuingStrategy when there's good support for it, currently in Node.js it fails due to size being a function { highWaterMark: 1e3 // 1_000 chunks for ~1MB of RAM } ); } async arrayBuffer() { const result = await this.#fetch(); return new Response(result).arrayBuffer(); } async text() { const result = await this.#fetch(); return new Response(result).text(); } async response() { const result = await this.#fetch(); return new Response(result); } stream() { const stream = new TransformStream(); this.#fetch().then((response) => response.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message)); return stream.readable; } }, _class2); var jwtPromises = /* @__PURE__ */ new Map(); var jwts = /* @__PURE__ */ new Map(); function cacheKey(params) { return JSON.stringify([params.refreshUrl, params.initialAccessToken]); } function bg4_regroup_bytes(bytes) { const split = Math.floor(bytes.byteLength / 4); const rem = bytes.byteLength % 4; const g1_pos = split + (rem >= 1 ? 1 : 0); const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); const ret = new Uint8Array(bytes.byteLength); for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { ret[i] = bytes[j]; } for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { ret[i] = bytes[j]; } for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { ret[i] = bytes[j]; } for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { ret[i] = bytes[j]; } return ret; } function bg4_split_bytes(bytes) { const ret = new Uint8Array(bytes.byteLength); const split = Math.floor(bytes.byteLength / 4); const rem = bytes.byteLength % 4; const g1_pos = split + (rem >= 1 ? 1 : 0); const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { ret[j] = bytes[i]; } for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { ret[j] = bytes[i]; } for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { ret[j] = bytes[i]; } for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { ret[j] = bytes[i]; } return ret; } async function getAccessToken(initialAccessToken, customFetch, refreshUrl) { const key = cacheKey({ refreshUrl, initialAccessToken }); const jwt = jwts.get(key); if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD)) { return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; } const existingPromise = jwtPromises.get(key); if (existingPromise) { return existingPromise; } const promise = (async () => { const resp = await customFetch(refreshUrl, { headers: { ...initialAccessToken ? { Authorization: `Bearer ${initialAccessToken}` } : {} } }); if (!resp.ok) { throw new Error(`Failed to get JWT token: ${resp.status} ${await resp.text()}`); } const json = await resp.json(); const jwt2 = { accessToken: json.accessToken, expiresAt: new Date(json.exp * 1e3), casUrl: json.casUrl }; jwtPromises.delete(key); for (const [key2, value] of jwts.entries()) { if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD)) { jwts.delete(key2); } else { break; } } if (jwts.size >= JWT_CACHE_SIZE) { const keyToDelete = jwts.keys().next().value; if (keyToDelete) { jwts.delete(keyToDelete); } } jwts.set(key, jwt2); return { accessToken: json.accessToken, casUrl: json.casUrl }; })(); jwtPromises.set(key, promise); return promise; } // src/utils/ChunkCache.ts var CHUNK_CACHE_INITIAL_SIZE = 1e4; var CHUNK_CACHE_GROW_FACTOR = 1.5; var CHUNK_CACHE_MAX_SIZE = 1e6; var ChunkCache = (_class3 = class { __init5() {this.index = 0} // Index >= 0 means local xorb, < 0 means remote xorb // Max 8K chunks per xorb, less than 64K uint16_t __init6() {this.map = /* @__PURE__ */ new Map()} // hash -> chunkCacheIndex. Less overhead that way, empty object is 60+B and empty array is 40+B __init7() {this.hmacs = /* @__PURE__ */ new Set()} // todo : remove old hmacs constructor(maxSize = CHUNK_CACHE_MAX_SIZE) {;_class3.prototype.__init5.call(this);_class3.prototype.__init6.call(this);_class3.prototype.__init7.call(this); if (maxSize < 1) { throw new Error("maxSize must be at least 1"); } this.maxSize = maxSize; this.xorbIndices = new Int32Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); this.chunkIndices = new Uint16Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); } addChunkToCache(hash2, xorbIndex, chunkIndex, hmac) { if (this.map.has(hash2)) { return; } if (this.map.values().next().value === this.index) { this.map.delete(this.map.keys().next().value); } this.map.set(hash2, this.index); if (hmac !== null) { this.hmacs.add(hmac); } if (this.index >= this.xorbIndices.length) { const oldXorbIndices = this.xorbIndices; const oldChunkIndices = this.chunkIndices; this.xorbIndices = new Int32Array(Math.min(this.xorbIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); this.chunkIndices = new Uint16Array(Math.min(this.chunkIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); this.xorbIndices.set(oldXorbIndices); this.chunkIndices.set(oldChunkIndices); } this.xorbIndices[this.index] = xorbIndex; this.chunkIndices[this.index] = chunkIndex; this.index = (this.index + 1) % this.maxSize; } getChunk(hash2, hmacFunction) { let index = this.map.get(hash2); if (index === void 0 && hmacFunction !== null) { for (const hmac of this.hmacs) { index = this.map.get(hmacFunction(hash2, hmac)); if (index !== void 0) { break; } } } if (index === void 0) { return void 0; } return { xorbIndex: this.xorbIndices[index], chunkIndex: this.chunkIndices[index] }; } updateChunkIndex(hash2, chunkIndex) { const index = this.map.get(hash2); if (index === void 0) { throw new Error(`Chunk not found in cache: ${hash2}`); } this.chunkIndices[index] = chunkIndex; } removeChunkFromCache(hash2) { this.map.delete(hash2); } }, _class3); // src/utils/xetWriteToken.ts var JWT_SAFETY_PERIOD2 = 6e4; var JWT_CACHE_SIZE2 = 1e3; var jwtPromises2 = /* @__PURE__ */ new Map(); var jwts2 = /* @__PURE__ */ new Map(); async function xetWriteToken(params) { if (params.xetParams.expiresAt && params.xetParams.casUrl && params.xetParams.accessToken && params.xetParams.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD2)) { return { accessToken: params.xetParams.accessToken, casUrl: params.xetParams.casUrl }; } const key = params.xetParams.refreshWriteTokenUrl; const jwt = jwts2.get(key); if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD2)) { return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; } const existingPromise = jwtPromises2.get(key); if (existingPromise) { return existingPromise; } const promise = (async () => { const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(params.xetParams.refreshWriteTokenUrl, { headers: { ...params.accessToken ? { Authorization: `Bearer ${params.accessToken}` } : {}, ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} } }); if (!resp.ok) { throw await createApiError(resp); } const json = await resp.json(); const jwt2 = { accessToken: json.accessToken, expiresAt: new Date(json.exp * 1e3), casUrl: json.casUrl }; jwtPromises2.delete(key); for (const [key2, value] of jwts2.entries()) { if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD2)) { jwts2.delete(key2); } else { break; } } if (jwts2.size >= JWT_CACHE_SIZE2) { const keyToDelete = jwts2.keys().next().value; if (keyToDelete) { jwts2.delete(keyToDelete); } } jwts2.set(key, jwt2); return { accessToken: json.accessToken, casUrl: json.casUrl }; })(); jwtPromises2.set(key, promise); return promise; } // src/utils/shardParser.ts var HASH_LENGTH = 32; var XORB_HASH_BOOKEND = "ff".repeat(HASH_LENGTH); function readHashFromArray(array, offset) { let hash2 = ""; for (let i = 0; i < HASH_LENGTH; i += 8) { hash2 += `${array[offset + i + 7].toString(16).padStart(2, "0")}${array[offset + i + 6].toString(16).padStart(2, "0")}${array[offset + i + 5].toString(16).padStart(2, "0")}${array[offset + i + 4].toString(16).padStart(2, "0")}${array[offset + i + 3].toString(16).padStart(2, "0")}${array[offset + i + 2].toString(16).padStart(2, "0")}${array[offset + i + 1].toString(16).padStart(2, "0")}${array[offset + i].toString(16).padStart(2, "0")}`; } return hash2; } async function parseShardData(shardBlob) { const shard = new Uint8Array(await shardBlob.arrayBuffer()); const shardView = new DataView(shard.buffer); const magicTag = shard.slice(0, SHARD_MAGIC_TAG.length); if (!magicTag.every((byte, i) => byte === SHARD_MAGIC_TAG[i])) { throw new Error("Invalid shard magic tag"); } const version = shardView.getBigUint64(SHARD_MAGIC_TAG.length, true); if (version !== SHARD_HEADER_VERSION) { throw new Error(`Invalid shard version: ${version}`); } const footerSize = Number(shardView.getBigUint64(SHARD_MAGIC_TAG.length + 8, true)); const footerStart = shard.length - footerSize; const footerVersion = shardView.getBigUint64(footerStart, true); if (footerVersion !== SHARD_FOOTER_VERSION) { throw new Error(`Invalid shard footer version: ${footerVersion}`); } const xorbInfoStart = Number(shardView.getBigUint64(footerStart + 16, true)); const fileLookupStart = Number(shardView.getBigUint64(footerStart + 24, true)); const hmacKey = readHashFromArray(shard, footerStart + 72); const xorbs = []; let offset = xorbInfoStart; while (offset < fileLookupStart) { const xorbHash = readHashFromArray(shard, offset); offset += HASH_LENGTH; if (xorbHash === XORB_HASH_BOOKEND) { break; } offset += 4; const chunkCount = shardView.getUint32(offset, true); offset += 4; offset += 4; offset += 4; const chunks = []; for (let i = 0; i < chunkCount; i++) { const chunkHash = readHashFromArray(shard, offset); offset += HASH_LENGTH; const startOffset = shardView.getUint32(offset, true); offset += 4; const length = shardView.getUint32(offset, true); offset += 4; offset += 8; chunks.push({ hash: chunkHash, startOffset, unpackedLength: length }); } xorbs.push({ hash: xorbHash, chunks }); } return { hmacKey, xorbs }; } // src/utils/sum.ts function sum(arr) { return arr.reduce((a, b) => a + b, 0); } // src/utils/SplicedBlob.ts var SplicedBlob = class extends Blob { constructor(originalBlob, spliceOperations) { super(); this.originalBlob = originalBlob; this.spliceOperations = spliceOperations; } static create(originalBlob, operations) { for (const op of operations) { if (op.start < 0 || op.end < 0) { throw new Error("Invalid start/end positions for SplicedBlob"); } if (op.start > originalBlob.size || op.end > originalBlob.size) { throw new Error("Invalid start/end positions for SplicedBlob"); } if (op.start > op.end) { throw new Error("Invalid start/end positions for SplicedBlob"); } } const sortedOps = [...operations].sort((a, b) => a.start - b.start); for (let i = 0; i < sortedOps.length - 1; i++) { if (sortedOps[i].end > sortedOps[i + 1].start) { throw new Error("Overlapping splice operations are not supported"); } } return new SplicedBlob(originalBlob, sortedOps); } /** * Returns the size of the spliced blob. * Size = original size - total replaced size + total insert size */ get size() { let totalReplacedSize = 0; let totalInsertSize = 0; for (const op of this.spliceOperations) { totalReplacedSize += op.end - op.start; totalInsertSize += op.insert.size; } return this.originalBlob.size - totalReplacedSize + totalInsertSize; } /** * Returns the MIME type of the original blob. */ get type() { return this.originalBlob.type; } /** * Returns a new instance of SplicedBlob that is a slice of the current one. * * The slice is inclusive of the start and exclusive of the end. * The slice method does not support negative start/end. * * @param start beginning of the slice * @param end end of the slice */ slice(start = 0, end = this.size) { if (start < 0 || end < 0) { throw new TypeError("Unsupported negative start/end on SplicedBlob.slice"); } start = Math.min(start, this.size); end = Math.min(end, this.size); if (start >= end) { return new Blob([]); } const segments = this.segments; const segmentBoundaries = [0]; let cumulativeSize = 0; for (const segment of segments) { cumulativeSize += segment.size; segmentBoundaries.push(cumulativeSize); } const resultSegments = []; for (let i = 0; i < segments.length; i++) { const segmentStart = segmentBoundaries[i]; const segmentEnd = segmentBoundaries[i + 1]; if (segmentEnd <= start) { continue; } if (segmentStart >= end) { break; } const sliceStart = Math.max(0, start - segmentStart); const sliceEnd = Math.min(segments[i].size, end - segmentStart); if (sliceStart < sliceEnd) { resultSegments.push(segments[i].slice(sliceStart, sliceEnd)); } } return new Blob(resultSegments); } get firstSpliceIndex() { return _nullishCoalesce(_optionalChain([this, 'access', _71 => _71.spliceOperations, 'access', _72 => _72[0], 'optionalAccess', _73 => _73.start]), () => ( Infinity)); } /** * Read the spliced blob content and returns it as an ArrayBuffer. */ async arrayBuffer() { const segments = this.segments; const buffers = await Promise.all(segments.map((segment) => segment.arrayBuffer())); const totalSize = sum(buffers.map((buffer) => buffer.byteLength)); const result = new Uint8Array(totalSize); let offset = 0; for (const buffer of buffers) { result.set(new Uint8Array(buffer), offset); offset += buffer.byteLength; } return result.buffer; } /** * Read the spliced blob content and returns it as a string. */ async text() { const buffer = await this.arrayBuffer(); return new TextDecoder().decode(buffer); } /** * Returns a stream around the spliced blob content. */ stream() { const readable = new ReadableStream({ start: async (controller) => { try { const segments = this.segments; for (const segment of segments) { const reader = segment.stream().getReader(); try { while (true) { const { done, value } = await reader.read(); if (done) { break; } controller.enqueue(value); } } finally { reader.releaseLock(); } } controller.close(); } catch (error) { controller.error(error); } } }); return readable; } /** * Get all segments that make up the spliced blob. * This includes original blob segments between splice operations and insert blobs. */ get segments() { const segments = []; let currentPosition = 0; const sortedOps = [...this.spliceOperations].sort((a, b) => a.start - b.start); for (const op of sortedOps) { if (currentPosition < op.start) { segments.push(this.originalBlob.slice(currentPosition, op.start)); } if (op.insert.size > 0) { segments.push(op.insert); } currentPosition = op.end; } if (currentPosition < this.originalBlob.size) { segments.push(this.originalBlob.slice(currentPosition)); } return segments; } }; // src/utils/createXorbs.ts var TARGET_CHUNK_SIZE = 64 * 1024; var MAX_CHUNK_SIZE = 2 * TARGET_CHUNK_SIZE; var XORB_SIZE = 64 * 1024 * 1024; var MAX_XORB_CHUNKS = 8 * 1024; var INTERVAL_BETWEEN_REMOTE_DEDUP = 4e6; var PROCESSING_PROGRESS_RATIO = 0.1; var UPLOADING_PROGRESS_RATIO = 1 - PROCESSING_PROGRESS_RATIO; var CurrentXorbInfo = class { constructor() { this.id = 0; this.offset = 0; this.chunks = []; this.fileProcessedBytes = {}; this.fileUploadedBytes = {}; this.fileSize = {}; this.data = new Uint8Array(XORB_SIZE); this.immutableData = null; } event(computeXorbHash) { const xorbChunksCleaned = this.chunks.map((chunk2) => ({ hash: chunk2.hash, length: chunk2.length })); return { event: "xorb", xorb: this.data.subarray(0, this.offset), hash: computeXorbHash(xorbChunksCleaned), chunks: xorbChunksCleaned, id: this.id, files: Object.entries(this.fileProcessedBytes).map(([path, processedBytes]) => ({ path, progress: processedBytes / this.fileSize[path], lastSentProgress: ((_nullishCoalesce(this.fileUploadedBytes[path], () => ( 0))) + (processedBytes - (_nullishCoalesce(this.fileUploadedBytes[path], () => ( 0)))) * PROCESSING_PROGRESS_RATIO) / this.fileSize[path] })) }; } }; async function* createXorbs(fileSources, params) { const alreadyDoneFileSha256s = /* @__PURE__ */ new Set(); const chunkModule = await Promise.resolve().then(() => _interopRequireWildcard(require("./chunker_wasm-FWDBRYQI.js"))); let xorbId = 0; await chunkModule.init(); const chunkCache = new ChunkCache(); let xorb = new CurrentXorbInfo(); const nextXorb = (currentFile) => { const event = xorb.event(chunkModule.compute_xorb_hash.bind(chunkModule)); xorbId++; xorb = new CurrentXorbInfo(); xorb.id = xorbId; xorb.fileUploadedBytes = { [currentFile.path]: currentFile.uploadedBytes }; xorb.fileSize[currentFile.path] = currentFile.size; return event; }; const pendingFileEvents = []; const remoteXorbHashes = [""]; for await (const fileSource of fileSources) { _optionalChain([params, 'access', _74 => _74.yieldCallback, 'optionalCall', _75 => _75({ event: "fileProgress", path: fileSource.path, progress: 0 })]); if (fileSource.sha256 && alreadyDoneFileSha256s.has(fileSource.sha256)) { _optionalChain([params, 'access', _76 => _76.yieldCallback, 'optionalCall', _77 => _77({ event: "fileProgress", path: fileSource.path, progress: 1 })]); continue; } if (fileSource.sha256) { alreadyDoneFileSha256s.add(fileSource.sha256); } const chunker = new chunkModule.Chunker(TARGET_CHUNK_SIZE); try { xorb.fileSize[fileSource.path] = fileSource.content.size; if (fileSource.content instanceof SplicedBlob && fileSource.content.firstSpliceIndex < MAX_CHUNK_SIZE) { await loadDedupInfoToCache( fileSource.content.originalBlob.slice(0, MAX_CHUNK_SIZE), remoteXorbHashes, params, chunkCache, chunkModule, { maxChunks: 1, isAtBeginning: true } ); } let bytesSinceRemoteDedup = Infinity; let bytesSinceLastProgressEvent = 0; let isFirstFileChunk = true; const sourceChunks = []; const reader = fileSource.content.stream().getReader(); let processedBytes = 0; let dedupedBytes = 0; const fileChunks = []; const chunkMetadata = []; const addChunks = async function* (chunks) { for (const chunk2 of chunks) { if (isFirstFileChunk) { chunk2.dedup = true; isFirstFileChunk = false; } let chunkIndex = xorb.chunks.length; let chunkXorbId = xorbId; const chunkToCopy = removeChunkFromSourceData(sourceChunks, chunk2.length); let cacheData = chunkCache.getChunk(chunk2.hash, chunkModule.compute_hmac); if (cacheData === void 0 && chunk2.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { const token = await xetWriteToken(params); bytesSinceRemoteDedup = 0; const shardResp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(token.casUrl + "/v1/chunks/default/" + chunk2.hash, { headers: { Authorization: `Bearer ${token.accessToken}` } }); if (shardResp.ok) { const shard = await shardResp.blob(); const shardData = await parseShardData(shard); for (const xorb2 of shardData.xorbs) { const remoteXorbId = -remoteXorbHashes.length; remoteXorbHashes.push(xorb2.hash); let i = 0; for (const chunk3 of xorb2.chunks) { chunkCache.addChunkToCache(chunk3.hash, remoteXorbId, i++, shardData.hmacKey); } } cacheData = chunkCache.getChunk(chunk2.hash, chunkModule.compute_hmac); const oldDedupedBytes = dedupedBytes; dedupedBytes = backtrackDedup( xorb, chunkModule.compute_hmac.bind(chunkModule), shardData, chunkCache, chunkMetadata, dedupedBytes ); if (dedupedBytes > oldDedupedBytes) { xorb.fileUploadedBytes[fileSource.path] ??= 0; xorb.fileUploadedBytes[fileSource.path] += dedupedBytes - oldDedupedBytes; } } } if (cacheData === void 0) { if (!writeChunk(xorb, chunkToCopy, chunk2.hash)) { yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); chunkIndex = 0; chunkXorbId = xorbId; for (const event of pendingFileEvents) { event.representation = event.representation.map((rep) => ({ ...rep, xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] })); yield event; } pendingFileEvents.length = 0; if (!writeChunk(xorb, chunkToCopy, chunk2.hash)) { throw new Error("Failed to write chunk into xorb"); } } chunkCache.addChunkToCache(chunk2.hash, xorbId, chunkIndex, null); } else { chunkXorbId = cacheData.xorbIndex; chunkIndex = cacheData.chunkIndex; dedupedBytes += chunk2.length; xorb.fileUploadedBytes[fileSource.path] ??= 0; xorb.fileUploadedBytes[fileSource.path] += chunk2.length; } bytesSinceRemoteDedup += chunk2.length; bytesSinceLastProgressEvent += chunk2.length; fileChunks.push({ hash: chunk2.hash, length: chunk2.length }); chunkMetadata.push({ xorbId: chunkXorbId, chunkIndex, length: chunk2.length }); xorb.fileProcessedBytes[fileSource.path] = processedBytes; if (bytesSinceLastProgressEvent >= 1e6) { bytesSinceLastProgressEvent = 0; _optionalChain([params, 'access', _78 => _78.yieldCallback, 'optionalCall', _79 => _79({ event: "fileProgress", path: fileSource.path, progress: ((_nullishCoalesce(xorb.fileUploadedBytes[fileSource.path], () => ( 0))) + (xorb.fileProcessedBytes[fileSource.path] - (_nullishCoalesce(xorb.fileUploadedBytes[fileSource.path], () => ( 0)))) * PROCESSING_PROGRESS_RATIO) / fileSource.content.size })]); } if (xorb.chunks.length >= MAX_XORB_CHUNKS) { yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); for (const event of pendingFileEvents) { event.representation = event.representation.map((rep) => ({ ...rep, xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] })); yield event; } pendingFileEvents.length = 0; } } }; while (true) { const { done, value } = await reader.read(); if (done) { yield* addChunks(chunker.finish()); break; } processedBytes += value.length; sourceChunks.push(value); yield* addChunks(chunker.add_data(value)); } const fileRepresentation = buildFileRepresentation( chunkMetadata, fileChunks, chunkModule.compute_verification_hash.bind(chunkModule) ); xorb.immutableData = { chunkIndex: xorb.chunks.length, offset: xorb.offset }; const dedupRatio = fileSource.content.size > 0 ? dedupedBytes / fileSource.content.size : 0; pendingFileEvents.push({ event: "file", path: fileSource.path, hash: chunkModule.compute_file_hash(fileChunks), sha256: fileSource.sha256, dedupRatio, representation: fileRepresentation }); } finally { chunker.free(); } } if (xorb.offset > 0) { yield xorb.event(chunkModule.compute_xorb_hash.bind(chunkModule)); } for (const event of pendingFileEvents) { event.representation = event.representation.map((rep) => ({ ...rep, xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] })); yield event; } } function backtrackDedup(xorb, computeHmac, shardData, chunkCache, chunkMetadata, dedupedBytes) { const chunkIndexesToBacktrackFor = /* @__PURE__ */ new Map(); for (let chunkToRecheckIndex = _nullishCoalesce(_optionalChain([xorb, 'access', _80 => _80.immutableData, 'optionalAccess', _81 => _81.chunkIndex]), () => ( 0)); chunkToRecheckIndex < xorb.chunks.length; chunkToRecheckIndex++) { const chunk2 = xorb.chunks[chunkToRecheckIndex]; const hmacHash = computeHmac(chunk2.hash, shardData.hmacKey); const cacheData = chunkCache.getChunk(hmacHash, null); if (cacheData !== void 0) { chunkIndexesToBacktrackFor.set(chunkToRecheckIndex, { xorbId: cacheData.xorbIndex, chunkIndex: cacheData.chunkIndex }); chunkCache.removeChunkFromCache(chunk2.hash); } } for (const metadata of chunkMetadata) { if (metadata.xorbId === xorb.id && chunkIndexesToBacktrackFor.has(metadata.chunkIndex)) { const backtrackData = chunkIndexesToBacktrackFor.get(metadata.chunkIndex); if (backtrackData !== void 0) { metadata.xorbId = backtrackData.xorbId; metadata.chunkIndex = backtrackData.chunkIndex; dedupedBytes += metadata.length; } } } const xorbRangesToErase = []; for (let i = 0; i < xorb.chunks.length; i++) { const chunk2 = xorb.chunks[i]; if (chunkIndexesToBacktrackFor.has(i)) { xorbRangesToErase.push({ start: chunk2.offset, end: i < xorb.chunks.length - 1 ? xorb.chunks[i + 1].offset : xorb.offset }); } } const xorbRangesToKeep = []; let currentStart = 0; for (let i = 0; i < xorbRangesToErase.length; i++) { const range2 = xorbRangesToErase[i]; if (currentStart !== range2.start) { xorbRangesToKeep.push({ start: currentStart, end: range2.start }); } currentStart = range2.end; } if (currentStart !== xorb.offset) { xorbRangesToKeep.push({ start: currentStart, end: xorb.offset }); } let currentOffset = 0; for (const range2 of xorbRangesToKeep) { if (range2.start !== currentOffset) { xorb.data.set(xorb.data.subarray(range2.start, range2.end), currentOffset); } currentOffset += range2.end - range2.start; } const newXorbChunks = []; const oldIndexToNewIndex = /* @__PURE__ */ new Map(); let erasedOffset = 0; for (let i = 0; i < xorb.chunks.length; i++) { const chunk2 = xorb.chunks[i]; if (chunkIndexesToBacktrackFor.has(i)) { if (i < xorb.chunks.length - 1) { erasedOffset += xorb.chunks[i + 1].offset - chunk2.offset; } } else { newXorbChunks.push({ hash: chunk2.hash, length: chunk2.length, offset: chunk2.offset - erasedOffset }); if (erasedOffset > 0) { oldIndexToNewIndex.set(i, newXorbChunks.length - 1); } } } xorb.chunks = newXorbChunks; xorb.offset = currentOffset; for (const chunk2 of chunkMetadata) { if (chunk2.xorbId === xorb.id) { const newIndex = oldIndexToNewIndex.get(chunk2.chunkIndex); if (newIndex !== void 0) { const cached = chunkCache.getChunk(xorb.chunks[newIndex].hash, null); if (cached !== void 0 && cached.xorbIndex === chunk2.xorbId && cached.chunkIndex === chunk2.chunkIndex) { chunkCache.updateChunkIndex(xorb.chunks[newIndex].hash, newIndex); } chunk2.chunkIndex = newIndex; } } } return dedupedBytes; } function removeChunkFromSourceData(sourceChunks, chunkLength) { if (chunkLength === sourceChunks[0].length) { const chunkToCopy = sourceChunks[0]; sourceChunks.shift(); return chunkToCopy; } else if (chunkLength < sourceChunks[0].length) { const chunkToCopy = sourceChunks[0].subarray(0, chunkLength); sourceChunks[0] = sourceChunks[0].subarray(chunkLength); return chunkToCopy; } else { const chunkToCopy = new Uint8Array(chunkLength); let copyOffset = 0; let index = 0; let toSlice = -1; while (copyOffset < chunkLength) { const nToCopy = Math.min(sourceChunks[index].length, chunkLength - copyOffset); chunkToCopy.set(sourceChunks[index].subarray(0, nToCopy), copyOffset); copyOffset += nToCopy; if (nToCopy === sourceChunks[index].length) { index++; } else { toSlice = nToCopy; } } sourceChunks.splice(0, index); if (toSlice !== -1) { sourceChunks[0] = sourceChunks[0].subarray(toSlice); } return chunkToCopy; } } function writeChunk(xorb, chunk2, hash2) { const regularCompressedChunk = compress(chunk2); const bgCompressedChunk = compress(bg4_split_bytes(chunk2)); const compressedChunk = bgCompressedChunk.length < regularCompressedChunk.length ? bgCompressedChunk : regularCompressedChunk; const chunkToWrite = compressedChunk.length < chunk2.length ? compressedChunk : chunk2; if (xorb.offset + XET_CHUNK_HEADER_BYTES + chunkToWrite.length > XORB_SIZE) { return false; } xorb.data[xorb.offset] = 0; xorb.data[xorb.offset + 1] = chunkToWrite.length & 255; xorb.data[xorb.offset + 2] = chunkToWrite.length >> 8 & 255; xorb.data[xorb.offset + 3] = chunkToWrite.length >> 16 & 255; xorb.data[xorb.offset + 4] = chunkToWrite.length < chunk2.length ? bgCompressedChunk.length < regularCompressedChunk.length ? 2 /* ByteGroupingLZ4 */ : 1 /* LZ4 */ : 0 /* None */; xorb.data[xorb.offset + 5] = chunk2.length & 255; xorb.data[xorb.offset + 6] = chunk2.length >> 8 & 255; xorb.data[xorb.offset + 7] = chunk2.length >> 16 & 255; xorb.data.set(chunkToWrite, xorb.offset + XET_CHUNK_HEADER_BYTES); xorb.chunks.push({ hash: hash2, length: chunk2.length, offset: xorb.offset }); xorb.offset += XET_CHUNK_HEADER_BYTES + chunkToWrite.length; return true; } var buildFileRepresentation = (metadata, chunks, computeVerificationHash) => { if (metadata.length === 0) { return []; } const representation = []; let currentRange = { xorbId: metadata[0].xorbId, indexStart: metadata[0].chunkIndex, indexEnd: metadata[0].chunkIndex + 1, length: metadata[0].length, chunkHashStart: 0 }; for (let i = 1; i < metadata.length; i++) { const chunk2 = metadata[i]; if (currentRange.xorbId === chunk2.xorbId && currentRange.indexEnd === chunk2.chunkIndex) { currentRange.indexEnd = chunk2.chunkIndex + 1; currentRange.length += chunk2.length; } else { const rangeHash2 = computeVerificationHash(chunks.slice(currentRange.chunkHashStart, i).map((x) => x.hash)); representation.push({ xorbId: currentRange.xorbId, indexStart: currentRange.indexStart, indexEnd: currentRange.indexEnd, length: currentRange.length, rangeHash: rangeHash2 }); currentRange = { xorbId: chunk2.xorbId, indexStart: chunk2.chunkIndex, indexEnd: chunk2.chunkIndex + 1, length: chunk2.length, chunkHashStart: i }; } } const rangeHash = computeVerificationHash(chunks.slice(currentRange.chunkHashStart).map((x) => x.hash)); representation.push({ xorbId: currentRange.xorbId, indexStart: currentRange.indexStart, indexEnd: currentRange.indexEnd, length: currentRange.length, rangeHash }); return representation; }; async function loadDedupInfoToCache(content, remoteXorbHashes, params, chunkCache, chunkModule, opts) { const chunker = new chunkModule.Chunker(TARGET_CHUNK_SIZE); const cache = chunkCache; let dedupedBytes = 0; let chunksProcessed = 0; let totalBytes = 0; let bytesSinceRemoteDedup = Infinity; const sourceChunks = []; try { const reader = content.stream().getReader(); const processChunks = async (chunkData) => { for (const chunk2 of chunkData) { chunksProcessed++; if (_optionalChain([opts, 'optionalAccess', _82 => _82.isAtBeginning]) && chunksProcessed === 1) { chunk2.dedup = true; } totalBytes += chunk2.length; removeChunkFromSourceData(sourceChunks, chunk2.length); let cacheData = cache.getChunk(chunk2.hash, chunkModule.compute_hmac); if (cacheData !== void 0) { dedupedBytes += chunk2.length; bytesSinceRemoteDedup += chunk2.length; continue; } if (chunk2.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { const token = await xetWriteToken(params); bytesSinceRemoteDedup = 0; const shardResp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(token.casUrl + "/v1/chunks/default/" + chunk2.hash, { headers: { Authorization: `Bearer ${token.accessToken}` } }); if (shardResp.ok) { const shard = await shardResp.blob(); const shardData = await parseShardData(shard); for (const xorb of shardData.xorbs) { const remoteXorbId = -remoteXorbHashes.length; remoteXorbHashes.push(xorb.hash); let i = 0; for (const xorbChunk of xorb.chunks) { cache.addChunkToCache(xorbChunk.hash, remoteXorbId, i++, shardData.hmacKey); } } cacheData = cache.getChunk(chunk2.hash, chunkModule.compute_hmac); } } if (cacheData !== void 0) { dedupedBytes += chunk2.length; } bytesSinceRemoteDedup += chunk2.length; } }; while (true) { if (_optionalChain([opts, 'optionalAccess', _83 => _83.end]) !== void 0 && totalBytes >= opts.end) { break; } if (_optionalChain([opts, 'optionalAccess', _84 => _84.maxChunks]) !== void 0 && chunksProcessed >= opts.maxChunks) { break; } const { done, value } = await reader.read(); if (done) { await processChunks(chunker.finish()); break; } sourceChunks.push(value); await processChunks(chunker.add_data(value)); } } finally { chunker.free(); } } // src/utils/uploadShards.ts var SHARD_MAX_SIZE = 64 * 1024 * 1024; var SHARD_HEADER_SIZE = 48; var SHARD_FOOTER_SIZE = 200; var HASH_LENGTH2 = 32; var XORB_FOOTER_LENGTH = 48; var FILE_FOOTER_LENGTH = 48; var SHARD_HEADER_VERSION = 2n; var SHARD_FOOTER_VERSION = 1n; var MDB_FILE_FLAG_WITH_VERIFICATION = 2147483648; var MDB_FILE_FLAG_WITH_METADATA_EXT = 1073741824; var EMPTY_HEX_HASH = "0".repeat(HASH_LENGTH2 * 2); var SHARD_MAGIC_TAG = new Uint8Array([ "H".charCodeAt(0), "F".charCodeAt(0), "R".charCodeAt(0), "e".charCodeAt(0), "p".charCodeAt(0), "o".charCodeAt(0), "M".charCodeAt(0), "e".charCodeAt(0), "t".charCodeAt(0), "a".charCodeAt(0), "D".charCodeAt(0), "a".charCodeAt(0), "t".charCodeAt(0), "a".charCodeAt(0), 0, 85, 105, 103, 69, 106, 123, 129, 87, 131, 165, 189, 217, 92, 205, 209, 74, 169 ]); async function* uploadShards(source, params) { const xorbHashes = []; const fileInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.25); const xorbInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.75); const xorbView = new DataView(xorbInfoSection.buffer); let xorbViewOffset = 0; const fileInfoView = new DataView(fileInfoSection.buffer); let fileViewOffset = 0; let xorbTotalSize = 0n; let fileTotalSize = 0n; let xorbTotalUnpackedSize = 0n; for await (const output of createXorbs(source, params)) { switch (output.event) { case "xorb": { xorbHashes.push(output.hash); const xorbEntrySize = HASH_LENGTH2 + 4 + 4 + 4 + 4; const chunksSize = output.chunks.length * (HASH_LENGTH2 + 4 + 4 + 8); const totalXorbSize = xorbEntrySize + chunksSize; if (xorbViewOffset + totalXorbSize > xorbInfoSection.length) { if (xorbViewOffset > 0 || fileViewOffset > 0) { await uploadShard(createShard(), params); } } writeHashToArray(output.hash, xorbInfoSection, xorbViewOffset); xorbViewOffset += HASH_LENGTH2; xorbView.setUint32(xorbViewOffset, 0, true); xorbViewOffset += 4; xorbView.setUint32(xorbViewOffset, output.chunks.length, true); xorbViewOffset += 4; const xorbUnpackedSize = sum(output.chunks.map((x) => x.length)); xorbView.setUint32(xorbViewOffset, xorbUnpackedSize, true); xorbTotalUnpackedSize += BigInt(xorbUnpackedSize); xorbTotalSize += BigInt(output.xorb.byteLength); xorbViewOffset += 4; xorbView.setUint32(xorbViewOffset, output.xorb.byteLength, true); xorbViewOffset += 4; let chunkBytes = 0; for (const chunk2 of output.chunks) { writeHashToArray(chunk2.hash, xorbInfoSection, xorbViewOffset); xorbViewOffset += HASH_LENGTH2; xorbView.setUint32(xorbViewOffset, chunkBytes, true); xorbViewOffset += 4; xorbView.setUint32(xorbViewOffset, chunk2.length, true); xorbViewOffset += 4; xorbView.setBigUint64(xorbViewOffset, 0n, true); xorbViewOffset += 8; chunkBytes += chunk2.length; } for (const file of output.files) { yield { event: "fileProgress", path: file.path, progress: file.lastSentProgress }; } await uploadXorb(output, params); for (const file of output.files) { yield { event: "fileProgress", path: file.path, progress: file.progress }; } break; } case "file": { yield { event: "file", path: output.path, xetHash: output.hash, sha256: output.sha256, dedupRatio: output.dedupRatio }; const fileHeaderSize = HASH_LENGTH2 + 4 + 4 + 8; const representationSize = output.representation.length * (HASH_LENGTH2 + 4 + 4 + 4 + 4); const verificationSize = output.representation.length * (HASH_LENGTH2 + 16); const metadataSize = HASH_LENGTH2 + 16; const totalFileSize = fileHeaderSize + representationSize + verificationSize + metadataSize; if (fileViewOffset + totalFileSize > fileInfoSection.length) { if (xorbViewOffset > 0 || fileViewOffset > 0) { await uploadShard(createShard(), params); } } writeHashToArray(output.hash, fileInfoSection, fileViewOffset); fileViewOffset += HASH_LENGTH2; fileInfoView.setUint32(fileViewOffset, MDB_FILE_FLAG_WITH_METADATA_EXT + MDB_FILE_FLAG_WITH_VERIFICATION, true); fileViewOffset += 4; fileInfoView.setUint32(fileViewOffset, output.representation.length, true); fileViewOffset += 4; fileInfoView.setBigUint64(fileViewOffset, 0n, true); fileViewOffset += 8; for (const repItem of output.representation) { writeHashToArray( typeof repItem.xorbId === "number" ? xorbHashes[repItem.xorbId] : repItem.xorbId, fileInfoSection, fileViewOffset ); fileViewOffset += HASH_LENGTH2; fileInfoView.setUint32(fileViewOffset, 0, true); fileViewOffset += 4; fileInfoView.setUint32(fileViewOffset, repItem.length, true); fileViewOffset += 4; fileInfoView.setUint32(fileViewOffset, repItem.indexStart, true); fileViewOffset += 4; fileInfoView.setUint32(fileViewOffset, repItem.indexEnd, true); fileViewOffset += 4; } for (const repItem of output.representation) { writeHashToArray(repItem.rangeHash, fileInfoSection, fileViewOffset); fileViewOffset += HASH_LENGTH2; for (let i = 0; i < 16; i++) { fileInfoSection[fileViewOffset + i] = 0; } fileViewOffset += 16; } writeHashToArray(_nullishCoalesce(output.sha256, () => ( EMPTY_HEX_HASH)), fileInfoSection, fileViewOffset); fileViewOffset += HASH_LENGTH2; for (let i = 0; i < 16; i++) { fileInfoSection[fileViewOffset + i] = 0; } fileViewOffset += 16; break; } } } function createShard() { const shard = new Uint8Array( SHARD_HEADER_SIZE + SHARD_FOOTER_SIZE + xorbViewOffset + XORB_FOOTER_LENGTH + fileViewOffset + FILE_FOOTER_LENGTH ); const shardView = new DataView(shard.buffer); let shardOffset = 0; shard.set(SHARD_MAGIC_TAG, shardOffset); shardOffset += SHARD_MAGIC_TAG.length; shardView.setBigUint64(shardOffset, SHARD_HEADER_VERSION, true); shardOffset += 8; shardView.setBigUint64(shardOffset, BigInt(SHARD_FOOTER_SIZE), true); shardOffset += 8; shard.set(fileInfoSection.slice(0, fileViewOffset), shardOffset); shardOffset += fileViewOffset; for (let i = 0; i < 32; i++) { shard[shardOffset + i] = 255; } shardOffset += 32; for (let i = 0; i < 16; i++) { shard[shardOffset + i] = 0; } shardOffset += 16; const xorbInfoOffset = shardOffset; shard.set(xorbInfoSection.slice(0, xorbViewOffset), shardOffset); shardOffset += xorbViewOffset; for (let i = 0; i < 32; i++) { shard[shardOffset + i] = 255; } shardOffset += 32; for (let i = 0; i < 16; i++) { shard[shardOffset + i] = 0; } shardOffset += 16; const footerOffset = shardOffset; shardView.setBigUint64(shardOffset, SHARD_FOOTER_VERSION, true); shardOffset += 8; shardView.setBigUint64(shardOffset, BigInt(SHARD_HEADER_SIZE), true); shardOffset += 8; shardView.setBigUint64(shardOffset, BigInt(xorbInfoOffset), true); shardOffset += 8; for (let i = 0; i < 48; i++) { shardView.setUint8(shardOffset + i, 0); } shardOffset += 48; for (let i = 0; i < 32; i++) { shardView.setUint8(shardOffset + i, 0); } shardOffset += 32; shardView.setBigUint64(shardOffset, BigInt(Math.floor(Date.now() / 1e3)), true); shardOffset += 8; shardView.setBigUint64(shardOffset, 0n, true); shardOffset += 8; for (let i = 0; i < 48; i++) { shardView.setUint8(shardOffset + i, 0); } shardOffset += 48; shardView.setBigUint64(shardOffset, xorbTotalSize, true); shardOffset += 8; shardView.setBigUint64(shardOffset, fileTotalSize, true); shardOffset += 8; shardView.setBigUint64(shardOffset, xorbTotalUnpackedSize, true); shardOffset += 8; shardView.setBigUint64(shardOffset, BigInt(footerOffset), true); xorbViewOffset = 0; fileViewOffset = 0; xorbTotalSize = 0n; xorbTotalUnpackedSize = 0n; fileTotalSize = 0n; return shard; } if (xorbViewOffset || fileViewOffset) { await uploadShard(createShard(), params); } } function writeHashToArray(hash2, array, offset) { for (let i = 0; i < hash2.length; i += 16) { array[offset + i / 2] = parseInt(hash2.substring(i + 2 * 7, i + 2 * 8), 16); array[offset + i / 2 + 1] = parseInt(hash2.substring(i + 2 * 6, i + 2 * 7), 16); array[offset + i / 2 + 2] = parseInt(hash2.substring(i + 2 * 5, i + 2 * 6), 16); array[offset + i / 2 + 3] = parseInt(hash2.substring(i + 2 * 4, i + 2 * 5), 16); array[offset + i / 2 + 4] = parseInt(hash2.substring(i + 2 * 3, i + 2 * 4), 16); array[offset + i / 2 + 5] = parseInt(hash2.substring(i + 2 * 2, i + 2 * 3), 16); array[offset + i / 2 + 6] = parseInt(hash2.substring(i + 2 * 1, i + 2 * 2), 16); array[offset + i / 2 + 7] = parseInt(hash2.substring(i + 2 * 0, i + 2 * 1), 16); } } async function uploadXorb(xorb, params) { const token = await xetWriteToken(params); const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${token.casUrl}/v1/xorbs/default/${xorb.hash}`, { method: "POST", body: xorb.xorb, headers: { Authorization: `Bearer ${token.accessToken}`, ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} }, ...{ progressHint: { progressCallback: (progress) => { for (const file of xorb.files) { _optionalChain([params, 'access', _85 => _85.yieldCallback, 'optionalCall', _86 => _86({ event: "fileProgress", path: file.path, progress: file.lastSentProgress + (file.progress - file.lastSentProgress) * progress })]); } } } } }); if (!resp.ok) { throw await createApiError(resp); } } async function uploadShard(shard, params) { const token = await xetWriteToken(params); const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${token.casUrl}/v1/shards`, { method: "POST", body: shard, headers: { Authorization: `Bearer ${token.accessToken}`, ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} } }); if (!resp.ok) { throw await createApiError(resp); } } // src/utils/splitAsyncGenerator.ts function splitAsyncGenerator(source, n) { if (n <= 0) { return []; } const sleep = (ms) => new Promise((resolve2) => setTimeout(resolve2, ms)); let takenIndex = null; const generators = []; let remaining = n; for (let i = 0; i < n; i++) { generators.push({ next: async () => { while (takenIndex !== null) { await sleep(1); } takenIndex = i; return source.next().then((r) => { takenIndex = null; return r; }); }, return: async () => { remaining--; if (remaining === 0) { return source.return(void 0); } return { done: true, value: void 0 }; }, throw: async (error) => { return source.throw(error); }, [Symbol.asyncIterator]: () => generators[i] }); } return generators; } // src/lib/commit.ts var CONCURRENT_SHAS = 5; var CONCURRENT_LFS_UPLOADS = 5; var MULTIPART_PARALLEL_UPLOAD = 5; function isFileOperation(op) { const ret = op.operation === "addOrUpdate"; if (ret && !(op.content instanceof Blob)) { throw new TypeError("Precondition failed: op.content should be a Blob"); } return ret; } async function* commitIter(params) { const accessToken = checkCredentials(params); const repoId = toRepoId(params.repo); if (repoId.type === "bucket") { return yield* commitIterBucket(params); } yield { event: "phase", phase: "preuploading" }; let useXet = _nullishCoalesce(params.useXet, () => ( true)); const lfsShas = /* @__PURE__ */ new Map(); const abortController = new AbortController(); const abortSignal = abortController.signal; if (!abortSignal.throwIfAborted) { abortSignal.throwIfAborted = () => { if (abortSignal.aborted) { throw new DOMException("Aborted", "AbortError"); } }; } if (params.abortSignal) { params.abortSignal.addEventListener("abort", () => abortController.abort()); } try { const allOperations = (await Promise.all( params.operations.map(async (operation) => { if (operation.operation === "edit") { const splicedBlob = SplicedBlob.create( operation.originalContent, operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })) ); return { operation: "addOrUpdate", path: operation.path, content: splicedBlob }; } if (operation.operation !== "addOrUpdate") { return operation; } if (!(operation.content instanceof URL)) { return { ...operation, content: operation.content }; } const lazyBlobs = await createBlobs(operation.content, operation.path, { fetch: params.fetch, maxFolderDepth: params.maxFolderDepth }); _optionalChain([abortSignal, 'optionalAccess', _87 => _87.throwIfAborted, 'call', _88 => _88()]); return lazyBlobs.map((blob) => ({ ...operation, content: blob.blob, path: blob.path })); }) )).flat(1); const gitAttributes = _optionalChain([allOperations, 'access', _89 => _89.filter, 'call', _90 => _90(isFileOperation), 'access', _91 => _91.find, 'call', _92 => _92((op) => op.path === ".gitattributes"), 'optionalAccess', _93 => _93.content]); for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { const payload = { gitAttributes: gitAttributes && await gitAttributes.text(), files: await Promise.all( operations.map(async (operation) => ({ path: operation.path, size: operation.content.size, sample: base64FromBytes(new Uint8Array(await operation.content.slice(0, 512).arrayBuffer())) })) ) }; _optionalChain([abortSignal, 'optionalAccess', _94 => _94.throwIfAborted, 'call', _95 => _95()]); const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/preupload/${encodeURIComponent( _nullishCoalesce(params.branch, () => ( "main")) )}` + (params.isPullRequest ? "?create_pr=1" : ""), { method: "POST", headers: { ...accessToken && { Authorization: `Bearer ${accessToken}` }, "Content-Type": "application/json" }, body: JSON.stringify(payload), signal: abortSignal } ); if (!res.ok) { throw await createApiError(res); } const json = await res.json(); for (const file of json.files) { if (file.uploadMode === "lfs") { lfsShas.set(file.path, null); } } } yield { event: "phase", phase: "uploadingLargeFiles" }; for (const operations of chunk( allOperations.filter(isFileOperation).filter((op) => lfsShas.has(op.path)), 100 )) { const shas = yield* eventToGenerator((yieldCallback, returnCallback, rejectCallack) => { return promisesQueue( operations.map((op) => async () => { const iterator = sha256(op.content, { useWebWorker: params.useWebWorkers, abortSignal }); let res2; do { res2 = await iterator.next(); if (!res2.done) { yieldCallback({ event: "fileProgress", path: op.path, progress: res2.value, state: "hashing" }); } } while (!res2.done); const sha = res2.value; lfsShas.set(op.path, res2.value); return sha; }), CONCURRENT_SHAS ).then(returnCallback, rejectCallack); }); _optionalChain([abortSignal, 'optionalAccess', _96 => _96.throwIfAborted, 'call', _97 => _97()]); const payload = { operation: "upload", // multipart is a custom protocol for HF transfers: ["basic", "multipart", ...useXet ? ["xet"] : []], hash_algo: "sha_256", ...!params.isPullRequest && { ref: { name: _nullishCoalesce(params.branch, () => ( "main")) } }, objects: operations.map((op, i) => ({ oid: shas[i], size: op.content.size })) }; const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/${repoId.type === "model" ? "" : repoId.type + "s/"}${repoId.name}.git/info/lfs/objects/batch`, { method: "POST", headers: { ...accessToken && { Authorization: `Bearer ${accessToken}` }, Accept: "application/vnd.git-lfs+json", "Content-Type": "application/vnd.git-lfs+json" }, body: JSON.stringify(payload), signal: abortSignal } ); if (!res.ok) { throw await createApiError(res); } const json = await res.json(); const batchRequestId = res.headers.get("X-Request-Id") || void 0; const shaToOperation = new Map(operations.map((op, i) => [shas[i], op])); if (useXet && json.transfer !== "xet") { useXet = false; } let xetParams = null; if (useXet) { for (const obj of json.objects) { const op = shaToOperation.get(obj.oid); if (!op) { throw new InvalidApiResponseFormatError("Unrequested object ID in response"); } if (obj.error) { const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${obj.error.message}${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); } if (!_optionalChain([obj, 'access', _98 => _98.actions, 'optionalAccess', _99 => _99.upload])) { yield { event: "fileProgress", path: op.path, progress: 1, state: "uploading" }; } else { const headers = new Headers(obj.actions.upload.header); xetParams = { sessionId: _nullishCoalesce(headers.get("X-Xet-Session-Id"), () => ( void 0)), casUrl: _nullishCoalesce(headers.get("X-Xet-Cas-Url"), () => ( void 0)), accessToken: _nullishCoalesce(headers.get("X-Xet-Access-Token"), () => ( void 0)), expiresAt: headers.get("X-Xet-Token-Expiration") ? new Date(parseInt(_nullishCoalesce(headers.get("X-Xet-Token-Expiration"), () => ( "0"))) * 1e3) : void 0, refreshWriteTokenUrl: obj.actions.upload.href }; } } const source = async function* () { for (const obj of json.objects) { const op = shaToOperation.get(obj.oid); if (!op || !_optionalChain([obj, 'access', _100 => _100.actions, 'optionalAccess', _101 => _101.upload])) { continue; } _optionalChain([abortSignal, 'optionalAccess', _102 => _102.throwIfAborted, 'call', _103 => _103()]); yield { content: op.content, path: op.path, sha256: obj.oid }; } }(); if (xetParams) { const fixedXetParams = xetParams; const sources = splitAsyncGenerator(source, 5); yield* eventToGenerator( (yieldCallback, returnCallback, rejectCallback) => Promise.all( sources.map(async function(source2) { for await (const event of uploadShards(source2, { fetch: params.fetch, accessToken, hubUrl: _nullishCoalesce(params.hubUrl, () => ( HUB_URL)), repo: repoId, xetParams: fixedXetParams, // todo: maybe leave empty if PR? rev: _nullishCoalesce(params.branch, () => ( "main")), isPullRequest: params.isPullRequest, yieldCallback: (event2) => yieldCallback({ ...event2, state: "uploading" }) })) { if (event.event === "file") { yieldCallback({ event: "fileProgress", path: event.path, progress: 1, state: "uploading" }); } else if (event.event === "fileProgress") { yieldCallback({ event: "fileProgress", path: event.path, progress: event.progress, state: "uploading" }); } } }) ).then(() => returnCallback(void 0), rejectCallback) ); } else { } } else { yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { return promisesQueueStreaming( json.objects.map((obj) => async () => { const op = shaToOperation.get(obj.oid); if (!op) { throw new InvalidApiResponseFormatError("Unrequested object ID in response"); } _optionalChain([abortSignal, 'optionalAccess', _104 => _104.throwIfAborted, 'call', _105 => _105()]); if (obj.error) { const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${obj.error.message}${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); } if (!_optionalChain([obj, 'access', _106 => _106.actions, 'optionalAccess', _107 => _107.upload])) { yieldCallback({ event: "fileProgress", path: op.path, progress: 1, state: "uploading" }); return; } yieldCallback({ event: "fileProgress", path: op.path, progress: 0, state: "uploading" }); const content = op.content; const header = obj.actions.upload.header; if (_optionalChain([header, 'optionalAccess', _108 => _108.chunk_size])) { const chunkSize = parseInt(header.chunk_size); const completionUrl = obj.actions.upload.href; const parts = Object.keys(header).filter((key) => /^[0-9]+$/.test(key)); if (parts.length !== Math.ceil(content.size / chunkSize)) { throw new Error("Invalid server response to upload large LFS file, wrong number of parts"); } const completeReq = { oid: obj.oid, parts: parts.map((part) => ({ partNumber: +part, etag: "" })) }; const progressCallback = (progress) => yieldCallback({ event: "fileProgress", path: op.path, progress, state: "uploading" }); await promisesQueueStreaming( parts.map((part) => async () => { _optionalChain([abortSignal, 'optionalAccess', _109 => _109.throwIfAborted, 'call', _110 => _110()]); const index = parseInt(part) - 1; const slice = content.slice(index * chunkSize, (index + 1) * chunkSize); const res3 = await (_nullishCoalesce(params.fetch, () => ( fetch)))(header[part], { method: "PUT", /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ body: slice instanceof WebBlob && isFrontend ? await slice.arrayBuffer() : slice, signal: abortSignal, ...{ progressHint: { path: op.path, part: index, numParts: parts.length, progressCallback } // eslint-disable-next-line @typescript-eslint/no-explicit-any } }); if (!res3.ok) { throw await createApiError(res3, { requestId: batchRequestId, message: `Error while uploading part ${part} of ${operations[shas.indexOf(obj.oid)].path} to LFS storage` }); } const eTag = res3.headers.get("ETag"); if (!eTag) { throw new Error("Cannot get ETag of part during multipart upload"); } completeReq.parts[Number(part) - 1].etag = eTag; }), MULTIPART_PARALLEL_UPLOAD ); _optionalChain([abortSignal, 'optionalAccess', _111 => _111.throwIfAborted, 'call', _112 => _112()]); const res2 = await (_nullishCoalesce(params.fetch, () => ( fetch)))(completionUrl, { method: "POST", body: JSON.stringify(completeReq), headers: { Accept: "application/vnd.git-lfs+json", "Content-Type": "application/vnd.git-lfs+json" }, signal: abortSignal }); if (!res2.ok) { throw await createApiError(res2, { requestId: batchRequestId, message: `Error completing multipart upload of ${operations[shas.indexOf(obj.oid)].path} to LFS storage` }); } yieldCallback({ event: "fileProgress", path: op.path, progress: 1, state: "uploading" }); } else { const res2 = await (_nullishCoalesce(params.fetch, () => ( fetch)))(obj.actions.upload.href, { method: "PUT", headers: { ...batchRequestId ? { "X-Request-Id": batchRequestId } : void 0 }, /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ body: content instanceof WebBlob && isFrontend ? await content.arrayBuffer() : content, signal: abortSignal, ...{ progressHint: { path: op.path, progressCallback: (progress) => yieldCallback({ event: "fileProgress", path: op.path, progress, state: "uploading" }) } // eslint-disable-next-line @typescript-eslint/no-explicit-any } }); if (!res2.ok) { throw await createApiError(res2, { requestId: batchRequestId, message: `Error while uploading ${operations[shas.indexOf(obj.oid)].path} to LFS storage` }); } yieldCallback({ event: "fileProgress", path: op.path, progress: 1, state: "uploading" }); } }), CONCURRENT_LFS_UPLOADS ).then(returnCallback, rejectCallback); }); } } _optionalChain([abortSignal, 'optionalAccess', _113 => _113.throwIfAborted, 'call', _114 => _114()]); yield { event: "phase", phase: "committing" }; return yield* eventToGenerator( async (yieldCallback, returnCallback, rejectCallback) => (_nullishCoalesce(params.fetch, () => ( fetch)))( `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/commit/${encodeURIComponent( _nullishCoalesce(params.branch, () => ( "main")) )}` + (params.isPullRequest ? "?create_pr=1" : ""), { method: "POST", headers: { ...accessToken && { Authorization: `Bearer ${accessToken}` }, "Content-Type": "application/x-ndjson" }, body: [ { key: "header", value: { summary: params.title, description: params.description, parentCommit: params.parentCommit } }, ...await Promise.all( allOperations.map((operation) => { if (isFileOperation(operation)) { const sha = lfsShas.get(operation.path); if (sha) { return { key: "lfsFile", value: { path: operation.path, algo: "sha256", size: operation.content.size, oid: sha } }; } } return convertOperationToNdJson(operation); }) ) ].map((x) => JSON.stringify(x)).join("\n"), signal: abortSignal, ...{ progressHint: { progressCallback: (progress) => { for (const op of allOperations) { if (isFileOperation(op) && !lfsShas.has(op.path)) { yieldCallback({ event: "fileProgress", path: op.path, progress, state: "uploading" }); } } } } // eslint-disable-next-line @typescript-eslint/no-explicit-any } } ).then(async (res) => { if (!res.ok) { throw await createApiError(res); } const json = await res.json(); returnCallback({ pullRequestUrl: json.pullRequestUrl, commit: { oid: json.commitOid, url: json.commitUrl }, hookOutput: json.hookOutput }); }).catch(rejectCallback) ); } catch (err) { abortController.abort(); throw err; } } async function* commitIterBucket(params) { const accessToken = checkCredentials(params); const repoId = toRepoId(params.repo); if (params.useXet === false) { throw new Error("useXet must be true or undefined for buckets"); } const abortController = new AbortController(); const abortSignal = abortController.signal; if (!abortSignal.throwIfAborted) { abortSignal.throwIfAborted = () => { if (abortSignal.aborted) { throw new DOMException("Aborted", "AbortError"); } }; } if (params.abortSignal) { params.abortSignal.addEventListener("abort", () => abortController.abort()); } try { const allOperations = (await Promise.all( params.operations.map(async (operation) => { if (operation.operation === "edit") { const splicedBlob = SplicedBlob.create( operation.originalContent, operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })) ); return { operation: "addOrUpdate", path: operation.path, content: splicedBlob }; } if (operation.operation !== "addOrUpdate") { return operation; } if (!(operation.content instanceof URL)) { return { ...operation, content: operation.content }; } const lazyBlobs = await createBlobs(operation.content, operation.path, { fetch: params.fetch, maxFolderDepth: params.maxFolderDepth }); _optionalChain([abortSignal, 'optionalAccess', _115 => _115.throwIfAborted, 'call', _116 => _116()]); return lazyBlobs.map((blob) => ({ ...operation, content: blob.blob, path: blob.path })); }) )).flat(1); yield { event: "phase", phase: "uploadingLargeFiles" }; for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { const xetHashes = /* @__PURE__ */ new Map(); _optionalChain([abortSignal, 'optionalAccess', _117 => _117.throwIfAborted, 'call', _118 => _118()]); const source = async function* () { for (const operation of operations) { _optionalChain([abortSignal, 'optionalAccess', _119 => _119.throwIfAborted, 'call', _120 => _120()]); yield { content: operation.content, path: operation.path }; } }(); const xetParams = { sessionId: crypto.randomUUID(), refreshWriteTokenUrl: `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/xet-write-token` }; const sources = splitAsyncGenerator(source, 5); yield* eventToGenerator( (yieldCallback, returnCallback, rejectCallback) => Promise.all( sources.map(async function(source2) { for await (const event of uploadShards(source2, { fetch: params.fetch, accessToken, hubUrl: _nullishCoalesce(params.hubUrl, () => ( HUB_URL)), repo: repoId, xetParams, rev: _nullishCoalesce(params.branch, () => ( "main")), yieldCallback: (event2) => yieldCallback({ ...event2, state: "uploading" }) })) { if (event.event === "file") { yieldCallback({ event: "fileProgress", path: event.path, progress: 1, state: "uploading" }); xetHashes.set(event.path, event.xetHash); } else if (event.event === "fileProgress") { yieldCallback({ event: "fileProgress", path: event.path, progress: event.progress, state: "uploading" }); } } }) ).then(() => returnCallback(void 0), rejectCallback) ); const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))( `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/batch`, { method: "POST", headers: { ...accessToken && { Authorization: `Bearer ${accessToken}` }, "Content-Type": "application/x-ndjson" }, body: [...xetHashes.entries()].map( ([path, xetHash]) => JSON.stringify({ type: "addFile", path, xetHash }) ).join("\n"), signal: abortSignal } ); if (!resp.ok && resp.status !== 422) { throw await createApiError(resp); } const json = await resp.json(); for (const failed of json.failed) { yield { event: "fileProgress", path: failed.path, progress: 0, state: "error" }; } } _optionalChain([abortSignal, 'optionalAccess', _121 => _121.throwIfAborted, 'call', _122 => _122()]); const deletedOperations = allOperations.filter((operation) => operation.operation === "delete"); if (deletedOperations.length > 0) { const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))( `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/batch`, { method: "POST", headers: { ...accessToken && { Authorization: `Bearer ${accessToken}` }, "Content-Type": "application/x-ndjson" }, body: deletedOperations.map( (operation) => JSON.stringify({ type: "deleteFile", path: operation.path }) ).join("\n"), signal: abortSignal } ); if (!resp.ok) { throw await createApiError(resp); } const json = await resp.json(); if (json.failed.length > 0) { const failedPaths = json.failed.slice(0, 5).map((f) => f.path); throw new Error( `Failed to delete ${json.failed.length} file(s): ${failedPaths.join(", ")}${json.failed.length > 5 ? "..." : ""}, request ID: ${resp.headers.get("X-Request-Id")}` ); } } _optionalChain([abortSignal, 'optionalAccess', _123 => _123.throwIfAborted, 'call', _124 => _124()]); } catch (err) { abortController.abort(); throw err; } } async function commit(params) { const iterator = commitIter(params); const failedPaths = []; let failedCount = 0; let res = await iterator.next(); while (!res.done) { if (res.value.event === "fileProgress" && res.value.state === "error") { failedCount++; if (failedPaths.length < 5) { failedPaths.push(res.value.path); } } res = await iterator.next(); } if (failedCount > 0) { throw new Error( `Failed to upload ${failedCount} file(s): ${failedPaths.join(", ")}${failedCount > 5 ? "..." : ""}` ); } return res.value; } async function convertOperationToNdJson(operation) { switch (operation.operation) { case "addOrUpdate": { return { key: "file", value: { content: base64FromBytes(new Uint8Array(await operation.content.arrayBuffer())), path: operation.path, encoding: "base64" } }; } case "delete": { return { key: "deletedFile", value: { path: operation.path } }; } case "edit": { throw new Error( "Edit operations should be converted to addOrUpdate operations before reaching convertOperationToNdJson" ); } default: throw new TypeError("Unknown operation: " + operation.operation); } } // src/lib/count-commits.ts async function countCommits(params) { const accessToken = checkCredentials(params); const repoId = toRepoId(params.repo); const url = `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/commits/${_nullishCoalesce(params.revision, () => ( "main"))}?limit=1`; const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} }); if (!res.ok) { throw await createApiError(res); } return parseInt(_nullishCoalesce(res.headers.get("x-total-count"), () => ( "0")), 10); } // src/lib/create-repo.ts async function createRepo(params) { const accessToken = checkCredentials(params); const repoId = toRepoId(params.repo); const [namespace, repoName] = repoId.name.split("/"); if (!namespace || !repoName) { throw new TypeError( `"${repoId.name}" is not a fully qualified repo name. It should be of the form "{namespace}/{repoName}".` ); } const res = repoId.type === "bucket" ? await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/buckets/${namespace}/${repoName}`, { method: "POST", body: JSON.stringify({ private: params.private, resourceGroupId: params.resourceGroupId }), headers: { Authorization: `Bearer ${accessToken}`, "Content-Type": "application/json" } }) : await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/repos/create`, { method: "POST", body: JSON.stringify({ name: repoName, private: params.private, organization: namespace, resourceGroupId: params.resourceGroupId, license: params.license, ...repoId.type === "space" ? { type: "space", sdk: _nullishCoalesce(params.sdk, () => ( "static")) } : { type: repoId.type }, files: params.files ? await Promise.all( params.files.map(async (file) => ({ encoding: "base64", path: file.path, content: base64FromBytes( new Uint8Array(file.content instanceof Blob ? await file.content.arrayBuffer() : file.content) ) })) ) : void 0 }), headers: { Authorization: `Bearer ${accessToken}`, "Content-Type": "application/json" } }); if (!res.ok) { throw await createApiError(res); } const output = await res.json(); return { repoUrl: output.url, id: output.id }; } // src/lib/create-branch.ts async function createBranch(params) { const repoId = toRepoId(params.repo); const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, { method: "POST", headers: { "Content-Type": "application/json", ...params.accessToken && { Authorization: `Bearer ${params.accessToken}` } }, body: JSON.stringify({ startingPoint: params.revision, ...params.empty && { emptyBranch: true }, overwrite: params.overwrite }) } ); if (!res.ok) { throw await createApiError(res); } } // src/lib/create-collection.ts async function createCollection(params) { const accessToken = checkCredentials(params); const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/collections`, { method: "POST", body: JSON.stringify(params.collection), headers: { Authorization: `Bearer ${accessToken}`, "Content-Type": "application/json" } }); if (!res.ok) { throw await createApiError(res); } const output = await res.json(); return { slug: output.slug }; } // src/utils/pick.ts function pick(o, props) { return Object.assign( {}, ...props.map((prop) => { if (o[prop] !== void 0) { return { [prop]: o[prop] }; } }) ); } // src/utils/parseLinkHeader.ts function parseLinkHeader(header) { const regex = /<(https?:[/][/][^>]+)>;\s+rel="([^"]+)"/g; return Object.fromEntries([...header.matchAll(regex)].map(([, url, rel]) => [rel, url])); } // src/lib/list-datasets.ts var DATASET_EXPAND_KEYS = [ "private", "downloads", "gated", "likes", "lastModified" ]; var DATASET_EXPANDABLE_KEYS = [ "author", "cardData", "citation", "createdAt", "disabled", "description", "downloads", "downloadsAllTime", "gated", "gitalyUid", "lastModified", "likes", "paperswithcode_id", "private", // "siblings", "sha", "tags" ]; async function* listDatasets(params) { const accessToken = params && checkCredentials(params); let totalToFetch = _nullishCoalesce(_optionalChain([params, 'optionalAccess', _125 => _125.limit]), () => ( Infinity)); const search = new URLSearchParams([ ...Object.entries({ limit: String(Math.min(totalToFetch, 500)), ..._optionalChain([params, 'optionalAccess', _126 => _126.search, 'optionalAccess', _127 => _127.owner]) ? { author: params.search.owner } : void 0, ..._optionalChain([params, 'optionalAccess', _128 => _128.search, 'optionalAccess', _129 => _129.query]) ? { search: params.search.query } : void 0, ..._optionalChain([params, 'optionalAccess', _130 => _130.sort]) ? { sort: params.sort } : void 0 }), ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _131 => _131.search, 'optionalAccess', _132 => _132.tags, 'optionalAccess', _133 => _133.map, 'call', _134 => _134((tag) => ["filter", tag])]), () => ( [])), ...DATASET_EXPAND_KEYS.map((val) => ["expand", val]), ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _135 => _135.additionalFields, 'optionalAccess', _136 => _136.map, 'call', _137 => _137((val) => ["expand", val])]), () => ( [])) ]).toString(); let url = `${_optionalChain([params, 'optionalAccess', _138 => _138.hubUrl]) || HUB_URL}/api/datasets` + (search ? "?" + search : ""); while (url) { const res = await (_nullishCoalesce(_optionalChain([params, 'optionalAccess', _139 => _139.fetch]), () => ( fetch)))(url, { headers: { accept: "application/json", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 } }); if (!res.ok) { throw await createApiError(res); } const items = await res.json(); for (const item of items) { yield { ..._optionalChain([params, 'optionalAccess', _140 => _140.additionalFields]) && pick(item, params.additionalFields), id: item._id, name: item.id, private: item.private, downloads: item.downloads, likes: item.likes, gated: item.gated, updatedAt: new Date(item.lastModified) }; totalToFetch--; if (totalToFetch <= 0) { return; } } const linkHeader = res.headers.get("Link"); url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; } } // src/lib/dataset-info.ts async function datasetInfo(params) { const accessToken = params && checkCredentials(params); const search = new URLSearchParams([ ...DATASET_EXPAND_KEYS.map((val) => ["expand", val]), ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _141 => _141.additionalFields, 'optionalAccess', _142 => _142.map, 'call', _143 => _143((val) => ["expand", val])]), () => ( [])) ]).toString(); const response = await (params.fetch || fetch)( `${_optionalChain([params, 'optionalAccess', _144 => _144.hubUrl]) || HUB_URL}/api/datasets/${params.name}/revision/${encodeURIComponent( _nullishCoalesce(params.revision, () => ( "HEAD")) )}?${search.toString()}`, { headers: { ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } } ); if (!response.ok) { throw await createApiError(response); } const data = await response.json(); return { ..._optionalChain([params, 'optionalAccess', _145 => _145.additionalFields]) && pick(data, params.additionalFields), id: data._id, name: data.id, private: data.private, downloads: data.downloads, likes: data.likes, gated: data.gated, updatedAt: new Date(data.lastModified) }; } // src/lib/delete-branch.ts async function deleteBranch(params) { const repoId = toRepoId(params.repo); const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, { method: "DELETE", headers: { ...params.accessToken && { Authorization: `Bearer ${params.accessToken}` } } } ); if (!res.ok) { throw await createApiError(res); } } // src/lib/delete-file.ts function deleteFile(params) { return commit({ ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, repo: params.repo, operations: [ { operation: "delete", path: params.path } ], title: _nullishCoalesce(params.commitTitle, () => ( `Delete ${params.path}`)), description: params.commitDescription, hubUrl: params.hubUrl, branch: params.branch, isPullRequest: params.isPullRequest, parentCommit: params.parentCommit, fetch: params.fetch }); } // src/lib/delete-files.ts function deleteFiles(params) { return commit({ ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, repo: params.repo, operations: params.paths.map((path) => ({ operation: "delete", path })), title: _nullishCoalesce(params.commitTitle, () => ( `Deletes ${params.paths.length} files`)), description: params.commitDescription, hubUrl: params.hubUrl, branch: params.branch, isPullRequest: params.isPullRequest, parentCommit: params.parentCommit, fetch: params.fetch }); } // src/lib/delete-repo.ts async function deleteRepo(params) { const accessToken = checkCredentials(params); const repoId = toRepoId(params.repo); const [namespace, repoName] = repoId.name.split("/"); const res = repoId.type === "bucket" ? await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/buckets/${namespace}/${repoName}`, { method: "DELETE", headers: { Authorization: `Bearer ${accessToken}`, "Content-Type": "application/json" } }) : await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/repos/delete`, { method: "DELETE", body: JSON.stringify({ name: repoName, organization: namespace, type: repoId.type }), headers: { Authorization: `Bearer ${accessToken}`, "Content-Type": "application/json" } }); if (!res.ok) { throw await createApiError(res); } } // src/lib/delete-collection.ts async function deleteCollection(params) { if (!params.slug) { throw new TypeError("slug is required"); } const accessToken = checkCredentials(params); const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/collections/${params.slug}`, { method: "DELETE", headers: { Authorization: `Bearer ${accessToken}`, "Content-Type": "application/json" } }); if (!res.ok) { throw await createApiError(res); } } // src/lib/file-download-info.ts async function fileDownloadInfo(params) { const accessToken = checkCredentials(params); const repoId = toRepoId(params.repo); const hubUrl = _nullishCoalesce(params.hubUrl, () => ( HUB_URL)); const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${params.raw ? "raw" : "resolve"}/${encodeURIComponent(_nullishCoalesce(params.revision, () => ( "main")))}/${params.path}` + (params.noContentDisposition ? "?noContentDisposition=1" : ""); const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { method: "GET", headers: { ...accessToken && { Authorization: `Bearer ${accessToken}` }, Range: "bytes=0-0", Accept: "application/vnd.xet-fileinfo+json, */*" } }); if (resp.status === 404 && resp.headers.get("X-Error-Code") === "EntryNotFound") { return null; } if (!resp.ok) { throw await createApiError(resp); } let size; let xetInfo; if (_optionalChain([resp, 'access', _146 => _146.headers, 'access', _147 => _147.get, 'call', _148 => _148("Content-Type"), 'optionalAccess', _149 => _149.includes, 'call', _150 => _150("application/vnd.xet-fileinfo+json")])) { size = parseInt(_nullishCoalesce(resp.headers.get("X-Linked-Size"), () => ( "invalid"))); if (isNaN(size)) { throw new InvalidApiResponseFormatError("Invalid file size received in X-Linked-Size header"); } const hash2 = resp.headers.get("X-Xet-Hash"); const links = parseLinkHeader(_nullishCoalesce(resp.headers.get("Link"), () => ( ""))); const reconstructionUrl = (() => { try { return new URL(links["xet-reconstruction-info"]); } catch (e2) { return null; } })(); const refreshUrl = (() => { try { return new URL(links["xet-auth"]); } catch (e3) { return null; } })(); if (!hash2) { throw new InvalidApiResponseFormatError("No hash received in X-Xet-Hash header"); } if (!reconstructionUrl || !refreshUrl) { throw new InvalidApiResponseFormatError("No xet-reconstruction-info or xet-auth link header"); } xetInfo = { hash: hash2, refreshUrl, reconstructionUrl }; } if (size === void 0 || isNaN(size)) { const contentRangeHeader = resp.headers.get("content-range"); if (!contentRangeHeader) { throw new InvalidApiResponseFormatError("Expected size information"); } const [, parsedSize] = contentRangeHeader.split("/"); size = parseInt(parsedSize); if (isNaN(size)) { throw new InvalidApiResponseFormatError("Invalid file size received"); } } const etag = _nullishCoalesce(_nullishCoalesce(resp.headers.get("X-Linked-ETag"), () => ( resp.headers.get("ETag"))), () => ( void 0)); if (!etag) { throw new InvalidApiResponseFormatError("Expected ETag"); } return { etag, size, xet: xetInfo, // Cannot use resp.url in case it's a S3 url and the user adds an Authorization header to it. url: resp.url && (new URL(resp.url).origin === new URL(hubUrl).origin || _optionalChain([resp, 'access', _151 => _151.headers, 'access', _152 => _152.get, 'call', _153 => _153("X-Cache"), 'optionalAccess', _154 => _154.endsWith, 'call', _155 => _155(" cloudfront")])) ? resp.url : url }; } // src/lib/download-file.ts async function downloadFile(params) { const accessToken = checkCredentials(params); const info = await _asyncNullishCoalesce(params.downloadInfo, async () => ( await fileDownloadInfo({ accessToken, repo: params.repo, path: params.path, revision: params.revision, hubUrl: params.hubUrl, fetch: params.fetch, raw: params.raw }))); if (!info) { return null; } if (info.xet && params.xet !== false) { return new XetBlob({ refreshUrl: info.xet.refreshUrl.href, reconstructionUrl: info.xet.reconstructionUrl.href, fetch: params.fetch, accessToken, size: info.size, readToken: typeof params.xet === "object" ? params.xet.readToken : void 0 }); } return new WebBlob(new URL(info.url), 0, info.size, "", true, _nullishCoalesce(params.fetch, () => ( fetch)), accessToken); } // src/lib/file-exists.ts async function fileExists(params) { const accessToken = checkCredentials(params); const repoId = toRepoId(params.repo); const hubUrl = _nullishCoalesce(params.hubUrl, () => ( HUB_URL)); const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/raw/${encodeURIComponent( _nullishCoalesce(params.revision, () => ( "main")) )}/${params.path}`; const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { method: "HEAD", headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} }); if (resp.status === 404) { return false; } if (!resp.ok) { throw await createApiError(resp); } return true; } // src/lib/jobs/cancel-job.ts async function cancelJob(params) { const accessToken = checkCredentials(params); const response = await (params.fetch || fetch)( `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/cancel`, { method: "POST", headers: { "Content-Type": "application/json", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } } ); if (!response.ok) { throw await createApiError(response); } return await response.json(); } // src/lib/jobs/create-scheduled-job.ts async function createScheduledJob(params) { const accessToken = checkCredentials(params); const { namespace, hubUrl, fetch: customFetch, ...rest } = params; if (!rest.jobSpec.dockerImage && !rest.jobSpec.spaceId) { throw new Error("Either dockerImage or spaceId must be provided in jobSpec"); } if (rest.jobSpec.dockerImage && rest.jobSpec.spaceId) { throw new Error("Cannot provide both dockerImage and spaceId in jobSpec"); } const body = { jobSpec: { flavor: rest.jobSpec.flavor }, schedule: rest.schedule, suspend: _nullishCoalesce(rest.suspend, () => ( false)), concurrency: _nullishCoalesce(rest.concurrency, () => ( false)) }; if (rest.jobSpec.dockerImage) { body.jobSpec.dockerImage = rest.jobSpec.dockerImage; } if (rest.jobSpec.spaceId) { body.jobSpec.spaceId = rest.jobSpec.spaceId; } if (rest.jobSpec.command) { body.jobSpec.command = rest.jobSpec.command; } body.jobSpec.environment = rest.jobSpec.environment || {}; if (rest.jobSpec.secrets) { body.jobSpec.secrets = rest.jobSpec.secrets; } if (rest.jobSpec.arch) { body.jobSpec.arch = rest.jobSpec.arch; } if (rest.jobSpec.timeoutSeconds !== void 0) { body.jobSpec.timeoutSeconds = rest.jobSpec.timeoutSeconds; } if (rest.jobSpec.attempts !== void 0) { body.jobSpec.attempts = rest.jobSpec.attempts; } if (rest.jobSpec.labels) { body.jobSpec.labels = rest.jobSpec.labels; } const response = await (customFetch || fetch)(`${hubUrl || HUB_URL}/api/scheduled-jobs/${namespace}`, { method: "POST", headers: { "Content-Type": "application/json", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} }, body: JSON.stringify(body) }); if (!response.ok) { throw await createApiError(response); } return await response.json(); } // src/lib/jobs/delete-scheduled-job.ts async function deleteScheduledJob(params) { const accessToken = checkCredentials(params); const response = await (params.fetch || fetch)( `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}`, { method: "DELETE", headers: { "Content-Type": "application/json", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } } ); if (!response.ok) { throw await createApiError(response); } } // src/lib/jobs/duplicate-job.ts async function duplicateJob(params) { const accessToken = checkCredentials(params); const response = await (params.fetch || fetch)( `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/duplicate`, { method: "POST", headers: { "Content-Type": "application/json", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } } ); if (!response.ok) { throw await createApiError(response); } return await response.json(); } // src/lib/jobs/get-job.ts async function getJob(params) { const accessToken = checkCredentials(params); const response = await (params.fetch || fetch)( `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}`, { headers: { ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } } ); if (!response.ok) { throw await createApiError(response); } return await response.json(); } // src/lib/jobs/get-scheduled-job.ts async function getScheduledJob(params) { const accessToken = checkCredentials(params); const response = await (params.fetch || fetch)( `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}`, { headers: { ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } } ); if (!response.ok) { throw await createApiError(response); } return await response.json(); } // src/lib/jobs/list-job-hardware.ts async function listJobHardware(params) { const accessToken = checkCredentials(_nullishCoalesce(params, () => ( {}))); const headers = {}; if (accessToken) { headers.Authorization = `Bearer ${accessToken}`; } const response = await (_optionalChain([params, 'optionalAccess', _156 => _156.fetch]) || fetch)(`${_optionalChain([params, 'optionalAccess', _157 => _157.hubUrl]) || HUB_URL}/api/jobs/hardware`, { headers }); if (!response.ok) { throw await createApiError(response); } return await response.json(); } // src/lib/jobs/list-jobs.ts async function listJobs(params) { const accessToken = checkCredentials(params); const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { headers: { ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } }); if (!response.ok) { throw await createApiError(response); } return await response.json(); } // src/lib/jobs/list-scheduled-jobs.ts async function listScheduledJobs(params) { const accessToken = checkCredentials(params); const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}`, { headers: { ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } }); if (!response.ok) { throw await createApiError(response); } return await response.json(); } // src/lib/jobs/resume-scheduled-job.ts async function resumeScheduledJob(params) { const accessToken = checkCredentials(params); const response = await (params.fetch || fetch)( `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/resume`, { method: "POST", headers: { ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } } ); if (!response.ok) { throw await createApiError(response); } } // src/lib/jobs/run-job.ts async function runJob(params) { const accessToken = checkCredentials(params); if (!params.dockerImage && !params.spaceId) { throw new Error("Either dockerImage or spaceId must be provided"); } if (params.dockerImage && params.spaceId) { throw new Error("Cannot provide both dockerImage and spaceId"); } const body = { flavor: params.flavor, environment: params.environment || {} }; if (params.dockerImage) { body.dockerImage = params.dockerImage; } if (params.spaceId) { body.spaceId = params.spaceId; } if (params.command) { body.command = params.command; } if (params.arguments) { body.arguments = params.arguments; } if (params.secrets) { body.secrets = params.secrets; } if (params.arch) { body.arch = params.arch; } if (params.timeoutSeconds !== void 0) { body.timeoutSeconds = params.timeoutSeconds; } if (params.attempts !== void 0) { body.attempts = params.attempts; } if (params.labels) { body.labels = params.labels; } const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { method: "POST", headers: { "Content-Type": "application/json", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} }, body: JSON.stringify(body) }); if (!response.ok) { throw await createApiError(response); } return await response.json(); } // src/lib/jobs/run-scheduled-job.ts async function runScheduledJob(params) { const accessToken = checkCredentials(params); const response = await (params.fetch || fetch)( `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/run`, { method: "POST", headers: { "Content-Type": "application/json", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } } ); if (!response.ok) { if (response.status === 409) { return null; } throw await createApiError(response); } return await response.json(); } // src/lib/jobs/stream-job-events.ts async function* streamJobEvents(params) { const accessToken = checkCredentials(params); const response = await (params.fetch || fetch)( `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/events`, { headers: { Accept: "text/event-stream", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } } ); if (!response.ok) { throw await createApiError(response); } if (!response.body) { return; } const reader = response.body.getReader(); const decoder = new TextDecoder(); let buffer = ""; try { while (true) { const { done, value } = await reader.read(); if (done) break; buffer += decoder.decode(value, { stream: true }); const lines = buffer.split("\n"); buffer = lines.pop() || ""; for (const line of lines) { if (line.startsWith("data: ")) { yield line.slice(6); } } } if (buffer) { const lines = buffer.split("\n"); for (const line of lines) { if (line.startsWith("data: ")) { yield line.slice(6); } } } } finally { reader.releaseLock(); } } // src/lib/jobs/stream-job-logs.ts async function* streamJobLogs(params) { const accessToken = checkCredentials(params); const response = await (params.fetch || fetch)( `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/logs`, { headers: { Accept: "text/event-stream", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } } ); if (!response.ok) { throw await createApiError(response); } if (!response.body) { return; } const reader = response.body.getReader(); const decoder = new TextDecoder(); let buffer = ""; try { while (true) { const { done, value } = await reader.read(); if (done) break; buffer += decoder.decode(value, { stream: true }); const lines = buffer.split("\n"); buffer = lines.pop() || ""; for (const line of lines) { if (line.startsWith("data: ")) { try { const data = JSON.parse(line.slice(6)); yield { message: data.data, timestamp: new Date(data.timestamp) }; } catch (e4) { yield { message: line.slice(6), timestamp: /* @__PURE__ */ new Date() }; } } } } if (buffer) { const lines = buffer.split("\n"); for (const line of lines) { if (line.startsWith("data: ")) { try { const data = JSON.parse(line.slice(6)); yield { message: data.data, timestamp: new Date(data.timestamp) }; } catch (e5) { yield { message: line.slice(6), timestamp: /* @__PURE__ */ new Date() }; } } } } } finally { reader.releaseLock(); } } // src/lib/jobs/stream-job-metrics.ts async function* streamJobMetrics(params) { const accessToken = checkCredentials(params); const response = await (params.fetch || fetch)( `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/metrics`, { headers: { Accept: "text/event-stream", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } } ); if (!response.ok) { throw await createApiError(response); } if (!response.body) { return; } const reader = response.body.getReader(); const decoder = new TextDecoder(); let buffer = ""; try { while (true) { const { done, value } = await reader.read(); if (done) break; buffer += decoder.decode(value, { stream: true }); const lines = buffer.split("\n"); buffer = lines.pop() || ""; for (const line of lines) { if (line.startsWith("data: ")) { yield line.slice(6); } } } if (buffer) { const lines = buffer.split("\n"); for (const line of lines) { if (line.startsWith("data: ")) { yield line.slice(6); } } } } finally { reader.releaseLock(); } } // src/lib/jobs/suspend-scheduled-job.ts async function suspendScheduledJob(params) { const accessToken = checkCredentials(params); const response = await (params.fetch || fetch)( `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/suspend`, { method: "POST", headers: { "Content-Type": "application/json", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } } ); if (!response.ok) { throw await createApiError(response); } } // src/lib/list-commits.ts async function* listCommits(params) { const accessToken = checkCredentials(params); const repoId = toRepoId(params.repo); let url = `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/commits/${_nullishCoalesce(params.revision, () => ( "main"))}?limit=${_nullishCoalesce(params.batchSize, () => ( 100))}`; while (url) { const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} }); if (!res.ok) { throw await createApiError(res); } const resJson = await res.json(); for (const commit2 of resJson) { yield { oid: commit2.id, title: commit2.title, message: commit2.message, authors: commit2.authors.map((author) => ({ username: author.user, avatarUrl: author.avatar })), date: new Date(commit2.date) }; } const linkHeader = res.headers.get("Link"); url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; } } // src/lib/list-files.ts async function* listFiles(params) { const accessToken = checkCredentials(params); const repoId = toRepoId(params.repo); let url = `${params.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}/tree/${params.revision || "main"}${params.path ? "/" + params.path : ""}?recursive=${!!params.recursive}&expand=${!!params.expand}`; while (url) { const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { headers: { accept: "application/json", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 } }); if (!res.ok) { throw await createApiError(res); } const items = await res.json(); for (const item of items) { yield item; } const linkHeader = res.headers.get("Link"); url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; } } // src/utils/normalizeInferenceProviderMapping.ts function normalizeInferenceProviderMapping(hfModelId, inferenceProviderMapping) { if (!inferenceProviderMapping) { return []; } if (Array.isArray(inferenceProviderMapping)) { return inferenceProviderMapping.map((entry) => ({ ...entry, hfModelId })); } return Object.entries(inferenceProviderMapping).map(([provider, mapping]) => ({ provider, hfModelId, providerId: mapping.providerId, status: mapping.status, task: mapping.task })); } // src/lib/list-models.ts var MODEL_EXPAND_KEYS = [ "pipeline_tag", "private", "gated", "downloads", "likes", "lastModified" ]; var MODEL_EXPANDABLE_KEYS = [ "author", "cardData", "config", "createdAt", "disabled", "downloads", "downloadsAllTime", "gated", "gitalyUid", "inferenceProviderMapping", "lastModified", "library_name", "likes", "model-index", "pipeline_tag", "private", "safetensors", "sha", // "siblings", "spaces", "tags", "transformersInfo" ]; async function* listModels(params) { const accessToken = params && checkCredentials(params); let totalToFetch = _nullishCoalesce(_optionalChain([params, 'optionalAccess', _158 => _158.limit]), () => ( Infinity)); const search = new URLSearchParams([ ...Object.entries({ limit: String(Math.min(totalToFetch, 500)), ..._optionalChain([params, 'optionalAccess', _159 => _159.search, 'optionalAccess', _160 => _160.owner]) ? { author: params.search.owner } : void 0, ..._optionalChain([params, 'optionalAccess', _161 => _161.search, 'optionalAccess', _162 => _162.task]) ? { pipeline_tag: params.search.task } : void 0, ..._optionalChain([params, 'optionalAccess', _163 => _163.search, 'optionalAccess', _164 => _164.query]) ? { search: params.search.query } : void 0, ..._optionalChain([params, 'optionalAccess', _165 => _165.search, 'optionalAccess', _166 => _166.inferenceProviders]) ? { inference_provider: params.search.inferenceProviders.join(",") } : void 0, ..._optionalChain([params, 'optionalAccess', _167 => _167.search, 'optionalAccess', _168 => _168.apps]) ? { apps: params.search.apps.join(",") } : void 0, ..._optionalChain([params, 'optionalAccess', _169 => _169.sort]) ? { sort: params.sort } : void 0 }), ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _170 => _170.search, 'optionalAccess', _171 => _171.tags, 'optionalAccess', _172 => _172.map, 'call', _173 => _173((tag) => ["filter", tag])]), () => ( [])), ...MODEL_EXPAND_KEYS.map((val) => ["expand", val]), ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _174 => _174.additionalFields, 'optionalAccess', _175 => _175.map, 'call', _176 => _176((val) => ["expand", val])]), () => ( [])) ]).toString(); let url = `${_optionalChain([params, 'optionalAccess', _177 => _177.hubUrl]) || HUB_URL}/api/models?${search}`; while (url) { const res = await (_nullishCoalesce(_optionalChain([params, 'optionalAccess', _178 => _178.fetch]), () => ( fetch)))(url, { headers: { accept: "application/json", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 } }); if (!res.ok) { throw await createApiError(res); } const items = await res.json(); for (const item of items) { const normalizedItem = { ...item }; if (_optionalChain([params, 'optionalAccess', _179 => _179.additionalFields, 'optionalAccess', _180 => _180.includes, 'call', _181 => _181("inferenceProviderMapping")]) && item.inferenceProviderMapping) { normalizedItem.inferenceProviderMapping = normalizeInferenceProviderMapping( item.id, item.inferenceProviderMapping ); } yield { ..._optionalChain([params, 'optionalAccess', _182 => _182.additionalFields]) && pick(normalizedItem, params.additionalFields), id: item._id, name: item.id, private: item.private, task: item.pipeline_tag, downloads: item.downloads, gated: item.gated, likes: item.likes, updatedAt: new Date(item.lastModified) }; totalToFetch--; if (totalToFetch <= 0) { return; } } const linkHeader = res.headers.get("Link"); url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; } } // src/lib/list-spaces.ts var SPACE_EXPAND_KEYS = [ "sdk", "likes", "private", "lastModified" ]; var SPACE_EXPANDABLE_KEYS = [ "author", "cardData", "datasets", "disabled", "gitalyUid", "lastModified", "createdAt", "likes", "private", "runtime", "sdk", // "siblings", "sha", "subdomain", "tags", "models" ]; async function* listSpaces(params) { const accessToken = params && checkCredentials(params); const search = new URLSearchParams([ ...Object.entries({ limit: "500", ..._optionalChain([params, 'optionalAccess', _183 => _183.search, 'optionalAccess', _184 => _184.owner]) ? { author: params.search.owner } : void 0, ..._optionalChain([params, 'optionalAccess', _185 => _185.search, 'optionalAccess', _186 => _186.query]) ? { search: params.search.query } : void 0, ..._optionalChain([params, 'optionalAccess', _187 => _187.sort]) ? { sort: params.sort } : void 0 }), ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _188 => _188.search, 'optionalAccess', _189 => _189.tags, 'optionalAccess', _190 => _190.map, 'call', _191 => _191((tag) => ["filter", tag])]), () => ( [])), ...[...SPACE_EXPAND_KEYS, ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _192 => _192.additionalFields]), () => ( []))].map( (val) => ["expand", val] ) ]).toString(); let url = `${_optionalChain([params, 'optionalAccess', _193 => _193.hubUrl]) || HUB_URL}/api/spaces?${search}`; while (url) { const res = await (_nullishCoalesce(_optionalChain([params, 'optionalAccess', _194 => _194.fetch]), () => ( fetch)))(url, { headers: { accept: "application/json", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 } }); if (!res.ok) { throw await createApiError(res); } const items = await res.json(); for (const item of items) { yield { ..._optionalChain([params, 'optionalAccess', _195 => _195.additionalFields]) && pick(item, params.additionalFields), id: item._id, name: item.id, sdk: item.sdk, likes: item.likes, private: item.private, updatedAt: new Date(item.lastModified) }; } const linkHeader = res.headers.get("Link"); url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; } } // src/lib/list-collections.ts async function* listCollections(params) { const accessToken = params && checkCredentials(params); const searchParams = new URLSearchParams(); let totalToFetch = _nullishCoalesce(_optionalChain([params, 'optionalAccess', _196 => _196.limit]), () => ( Infinity)); searchParams.append("limit", String(Math.min(totalToFetch, 100))); if (_optionalChain([params, 'optionalAccess', _197 => _197.sort])) { searchParams.append("sort", params.sort); } if (_optionalChain([params, 'optionalAccess', _198 => _198.search, 'optionalAccess', _199 => _199.owner])) { for (const owner of params.search.owner) { searchParams.append("owner", owner); } } if (_optionalChain([params, 'optionalAccess', _200 => _200.search, 'optionalAccess', _201 => _201.item])) { for (const item of params.search.item) { searchParams.append("item", item); } } if (_optionalChain([params, 'optionalAccess', _202 => _202.search, 'optionalAccess', _203 => _203.q])) { searchParams.append("q", params.search.q); } let url = `${_optionalChain([params, 'optionalAccess', _204 => _204.hubUrl]) || HUB_URL}/api/collections?${searchParams}`; while (url) { const res = await (_nullishCoalesce(_optionalChain([params, 'optionalAccess', _205 => _205.fetch]), () => ( fetch)))(url, { headers: { accept: "application/json", ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 } }); if (!res.ok) { throw await createApiError(res); } const collections = await res.json(); for (const collection of collections) { yield collection; totalToFetch--; if (totalToFetch <= 0) { return; } } const linkHeader = res.headers.get("Link"); url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; } } // src/lib/model-info.ts async function modelInfo(params) { const accessToken = params && checkCredentials(params); const search = new URLSearchParams([ ...MODEL_EXPAND_KEYS.map((val) => ["expand", val]), ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _206 => _206.additionalFields, 'optionalAccess', _207 => _207.map, 'call', _208 => _208((val) => ["expand", val])]), () => ( [])) ]).toString(); const response = await (params.fetch || fetch)( `${_optionalChain([params, 'optionalAccess', _209 => _209.hubUrl]) || HUB_URL}/api/models/${params.name}/revision/${encodeURIComponent( _nullishCoalesce(params.revision, () => ( "HEAD")) )}?${search.toString()}`, { headers: { ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } } ); if (!response.ok) { throw await createApiError(response); } const data = await response.json(); const normalizedData = { ...data }; if (_optionalChain([params, 'optionalAccess', _210 => _210.additionalFields, 'optionalAccess', _211 => _211.includes, 'call', _212 => _212("inferenceProviderMapping")]) && data.inferenceProviderMapping) { normalizedData.inferenceProviderMapping = normalizeInferenceProviderMapping(data.id, data.inferenceProviderMapping); } return { ..._optionalChain([params, 'optionalAccess', _213 => _213.additionalFields]) && pick(normalizedData, params.additionalFields), id: data._id, name: data.id, private: data.private, task: data.pipeline_tag, downloads: data.downloads, gated: data.gated, likes: data.likes, updatedAt: new Date(data.lastModified) }; } // src/lib/oauth-handle-redirect.ts async function oauthHandleRedirect(opts) { if (typeof window === "undefined" && !_optionalChain([opts, 'optionalAccess', _214 => _214.redirectedUrl])) { throw new Error("oauthHandleRedirect is only available in the browser, unless you provide redirectedUrl"); } if (typeof localStorage === "undefined" && (!_optionalChain([opts, 'optionalAccess', _215 => _215.nonce]) || !_optionalChain([opts, 'optionalAccess', _216 => _216.codeVerifier]))) { throw new Error( "oauthHandleRedirect requires localStorage to be available, unless you provide nonce and codeVerifier" ); } const redirectedUrl = _nullishCoalesce(_optionalChain([opts, 'optionalAccess', _217 => _217.redirectedUrl]), () => ( window.location.href)); const searchParams = (() => { try { return new URL(redirectedUrl).searchParams; } catch (err) { throw new Error("Failed to parse redirected URL: " + redirectedUrl); } })(); const [error, errorDescription] = [searchParams.get("error"), searchParams.get("error_description")]; if (error) { throw new Error(`${error}: ${errorDescription}`); } const code = searchParams.get("code"); const nonce = _nullishCoalesce(_optionalChain([opts, 'optionalAccess', _218 => _218.nonce]), () => ( localStorage.getItem("huggingface.co:oauth:nonce"))); if (!code) { throw new Error("Missing oauth code from query parameters in redirected URL: " + redirectedUrl); } if (!nonce) { throw new Error("Missing oauth nonce from localStorage"); } const codeVerifier = _nullishCoalesce(_optionalChain([opts, 'optionalAccess', _219 => _219.codeVerifier]), () => ( localStorage.getItem("huggingface.co:oauth:code_verifier"))); if (!codeVerifier) { throw new Error("Missing oauth code_verifier from localStorage"); } const state = searchParams.get("state"); if (!state) { throw new Error("Missing oauth state from query parameters in redirected URL"); } let parsedState; try { parsedState = JSON.parse(state); } catch (e6) { throw new Error("Invalid oauth state in redirected URL, unable to parse JSON: " + state); } if (parsedState.nonce !== nonce) { throw new Error("Invalid oauth state in redirected URL"); } const hubUrl = _optionalChain([opts, 'optionalAccess', _220 => _220.hubUrl]) || HUB_URL; const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; const openidConfigRes = await fetch(openidConfigUrl, { headers: { Accept: "application/json" } }); if (!openidConfigRes.ok) { throw await createApiError(openidConfigRes); } const openidConfig = await openidConfigRes.json(); const tokenRes = await fetch(openidConfig.token_endpoint, { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded" }, body: new URLSearchParams({ grant_type: "authorization_code", code, redirect_uri: parsedState.redirectUri, code_verifier: codeVerifier }).toString() }); if (!_optionalChain([opts, 'optionalAccess', _221 => _221.codeVerifier])) { localStorage.removeItem("huggingface.co:oauth:code_verifier"); } if (!_optionalChain([opts, 'optionalAccess', _222 => _222.nonce])) { localStorage.removeItem("huggingface.co:oauth:nonce"); } if (!tokenRes.ok) { throw await createApiError(tokenRes); } const token = await tokenRes.json(); const accessTokenExpiresAt = new Date(Date.now() + token.expires_in * 1e3); const userInfoRes = await fetch(openidConfig.userinfo_endpoint, { headers: { Authorization: `Bearer ${token.access_token}` } }); if (!userInfoRes.ok) { throw await createApiError(userInfoRes); } const userInfo = await userInfoRes.json(); return { accessToken: token.access_token, accessTokenExpiresAt, userInfo, state: parsedState.state, scope: token.scope }; } async function oauthHandleRedirectIfPresent(opts) { if (typeof window === "undefined" && !_optionalChain([opts, 'optionalAccess', _223 => _223.redirectedUrl])) { throw new Error("oauthHandleRedirect is only available in the browser, unless you provide redirectedUrl"); } if (typeof localStorage === "undefined" && (!_optionalChain([opts, 'optionalAccess', _224 => _224.nonce]) || !_optionalChain([opts, 'optionalAccess', _225 => _225.codeVerifier]))) { throw new Error( "oauthHandleRedirect requires localStorage to be available, unless you provide nonce and codeVerifier" ); } const searchParams = new URLSearchParams(_nullishCoalesce(_optionalChain([opts, 'optionalAccess', _226 => _226.redirectedUrl]), () => ( window.location.search))); if (searchParams.has("error")) { return oauthHandleRedirect(opts); } if (searchParams.has("code")) { if (!localStorage.getItem("huggingface.co:oauth:nonce")) { console.warn( "Missing oauth nonce from localStorage. This can happen when the user refreshes the page after logging in, without changing the URL." ); return false; } return oauthHandleRedirect(opts); } return false; } // src/lib/oauth-login-url.ts async function oauthLoginUrl(opts) { if (typeof window === "undefined" && (!_optionalChain([opts, 'optionalAccess', _227 => _227.redirectUrl]) || !_optionalChain([opts, 'optionalAccess', _228 => _228.clientId]))) { throw new Error("oauthLogin is only available in the browser, unless you provide clientId and redirectUrl"); } if (typeof localStorage === "undefined" && !_optionalChain([opts, 'optionalAccess', _229 => _229.localStorage])) { throw new Error( "oauthLogin requires localStorage to be available in the context, unless you provide a localStorage empty object as argument" ); } const hubUrl = _optionalChain([opts, 'optionalAccess', _230 => _230.hubUrl]) || HUB_URL; const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; const openidConfigRes = await fetch(openidConfigUrl, { headers: { Accept: "application/json" } }); if (!openidConfigRes.ok) { throw await createApiError(openidConfigRes); } const opendidConfig = await openidConfigRes.json(); const newNonce = globalThis.crypto.randomUUID(); const newCodeVerifier = globalThis.crypto.randomUUID() + globalThis.crypto.randomUUID(); if (_optionalChain([opts, 'optionalAccess', _231 => _231.localStorage])) { if (opts.localStorage.codeVerifier !== void 0 && opts.localStorage.codeVerifier !== null) { throw new Error( "localStorage.codeVerifier must be initially set to null or undefined, and will be filled by oauthLoginUrl" ); } if (opts.localStorage.nonce !== void 0 && opts.localStorage.nonce !== null) { throw new Error( "localStorage.nonce must be initially set to null or undefined, and will be filled by oauthLoginUrl" ); } opts.localStorage.codeVerifier = newCodeVerifier; opts.localStorage.nonce = newNonce; } else { localStorage.setItem("huggingface.co:oauth:nonce", newNonce); localStorage.setItem("huggingface.co:oauth:code_verifier", newCodeVerifier); } const redirectUri = _optionalChain([opts, 'optionalAccess', _232 => _232.redirectUrl]) || (typeof window !== "undefined" ? window.location.href : void 0); if (!redirectUri) { throw new Error("Missing redirectUrl"); } const state = JSON.stringify({ nonce: newNonce, redirectUri, state: _optionalChain([opts, 'optionalAccess', _233 => _233.state]) }); const variables = ( // @ts-expect-error window.huggingface is defined inside static Spaces. typeof window !== "undefined" ? _nullishCoalesce(_optionalChain([window, 'access', _234 => _234.huggingface, 'optionalAccess', _235 => _235.variables]), () => ( null)) : null ); const clientId = _optionalChain([opts, 'optionalAccess', _236 => _236.clientId]) || _optionalChain([variables, 'optionalAccess', _237 => _237.OAUTH_CLIENT_ID]); if (!clientId) { if (variables) { throw new Error("Missing clientId, please add hf_oauth: true to the README.md's metadata in your static Space"); } throw new Error("Missing clientId"); } const challenge = base64FromBytes( new Uint8Array(await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(newCodeVerifier))) ).replace(/[+]/g, "-").replace(/[/]/g, "_").replace(/=/g, ""); return `${opendidConfig.authorization_endpoint}?${new URLSearchParams({ client_id: clientId, scope: _optionalChain([opts, 'optionalAccess', _238 => _238.scopes]) || _optionalChain([variables, 'optionalAccess', _239 => _239.OAUTH_SCOPES]) || "openid profile", response_type: "code", redirect_uri: redirectUri, state, code_challenge: challenge, code_challenge_method: "S256" }).toString()}`; } // src/utils/typedInclude.ts function typedInclude(arr, v) { return arr.includes(v); } // src/utils/omit.ts function omit(o, props) { const propsArr = Array.isArray(props) ? props : [props]; const letsKeep = Object.keys(o).filter((prop) => !typedInclude(propsArr, prop)); return pick(o, letsKeep); } // src/utils/typedEntries.ts function typedEntries(obj) { return Object.entries(obj); } // src/lib/parse-safetensors-metadata.ts var SAFETENSORS_FILE = "model.safetensors"; var SAFETENSORS_INDEX_FILE = "model.safetensors.index.json"; var RE_SAFETENSORS_FILE = /\.safetensors$/; var RE_SAFETENSORS_INDEX_FILE = /\.safetensors\.index\.json$/; var RE_SAFETENSORS_SHARD_FILE = /^(?(?.*?)[_-])(?\d{5,6})-of-(?\d{5,6})\.safetensors$/; function parseSafetensorsShardFilename(filename) { const match = RE_SAFETENSORS_SHARD_FILE.exec(filename); if (match && match.groups) { return { prefix: match.groups["prefix"], basePrefix: match.groups["basePrefix"], shard: match.groups["shard"], total: match.groups["total"] }; } return null; } var PARALLEL_DOWNLOADS = 20; var MAX_HEADER_LENGTH = 25e6; var GPTQ_QWEIGHT_SUFFIX = "qweight"; var SafetensorParseError = class extends Error { }; async function fetchModelConfig(params) { try { const configBlob = await downloadFile({ ...params, path: "config.json" }); if (!configBlob) { return null; } const config = JSON.parse(await configBlob.text()); return config; } catch (error) { return null; } } async function parseSingleFile(path, params) { const blob = await downloadFile({ ...params, path }); if (!blob) { throw new SafetensorParseError(`Failed to parse file ${path}: failed to fetch safetensors header length.`); } const bufLengthOfHeaderLE = await blob.slice(0, 8).arrayBuffer(); const lengthOfHeader = new DataView(bufLengthOfHeaderLE).getBigUint64(0, true); if (lengthOfHeader <= 0) { throw new SafetensorParseError(`Failed to parse file ${path}: safetensors header is malformed.`); } if (lengthOfHeader > MAX_HEADER_LENGTH) { throw new SafetensorParseError( `Failed to parse file ${path}: safetensor header is too big. Maximum supported size is ${MAX_HEADER_LENGTH} bytes.` ); } try { const header = JSON.parse(await blob.slice(8, 8 + Number(lengthOfHeader)).text()); return header; } catch (err) { throw new SafetensorParseError(`Failed to parse file ${path}: safetensors header is not valid JSON.`); } } async function parseShardedIndex(path, params) { const indexBlob = await downloadFile({ ...params, path }); if (!indexBlob) { throw new SafetensorParseError(`Failed to parse file ${path}: failed to fetch safetensors index.`); } try { const index = JSON.parse(await indexBlob.slice(0, MAX_HEADER_LENGTH).text()); return index; } catch (error) { throw new SafetensorParseError(`Failed to parse file ${path}: not a valid JSON.`); } } async function fetchAllHeaders(path, index, params) { const pathPrefix = path.slice(0, path.lastIndexOf("/") + 1); const filenames = [...new Set(Object.values(index.weight_map))]; const shardedMap = Object.fromEntries( await promisesQueue( filenames.map( (filename) => async () => [filename, await parseSingleFile(pathPrefix + filename, params)] ), PARALLEL_DOWNLOADS ) ); return shardedMap; } async function parseSafetensorsMetadata(params) { const repoId = toRepoId(params.repo); if (repoId.type !== "model") { throw new TypeError("Only model repos should contain safetensors files."); } const modelConfig = params.computeParametersCount ? await fetchModelConfig(params) : null; const quantConfig = _optionalChain([modelConfig, 'optionalAccess', _240 => _240.quantization_config]); if (params.path && RE_SAFETENSORS_FILE.test(params.path) || await fileExists({ ...params, path: SAFETENSORS_FILE })) { const header = await parseSingleFile(_nullishCoalesce(params.path, () => ( SAFETENSORS_FILE)), params); return { sharded: false, header, ...params.computeParametersCount ? { parameterCount: computeNumOfParamsByDtypeSingleFile(header, quantConfig), parameterTotal: ( /// shortcut: get param count directly from metadata header.__metadata__.total_parameters ? typeof header.__metadata__.total_parameters === "number" ? header.__metadata__.total_parameters : typeof header.__metadata__.total_parameters === "string" ? parseInt(header.__metadata__.total_parameters) : void 0 : void 0 ) } : void 0 }; } else if (params.path && RE_SAFETENSORS_INDEX_FILE.test(params.path) || await fileExists({ ...params, path: SAFETENSORS_INDEX_FILE })) { const path = _nullishCoalesce(params.path, () => ( SAFETENSORS_INDEX_FILE)); const index = await parseShardedIndex(path, params); const shardedMap = await fetchAllHeaders(path, index, params); return { sharded: true, index, headers: shardedMap, ...params.computeParametersCount ? { parameterCount: computeNumOfParamsByDtypeSharded(shardedMap, quantConfig), parameterTotal: ( /// shortcut: get param count directly from metadata _optionalChain([index, 'access', _241 => _241.metadata, 'optionalAccess', _242 => _242.total_parameters]) ? typeof index.metadata.total_parameters === "number" ? index.metadata.total_parameters : typeof index.metadata.total_parameters === "string" ? parseInt(index.metadata.total_parameters) : void 0 : void 0 ) } : void 0 }; } else { throw new Error("model id does not seem to contain safetensors weights"); } } function isQuantizedTensor(tensorName, quantConfig) { if (!quantConfig) { return false; } if (!quantConfig.modules_to_not_convert || quantConfig.modules_to_not_convert.length === 0) { return true; } for (const pattern of quantConfig.modules_to_not_convert) { const regexPattern = pattern.replace(/\*/g, ".*"); const regex = new RegExp(regexPattern); if (regex.test(tensorName)) { return false; } } return true; } function getQuantizationMultiplier(tensorName, dtype, quantConfig) { if (!quantConfig || !isQuantizedTensor(tensorName, quantConfig)) { return 1; } const quantMethod = _optionalChain([quantConfig, 'access', _243 => _243.quant_method, 'optionalAccess', _244 => _244.toLowerCase, 'call', _245 => _245()]); switch (quantMethod) { case "mxfp4": if (dtype === "U8" && tensorName.includes("_blocks")) { return 2; } return 1; case "gptq": case "awq": if (getTensorSuffix(tensorName) === GPTQ_QWEIGHT_SUFFIX) { const bits = quantConfig.bits && quantConfig.bits > 0 ? quantConfig.bits : 4; return Math.max(1, Math.floor(32 / bits)); } if (quantConfig.bits === 4 && dtype === "U8") { return 2; } if (quantConfig.bits === 2 && dtype === "U8") { return 4; } return 1; case "bitsandbytes": if (quantConfig.load_in_4bit && dtype === "U8") { return 2; } if (quantConfig.load_in_8bit) { return 1; } return 1; default: if (quantConfig.load_in_4bit && dtype === "U8") { return 2; } if (quantConfig.bits === 4 && dtype === "U8") { return 2; } return 1; } } function computeNumOfParamsByDtypeSingleFile(header, quantConfig) { const counter = {}; const tensors = omit(header, "__metadata__"); for (const [tensorName, v] of typedEntries(tensors)) { if (shouldSkipTensor(tensorName, quantConfig)) { continue; } if (v.shape.length === 0) { continue; } const elements = v.shape.reduce((a, b) => a * b); const multiplier = quantConfig ? getQuantizationMultiplier(tensorName, v.dtype, quantConfig) : 1; if (multiplier === 0) { continue; } counter[v.dtype] = (_nullishCoalesce(counter[v.dtype], () => ( 0))) + elements * multiplier; } return counter; } function computeNumOfParamsByDtypeSharded(shardedMap, quantConfig) { const counter = {}; for (const header of Object.values(shardedMap)) { for (const [k, v] of typedEntries(computeNumOfParamsByDtypeSingleFile(header, quantConfig))) { counter[k] = (_nullishCoalesce(counter[k], () => ( 0))) + (_nullishCoalesce(v, () => ( 0))); } } return counter; } function getTensorSuffix(tensorName) { const lastDotIndex = tensorName.lastIndexOf("."); return lastDotIndex === -1 ? tensorName : tensorName.slice(lastDotIndex + 1); } function shouldSkipTensor(tensorName, quantConfig) { const GPTQ_AWQ_AUXILIARY_SUFFIXES = ["qzeros", "g_idx", "scales"]; if (!quantConfig) { return false; } const quantMethod = _optionalChain([quantConfig, 'access', _246 => _246.quant_method, 'optionalAccess', _247 => _247.toLowerCase, 'call', _248 => _248()]); if (!quantMethod || quantMethod !== "gptq" && quantMethod !== "awq") { return false; } if (!isQuantizedTensor(tensorName, quantConfig)) { return false; } const suffix = getTensorSuffix(tensorName); if (suffix === GPTQ_QWEIGHT_SUFFIX) { return false; } return GPTQ_AWQ_AUXILIARY_SUFFIXES.includes(suffix); } // src/lib/paths-info.ts async function pathsInfo(params) { const accessToken = checkCredentials(params); const repoId = toRepoId(params.repo); const hubUrl = _nullishCoalesce(params.hubUrl, () => ( HUB_URL)); const url = `${hubUrl}/api/${repoId.type}s/${repoId.name}/paths-info/${encodeURIComponent( _nullishCoalesce(params.revision, () => ( "main")) )}`; const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { method: "POST", headers: { ...accessToken && { Authorization: `Bearer ${accessToken}` }, Accept: "application/json", "Content-Type": "application/json" }, body: JSON.stringify({ paths: params.paths, expand: params.expand }) }); if (!resp.ok) { throw await createApiError(resp); } const json = await resp.json(); if (!Array.isArray(json)) throw new Error("malformed response: expected array"); return json.map((item) => ({ path: item.path, lfs: item.lfs, type: item.type, oid: item.oid, size: item.size, // expand fields securityFileStatus: item.securityFileStatus, lastCommit: item.lastCommit ? { date: new Date(item.lastCommit.date), title: item.lastCommit.title, id: item.lastCommit.id } : void 0 })); } // src/lib/repo-exists.ts async function repoExists(params) { const repoId = toRepoId(params.repo); const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}?expand[]=likes`, { method: "GET", headers: { ...params.accessToken && { Authorization: `Bearer ${params.accessToken}` } } } ); if (res.status === 404 || res.status === 401) { return false; } if (!res.ok) { throw await createApiError(res); } return true; } // src/lib/space-info.ts async function spaceInfo(params) { const accessToken = params && checkCredentials(params); const search = new URLSearchParams([ ...SPACE_EXPAND_KEYS.map((val) => ["expand", val]), ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _249 => _249.additionalFields, 'optionalAccess', _250 => _250.map, 'call', _251 => _251((val) => ["expand", val])]), () => ( [])) ]).toString(); const response = await (params.fetch || fetch)( `${_optionalChain([params, 'optionalAccess', _252 => _252.hubUrl]) || HUB_URL}/api/spaces/${params.name}/revision/${encodeURIComponent( _nullishCoalesce(params.revision, () => ( "HEAD")) )}?${search.toString()}`, { headers: { ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} } } ); if (!response.ok) { throw await createApiError(response); } const data = await response.json(); return { ..._optionalChain([params, 'optionalAccess', _253 => _253.additionalFields]) && pick(data, params.additionalFields), id: data._id, name: data.id, sdk: data.sdk, likes: data.likes, private: data.private, updatedAt: new Date(data.lastModified) }; } // src/lib/upload-file.ts function uploadFile(params) { const path = params.file instanceof URL ? _nullishCoalesce(params.file.pathname.split("/").at(-1), () => ( "file")) : "path" in params.file ? params.file.path : params.file.name; return commit({ ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, repo: params.repo, operations: [ { operation: "addOrUpdate", path, content: "content" in params.file ? params.file.content : params.file } ], title: _nullishCoalesce(params.commitTitle, () => ( `Add ${path}`)), description: params.commitDescription, hubUrl: params.hubUrl, branch: params.branch, isPullRequest: params.isPullRequest, parentCommit: params.parentCommit, fetch: params.fetch, useWebWorkers: params.useWebWorkers, abortSignal: params.abortSignal, useXet: params.useXet }); } // src/lib/upload-files.ts function uploadFiles(params) { return commit({ ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, repo: params.repo, operations: params.files.map((file) => ({ operation: "addOrUpdate", path: file instanceof URL ? _nullishCoalesce(file.pathname.split("/").at(-1), () => ( "file")) : "path" in file ? file.path : file.name, content: "content" in file ? file.content : file })), title: _nullishCoalesce(params.commitTitle, () => ( `Add ${params.files.length} files`)), description: params.commitDescription, hubUrl: params.hubUrl, branch: params.branch, isPullRequest: params.isPullRequest, parentCommit: params.parentCommit, fetch: params.fetch, useWebWorkers: params.useWebWorkers, abortSignal: params.abortSignal, useXet: params.useXet }); } // src/lib/upload-files-with-progress.ts var multipartUploadTracking = /* @__PURE__ */ new WeakMap(); async function* uploadFilesWithProgress(params) { return yield* commitIter({ ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, repo: params.repo, operations: params.files.map((file) => ({ operation: "addOrUpdate", path: file instanceof URL ? _nullishCoalesce(file.pathname.split("/").at(-1), () => ( "file")) : "path" in file ? file.path : file.name, content: "content" in file ? file.content : file })), title: _nullishCoalesce(params.commitTitle, () => ( `Add ${params.files.length} files`)), description: params.commitDescription, hubUrl: params.hubUrl, branch: params.branch, isPullRequest: params.isPullRequest, parentCommit: params.parentCommit, useWebWorkers: params.useWebWorkers, abortSignal: params.abortSignal, useXet: params.useXet, fetch: async (input, init) => { if (!init) { return fetch(input); } if (!typedInclude(["PUT", "POST"], init.method) || !("progressHint" in init) || !init.progressHint || typeof XMLHttpRequest === "undefined" || typeof input !== "string" || !(init.body instanceof ArrayBuffer) && !(init.body instanceof Blob) && !(init.body instanceof File) && typeof init.body !== "string") { return fetch(input, init); } const progressHint = init.progressHint; const progressCallback = progressHint.progressCallback; const xhr = new XMLHttpRequest(); xhr.upload.addEventListener("progress", (event) => { if (event.lengthComputable) { if (progressHint.part !== void 0) { let tracking = multipartUploadTracking.get(progressCallback); if (!tracking) { tracking = { numParts: progressHint.numParts, partsProgress: {} }; multipartUploadTracking.set(progressCallback, tracking); } tracking.partsProgress[progressHint.part] = event.loaded / event.total; let totalProgress = 0; for (const partProgress of Object.values(tracking.partsProgress)) { totalProgress += partProgress; } if (totalProgress === tracking.numParts) { progressCallback(0.9999999999); } else { progressCallback(totalProgress / tracking.numParts); } } else { if (event.loaded === event.total) { progressCallback(0.9999999999); } else { progressCallback(event.loaded / event.total); } } } }); xhr.open(init.method, input, true); if (init.headers) { const headers = new Headers(init.headers); headers.forEach((value, key) => { xhr.setRequestHeader(key, value); }); } _optionalChain([init, 'access', _254 => _254.signal, 'optionalAccess', _255 => _255.throwIfAborted, 'call', _256 => _256()]); xhr.send(init.body); return new Promise((resolve2, reject) => { xhr.addEventListener("load", () => { resolve2( new Response(xhr.responseText, { status: xhr.status, statusText: xhr.statusText, headers: Object.fromEntries( xhr.getAllResponseHeaders().trim().split("\n").map((header) => [ header.slice(0, header.indexOf(":")), header.slice(header.indexOf(":") + 1).trim() ]) ) }) ); }); xhr.addEventListener("error", () => { reject(new Error(xhr.statusText)); }); if (init.signal) { init.signal.addEventListener("abort", () => { xhr.abort(); try { _optionalChain([init, 'access', _257 => _257.signal, 'optionalAccess', _258 => _258.throwIfAborted, 'call', _259 => _259()]); } catch (err) { reject(err); } }); } }); } }); } // src/lib/who-am-i.ts async function whoAmI(params) { const accessToken = checkCredentials(params); const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/whoami-v2`, { headers: { Authorization: `Bearer ${accessToken}` } }); if (!res.ok) { throw await createApiError(res); } const response = await res.json(); if (typeof _optionalChain([response, 'access', _260 => _260.auth, 'access', _261 => _261.accessToken, 'optionalAccess', _262 => _262.createdAt]) === "string") { response.auth.accessToken.createdAt = new Date(response.auth.accessToken.createdAt); } return response; } exports.DATASET_EXPANDABLE_KEYS = DATASET_EXPANDABLE_KEYS; exports.DATASET_EXPAND_KEYS = DATASET_EXPAND_KEYS; exports.HUB_URL = HUB_URL; exports.HubApiError = HubApiError; exports.InvalidApiResponseFormatError = InvalidApiResponseFormatError; exports.MODEL_EXPANDABLE_KEYS = MODEL_EXPANDABLE_KEYS; exports.MODEL_EXPAND_KEYS = MODEL_EXPAND_KEYS; exports.RE_SAFETENSORS_FILE = RE_SAFETENSORS_FILE; exports.RE_SAFETENSORS_INDEX_FILE = RE_SAFETENSORS_INDEX_FILE; exports.RE_SAFETENSORS_SHARD_FILE = RE_SAFETENSORS_SHARD_FILE; exports.SAFETENSORS_FILE = SAFETENSORS_FILE; exports.SAFETENSORS_INDEX_FILE = SAFETENSORS_INDEX_FILE; exports.SPACE_EXPANDABLE_KEYS = SPACE_EXPANDABLE_KEYS; exports.SPACE_EXPAND_KEYS = SPACE_EXPAND_KEYS; exports.__internal_XetBlob = XetBlob; exports.__internal_sha256 = sha256; exports.cancelJob = cancelJob; exports.checkRepoAccess = checkRepoAccess; exports.commit = commit; exports.commitIter = commitIter; exports.commitIterBucket = commitIterBucket; exports.countCommits = countCommits; exports.createBranch = createBranch; exports.createCollection = createCollection; exports.createRepo = createRepo; exports.createScheduledJob = createScheduledJob; exports.datasetInfo = datasetInfo; exports.deleteBranch = deleteBranch; exports.deleteCollection = deleteCollection; exports.deleteFile = deleteFile; exports.deleteFiles = deleteFiles; exports.deleteRepo = deleteRepo; exports.deleteScheduledJob = deleteScheduledJob; exports.downloadFile = downloadFile; exports.duplicateJob = duplicateJob; exports.fileDownloadInfo = fileDownloadInfo; exports.fileExists = fileExists; exports.getJob = getJob; exports.getScheduledJob = getScheduledJob; exports.listCollections = listCollections; exports.listCommits = listCommits; exports.listDatasets = listDatasets; exports.listFiles = listFiles; exports.listJobHardware = listJobHardware; exports.listJobs = listJobs; exports.listModels = listModels; exports.listScheduledJobs = listScheduledJobs; exports.listSpaces = listSpaces; exports.modelInfo = modelInfo; exports.oauthHandleRedirect = oauthHandleRedirect; exports.oauthHandleRedirectIfPresent = oauthHandleRedirectIfPresent; exports.oauthLoginUrl = oauthLoginUrl; exports.parseSafetensorsMetadata = parseSafetensorsMetadata; exports.parseSafetensorsShardFilename = parseSafetensorsShardFilename; exports.pathsInfo = pathsInfo; exports.repoExists = repoExists; exports.resumeScheduledJob = resumeScheduledJob; exports.runJob = runJob; exports.runScheduledJob = runScheduledJob; exports.spaceInfo = spaceInfo; exports.streamJobEvents = streamJobEvents; exports.streamJobLogs = streamJobLogs; exports.streamJobMetrics = streamJobMetrics; exports.suspendScheduledJob = suspendScheduledJob; exports.uploadFile = uploadFile; exports.uploadFiles = uploadFiles; exports.uploadFilesWithProgress = uploadFilesWithProgress; exports.whoAmI = whoAmI;