Spaces:
Sleeping
Sleeping
| ;Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { newObj[key] = obj[key]; } } } newObj.default = obj; return newObj; } } function _nullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return rhsFn(); } } async function _asyncNullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return await rhsFn(); } } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; } var _class; var _class2; var _class3;require('./chunk-UICA3PK6.js'); | |
| // src/consts.ts | |
| var HUB_URL = "https://huggingface.co"; | |
| // src/error.ts | |
| async function createApiError(response, opts) { | |
| const error = new HubApiError(response.url, response.status, _nullishCoalesce(response.headers.get("X-Request-Id"), () => ( _optionalChain([opts, 'optionalAccess', _2 => _2.requestId])))); | |
| error.message = `Api error with status ${error.statusCode}${_optionalChain([opts, 'optionalAccess', _3 => _3.message]) ? `. ${opts.message}` : ""}`; | |
| const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : void 0].filter(Boolean).join(". "); | |
| if (_optionalChain([response, 'access', _4 => _4.headers, 'access', _5 => _5.get, 'call', _6 => _6("Content-Type"), 'optionalAccess', _7 => _7.startsWith, 'call', _8 => _8("application/json")])) { | |
| const json = await response.json(); | |
| error.message = json.error || json.message || error.message; | |
| if (json.error_description) { | |
| error.message = error.message ? error.message + `: ${json.error_description}` : json.error_description; | |
| } | |
| error.data = json; | |
| } else { | |
| error.data = { message: await response.text() }; | |
| } | |
| error.message += `. ${trailer}`; | |
| throw error; | |
| } | |
| var HubApiError = class extends Error { | |
| constructor(url, statusCode, requestId, message) { | |
| super(message); | |
| this.statusCode = statusCode; | |
| this.requestId = requestId; | |
| this.url = url; | |
| } | |
| }; | |
| var InvalidApiResponseFormatError = class extends Error { | |
| }; | |
| // src/utils/checkCredentials.ts | |
| function checkAccessToken(accessToken) { | |
| if (!accessToken.startsWith("hf_")) { | |
| throw new TypeError("Your access token must start with 'hf_'"); | |
| } | |
| } | |
| function checkCredentials(params) { | |
| if (params.accessToken) { | |
| checkAccessToken(params.accessToken); | |
| return params.accessToken; | |
| } | |
| if (_optionalChain([params, 'access', _9 => _9.credentials, 'optionalAccess', _10 => _10.accessToken])) { | |
| checkAccessToken(params.credentials.accessToken); | |
| return params.credentials.accessToken; | |
| } | |
| } | |
| // src/utils/toRepoId.ts | |
| function toRepoId(repo) { | |
| if (typeof repo !== "string") { | |
| return repo; | |
| } | |
| if (repo.startsWith("model/") || repo.startsWith("models/")) { | |
| throw new TypeError( | |
| "A repo designation for a model should not start with 'models/', directly specify the model namespace / name" | |
| ); | |
| } | |
| if (repo.startsWith("space/")) { | |
| throw new TypeError("Spaces should start with 'spaces/', plural, not 'space/'"); | |
| } | |
| if (repo.startsWith("dataset/")) { | |
| throw new TypeError("Datasets should start with 'datasets/', plural, not 'dataset/'"); | |
| } | |
| if (repo.startsWith("bucket/")) { | |
| throw new TypeError("Buckets should start with 'buckets/', plural, not 'bucket/'"); | |
| } | |
| const slashes = repo.split("/").length - 1; | |
| if (repo.startsWith("spaces/")) { | |
| if (slashes !== 2) { | |
| throw new TypeError("Space Id must include namespace and name of the space"); | |
| } | |
| return { | |
| type: "space", | |
| name: repo.slice("spaces/".length) | |
| }; | |
| } | |
| if (repo.startsWith("datasets/")) { | |
| if (slashes > 2) { | |
| throw new TypeError("Too many slashes in repo designation: " + repo); | |
| } | |
| return { | |
| type: "dataset", | |
| name: repo.slice("datasets/".length) | |
| }; | |
| } | |
| if (repo.startsWith("buckets/")) { | |
| if (slashes !== 2) { | |
| throw new TypeError("Bucket Id must include namespace and name of the bucket"); | |
| } | |
| return { | |
| type: "bucket", | |
| name: repo.slice("buckets/".length) | |
| }; | |
| } | |
| if (slashes > 1) { | |
| throw new TypeError("Too many slashes in repo designation: " + repo); | |
| } | |
| return { | |
| type: "model", | |
| name: repo | |
| }; | |
| } | |
| // src/lib/check-repo-access.ts | |
| async function checkRepoAccess(params) { | |
| const accessToken = params && checkCredentials(params); | |
| const repoId = toRepoId(params.repo); | |
| const response = await (params.fetch || fetch)(`${_optionalChain([params, 'optionalAccess', _11 => _11.hubUrl]) || HUB_URL}/api/${repoId.type}s/${repoId.name}`, { | |
| headers: { | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| }); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| } | |
| // src/utils/range.ts | |
| function range(n, b) { | |
| return b ? Array(b - n).fill(0).map((_, i) => n + i) : Array(n).fill(0).map((_, i) => i); | |
| } | |
| // src/utils/chunk.ts | |
| function chunk(arr, chunkSize) { | |
| if (isNaN(chunkSize) || chunkSize < 1) { | |
| throw new RangeError("Invalid chunk size: " + chunkSize); | |
| } | |
| if (!arr.length) { | |
| return []; | |
| } | |
| if (arr.length <= chunkSize) { | |
| return [arr]; | |
| } | |
| return range(Math.ceil(arr.length / chunkSize)).map((i) => { | |
| return arr.slice(i * chunkSize, (i + 1) * chunkSize); | |
| }); | |
| } | |
| // src/utils/promisesQueue.ts | |
| async function promisesQueue(factories, concurrency) { | |
| const results = []; | |
| const executing = /* @__PURE__ */ new Set(); | |
| let index = 0; | |
| for (const factory of factories) { | |
| const closureIndex = index++; | |
| const e = factory().then((r) => { | |
| results[closureIndex] = r; | |
| executing.delete(e); | |
| }); | |
| executing.add(e); | |
| if (executing.size >= concurrency) { | |
| await Promise.race(executing); | |
| } | |
| } | |
| await Promise.all(executing); | |
| return results; | |
| } | |
| // src/utils/promisesQueueStreaming.ts | |
| async function promisesQueueStreaming(factories, concurrency) { | |
| const executing = []; | |
| for await (const factory of factories) { | |
| const e = factory().then(() => { | |
| executing.splice(executing.indexOf(e), 1); | |
| }); | |
| executing.push(e); | |
| if (executing.length >= concurrency) { | |
| await Promise.race(executing); | |
| } | |
| } | |
| await Promise.all(executing); | |
| } | |
| // src/utils/eventToGenerator.ts | |
| async function* eventToGenerator(cb) { | |
| const promises = []; | |
| function addPromise() { | |
| let resolve2; | |
| let reject; | |
| const p = new Promise((res, rej) => { | |
| resolve2 = res; | |
| reject = rej; | |
| }); | |
| promises.push({ p, resolve: resolve2, reject }); | |
| } | |
| addPromise(); | |
| const callbackRes = Promise.resolve().then( | |
| () => cb( | |
| (y) => { | |
| addPromise(); | |
| _optionalChain([promises, 'access', _12 => _12.at, 'call', _13 => _13(-2), 'optionalAccess', _14 => _14.resolve, 'call', _15 => _15({ done: false, value: y })]); | |
| }, | |
| (r) => { | |
| addPromise(); | |
| _optionalChain([promises, 'access', _16 => _16.at, 'call', _17 => _17(-2), 'optionalAccess', _18 => _18.resolve, 'call', _19 => _19({ done: true, value: r })]); | |
| }, | |
| (err) => _optionalChain([promises, 'access', _20 => _20.shift, 'call', _21 => _21(), 'optionalAccess', _22 => _22.reject, 'call', _23 => _23(err)]) | |
| ) | |
| ).catch((err) => _optionalChain([promises, 'access', _24 => _24.shift, 'call', _25 => _25(), 'optionalAccess', _26 => _26.reject, 'call', _27 => _27(err)])); | |
| while (1) { | |
| const p = promises[0]; | |
| if (!p) { | |
| throw new Error("Logic error in eventGenerator, promises should never be empty"); | |
| } | |
| const result = await p.p; | |
| promises.shift(); | |
| if (result.done) { | |
| await callbackRes; | |
| return result.value; | |
| } | |
| yield result.value; | |
| } | |
| throw new Error("Unreachable"); | |
| } | |
| // src/utils/hexFromBytes.ts | |
| function hexFromBytes(arr) { | |
| if (globalThis.Buffer) { | |
| return globalThis.Buffer.from(arr).toString("hex"); | |
| } else { | |
| const bin = []; | |
| arr.forEach((byte) => { | |
| bin.push(byte.toString(16).padStart(2, "0")); | |
| }); | |
| return bin.join(""); | |
| } | |
| } | |
| // src/utils/isBackend.ts | |
| var isBrowser = typeof window !== "undefined" && typeof window.document !== "undefined"; | |
| var isWebWorker = typeof self === "object" && self.constructor && self.constructor.name === "DedicatedWorkerGlobalScope"; | |
| var isBackend = !isBrowser && !isWebWorker; | |
| // src/utils/isFrontend.ts | |
| var isFrontend = !isBackend; | |
| // src/utils/sha256.ts | |
| async function getWebWorkerCode() { | |
| const sha256Module = await Promise.resolve().then(() => _interopRequireWildcard(require("./sha256-wrapper-DHTT2DPH.js"))); | |
| return URL.createObjectURL(new Blob([sha256Module.createSHA256WorkerCode()])); | |
| } | |
| var pendingWorkers = []; | |
| var runningWorkers = /* @__PURE__ */ new Set(); | |
| var resolve; | |
| var waitPromise = new Promise((r) => { | |
| resolve = r; | |
| }); | |
| async function getWorker(poolSize) { | |
| { | |
| const worker2 = pendingWorkers.pop(); | |
| if (worker2) { | |
| runningWorkers.add(worker2); | |
| return worker2; | |
| } | |
| } | |
| if (!poolSize) { | |
| const worker2 = new Worker(await getWebWorkerCode()); | |
| runningWorkers.add(worker2); | |
| return worker2; | |
| } | |
| if (poolSize <= 0) { | |
| throw new TypeError("Invalid webworker pool size: " + poolSize); | |
| } | |
| while (runningWorkers.size >= poolSize) { | |
| await waitPromise; | |
| } | |
| const worker = new Worker(await getWebWorkerCode()); | |
| runningWorkers.add(worker); | |
| return worker; | |
| } | |
| async function freeWorker(worker, poolSize) { | |
| if (!poolSize) { | |
| return destroyWorker(worker); | |
| } | |
| runningWorkers.delete(worker); | |
| pendingWorkers.push(worker); | |
| const r = resolve; | |
| waitPromise = new Promise((r2) => { | |
| resolve = r2; | |
| }); | |
| r(); | |
| } | |
| function destroyWorker(worker) { | |
| runningWorkers.delete(worker); | |
| worker.terminate(); | |
| const r = resolve; | |
| waitPromise = new Promise((r2) => { | |
| resolve = r2; | |
| }); | |
| r(); | |
| } | |
| async function* sha256(buffer, opts) { | |
| yield 0; | |
| const maxCryptoSize = typeof _optionalChain([opts, 'optionalAccess', _28 => _28.useWebWorker]) === "object" && _optionalChain([opts, 'optionalAccess', _29 => _29.useWebWorker, 'access', _30 => _30.minSize]) !== void 0 ? opts.useWebWorker.minSize : 1e7; | |
| if (buffer.size < maxCryptoSize && _optionalChain([globalThis, 'access', _31 => _31.crypto, 'optionalAccess', _32 => _32.subtle])) { | |
| const res = hexFromBytes( | |
| new Uint8Array( | |
| await globalThis.crypto.subtle.digest("SHA-256", buffer instanceof Blob ? await buffer.arrayBuffer() : buffer) | |
| ) | |
| ); | |
| yield 1; | |
| return res; | |
| } | |
| if (isFrontend) { | |
| if (_optionalChain([opts, 'optionalAccess', _33 => _33.useWebWorker])) { | |
| try { | |
| const poolSize = typeof _optionalChain([opts, 'optionalAccess', _34 => _34.useWebWorker]) === "object" ? opts.useWebWorker.poolSize : void 0; | |
| const worker = await getWorker(poolSize); | |
| let messageHandler; | |
| let errorHandler; | |
| const cleanup = () => { | |
| worker.removeEventListener("message", messageHandler); | |
| worker.removeEventListener("error", errorHandler); | |
| }; | |
| return yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { | |
| messageHandler = (event) => { | |
| if (event.data.sha256) { | |
| cleanup(); | |
| freeWorker(worker, poolSize); | |
| returnCallback(event.data.sha256); | |
| } else if (event.data.progress) { | |
| yieldCallback(event.data.progress); | |
| try { | |
| _optionalChain([opts, 'access', _35 => _35.abortSignal, 'optionalAccess', _36 => _36.throwIfAborted, 'call', _37 => _37()]); | |
| } catch (err) { | |
| cleanup(); | |
| destroyWorker(worker); | |
| rejectCallback(err); | |
| } | |
| } else { | |
| cleanup(); | |
| destroyWorker(worker); | |
| rejectCallback(event); | |
| } | |
| }; | |
| errorHandler = (event) => { | |
| cleanup(); | |
| destroyWorker(worker); | |
| rejectCallback(event.error); | |
| }; | |
| if (_optionalChain([opts, 'optionalAccess', _38 => _38.abortSignal])) { | |
| try { | |
| _optionalChain([opts, 'access', _39 => _39.abortSignal, 'optionalAccess', _40 => _40.throwIfAborted, 'call', _41 => _41()]); | |
| } catch (err) { | |
| cleanup(); | |
| destroyWorker(worker); | |
| rejectCallback(_nullishCoalesce(opts.abortSignal.reason, () => ( new DOMException("Aborted", "AbortError")))); | |
| return; | |
| } | |
| const abortListener = () => { | |
| cleanup(); | |
| destroyWorker(worker); | |
| rejectCallback(_nullishCoalesce(_optionalChain([opts, 'access', _42 => _42.abortSignal, 'optionalAccess', _43 => _43.reason]), () => ( new DOMException("Aborted", "AbortError")))); | |
| _optionalChain([opts, 'access', _44 => _44.abortSignal, 'optionalAccess', _45 => _45.removeEventListener, 'call', _46 => _46("abort", abortListener)]); | |
| }; | |
| opts.abortSignal.addEventListener("abort", abortListener); | |
| } | |
| worker.addEventListener("message", messageHandler); | |
| worker.addEventListener("error", errorHandler); | |
| worker.postMessage({ file: buffer }); | |
| }); | |
| } catch (err) { | |
| console.warn("Failed to use web worker for sha256", err); | |
| } | |
| } | |
| if (!wasmModule) { | |
| wasmModule = await Promise.resolve().then(() => _interopRequireWildcard(require("./sha256-wrapper-DHTT2DPH.js"))); | |
| } | |
| const sha2562 = await wasmModule.createSHA256(); | |
| sha2562.init(); | |
| const reader = buffer.stream().getReader(); | |
| const total = buffer.size; | |
| let bytesDone = 0; | |
| while (true) { | |
| const { done, value } = await reader.read(); | |
| if (done) { | |
| break; | |
| } | |
| sha2562.update(value); | |
| bytesDone += value.length; | |
| yield bytesDone / total; | |
| _optionalChain([opts, 'optionalAccess', _47 => _47.abortSignal, 'optionalAccess', _48 => _48.throwIfAborted, 'call', _49 => _49()]); | |
| } | |
| return sha2562.digest("hex"); | |
| } | |
| if (!cryptoModule) { | |
| cryptoModule = await Promise.resolve().then(() => _interopRequireWildcard(require("./sha256-node-FT2Y3VXD.js"))); | |
| } | |
| return yield* cryptoModule.sha256Node(buffer, { abortSignal: _optionalChain([opts, 'optionalAccess', _50 => _50.abortSignal]) }); | |
| } | |
| var cryptoModule; | |
| var wasmModule; | |
| // src/utils/WebBlob.ts | |
| var WebBlob = class extends Blob { | |
| static async create(url, opts) { | |
| const customFetch = _nullishCoalesce(_optionalChain([opts, 'optionalAccess', _51 => _51.fetch]), () => ( fetch)); | |
| const response = await customFetch(url, { | |
| method: "HEAD", | |
| ..._optionalChain([opts, 'optionalAccess', _52 => _52.accessToken]) && { | |
| headers: { | |
| Authorization: `Bearer ${opts.accessToken}` | |
| } | |
| } | |
| }); | |
| const size = Number(response.headers.get("content-length")); | |
| const contentType = response.headers.get("content-type") || ""; | |
| const supportRange = response.headers.get("accept-ranges") === "bytes"; | |
| if (!supportRange || size < (_nullishCoalesce(_optionalChain([opts, 'optionalAccess', _53 => _53.cacheBelow]), () => ( 1e6)))) { | |
| return await (await customFetch(url)).blob(); | |
| } | |
| return new WebBlob(url, 0, size, contentType, true, customFetch, _optionalChain([opts, 'optionalAccess', _54 => _54.accessToken])); | |
| } | |
| constructor(url, start, end, contentType, full, customFetch, accessToken) { | |
| super([]); | |
| this.url = url; | |
| this.start = start; | |
| this.end = end; | |
| this.contentType = contentType; | |
| this.full = full; | |
| this.fetch = customFetch; | |
| this.accessToken = accessToken; | |
| } | |
| get size() { | |
| return this.end - this.start; | |
| } | |
| get type() { | |
| return this.contentType; | |
| } | |
| slice(start = 0, end = this.size) { | |
| if (start < 0 || end < 0) { | |
| new TypeError("Unsupported negative start/end on WebBlob.slice"); | |
| } | |
| const slice = new WebBlob( | |
| this.url, | |
| this.start + start, | |
| Math.min(this.start + end, this.end), | |
| this.contentType, | |
| start === 0 && end === this.size ? this.full : false, | |
| this.fetch, | |
| this.accessToken | |
| ); | |
| return slice; | |
| } | |
| async arrayBuffer() { | |
| const result = await this.fetchRange(); | |
| return result.arrayBuffer(); | |
| } | |
| async text() { | |
| const result = await this.fetchRange(); | |
| return result.text(); | |
| } | |
| stream() { | |
| const stream = new TransformStream(); | |
| this.fetchRange().then((response) => _optionalChain([response, 'access', _55 => _55.body, 'optionalAccess', _56 => _56.pipeThrough, 'call', _57 => _57(stream)])).catch((error) => stream.writable.abort(error.message)); | |
| return stream.readable; | |
| } | |
| fetchRange() { | |
| const fetch2 = this.fetch; | |
| if (this.full) { | |
| return fetch2(this.url, { | |
| ...this.accessToken && { | |
| headers: { | |
| Authorization: `Bearer ${this.accessToken}` | |
| } | |
| } | |
| }).then((resp) => resp.ok ? resp : createApiError(resp)); | |
| } | |
| return fetch2(this.url, { | |
| headers: { | |
| Range: `bytes=${this.start}-${this.end - 1}`, | |
| ...this.accessToken && { Authorization: `Bearer ${this.accessToken}` } | |
| } | |
| }).then((resp) => resp.ok ? resp : createApiError(resp)); | |
| } | |
| }; | |
| // src/utils/base64FromBytes.ts | |
| function base64FromBytes(arr) { | |
| if (globalThis.Buffer) { | |
| return globalThis.Buffer.from(arr).toString("base64"); | |
| } else { | |
| const bin = []; | |
| arr.forEach((byte) => { | |
| bin.push(String.fromCharCode(byte)); | |
| }); | |
| return globalThis.btoa(bin.join("")); | |
| } | |
| } | |
| // src/utils/createBlobs.ts | |
| async function createBlobs(url, destPath, opts) { | |
| if (url.protocol === "http:" || url.protocol === "https:") { | |
| const blob = await WebBlob.create(url, { fetch: _optionalChain([opts, 'optionalAccess', _58 => _58.fetch]), accessToken: _optionalChain([opts, 'optionalAccess', _59 => _59.accessToken]) }); | |
| return [{ path: destPath, blob }]; | |
| } | |
| if (isFrontend) { | |
| throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); | |
| } | |
| if (url.protocol === "file:") { | |
| const { FileBlob } = await Promise.resolve().then(() => _interopRequireWildcard(require("./FileBlob-YC2EPDW4.js"))); | |
| const { subPaths } = await Promise.resolve().then(() => _interopRequireWildcard(require("./sub-paths-RH3O65LG.js"))); | |
| const paths = await subPaths(url, _optionalChain([opts, 'optionalAccess', _60 => _60.maxFolderDepth])); | |
| if (paths.length === 1 && paths[0].relativePath === ".") { | |
| const blob = await FileBlob.create(url); | |
| return [{ path: destPath, blob }]; | |
| } | |
| return Promise.all( | |
| paths.map(async (path) => ({ | |
| path: `${destPath}/${path.relativePath}`.replace(/\/[.]$/, "").replaceAll("//", "/").replace(/^[.]?\//, ""), | |
| blob: await FileBlob.create(new URL(path.path)) | |
| })) | |
| ); | |
| } | |
| throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); | |
| } | |
| // src/utils/combineUint8Arrays.ts | |
| function combineUint8Arrays(a, b) { | |
| const aLength = a.length; | |
| const combinedBytes = new Uint8Array(aLength + b.length); | |
| combinedBytes.set(a); | |
| combinedBytes.set(b, aLength); | |
| return combinedBytes; | |
| } | |
| // src/vendor/lz4js/util.ts | |
| function hashU32(a) { | |
| a = a | 0; | |
| a = a + 2127912214 + (a << 12) | 0; | |
| a = a ^ -949894596 ^ a >>> 19; | |
| a = a + 374761393 + (a << 5) | 0; | |
| a = a + -744332180 ^ a << 9; | |
| a = a + -42973499 + (a << 3) | 0; | |
| return a ^ -1252372727 ^ a >>> 16 | 0; | |
| } | |
| function readU64(b, n) { | |
| let x = 0; | |
| x |= b[n++] << 0; | |
| x |= b[n++] << 8; | |
| x |= b[n++] << 16; | |
| x |= b[n++] << 24; | |
| x |= b[n++] << 32; | |
| x |= b[n++] << 40; | |
| x |= b[n++] << 48; | |
| x |= b[n++] << 56; | |
| return x; | |
| } | |
| function readU32(b, n) { | |
| let x = 0; | |
| x |= b[n++] << 0; | |
| x |= b[n++] << 8; | |
| x |= b[n++] << 16; | |
| x |= b[n++] << 24; | |
| return x; | |
| } | |
| function writeU32(b, n, x) { | |
| b[n++] = x >> 0 & 255; | |
| b[n++] = x >> 8 & 255; | |
| b[n++] = x >> 16 & 255; | |
| b[n++] = x >> 24 & 255; | |
| } | |
| function imul(a, b) { | |
| const ah = a >>> 16; | |
| const al = a & 65535; | |
| const bh = b >>> 16; | |
| const bl = b & 65535; | |
| return al * bl + (ah * bl + al * bh << 16) | 0; | |
| } | |
| // src/vendor/lz4js/xxh32.ts | |
| var prime1 = 2654435761; | |
| var prime2 = 2246822519; | |
| var prime3 = 3266489917; | |
| var prime4 = 668265263; | |
| var prime5 = 374761393; | |
| function rotl32(x, r) { | |
| x = x | 0; | |
| r = r | 0; | |
| return x >>> (32 - r | 0) | x << r | 0; | |
| } | |
| function rotmul32(h, r, m) { | |
| h = h | 0; | |
| r = r | 0; | |
| m = m | 0; | |
| return imul(h >>> (32 - r | 0) | h << r, m) | 0; | |
| } | |
| function shiftxor32(h, s) { | |
| h = h | 0; | |
| s = s | 0; | |
| return h >>> s ^ h | 0; | |
| } | |
| function xxhapply(h, src, m0, s, m1) { | |
| return rotmul32(imul(src, m0) + h, s, m1); | |
| } | |
| function xxh1(h, src, index) { | |
| return rotmul32(h + imul(src[index], prime5), 11, prime1); | |
| } | |
| function xxh4(h, src, index) { | |
| return xxhapply(h, readU32(src, index), prime3, 17, prime4); | |
| } | |
| function xxh16(h, src, index) { | |
| return [ | |
| xxhapply(h[0], readU32(src, index + 0), prime2, 13, prime1), | |
| xxhapply(h[1], readU32(src, index + 4), prime2, 13, prime1), | |
| xxhapply(h[2], readU32(src, index + 8), prime2, 13, prime1), | |
| xxhapply(h[3], readU32(src, index + 12), prime2, 13, prime1) | |
| ]; | |
| } | |
| function xxh32(seed, src, index, len) { | |
| let h; | |
| const l = len; | |
| if (len >= 16) { | |
| h = [seed + prime1 + prime2, seed + prime2, seed, seed - prime1]; | |
| while (len >= 16) { | |
| h = xxh16(h, src, index); | |
| index += 16; | |
| len -= 16; | |
| } | |
| h = rotl32(h[0], 1) + rotl32(h[1], 7) + rotl32(h[2], 12) + rotl32(h[3], 18) + l; | |
| } else { | |
| h = seed + prime5 + len >>> 0; | |
| } | |
| while (len >= 4) { | |
| h = xxh4(h, src, index); | |
| index += 4; | |
| len -= 4; | |
| } | |
| while (len > 0) { | |
| h = xxh1(h, src, index); | |
| index++; | |
| len--; | |
| } | |
| h = shiftxor32(imul(shiftxor32(imul(shiftxor32(h, 15), prime2), 13), prime3), 16); | |
| return h >>> 0; | |
| } | |
| var hash = xxh32; | |
| // src/vendor/lz4js/index.ts | |
| var minMatch = 4; | |
| var matchSearchLimit = 12; | |
| var minTrailingLitterals = 5; | |
| var skipTrigger = 6; | |
| var hashSize = 1 << 16; | |
| var mlBits = 4; | |
| var mlMask = (1 << mlBits) - 1; | |
| var runBits = 4; | |
| var runMask = (1 << runBits) - 1; | |
| var blockBuf = makeBuffer(5 << 20); | |
| var hashTable = makeHashTable(); | |
| var magicNum = 407708164; | |
| var fdContentChksum = 4; | |
| var fdContentSize = 8; | |
| var fdBlockChksum = 16; | |
| var fdVersion = 64; | |
| var fdVersionMask = 192; | |
| var bsUncompressed = 2147483648; | |
| var bsDefault = 7; | |
| var bsShift = 4; | |
| var bsMask = 7; | |
| var bsMap = { | |
| 4: 65536, | |
| 5: 262144, | |
| 6: 1048576, | |
| 7: 4194304 | |
| }; | |
| function makeHashTable() { | |
| try { | |
| return new Uint32Array(hashSize); | |
| } catch (error) { | |
| const hashTable2 = new Array(hashSize); | |
| for (let i = 0; i < hashSize; i++) { | |
| hashTable2[i] = 0; | |
| } | |
| return hashTable2; | |
| } | |
| } | |
| function clearHashTable(table) { | |
| for (let i = 0; i < hashSize; i++) { | |
| table[i] = 0; | |
| } | |
| } | |
| function makeBuffer(size) { | |
| return new Uint8Array(size); | |
| } | |
| function sliceArray(array, start, end) { | |
| return array.slice(start, end); | |
| } | |
| function compressBound(n) { | |
| return n + n / 255 + 16 | 0; | |
| } | |
| function decompressBound(src) { | |
| let sIndex = 0; | |
| if (readU32(src, sIndex) !== magicNum) { | |
| throw new Error("invalid magic number"); | |
| } | |
| sIndex += 4; | |
| const descriptor = src[sIndex++]; | |
| if ((descriptor & fdVersionMask) !== fdVersion) { | |
| throw new Error("incompatible descriptor version " + (descriptor & fdVersionMask)); | |
| } | |
| const useBlockSum = (descriptor & fdBlockChksum) !== 0; | |
| const useContentSize = (descriptor & fdContentSize) !== 0; | |
| const bsIdx = src[sIndex++] >> bsShift & bsMask; | |
| if (bsMap[bsIdx] === void 0) { | |
| throw new Error("invalid block size " + bsIdx); | |
| } | |
| const maxBlockSize = bsMap[bsIdx]; | |
| if (useContentSize) { | |
| return readU64(src, sIndex); | |
| } | |
| sIndex++; | |
| let maxSize = 0; | |
| while (true) { | |
| let blockSize = readU32(src, sIndex); | |
| sIndex += 4; | |
| if (blockSize & bsUncompressed) { | |
| blockSize &= ~bsUncompressed; | |
| maxSize += blockSize; | |
| } else if (blockSize > 0) { | |
| maxSize += maxBlockSize; | |
| } | |
| if (blockSize === 0) { | |
| return maxSize; | |
| } | |
| if (useBlockSum) { | |
| sIndex += 4; | |
| } | |
| sIndex += blockSize; | |
| } | |
| } | |
| function decompressBlock(src, dst, sIndex, sLength, dIndex) { | |
| let mLength, mOffset, sEnd, n, i; | |
| const hasCopyWithin = dst.copyWithin !== void 0 && dst.fill !== void 0; | |
| sEnd = sIndex + sLength; | |
| while (sIndex < sEnd) { | |
| const token = src[sIndex++]; | |
| let literalCount = token >> 4; | |
| if (literalCount > 0) { | |
| if (literalCount === 15) { | |
| while (true) { | |
| literalCount += src[sIndex]; | |
| if (src[sIndex++] !== 255) { | |
| break; | |
| } | |
| } | |
| } | |
| for (n = sIndex + literalCount; sIndex < n; ) { | |
| dst[dIndex++] = src[sIndex++]; | |
| } | |
| } | |
| if (sIndex >= sEnd) { | |
| break; | |
| } | |
| mLength = token & 15; | |
| mOffset = src[sIndex++] | src[sIndex++] << 8; | |
| if (mLength === 15) { | |
| while (true) { | |
| mLength += src[sIndex]; | |
| if (src[sIndex++] !== 255) { | |
| break; | |
| } | |
| } | |
| } | |
| mLength += minMatch; | |
| if (hasCopyWithin && mOffset === 1) { | |
| dst.fill(dst[dIndex - 1] | 0, dIndex, dIndex + mLength); | |
| dIndex += mLength; | |
| } else if (hasCopyWithin && mOffset > mLength && mLength > 31) { | |
| dst.copyWithin(dIndex, dIndex - mOffset, dIndex - mOffset + mLength); | |
| dIndex += mLength; | |
| } else { | |
| for (i = dIndex - mOffset, n = i + mLength; i < n; ) { | |
| dst[dIndex++] = dst[i++] | 0; | |
| } | |
| } | |
| } | |
| return dIndex; | |
| } | |
| function compressBlock(src, dst, sIndex, sLength, hashTable2) { | |
| let mIndex, mAnchor, mLength, mOffset, mStep; | |
| let literalCount, dIndex, sEnd, n; | |
| dIndex = 0; | |
| sEnd = sLength + sIndex; | |
| mAnchor = sIndex; | |
| let searchMatchCount = (1 << skipTrigger) + 3; | |
| while (sIndex <= sEnd - matchSearchLimit) { | |
| const seq = readU32(src, sIndex); | |
| let hash2 = hashU32(seq) >>> 0; | |
| hash2 = (hash2 >> 16 ^ hash2) >>> 0 & 65535; | |
| mIndex = hashTable2[hash2] - 1; | |
| hashTable2[hash2] = sIndex + 1; | |
| if (mIndex < 0 || sIndex - mIndex >>> 16 > 0 || readU32(src, mIndex) !== seq) { | |
| mStep = searchMatchCount++ >> skipTrigger; | |
| sIndex += mStep; | |
| continue; | |
| } | |
| searchMatchCount = (1 << skipTrigger) + 3; | |
| literalCount = sIndex - mAnchor; | |
| mOffset = sIndex - mIndex; | |
| sIndex += minMatch; | |
| mIndex += minMatch; | |
| mLength = sIndex; | |
| while (sIndex < sEnd - minTrailingLitterals && src[sIndex] === src[mIndex]) { | |
| sIndex++; | |
| mIndex++; | |
| } | |
| mLength = sIndex - mLength; | |
| const token = mLength < mlMask ? mLength : mlMask; | |
| if (literalCount >= runMask) { | |
| dst[dIndex++] = (runMask << mlBits) + token; | |
| for (n = literalCount - runMask; n >= 255; n -= 255) { | |
| dst[dIndex++] = 255; | |
| } | |
| dst[dIndex++] = n; | |
| } else { | |
| dst[dIndex++] = (literalCount << mlBits) + token; | |
| } | |
| for (let i = 0; i < literalCount; i++) { | |
| dst[dIndex++] = src[mAnchor + i]; | |
| } | |
| dst[dIndex++] = mOffset; | |
| dst[dIndex++] = mOffset >> 8; | |
| if (mLength >= mlMask) { | |
| for (n = mLength - mlMask; n >= 255; n -= 255) { | |
| dst[dIndex++] = 255; | |
| } | |
| dst[dIndex++] = n; | |
| } | |
| mAnchor = sIndex; | |
| } | |
| if (mAnchor === 0) { | |
| return 0; | |
| } | |
| literalCount = sEnd - mAnchor; | |
| if (literalCount >= runMask) { | |
| dst[dIndex++] = runMask << mlBits; | |
| for (n = literalCount - runMask; n >= 255; n -= 255) { | |
| dst[dIndex++] = 255; | |
| } | |
| dst[dIndex++] = n; | |
| } else { | |
| dst[dIndex++] = literalCount << mlBits; | |
| } | |
| sIndex = mAnchor; | |
| while (sIndex < sEnd) { | |
| dst[dIndex++] = src[sIndex++]; | |
| } | |
| return dIndex; | |
| } | |
| function decompressFrame(src, dst) { | |
| let useBlockSum, useContentSum, useContentSize, descriptor; | |
| let sIndex = 0; | |
| let dIndex = 0; | |
| if (readU32(src, sIndex) !== magicNum) { | |
| throw new Error("invalid magic number"); | |
| } | |
| sIndex += 4; | |
| descriptor = src[sIndex++]; | |
| if ((descriptor & fdVersionMask) !== fdVersion) { | |
| throw new Error("incompatible descriptor version"); | |
| } | |
| useBlockSum = (descriptor & fdBlockChksum) !== 0; | |
| useContentSum = (descriptor & fdContentChksum) !== 0; | |
| useContentSize = (descriptor & fdContentSize) !== 0; | |
| const bsIdx = src[sIndex++] >> bsShift & bsMask; | |
| if (bsMap[bsIdx] === void 0) { | |
| throw new Error("invalid block size"); | |
| } | |
| if (useContentSize) { | |
| sIndex += 8; | |
| } | |
| sIndex++; | |
| while (true) { | |
| var compSize; | |
| compSize = readU32(src, sIndex); | |
| sIndex += 4; | |
| if (compSize === 0) { | |
| break; | |
| } | |
| if (useBlockSum) { | |
| sIndex += 4; | |
| } | |
| if ((compSize & bsUncompressed) !== 0) { | |
| compSize &= ~bsUncompressed; | |
| for (let j = 0; j < compSize; j++) { | |
| dst[dIndex++] = src[sIndex++]; | |
| } | |
| } else { | |
| dIndex = decompressBlock(src, dst, sIndex, compSize, dIndex); | |
| sIndex += compSize; | |
| } | |
| } | |
| if (useContentSum) { | |
| sIndex += 4; | |
| } | |
| return dIndex; | |
| } | |
| function compressFrame(src, dst) { | |
| let dIndex = 0; | |
| writeU32(dst, dIndex, magicNum); | |
| dIndex += 4; | |
| dst[dIndex++] = fdVersion; | |
| dst[dIndex++] = bsDefault << bsShift; | |
| dst[dIndex] = hash(0, dst, 4, dIndex - 4) >> 8; | |
| dIndex++; | |
| const maxBlockSize = bsMap[bsDefault]; | |
| let remaining = src.length; | |
| let sIndex = 0; | |
| clearHashTable(hashTable); | |
| while (remaining > 0) { | |
| let compSize = 0; | |
| const blockSize = remaining > maxBlockSize ? maxBlockSize : remaining; | |
| compSize = compressBlock(src, blockBuf, sIndex, blockSize, hashTable); | |
| if (compSize > blockSize || compSize === 0) { | |
| writeU32(dst, dIndex, 2147483648 | blockSize); | |
| dIndex += 4; | |
| for (let z = sIndex + blockSize; sIndex < z; ) { | |
| dst[dIndex++] = src[sIndex++]; | |
| } | |
| remaining -= blockSize; | |
| } else { | |
| writeU32(dst, dIndex, compSize); | |
| dIndex += 4; | |
| for (let j = 0; j < compSize; ) { | |
| dst[dIndex++] = blockBuf[j++]; | |
| } | |
| sIndex += blockSize; | |
| remaining -= blockSize; | |
| } | |
| } | |
| writeU32(dst, dIndex, 0); | |
| dIndex += 4; | |
| return dIndex; | |
| } | |
| function decompress(src, maxSize) { | |
| let dst, size; | |
| if (maxSize === void 0) { | |
| maxSize = decompressBound(src); | |
| } | |
| dst = makeBuffer(maxSize); | |
| size = decompressFrame(src, dst); | |
| if (size !== maxSize) { | |
| dst = sliceArray(dst, 0, size); | |
| } | |
| return dst; | |
| } | |
| function compress(src, maxSize) { | |
| let dst, size; | |
| if (maxSize === void 0) { | |
| maxSize = compressBound(src.length); | |
| } | |
| dst = makeBuffer(maxSize); | |
| size = compressFrame(src, dst); | |
| if (size !== maxSize) { | |
| dst = sliceArray(dst, 0, size); | |
| } | |
| return dst; | |
| } | |
| // src/utils/RangeList.ts | |
| var RangeList = (_class = class {constructor() { _class.prototype.__init.call(this); } | |
| __init() {this.ranges = []} | |
| /** | |
| * Add a range to the list. If it overlaps with existing ranges, | |
| * it will split them and increment reference counts accordingly. | |
| */ | |
| add(start, end) { | |
| if (end <= start) { | |
| throw new TypeError("End must be greater than start"); | |
| } | |
| const overlappingRanges = []; | |
| for (let i = 0; i < this.ranges.length; i++) { | |
| const range2 = this.ranges[i]; | |
| if (start < range2.end && end > range2.start) { | |
| overlappingRanges.push({ index: i, range: range2 }); | |
| } | |
| if (range2.data !== null) { | |
| throw new Error("Overlapping range already has data"); | |
| } | |
| } | |
| if (overlappingRanges.length === 0) { | |
| this.ranges.push({ start, end, refCount: 1, data: null }); | |
| this.ranges.sort((a, b) => a.start - b.start); | |
| return; | |
| } | |
| const newRanges = []; | |
| let currentPos = start; | |
| for (let i = 0; i < overlappingRanges.length; i++) { | |
| const { range: range2 } = overlappingRanges[i]; | |
| if (currentPos < range2.start) { | |
| newRanges.push({ | |
| start: currentPos, | |
| end: range2.start, | |
| refCount: 1, | |
| data: null | |
| }); | |
| } else if (range2.start < currentPos) { | |
| newRanges.push({ | |
| start: range2.start, | |
| end: currentPos, | |
| refCount: range2.refCount, | |
| data: null | |
| }); | |
| } | |
| newRanges.push({ | |
| start: Math.max(currentPos, range2.start), | |
| end: Math.min(end, range2.end), | |
| refCount: range2.refCount + 1, | |
| data: null | |
| }); | |
| if (range2.end > end) { | |
| newRanges.push({ | |
| start: end, | |
| end: range2.end, | |
| refCount: range2.refCount, | |
| data: null | |
| }); | |
| } | |
| currentPos = Math.max(currentPos, range2.end); | |
| } | |
| if (currentPos < end) { | |
| newRanges.push({ | |
| start: currentPos, | |
| end, | |
| refCount: 1, | |
| data: null | |
| }); | |
| } | |
| const firstIndex = overlappingRanges[0].index; | |
| const lastIndex = overlappingRanges[overlappingRanges.length - 1].index; | |
| this.ranges.splice(firstIndex, lastIndex - firstIndex + 1, ...newRanges); | |
| this.ranges.sort((a, b) => a.start - b.start); | |
| } | |
| /** | |
| * Remove a range from the list. The range must start and end at existing boundaries. | |
| */ | |
| remove(start, end) { | |
| if (end <= start) { | |
| throw new TypeError("End must be greater than start"); | |
| } | |
| const affectedRanges = []; | |
| for (let i = 0; i < this.ranges.length; i++) { | |
| const range2 = this.ranges[i]; | |
| if (start < range2.end && end > range2.start) { | |
| affectedRanges.push({ index: i, range: range2 }); | |
| } | |
| } | |
| if (affectedRanges.length === 0) { | |
| throw new Error("No ranges found to remove"); | |
| } | |
| if (start !== affectedRanges[0].range.start || end !== affectedRanges[affectedRanges.length - 1].range.end) { | |
| throw new Error("Range boundaries must match existing boundaries"); | |
| } | |
| for (let i = 0; i < affectedRanges.length; i++) { | |
| const { range: range2 } = affectedRanges[i]; | |
| range2.refCount--; | |
| } | |
| this.ranges = this.ranges.filter((range2) => range2.refCount > 0); | |
| } | |
| /** | |
| * Get all ranges within the specified boundaries. | |
| */ | |
| getRanges(start, end) { | |
| if (end <= start) { | |
| throw new TypeError("End must be greater than start"); | |
| } | |
| return this.ranges.filter((range2) => start < range2.end && end > range2.start); | |
| } | |
| /** | |
| * Get all ranges in the list | |
| */ | |
| getAllRanges() { | |
| return [...this.ranges]; | |
| } | |
| }, _class); | |
| // src/utils/XetBlob.ts | |
| var JWT_SAFETY_PERIOD = 6e4; | |
| var JWT_CACHE_SIZE = 1e3; | |
| var compressionSchemeLabels = { | |
| [0 /* None */]: "None", | |
| [1 /* LZ4 */]: "LZ4", | |
| [2 /* ByteGroupingLZ4 */]: "ByteGroupingLZ4" | |
| }; | |
| var XET_CHUNK_HEADER_BYTES = 8; | |
| var XetBlob = (_class2 = class extends Blob { | |
| __init2() {this.start = 0} | |
| __init3() {this.end = 0} | |
| __init4() {this.internalLogging = false} | |
| constructor(params) { | |
| super([]);_class2.prototype.__init2.call(this);_class2.prototype.__init3.call(this);_class2.prototype.__init4.call(this);; | |
| this.fetch = _nullishCoalesce(params.fetch, () => ( fetch.bind(globalThis))); | |
| this.accessToken = checkCredentials(params); | |
| this.refreshUrl = params.refreshUrl; | |
| this.end = params.size; | |
| this.reconstructionUrl = params.reconstructionUrl; | |
| this.hash = params.hash; | |
| this.listener = params.listener; | |
| this.internalLogging = _nullishCoalesce(params.internalLogging, () => ( false)); | |
| if (params.readToken) { | |
| const key = cacheKey({ refreshUrl: this.refreshUrl, initialAccessToken: this.accessToken }); | |
| jwts.set(key, { | |
| accessToken: params.readToken.accessToken, | |
| expiresAt: new Date(params.readToken.exp * 1e3), | |
| casUrl: params.readToken.casUrl | |
| }); | |
| } | |
| } | |
| get size() { | |
| return this.end - this.start; | |
| } | |
| #clone() { | |
| const blob = new XetBlob({ | |
| fetch: this.fetch, | |
| hash: this.hash, | |
| refreshUrl: this.refreshUrl, | |
| // eslint-disable-next-line @typescript-eslint/no-non-null-assertion | |
| reconstructionUrl: this.reconstructionUrl, | |
| size: this.size | |
| }); | |
| blob.accessToken = this.accessToken; | |
| blob.start = this.start; | |
| blob.end = this.end; | |
| blob.reconstructionInfo = this.reconstructionInfo; | |
| blob.listener = this.listener; | |
| blob.internalLogging = this.internalLogging; | |
| return blob; | |
| } | |
| slice(start = 0, end = this.size) { | |
| if (start < 0 || end < 0) { | |
| new TypeError("Unsupported negative start/end on XetBlob.slice"); | |
| } | |
| const slice = this.#clone(); | |
| slice.start = this.start + start; | |
| slice.end = Math.min(this.start + end, this.end); | |
| if (slice.start !== this.start || slice.end !== this.end) { | |
| slice.reconstructionInfo = void 0; | |
| } | |
| return slice; | |
| } | |
| #reconstructionInfoPromise; | |
| #loadReconstructionInfo() { | |
| if (this.#reconstructionInfoPromise) { | |
| return this.#reconstructionInfoPromise; | |
| } | |
| this.#reconstructionInfoPromise = (async () => { | |
| const connParams = await getAccessToken(this.accessToken, this.fetch, this.refreshUrl); | |
| const resp = await this.fetch(_nullishCoalesce(this.reconstructionUrl, () => ( `${connParams.casUrl}/v1/reconstructions/${this.hash}`)), { | |
| headers: { | |
| Authorization: `Bearer ${connParams.accessToken}`, | |
| Range: `bytes=${this.start}-${this.end - 1}` | |
| } | |
| }); | |
| if (!resp.ok) { | |
| throw await createApiError(resp); | |
| } | |
| this.reconstructionInfo = await resp.json(); | |
| return this.reconstructionInfo; | |
| })().finally(() => this.#reconstructionInfoPromise = void 0); | |
| return this.#reconstructionInfoPromise; | |
| } | |
| async #fetch() { | |
| if (!this.reconstructionInfo) { | |
| await this.#loadReconstructionInfo(); | |
| } | |
| const rangeLists = /* @__PURE__ */ new Map(); | |
| if (!this.reconstructionInfo) { | |
| throw new Error("Failed to load reconstruction info"); | |
| } | |
| for (const term of this.reconstructionInfo.terms) { | |
| let rangeList = rangeLists.get(term.hash); | |
| if (!rangeList) { | |
| rangeList = new RangeList(); | |
| rangeLists.set(term.hash, rangeList); | |
| } | |
| rangeList.add(term.range.start, term.range.end); | |
| } | |
| const listener = this.listener; | |
| const log = this.internalLogging ? (...args) => console.log(...args) : () => { | |
| }; | |
| async function* readData(reconstructionInfo, customFetch, maxBytes, reloadReconstructionInfo) { | |
| let totalBytesRead = 0; | |
| let readBytesToSkip = reconstructionInfo.offset_into_first_range; | |
| for (const term of reconstructionInfo.terms) { | |
| if (totalBytesRead >= maxBytes) { | |
| break; | |
| } | |
| const rangeList = rangeLists.get(term.hash); | |
| if (!rangeList) { | |
| throw new Error(`Failed to find range list for term ${term.hash}`); | |
| } | |
| { | |
| const termRanges = rangeList.getRanges(term.range.start, term.range.end); | |
| if (termRanges.every((range2) => range2.data)) { | |
| log("all data available for term", term.hash, readBytesToSkip); | |
| rangeLoop: | |
| for (const range2 of termRanges) { | |
| for (let chunk2 of range2.data) { | |
| if (readBytesToSkip) { | |
| const skipped = Math.min(readBytesToSkip, chunk2.byteLength); | |
| chunk2 = chunk2.slice(skipped); | |
| readBytesToSkip -= skipped; | |
| if (!chunk2.byteLength) { | |
| continue; | |
| } | |
| } | |
| if (chunk2.byteLength > maxBytes - totalBytesRead) { | |
| chunk2 = chunk2.slice(0, maxBytes - totalBytesRead); | |
| } | |
| totalBytesRead += chunk2.byteLength; | |
| yield range2.refCount > 1 ? chunk2.slice() : chunk2; | |
| _optionalChain([listener, 'optionalCall', _61 => _61({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } })]); | |
| if (totalBytesRead >= maxBytes) { | |
| break rangeLoop; | |
| } | |
| } | |
| } | |
| rangeList.remove(term.range.start, term.range.end); | |
| continue; | |
| } | |
| } | |
| const fetchInfo = reconstructionInfo.fetch_info[term.hash].find( | |
| (info) => info.range.start <= term.range.start && info.range.end >= term.range.end | |
| ); | |
| if (!fetchInfo) { | |
| throw new Error( | |
| `Failed to find fetch info for term ${term.hash} and range ${term.range.start}-${term.range.end}` | |
| ); | |
| } | |
| log("term", term); | |
| log("fetchinfo", fetchInfo); | |
| log("readBytesToSkip", readBytesToSkip); | |
| let resp = await customFetch(fetchInfo.url, { | |
| headers: { | |
| Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}` | |
| } | |
| }); | |
| if (resp.status === 403) { | |
| reconstructionInfo = await reloadReconstructionInfo(); | |
| resp = await customFetch(fetchInfo.url, { | |
| headers: { | |
| Range: `bytes=${fetchInfo.url_range.start}-${fetchInfo.url_range.end}` | |
| } | |
| }); | |
| } | |
| if (!resp.ok) { | |
| throw await createApiError(resp); | |
| } | |
| log( | |
| "expected content length", | |
| resp.headers.get("content-length"), | |
| "range", | |
| fetchInfo.url_range, | |
| resp.headers.get("content-range") | |
| ); | |
| const reader = _optionalChain([resp, 'access', _62 => _62.body, 'optionalAccess', _63 => _63.getReader, 'call', _64 => _64()]); | |
| if (!reader) { | |
| throw new Error("Failed to get reader from response body"); | |
| } | |
| let done = false; | |
| let chunkIndex = fetchInfo.range.start; | |
| const ranges = rangeList.getRanges(fetchInfo.range.start, fetchInfo.range.end); | |
| let leftoverBytes = void 0; | |
| let totalFetchBytes = 0; | |
| fetchData: | |
| while (!done && totalBytesRead < maxBytes) { | |
| const result = await reader.read(); | |
| _optionalChain([listener, 'optionalCall', _65 => _65({ event: "read" })]); | |
| done = result.done; | |
| log("read", _optionalChain([result, 'access', _66 => _66.value, 'optionalAccess', _67 => _67.byteLength]), "bytes", "total read", totalBytesRead, "toSkip", readBytesToSkip); | |
| if (!result.value) { | |
| log("no data in result, cancelled", result); | |
| continue; | |
| } | |
| totalFetchBytes += result.value.byteLength; | |
| if (leftoverBytes) { | |
| result.value = combineUint8Arrays(leftoverBytes, result.value); | |
| leftoverBytes = void 0; | |
| } | |
| while (totalBytesRead < maxBytes && _optionalChain([result, 'access', _68 => _68.value, 'optionalAccess', _69 => _69.byteLength])) { | |
| if (result.value.byteLength < 8) { | |
| leftoverBytes = result.value; | |
| continue fetchData; | |
| } | |
| const header = new DataView(result.value.buffer, result.value.byteOffset, XET_CHUNK_HEADER_BYTES); | |
| const chunkHeader = { | |
| version: header.getUint8(0), | |
| compressed_length: header.getUint8(1) | header.getUint8(2) << 8 | header.getUint8(3) << 16, | |
| compression_scheme: header.getUint8(4), | |
| uncompressed_length: header.getUint8(5) | header.getUint8(6) << 8 | header.getUint8(7) << 16 | |
| }; | |
| log("chunk header", chunkHeader, "to skip", readBytesToSkip); | |
| if (chunkHeader.version !== 0) { | |
| throw new Error(`Unsupported chunk version ${chunkHeader.version}`); | |
| } | |
| if (chunkHeader.compression_scheme !== 0 /* None */ && chunkHeader.compression_scheme !== 1 /* LZ4 */ && chunkHeader.compression_scheme !== 2 /* ByteGroupingLZ4 */) { | |
| throw new Error( | |
| `Unsupported compression scheme ${_nullishCoalesce(compressionSchemeLabels[chunkHeader.compression_scheme], () => ( chunkHeader.compression_scheme))}` | |
| ); | |
| } | |
| if (result.value.byteLength < chunkHeader.compressed_length + XET_CHUNK_HEADER_BYTES) { | |
| leftoverBytes = result.value; | |
| continue fetchData; | |
| } | |
| result.value = result.value.slice(XET_CHUNK_HEADER_BYTES); | |
| let uncompressed = chunkHeader.compression_scheme === 1 /* LZ4 */ ? decompress(result.value.slice(0, chunkHeader.compressed_length), chunkHeader.uncompressed_length) : chunkHeader.compression_scheme === 2 /* ByteGroupingLZ4 */ ? bg4_regroup_bytes( | |
| decompress( | |
| result.value.slice(0, chunkHeader.compressed_length), | |
| chunkHeader.uncompressed_length | |
| ) | |
| ) : result.value.slice(0, chunkHeader.compressed_length); | |
| const range2 = ranges.find((range3) => chunkIndex >= range3.start && chunkIndex < range3.end); | |
| const shouldYield = chunkIndex >= term.range.start && chunkIndex < term.range.end; | |
| const minRefCountToStore = shouldYield ? 2 : 1; | |
| let stored = false; | |
| if (range2 && range2.refCount >= minRefCountToStore) { | |
| range2.data ??= []; | |
| range2.data.push(uncompressed); | |
| stored = true; | |
| } | |
| if (shouldYield) { | |
| if (readBytesToSkip) { | |
| const skipped = Math.min(readBytesToSkip, uncompressed.byteLength); | |
| uncompressed = uncompressed.slice(readBytesToSkip); | |
| readBytesToSkip -= skipped; | |
| } | |
| if (uncompressed.byteLength > maxBytes - totalBytesRead) { | |
| uncompressed = uncompressed.slice(0, maxBytes - totalBytesRead); | |
| } | |
| if (uncompressed.byteLength) { | |
| log( | |
| "yield", | |
| uncompressed.byteLength, | |
| "bytes", | |
| result.value.byteLength, | |
| "total read", | |
| totalBytesRead, | |
| stored | |
| ); | |
| totalBytesRead += uncompressed.byteLength; | |
| yield stored ? uncompressed.slice() : uncompressed; | |
| _optionalChain([listener, 'optionalCall', _70 => _70({ event: "progress", progress: { read: totalBytesRead, total: maxBytes } })]); | |
| } | |
| } | |
| chunkIndex++; | |
| result.value = result.value.slice(chunkHeader.compressed_length); | |
| } | |
| } | |
| if (done && totalBytesRead < maxBytes && totalFetchBytes < fetchInfo.url_range.end - fetchInfo.url_range.start + 1) { | |
| log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes); | |
| log("failed to fetch all data for term", term.hash); | |
| throw new Error( | |
| `Failed to fetch all data for term ${term.hash}, fetched ${totalFetchBytes} bytes out of ${fetchInfo.url_range.end - fetchInfo.url_range.start + 1}` | |
| ); | |
| } | |
| log("done", done, "total read", totalBytesRead, maxBytes, totalFetchBytes); | |
| log("cancel reader"); | |
| await reader.cancel(); | |
| } | |
| } | |
| const iterator = readData( | |
| this.reconstructionInfo, | |
| this.fetch, | |
| this.end - this.start, | |
| this.#loadReconstructionInfo.bind(this) | |
| ); | |
| return new ReadableStream( | |
| { | |
| // todo: when Safari supports it, type controller as ReadableByteStreamController | |
| async pull(controller) { | |
| const result = await iterator.next(); | |
| if (result.value) { | |
| controller.enqueue(result.value); | |
| } | |
| if (result.done) { | |
| controller.close(); | |
| } | |
| }, | |
| type: "bytes" | |
| // todo: when Safari supports it, add autoAllocateChunkSize param | |
| }, | |
| // todo : use ByteLengthQueuingStrategy when there's good support for it, currently in Node.js it fails due to size being a function | |
| { | |
| highWaterMark: 1e3 | |
| // 1_000 chunks for ~1MB of RAM | |
| } | |
| ); | |
| } | |
| async arrayBuffer() { | |
| const result = await this.#fetch(); | |
| return new Response(result).arrayBuffer(); | |
| } | |
| async text() { | |
| const result = await this.#fetch(); | |
| return new Response(result).text(); | |
| } | |
| async response() { | |
| const result = await this.#fetch(); | |
| return new Response(result); | |
| } | |
| stream() { | |
| const stream = new TransformStream(); | |
| this.#fetch().then((response) => response.pipeThrough(stream)).catch((error) => stream.writable.abort(error.message)); | |
| return stream.readable; | |
| } | |
| }, _class2); | |
| var jwtPromises = /* @__PURE__ */ new Map(); | |
| var jwts = /* @__PURE__ */ new Map(); | |
| function cacheKey(params) { | |
| return JSON.stringify([params.refreshUrl, params.initialAccessToken]); | |
| } | |
| function bg4_regroup_bytes(bytes) { | |
| const split = Math.floor(bytes.byteLength / 4); | |
| const rem = bytes.byteLength % 4; | |
| const g1_pos = split + (rem >= 1 ? 1 : 0); | |
| const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); | |
| const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); | |
| const ret = new Uint8Array(bytes.byteLength); | |
| for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { | |
| ret[i] = bytes[j]; | |
| } | |
| for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { | |
| ret[i] = bytes[j]; | |
| } | |
| for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { | |
| ret[i] = bytes[j]; | |
| } | |
| for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { | |
| ret[i] = bytes[j]; | |
| } | |
| return ret; | |
| } | |
| function bg4_split_bytes(bytes) { | |
| const ret = new Uint8Array(bytes.byteLength); | |
| const split = Math.floor(bytes.byteLength / 4); | |
| const rem = bytes.byteLength % 4; | |
| const g1_pos = split + (rem >= 1 ? 1 : 0); | |
| const g2_pos = g1_pos + split + (rem >= 2 ? 1 : 0); | |
| const g3_pos = g2_pos + split + (rem == 3 ? 1 : 0); | |
| for (let i = 0, j = 0; i < bytes.byteLength; i += 4, j++) { | |
| ret[j] = bytes[i]; | |
| } | |
| for (let i = 1, j = g1_pos; i < bytes.byteLength; i += 4, j++) { | |
| ret[j] = bytes[i]; | |
| } | |
| for (let i = 2, j = g2_pos; i < bytes.byteLength; i += 4, j++) { | |
| ret[j] = bytes[i]; | |
| } | |
| for (let i = 3, j = g3_pos; i < bytes.byteLength; i += 4, j++) { | |
| ret[j] = bytes[i]; | |
| } | |
| return ret; | |
| } | |
| async function getAccessToken(initialAccessToken, customFetch, refreshUrl) { | |
| const key = cacheKey({ refreshUrl, initialAccessToken }); | |
| const jwt = jwts.get(key); | |
| if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD)) { | |
| return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; | |
| } | |
| const existingPromise = jwtPromises.get(key); | |
| if (existingPromise) { | |
| return existingPromise; | |
| } | |
| const promise = (async () => { | |
| const resp = await customFetch(refreshUrl, { | |
| headers: { | |
| ...initialAccessToken ? { | |
| Authorization: `Bearer ${initialAccessToken}` | |
| } : {} | |
| } | |
| }); | |
| if (!resp.ok) { | |
| throw new Error(`Failed to get JWT token: ${resp.status} ${await resp.text()}`); | |
| } | |
| const json = await resp.json(); | |
| const jwt2 = { | |
| accessToken: json.accessToken, | |
| expiresAt: new Date(json.exp * 1e3), | |
| casUrl: json.casUrl | |
| }; | |
| jwtPromises.delete(key); | |
| for (const [key2, value] of jwts.entries()) { | |
| if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD)) { | |
| jwts.delete(key2); | |
| } else { | |
| break; | |
| } | |
| } | |
| if (jwts.size >= JWT_CACHE_SIZE) { | |
| const keyToDelete = jwts.keys().next().value; | |
| if (keyToDelete) { | |
| jwts.delete(keyToDelete); | |
| } | |
| } | |
| jwts.set(key, jwt2); | |
| return { | |
| accessToken: json.accessToken, | |
| casUrl: json.casUrl | |
| }; | |
| })(); | |
| jwtPromises.set(key, promise); | |
| return promise; | |
| } | |
| // src/utils/ChunkCache.ts | |
| var CHUNK_CACHE_INITIAL_SIZE = 1e4; | |
| var CHUNK_CACHE_GROW_FACTOR = 1.5; | |
| var CHUNK_CACHE_MAX_SIZE = 1e6; | |
| var ChunkCache = (_class3 = class { | |
| __init5() {this.index = 0} | |
| // Index >= 0 means local xorb, < 0 means remote xorb | |
| // Max 8K chunks per xorb, less than 64K uint16_t | |
| __init6() {this.map = /* @__PURE__ */ new Map()} | |
| // hash -> chunkCacheIndex. Less overhead that way, empty object is 60+B and empty array is 40+B | |
| __init7() {this.hmacs = /* @__PURE__ */ new Set()} | |
| // todo : remove old hmacs | |
| constructor(maxSize = CHUNK_CACHE_MAX_SIZE) {;_class3.prototype.__init5.call(this);_class3.prototype.__init6.call(this);_class3.prototype.__init7.call(this); | |
| if (maxSize < 1) { | |
| throw new Error("maxSize must be at least 1"); | |
| } | |
| this.maxSize = maxSize; | |
| this.xorbIndices = new Int32Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); | |
| this.chunkIndices = new Uint16Array(Math.min(CHUNK_CACHE_INITIAL_SIZE, maxSize)); | |
| } | |
| addChunkToCache(hash2, xorbIndex, chunkIndex, hmac) { | |
| if (this.map.has(hash2)) { | |
| return; | |
| } | |
| if (this.map.values().next().value === this.index) { | |
| this.map.delete(this.map.keys().next().value); | |
| } | |
| this.map.set(hash2, this.index); | |
| if (hmac !== null) { | |
| this.hmacs.add(hmac); | |
| } | |
| if (this.index >= this.xorbIndices.length) { | |
| const oldXorbIndices = this.xorbIndices; | |
| const oldChunkIndices = this.chunkIndices; | |
| this.xorbIndices = new Int32Array(Math.min(this.xorbIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); | |
| this.chunkIndices = new Uint16Array(Math.min(this.chunkIndices.length * CHUNK_CACHE_GROW_FACTOR, this.maxSize)); | |
| this.xorbIndices.set(oldXorbIndices); | |
| this.chunkIndices.set(oldChunkIndices); | |
| } | |
| this.xorbIndices[this.index] = xorbIndex; | |
| this.chunkIndices[this.index] = chunkIndex; | |
| this.index = (this.index + 1) % this.maxSize; | |
| } | |
| getChunk(hash2, hmacFunction) { | |
| let index = this.map.get(hash2); | |
| if (index === void 0 && hmacFunction !== null) { | |
| for (const hmac of this.hmacs) { | |
| index = this.map.get(hmacFunction(hash2, hmac)); | |
| if (index !== void 0) { | |
| break; | |
| } | |
| } | |
| } | |
| if (index === void 0) { | |
| return void 0; | |
| } | |
| return { | |
| xorbIndex: this.xorbIndices[index], | |
| chunkIndex: this.chunkIndices[index] | |
| }; | |
| } | |
| updateChunkIndex(hash2, chunkIndex) { | |
| const index = this.map.get(hash2); | |
| if (index === void 0) { | |
| throw new Error(`Chunk not found in cache: ${hash2}`); | |
| } | |
| this.chunkIndices[index] = chunkIndex; | |
| } | |
| removeChunkFromCache(hash2) { | |
| this.map.delete(hash2); | |
| } | |
| }, _class3); | |
| // src/utils/xetWriteToken.ts | |
| var JWT_SAFETY_PERIOD2 = 6e4; | |
| var JWT_CACHE_SIZE2 = 1e3; | |
| var jwtPromises2 = /* @__PURE__ */ new Map(); | |
| var jwts2 = /* @__PURE__ */ new Map(); | |
| async function xetWriteToken(params) { | |
| if (params.xetParams.expiresAt && params.xetParams.casUrl && params.xetParams.accessToken && params.xetParams.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD2)) { | |
| return { accessToken: params.xetParams.accessToken, casUrl: params.xetParams.casUrl }; | |
| } | |
| const key = params.xetParams.refreshWriteTokenUrl; | |
| const jwt = jwts2.get(key); | |
| if (jwt && jwt.expiresAt > new Date(Date.now() + JWT_SAFETY_PERIOD2)) { | |
| return { accessToken: jwt.accessToken, casUrl: jwt.casUrl }; | |
| } | |
| const existingPromise = jwtPromises2.get(key); | |
| if (existingPromise) { | |
| return existingPromise; | |
| } | |
| const promise = (async () => { | |
| const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(params.xetParams.refreshWriteTokenUrl, { | |
| headers: { | |
| ...params.accessToken ? { | |
| Authorization: `Bearer ${params.accessToken}` | |
| } : {}, | |
| ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} | |
| } | |
| }); | |
| if (!resp.ok) { | |
| throw await createApiError(resp); | |
| } | |
| const json = await resp.json(); | |
| const jwt2 = { | |
| accessToken: json.accessToken, | |
| expiresAt: new Date(json.exp * 1e3), | |
| casUrl: json.casUrl | |
| }; | |
| jwtPromises2.delete(key); | |
| for (const [key2, value] of jwts2.entries()) { | |
| if (value.expiresAt < new Date(Date.now() + JWT_SAFETY_PERIOD2)) { | |
| jwts2.delete(key2); | |
| } else { | |
| break; | |
| } | |
| } | |
| if (jwts2.size >= JWT_CACHE_SIZE2) { | |
| const keyToDelete = jwts2.keys().next().value; | |
| if (keyToDelete) { | |
| jwts2.delete(keyToDelete); | |
| } | |
| } | |
| jwts2.set(key, jwt2); | |
| return { | |
| accessToken: json.accessToken, | |
| casUrl: json.casUrl | |
| }; | |
| })(); | |
| jwtPromises2.set(key, promise); | |
| return promise; | |
| } | |
| // src/utils/shardParser.ts | |
| var HASH_LENGTH = 32; | |
| var XORB_HASH_BOOKEND = "ff".repeat(HASH_LENGTH); | |
| function readHashFromArray(array, offset) { | |
| let hash2 = ""; | |
| for (let i = 0; i < HASH_LENGTH; i += 8) { | |
| hash2 += `${array[offset + i + 7].toString(16).padStart(2, "0")}${array[offset + i + 6].toString(16).padStart(2, "0")}${array[offset + i + 5].toString(16).padStart(2, "0")}${array[offset + i + 4].toString(16).padStart(2, "0")}${array[offset + i + 3].toString(16).padStart(2, "0")}${array[offset + i + 2].toString(16).padStart(2, "0")}${array[offset + i + 1].toString(16).padStart(2, "0")}${array[offset + i].toString(16).padStart(2, "0")}`; | |
| } | |
| return hash2; | |
| } | |
| async function parseShardData(shardBlob) { | |
| const shard = new Uint8Array(await shardBlob.arrayBuffer()); | |
| const shardView = new DataView(shard.buffer); | |
| const magicTag = shard.slice(0, SHARD_MAGIC_TAG.length); | |
| if (!magicTag.every((byte, i) => byte === SHARD_MAGIC_TAG[i])) { | |
| throw new Error("Invalid shard magic tag"); | |
| } | |
| const version = shardView.getBigUint64(SHARD_MAGIC_TAG.length, true); | |
| if (version !== SHARD_HEADER_VERSION) { | |
| throw new Error(`Invalid shard version: ${version}`); | |
| } | |
| const footerSize = Number(shardView.getBigUint64(SHARD_MAGIC_TAG.length + 8, true)); | |
| const footerStart = shard.length - footerSize; | |
| const footerVersion = shardView.getBigUint64(footerStart, true); | |
| if (footerVersion !== SHARD_FOOTER_VERSION) { | |
| throw new Error(`Invalid shard footer version: ${footerVersion}`); | |
| } | |
| const xorbInfoStart = Number(shardView.getBigUint64(footerStart + 16, true)); | |
| const fileLookupStart = Number(shardView.getBigUint64(footerStart + 24, true)); | |
| const hmacKey = readHashFromArray(shard, footerStart + 72); | |
| const xorbs = []; | |
| let offset = xorbInfoStart; | |
| while (offset < fileLookupStart) { | |
| const xorbHash = readHashFromArray(shard, offset); | |
| offset += HASH_LENGTH; | |
| if (xorbHash === XORB_HASH_BOOKEND) { | |
| break; | |
| } | |
| offset += 4; | |
| const chunkCount = shardView.getUint32(offset, true); | |
| offset += 4; | |
| offset += 4; | |
| offset += 4; | |
| const chunks = []; | |
| for (let i = 0; i < chunkCount; i++) { | |
| const chunkHash = readHashFromArray(shard, offset); | |
| offset += HASH_LENGTH; | |
| const startOffset = shardView.getUint32(offset, true); | |
| offset += 4; | |
| const length = shardView.getUint32(offset, true); | |
| offset += 4; | |
| offset += 8; | |
| chunks.push({ | |
| hash: chunkHash, | |
| startOffset, | |
| unpackedLength: length | |
| }); | |
| } | |
| xorbs.push({ | |
| hash: xorbHash, | |
| chunks | |
| }); | |
| } | |
| return { | |
| hmacKey, | |
| xorbs | |
| }; | |
| } | |
| // src/utils/sum.ts | |
| function sum(arr) { | |
| return arr.reduce((a, b) => a + b, 0); | |
| } | |
| // src/utils/SplicedBlob.ts | |
| var SplicedBlob = class extends Blob { | |
| constructor(originalBlob, spliceOperations) { | |
| super(); | |
| this.originalBlob = originalBlob; | |
| this.spliceOperations = spliceOperations; | |
| } | |
| static create(originalBlob, operations) { | |
| for (const op of operations) { | |
| if (op.start < 0 || op.end < 0) { | |
| throw new Error("Invalid start/end positions for SplicedBlob"); | |
| } | |
| if (op.start > originalBlob.size || op.end > originalBlob.size) { | |
| throw new Error("Invalid start/end positions for SplicedBlob"); | |
| } | |
| if (op.start > op.end) { | |
| throw new Error("Invalid start/end positions for SplicedBlob"); | |
| } | |
| } | |
| const sortedOps = [...operations].sort((a, b) => a.start - b.start); | |
| for (let i = 0; i < sortedOps.length - 1; i++) { | |
| if (sortedOps[i].end > sortedOps[i + 1].start) { | |
| throw new Error("Overlapping splice operations are not supported"); | |
| } | |
| } | |
| return new SplicedBlob(originalBlob, sortedOps); | |
| } | |
| /** | |
| * Returns the size of the spliced blob. | |
| * Size = original size - total replaced size + total insert size | |
| */ | |
| get size() { | |
| let totalReplacedSize = 0; | |
| let totalInsertSize = 0; | |
| for (const op of this.spliceOperations) { | |
| totalReplacedSize += op.end - op.start; | |
| totalInsertSize += op.insert.size; | |
| } | |
| return this.originalBlob.size - totalReplacedSize + totalInsertSize; | |
| } | |
| /** | |
| * Returns the MIME type of the original blob. | |
| */ | |
| get type() { | |
| return this.originalBlob.type; | |
| } | |
| /** | |
| * Returns a new instance of SplicedBlob that is a slice of the current one. | |
| * | |
| * The slice is inclusive of the start and exclusive of the end. | |
| * The slice method does not support negative start/end. | |
| * | |
| * @param start beginning of the slice | |
| * @param end end of the slice | |
| */ | |
| slice(start = 0, end = this.size) { | |
| if (start < 0 || end < 0) { | |
| throw new TypeError("Unsupported negative start/end on SplicedBlob.slice"); | |
| } | |
| start = Math.min(start, this.size); | |
| end = Math.min(end, this.size); | |
| if (start >= end) { | |
| return new Blob([]); | |
| } | |
| const segments = this.segments; | |
| const segmentBoundaries = [0]; | |
| let cumulativeSize = 0; | |
| for (const segment of segments) { | |
| cumulativeSize += segment.size; | |
| segmentBoundaries.push(cumulativeSize); | |
| } | |
| const resultSegments = []; | |
| for (let i = 0; i < segments.length; i++) { | |
| const segmentStart = segmentBoundaries[i]; | |
| const segmentEnd = segmentBoundaries[i + 1]; | |
| if (segmentEnd <= start) { | |
| continue; | |
| } | |
| if (segmentStart >= end) { | |
| break; | |
| } | |
| const sliceStart = Math.max(0, start - segmentStart); | |
| const sliceEnd = Math.min(segments[i].size, end - segmentStart); | |
| if (sliceStart < sliceEnd) { | |
| resultSegments.push(segments[i].slice(sliceStart, sliceEnd)); | |
| } | |
| } | |
| return new Blob(resultSegments); | |
| } | |
| get firstSpliceIndex() { | |
| return _nullishCoalesce(_optionalChain([this, 'access', _71 => _71.spliceOperations, 'access', _72 => _72[0], 'optionalAccess', _73 => _73.start]), () => ( Infinity)); | |
| } | |
| /** | |
| * Read the spliced blob content and returns it as an ArrayBuffer. | |
| */ | |
| async arrayBuffer() { | |
| const segments = this.segments; | |
| const buffers = await Promise.all(segments.map((segment) => segment.arrayBuffer())); | |
| const totalSize = sum(buffers.map((buffer) => buffer.byteLength)); | |
| const result = new Uint8Array(totalSize); | |
| let offset = 0; | |
| for (const buffer of buffers) { | |
| result.set(new Uint8Array(buffer), offset); | |
| offset += buffer.byteLength; | |
| } | |
| return result.buffer; | |
| } | |
| /** | |
| * Read the spliced blob content and returns it as a string. | |
| */ | |
| async text() { | |
| const buffer = await this.arrayBuffer(); | |
| return new TextDecoder().decode(buffer); | |
| } | |
| /** | |
| * Returns a stream around the spliced blob content. | |
| */ | |
| stream() { | |
| const readable = new ReadableStream({ | |
| start: async (controller) => { | |
| try { | |
| const segments = this.segments; | |
| for (const segment of segments) { | |
| const reader = segment.stream().getReader(); | |
| try { | |
| while (true) { | |
| const { done, value } = await reader.read(); | |
| if (done) { | |
| break; | |
| } | |
| controller.enqueue(value); | |
| } | |
| } finally { | |
| reader.releaseLock(); | |
| } | |
| } | |
| controller.close(); | |
| } catch (error) { | |
| controller.error(error); | |
| } | |
| } | |
| }); | |
| return readable; | |
| } | |
| /** | |
| * Get all segments that make up the spliced blob. | |
| * This includes original blob segments between splice operations and insert blobs. | |
| */ | |
| get segments() { | |
| const segments = []; | |
| let currentPosition = 0; | |
| const sortedOps = [...this.spliceOperations].sort((a, b) => a.start - b.start); | |
| for (const op of sortedOps) { | |
| if (currentPosition < op.start) { | |
| segments.push(this.originalBlob.slice(currentPosition, op.start)); | |
| } | |
| if (op.insert.size > 0) { | |
| segments.push(op.insert); | |
| } | |
| currentPosition = op.end; | |
| } | |
| if (currentPosition < this.originalBlob.size) { | |
| segments.push(this.originalBlob.slice(currentPosition)); | |
| } | |
| return segments; | |
| } | |
| }; | |
| // src/utils/createXorbs.ts | |
| var TARGET_CHUNK_SIZE = 64 * 1024; | |
| var MAX_CHUNK_SIZE = 2 * TARGET_CHUNK_SIZE; | |
| var XORB_SIZE = 64 * 1024 * 1024; | |
| var MAX_XORB_CHUNKS = 8 * 1024; | |
| var INTERVAL_BETWEEN_REMOTE_DEDUP = 4e6; | |
| var PROCESSING_PROGRESS_RATIO = 0.1; | |
| var UPLOADING_PROGRESS_RATIO = 1 - PROCESSING_PROGRESS_RATIO; | |
| var CurrentXorbInfo = class { | |
| constructor() { | |
| this.id = 0; | |
| this.offset = 0; | |
| this.chunks = []; | |
| this.fileProcessedBytes = {}; | |
| this.fileUploadedBytes = {}; | |
| this.fileSize = {}; | |
| this.data = new Uint8Array(XORB_SIZE); | |
| this.immutableData = null; | |
| } | |
| event(computeXorbHash) { | |
| const xorbChunksCleaned = this.chunks.map((chunk2) => ({ | |
| hash: chunk2.hash, | |
| length: chunk2.length | |
| })); | |
| return { | |
| event: "xorb", | |
| xorb: this.data.subarray(0, this.offset), | |
| hash: computeXorbHash(xorbChunksCleaned), | |
| chunks: xorbChunksCleaned, | |
| id: this.id, | |
| files: Object.entries(this.fileProcessedBytes).map(([path, processedBytes]) => ({ | |
| path, | |
| progress: processedBytes / this.fileSize[path], | |
| lastSentProgress: ((_nullishCoalesce(this.fileUploadedBytes[path], () => ( 0))) + (processedBytes - (_nullishCoalesce(this.fileUploadedBytes[path], () => ( 0)))) * PROCESSING_PROGRESS_RATIO) / this.fileSize[path] | |
| })) | |
| }; | |
| } | |
| }; | |
| async function* createXorbs(fileSources, params) { | |
| const alreadyDoneFileSha256s = /* @__PURE__ */ new Set(); | |
| const chunkModule = await Promise.resolve().then(() => _interopRequireWildcard(require("./chunker_wasm-FWDBRYQI.js"))); | |
| let xorbId = 0; | |
| await chunkModule.init(); | |
| const chunkCache = new ChunkCache(); | |
| let xorb = new CurrentXorbInfo(); | |
| const nextXorb = (currentFile) => { | |
| const event = xorb.event(chunkModule.compute_xorb_hash.bind(chunkModule)); | |
| xorbId++; | |
| xorb = new CurrentXorbInfo(); | |
| xorb.id = xorbId; | |
| xorb.fileUploadedBytes = { | |
| [currentFile.path]: currentFile.uploadedBytes | |
| }; | |
| xorb.fileSize[currentFile.path] = currentFile.size; | |
| return event; | |
| }; | |
| const pendingFileEvents = []; | |
| const remoteXorbHashes = [""]; | |
| for await (const fileSource of fileSources) { | |
| _optionalChain([params, 'access', _74 => _74.yieldCallback, 'optionalCall', _75 => _75({ | |
| event: "fileProgress", | |
| path: fileSource.path, | |
| progress: 0 | |
| })]); | |
| if (fileSource.sha256 && alreadyDoneFileSha256s.has(fileSource.sha256)) { | |
| _optionalChain([params, 'access', _76 => _76.yieldCallback, 'optionalCall', _77 => _77({ | |
| event: "fileProgress", | |
| path: fileSource.path, | |
| progress: 1 | |
| })]); | |
| continue; | |
| } | |
| if (fileSource.sha256) { | |
| alreadyDoneFileSha256s.add(fileSource.sha256); | |
| } | |
| const chunker = new chunkModule.Chunker(TARGET_CHUNK_SIZE); | |
| try { | |
| xorb.fileSize[fileSource.path] = fileSource.content.size; | |
| if (fileSource.content instanceof SplicedBlob && fileSource.content.firstSpliceIndex < MAX_CHUNK_SIZE) { | |
| await loadDedupInfoToCache( | |
| fileSource.content.originalBlob.slice(0, MAX_CHUNK_SIZE), | |
| remoteXorbHashes, | |
| params, | |
| chunkCache, | |
| chunkModule, | |
| { | |
| maxChunks: 1, | |
| isAtBeginning: true | |
| } | |
| ); | |
| } | |
| let bytesSinceRemoteDedup = Infinity; | |
| let bytesSinceLastProgressEvent = 0; | |
| let isFirstFileChunk = true; | |
| const sourceChunks = []; | |
| const reader = fileSource.content.stream().getReader(); | |
| let processedBytes = 0; | |
| let dedupedBytes = 0; | |
| const fileChunks = []; | |
| const chunkMetadata = []; | |
| const addChunks = async function* (chunks) { | |
| for (const chunk2 of chunks) { | |
| if (isFirstFileChunk) { | |
| chunk2.dedup = true; | |
| isFirstFileChunk = false; | |
| } | |
| let chunkIndex = xorb.chunks.length; | |
| let chunkXorbId = xorbId; | |
| const chunkToCopy = removeChunkFromSourceData(sourceChunks, chunk2.length); | |
| let cacheData = chunkCache.getChunk(chunk2.hash, chunkModule.compute_hmac); | |
| if (cacheData === void 0 && chunk2.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { | |
| const token = await xetWriteToken(params); | |
| bytesSinceRemoteDedup = 0; | |
| const shardResp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(token.casUrl + "/v1/chunks/default/" + chunk2.hash, { | |
| headers: { | |
| Authorization: `Bearer ${token.accessToken}` | |
| } | |
| }); | |
| if (shardResp.ok) { | |
| const shard = await shardResp.blob(); | |
| const shardData = await parseShardData(shard); | |
| for (const xorb2 of shardData.xorbs) { | |
| const remoteXorbId = -remoteXorbHashes.length; | |
| remoteXorbHashes.push(xorb2.hash); | |
| let i = 0; | |
| for (const chunk3 of xorb2.chunks) { | |
| chunkCache.addChunkToCache(chunk3.hash, remoteXorbId, i++, shardData.hmacKey); | |
| } | |
| } | |
| cacheData = chunkCache.getChunk(chunk2.hash, chunkModule.compute_hmac); | |
| const oldDedupedBytes = dedupedBytes; | |
| dedupedBytes = backtrackDedup( | |
| xorb, | |
| chunkModule.compute_hmac.bind(chunkModule), | |
| shardData, | |
| chunkCache, | |
| chunkMetadata, | |
| dedupedBytes | |
| ); | |
| if (dedupedBytes > oldDedupedBytes) { | |
| xorb.fileUploadedBytes[fileSource.path] ??= 0; | |
| xorb.fileUploadedBytes[fileSource.path] += dedupedBytes - oldDedupedBytes; | |
| } | |
| } | |
| } | |
| if (cacheData === void 0) { | |
| if (!writeChunk(xorb, chunkToCopy, chunk2.hash)) { | |
| yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); | |
| chunkIndex = 0; | |
| chunkXorbId = xorbId; | |
| for (const event of pendingFileEvents) { | |
| event.representation = event.representation.map((rep) => ({ | |
| ...rep, | |
| xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] | |
| })); | |
| yield event; | |
| } | |
| pendingFileEvents.length = 0; | |
| if (!writeChunk(xorb, chunkToCopy, chunk2.hash)) { | |
| throw new Error("Failed to write chunk into xorb"); | |
| } | |
| } | |
| chunkCache.addChunkToCache(chunk2.hash, xorbId, chunkIndex, null); | |
| } else { | |
| chunkXorbId = cacheData.xorbIndex; | |
| chunkIndex = cacheData.chunkIndex; | |
| dedupedBytes += chunk2.length; | |
| xorb.fileUploadedBytes[fileSource.path] ??= 0; | |
| xorb.fileUploadedBytes[fileSource.path] += chunk2.length; | |
| } | |
| bytesSinceRemoteDedup += chunk2.length; | |
| bytesSinceLastProgressEvent += chunk2.length; | |
| fileChunks.push({ hash: chunk2.hash, length: chunk2.length }); | |
| chunkMetadata.push({ | |
| xorbId: chunkXorbId, | |
| chunkIndex, | |
| length: chunk2.length | |
| }); | |
| xorb.fileProcessedBytes[fileSource.path] = processedBytes; | |
| if (bytesSinceLastProgressEvent >= 1e6) { | |
| bytesSinceLastProgressEvent = 0; | |
| _optionalChain([params, 'access', _78 => _78.yieldCallback, 'optionalCall', _79 => _79({ | |
| event: "fileProgress", | |
| path: fileSource.path, | |
| progress: ((_nullishCoalesce(xorb.fileUploadedBytes[fileSource.path], () => ( 0))) + (xorb.fileProcessedBytes[fileSource.path] - (_nullishCoalesce(xorb.fileUploadedBytes[fileSource.path], () => ( 0)))) * PROCESSING_PROGRESS_RATIO) / fileSource.content.size | |
| })]); | |
| } | |
| if (xorb.chunks.length >= MAX_XORB_CHUNKS) { | |
| yield nextXorb({ path: fileSource.path, uploadedBytes: processedBytes, size: fileSource.content.size }); | |
| for (const event of pendingFileEvents) { | |
| event.representation = event.representation.map((rep) => ({ | |
| ...rep, | |
| xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] | |
| })); | |
| yield event; | |
| } | |
| pendingFileEvents.length = 0; | |
| } | |
| } | |
| }; | |
| while (true) { | |
| const { done, value } = await reader.read(); | |
| if (done) { | |
| yield* addChunks(chunker.finish()); | |
| break; | |
| } | |
| processedBytes += value.length; | |
| sourceChunks.push(value); | |
| yield* addChunks(chunker.add_data(value)); | |
| } | |
| const fileRepresentation = buildFileRepresentation( | |
| chunkMetadata, | |
| fileChunks, | |
| chunkModule.compute_verification_hash.bind(chunkModule) | |
| ); | |
| xorb.immutableData = { | |
| chunkIndex: xorb.chunks.length, | |
| offset: xorb.offset | |
| }; | |
| const dedupRatio = fileSource.content.size > 0 ? dedupedBytes / fileSource.content.size : 0; | |
| pendingFileEvents.push({ | |
| event: "file", | |
| path: fileSource.path, | |
| hash: chunkModule.compute_file_hash(fileChunks), | |
| sha256: fileSource.sha256, | |
| dedupRatio, | |
| representation: fileRepresentation | |
| }); | |
| } finally { | |
| chunker.free(); | |
| } | |
| } | |
| if (xorb.offset > 0) { | |
| yield xorb.event(chunkModule.compute_xorb_hash.bind(chunkModule)); | |
| } | |
| for (const event of pendingFileEvents) { | |
| event.representation = event.representation.map((rep) => ({ | |
| ...rep, | |
| xorbId: rep.xorbId >= 0 ? rep.xorbId : remoteXorbHashes[-rep.xorbId] | |
| })); | |
| yield event; | |
| } | |
| } | |
| function backtrackDedup(xorb, computeHmac, shardData, chunkCache, chunkMetadata, dedupedBytes) { | |
| const chunkIndexesToBacktrackFor = /* @__PURE__ */ new Map(); | |
| for (let chunkToRecheckIndex = _nullishCoalesce(_optionalChain([xorb, 'access', _80 => _80.immutableData, 'optionalAccess', _81 => _81.chunkIndex]), () => ( 0)); chunkToRecheckIndex < xorb.chunks.length; chunkToRecheckIndex++) { | |
| const chunk2 = xorb.chunks[chunkToRecheckIndex]; | |
| const hmacHash = computeHmac(chunk2.hash, shardData.hmacKey); | |
| const cacheData = chunkCache.getChunk(hmacHash, null); | |
| if (cacheData !== void 0) { | |
| chunkIndexesToBacktrackFor.set(chunkToRecheckIndex, { | |
| xorbId: cacheData.xorbIndex, | |
| chunkIndex: cacheData.chunkIndex | |
| }); | |
| chunkCache.removeChunkFromCache(chunk2.hash); | |
| } | |
| } | |
| for (const metadata of chunkMetadata) { | |
| if (metadata.xorbId === xorb.id && chunkIndexesToBacktrackFor.has(metadata.chunkIndex)) { | |
| const backtrackData = chunkIndexesToBacktrackFor.get(metadata.chunkIndex); | |
| if (backtrackData !== void 0) { | |
| metadata.xorbId = backtrackData.xorbId; | |
| metadata.chunkIndex = backtrackData.chunkIndex; | |
| dedupedBytes += metadata.length; | |
| } | |
| } | |
| } | |
| const xorbRangesToErase = []; | |
| for (let i = 0; i < xorb.chunks.length; i++) { | |
| const chunk2 = xorb.chunks[i]; | |
| if (chunkIndexesToBacktrackFor.has(i)) { | |
| xorbRangesToErase.push({ | |
| start: chunk2.offset, | |
| end: i < xorb.chunks.length - 1 ? xorb.chunks[i + 1].offset : xorb.offset | |
| }); | |
| } | |
| } | |
| const xorbRangesToKeep = []; | |
| let currentStart = 0; | |
| for (let i = 0; i < xorbRangesToErase.length; i++) { | |
| const range2 = xorbRangesToErase[i]; | |
| if (currentStart !== range2.start) { | |
| xorbRangesToKeep.push({ start: currentStart, end: range2.start }); | |
| } | |
| currentStart = range2.end; | |
| } | |
| if (currentStart !== xorb.offset) { | |
| xorbRangesToKeep.push({ start: currentStart, end: xorb.offset }); | |
| } | |
| let currentOffset = 0; | |
| for (const range2 of xorbRangesToKeep) { | |
| if (range2.start !== currentOffset) { | |
| xorb.data.set(xorb.data.subarray(range2.start, range2.end), currentOffset); | |
| } | |
| currentOffset += range2.end - range2.start; | |
| } | |
| const newXorbChunks = []; | |
| const oldIndexToNewIndex = /* @__PURE__ */ new Map(); | |
| let erasedOffset = 0; | |
| for (let i = 0; i < xorb.chunks.length; i++) { | |
| const chunk2 = xorb.chunks[i]; | |
| if (chunkIndexesToBacktrackFor.has(i)) { | |
| if (i < xorb.chunks.length - 1) { | |
| erasedOffset += xorb.chunks[i + 1].offset - chunk2.offset; | |
| } | |
| } else { | |
| newXorbChunks.push({ | |
| hash: chunk2.hash, | |
| length: chunk2.length, | |
| offset: chunk2.offset - erasedOffset | |
| }); | |
| if (erasedOffset > 0) { | |
| oldIndexToNewIndex.set(i, newXorbChunks.length - 1); | |
| } | |
| } | |
| } | |
| xorb.chunks = newXorbChunks; | |
| xorb.offset = currentOffset; | |
| for (const chunk2 of chunkMetadata) { | |
| if (chunk2.xorbId === xorb.id) { | |
| const newIndex = oldIndexToNewIndex.get(chunk2.chunkIndex); | |
| if (newIndex !== void 0) { | |
| const cached = chunkCache.getChunk(xorb.chunks[newIndex].hash, null); | |
| if (cached !== void 0 && cached.xorbIndex === chunk2.xorbId && cached.chunkIndex === chunk2.chunkIndex) { | |
| chunkCache.updateChunkIndex(xorb.chunks[newIndex].hash, newIndex); | |
| } | |
| chunk2.chunkIndex = newIndex; | |
| } | |
| } | |
| } | |
| return dedupedBytes; | |
| } | |
| function removeChunkFromSourceData(sourceChunks, chunkLength) { | |
| if (chunkLength === sourceChunks[0].length) { | |
| const chunkToCopy = sourceChunks[0]; | |
| sourceChunks.shift(); | |
| return chunkToCopy; | |
| } else if (chunkLength < sourceChunks[0].length) { | |
| const chunkToCopy = sourceChunks[0].subarray(0, chunkLength); | |
| sourceChunks[0] = sourceChunks[0].subarray(chunkLength); | |
| return chunkToCopy; | |
| } else { | |
| const chunkToCopy = new Uint8Array(chunkLength); | |
| let copyOffset = 0; | |
| let index = 0; | |
| let toSlice = -1; | |
| while (copyOffset < chunkLength) { | |
| const nToCopy = Math.min(sourceChunks[index].length, chunkLength - copyOffset); | |
| chunkToCopy.set(sourceChunks[index].subarray(0, nToCopy), copyOffset); | |
| copyOffset += nToCopy; | |
| if (nToCopy === sourceChunks[index].length) { | |
| index++; | |
| } else { | |
| toSlice = nToCopy; | |
| } | |
| } | |
| sourceChunks.splice(0, index); | |
| if (toSlice !== -1) { | |
| sourceChunks[0] = sourceChunks[0].subarray(toSlice); | |
| } | |
| return chunkToCopy; | |
| } | |
| } | |
| function writeChunk(xorb, chunk2, hash2) { | |
| const regularCompressedChunk = compress(chunk2); | |
| const bgCompressedChunk = compress(bg4_split_bytes(chunk2)); | |
| const compressedChunk = bgCompressedChunk.length < regularCompressedChunk.length ? bgCompressedChunk : regularCompressedChunk; | |
| const chunkToWrite = compressedChunk.length < chunk2.length ? compressedChunk : chunk2; | |
| if (xorb.offset + XET_CHUNK_HEADER_BYTES + chunkToWrite.length > XORB_SIZE) { | |
| return false; | |
| } | |
| xorb.data[xorb.offset] = 0; | |
| xorb.data[xorb.offset + 1] = chunkToWrite.length & 255; | |
| xorb.data[xorb.offset + 2] = chunkToWrite.length >> 8 & 255; | |
| xorb.data[xorb.offset + 3] = chunkToWrite.length >> 16 & 255; | |
| xorb.data[xorb.offset + 4] = chunkToWrite.length < chunk2.length ? bgCompressedChunk.length < regularCompressedChunk.length ? 2 /* ByteGroupingLZ4 */ : 1 /* LZ4 */ : 0 /* None */; | |
| xorb.data[xorb.offset + 5] = chunk2.length & 255; | |
| xorb.data[xorb.offset + 6] = chunk2.length >> 8 & 255; | |
| xorb.data[xorb.offset + 7] = chunk2.length >> 16 & 255; | |
| xorb.data.set(chunkToWrite, xorb.offset + XET_CHUNK_HEADER_BYTES); | |
| xorb.chunks.push({ hash: hash2, length: chunk2.length, offset: xorb.offset }); | |
| xorb.offset += XET_CHUNK_HEADER_BYTES + chunkToWrite.length; | |
| return true; | |
| } | |
| var buildFileRepresentation = (metadata, chunks, computeVerificationHash) => { | |
| if (metadata.length === 0) { | |
| return []; | |
| } | |
| const representation = []; | |
| let currentRange = { | |
| xorbId: metadata[0].xorbId, | |
| indexStart: metadata[0].chunkIndex, | |
| indexEnd: metadata[0].chunkIndex + 1, | |
| length: metadata[0].length, | |
| chunkHashStart: 0 | |
| }; | |
| for (let i = 1; i < metadata.length; i++) { | |
| const chunk2 = metadata[i]; | |
| if (currentRange.xorbId === chunk2.xorbId && currentRange.indexEnd === chunk2.chunkIndex) { | |
| currentRange.indexEnd = chunk2.chunkIndex + 1; | |
| currentRange.length += chunk2.length; | |
| } else { | |
| const rangeHash2 = computeVerificationHash(chunks.slice(currentRange.chunkHashStart, i).map((x) => x.hash)); | |
| representation.push({ | |
| xorbId: currentRange.xorbId, | |
| indexStart: currentRange.indexStart, | |
| indexEnd: currentRange.indexEnd, | |
| length: currentRange.length, | |
| rangeHash: rangeHash2 | |
| }); | |
| currentRange = { | |
| xorbId: chunk2.xorbId, | |
| indexStart: chunk2.chunkIndex, | |
| indexEnd: chunk2.chunkIndex + 1, | |
| length: chunk2.length, | |
| chunkHashStart: i | |
| }; | |
| } | |
| } | |
| const rangeHash = computeVerificationHash(chunks.slice(currentRange.chunkHashStart).map((x) => x.hash)); | |
| representation.push({ | |
| xorbId: currentRange.xorbId, | |
| indexStart: currentRange.indexStart, | |
| indexEnd: currentRange.indexEnd, | |
| length: currentRange.length, | |
| rangeHash | |
| }); | |
| return representation; | |
| }; | |
| async function loadDedupInfoToCache(content, remoteXorbHashes, params, chunkCache, chunkModule, opts) { | |
| const chunker = new chunkModule.Chunker(TARGET_CHUNK_SIZE); | |
| const cache = chunkCache; | |
| let dedupedBytes = 0; | |
| let chunksProcessed = 0; | |
| let totalBytes = 0; | |
| let bytesSinceRemoteDedup = Infinity; | |
| const sourceChunks = []; | |
| try { | |
| const reader = content.stream().getReader(); | |
| const processChunks = async (chunkData) => { | |
| for (const chunk2 of chunkData) { | |
| chunksProcessed++; | |
| if (_optionalChain([opts, 'optionalAccess', _82 => _82.isAtBeginning]) && chunksProcessed === 1) { | |
| chunk2.dedup = true; | |
| } | |
| totalBytes += chunk2.length; | |
| removeChunkFromSourceData(sourceChunks, chunk2.length); | |
| let cacheData = cache.getChunk(chunk2.hash, chunkModule.compute_hmac); | |
| if (cacheData !== void 0) { | |
| dedupedBytes += chunk2.length; | |
| bytesSinceRemoteDedup += chunk2.length; | |
| continue; | |
| } | |
| if (chunk2.dedup && bytesSinceRemoteDedup >= INTERVAL_BETWEEN_REMOTE_DEDUP) { | |
| const token = await xetWriteToken(params); | |
| bytesSinceRemoteDedup = 0; | |
| const shardResp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(token.casUrl + "/v1/chunks/default/" + chunk2.hash, { | |
| headers: { | |
| Authorization: `Bearer ${token.accessToken}` | |
| } | |
| }); | |
| if (shardResp.ok) { | |
| const shard = await shardResp.blob(); | |
| const shardData = await parseShardData(shard); | |
| for (const xorb of shardData.xorbs) { | |
| const remoteXorbId = -remoteXorbHashes.length; | |
| remoteXorbHashes.push(xorb.hash); | |
| let i = 0; | |
| for (const xorbChunk of xorb.chunks) { | |
| cache.addChunkToCache(xorbChunk.hash, remoteXorbId, i++, shardData.hmacKey); | |
| } | |
| } | |
| cacheData = cache.getChunk(chunk2.hash, chunkModule.compute_hmac); | |
| } | |
| } | |
| if (cacheData !== void 0) { | |
| dedupedBytes += chunk2.length; | |
| } | |
| bytesSinceRemoteDedup += chunk2.length; | |
| } | |
| }; | |
| while (true) { | |
| if (_optionalChain([opts, 'optionalAccess', _83 => _83.end]) !== void 0 && totalBytes >= opts.end) { | |
| break; | |
| } | |
| if (_optionalChain([opts, 'optionalAccess', _84 => _84.maxChunks]) !== void 0 && chunksProcessed >= opts.maxChunks) { | |
| break; | |
| } | |
| const { done, value } = await reader.read(); | |
| if (done) { | |
| await processChunks(chunker.finish()); | |
| break; | |
| } | |
| sourceChunks.push(value); | |
| await processChunks(chunker.add_data(value)); | |
| } | |
| } finally { | |
| chunker.free(); | |
| } | |
| } | |
| // src/utils/uploadShards.ts | |
| var SHARD_MAX_SIZE = 64 * 1024 * 1024; | |
| var SHARD_HEADER_SIZE = 48; | |
| var SHARD_FOOTER_SIZE = 200; | |
| var HASH_LENGTH2 = 32; | |
| var XORB_FOOTER_LENGTH = 48; | |
| var FILE_FOOTER_LENGTH = 48; | |
| var SHARD_HEADER_VERSION = 2n; | |
| var SHARD_FOOTER_VERSION = 1n; | |
| var MDB_FILE_FLAG_WITH_VERIFICATION = 2147483648; | |
| var MDB_FILE_FLAG_WITH_METADATA_EXT = 1073741824; | |
| var EMPTY_HEX_HASH = "0".repeat(HASH_LENGTH2 * 2); | |
| var SHARD_MAGIC_TAG = new Uint8Array([ | |
| "H".charCodeAt(0), | |
| "F".charCodeAt(0), | |
| "R".charCodeAt(0), | |
| "e".charCodeAt(0), | |
| "p".charCodeAt(0), | |
| "o".charCodeAt(0), | |
| "M".charCodeAt(0), | |
| "e".charCodeAt(0), | |
| "t".charCodeAt(0), | |
| "a".charCodeAt(0), | |
| "D".charCodeAt(0), | |
| "a".charCodeAt(0), | |
| "t".charCodeAt(0), | |
| "a".charCodeAt(0), | |
| 0, | |
| 85, | |
| 105, | |
| 103, | |
| 69, | |
| 106, | |
| 123, | |
| 129, | |
| 87, | |
| 131, | |
| 165, | |
| 189, | |
| 217, | |
| 92, | |
| 205, | |
| 209, | |
| 74, | |
| 169 | |
| ]); | |
| async function* uploadShards(source, params) { | |
| const xorbHashes = []; | |
| const fileInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.25); | |
| const xorbInfoSection = new Uint8Array(Math.floor(SHARD_MAX_SIZE - SHARD_HEADER_SIZE - SHARD_FOOTER_SIZE) * 0.75); | |
| const xorbView = new DataView(xorbInfoSection.buffer); | |
| let xorbViewOffset = 0; | |
| const fileInfoView = new DataView(fileInfoSection.buffer); | |
| let fileViewOffset = 0; | |
| let xorbTotalSize = 0n; | |
| let fileTotalSize = 0n; | |
| let xorbTotalUnpackedSize = 0n; | |
| for await (const output of createXorbs(source, params)) { | |
| switch (output.event) { | |
| case "xorb": { | |
| xorbHashes.push(output.hash); | |
| const xorbEntrySize = HASH_LENGTH2 + 4 + 4 + 4 + 4; | |
| const chunksSize = output.chunks.length * (HASH_LENGTH2 + 4 + 4 + 8); | |
| const totalXorbSize = xorbEntrySize + chunksSize; | |
| if (xorbViewOffset + totalXorbSize > xorbInfoSection.length) { | |
| if (xorbViewOffset > 0 || fileViewOffset > 0) { | |
| await uploadShard(createShard(), params); | |
| } | |
| } | |
| writeHashToArray(output.hash, xorbInfoSection, xorbViewOffset); | |
| xorbViewOffset += HASH_LENGTH2; | |
| xorbView.setUint32(xorbViewOffset, 0, true); | |
| xorbViewOffset += 4; | |
| xorbView.setUint32(xorbViewOffset, output.chunks.length, true); | |
| xorbViewOffset += 4; | |
| const xorbUnpackedSize = sum(output.chunks.map((x) => x.length)); | |
| xorbView.setUint32(xorbViewOffset, xorbUnpackedSize, true); | |
| xorbTotalUnpackedSize += BigInt(xorbUnpackedSize); | |
| xorbTotalSize += BigInt(output.xorb.byteLength); | |
| xorbViewOffset += 4; | |
| xorbView.setUint32(xorbViewOffset, output.xorb.byteLength, true); | |
| xorbViewOffset += 4; | |
| let chunkBytes = 0; | |
| for (const chunk2 of output.chunks) { | |
| writeHashToArray(chunk2.hash, xorbInfoSection, xorbViewOffset); | |
| xorbViewOffset += HASH_LENGTH2; | |
| xorbView.setUint32(xorbViewOffset, chunkBytes, true); | |
| xorbViewOffset += 4; | |
| xorbView.setUint32(xorbViewOffset, chunk2.length, true); | |
| xorbViewOffset += 4; | |
| xorbView.setBigUint64(xorbViewOffset, 0n, true); | |
| xorbViewOffset += 8; | |
| chunkBytes += chunk2.length; | |
| } | |
| for (const file of output.files) { | |
| yield { | |
| event: "fileProgress", | |
| path: file.path, | |
| progress: file.lastSentProgress | |
| }; | |
| } | |
| await uploadXorb(output, params); | |
| for (const file of output.files) { | |
| yield { event: "fileProgress", path: file.path, progress: file.progress }; | |
| } | |
| break; | |
| } | |
| case "file": { | |
| yield { | |
| event: "file", | |
| path: output.path, | |
| xetHash: output.hash, | |
| sha256: output.sha256, | |
| dedupRatio: output.dedupRatio | |
| }; | |
| const fileHeaderSize = HASH_LENGTH2 + 4 + 4 + 8; | |
| const representationSize = output.representation.length * (HASH_LENGTH2 + 4 + 4 + 4 + 4); | |
| const verificationSize = output.representation.length * (HASH_LENGTH2 + 16); | |
| const metadataSize = HASH_LENGTH2 + 16; | |
| const totalFileSize = fileHeaderSize + representationSize + verificationSize + metadataSize; | |
| if (fileViewOffset + totalFileSize > fileInfoSection.length) { | |
| if (xorbViewOffset > 0 || fileViewOffset > 0) { | |
| await uploadShard(createShard(), params); | |
| } | |
| } | |
| writeHashToArray(output.hash, fileInfoSection, fileViewOffset); | |
| fileViewOffset += HASH_LENGTH2; | |
| fileInfoView.setUint32(fileViewOffset, MDB_FILE_FLAG_WITH_METADATA_EXT + MDB_FILE_FLAG_WITH_VERIFICATION, true); | |
| fileViewOffset += 4; | |
| fileInfoView.setUint32(fileViewOffset, output.representation.length, true); | |
| fileViewOffset += 4; | |
| fileInfoView.setBigUint64(fileViewOffset, 0n, true); | |
| fileViewOffset += 8; | |
| for (const repItem of output.representation) { | |
| writeHashToArray( | |
| typeof repItem.xorbId === "number" ? xorbHashes[repItem.xorbId] : repItem.xorbId, | |
| fileInfoSection, | |
| fileViewOffset | |
| ); | |
| fileViewOffset += HASH_LENGTH2; | |
| fileInfoView.setUint32(fileViewOffset, 0, true); | |
| fileViewOffset += 4; | |
| fileInfoView.setUint32(fileViewOffset, repItem.length, true); | |
| fileViewOffset += 4; | |
| fileInfoView.setUint32(fileViewOffset, repItem.indexStart, true); | |
| fileViewOffset += 4; | |
| fileInfoView.setUint32(fileViewOffset, repItem.indexEnd, true); | |
| fileViewOffset += 4; | |
| } | |
| for (const repItem of output.representation) { | |
| writeHashToArray(repItem.rangeHash, fileInfoSection, fileViewOffset); | |
| fileViewOffset += HASH_LENGTH2; | |
| for (let i = 0; i < 16; i++) { | |
| fileInfoSection[fileViewOffset + i] = 0; | |
| } | |
| fileViewOffset += 16; | |
| } | |
| writeHashToArray(_nullishCoalesce(output.sha256, () => ( EMPTY_HEX_HASH)), fileInfoSection, fileViewOffset); | |
| fileViewOffset += HASH_LENGTH2; | |
| for (let i = 0; i < 16; i++) { | |
| fileInfoSection[fileViewOffset + i] = 0; | |
| } | |
| fileViewOffset += 16; | |
| break; | |
| } | |
| } | |
| } | |
| function createShard() { | |
| const shard = new Uint8Array( | |
| SHARD_HEADER_SIZE + SHARD_FOOTER_SIZE + xorbViewOffset + XORB_FOOTER_LENGTH + fileViewOffset + FILE_FOOTER_LENGTH | |
| ); | |
| const shardView = new DataView(shard.buffer); | |
| let shardOffset = 0; | |
| shard.set(SHARD_MAGIC_TAG, shardOffset); | |
| shardOffset += SHARD_MAGIC_TAG.length; | |
| shardView.setBigUint64(shardOffset, SHARD_HEADER_VERSION, true); | |
| shardOffset += 8; | |
| shardView.setBigUint64(shardOffset, BigInt(SHARD_FOOTER_SIZE), true); | |
| shardOffset += 8; | |
| shard.set(fileInfoSection.slice(0, fileViewOffset), shardOffset); | |
| shardOffset += fileViewOffset; | |
| for (let i = 0; i < 32; i++) { | |
| shard[shardOffset + i] = 255; | |
| } | |
| shardOffset += 32; | |
| for (let i = 0; i < 16; i++) { | |
| shard[shardOffset + i] = 0; | |
| } | |
| shardOffset += 16; | |
| const xorbInfoOffset = shardOffset; | |
| shard.set(xorbInfoSection.slice(0, xorbViewOffset), shardOffset); | |
| shardOffset += xorbViewOffset; | |
| for (let i = 0; i < 32; i++) { | |
| shard[shardOffset + i] = 255; | |
| } | |
| shardOffset += 32; | |
| for (let i = 0; i < 16; i++) { | |
| shard[shardOffset + i] = 0; | |
| } | |
| shardOffset += 16; | |
| const footerOffset = shardOffset; | |
| shardView.setBigUint64(shardOffset, SHARD_FOOTER_VERSION, true); | |
| shardOffset += 8; | |
| shardView.setBigUint64(shardOffset, BigInt(SHARD_HEADER_SIZE), true); | |
| shardOffset += 8; | |
| shardView.setBigUint64(shardOffset, BigInt(xorbInfoOffset), true); | |
| shardOffset += 8; | |
| for (let i = 0; i < 48; i++) { | |
| shardView.setUint8(shardOffset + i, 0); | |
| } | |
| shardOffset += 48; | |
| for (let i = 0; i < 32; i++) { | |
| shardView.setUint8(shardOffset + i, 0); | |
| } | |
| shardOffset += 32; | |
| shardView.setBigUint64(shardOffset, BigInt(Math.floor(Date.now() / 1e3)), true); | |
| shardOffset += 8; | |
| shardView.setBigUint64(shardOffset, 0n, true); | |
| shardOffset += 8; | |
| for (let i = 0; i < 48; i++) { | |
| shardView.setUint8(shardOffset + i, 0); | |
| } | |
| shardOffset += 48; | |
| shardView.setBigUint64(shardOffset, xorbTotalSize, true); | |
| shardOffset += 8; | |
| shardView.setBigUint64(shardOffset, fileTotalSize, true); | |
| shardOffset += 8; | |
| shardView.setBigUint64(shardOffset, xorbTotalUnpackedSize, true); | |
| shardOffset += 8; | |
| shardView.setBigUint64(shardOffset, BigInt(footerOffset), true); | |
| xorbViewOffset = 0; | |
| fileViewOffset = 0; | |
| xorbTotalSize = 0n; | |
| xorbTotalUnpackedSize = 0n; | |
| fileTotalSize = 0n; | |
| return shard; | |
| } | |
| if (xorbViewOffset || fileViewOffset) { | |
| await uploadShard(createShard(), params); | |
| } | |
| } | |
| function writeHashToArray(hash2, array, offset) { | |
| for (let i = 0; i < hash2.length; i += 16) { | |
| array[offset + i / 2] = parseInt(hash2.substring(i + 2 * 7, i + 2 * 8), 16); | |
| array[offset + i / 2 + 1] = parseInt(hash2.substring(i + 2 * 6, i + 2 * 7), 16); | |
| array[offset + i / 2 + 2] = parseInt(hash2.substring(i + 2 * 5, i + 2 * 6), 16); | |
| array[offset + i / 2 + 3] = parseInt(hash2.substring(i + 2 * 4, i + 2 * 5), 16); | |
| array[offset + i / 2 + 4] = parseInt(hash2.substring(i + 2 * 3, i + 2 * 4), 16); | |
| array[offset + i / 2 + 5] = parseInt(hash2.substring(i + 2 * 2, i + 2 * 3), 16); | |
| array[offset + i / 2 + 6] = parseInt(hash2.substring(i + 2 * 1, i + 2 * 2), 16); | |
| array[offset + i / 2 + 7] = parseInt(hash2.substring(i + 2 * 0, i + 2 * 1), 16); | |
| } | |
| } | |
| async function uploadXorb(xorb, params) { | |
| const token = await xetWriteToken(params); | |
| const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${token.casUrl}/v1/xorbs/default/${xorb.hash}`, { | |
| method: "POST", | |
| body: xorb.xorb, | |
| headers: { | |
| Authorization: `Bearer ${token.accessToken}`, | |
| ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} | |
| }, | |
| ...{ | |
| progressHint: { | |
| progressCallback: (progress) => { | |
| for (const file of xorb.files) { | |
| _optionalChain([params, 'access', _85 => _85.yieldCallback, 'optionalCall', _86 => _86({ | |
| event: "fileProgress", | |
| path: file.path, | |
| progress: file.lastSentProgress + (file.progress - file.lastSentProgress) * progress | |
| })]); | |
| } | |
| } | |
| } | |
| } | |
| }); | |
| if (!resp.ok) { | |
| throw await createApiError(resp); | |
| } | |
| } | |
| async function uploadShard(shard, params) { | |
| const token = await xetWriteToken(params); | |
| const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${token.casUrl}/v1/shards`, { | |
| method: "POST", | |
| body: shard, | |
| headers: { | |
| Authorization: `Bearer ${token.accessToken}`, | |
| ...params.xetParams.sessionId ? { "X-Xet-Session-Id": params.xetParams.sessionId } : {} | |
| } | |
| }); | |
| if (!resp.ok) { | |
| throw await createApiError(resp); | |
| } | |
| } | |
| // src/utils/splitAsyncGenerator.ts | |
| function splitAsyncGenerator(source, n) { | |
| if (n <= 0) { | |
| return []; | |
| } | |
| const sleep = (ms) => new Promise((resolve2) => setTimeout(resolve2, ms)); | |
| let takenIndex = null; | |
| const generators = []; | |
| let remaining = n; | |
| for (let i = 0; i < n; i++) { | |
| generators.push({ | |
| next: async () => { | |
| while (takenIndex !== null) { | |
| await sleep(1); | |
| } | |
| takenIndex = i; | |
| return source.next().then((r) => { | |
| takenIndex = null; | |
| return r; | |
| }); | |
| }, | |
| return: async () => { | |
| remaining--; | |
| if (remaining === 0) { | |
| return source.return(void 0); | |
| } | |
| return { | |
| done: true, | |
| value: void 0 | |
| }; | |
| }, | |
| throw: async (error) => { | |
| return source.throw(error); | |
| }, | |
| [Symbol.asyncIterator]: () => generators[i] | |
| }); | |
| } | |
| return generators; | |
| } | |
| // src/lib/commit.ts | |
| var CONCURRENT_SHAS = 5; | |
| var CONCURRENT_LFS_UPLOADS = 5; | |
| var MULTIPART_PARALLEL_UPLOAD = 5; | |
| function isFileOperation(op) { | |
| const ret = op.operation === "addOrUpdate"; | |
| if (ret && !(op.content instanceof Blob)) { | |
| throw new TypeError("Precondition failed: op.content should be a Blob"); | |
| } | |
| return ret; | |
| } | |
| async function* commitIter(params) { | |
| const accessToken = checkCredentials(params); | |
| const repoId = toRepoId(params.repo); | |
| if (repoId.type === "bucket") { | |
| return yield* commitIterBucket(params); | |
| } | |
| yield { event: "phase", phase: "preuploading" }; | |
| let useXet = _nullishCoalesce(params.useXet, () => ( true)); | |
| const lfsShas = /* @__PURE__ */ new Map(); | |
| const abortController = new AbortController(); | |
| const abortSignal = abortController.signal; | |
| if (!abortSignal.throwIfAborted) { | |
| abortSignal.throwIfAborted = () => { | |
| if (abortSignal.aborted) { | |
| throw new DOMException("Aborted", "AbortError"); | |
| } | |
| }; | |
| } | |
| if (params.abortSignal) { | |
| params.abortSignal.addEventListener("abort", () => abortController.abort()); | |
| } | |
| try { | |
| const allOperations = (await Promise.all( | |
| params.operations.map(async (operation) => { | |
| if (operation.operation === "edit") { | |
| const splicedBlob = SplicedBlob.create( | |
| operation.originalContent, | |
| operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })) | |
| ); | |
| return { | |
| operation: "addOrUpdate", | |
| path: operation.path, | |
| content: splicedBlob | |
| }; | |
| } | |
| if (operation.operation !== "addOrUpdate") { | |
| return operation; | |
| } | |
| if (!(operation.content instanceof URL)) { | |
| return { ...operation, content: operation.content }; | |
| } | |
| const lazyBlobs = await createBlobs(operation.content, operation.path, { | |
| fetch: params.fetch, | |
| maxFolderDepth: params.maxFolderDepth | |
| }); | |
| _optionalChain([abortSignal, 'optionalAccess', _87 => _87.throwIfAborted, 'call', _88 => _88()]); | |
| return lazyBlobs.map((blob) => ({ | |
| ...operation, | |
| content: blob.blob, | |
| path: blob.path | |
| })); | |
| }) | |
| )).flat(1); | |
| const gitAttributes = _optionalChain([allOperations, 'access', _89 => _89.filter, 'call', _90 => _90(isFileOperation), 'access', _91 => _91.find, 'call', _92 => _92((op) => op.path === ".gitattributes"), 'optionalAccess', _93 => _93.content]); | |
| for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { | |
| const payload = { | |
| gitAttributes: gitAttributes && await gitAttributes.text(), | |
| files: await Promise.all( | |
| operations.map(async (operation) => ({ | |
| path: operation.path, | |
| size: operation.content.size, | |
| sample: base64FromBytes(new Uint8Array(await operation.content.slice(0, 512).arrayBuffer())) | |
| })) | |
| ) | |
| }; | |
| _optionalChain([abortSignal, 'optionalAccess', _94 => _94.throwIfAborted, 'call', _95 => _95()]); | |
| const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( | |
| `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/preupload/${encodeURIComponent( | |
| _nullishCoalesce(params.branch, () => ( "main")) | |
| )}` + (params.isPullRequest ? "?create_pr=1" : ""), | |
| { | |
| method: "POST", | |
| headers: { | |
| ...accessToken && { Authorization: `Bearer ${accessToken}` }, | |
| "Content-Type": "application/json" | |
| }, | |
| body: JSON.stringify(payload), | |
| signal: abortSignal | |
| } | |
| ); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| const json = await res.json(); | |
| for (const file of json.files) { | |
| if (file.uploadMode === "lfs") { | |
| lfsShas.set(file.path, null); | |
| } | |
| } | |
| } | |
| yield { event: "phase", phase: "uploadingLargeFiles" }; | |
| for (const operations of chunk( | |
| allOperations.filter(isFileOperation).filter((op) => lfsShas.has(op.path)), | |
| 100 | |
| )) { | |
| const shas = yield* eventToGenerator((yieldCallback, returnCallback, rejectCallack) => { | |
| return promisesQueue( | |
| operations.map((op) => async () => { | |
| const iterator = sha256(op.content, { useWebWorker: params.useWebWorkers, abortSignal }); | |
| let res2; | |
| do { | |
| res2 = await iterator.next(); | |
| if (!res2.done) { | |
| yieldCallback({ event: "fileProgress", path: op.path, progress: res2.value, state: "hashing" }); | |
| } | |
| } while (!res2.done); | |
| const sha = res2.value; | |
| lfsShas.set(op.path, res2.value); | |
| return sha; | |
| }), | |
| CONCURRENT_SHAS | |
| ).then(returnCallback, rejectCallack); | |
| }); | |
| _optionalChain([abortSignal, 'optionalAccess', _96 => _96.throwIfAborted, 'call', _97 => _97()]); | |
| const payload = { | |
| operation: "upload", | |
| // multipart is a custom protocol for HF | |
| transfers: ["basic", "multipart", ...useXet ? ["xet"] : []], | |
| hash_algo: "sha_256", | |
| ...!params.isPullRequest && { | |
| ref: { | |
| name: _nullishCoalesce(params.branch, () => ( "main")) | |
| } | |
| }, | |
| objects: operations.map((op, i) => ({ | |
| oid: shas[i], | |
| size: op.content.size | |
| })) | |
| }; | |
| const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( | |
| `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/${repoId.type === "model" ? "" : repoId.type + "s/"}${repoId.name}.git/info/lfs/objects/batch`, | |
| { | |
| method: "POST", | |
| headers: { | |
| ...accessToken && { Authorization: `Bearer ${accessToken}` }, | |
| Accept: "application/vnd.git-lfs+json", | |
| "Content-Type": "application/vnd.git-lfs+json" | |
| }, | |
| body: JSON.stringify(payload), | |
| signal: abortSignal | |
| } | |
| ); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| const json = await res.json(); | |
| const batchRequestId = res.headers.get("X-Request-Id") || void 0; | |
| const shaToOperation = new Map(operations.map((op, i) => [shas[i], op])); | |
| if (useXet && json.transfer !== "xet") { | |
| useXet = false; | |
| } | |
| let xetParams = null; | |
| if (useXet) { | |
| for (const obj of json.objects) { | |
| const op = shaToOperation.get(obj.oid); | |
| if (!op) { | |
| throw new InvalidApiResponseFormatError("Unrequested object ID in response"); | |
| } | |
| if (obj.error) { | |
| const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${obj.error.message}${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; | |
| throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); | |
| } | |
| if (!_optionalChain([obj, 'access', _98 => _98.actions, 'optionalAccess', _99 => _99.upload])) { | |
| yield { | |
| event: "fileProgress", | |
| path: op.path, | |
| progress: 1, | |
| state: "uploading" | |
| }; | |
| } else { | |
| const headers = new Headers(obj.actions.upload.header); | |
| xetParams = { | |
| sessionId: _nullishCoalesce(headers.get("X-Xet-Session-Id"), () => ( void 0)), | |
| casUrl: _nullishCoalesce(headers.get("X-Xet-Cas-Url"), () => ( void 0)), | |
| accessToken: _nullishCoalesce(headers.get("X-Xet-Access-Token"), () => ( void 0)), | |
| expiresAt: headers.get("X-Xet-Token-Expiration") ? new Date(parseInt(_nullishCoalesce(headers.get("X-Xet-Token-Expiration"), () => ( "0"))) * 1e3) : void 0, | |
| refreshWriteTokenUrl: obj.actions.upload.href | |
| }; | |
| } | |
| } | |
| const source = async function* () { | |
| for (const obj of json.objects) { | |
| const op = shaToOperation.get(obj.oid); | |
| if (!op || !_optionalChain([obj, 'access', _100 => _100.actions, 'optionalAccess', _101 => _101.upload])) { | |
| continue; | |
| } | |
| _optionalChain([abortSignal, 'optionalAccess', _102 => _102.throwIfAborted, 'call', _103 => _103()]); | |
| yield { content: op.content, path: op.path, sha256: obj.oid }; | |
| } | |
| }(); | |
| if (xetParams) { | |
| const fixedXetParams = xetParams; | |
| const sources = splitAsyncGenerator(source, 5); | |
| yield* eventToGenerator( | |
| (yieldCallback, returnCallback, rejectCallback) => Promise.all( | |
| sources.map(async function(source2) { | |
| for await (const event of uploadShards(source2, { | |
| fetch: params.fetch, | |
| accessToken, | |
| hubUrl: _nullishCoalesce(params.hubUrl, () => ( HUB_URL)), | |
| repo: repoId, | |
| xetParams: fixedXetParams, | |
| // todo: maybe leave empty if PR? | |
| rev: _nullishCoalesce(params.branch, () => ( "main")), | |
| isPullRequest: params.isPullRequest, | |
| yieldCallback: (event2) => yieldCallback({ ...event2, state: "uploading" }) | |
| })) { | |
| if (event.event === "file") { | |
| yieldCallback({ | |
| event: "fileProgress", | |
| path: event.path, | |
| progress: 1, | |
| state: "uploading" | |
| }); | |
| } else if (event.event === "fileProgress") { | |
| yieldCallback({ | |
| event: "fileProgress", | |
| path: event.path, | |
| progress: event.progress, | |
| state: "uploading" | |
| }); | |
| } | |
| } | |
| }) | |
| ).then(() => returnCallback(void 0), rejectCallback) | |
| ); | |
| } else { | |
| } | |
| } else { | |
| yield* eventToGenerator((yieldCallback, returnCallback, rejectCallback) => { | |
| return promisesQueueStreaming( | |
| json.objects.map((obj) => async () => { | |
| const op = shaToOperation.get(obj.oid); | |
| if (!op) { | |
| throw new InvalidApiResponseFormatError("Unrequested object ID in response"); | |
| } | |
| _optionalChain([abortSignal, 'optionalAccess', _104 => _104.throwIfAborted, 'call', _105 => _105()]); | |
| if (obj.error) { | |
| const errorMessage = `Error while doing LFS batch call for ${operations[shas.indexOf(obj.oid)].path}: ${obj.error.message}${batchRequestId ? ` - Request ID: ${batchRequestId}` : ""}`; | |
| throw new HubApiError(res.url, obj.error.code, batchRequestId, errorMessage); | |
| } | |
| if (!_optionalChain([obj, 'access', _106 => _106.actions, 'optionalAccess', _107 => _107.upload])) { | |
| yieldCallback({ | |
| event: "fileProgress", | |
| path: op.path, | |
| progress: 1, | |
| state: "uploading" | |
| }); | |
| return; | |
| } | |
| yieldCallback({ | |
| event: "fileProgress", | |
| path: op.path, | |
| progress: 0, | |
| state: "uploading" | |
| }); | |
| const content = op.content; | |
| const header = obj.actions.upload.header; | |
| if (_optionalChain([header, 'optionalAccess', _108 => _108.chunk_size])) { | |
| const chunkSize = parseInt(header.chunk_size); | |
| const completionUrl = obj.actions.upload.href; | |
| const parts = Object.keys(header).filter((key) => /^[0-9]+$/.test(key)); | |
| if (parts.length !== Math.ceil(content.size / chunkSize)) { | |
| throw new Error("Invalid server response to upload large LFS file, wrong number of parts"); | |
| } | |
| const completeReq = { | |
| oid: obj.oid, | |
| parts: parts.map((part) => ({ | |
| partNumber: +part, | |
| etag: "" | |
| })) | |
| }; | |
| const progressCallback = (progress) => yieldCallback({ event: "fileProgress", path: op.path, progress, state: "uploading" }); | |
| await promisesQueueStreaming( | |
| parts.map((part) => async () => { | |
| _optionalChain([abortSignal, 'optionalAccess', _109 => _109.throwIfAborted, 'call', _110 => _110()]); | |
| const index = parseInt(part) - 1; | |
| const slice = content.slice(index * chunkSize, (index + 1) * chunkSize); | |
| const res3 = await (_nullishCoalesce(params.fetch, () => ( fetch)))(header[part], { | |
| method: "PUT", | |
| /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ | |
| body: slice instanceof WebBlob && isFrontend ? await slice.arrayBuffer() : slice, | |
| signal: abortSignal, | |
| ...{ | |
| progressHint: { | |
| path: op.path, | |
| part: index, | |
| numParts: parts.length, | |
| progressCallback | |
| } | |
| // eslint-disable-next-line @typescript-eslint/no-explicit-any | |
| } | |
| }); | |
| if (!res3.ok) { | |
| throw await createApiError(res3, { | |
| requestId: batchRequestId, | |
| message: `Error while uploading part ${part} of ${operations[shas.indexOf(obj.oid)].path} to LFS storage` | |
| }); | |
| } | |
| const eTag = res3.headers.get("ETag"); | |
| if (!eTag) { | |
| throw new Error("Cannot get ETag of part during multipart upload"); | |
| } | |
| completeReq.parts[Number(part) - 1].etag = eTag; | |
| }), | |
| MULTIPART_PARALLEL_UPLOAD | |
| ); | |
| _optionalChain([abortSignal, 'optionalAccess', _111 => _111.throwIfAborted, 'call', _112 => _112()]); | |
| const res2 = await (_nullishCoalesce(params.fetch, () => ( fetch)))(completionUrl, { | |
| method: "POST", | |
| body: JSON.stringify(completeReq), | |
| headers: { | |
| Accept: "application/vnd.git-lfs+json", | |
| "Content-Type": "application/vnd.git-lfs+json" | |
| }, | |
| signal: abortSignal | |
| }); | |
| if (!res2.ok) { | |
| throw await createApiError(res2, { | |
| requestId: batchRequestId, | |
| message: `Error completing multipart upload of ${operations[shas.indexOf(obj.oid)].path} to LFS storage` | |
| }); | |
| } | |
| yieldCallback({ | |
| event: "fileProgress", | |
| path: op.path, | |
| progress: 1, | |
| state: "uploading" | |
| }); | |
| } else { | |
| const res2 = await (_nullishCoalesce(params.fetch, () => ( fetch)))(obj.actions.upload.href, { | |
| method: "PUT", | |
| headers: { | |
| ...batchRequestId ? { "X-Request-Id": batchRequestId } : void 0 | |
| }, | |
| /** Unfortunately, browsers don't support our inherited version of Blob in fetch calls */ | |
| body: content instanceof WebBlob && isFrontend ? await content.arrayBuffer() : content, | |
| signal: abortSignal, | |
| ...{ | |
| progressHint: { | |
| path: op.path, | |
| progressCallback: (progress) => yieldCallback({ | |
| event: "fileProgress", | |
| path: op.path, | |
| progress, | |
| state: "uploading" | |
| }) | |
| } | |
| // eslint-disable-next-line @typescript-eslint/no-explicit-any | |
| } | |
| }); | |
| if (!res2.ok) { | |
| throw await createApiError(res2, { | |
| requestId: batchRequestId, | |
| message: `Error while uploading ${operations[shas.indexOf(obj.oid)].path} to LFS storage` | |
| }); | |
| } | |
| yieldCallback({ | |
| event: "fileProgress", | |
| path: op.path, | |
| progress: 1, | |
| state: "uploading" | |
| }); | |
| } | |
| }), | |
| CONCURRENT_LFS_UPLOADS | |
| ).then(returnCallback, rejectCallback); | |
| }); | |
| } | |
| } | |
| _optionalChain([abortSignal, 'optionalAccess', _113 => _113.throwIfAborted, 'call', _114 => _114()]); | |
| yield { event: "phase", phase: "committing" }; | |
| return yield* eventToGenerator( | |
| async (yieldCallback, returnCallback, rejectCallback) => (_nullishCoalesce(params.fetch, () => ( fetch)))( | |
| `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/commit/${encodeURIComponent( | |
| _nullishCoalesce(params.branch, () => ( "main")) | |
| )}` + (params.isPullRequest ? "?create_pr=1" : ""), | |
| { | |
| method: "POST", | |
| headers: { | |
| ...accessToken && { Authorization: `Bearer ${accessToken}` }, | |
| "Content-Type": "application/x-ndjson" | |
| }, | |
| body: [ | |
| { | |
| key: "header", | |
| value: { | |
| summary: params.title, | |
| description: params.description, | |
| parentCommit: params.parentCommit | |
| } | |
| }, | |
| ...await Promise.all( | |
| allOperations.map((operation) => { | |
| if (isFileOperation(operation)) { | |
| const sha = lfsShas.get(operation.path); | |
| if (sha) { | |
| return { | |
| key: "lfsFile", | |
| value: { | |
| path: operation.path, | |
| algo: "sha256", | |
| size: operation.content.size, | |
| oid: sha | |
| } | |
| }; | |
| } | |
| } | |
| return convertOperationToNdJson(operation); | |
| }) | |
| ) | |
| ].map((x) => JSON.stringify(x)).join("\n"), | |
| signal: abortSignal, | |
| ...{ | |
| progressHint: { | |
| progressCallback: (progress) => { | |
| for (const op of allOperations) { | |
| if (isFileOperation(op) && !lfsShas.has(op.path)) { | |
| yieldCallback({ | |
| event: "fileProgress", | |
| path: op.path, | |
| progress, | |
| state: "uploading" | |
| }); | |
| } | |
| } | |
| } | |
| } | |
| // eslint-disable-next-line @typescript-eslint/no-explicit-any | |
| } | |
| } | |
| ).then(async (res) => { | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| const json = await res.json(); | |
| returnCallback({ | |
| pullRequestUrl: json.pullRequestUrl, | |
| commit: { | |
| oid: json.commitOid, | |
| url: json.commitUrl | |
| }, | |
| hookOutput: json.hookOutput | |
| }); | |
| }).catch(rejectCallback) | |
| ); | |
| } catch (err) { | |
| abortController.abort(); | |
| throw err; | |
| } | |
| } | |
| async function* commitIterBucket(params) { | |
| const accessToken = checkCredentials(params); | |
| const repoId = toRepoId(params.repo); | |
| if (params.useXet === false) { | |
| throw new Error("useXet must be true or undefined for buckets"); | |
| } | |
| const abortController = new AbortController(); | |
| const abortSignal = abortController.signal; | |
| if (!abortSignal.throwIfAborted) { | |
| abortSignal.throwIfAborted = () => { | |
| if (abortSignal.aborted) { | |
| throw new DOMException("Aborted", "AbortError"); | |
| } | |
| }; | |
| } | |
| if (params.abortSignal) { | |
| params.abortSignal.addEventListener("abort", () => abortController.abort()); | |
| } | |
| try { | |
| const allOperations = (await Promise.all( | |
| params.operations.map(async (operation) => { | |
| if (operation.operation === "edit") { | |
| const splicedBlob = SplicedBlob.create( | |
| operation.originalContent, | |
| operation.edits.map((splice) => ({ insert: splice.content, start: splice.start, end: splice.end })) | |
| ); | |
| return { | |
| operation: "addOrUpdate", | |
| path: operation.path, | |
| content: splicedBlob | |
| }; | |
| } | |
| if (operation.operation !== "addOrUpdate") { | |
| return operation; | |
| } | |
| if (!(operation.content instanceof URL)) { | |
| return { ...operation, content: operation.content }; | |
| } | |
| const lazyBlobs = await createBlobs(operation.content, operation.path, { | |
| fetch: params.fetch, | |
| maxFolderDepth: params.maxFolderDepth | |
| }); | |
| _optionalChain([abortSignal, 'optionalAccess', _115 => _115.throwIfAborted, 'call', _116 => _116()]); | |
| return lazyBlobs.map((blob) => ({ | |
| ...operation, | |
| content: blob.blob, | |
| path: blob.path | |
| })); | |
| }) | |
| )).flat(1); | |
| yield { event: "phase", phase: "uploadingLargeFiles" }; | |
| for (const operations of chunk(allOperations.filter(isFileOperation), 100)) { | |
| const xetHashes = /* @__PURE__ */ new Map(); | |
| _optionalChain([abortSignal, 'optionalAccess', _117 => _117.throwIfAborted, 'call', _118 => _118()]); | |
| const source = async function* () { | |
| for (const operation of operations) { | |
| _optionalChain([abortSignal, 'optionalAccess', _119 => _119.throwIfAborted, 'call', _120 => _120()]); | |
| yield { content: operation.content, path: operation.path }; | |
| } | |
| }(); | |
| const xetParams = { | |
| sessionId: crypto.randomUUID(), | |
| refreshWriteTokenUrl: `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/xet-write-token` | |
| }; | |
| const sources = splitAsyncGenerator(source, 5); | |
| yield* eventToGenerator( | |
| (yieldCallback, returnCallback, rejectCallback) => Promise.all( | |
| sources.map(async function(source2) { | |
| for await (const event of uploadShards(source2, { | |
| fetch: params.fetch, | |
| accessToken, | |
| hubUrl: _nullishCoalesce(params.hubUrl, () => ( HUB_URL)), | |
| repo: repoId, | |
| xetParams, | |
| rev: _nullishCoalesce(params.branch, () => ( "main")), | |
| yieldCallback: (event2) => yieldCallback({ ...event2, state: "uploading" }) | |
| })) { | |
| if (event.event === "file") { | |
| yieldCallback({ | |
| event: "fileProgress", | |
| path: event.path, | |
| progress: 1, | |
| state: "uploading" | |
| }); | |
| xetHashes.set(event.path, event.xetHash); | |
| } else if (event.event === "fileProgress") { | |
| yieldCallback({ | |
| event: "fileProgress", | |
| path: event.path, | |
| progress: event.progress, | |
| state: "uploading" | |
| }); | |
| } | |
| } | |
| }) | |
| ).then(() => returnCallback(void 0), rejectCallback) | |
| ); | |
| const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))( | |
| `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/batch`, | |
| { | |
| method: "POST", | |
| headers: { | |
| ...accessToken && { Authorization: `Bearer ${accessToken}` }, | |
| "Content-Type": "application/x-ndjson" | |
| }, | |
| body: [...xetHashes.entries()].map( | |
| ([path, xetHash]) => JSON.stringify({ | |
| type: "addFile", | |
| path, | |
| xetHash | |
| }) | |
| ).join("\n"), | |
| signal: abortSignal | |
| } | |
| ); | |
| if (!resp.ok && resp.status !== 422) { | |
| throw await createApiError(resp); | |
| } | |
| const json = await resp.json(); | |
| for (const failed of json.failed) { | |
| yield { | |
| event: "fileProgress", | |
| path: failed.path, | |
| progress: 0, | |
| state: "error" | |
| }; | |
| } | |
| } | |
| _optionalChain([abortSignal, 'optionalAccess', _121 => _121.throwIfAborted, 'call', _122 => _122()]); | |
| const deletedOperations = allOperations.filter((operation) => operation.operation === "delete"); | |
| if (deletedOperations.length > 0) { | |
| const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))( | |
| `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/batch`, | |
| { | |
| method: "POST", | |
| headers: { | |
| ...accessToken && { Authorization: `Bearer ${accessToken}` }, | |
| "Content-Type": "application/x-ndjson" | |
| }, | |
| body: deletedOperations.map( | |
| (operation) => JSON.stringify({ | |
| type: "deleteFile", | |
| path: operation.path | |
| }) | |
| ).join("\n"), | |
| signal: abortSignal | |
| } | |
| ); | |
| if (!resp.ok) { | |
| throw await createApiError(resp); | |
| } | |
| const json = await resp.json(); | |
| if (json.failed.length > 0) { | |
| const failedPaths = json.failed.slice(0, 5).map((f) => f.path); | |
| throw new Error( | |
| `Failed to delete ${json.failed.length} file(s): ${failedPaths.join(", ")}${json.failed.length > 5 ? "..." : ""}, request ID: ${resp.headers.get("X-Request-Id")}` | |
| ); | |
| } | |
| } | |
| _optionalChain([abortSignal, 'optionalAccess', _123 => _123.throwIfAborted, 'call', _124 => _124()]); | |
| } catch (err) { | |
| abortController.abort(); | |
| throw err; | |
| } | |
| } | |
| async function commit(params) { | |
| const iterator = commitIter(params); | |
| const failedPaths = []; | |
| let failedCount = 0; | |
| let res = await iterator.next(); | |
| while (!res.done) { | |
| if (res.value.event === "fileProgress" && res.value.state === "error") { | |
| failedCount++; | |
| if (failedPaths.length < 5) { | |
| failedPaths.push(res.value.path); | |
| } | |
| } | |
| res = await iterator.next(); | |
| } | |
| if (failedCount > 0) { | |
| throw new Error( | |
| `Failed to upload ${failedCount} file(s): ${failedPaths.join(", ")}${failedCount > 5 ? "..." : ""}` | |
| ); | |
| } | |
| return res.value; | |
| } | |
| async function convertOperationToNdJson(operation) { | |
| switch (operation.operation) { | |
| case "addOrUpdate": { | |
| return { | |
| key: "file", | |
| value: { | |
| content: base64FromBytes(new Uint8Array(await operation.content.arrayBuffer())), | |
| path: operation.path, | |
| encoding: "base64" | |
| } | |
| }; | |
| } | |
| case "delete": { | |
| return { | |
| key: "deletedFile", | |
| value: { | |
| path: operation.path | |
| } | |
| }; | |
| } | |
| case "edit": { | |
| throw new Error( | |
| "Edit operations should be converted to addOrUpdate operations before reaching convertOperationToNdJson" | |
| ); | |
| } | |
| default: | |
| throw new TypeError("Unknown operation: " + operation.operation); | |
| } | |
| } | |
| // src/lib/count-commits.ts | |
| async function countCommits(params) { | |
| const accessToken = checkCredentials(params); | |
| const repoId = toRepoId(params.repo); | |
| const url = `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/commits/${_nullishCoalesce(params.revision, () => ( "main"))}?limit=1`; | |
| const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { | |
| headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| }); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| return parseInt(_nullishCoalesce(res.headers.get("x-total-count"), () => ( "0")), 10); | |
| } | |
| // src/lib/create-repo.ts | |
| async function createRepo(params) { | |
| const accessToken = checkCredentials(params); | |
| const repoId = toRepoId(params.repo); | |
| const [namespace, repoName] = repoId.name.split("/"); | |
| if (!namespace || !repoName) { | |
| throw new TypeError( | |
| `"${repoId.name}" is not a fully qualified repo name. It should be of the form "{namespace}/{repoName}".` | |
| ); | |
| } | |
| const res = repoId.type === "bucket" ? await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/buckets/${namespace}/${repoName}`, { | |
| method: "POST", | |
| body: JSON.stringify({ | |
| private: params.private, | |
| resourceGroupId: params.resourceGroupId | |
| }), | |
| headers: { | |
| Authorization: `Bearer ${accessToken}`, | |
| "Content-Type": "application/json" | |
| } | |
| }) : await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/repos/create`, { | |
| method: "POST", | |
| body: JSON.stringify({ | |
| name: repoName, | |
| private: params.private, | |
| organization: namespace, | |
| resourceGroupId: params.resourceGroupId, | |
| license: params.license, | |
| ...repoId.type === "space" ? { | |
| type: "space", | |
| sdk: _nullishCoalesce(params.sdk, () => ( "static")) | |
| } : { | |
| type: repoId.type | |
| }, | |
| files: params.files ? await Promise.all( | |
| params.files.map(async (file) => ({ | |
| encoding: "base64", | |
| path: file.path, | |
| content: base64FromBytes( | |
| new Uint8Array(file.content instanceof Blob ? await file.content.arrayBuffer() : file.content) | |
| ) | |
| })) | |
| ) : void 0 | |
| }), | |
| headers: { | |
| Authorization: `Bearer ${accessToken}`, | |
| "Content-Type": "application/json" | |
| } | |
| }); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| const output = await res.json(); | |
| return { repoUrl: output.url, id: output.id }; | |
| } | |
| // src/lib/create-branch.ts | |
| async function createBranch(params) { | |
| const repoId = toRepoId(params.repo); | |
| const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( | |
| `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, | |
| { | |
| method: "POST", | |
| headers: { | |
| "Content-Type": "application/json", | |
| ...params.accessToken && { | |
| Authorization: `Bearer ${params.accessToken}` | |
| } | |
| }, | |
| body: JSON.stringify({ | |
| startingPoint: params.revision, | |
| ...params.empty && { emptyBranch: true }, | |
| overwrite: params.overwrite | |
| }) | |
| } | |
| ); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| } | |
| // src/lib/create-collection.ts | |
| async function createCollection(params) { | |
| const accessToken = checkCredentials(params); | |
| const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/collections`, { | |
| method: "POST", | |
| body: JSON.stringify(params.collection), | |
| headers: { | |
| Authorization: `Bearer ${accessToken}`, | |
| "Content-Type": "application/json" | |
| } | |
| }); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| const output = await res.json(); | |
| return { slug: output.slug }; | |
| } | |
| // src/utils/pick.ts | |
| function pick(o, props) { | |
| return Object.assign( | |
| {}, | |
| ...props.map((prop) => { | |
| if (o[prop] !== void 0) { | |
| return { [prop]: o[prop] }; | |
| } | |
| }) | |
| ); | |
| } | |
| // src/utils/parseLinkHeader.ts | |
| function parseLinkHeader(header) { | |
| const regex = /<(https?:[/][/][^>]+)>;\s+rel="([^"]+)"/g; | |
| return Object.fromEntries([...header.matchAll(regex)].map(([, url, rel]) => [rel, url])); | |
| } | |
| // src/lib/list-datasets.ts | |
| var DATASET_EXPAND_KEYS = [ | |
| "private", | |
| "downloads", | |
| "gated", | |
| "likes", | |
| "lastModified" | |
| ]; | |
| var DATASET_EXPANDABLE_KEYS = [ | |
| "author", | |
| "cardData", | |
| "citation", | |
| "createdAt", | |
| "disabled", | |
| "description", | |
| "downloads", | |
| "downloadsAllTime", | |
| "gated", | |
| "gitalyUid", | |
| "lastModified", | |
| "likes", | |
| "paperswithcode_id", | |
| "private", | |
| // "siblings", | |
| "sha", | |
| "tags" | |
| ]; | |
| async function* listDatasets(params) { | |
| const accessToken = params && checkCredentials(params); | |
| let totalToFetch = _nullishCoalesce(_optionalChain([params, 'optionalAccess', _125 => _125.limit]), () => ( Infinity)); | |
| const search = new URLSearchParams([ | |
| ...Object.entries({ | |
| limit: String(Math.min(totalToFetch, 500)), | |
| ..._optionalChain([params, 'optionalAccess', _126 => _126.search, 'optionalAccess', _127 => _127.owner]) ? { author: params.search.owner } : void 0, | |
| ..._optionalChain([params, 'optionalAccess', _128 => _128.search, 'optionalAccess', _129 => _129.query]) ? { search: params.search.query } : void 0, | |
| ..._optionalChain([params, 'optionalAccess', _130 => _130.sort]) ? { sort: params.sort } : void 0 | |
| }), | |
| ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _131 => _131.search, 'optionalAccess', _132 => _132.tags, 'optionalAccess', _133 => _133.map, 'call', _134 => _134((tag) => ["filter", tag])]), () => ( [])), | |
| ...DATASET_EXPAND_KEYS.map((val) => ["expand", val]), | |
| ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _135 => _135.additionalFields, 'optionalAccess', _136 => _136.map, 'call', _137 => _137((val) => ["expand", val])]), () => ( [])) | |
| ]).toString(); | |
| let url = `${_optionalChain([params, 'optionalAccess', _138 => _138.hubUrl]) || HUB_URL}/api/datasets` + (search ? "?" + search : ""); | |
| while (url) { | |
| const res = await (_nullishCoalesce(_optionalChain([params, 'optionalAccess', _139 => _139.fetch]), () => ( fetch)))(url, { | |
| headers: { | |
| accept: "application/json", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 | |
| } | |
| }); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| const items = await res.json(); | |
| for (const item of items) { | |
| yield { | |
| ..._optionalChain([params, 'optionalAccess', _140 => _140.additionalFields]) && pick(item, params.additionalFields), | |
| id: item._id, | |
| name: item.id, | |
| private: item.private, | |
| downloads: item.downloads, | |
| likes: item.likes, | |
| gated: item.gated, | |
| updatedAt: new Date(item.lastModified) | |
| }; | |
| totalToFetch--; | |
| if (totalToFetch <= 0) { | |
| return; | |
| } | |
| } | |
| const linkHeader = res.headers.get("Link"); | |
| url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; | |
| } | |
| } | |
| // src/lib/dataset-info.ts | |
| async function datasetInfo(params) { | |
| const accessToken = params && checkCredentials(params); | |
| const search = new URLSearchParams([ | |
| ...DATASET_EXPAND_KEYS.map((val) => ["expand", val]), | |
| ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _141 => _141.additionalFields, 'optionalAccess', _142 => _142.map, 'call', _143 => _143((val) => ["expand", val])]), () => ( [])) | |
| ]).toString(); | |
| const response = await (params.fetch || fetch)( | |
| `${_optionalChain([params, 'optionalAccess', _144 => _144.hubUrl]) || HUB_URL}/api/datasets/${params.name}/revision/${encodeURIComponent( | |
| _nullishCoalesce(params.revision, () => ( "HEAD")) | |
| )}?${search.toString()}`, | |
| { | |
| headers: { | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| const data = await response.json(); | |
| return { | |
| ..._optionalChain([params, 'optionalAccess', _145 => _145.additionalFields]) && pick(data, params.additionalFields), | |
| id: data._id, | |
| name: data.id, | |
| private: data.private, | |
| downloads: data.downloads, | |
| likes: data.likes, | |
| gated: data.gated, | |
| updatedAt: new Date(data.lastModified) | |
| }; | |
| } | |
| // src/lib/delete-branch.ts | |
| async function deleteBranch(params) { | |
| const repoId = toRepoId(params.repo); | |
| const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( | |
| `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, | |
| { | |
| method: "DELETE", | |
| headers: { | |
| ...params.accessToken && { | |
| Authorization: `Bearer ${params.accessToken}` | |
| } | |
| } | |
| } | |
| ); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| } | |
| // src/lib/delete-file.ts | |
| function deleteFile(params) { | |
| return commit({ | |
| ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, | |
| repo: params.repo, | |
| operations: [ | |
| { | |
| operation: "delete", | |
| path: params.path | |
| } | |
| ], | |
| title: _nullishCoalesce(params.commitTitle, () => ( `Delete ${params.path}`)), | |
| description: params.commitDescription, | |
| hubUrl: params.hubUrl, | |
| branch: params.branch, | |
| isPullRequest: params.isPullRequest, | |
| parentCommit: params.parentCommit, | |
| fetch: params.fetch | |
| }); | |
| } | |
| // src/lib/delete-files.ts | |
| function deleteFiles(params) { | |
| return commit({ | |
| ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, | |
| repo: params.repo, | |
| operations: params.paths.map((path) => ({ | |
| operation: "delete", | |
| path | |
| })), | |
| title: _nullishCoalesce(params.commitTitle, () => ( `Deletes ${params.paths.length} files`)), | |
| description: params.commitDescription, | |
| hubUrl: params.hubUrl, | |
| branch: params.branch, | |
| isPullRequest: params.isPullRequest, | |
| parentCommit: params.parentCommit, | |
| fetch: params.fetch | |
| }); | |
| } | |
| // src/lib/delete-repo.ts | |
| async function deleteRepo(params) { | |
| const accessToken = checkCredentials(params); | |
| const repoId = toRepoId(params.repo); | |
| const [namespace, repoName] = repoId.name.split("/"); | |
| const res = repoId.type === "bucket" ? await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/buckets/${namespace}/${repoName}`, { | |
| method: "DELETE", | |
| headers: { | |
| Authorization: `Bearer ${accessToken}`, | |
| "Content-Type": "application/json" | |
| } | |
| }) : await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/repos/delete`, { | |
| method: "DELETE", | |
| body: JSON.stringify({ | |
| name: repoName, | |
| organization: namespace, | |
| type: repoId.type | |
| }), | |
| headers: { | |
| Authorization: `Bearer ${accessToken}`, | |
| "Content-Type": "application/json" | |
| } | |
| }); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| } | |
| // src/lib/delete-collection.ts | |
| async function deleteCollection(params) { | |
| if (!params.slug) { | |
| throw new TypeError("slug is required"); | |
| } | |
| const accessToken = checkCredentials(params); | |
| const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/collections/${params.slug}`, { | |
| method: "DELETE", | |
| headers: { | |
| Authorization: `Bearer ${accessToken}`, | |
| "Content-Type": "application/json" | |
| } | |
| }); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| } | |
| // src/lib/file-download-info.ts | |
| async function fileDownloadInfo(params) { | |
| const accessToken = checkCredentials(params); | |
| const repoId = toRepoId(params.repo); | |
| const hubUrl = _nullishCoalesce(params.hubUrl, () => ( HUB_URL)); | |
| const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/${params.raw ? "raw" : "resolve"}/${encodeURIComponent(_nullishCoalesce(params.revision, () => ( "main")))}/${params.path}` + (params.noContentDisposition ? "?noContentDisposition=1" : ""); | |
| const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { | |
| method: "GET", | |
| headers: { | |
| ...accessToken && { | |
| Authorization: `Bearer ${accessToken}` | |
| }, | |
| Range: "bytes=0-0", | |
| Accept: "application/vnd.xet-fileinfo+json, */*" | |
| } | |
| }); | |
| if (resp.status === 404 && resp.headers.get("X-Error-Code") === "EntryNotFound") { | |
| return null; | |
| } | |
| if (!resp.ok) { | |
| throw await createApiError(resp); | |
| } | |
| let size; | |
| let xetInfo; | |
| if (_optionalChain([resp, 'access', _146 => _146.headers, 'access', _147 => _147.get, 'call', _148 => _148("Content-Type"), 'optionalAccess', _149 => _149.includes, 'call', _150 => _150("application/vnd.xet-fileinfo+json")])) { | |
| size = parseInt(_nullishCoalesce(resp.headers.get("X-Linked-Size"), () => ( "invalid"))); | |
| if (isNaN(size)) { | |
| throw new InvalidApiResponseFormatError("Invalid file size received in X-Linked-Size header"); | |
| } | |
| const hash2 = resp.headers.get("X-Xet-Hash"); | |
| const links = parseLinkHeader(_nullishCoalesce(resp.headers.get("Link"), () => ( ""))); | |
| const reconstructionUrl = (() => { | |
| try { | |
| return new URL(links["xet-reconstruction-info"]); | |
| } catch (e2) { | |
| return null; | |
| } | |
| })(); | |
| const refreshUrl = (() => { | |
| try { | |
| return new URL(links["xet-auth"]); | |
| } catch (e3) { | |
| return null; | |
| } | |
| })(); | |
| if (!hash2) { | |
| throw new InvalidApiResponseFormatError("No hash received in X-Xet-Hash header"); | |
| } | |
| if (!reconstructionUrl || !refreshUrl) { | |
| throw new InvalidApiResponseFormatError("No xet-reconstruction-info or xet-auth link header"); | |
| } | |
| xetInfo = { | |
| hash: hash2, | |
| refreshUrl, | |
| reconstructionUrl | |
| }; | |
| } | |
| if (size === void 0 || isNaN(size)) { | |
| const contentRangeHeader = resp.headers.get("content-range"); | |
| if (!contentRangeHeader) { | |
| throw new InvalidApiResponseFormatError("Expected size information"); | |
| } | |
| const [, parsedSize] = contentRangeHeader.split("/"); | |
| size = parseInt(parsedSize); | |
| if (isNaN(size)) { | |
| throw new InvalidApiResponseFormatError("Invalid file size received"); | |
| } | |
| } | |
| const etag = _nullishCoalesce(_nullishCoalesce(resp.headers.get("X-Linked-ETag"), () => ( resp.headers.get("ETag"))), () => ( void 0)); | |
| if (!etag) { | |
| throw new InvalidApiResponseFormatError("Expected ETag"); | |
| } | |
| return { | |
| etag, | |
| size, | |
| xet: xetInfo, | |
| // Cannot use resp.url in case it's a S3 url and the user adds an Authorization header to it. | |
| url: resp.url && (new URL(resp.url).origin === new URL(hubUrl).origin || _optionalChain([resp, 'access', _151 => _151.headers, 'access', _152 => _152.get, 'call', _153 => _153("X-Cache"), 'optionalAccess', _154 => _154.endsWith, 'call', _155 => _155(" cloudfront")])) ? resp.url : url | |
| }; | |
| } | |
| // src/lib/download-file.ts | |
| async function downloadFile(params) { | |
| const accessToken = checkCredentials(params); | |
| const info = await _asyncNullishCoalesce(params.downloadInfo, async () => ( await fileDownloadInfo({ | |
| accessToken, | |
| repo: params.repo, | |
| path: params.path, | |
| revision: params.revision, | |
| hubUrl: params.hubUrl, | |
| fetch: params.fetch, | |
| raw: params.raw | |
| }))); | |
| if (!info) { | |
| return null; | |
| } | |
| if (info.xet && params.xet !== false) { | |
| return new XetBlob({ | |
| refreshUrl: info.xet.refreshUrl.href, | |
| reconstructionUrl: info.xet.reconstructionUrl.href, | |
| fetch: params.fetch, | |
| accessToken, | |
| size: info.size, | |
| readToken: typeof params.xet === "object" ? params.xet.readToken : void 0 | |
| }); | |
| } | |
| return new WebBlob(new URL(info.url), 0, info.size, "", true, _nullishCoalesce(params.fetch, () => ( fetch)), accessToken); | |
| } | |
| // src/lib/file-exists.ts | |
| async function fileExists(params) { | |
| const accessToken = checkCredentials(params); | |
| const repoId = toRepoId(params.repo); | |
| const hubUrl = _nullishCoalesce(params.hubUrl, () => ( HUB_URL)); | |
| const url = `${hubUrl}/${repoId.type === "model" ? "" : `${repoId.type}s/`}${repoId.name}/raw/${encodeURIComponent( | |
| _nullishCoalesce(params.revision, () => ( "main")) | |
| )}/${params.path}`; | |
| const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { | |
| method: "HEAD", | |
| headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| }); | |
| if (resp.status === 404) { | |
| return false; | |
| } | |
| if (!resp.ok) { | |
| throw await createApiError(resp); | |
| } | |
| return true; | |
| } | |
| // src/lib/jobs/cancel-job.ts | |
| async function cancelJob(params) { | |
| const accessToken = checkCredentials(params); | |
| const response = await (params.fetch || fetch)( | |
| `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/cancel`, | |
| { | |
| method: "POST", | |
| headers: { | |
| "Content-Type": "application/json", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| return await response.json(); | |
| } | |
| // src/lib/jobs/create-scheduled-job.ts | |
| async function createScheduledJob(params) { | |
| const accessToken = checkCredentials(params); | |
| const { namespace, hubUrl, fetch: customFetch, ...rest } = params; | |
| if (!rest.jobSpec.dockerImage && !rest.jobSpec.spaceId) { | |
| throw new Error("Either dockerImage or spaceId must be provided in jobSpec"); | |
| } | |
| if (rest.jobSpec.dockerImage && rest.jobSpec.spaceId) { | |
| throw new Error("Cannot provide both dockerImage and spaceId in jobSpec"); | |
| } | |
| const body = { | |
| jobSpec: { | |
| flavor: rest.jobSpec.flavor | |
| }, | |
| schedule: rest.schedule, | |
| suspend: _nullishCoalesce(rest.suspend, () => ( false)), | |
| concurrency: _nullishCoalesce(rest.concurrency, () => ( false)) | |
| }; | |
| if (rest.jobSpec.dockerImage) { | |
| body.jobSpec.dockerImage = rest.jobSpec.dockerImage; | |
| } | |
| if (rest.jobSpec.spaceId) { | |
| body.jobSpec.spaceId = rest.jobSpec.spaceId; | |
| } | |
| if (rest.jobSpec.command) { | |
| body.jobSpec.command = rest.jobSpec.command; | |
| } | |
| body.jobSpec.environment = rest.jobSpec.environment || {}; | |
| if (rest.jobSpec.secrets) { | |
| body.jobSpec.secrets = rest.jobSpec.secrets; | |
| } | |
| if (rest.jobSpec.arch) { | |
| body.jobSpec.arch = rest.jobSpec.arch; | |
| } | |
| if (rest.jobSpec.timeoutSeconds !== void 0) { | |
| body.jobSpec.timeoutSeconds = rest.jobSpec.timeoutSeconds; | |
| } | |
| if (rest.jobSpec.attempts !== void 0) { | |
| body.jobSpec.attempts = rest.jobSpec.attempts; | |
| } | |
| if (rest.jobSpec.labels) { | |
| body.jobSpec.labels = rest.jobSpec.labels; | |
| } | |
| const response = await (customFetch || fetch)(`${hubUrl || HUB_URL}/api/scheduled-jobs/${namespace}`, { | |
| method: "POST", | |
| headers: { | |
| "Content-Type": "application/json", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| }, | |
| body: JSON.stringify(body) | |
| }); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| return await response.json(); | |
| } | |
| // src/lib/jobs/delete-scheduled-job.ts | |
| async function deleteScheduledJob(params) { | |
| const accessToken = checkCredentials(params); | |
| const response = await (params.fetch || fetch)( | |
| `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}`, | |
| { | |
| method: "DELETE", | |
| headers: { | |
| "Content-Type": "application/json", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| } | |
| // src/lib/jobs/duplicate-job.ts | |
| async function duplicateJob(params) { | |
| const accessToken = checkCredentials(params); | |
| const response = await (params.fetch || fetch)( | |
| `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/duplicate`, | |
| { | |
| method: "POST", | |
| headers: { | |
| "Content-Type": "application/json", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| return await response.json(); | |
| } | |
| // src/lib/jobs/get-job.ts | |
| async function getJob(params) { | |
| const accessToken = checkCredentials(params); | |
| const response = await (params.fetch || fetch)( | |
| `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}`, | |
| { | |
| headers: { | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| return await response.json(); | |
| } | |
| // src/lib/jobs/get-scheduled-job.ts | |
| async function getScheduledJob(params) { | |
| const accessToken = checkCredentials(params); | |
| const response = await (params.fetch || fetch)( | |
| `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}`, | |
| { | |
| headers: { | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| return await response.json(); | |
| } | |
| // src/lib/jobs/list-job-hardware.ts | |
| async function listJobHardware(params) { | |
| const accessToken = checkCredentials(_nullishCoalesce(params, () => ( {}))); | |
| const headers = {}; | |
| if (accessToken) { | |
| headers.Authorization = `Bearer ${accessToken}`; | |
| } | |
| const response = await (_optionalChain([params, 'optionalAccess', _156 => _156.fetch]) || fetch)(`${_optionalChain([params, 'optionalAccess', _157 => _157.hubUrl]) || HUB_URL}/api/jobs/hardware`, { | |
| headers | |
| }); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| return await response.json(); | |
| } | |
| // src/lib/jobs/list-jobs.ts | |
| async function listJobs(params) { | |
| const accessToken = checkCredentials(params); | |
| const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { | |
| headers: { | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| }); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| return await response.json(); | |
| } | |
| // src/lib/jobs/list-scheduled-jobs.ts | |
| async function listScheduledJobs(params) { | |
| const accessToken = checkCredentials(params); | |
| const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}`, { | |
| headers: { | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| }); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| return await response.json(); | |
| } | |
| // src/lib/jobs/resume-scheduled-job.ts | |
| async function resumeScheduledJob(params) { | |
| const accessToken = checkCredentials(params); | |
| const response = await (params.fetch || fetch)( | |
| `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/resume`, | |
| { | |
| method: "POST", | |
| headers: { | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| } | |
| // src/lib/jobs/run-job.ts | |
| async function runJob(params) { | |
| const accessToken = checkCredentials(params); | |
| if (!params.dockerImage && !params.spaceId) { | |
| throw new Error("Either dockerImage or spaceId must be provided"); | |
| } | |
| if (params.dockerImage && params.spaceId) { | |
| throw new Error("Cannot provide both dockerImage and spaceId"); | |
| } | |
| const body = { | |
| flavor: params.flavor, | |
| environment: params.environment || {} | |
| }; | |
| if (params.dockerImage) { | |
| body.dockerImage = params.dockerImage; | |
| } | |
| if (params.spaceId) { | |
| body.spaceId = params.spaceId; | |
| } | |
| if (params.command) { | |
| body.command = params.command; | |
| } | |
| if (params.arguments) { | |
| body.arguments = params.arguments; | |
| } | |
| if (params.secrets) { | |
| body.secrets = params.secrets; | |
| } | |
| if (params.arch) { | |
| body.arch = params.arch; | |
| } | |
| if (params.timeoutSeconds !== void 0) { | |
| body.timeoutSeconds = params.timeoutSeconds; | |
| } | |
| if (params.attempts !== void 0) { | |
| body.attempts = params.attempts; | |
| } | |
| if (params.labels) { | |
| body.labels = params.labels; | |
| } | |
| const response = await (params.fetch || fetch)(`${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}`, { | |
| method: "POST", | |
| headers: { | |
| "Content-Type": "application/json", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| }, | |
| body: JSON.stringify(body) | |
| }); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| return await response.json(); | |
| } | |
| // src/lib/jobs/run-scheduled-job.ts | |
| async function runScheduledJob(params) { | |
| const accessToken = checkCredentials(params); | |
| const response = await (params.fetch || fetch)( | |
| `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/run`, | |
| { | |
| method: "POST", | |
| headers: { | |
| "Content-Type": "application/json", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| } | |
| ); | |
| if (!response.ok) { | |
| if (response.status === 409) { | |
| return null; | |
| } | |
| throw await createApiError(response); | |
| } | |
| return await response.json(); | |
| } | |
| // src/lib/jobs/stream-job-events.ts | |
| async function* streamJobEvents(params) { | |
| const accessToken = checkCredentials(params); | |
| const response = await (params.fetch || fetch)( | |
| `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/events`, | |
| { | |
| headers: { | |
| Accept: "text/event-stream", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| if (!response.body) { | |
| return; | |
| } | |
| const reader = response.body.getReader(); | |
| const decoder = new TextDecoder(); | |
| let buffer = ""; | |
| try { | |
| while (true) { | |
| const { done, value } = await reader.read(); | |
| if (done) | |
| break; | |
| buffer += decoder.decode(value, { stream: true }); | |
| const lines = buffer.split("\n"); | |
| buffer = lines.pop() || ""; | |
| for (const line of lines) { | |
| if (line.startsWith("data: ")) { | |
| yield line.slice(6); | |
| } | |
| } | |
| } | |
| if (buffer) { | |
| const lines = buffer.split("\n"); | |
| for (const line of lines) { | |
| if (line.startsWith("data: ")) { | |
| yield line.slice(6); | |
| } | |
| } | |
| } | |
| } finally { | |
| reader.releaseLock(); | |
| } | |
| } | |
| // src/lib/jobs/stream-job-logs.ts | |
| async function* streamJobLogs(params) { | |
| const accessToken = checkCredentials(params); | |
| const response = await (params.fetch || fetch)( | |
| `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/logs`, | |
| { | |
| headers: { | |
| Accept: "text/event-stream", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| if (!response.body) { | |
| return; | |
| } | |
| const reader = response.body.getReader(); | |
| const decoder = new TextDecoder(); | |
| let buffer = ""; | |
| try { | |
| while (true) { | |
| const { done, value } = await reader.read(); | |
| if (done) | |
| break; | |
| buffer += decoder.decode(value, { stream: true }); | |
| const lines = buffer.split("\n"); | |
| buffer = lines.pop() || ""; | |
| for (const line of lines) { | |
| if (line.startsWith("data: ")) { | |
| try { | |
| const data = JSON.parse(line.slice(6)); | |
| yield { message: data.data, timestamp: new Date(data.timestamp) }; | |
| } catch (e4) { | |
| yield { message: line.slice(6), timestamp: /* @__PURE__ */ new Date() }; | |
| } | |
| } | |
| } | |
| } | |
| if (buffer) { | |
| const lines = buffer.split("\n"); | |
| for (const line of lines) { | |
| if (line.startsWith("data: ")) { | |
| try { | |
| const data = JSON.parse(line.slice(6)); | |
| yield { message: data.data, timestamp: new Date(data.timestamp) }; | |
| } catch (e5) { | |
| yield { message: line.slice(6), timestamp: /* @__PURE__ */ new Date() }; | |
| } | |
| } | |
| } | |
| } | |
| } finally { | |
| reader.releaseLock(); | |
| } | |
| } | |
| // src/lib/jobs/stream-job-metrics.ts | |
| async function* streamJobMetrics(params) { | |
| const accessToken = checkCredentials(params); | |
| const response = await (params.fetch || fetch)( | |
| `${params.hubUrl || HUB_URL}/api/jobs/${params.namespace}/${params.jobId}/metrics`, | |
| { | |
| headers: { | |
| Accept: "text/event-stream", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| if (!response.body) { | |
| return; | |
| } | |
| const reader = response.body.getReader(); | |
| const decoder = new TextDecoder(); | |
| let buffer = ""; | |
| try { | |
| while (true) { | |
| const { done, value } = await reader.read(); | |
| if (done) | |
| break; | |
| buffer += decoder.decode(value, { stream: true }); | |
| const lines = buffer.split("\n"); | |
| buffer = lines.pop() || ""; | |
| for (const line of lines) { | |
| if (line.startsWith("data: ")) { | |
| yield line.slice(6); | |
| } | |
| } | |
| } | |
| if (buffer) { | |
| const lines = buffer.split("\n"); | |
| for (const line of lines) { | |
| if (line.startsWith("data: ")) { | |
| yield line.slice(6); | |
| } | |
| } | |
| } | |
| } finally { | |
| reader.releaseLock(); | |
| } | |
| } | |
| // src/lib/jobs/suspend-scheduled-job.ts | |
| async function suspendScheduledJob(params) { | |
| const accessToken = checkCredentials(params); | |
| const response = await (params.fetch || fetch)( | |
| `${params.hubUrl || HUB_URL}/api/scheduled-jobs/${params.namespace}/${params.jobId}/suspend`, | |
| { | |
| method: "POST", | |
| headers: { | |
| "Content-Type": "application/json", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| } | |
| // src/lib/list-commits.ts | |
| async function* listCommits(params) { | |
| const accessToken = checkCredentials(params); | |
| const repoId = toRepoId(params.repo); | |
| let url = `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}/commits/${_nullishCoalesce(params.revision, () => ( "main"))}?limit=${_nullishCoalesce(params.batchSize, () => ( 100))}`; | |
| while (url) { | |
| const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { | |
| headers: accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| }); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| const resJson = await res.json(); | |
| for (const commit2 of resJson) { | |
| yield { | |
| oid: commit2.id, | |
| title: commit2.title, | |
| message: commit2.message, | |
| authors: commit2.authors.map((author) => ({ | |
| username: author.user, | |
| avatarUrl: author.avatar | |
| })), | |
| date: new Date(commit2.date) | |
| }; | |
| } | |
| const linkHeader = res.headers.get("Link"); | |
| url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; | |
| } | |
| } | |
| // src/lib/list-files.ts | |
| async function* listFiles(params) { | |
| const accessToken = checkCredentials(params); | |
| const repoId = toRepoId(params.repo); | |
| let url = `${params.hubUrl || HUB_URL}/api/${repoId.type}s/${repoId.name}/tree/${params.revision || "main"}${params.path ? "/" + params.path : ""}?recursive=${!!params.recursive}&expand=${!!params.expand}`; | |
| while (url) { | |
| const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { | |
| headers: { | |
| accept: "application/json", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 | |
| } | |
| }); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| const items = await res.json(); | |
| for (const item of items) { | |
| yield item; | |
| } | |
| const linkHeader = res.headers.get("Link"); | |
| url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; | |
| } | |
| } | |
| // src/utils/normalizeInferenceProviderMapping.ts | |
| function normalizeInferenceProviderMapping(hfModelId, inferenceProviderMapping) { | |
| if (!inferenceProviderMapping) { | |
| return []; | |
| } | |
| if (Array.isArray(inferenceProviderMapping)) { | |
| return inferenceProviderMapping.map((entry) => ({ | |
| ...entry, | |
| hfModelId | |
| })); | |
| } | |
| return Object.entries(inferenceProviderMapping).map(([provider, mapping]) => ({ | |
| provider, | |
| hfModelId, | |
| providerId: mapping.providerId, | |
| status: mapping.status, | |
| task: mapping.task | |
| })); | |
| } | |
| // src/lib/list-models.ts | |
| var MODEL_EXPAND_KEYS = [ | |
| "pipeline_tag", | |
| "private", | |
| "gated", | |
| "downloads", | |
| "likes", | |
| "lastModified" | |
| ]; | |
| var MODEL_EXPANDABLE_KEYS = [ | |
| "author", | |
| "cardData", | |
| "config", | |
| "createdAt", | |
| "disabled", | |
| "downloads", | |
| "downloadsAllTime", | |
| "gated", | |
| "gitalyUid", | |
| "inferenceProviderMapping", | |
| "lastModified", | |
| "library_name", | |
| "likes", | |
| "model-index", | |
| "pipeline_tag", | |
| "private", | |
| "safetensors", | |
| "sha", | |
| // "siblings", | |
| "spaces", | |
| "tags", | |
| "transformersInfo" | |
| ]; | |
| async function* listModels(params) { | |
| const accessToken = params && checkCredentials(params); | |
| let totalToFetch = _nullishCoalesce(_optionalChain([params, 'optionalAccess', _158 => _158.limit]), () => ( Infinity)); | |
| const search = new URLSearchParams([ | |
| ...Object.entries({ | |
| limit: String(Math.min(totalToFetch, 500)), | |
| ..._optionalChain([params, 'optionalAccess', _159 => _159.search, 'optionalAccess', _160 => _160.owner]) ? { author: params.search.owner } : void 0, | |
| ..._optionalChain([params, 'optionalAccess', _161 => _161.search, 'optionalAccess', _162 => _162.task]) ? { pipeline_tag: params.search.task } : void 0, | |
| ..._optionalChain([params, 'optionalAccess', _163 => _163.search, 'optionalAccess', _164 => _164.query]) ? { search: params.search.query } : void 0, | |
| ..._optionalChain([params, 'optionalAccess', _165 => _165.search, 'optionalAccess', _166 => _166.inferenceProviders]) ? { inference_provider: params.search.inferenceProviders.join(",") } : void 0, | |
| ..._optionalChain([params, 'optionalAccess', _167 => _167.search, 'optionalAccess', _168 => _168.apps]) ? { apps: params.search.apps.join(",") } : void 0, | |
| ..._optionalChain([params, 'optionalAccess', _169 => _169.sort]) ? { sort: params.sort } : void 0 | |
| }), | |
| ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _170 => _170.search, 'optionalAccess', _171 => _171.tags, 'optionalAccess', _172 => _172.map, 'call', _173 => _173((tag) => ["filter", tag])]), () => ( [])), | |
| ...MODEL_EXPAND_KEYS.map((val) => ["expand", val]), | |
| ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _174 => _174.additionalFields, 'optionalAccess', _175 => _175.map, 'call', _176 => _176((val) => ["expand", val])]), () => ( [])) | |
| ]).toString(); | |
| let url = `${_optionalChain([params, 'optionalAccess', _177 => _177.hubUrl]) || HUB_URL}/api/models?${search}`; | |
| while (url) { | |
| const res = await (_nullishCoalesce(_optionalChain([params, 'optionalAccess', _178 => _178.fetch]), () => ( fetch)))(url, { | |
| headers: { | |
| accept: "application/json", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 | |
| } | |
| }); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| const items = await res.json(); | |
| for (const item of items) { | |
| const normalizedItem = { ...item }; | |
| if (_optionalChain([params, 'optionalAccess', _179 => _179.additionalFields, 'optionalAccess', _180 => _180.includes, 'call', _181 => _181("inferenceProviderMapping")]) && item.inferenceProviderMapping) { | |
| normalizedItem.inferenceProviderMapping = normalizeInferenceProviderMapping( | |
| item.id, | |
| item.inferenceProviderMapping | |
| ); | |
| } | |
| yield { | |
| ..._optionalChain([params, 'optionalAccess', _182 => _182.additionalFields]) && pick(normalizedItem, params.additionalFields), | |
| id: item._id, | |
| name: item.id, | |
| private: item.private, | |
| task: item.pipeline_tag, | |
| downloads: item.downloads, | |
| gated: item.gated, | |
| likes: item.likes, | |
| updatedAt: new Date(item.lastModified) | |
| }; | |
| totalToFetch--; | |
| if (totalToFetch <= 0) { | |
| return; | |
| } | |
| } | |
| const linkHeader = res.headers.get("Link"); | |
| url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; | |
| } | |
| } | |
| // src/lib/list-spaces.ts | |
| var SPACE_EXPAND_KEYS = [ | |
| "sdk", | |
| "likes", | |
| "private", | |
| "lastModified" | |
| ]; | |
| var SPACE_EXPANDABLE_KEYS = [ | |
| "author", | |
| "cardData", | |
| "datasets", | |
| "disabled", | |
| "gitalyUid", | |
| "lastModified", | |
| "createdAt", | |
| "likes", | |
| "private", | |
| "runtime", | |
| "sdk", | |
| // "siblings", | |
| "sha", | |
| "subdomain", | |
| "tags", | |
| "models" | |
| ]; | |
| async function* listSpaces(params) { | |
| const accessToken = params && checkCredentials(params); | |
| const search = new URLSearchParams([ | |
| ...Object.entries({ | |
| limit: "500", | |
| ..._optionalChain([params, 'optionalAccess', _183 => _183.search, 'optionalAccess', _184 => _184.owner]) ? { author: params.search.owner } : void 0, | |
| ..._optionalChain([params, 'optionalAccess', _185 => _185.search, 'optionalAccess', _186 => _186.query]) ? { search: params.search.query } : void 0, | |
| ..._optionalChain([params, 'optionalAccess', _187 => _187.sort]) ? { sort: params.sort } : void 0 | |
| }), | |
| ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _188 => _188.search, 'optionalAccess', _189 => _189.tags, 'optionalAccess', _190 => _190.map, 'call', _191 => _191((tag) => ["filter", tag])]), () => ( [])), | |
| ...[...SPACE_EXPAND_KEYS, ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _192 => _192.additionalFields]), () => ( []))].map( | |
| (val) => ["expand", val] | |
| ) | |
| ]).toString(); | |
| let url = `${_optionalChain([params, 'optionalAccess', _193 => _193.hubUrl]) || HUB_URL}/api/spaces?${search}`; | |
| while (url) { | |
| const res = await (_nullishCoalesce(_optionalChain([params, 'optionalAccess', _194 => _194.fetch]), () => ( fetch)))(url, { | |
| headers: { | |
| accept: "application/json", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 | |
| } | |
| }); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| const items = await res.json(); | |
| for (const item of items) { | |
| yield { | |
| ..._optionalChain([params, 'optionalAccess', _195 => _195.additionalFields]) && pick(item, params.additionalFields), | |
| id: item._id, | |
| name: item.id, | |
| sdk: item.sdk, | |
| likes: item.likes, | |
| private: item.private, | |
| updatedAt: new Date(item.lastModified) | |
| }; | |
| } | |
| const linkHeader = res.headers.get("Link"); | |
| url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; | |
| } | |
| } | |
| // src/lib/list-collections.ts | |
| async function* listCollections(params) { | |
| const accessToken = params && checkCredentials(params); | |
| const searchParams = new URLSearchParams(); | |
| let totalToFetch = _nullishCoalesce(_optionalChain([params, 'optionalAccess', _196 => _196.limit]), () => ( Infinity)); | |
| searchParams.append("limit", String(Math.min(totalToFetch, 100))); | |
| if (_optionalChain([params, 'optionalAccess', _197 => _197.sort])) { | |
| searchParams.append("sort", params.sort); | |
| } | |
| if (_optionalChain([params, 'optionalAccess', _198 => _198.search, 'optionalAccess', _199 => _199.owner])) { | |
| for (const owner of params.search.owner) { | |
| searchParams.append("owner", owner); | |
| } | |
| } | |
| if (_optionalChain([params, 'optionalAccess', _200 => _200.search, 'optionalAccess', _201 => _201.item])) { | |
| for (const item of params.search.item) { | |
| searchParams.append("item", item); | |
| } | |
| } | |
| if (_optionalChain([params, 'optionalAccess', _202 => _202.search, 'optionalAccess', _203 => _203.q])) { | |
| searchParams.append("q", params.search.q); | |
| } | |
| let url = `${_optionalChain([params, 'optionalAccess', _204 => _204.hubUrl]) || HUB_URL}/api/collections?${searchParams}`; | |
| while (url) { | |
| const res = await (_nullishCoalesce(_optionalChain([params, 'optionalAccess', _205 => _205.fetch]), () => ( fetch)))(url, { | |
| headers: { | |
| accept: "application/json", | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : void 0 | |
| } | |
| }); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| const collections = await res.json(); | |
| for (const collection of collections) { | |
| yield collection; | |
| totalToFetch--; | |
| if (totalToFetch <= 0) { | |
| return; | |
| } | |
| } | |
| const linkHeader = res.headers.get("Link"); | |
| url = linkHeader ? parseLinkHeader(linkHeader).next : void 0; | |
| } | |
| } | |
| // src/lib/model-info.ts | |
| async function modelInfo(params) { | |
| const accessToken = params && checkCredentials(params); | |
| const search = new URLSearchParams([ | |
| ...MODEL_EXPAND_KEYS.map((val) => ["expand", val]), | |
| ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _206 => _206.additionalFields, 'optionalAccess', _207 => _207.map, 'call', _208 => _208((val) => ["expand", val])]), () => ( [])) | |
| ]).toString(); | |
| const response = await (params.fetch || fetch)( | |
| `${_optionalChain([params, 'optionalAccess', _209 => _209.hubUrl]) || HUB_URL}/api/models/${params.name}/revision/${encodeURIComponent( | |
| _nullishCoalesce(params.revision, () => ( "HEAD")) | |
| )}?${search.toString()}`, | |
| { | |
| headers: { | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| const data = await response.json(); | |
| const normalizedData = { ...data }; | |
| if (_optionalChain([params, 'optionalAccess', _210 => _210.additionalFields, 'optionalAccess', _211 => _211.includes, 'call', _212 => _212("inferenceProviderMapping")]) && data.inferenceProviderMapping) { | |
| normalizedData.inferenceProviderMapping = normalizeInferenceProviderMapping(data.id, data.inferenceProviderMapping); | |
| } | |
| return { | |
| ..._optionalChain([params, 'optionalAccess', _213 => _213.additionalFields]) && pick(normalizedData, params.additionalFields), | |
| id: data._id, | |
| name: data.id, | |
| private: data.private, | |
| task: data.pipeline_tag, | |
| downloads: data.downloads, | |
| gated: data.gated, | |
| likes: data.likes, | |
| updatedAt: new Date(data.lastModified) | |
| }; | |
| } | |
| // src/lib/oauth-handle-redirect.ts | |
| async function oauthHandleRedirect(opts) { | |
| if (typeof window === "undefined" && !_optionalChain([opts, 'optionalAccess', _214 => _214.redirectedUrl])) { | |
| throw new Error("oauthHandleRedirect is only available in the browser, unless you provide redirectedUrl"); | |
| } | |
| if (typeof localStorage === "undefined" && (!_optionalChain([opts, 'optionalAccess', _215 => _215.nonce]) || !_optionalChain([opts, 'optionalAccess', _216 => _216.codeVerifier]))) { | |
| throw new Error( | |
| "oauthHandleRedirect requires localStorage to be available, unless you provide nonce and codeVerifier" | |
| ); | |
| } | |
| const redirectedUrl = _nullishCoalesce(_optionalChain([opts, 'optionalAccess', _217 => _217.redirectedUrl]), () => ( window.location.href)); | |
| const searchParams = (() => { | |
| try { | |
| return new URL(redirectedUrl).searchParams; | |
| } catch (err) { | |
| throw new Error("Failed to parse redirected URL: " + redirectedUrl); | |
| } | |
| })(); | |
| const [error, errorDescription] = [searchParams.get("error"), searchParams.get("error_description")]; | |
| if (error) { | |
| throw new Error(`${error}: ${errorDescription}`); | |
| } | |
| const code = searchParams.get("code"); | |
| const nonce = _nullishCoalesce(_optionalChain([opts, 'optionalAccess', _218 => _218.nonce]), () => ( localStorage.getItem("huggingface.co:oauth:nonce"))); | |
| if (!code) { | |
| throw new Error("Missing oauth code from query parameters in redirected URL: " + redirectedUrl); | |
| } | |
| if (!nonce) { | |
| throw new Error("Missing oauth nonce from localStorage"); | |
| } | |
| const codeVerifier = _nullishCoalesce(_optionalChain([opts, 'optionalAccess', _219 => _219.codeVerifier]), () => ( localStorage.getItem("huggingface.co:oauth:code_verifier"))); | |
| if (!codeVerifier) { | |
| throw new Error("Missing oauth code_verifier from localStorage"); | |
| } | |
| const state = searchParams.get("state"); | |
| if (!state) { | |
| throw new Error("Missing oauth state from query parameters in redirected URL"); | |
| } | |
| let parsedState; | |
| try { | |
| parsedState = JSON.parse(state); | |
| } catch (e6) { | |
| throw new Error("Invalid oauth state in redirected URL, unable to parse JSON: " + state); | |
| } | |
| if (parsedState.nonce !== nonce) { | |
| throw new Error("Invalid oauth state in redirected URL"); | |
| } | |
| const hubUrl = _optionalChain([opts, 'optionalAccess', _220 => _220.hubUrl]) || HUB_URL; | |
| const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; | |
| const openidConfigRes = await fetch(openidConfigUrl, { | |
| headers: { | |
| Accept: "application/json" | |
| } | |
| }); | |
| if (!openidConfigRes.ok) { | |
| throw await createApiError(openidConfigRes); | |
| } | |
| const openidConfig = await openidConfigRes.json(); | |
| const tokenRes = await fetch(openidConfig.token_endpoint, { | |
| method: "POST", | |
| headers: { | |
| "Content-Type": "application/x-www-form-urlencoded" | |
| }, | |
| body: new URLSearchParams({ | |
| grant_type: "authorization_code", | |
| code, | |
| redirect_uri: parsedState.redirectUri, | |
| code_verifier: codeVerifier | |
| }).toString() | |
| }); | |
| if (!_optionalChain([opts, 'optionalAccess', _221 => _221.codeVerifier])) { | |
| localStorage.removeItem("huggingface.co:oauth:code_verifier"); | |
| } | |
| if (!_optionalChain([opts, 'optionalAccess', _222 => _222.nonce])) { | |
| localStorage.removeItem("huggingface.co:oauth:nonce"); | |
| } | |
| if (!tokenRes.ok) { | |
| throw await createApiError(tokenRes); | |
| } | |
| const token = await tokenRes.json(); | |
| const accessTokenExpiresAt = new Date(Date.now() + token.expires_in * 1e3); | |
| const userInfoRes = await fetch(openidConfig.userinfo_endpoint, { | |
| headers: { | |
| Authorization: `Bearer ${token.access_token}` | |
| } | |
| }); | |
| if (!userInfoRes.ok) { | |
| throw await createApiError(userInfoRes); | |
| } | |
| const userInfo = await userInfoRes.json(); | |
| return { | |
| accessToken: token.access_token, | |
| accessTokenExpiresAt, | |
| userInfo, | |
| state: parsedState.state, | |
| scope: token.scope | |
| }; | |
| } | |
| async function oauthHandleRedirectIfPresent(opts) { | |
| if (typeof window === "undefined" && !_optionalChain([opts, 'optionalAccess', _223 => _223.redirectedUrl])) { | |
| throw new Error("oauthHandleRedirect is only available in the browser, unless you provide redirectedUrl"); | |
| } | |
| if (typeof localStorage === "undefined" && (!_optionalChain([opts, 'optionalAccess', _224 => _224.nonce]) || !_optionalChain([opts, 'optionalAccess', _225 => _225.codeVerifier]))) { | |
| throw new Error( | |
| "oauthHandleRedirect requires localStorage to be available, unless you provide nonce and codeVerifier" | |
| ); | |
| } | |
| const searchParams = new URLSearchParams(_nullishCoalesce(_optionalChain([opts, 'optionalAccess', _226 => _226.redirectedUrl]), () => ( window.location.search))); | |
| if (searchParams.has("error")) { | |
| return oauthHandleRedirect(opts); | |
| } | |
| if (searchParams.has("code")) { | |
| if (!localStorage.getItem("huggingface.co:oauth:nonce")) { | |
| console.warn( | |
| "Missing oauth nonce from localStorage. This can happen when the user refreshes the page after logging in, without changing the URL." | |
| ); | |
| return false; | |
| } | |
| return oauthHandleRedirect(opts); | |
| } | |
| return false; | |
| } | |
| // src/lib/oauth-login-url.ts | |
| async function oauthLoginUrl(opts) { | |
| if (typeof window === "undefined" && (!_optionalChain([opts, 'optionalAccess', _227 => _227.redirectUrl]) || !_optionalChain([opts, 'optionalAccess', _228 => _228.clientId]))) { | |
| throw new Error("oauthLogin is only available in the browser, unless you provide clientId and redirectUrl"); | |
| } | |
| if (typeof localStorage === "undefined" && !_optionalChain([opts, 'optionalAccess', _229 => _229.localStorage])) { | |
| throw new Error( | |
| "oauthLogin requires localStorage to be available in the context, unless you provide a localStorage empty object as argument" | |
| ); | |
| } | |
| const hubUrl = _optionalChain([opts, 'optionalAccess', _230 => _230.hubUrl]) || HUB_URL; | |
| const openidConfigUrl = `${new URL(hubUrl).origin}/.well-known/openid-configuration`; | |
| const openidConfigRes = await fetch(openidConfigUrl, { | |
| headers: { | |
| Accept: "application/json" | |
| } | |
| }); | |
| if (!openidConfigRes.ok) { | |
| throw await createApiError(openidConfigRes); | |
| } | |
| const opendidConfig = await openidConfigRes.json(); | |
| const newNonce = globalThis.crypto.randomUUID(); | |
| const newCodeVerifier = globalThis.crypto.randomUUID() + globalThis.crypto.randomUUID(); | |
| if (_optionalChain([opts, 'optionalAccess', _231 => _231.localStorage])) { | |
| if (opts.localStorage.codeVerifier !== void 0 && opts.localStorage.codeVerifier !== null) { | |
| throw new Error( | |
| "localStorage.codeVerifier must be initially set to null or undefined, and will be filled by oauthLoginUrl" | |
| ); | |
| } | |
| if (opts.localStorage.nonce !== void 0 && opts.localStorage.nonce !== null) { | |
| throw new Error( | |
| "localStorage.nonce must be initially set to null or undefined, and will be filled by oauthLoginUrl" | |
| ); | |
| } | |
| opts.localStorage.codeVerifier = newCodeVerifier; | |
| opts.localStorage.nonce = newNonce; | |
| } else { | |
| localStorage.setItem("huggingface.co:oauth:nonce", newNonce); | |
| localStorage.setItem("huggingface.co:oauth:code_verifier", newCodeVerifier); | |
| } | |
| const redirectUri = _optionalChain([opts, 'optionalAccess', _232 => _232.redirectUrl]) || (typeof window !== "undefined" ? window.location.href : void 0); | |
| if (!redirectUri) { | |
| throw new Error("Missing redirectUrl"); | |
| } | |
| const state = JSON.stringify({ | |
| nonce: newNonce, | |
| redirectUri, | |
| state: _optionalChain([opts, 'optionalAccess', _233 => _233.state]) | |
| }); | |
| const variables = ( | |
| // @ts-expect-error window.huggingface is defined inside static Spaces. | |
| typeof window !== "undefined" ? _nullishCoalesce(_optionalChain([window, 'access', _234 => _234.huggingface, 'optionalAccess', _235 => _235.variables]), () => ( null)) : null | |
| ); | |
| const clientId = _optionalChain([opts, 'optionalAccess', _236 => _236.clientId]) || _optionalChain([variables, 'optionalAccess', _237 => _237.OAUTH_CLIENT_ID]); | |
| if (!clientId) { | |
| if (variables) { | |
| throw new Error("Missing clientId, please add hf_oauth: true to the README.md's metadata in your static Space"); | |
| } | |
| throw new Error("Missing clientId"); | |
| } | |
| const challenge = base64FromBytes( | |
| new Uint8Array(await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(newCodeVerifier))) | |
| ).replace(/[+]/g, "-").replace(/[/]/g, "_").replace(/=/g, ""); | |
| return `${opendidConfig.authorization_endpoint}?${new URLSearchParams({ | |
| client_id: clientId, | |
| scope: _optionalChain([opts, 'optionalAccess', _238 => _238.scopes]) || _optionalChain([variables, 'optionalAccess', _239 => _239.OAUTH_SCOPES]) || "openid profile", | |
| response_type: "code", | |
| redirect_uri: redirectUri, | |
| state, | |
| code_challenge: challenge, | |
| code_challenge_method: "S256" | |
| }).toString()}`; | |
| } | |
| // src/utils/typedInclude.ts | |
| function typedInclude(arr, v) { | |
| return arr.includes(v); | |
| } | |
| // src/utils/omit.ts | |
| function omit(o, props) { | |
| const propsArr = Array.isArray(props) ? props : [props]; | |
| const letsKeep = Object.keys(o).filter((prop) => !typedInclude(propsArr, prop)); | |
| return pick(o, letsKeep); | |
| } | |
| // src/utils/typedEntries.ts | |
| function typedEntries(obj) { | |
| return Object.entries(obj); | |
| } | |
| // src/lib/parse-safetensors-metadata.ts | |
| var SAFETENSORS_FILE = "model.safetensors"; | |
| var SAFETENSORS_INDEX_FILE = "model.safetensors.index.json"; | |
| var RE_SAFETENSORS_FILE = /\.safetensors$/; | |
| var RE_SAFETENSORS_INDEX_FILE = /\.safetensors\.index\.json$/; | |
| var RE_SAFETENSORS_SHARD_FILE = /^(?<prefix>(?<basePrefix>.*?)[_-])(?<shard>\d{5,6})-of-(?<total>\d{5,6})\.safetensors$/; | |
| function parseSafetensorsShardFilename(filename) { | |
| const match = RE_SAFETENSORS_SHARD_FILE.exec(filename); | |
| if (match && match.groups) { | |
| return { | |
| prefix: match.groups["prefix"], | |
| basePrefix: match.groups["basePrefix"], | |
| shard: match.groups["shard"], | |
| total: match.groups["total"] | |
| }; | |
| } | |
| return null; | |
| } | |
| var PARALLEL_DOWNLOADS = 20; | |
| var MAX_HEADER_LENGTH = 25e6; | |
| var GPTQ_QWEIGHT_SUFFIX = "qweight"; | |
| var SafetensorParseError = class extends Error { | |
| }; | |
| async function fetchModelConfig(params) { | |
| try { | |
| const configBlob = await downloadFile({ | |
| ...params, | |
| path: "config.json" | |
| }); | |
| if (!configBlob) { | |
| return null; | |
| } | |
| const config = JSON.parse(await configBlob.text()); | |
| return config; | |
| } catch (error) { | |
| return null; | |
| } | |
| } | |
| async function parseSingleFile(path, params) { | |
| const blob = await downloadFile({ ...params, path }); | |
| if (!blob) { | |
| throw new SafetensorParseError(`Failed to parse file ${path}: failed to fetch safetensors header length.`); | |
| } | |
| const bufLengthOfHeaderLE = await blob.slice(0, 8).arrayBuffer(); | |
| const lengthOfHeader = new DataView(bufLengthOfHeaderLE).getBigUint64(0, true); | |
| if (lengthOfHeader <= 0) { | |
| throw new SafetensorParseError(`Failed to parse file ${path}: safetensors header is malformed.`); | |
| } | |
| if (lengthOfHeader > MAX_HEADER_LENGTH) { | |
| throw new SafetensorParseError( | |
| `Failed to parse file ${path}: safetensor header is too big. Maximum supported size is ${MAX_HEADER_LENGTH} bytes.` | |
| ); | |
| } | |
| try { | |
| const header = JSON.parse(await blob.slice(8, 8 + Number(lengthOfHeader)).text()); | |
| return header; | |
| } catch (err) { | |
| throw new SafetensorParseError(`Failed to parse file ${path}: safetensors header is not valid JSON.`); | |
| } | |
| } | |
| async function parseShardedIndex(path, params) { | |
| const indexBlob = await downloadFile({ | |
| ...params, | |
| path | |
| }); | |
| if (!indexBlob) { | |
| throw new SafetensorParseError(`Failed to parse file ${path}: failed to fetch safetensors index.`); | |
| } | |
| try { | |
| const index = JSON.parse(await indexBlob.slice(0, MAX_HEADER_LENGTH).text()); | |
| return index; | |
| } catch (error) { | |
| throw new SafetensorParseError(`Failed to parse file ${path}: not a valid JSON.`); | |
| } | |
| } | |
| async function fetchAllHeaders(path, index, params) { | |
| const pathPrefix = path.slice(0, path.lastIndexOf("/") + 1); | |
| const filenames = [...new Set(Object.values(index.weight_map))]; | |
| const shardedMap = Object.fromEntries( | |
| await promisesQueue( | |
| filenames.map( | |
| (filename) => async () => [filename, await parseSingleFile(pathPrefix + filename, params)] | |
| ), | |
| PARALLEL_DOWNLOADS | |
| ) | |
| ); | |
| return shardedMap; | |
| } | |
| async function parseSafetensorsMetadata(params) { | |
| const repoId = toRepoId(params.repo); | |
| if (repoId.type !== "model") { | |
| throw new TypeError("Only model repos should contain safetensors files."); | |
| } | |
| const modelConfig = params.computeParametersCount ? await fetchModelConfig(params) : null; | |
| const quantConfig = _optionalChain([modelConfig, 'optionalAccess', _240 => _240.quantization_config]); | |
| if (params.path && RE_SAFETENSORS_FILE.test(params.path) || await fileExists({ ...params, path: SAFETENSORS_FILE })) { | |
| const header = await parseSingleFile(_nullishCoalesce(params.path, () => ( SAFETENSORS_FILE)), params); | |
| return { | |
| sharded: false, | |
| header, | |
| ...params.computeParametersCount ? { | |
| parameterCount: computeNumOfParamsByDtypeSingleFile(header, quantConfig), | |
| parameterTotal: ( | |
| /// shortcut: get param count directly from metadata | |
| header.__metadata__.total_parameters ? typeof header.__metadata__.total_parameters === "number" ? header.__metadata__.total_parameters : typeof header.__metadata__.total_parameters === "string" ? parseInt(header.__metadata__.total_parameters) : void 0 : void 0 | |
| ) | |
| } : void 0 | |
| }; | |
| } else if (params.path && RE_SAFETENSORS_INDEX_FILE.test(params.path) || await fileExists({ ...params, path: SAFETENSORS_INDEX_FILE })) { | |
| const path = _nullishCoalesce(params.path, () => ( SAFETENSORS_INDEX_FILE)); | |
| const index = await parseShardedIndex(path, params); | |
| const shardedMap = await fetchAllHeaders(path, index, params); | |
| return { | |
| sharded: true, | |
| index, | |
| headers: shardedMap, | |
| ...params.computeParametersCount ? { | |
| parameterCount: computeNumOfParamsByDtypeSharded(shardedMap, quantConfig), | |
| parameterTotal: ( | |
| /// shortcut: get param count directly from metadata | |
| _optionalChain([index, 'access', _241 => _241.metadata, 'optionalAccess', _242 => _242.total_parameters]) ? typeof index.metadata.total_parameters === "number" ? index.metadata.total_parameters : typeof index.metadata.total_parameters === "string" ? parseInt(index.metadata.total_parameters) : void 0 : void 0 | |
| ) | |
| } : void 0 | |
| }; | |
| } else { | |
| throw new Error("model id does not seem to contain safetensors weights"); | |
| } | |
| } | |
| function isQuantizedTensor(tensorName, quantConfig) { | |
| if (!quantConfig) { | |
| return false; | |
| } | |
| if (!quantConfig.modules_to_not_convert || quantConfig.modules_to_not_convert.length === 0) { | |
| return true; | |
| } | |
| for (const pattern of quantConfig.modules_to_not_convert) { | |
| const regexPattern = pattern.replace(/\*/g, ".*"); | |
| const regex = new RegExp(regexPattern); | |
| if (regex.test(tensorName)) { | |
| return false; | |
| } | |
| } | |
| return true; | |
| } | |
| function getQuantizationMultiplier(tensorName, dtype, quantConfig) { | |
| if (!quantConfig || !isQuantizedTensor(tensorName, quantConfig)) { | |
| return 1; | |
| } | |
| const quantMethod = _optionalChain([quantConfig, 'access', _243 => _243.quant_method, 'optionalAccess', _244 => _244.toLowerCase, 'call', _245 => _245()]); | |
| switch (quantMethod) { | |
| case "mxfp4": | |
| if (dtype === "U8" && tensorName.includes("_blocks")) { | |
| return 2; | |
| } | |
| return 1; | |
| case "gptq": | |
| case "awq": | |
| if (getTensorSuffix(tensorName) === GPTQ_QWEIGHT_SUFFIX) { | |
| const bits = quantConfig.bits && quantConfig.bits > 0 ? quantConfig.bits : 4; | |
| return Math.max(1, Math.floor(32 / bits)); | |
| } | |
| if (quantConfig.bits === 4 && dtype === "U8") { | |
| return 2; | |
| } | |
| if (quantConfig.bits === 2 && dtype === "U8") { | |
| return 4; | |
| } | |
| return 1; | |
| case "bitsandbytes": | |
| if (quantConfig.load_in_4bit && dtype === "U8") { | |
| return 2; | |
| } | |
| if (quantConfig.load_in_8bit) { | |
| return 1; | |
| } | |
| return 1; | |
| default: | |
| if (quantConfig.load_in_4bit && dtype === "U8") { | |
| return 2; | |
| } | |
| if (quantConfig.bits === 4 && dtype === "U8") { | |
| return 2; | |
| } | |
| return 1; | |
| } | |
| } | |
| function computeNumOfParamsByDtypeSingleFile(header, quantConfig) { | |
| const counter = {}; | |
| const tensors = omit(header, "__metadata__"); | |
| for (const [tensorName, v] of typedEntries(tensors)) { | |
| if (shouldSkipTensor(tensorName, quantConfig)) { | |
| continue; | |
| } | |
| if (v.shape.length === 0) { | |
| continue; | |
| } | |
| const elements = v.shape.reduce((a, b) => a * b); | |
| const multiplier = quantConfig ? getQuantizationMultiplier(tensorName, v.dtype, quantConfig) : 1; | |
| if (multiplier === 0) { | |
| continue; | |
| } | |
| counter[v.dtype] = (_nullishCoalesce(counter[v.dtype], () => ( 0))) + elements * multiplier; | |
| } | |
| return counter; | |
| } | |
| function computeNumOfParamsByDtypeSharded(shardedMap, quantConfig) { | |
| const counter = {}; | |
| for (const header of Object.values(shardedMap)) { | |
| for (const [k, v] of typedEntries(computeNumOfParamsByDtypeSingleFile(header, quantConfig))) { | |
| counter[k] = (_nullishCoalesce(counter[k], () => ( 0))) + (_nullishCoalesce(v, () => ( 0))); | |
| } | |
| } | |
| return counter; | |
| } | |
| function getTensorSuffix(tensorName) { | |
| const lastDotIndex = tensorName.lastIndexOf("."); | |
| return lastDotIndex === -1 ? tensorName : tensorName.slice(lastDotIndex + 1); | |
| } | |
| function shouldSkipTensor(tensorName, quantConfig) { | |
| const GPTQ_AWQ_AUXILIARY_SUFFIXES = ["qzeros", "g_idx", "scales"]; | |
| if (!quantConfig) { | |
| return false; | |
| } | |
| const quantMethod = _optionalChain([quantConfig, 'access', _246 => _246.quant_method, 'optionalAccess', _247 => _247.toLowerCase, 'call', _248 => _248()]); | |
| if (!quantMethod || quantMethod !== "gptq" && quantMethod !== "awq") { | |
| return false; | |
| } | |
| if (!isQuantizedTensor(tensorName, quantConfig)) { | |
| return false; | |
| } | |
| const suffix = getTensorSuffix(tensorName); | |
| if (suffix === GPTQ_QWEIGHT_SUFFIX) { | |
| return false; | |
| } | |
| return GPTQ_AWQ_AUXILIARY_SUFFIXES.includes(suffix); | |
| } | |
| // src/lib/paths-info.ts | |
| async function pathsInfo(params) { | |
| const accessToken = checkCredentials(params); | |
| const repoId = toRepoId(params.repo); | |
| const hubUrl = _nullishCoalesce(params.hubUrl, () => ( HUB_URL)); | |
| const url = `${hubUrl}/api/${repoId.type}s/${repoId.name}/paths-info/${encodeURIComponent( | |
| _nullishCoalesce(params.revision, () => ( "main")) | |
| )}`; | |
| const resp = await (_nullishCoalesce(params.fetch, () => ( fetch)))(url, { | |
| method: "POST", | |
| headers: { | |
| ...accessToken && { | |
| Authorization: `Bearer ${accessToken}` | |
| }, | |
| Accept: "application/json", | |
| "Content-Type": "application/json" | |
| }, | |
| body: JSON.stringify({ | |
| paths: params.paths, | |
| expand: params.expand | |
| }) | |
| }); | |
| if (!resp.ok) { | |
| throw await createApiError(resp); | |
| } | |
| const json = await resp.json(); | |
| if (!Array.isArray(json)) | |
| throw new Error("malformed response: expected array"); | |
| return json.map((item) => ({ | |
| path: item.path, | |
| lfs: item.lfs, | |
| type: item.type, | |
| oid: item.oid, | |
| size: item.size, | |
| // expand fields | |
| securityFileStatus: item.securityFileStatus, | |
| lastCommit: item.lastCommit ? { | |
| date: new Date(item.lastCommit.date), | |
| title: item.lastCommit.title, | |
| id: item.lastCommit.id | |
| } : void 0 | |
| })); | |
| } | |
| // src/lib/repo-exists.ts | |
| async function repoExists(params) { | |
| const repoId = toRepoId(params.repo); | |
| const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))( | |
| `${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/${repoId.type}s/${repoId.name}?expand[]=likes`, | |
| { | |
| method: "GET", | |
| headers: { | |
| ...params.accessToken && { | |
| Authorization: `Bearer ${params.accessToken}` | |
| } | |
| } | |
| } | |
| ); | |
| if (res.status === 404 || res.status === 401) { | |
| return false; | |
| } | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| return true; | |
| } | |
| // src/lib/space-info.ts | |
| async function spaceInfo(params) { | |
| const accessToken = params && checkCredentials(params); | |
| const search = new URLSearchParams([ | |
| ...SPACE_EXPAND_KEYS.map((val) => ["expand", val]), | |
| ..._nullishCoalesce(_optionalChain([params, 'optionalAccess', _249 => _249.additionalFields, 'optionalAccess', _250 => _250.map, 'call', _251 => _251((val) => ["expand", val])]), () => ( [])) | |
| ]).toString(); | |
| const response = await (params.fetch || fetch)( | |
| `${_optionalChain([params, 'optionalAccess', _252 => _252.hubUrl]) || HUB_URL}/api/spaces/${params.name}/revision/${encodeURIComponent( | |
| _nullishCoalesce(params.revision, () => ( "HEAD")) | |
| )}?${search.toString()}`, | |
| { | |
| headers: { | |
| ...accessToken ? { Authorization: `Bearer ${accessToken}` } : {} | |
| } | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw await createApiError(response); | |
| } | |
| const data = await response.json(); | |
| return { | |
| ..._optionalChain([params, 'optionalAccess', _253 => _253.additionalFields]) && pick(data, params.additionalFields), | |
| id: data._id, | |
| name: data.id, | |
| sdk: data.sdk, | |
| likes: data.likes, | |
| private: data.private, | |
| updatedAt: new Date(data.lastModified) | |
| }; | |
| } | |
| // src/lib/upload-file.ts | |
| function uploadFile(params) { | |
| const path = params.file instanceof URL ? _nullishCoalesce(params.file.pathname.split("/").at(-1), () => ( "file")) : "path" in params.file ? params.file.path : params.file.name; | |
| return commit({ | |
| ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, | |
| repo: params.repo, | |
| operations: [ | |
| { | |
| operation: "addOrUpdate", | |
| path, | |
| content: "content" in params.file ? params.file.content : params.file | |
| } | |
| ], | |
| title: _nullishCoalesce(params.commitTitle, () => ( `Add ${path}`)), | |
| description: params.commitDescription, | |
| hubUrl: params.hubUrl, | |
| branch: params.branch, | |
| isPullRequest: params.isPullRequest, | |
| parentCommit: params.parentCommit, | |
| fetch: params.fetch, | |
| useWebWorkers: params.useWebWorkers, | |
| abortSignal: params.abortSignal, | |
| useXet: params.useXet | |
| }); | |
| } | |
| // src/lib/upload-files.ts | |
| function uploadFiles(params) { | |
| return commit({ | |
| ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, | |
| repo: params.repo, | |
| operations: params.files.map((file) => ({ | |
| operation: "addOrUpdate", | |
| path: file instanceof URL ? _nullishCoalesce(file.pathname.split("/").at(-1), () => ( "file")) : "path" in file ? file.path : file.name, | |
| content: "content" in file ? file.content : file | |
| })), | |
| title: _nullishCoalesce(params.commitTitle, () => ( `Add ${params.files.length} files`)), | |
| description: params.commitDescription, | |
| hubUrl: params.hubUrl, | |
| branch: params.branch, | |
| isPullRequest: params.isPullRequest, | |
| parentCommit: params.parentCommit, | |
| fetch: params.fetch, | |
| useWebWorkers: params.useWebWorkers, | |
| abortSignal: params.abortSignal, | |
| useXet: params.useXet | |
| }); | |
| } | |
| // src/lib/upload-files-with-progress.ts | |
| var multipartUploadTracking = /* @__PURE__ */ new WeakMap(); | |
| async function* uploadFilesWithProgress(params) { | |
| return yield* commitIter({ | |
| ...params.accessToken ? { accessToken: params.accessToken } : { credentials: params.credentials }, | |
| repo: params.repo, | |
| operations: params.files.map((file) => ({ | |
| operation: "addOrUpdate", | |
| path: file instanceof URL ? _nullishCoalesce(file.pathname.split("/").at(-1), () => ( "file")) : "path" in file ? file.path : file.name, | |
| content: "content" in file ? file.content : file | |
| })), | |
| title: _nullishCoalesce(params.commitTitle, () => ( `Add ${params.files.length} files`)), | |
| description: params.commitDescription, | |
| hubUrl: params.hubUrl, | |
| branch: params.branch, | |
| isPullRequest: params.isPullRequest, | |
| parentCommit: params.parentCommit, | |
| useWebWorkers: params.useWebWorkers, | |
| abortSignal: params.abortSignal, | |
| useXet: params.useXet, | |
| fetch: async (input, init) => { | |
| if (!init) { | |
| return fetch(input); | |
| } | |
| if (!typedInclude(["PUT", "POST"], init.method) || !("progressHint" in init) || !init.progressHint || typeof XMLHttpRequest === "undefined" || typeof input !== "string" || !(init.body instanceof ArrayBuffer) && !(init.body instanceof Blob) && !(init.body instanceof File) && typeof init.body !== "string") { | |
| return fetch(input, init); | |
| } | |
| const progressHint = init.progressHint; | |
| const progressCallback = progressHint.progressCallback; | |
| const xhr = new XMLHttpRequest(); | |
| xhr.upload.addEventListener("progress", (event) => { | |
| if (event.lengthComputable) { | |
| if (progressHint.part !== void 0) { | |
| let tracking = multipartUploadTracking.get(progressCallback); | |
| if (!tracking) { | |
| tracking = { numParts: progressHint.numParts, partsProgress: {} }; | |
| multipartUploadTracking.set(progressCallback, tracking); | |
| } | |
| tracking.partsProgress[progressHint.part] = event.loaded / event.total; | |
| let totalProgress = 0; | |
| for (const partProgress of Object.values(tracking.partsProgress)) { | |
| totalProgress += partProgress; | |
| } | |
| if (totalProgress === tracking.numParts) { | |
| progressCallback(0.9999999999); | |
| } else { | |
| progressCallback(totalProgress / tracking.numParts); | |
| } | |
| } else { | |
| if (event.loaded === event.total) { | |
| progressCallback(0.9999999999); | |
| } else { | |
| progressCallback(event.loaded / event.total); | |
| } | |
| } | |
| } | |
| }); | |
| xhr.open(init.method, input, true); | |
| if (init.headers) { | |
| const headers = new Headers(init.headers); | |
| headers.forEach((value, key) => { | |
| xhr.setRequestHeader(key, value); | |
| }); | |
| } | |
| _optionalChain([init, 'access', _254 => _254.signal, 'optionalAccess', _255 => _255.throwIfAborted, 'call', _256 => _256()]); | |
| xhr.send(init.body); | |
| return new Promise((resolve2, reject) => { | |
| xhr.addEventListener("load", () => { | |
| resolve2( | |
| new Response(xhr.responseText, { | |
| status: xhr.status, | |
| statusText: xhr.statusText, | |
| headers: Object.fromEntries( | |
| xhr.getAllResponseHeaders().trim().split("\n").map((header) => [ | |
| header.slice(0, header.indexOf(":")), | |
| header.slice(header.indexOf(":") + 1).trim() | |
| ]) | |
| ) | |
| }) | |
| ); | |
| }); | |
| xhr.addEventListener("error", () => { | |
| reject(new Error(xhr.statusText)); | |
| }); | |
| if (init.signal) { | |
| init.signal.addEventListener("abort", () => { | |
| xhr.abort(); | |
| try { | |
| _optionalChain([init, 'access', _257 => _257.signal, 'optionalAccess', _258 => _258.throwIfAborted, 'call', _259 => _259()]); | |
| } catch (err) { | |
| reject(err); | |
| } | |
| }); | |
| } | |
| }); | |
| } | |
| }); | |
| } | |
| // src/lib/who-am-i.ts | |
| async function whoAmI(params) { | |
| const accessToken = checkCredentials(params); | |
| const res = await (_nullishCoalesce(params.fetch, () => ( fetch)))(`${_nullishCoalesce(params.hubUrl, () => ( HUB_URL))}/api/whoami-v2`, { | |
| headers: { | |
| Authorization: `Bearer ${accessToken}` | |
| } | |
| }); | |
| if (!res.ok) { | |
| throw await createApiError(res); | |
| } | |
| const response = await res.json(); | |
| if (typeof _optionalChain([response, 'access', _260 => _260.auth, 'access', _261 => _261.accessToken, 'optionalAccess', _262 => _262.createdAt]) === "string") { | |
| response.auth.accessToken.createdAt = new Date(response.auth.accessToken.createdAt); | |
| } | |
| return response; | |
| } | |
| exports.DATASET_EXPANDABLE_KEYS = DATASET_EXPANDABLE_KEYS; exports.DATASET_EXPAND_KEYS = DATASET_EXPAND_KEYS; exports.HUB_URL = HUB_URL; exports.HubApiError = HubApiError; exports.InvalidApiResponseFormatError = InvalidApiResponseFormatError; exports.MODEL_EXPANDABLE_KEYS = MODEL_EXPANDABLE_KEYS; exports.MODEL_EXPAND_KEYS = MODEL_EXPAND_KEYS; exports.RE_SAFETENSORS_FILE = RE_SAFETENSORS_FILE; exports.RE_SAFETENSORS_INDEX_FILE = RE_SAFETENSORS_INDEX_FILE; exports.RE_SAFETENSORS_SHARD_FILE = RE_SAFETENSORS_SHARD_FILE; exports.SAFETENSORS_FILE = SAFETENSORS_FILE; exports.SAFETENSORS_INDEX_FILE = SAFETENSORS_INDEX_FILE; exports.SPACE_EXPANDABLE_KEYS = SPACE_EXPANDABLE_KEYS; exports.SPACE_EXPAND_KEYS = SPACE_EXPAND_KEYS; exports.__internal_XetBlob = XetBlob; exports.__internal_sha256 = sha256; exports.cancelJob = cancelJob; exports.checkRepoAccess = checkRepoAccess; exports.commit = commit; exports.commitIter = commitIter; exports.commitIterBucket = commitIterBucket; exports.countCommits = countCommits; exports.createBranch = createBranch; exports.createCollection = createCollection; exports.createRepo = createRepo; exports.createScheduledJob = createScheduledJob; exports.datasetInfo = datasetInfo; exports.deleteBranch = deleteBranch; exports.deleteCollection = deleteCollection; exports.deleteFile = deleteFile; exports.deleteFiles = deleteFiles; exports.deleteRepo = deleteRepo; exports.deleteScheduledJob = deleteScheduledJob; exports.downloadFile = downloadFile; exports.duplicateJob = duplicateJob; exports.fileDownloadInfo = fileDownloadInfo; exports.fileExists = fileExists; exports.getJob = getJob; exports.getScheduledJob = getScheduledJob; exports.listCollections = listCollections; exports.listCommits = listCommits; exports.listDatasets = listDatasets; exports.listFiles = listFiles; exports.listJobHardware = listJobHardware; exports.listJobs = listJobs; exports.listModels = listModels; exports.listScheduledJobs = listScheduledJobs; exports.listSpaces = listSpaces; exports.modelInfo = modelInfo; exports.oauthHandleRedirect = oauthHandleRedirect; exports.oauthHandleRedirectIfPresent = oauthHandleRedirectIfPresent; exports.oauthLoginUrl = oauthLoginUrl; exports.parseSafetensorsMetadata = parseSafetensorsMetadata; exports.parseSafetensorsShardFilename = parseSafetensorsShardFilename; exports.pathsInfo = pathsInfo; exports.repoExists = repoExists; exports.resumeScheduledJob = resumeScheduledJob; exports.runJob = runJob; exports.runScheduledJob = runScheduledJob; exports.spaceInfo = spaceInfo; exports.streamJobEvents = streamJobEvents; exports.streamJobLogs = streamJobLogs; exports.streamJobMetrics = streamJobMetrics; exports.suspendScheduledJob = suspendScheduledJob; exports.uploadFile = uploadFile; exports.uploadFiles = uploadFiles; exports.uploadFilesWithProgress = uploadFilesWithProgress; exports.whoAmI = whoAmI; | |