Spaces:
Running
Running
| // Incremental SHA-256 (FIPS 180-4). | |
| // | |
| // crypto.subtle.digest() is one-shot: it wants the whole input as a single | |
| // ArrayBuffer. A quantised GGUF is hundreds of MB (and can be several GB), so | |
| // hashing it that way would mean holding the entire model in JS memory twice. | |
| // This implementation takes the bytes 4 MiB at a time as they stream out of | |
| // IndexedDB (or off the user's disk) and keeps a 64 byte block of state. | |
| // | |
| // Verified against node's crypto.createHash("sha256") in test/sha256-test.mjs, | |
| // including every padding boundary and non-block-aligned streaming. | |
| const K = new Uint32Array([ | |
| 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, | |
| 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, | |
| 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, | |
| 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, | |
| 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, | |
| 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, | |
| 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, | |
| 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, | |
| ]); | |
| export class Sha256 { | |
| constructor() { | |
| this.h = new Uint32Array([ | |
| 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, | |
| 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, | |
| ]); | |
| this.w = new Uint32Array(64); | |
| this.block = new Uint8Array(64); | |
| this.blockLen = 0; | |
| this.bytes = 0; | |
| this.finished = false; | |
| } | |
| update(data) { | |
| if (this.finished) throw new Error("Sha256: update() after digest()"); | |
| const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); | |
| const len = bytes.length; | |
| this.bytes += len; | |
| let off = 0; | |
| // top up a partially filled block first | |
| if (this.blockLen > 0) { | |
| const need = 64 - this.blockLen; | |
| if (len < need) { | |
| this.block.set(bytes, this.blockLen); | |
| this.blockLen += len; | |
| return this; | |
| } | |
| this.block.set(bytes.subarray(0, need), this.blockLen); | |
| this._compress(this.block, 0); | |
| this.blockLen = 0; | |
| off = need; | |
| } | |
| while (off + 64 <= len) { | |
| this._compress(bytes, off); | |
| off += 64; | |
| } | |
| if (off < len) { | |
| this.block.set(bytes.subarray(off), 0); | |
| this.blockLen = len - off; | |
| } | |
| return this; | |
| } | |
| _compress(buf, off) { | |
| const w = this.w; | |
| const h = this.h; | |
| for (let i = 0; i < 16; i++) { | |
| const j = off + i * 4; | |
| w[i] = (buf[j] << 24) | (buf[j + 1] << 16) | (buf[j + 2] << 8) | buf[j + 3]; | |
| } | |
| for (let i = 16; i < 64; i++) { | |
| const a = w[i - 15]; | |
| const b = w[i - 2]; | |
| const s0 = ((a >>> 7) | (a << 25)) ^ ((a >>> 18) | (a << 14)) ^ (a >>> 3); | |
| const s1 = ((b >>> 17) | (b << 15)) ^ ((b >>> 19) | (b << 13)) ^ (b >>> 10); | |
| w[i] = (w[i - 16] + s0 + w[i - 7] + s1) | 0; | |
| } | |
| let a = h[0], b = h[1], c = h[2], d = h[3]; | |
| let e = h[4], f = h[5], g = h[6], k = h[7]; | |
| for (let i = 0; i < 64; i++) { | |
| const S1 = ((e >>> 6) | (e << 26)) ^ ((e >>> 11) | (e << 21)) ^ ((e >>> 25) | (e << 7)); | |
| const ch = (e & f) ^ (~e & g); | |
| const t1 = (k + S1 + ch + K[i] + w[i]) | 0; | |
| const S0 = ((a >>> 2) | (a << 30)) ^ ((a >>> 13) | (a << 19)) ^ ((a >>> 22) | (a << 10)); | |
| const maj = (a & b) ^ (a & c) ^ (b & c); | |
| const t2 = (S0 + maj) | 0; | |
| k = g; g = f; f = e; | |
| e = (d + t1) | 0; | |
| d = c; c = b; b = a; | |
| a = (t1 + t2) | 0; | |
| } | |
| h[0] = (h[0] + a) | 0; | |
| h[1] = (h[1] + b) | 0; | |
| h[2] = (h[2] + c) | 0; | |
| h[3] = (h[3] + d) | 0; | |
| h[4] = (h[4] + e) | 0; | |
| h[5] = (h[5] + f) | 0; | |
| h[6] = (h[6] + g) | 0; | |
| h[7] = (h[7] + k) | 0; | |
| } | |
| // 64 hex characters, lowercase; the object is unusable afterwards | |
| digestHex() { | |
| if (this.finished) throw new Error("Sha256: digest() twice"); | |
| this.finished = true; | |
| const tail = new Uint8Array(this.blockLen < 56 ? 64 : 128); | |
| tail.set(this.block.subarray(0, this.blockLen)); | |
| tail[this.blockLen] = 0x80; | |
| // message length in bits, as a 64-bit big-endian integer | |
| const bits = this.bytes * 8; | |
| const hi = Math.floor(bits / 4294967296); | |
| const lo = bits % 4294967296; | |
| const end = tail.length; | |
| tail[end - 8] = (hi >>> 24) & 0xff; | |
| tail[end - 7] = (hi >>> 16) & 0xff; | |
| tail[end - 6] = (hi >>> 8) & 0xff; | |
| tail[end - 5] = hi & 0xff; | |
| tail[end - 4] = (lo >>> 24) & 0xff; | |
| tail[end - 3] = (lo >>> 16) & 0xff; | |
| tail[end - 2] = (lo >>> 8) & 0xff; | |
| tail[end - 1] = lo & 0xff; | |
| for (let off = 0; off < tail.length; off += 64) this._compress(tail, off); | |
| let hex = ""; | |
| for (let i = 0; i < 8; i++) hex += (this.h[i] >>> 0).toString(16).padStart(8, "0"); | |
| return hex; | |
| } | |
| } | |
| /** | |
| * Hash an async iterable of Uint8Array/ArrayBuffer chunks. | |
| * | |
| * @param {AsyncIterable<Uint8Array|ArrayBuffer>} source | |
| * @param {(done:number)=>void} [onProgress] bytes hashed so far | |
| * @returns {Promise<string>} lowercase hex digest | |
| */ | |
| export async function sha256Chunks(source, onProgress) { | |
| const hasher = new Sha256(); | |
| let done = 0; | |
| for await (const chunk of source) { | |
| const bytes = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk); | |
| hasher.update(bytes); | |
| done += bytes.length; | |
| if (onProgress) onProgress(done); | |
| } | |
| return hasher.digestHex(); | |
| } | |