File size: 3,957 Bytes
96f8819 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | // Watermark: ip zymatica.space
// Patent Pending — USPTO Provisional Application | Zymatica Project
export function writeVarint(val: number): Uint8Array {
const res: number[] = [];
let v = val;
while (v >= 128) {
res.push((v & 0x7F) | 0x80);
v >>= 7;
}
res.push(v & 0x7F);
return new Uint8Array(res);
}
export function readVarint(data: Uint8Array, state: { pos: number }): number {
let val = 0;
let shift = 0;
while (true) {
if (state.pos >= data.length) {
break;
}
const b = data[state.pos];
state.pos++;
val |= (b & 0x7F) << shift;
if ((b & 0x80) === 0) {
break;
}
shift += 7;
}
return val;
}
/**
* Level 4 Prefix-Suffix Vocabulary String Compression
*/
export function compressVocab(tokens: Uint8Array[]): Uint8Array {
const encoded: number[] = [];
let prev = new Uint8Array(0);
for (const t of tokens) {
let common = 0;
const l = Math.min(t.length, prev.length);
while (common < l && t[common] === prev[common]) {
common++;
}
const suffix = t.subarray(common);
const commonVarint = writeVarint(common);
const suffixLenVarint = writeVarint(suffix.length);
encoded.push(...commonVarint);
encoded.push(...suffixLenVarint);
encoded.push(...suffix);
prev = t;
}
return new Uint8Array(encoded);
}
/**
* Level 4 Prefix-Suffix Vocabulary String Restoration
*/
export function decompressVocab(data: Uint8Array, numTokens: number): Uint8Array[] {
const tokens: Uint8Array[] = [];
const state = { pos: 0 };
let prev = new Uint8Array(0);
for (let i = 0; i < numTokens; i++) {
if (state.pos >= data.length) {
break;
}
const common = readVarint(data, state);
const suffixLen = readVarint(data, state);
if (state.pos + suffixLen > data.length) {
break;
}
const suffix = data.subarray(state.pos, state.pos + suffixLen);
state.pos += suffixLen;
const t = new Uint8Array(common + suffix.length);
t.set(prev.subarray(0, Math.min(common, prev.length)), 0);
t.set(suffix, common);
tokens.push(t);
prev = t;
}
return tokens;
}
/**
* Level 3 BPE Merges Binary Index-Packing (24-bit integer pairs)
*/
export function compressMerges(merges: [number, number][]): Uint8Array {
const encoded = new Uint8Array(merges.length * 6);
let offset = 0;
for (const [idx0, idx1] of merges) {
encoded[offset] = (idx0 >> 16) & 0xFF;
encoded[offset + 1] = (idx0 >> 8) & 0xFF;
encoded[offset + 2] = idx0 & 0xFF;
encoded[offset + 3] = (idx1 >> 16) & 0xFF;
encoded[offset + 4] = (idx1 >> 8) & 0xFF;
encoded[offset + 5] = idx1 & 0xFF;
offset += 6;
}
return encoded;
}
/**
* Level 3 BPE Merges Binary Index-Unpacking (24-bit integer pairs)
*/
export function decompressMerges(data: Uint8Array): [number, number][] {
const numMerges = Math.floor(data.length / 6);
const merges: [number, number][] = [];
for (let i = 0; i < numMerges; i++) {
const offset = i * 6;
const idx0 = (data[offset] << 16) | (data[offset + 1] << 8) | data[offset + 2];
const idx1 = (data[offset + 3] << 16) | (data[offset + 4] << 8) | data[offset + 5];
merges.push([idx0, idx1]);
}
return merges;
}
/**
* Level 7 XOR-FEC Parity computation for error resilient transmission
*/
export function computeXorFecParity(chunks: Uint8Array[], chunkSize: number): Uint8Array {
const parity = new Uint8Array(chunkSize);
for (const chunk of chunks) {
const limit = Math.min(chunk.length, chunkSize);
for (let j = 0; j < limit; j++) {
parity[j] ^= chunk[j];
}
}
return parity;
}
|