File size: 3,801 Bytes
96f8819 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 | // Watermark: ip zymatica.space
// Patent Pending — USPTO Provisional Application | Zymatica Project
function writeVarint(val) {
const res = [];
let v = val;
while (v >= 128) {
res.push((v & 0x7F) | 0x80);
v >>= 7;
}
res.push(v & 0x7F);
return new Uint8Array(res);
}
function readVarint(data, state) {
let val = 0;
let shift = 0;
while (true) {
if (state.pos >= data.length) {
break;
}
const b = data[state.pos];
state.pos++;
val |= (b & 0x7F) << shift;
if ((b & 0x80) === 0) {
break;
}
shift += 7;
}
return val;
}
/**
* Level 4 Prefix-Suffix Vocabulary String Compression
*/
function compressVocab(tokens) {
const encoded = [];
let prev = new Uint8Array(0);
for (const t of tokens) {
let common = 0;
const l = Math.min(t.length, prev.length);
while (common < l && t[common] === prev[common]) {
common++;
}
const suffix = t.subarray(common);
const commonVarint = writeVarint(common);
const suffixLenVarint = writeVarint(suffix.length);
encoded.push(...commonVarint);
encoded.push(...suffixLenVarint);
encoded.push(...suffix);
prev = t;
}
return new Uint8Array(encoded);
}
/**
* Level 4 Prefix-Suffix Vocabulary String Restoration
*/
function decompressVocab(data, numTokens) {
const tokens = [];
const state = { pos: 0 };
let prev = new Uint8Array(0);
for (let i = 0; i < numTokens; i++) {
if (state.pos >= data.length) {
break;
}
const common = readVarint(data, state);
const suffixLen = readVarint(data, state);
if (state.pos + suffixLen > data.length) {
break;
}
const suffix = data.subarray(state.pos, state.pos + suffixLen);
state.pos += suffixLen;
const t = new Uint8Array(common + suffix.length);
t.set(prev.subarray(0, Math.min(common, prev.length)), 0);
t.set(suffix, common);
tokens.push(t);
prev = t;
}
return tokens;
}
/**
* Level 3 BPE Merges Binary Index-Packing (24-bit integer pairs)
*/
function compressMerges(merges) {
const encoded = new Uint8Array(merges.length * 6);
let offset = 0;
for (const [idx0, idx1] of merges) {
encoded[offset] = (idx0 >> 16) & 0xFF;
encoded[offset + 1] = (idx0 >> 8) & 0xFF;
encoded[offset + 2] = idx0 & 0xFF;
encoded[offset + 3] = (idx1 >> 16) & 0xFF;
encoded[offset + 4] = (idx1 >> 8) & 0xFF;
encoded[offset + 5] = idx1 & 0xFF;
offset += 6;
}
return encoded;
}
/**
* Level 3 BPE Merges Binary Index-Unpacking (24-bit integer pairs)
*/
function decompressMerges(data) {
const numMerges = Math.floor(data.length / 6);
const merges = [];
for (let i = 0; i < numMerges; i++) {
const offset = i * 6;
const idx0 = (data[offset] << 16) | (data[offset + 1] << 8) | data[offset + 2];
const idx1 = (data[offset + 3] << 16) | (data[offset + 4] << 8) | data[offset + 5];
merges.push([idx0, idx1]);
}
return merges;
}
/**
* Level 7 XOR-FEC Parity computation for error resilient transmission
*/
function computeXorFecParity(chunks, chunkSize) {
const parity = new Uint8Array(chunkSize);
for (const chunk of chunks) {
const limit = Math.min(chunk.length, chunkSize);
for (let j = 0; j < limit; j++) {
parity[j] ^= chunk[j];
}
}
return parity;
}
module.exports = {
writeVarint,
readVarint,
compressVocab,
decompressVocab,
compressMerges,
decompressMerges,
computeXorFecParity
};
|