File size: 4,961 Bytes
945d026 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | // Watermark: ip zymatica.space
// Patent Pending — USPTO Provisional Application | Zymatica Project
package ufo;
import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
import java.util.List;
public class TokenizerCoder {
public static byte[] writeVarint(int val) {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
int v = val;
while (v >= 128) {
bos.write((v & 0x7F) | 0x80);
v >>>= 7;
}
bos.write(v & 0x7F);
return bos.toByteArray();
}
public static int readVarint(byte[] data, int[] state) {
int val = 0;
int shift = 0;
while (true) {
if (state[0] >= data.length) {
break;
}
byte b = data[state[0]];
state[0]++;
val |= (b & 0x7F) << shift;
if ((b & 0x80) == 0) {
break;
}
shift += 7;
}
return val;
}
/**
* Level 4 Prefix-Suffix Vocabulary String Compression
*/
public static byte[] compressVocab(List<byte[]> tokens) {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
byte[] prev = new byte[0];
for (byte[] t : tokens) {
int common = 0;
int l = Math.min(t.length, prev.length);
while (common < l && t[common] == prev[common]) {
common++;
}
int suffixLen = t.length - common;
byte[] suffix = new byte[suffixLen];
System.arraycopy(t, common, suffix, 0, suffixLen);
try {
bos.write(writeVarint(common));
bos.write(writeVarint(suffixLen));
bos.write(suffix);
} catch (Exception e) {
// Ignore
}
prev = t;
}
return bos.toByteArray();
}
/**
* Level 4 Prefix-Suffix Vocabulary String Restoration
*/
public static List<byte[]> decompressVocab(byte[] data, int numTokens) {
List<byte[]> tokens = new ArrayList<>(numTokens);
int[] state = new int[]{0};
byte[] prev = new byte[0];
for (int i = 0; i < numTokens; i++) {
if (state[0] >= data.length) {
break;
}
int common = readVarint(data, state);
int suffixLen = readVarint(data, state);
if (state[0] + suffixLen > data.length) {
break;
}
byte[] suffix = new byte[suffixLen];
System.arraycopy(data, state[0], suffix, 0, suffixLen);
state[0] += suffixLen;
byte[] t = new byte[common + suffixLen];
System.arraycopy(prev, 0, t, 0, Math.min(common, prev.length));
System.arraycopy(suffix, 0, t, common, suffixLen);
tokens.add(t);
prev = t;
}
return tokens;
}
/**
* Level 3 BPE Merges Binary Index-Packing (24-bit integer pairs)
*/
public static byte[] compressMerges(List<int[]> merges) {
byte[] encoded = new byte[merges.size() * 6];
int offset = 0;
for (int[] pair : merges) {
int idx0 = pair[0];
int int1 = pair[1];
encoded[offset] = (byte)((idx0 >> 16) & 0xFF);
encoded[offset + 1] = (byte)((idx0 >> 8) & 0xFF);
encoded[offset + 2] = (byte)(idx0 & 0xFF);
encoded[offset + 3] = (byte)((int1 >> 16) & 0xFF);
encoded[offset + 4] = (byte)((int1 >> 8) & 0xFF);
encoded[offset + 5] = (byte)(int1 & 0xFF);
offset += 6;
}
return encoded;
}
/**
* Level 3 BPE Merges Binary Index-Unpacking (24-bit integer pairs)
*/
public static List<int[]> decompressMerges(byte[] data) {
int numMerges = data.length / 6;
List<int[]> merges = new ArrayList<>(numMerges);
for (int i = 0; i < numMerges; i++) {
int offset = i * 6;
int idx0 = ((data[offset] & 0xFF) << 16) |
((data[offset + 1] & 0xFF) << 8) |
(data[offset + 2] & 0xFF);
int idx1 = ((data[offset + 3] & 0xFF) << 16) |
((data[offset + 4] & 0xFF) << 8) |
(data[offset + 5] & 0xFF);
merges.add(new int[]{idx0, idx1});
}
return merges;
}
/**
* Level 7 XOR-FEC Parity computation for error resilient transmission
*/
public static byte[] computeXorFecParity(List<byte[]> chunks, int chunkSize) {
byte[] parity = new byte[chunkSize];
for (byte[] chunk : chunks) {
int limit = Math.min(chunk.length, chunkSize);
for (int j = 0; j < limit; j++) {
parity[j] ^= chunk[j];
}
}
return parity;
}
}
|