File size: 3,738 Bytes
8b2a791 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | // Watermark: ip zymatica.space
// Patent Pending — USPTO Provisional Application | Zymatica Project
import Foundation
public func writeVarint(_ val: Int) -> [UInt8] {
var res = [UInt8]()
var v = val
while v >= 128 {
res.append(UInt8((v & 0x7F) | 0x80))
v >>= 7
}
res.append(UInt8(v & 0x7F))
return res
}
public func readVarint(_ data: [UInt8], _ pos: inout Int) -> Int {
var val = 0
var shift = 0
while true {
if pos >= data.count {
break
}
let b = data[pos]
pos += 1
val |= Int(b & 0x7F) << shift
if (b & 0x80) == 0 {
break
}
shift += 7
}
return val
}
/**
* Level 4 Prefix-Suffix Vocabulary String Compression
*/
public func compressVocab(_ tokens: [[UInt8]]) -> [UInt8] {
var encoded = [UInt8]()
var prev = [UInt8]()
for t in tokens {
var common = 0
let l = min(t.count, prev.count)
while common < l && t[common] == prev[common] {
common += 1
}
let suffix = Array(t[common...])
encoded.append(contentsOf: writeVarint(common))
encoded.append(contentsOf: writeVarint(suffix.count))
encoded.append(contentsOf: suffix)
prev = t
}
return encoded
}
/**
* Level 4 Prefix-Suffix Vocabulary String Restoration
*/
public func decompressVocab(_ data: [UInt8], _ numTokens: Int) -> [[UInt8]] {
var tokens = [[UInt8]]()
var pos = 0
var prev = [UInt8]()
for _ in 0..<numTokens {
if pos >= data.count {
break
}
let common = readVarint(data, &pos)
let suffixLen = readVarint(data, &pos)
if pos + suffixLen > data.count {
break
}
let suffix = Array(data[pos..<(pos + suffixLen)])
pos += suffixLen
var t = Array(prev[0..<min(common, prev.count)])
t.append(contentsOf: suffix)
tokens.append(t)
prev = t
}
return tokens
}
/**
* Level 3 BPE Merges Binary Index-Packing (24-bit integer pairs)
*/
public func compressMerges(_ merges: [(UInt32, UInt32)]) -> [UInt8] {
var encoded = [UInt8]()
encoded.reserveCapacity(merges.count * 6)
for pair in merges {
let idx0 = pair.0
let idx1 = pair.1
encoded.append(UInt8((idx0 >> 16) & 0xFF))
encoded.append(UInt8((idx0 >> 8) & 0xFF))
encoded.append(UInt8(idx0 & 0xFF))
encoded.append(UInt8((idx1 >> 16) & 0xFF))
encoded.append(UInt8((idx1 >> 8) & 0xFF))
encoded.append(UInt8(idx1 & 0xFF))
}
return encoded
}
/**
* Level 3 BPE Merges Binary Index-Unpacking (24-bit integer pairs)
*/
public func decompressMerges(_ data: [UInt8]) -> [(UInt32, UInt32)] {
let numMerges = data.count / 6
var merges = [(UInt32, UInt32)]()
merges.reserveCapacity(numMerges)
for i in 0..<numMerges {
let offset = i * 6
let idx0 = (UInt32(data[offset]) << 16) |
(UInt32(data[offset + 1]) << 8) |
UInt32(data[offset + 2])
let idx1 = (UInt32(data[offset + 3]) << 16) |
(UInt32(data[offset + 4]) << 8) |
UInt32(data[offset + 5])
merges.append((idx0, idx1))
}
return merges
}
/**
* Level 7 XOR-FEC Parity computation for error resilient transmission
*/
public func computeXorFecParity(_ chunks: [[UInt8]], _ chunkSize: Int) -> [UInt8] {
var parity = [UInt8](repeating: 0, count: chunkSize)
for chunk in chunks {
let limit = min(chunk.count, chunkSize)
for j in 0..<limit {
parity[j] ^= chunk[j]
}
}
return parity
}
|