File size: 3,067 Bytes
cdddfb8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 | package ufo
import (
"bytes"
"fmt"
"testing"
)
func TestTokenizerCoder(t *testing.T) {
fmt.Println("=========================================================")
fmt.Println(" RUNNING GO UFO TOKENIZER CODER VERIFICATION")
fmt.Println(" Watermark: ip zymatica.space")
fmt.Println("=========================================================")
// 1. Test Vocab Coder
fmt.Println("\n[Test 1] Prefix-Suffix Vocab Coder...")
originalVocab := [][]byte{
[]byte("hello"),
[]byte("hell"),
[]byte("heaven"),
[]byte("heavy"),
[]byte("world"),
[]byte("word"),
[]byte("work"),
[]byte("worker"),
[]byte("working"),
}
compressedVocab := CompressVocab(originalVocab)
fmt.Printf(" Original vocab items: %d\n", len(originalVocab))
fmt.Printf(" Compressed vocab size: %d bytes\n", len(compressedVocab))
restoredVocab := DecompressVocab(compressedVocab, len(originalVocab))
fmt.Printf(" Restored vocab items: %d\n", len(restoredVocab))
if len(originalVocab) != len(restoredVocab) {
t.Fatalf("Vocab length mismatch: expected %d, got %d", len(originalVocab), len(restoredVocab))
}
for i := range originalVocab {
if !bytes.Equal(originalVocab[i], restoredVocab[i]) {
t.Fatalf("Vocab item mismatch at index %d: expected %s, got %s", i, originalVocab[i], restoredVocab[i])
}
}
fmt.Println(" [+] Vocab round-trip: SUCCESS (100% Match)")
// 2. Test BPE Merges Coder
fmt.Println("\n[Test 2] BPE Merges Binary Index Coder...")
originalMerges := [][2]uint32{
{1015, 2030},
{45, 12},
{16777215, 50000},
{0, 1},
{100000, 200000},
}
compressedMerges := CompressMerges(originalMerges)
fmt.Printf(" Original merges items: %d\n", len(originalMerges))
fmt.Printf(" Compressed merges size: %d bytes\n", len(compressedMerges))
restoredMerges := DecompressMerges(compressedMerges)
fmt.Printf(" Restored merges items: %d\n", len(restoredMerges))
if len(originalMerges) != len(restoredMerges) {
t.Fatalf("Merges length mismatch: expected %d, got %d", len(originalMerges), len(restoredMerges))
}
for i := range originalMerges {
if originalMerges[i] != restoredMerges[i] {
t.Fatalf("Merges item mismatch at index %d: expected %v, got %v", i, originalMerges[i], restoredMerges[i])
}
}
fmt.Println(" [+] Merges round-trip: SUCCESS (100% Match)")
// 3. Test XOR-FEC Parity
fmt.Println("\n[Test 3] XOR-FEC Parity Calculation...")
c1 := []byte{0xAA, 0xBB, 0xCC, 0xDD}
c2 := []byte{0x11, 0x22, 0x33, 0x44}
c3 := []byte{0x55, 0x66, 0x77, 0x88}
chunks := [][]byte{c1, c2, c3}
parity := ComputeXorFecParity(chunks, 4)
expectedParity := []byte{
0xAA ^ 0x11 ^ 0x55,
0xBB ^ 0x22 ^ 0x66,
0xCC ^ 0x33 ^ 0x77,
0xDD ^ 0x44 ^ 0x88,
}
if !bytes.Equal(parity, expectedParity) {
t.Fatalf("Parity mismatch: expected %v, got %v", expectedParity, parity)
}
fmt.Println(" [+] XOR-FEC computation: SUCCESS")
fmt.Println("\n=========================================================")
fmt.Println(" ALL GO TESTS PASSED SUCCESSFULLY!")
fmt.Println("=========================================================")
}
|