tokenizer_coder_test.go

package ufo

import (
	"bytes"
	"fmt"
	"testing"
)

func TestTokenizerCoder(t *testing.T) {
	fmt.Println("=========================================================")
	fmt.Println("  RUNNING GO UFO TOKENIZER CODER VERIFICATION")
	fmt.Println("  Watermark: ip zymatica.space")
	fmt.Println("=========================================================")

	// 1. Test Vocab Coder
	fmt.Println("\n[Test 1] Prefix-Suffix Vocab Coder...")
	originalVocab := [][]byte{
		[]byte("hello"),
		[]byte("hell"),
		[]byte("heaven"),
		[]byte("heavy"),
		[]byte("world"),
		[]byte("word"),
		[]byte("work"),
		[]byte("worker"),
		[]byte("working"),
	}

	compressedVocab := CompressVocab(originalVocab)
	fmt.Printf("  Original vocab items: %d\n", len(originalVocab))
	fmt.Printf("  Compressed vocab size: %d bytes\n", len(compressedVocab))

	restoredVocab := DecompressVocab(compressedVocab, len(originalVocab))
	fmt.Printf("  Restored vocab items: %d\n", len(restoredVocab))

	if len(originalVocab) != len(restoredVocab) {
		t.Fatalf("Vocab length mismatch: expected %d, got %d", len(originalVocab), len(restoredVocab))
	}
	for i := range originalVocab {
		if !bytes.Equal(originalVocab[i], restoredVocab[i]) {
			t.Fatalf("Vocab item mismatch at index %d: expected %s, got %s", i, originalVocab[i], restoredVocab[i])
		}
	}
	fmt.Println("  [+] Vocab round-trip: SUCCESS (100% Match)")

	// 2. Test BPE Merges Coder
	fmt.Println("\n[Test 2] BPE Merges Binary Index Coder...")
	originalMerges := [][2]uint32{
		{1015, 2030},
		{45, 12},
		{16777215, 50000},
		{0, 1},
		{100000, 200000},
	}

	compressedMerges := CompressMerges(originalMerges)
	fmt.Printf("  Original merges items: %d\n", len(originalMerges))
	fmt.Printf("  Compressed merges size: %d bytes\n", len(compressedMerges))

	restoredMerges := DecompressMerges(compressedMerges)
	fmt.Printf("  Restored merges items: %d\n", len(restoredMerges))

	if len(originalMerges) != len(restoredMerges) {
		t.Fatalf("Merges length mismatch: expected %d, got %d", len(originalMerges), len(restoredMerges))
	}
	for i := range originalMerges {
		if originalMerges[i] != restoredMerges[i] {
			t.Fatalf("Merges item mismatch at index %d: expected %v, got %v", i, originalMerges[i], restoredMerges[i])
		}
	}
	fmt.Println("  [+] Merges round-trip: SUCCESS (100% Match)")

	// 3. Test XOR-FEC Parity
	fmt.Println("\n[Test 3] XOR-FEC Parity Calculation...")
	c1 := []byte{0xAA, 0xBB, 0xCC, 0xDD}
	c2 := []byte{0x11, 0x22, 0x33, 0x44}
	c3 := []byte{0x55, 0x66, 0x77, 0x88}
	chunks := [][]byte{c1, c2, c3}

	parity := ComputeXorFecParity(chunks, 4)
	expectedParity := []byte{
		0xAA ^ 0x11 ^ 0x55,
		0xBB ^ 0x22 ^ 0x66,
		0xCC ^ 0x33 ^ 0x77,
		0xDD ^ 0x44 ^ 0x88,
	}

	if !bytes.Equal(parity, expectedParity) {
		t.Fatalf("Parity mismatch: expected %v, got %v", expectedParity, parity)
	}
	fmt.Println("  [+] XOR-FEC computation: SUCCESS")

	fmt.Println("\n=========================================================")
	fmt.Println("  ALL GO TESTS PASSED SUCCESSFULLY!")
	fmt.Println("=========================================================")
}