| { |
| "arms": { |
| "fp32": { |
| "val": 2.0553244292736053, |
| "ppl": 7.809371050272343, |
| "params": 11159360 |
| }, |
| "qat_ternary": { |
| "val": 2.1759746789932253, |
| "ppl": 8.81076860951355, |
| "params": 11159360 |
| }, |
| "ptq_ternary": { |
| "val": 5.022902528444926, |
| "ppl": 151.8514178229577, |
| "params": 11159360 |
| }, |
| "fp16_d128": { |
| "val": 2.360668977101644, |
| "ppl": 10.598038926854809, |
| "params": 2104448 |
| }, |
| "qat_ternary_embed": { |
| "val": 2.310712154706319, |
| "ppl": 10.081601759246057, |
| "params": 11159360, |
| "embed_mode": "ternary", |
| "qat_recovery_of_ptq_embed_loss": 0.760477491978586 |
| } |
| }, |
| "baselines": { |
| "uniform": 8.3178, |
| "unigram": 6.038, |
| "bigram_bound": 3.614 |
| }, |
| "ladder_fp32": { |
| "rung2": 5.3828, |
| "rung3": 3.9106, |
| "rung4": 3.715, |
| "rung5": 3.2948, |
| "rung6": 2.4304, |
| "rung7": 2.0651 |
| }, |
| "config": { |
| "vocab": 4096, |
| "d": 320, |
| "n_layer": 8, |
| "n_head": 8, |
| "mult": 4, |
| "block_size": 256, |
| "weight_mode": "ternary", |
| "act_bits": 8 |
| }, |
| "training": { |
| "tokens": 19996672, |
| "steps": 2441, |
| "batch": 32, |
| "ctx": 256, |
| "lr": 0.001, |
| "warmup": 100, |
| "seed": 1337, |
| "optimizer": "AdamW(0.9,0.95) wd=0.1 clip=1.0", |
| "schedule": "linear warmup, cosine to 10%" |
| }, |
| "param_split": { |
| "ternary": 9830400, |
| "full_precision": 1328960 |
| }, |
| "packed_bytes": { |
| "ternary_body": 1947601, |
| "fp16_rest": 2657920, |
| "total": 4605521 |
| }, |
| "key_findings": { |
| "qat_gap_to_fp32_nats": 0.12065024971962002, |
| "qat_recovered_fraction_of_ptq_loss": 0.9593438670566713, |
| "equal_memory_win_vs_fp16_nats": 0.18469429810841875, |
| "ternarization_relative_error": 0.531, |
| "weight_zero_fraction": 0.314, |
| "qat_wallclock_multiplier": 1.1170731707317074 |
| }, |
| "note": "checkpoints store fp32 MASTER weights. ternary quantization happens in the forward pass. the 4.61 MB figure is the packed inference size.", |
| "pareto": { |
| "all_fp16": { |
| "bytes": 22318720, |
| "MB": 22.32, |
| "share_1p58bit": 0.0, |
| "share_fp16": 0.0016, |
| "compression_vs_fp16": 1.0, |
| "val": 2.0553244292736053 |
| }, |
| "ternary_body_fp16_embed": { |
| "bytes": 4605521, |
| "MB": 4.61, |
| "share_1p58bit": 0.4229, |
| "share_fp16": 0.0079, |
| "compression_vs_fp16": 4.85, |
| "val": 2.1759746789932253 |
| }, |
| "ternary_body_int8_embed": { |
| "bytes": 3294801, |
| "MB": 3.29, |
| "share_1p58bit": 0.5911, |
| "share_fp16": 0.0111, |
| "compression_vs_fp16": 6.77, |
| "val": 2.1795 |
| }, |
| "ternary_body_ternary_embed": { |
| "bytes": 2243762, |
| "MB": 2.24, |
| "share_1p58bit": 0.9837, |
| "share_fp16": 0.0163, |
| "compression_vs_fp16": 9.95, |
| "val": 2.310712154706319 |
| } |
| }, |
| "param_buckets": { |
| "ternary_body": 9830400, |
| "embedding_tied": 1310720, |
| "norms_always_fp16": 18240 |
| }, |
| "correction": "an earlier table bucketed embedding+norms as 'full precision' then quantized the bucket; the true fp16 share of the ternary-embed model is 1.63%, not 11.9%" |
| } |
