Add files via upload

ablation.c

@@ -25,7 +25,7 @@
 #define SCALE (1 << 14)
 
 /* ── Flags ── */
-#define FLAG_TWEEDIE    1
+#define FLAG_TWEEDIE    1   /* post-blend Tweedie (after match/word/highctx) */
 #define FLAG_MATCH      2
 #define FLAG_WORD       4
 #define FLAG_HIGHCTX    8
@@ -84,9 +84,6 @@ static uint8_t *do_compress(const uint8_t *data, size_t data_len,
 
         ppm_predict(&ppm, probs, &confidence, &order);
 
-        if (flags & FLAG_TWEEDIE) {
-            tweedie_denoise(&twd, probs, order, confidence);
-        }
         clamp_normalize(probs);
 
         if (flags & FLAG_MATCH) {
@@ -108,6 +105,11 @@ static uint8_t *do_compress(const uint8_t *data, size_t data_len,
                 blend_highctx(probs, hctx_probs, hctx_conf);
         }
 
+        if (flags & FLAG_TWEEDIE) {
+            tweedie_denoise(&twd, probs, order, confidence);
+            clamp_normalize(probs);
+        }
+
         probs_to_cumfreqs(probs, cumfreqs, &total);
         ae_encode(&enc, cumfreqs, byte, total);
 
@@ -178,9 +180,6 @@ static uint8_t *do_decompress(const uint8_t *compressed, size_t comp_len,
 
         ppm_predict(&ppm, probs, &confidence, &order);
 
-        if (flags & FLAG_TWEEDIE) {
-            tweedie_denoise(&twd, probs, order, confidence);
-        }
         clamp_normalize(probs);
 
         if (flags & FLAG_MATCH) {
@@ -202,6 +201,11 @@ static uint8_t *do_decompress(const uint8_t *compressed, size_t comp_len,
                 blend_highctx(probs, hctx_probs, hctx_conf);
         }
 
+        if (flags & FLAG_TWEEDIE) {
+            tweedie_denoise(&twd, probs, order, confidence);
+            clamp_normalize(probs);
+        }
+
         probs_to_cumfreqs(probs, cumfreqs, &total);
         int sym = ad_decode(&dec, cumfreqs, total);
         result[i] = (uint8_t)sym;
@@ -235,11 +239,11 @@ typedef struct {
 } AblationConfig;
 
 static const AblationConfig CONFIGS[] = {
-    { "Base PPM",                    0 },
-    { "+ Tweedie",                   FLAG_TWEEDIE },
-    { "+ Twd + Match",               FLAG_TWEEDIE | FLAG_MATCH },
-    { "+ Twd + Match + Word",        FLAG_TWEEDIE | FLAG_MATCH | FLAG_WORD },
-    { "+ Twd + M + W + H",           FLAG_TWEEDIE | FLAG_MATCH | FLAG_WORD | FLAG_HIGHCTX },
+    { "Base PPM",               0 },
+    { "+ Match",                FLAG_MATCH },
+    { "+ Match + Word",         FLAG_MATCH | FLAG_WORD },
+    { "+ Match + Word + HCtx",  FLAG_MATCH | FLAG_WORD | FLAG_HIGHCTX },
+    { "+ M + W + H + Tweedie",  FLAG_MATCH | FLAG_WORD | FLAG_HIGHCTX | FLAG_TWEEDIE },
 };
 #define N_CONFIGS 5
 

mdc.c

@@ -1,6 +1,6 @@
 /*
  * Midicoth Compressor — C implementation
- * Pipeline: PPM + Tweedie Denoising + Match + Word + HighCtx
+ * Pipeline: PPM + Match + Word + HighCtx + Tweedie Denoising
  *
  * Usage:
  *   ./mdc compress   <input> <output>
@@ -96,8 +96,6 @@ static int do_compress(const char *input_path, const char *output_path) {
         double confidence;
         int order;
         ppm_predict(&ppm, probs, &confidence, &order);
-
-        tweedie_denoise(&twd, probs, order, confidence);
         clamp_normalize(probs);
 
         int match_byte;
@@ -113,6 +111,9 @@ static int do_compress(const char *input_path, const char *output_path) {
         if (highctx_predict(&hctx, hctx_probs, &hctx_conf))
             blend_highctx(probs, hctx_probs, hctx_conf);
 
+        tweedie_denoise(&twd, probs, order, confidence);
+        clamp_normalize(probs);
+
         probs_to_cumfreqs(probs, cumfreqs, &total);
         ae_encode(&enc, cumfreqs, byte, total);
 
@@ -219,8 +220,6 @@ static int do_decompress(const char *input_path, const char *output_path) {
         double confidence;
         int order;
         ppm_predict(&ppm, probs, &confidence, &order);
-
-        tweedie_denoise(&twd, probs, order, confidence);
         clamp_normalize(probs);
 
         int match_byte;
@@ -236,6 +235,9 @@ static int do_decompress(const char *input_path, const char *output_path) {
         if (highctx_predict(&hctx, hctx_probs, &hctx_conf))
             blend_highctx(probs, hctx_probs, hctx_conf);
 
+        tweedie_denoise(&twd, probs, order, confidence);
+        clamp_normalize(probs);
+
         probs_to_cumfreqs(probs, cumfreqs, &total);
         int sym = ad_decode(&dec, cumfreqs, total);
         result[i] = (uint8_t)sym;

measure_delta.c

@@ -0,0 +1,277 @@
+/*
+ * Measure mean |delta| vs noise level (confidence) for the
+ * delta-vs-gamma table in the paper.
+ *
+ * Runs the full pipeline (PPM+Match+Word+HCtx+Tweedie) and
+ * instruments the Tweedie denoise to collect per-step, per-confidence
+ * delta statistics.
+ *
+ * Usage: ./measure_delta <input_file>
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+
+#include "fastmath.h"
+#include "arith.h"
+#include "ppm.h"
+#include "match.h"
+#include "word.h"
+#include "highctx.h"
+
+/* We need access to Tweedie internals, so include it but also
+ * define instrumentation hooks */
+#include "tweedie.h"
+
+/* Accumulate |delta| by [step][conf_bin] */
+#define N_CONF_REPORT 4
+static double delta_sum[TWD_STEPS][N_CONF_REPORT];
+static double delta_count[TWD_STEPS][N_CONF_REPORT];
+
+/* Map raw confidence to our 4 reporting bins:
+ *   bin 0: C ~ 128  (gamma ~ 0.500)
+ *   bin 1: C ~ 512  (gamma ~ 0.200)
+ *   bin 2: C ~ 2048 (gamma ~ 0.059)
+ *   bin 3: C ~ 8192 (gamma ~ 0.015) */
+static int conf_report_bin(double confidence) {
+    if (confidence < 256.0)  return 0;
+    if (confidence < 1024.0) return 1;
+    if (confidence < 4096.0) return 2;
+    return 3;
+}
+
+/* Instrumented denoise that collects delta stats */
+static void tweedie_denoise_instrumented(TweedieDenoiser *td, double *probs,
+                                          int ppm_order, double confidence) {
+    int og = twd_order_group(ppm_order);
+    int cb = twd_conf_bin(confidence);
+    int crb = conf_report_bin(confidence);
+
+    double max_p = 0.0;
+    for (int i = 0; i < TWD_NSYM; i++)
+        if (probs[i] > max_p) max_p = probs[i];
+    int sb = twd_shape_bin(max_p);
+
+    td->cached_ord = og;
+    td->cached_shape = sb;
+    td->cached_conf = cb;
+
+    double stree[512];
+    double scale[512];
+
+    for (int step = 0; step < TWD_STEPS; step++) {
+        for (int i = 0; i < TWD_NSYM; i++)
+            stree[TWD_NSYM + i] = probs[i];
+        for (int i = TWD_NSYM - 1; i >= 1; i--)
+            stree[i] = stree[2 * i] + stree[2 * i + 1];
+
+        scale[1] = 1.0;
+
+        for (int level = 0; level < TWD_N_LEVELS; level++) {
+            int level_start = 1 << level;
+            int level_end   = 1 << (level + 1);
+
+            for (int ni = level_start; ni < level_end; ni++) {
+                double node_total = stree[ni];
+                int node_id = ni - 1;
+                int node_at_level = ni - level_start;
+
+                if (node_total < 1e-15) {
+                    scale[2 * ni]     = scale[ni];
+                    scale[2 * ni + 1] = scale[ni];
+                    td->cached_p_right[step][node_id] = 0.5;
+                    td->cached_prob_bin[step][node_id] = twd_prob_bin(0.5);
+                    td->cached_bctx[step][node_id] = twd_bit_context(level, node_at_level);
+                    continue;
+                }
+
+                double sum_right = stree[2 * ni + 1];
+                double p_right = sum_right / node_total;
+                if (p_right < 1e-8) p_right = 1e-8;
+                if (p_right > 1.0 - 1e-8) p_right = 1.0 - 1e-8;
+
+                int bctx = twd_bit_context(level, node_at_level);
+                int pbin = twd_prob_bin(p_right);
+                td->cached_p_right[step][node_id] = p_right;
+                td->cached_prob_bin[step][node_id] = pbin;
+                td->cached_bctx[step][node_id] = bctx;
+
+                TwdCalibEntry *e = &td->table[step][bctx][og][sb][cb][pbin];
+                double avg_pred = e->sum_pred / e->total;
+                double emp_rate = e->hits / e->total;
+                double delta = emp_rate - avg_pred;
+
+                /* Apply same shrinkage as production code */
+                double var_err = e->sum_sq_err / e->total;
+                if (e->total > 10.0 && var_err > 1e-10) {
+                    double snr = delta * delta * e->total / var_err;
+                    double shrink = (snr > 4.0) ? 1.0 : snr / 4.0;
+                    delta *= shrink;
+                } else {
+                    delta = 0.0;
+                }
+
+                /* Collect stats: weight by node probability mass */
+                double weight = node_total / stree[1];
+                delta_sum[step][crb]   += fabs(delta) * weight;
+                delta_count[step][crb] += weight;
+
+                double p_right_corr = p_right + delta;
+                if (p_right_corr < 1e-8)       p_right_corr = 1e-8;
+                if (p_right_corr > 1.0 - 1e-8) p_right_corr = 1.0 - 1e-8;
+
+                double sl = (1.0 - p_right_corr) / (1.0 - p_right);
+                double sr = p_right_corr / p_right;
+                scale[2 * ni]     = scale[ni] * sl;
+                scale[2 * ni + 1] = scale[ni] * sr;
+            }
+        }
+
+        for (int i = 0; i < TWD_NSYM; i++)
+            probs[i] *= scale[TWD_NSYM + i];
+
+        double sum = 0.0;
+        for (int i = 0; i < TWD_NSYM; i++) {
+            if (probs[i] < 1e-10) probs[i] = 1e-10;
+            sum += probs[i];
+        }
+        double inv = 1.0 / sum;
+        for (int i = 0; i < TWD_NSYM; i++)
+            probs[i] *= inv;
+
+        max_p = 0.0;
+        for (int i = 0; i < TWD_NSYM; i++)
+            if (probs[i] > max_p) max_p = probs[i];
+        sb = twd_shape_bin(max_p);
+    }
+}
+
+static void my_clamp_normalize(double *p) {
+    double sum = 0;
+    for (int i = 0; i < 256; i++) {
+        if (p[i] < 1e-10) p[i] = 1e-10;
+        sum += p[i];
+    }
+    double inv = 1.0 / sum;
+    for (int i = 0; i < 256; i++) p[i] *= inv;
+}
+
+static void my_blend_match(double *probs, int match_byte, double match_conf) {
+    if (match_byte < 0 || match_conf < 0.01) return;
+    double w = match_conf * 0.85;
+    if (w > 0.95) w = 0.95;
+    for (int i = 0; i < 256; i++)
+        probs[i] *= (1.0 - w);
+    probs[match_byte] += w;
+}
+
+static void my_blend_word(double *probs, double *wprobs, double wconf) {
+    double w = wconf * 0.35;
+    if (w > 0.45) w = 0.45;
+    for (int i = 0; i < 256; i++)
+        probs[i] = (1.0 - w) * probs[i] + w * wprobs[i];
+}
+
+static void my_blend_hctx(double *probs, double *hprobs, double hconf) {
+    double w = hconf * 2.0;
+    if (w > 0.60) w = 0.60;
+    for (int i = 0; i < 256; i++)
+        probs[i] = (1.0 - w) * probs[i] + w * hprobs[i];
+}
+
+int main(int argc, char **argv) {
+    if (argc < 2) {
+        fprintf(stderr, "Usage: %s <input_file>\n", argv[0]);
+        return 1;
+    }
+
+    FILE *f = fopen(argv[1], "rb");
+    if (!f) { perror(argv[1]); return 1; }
+    fseek(f, 0, SEEK_END);
+    size_t len = ftell(f);
+    fseek(f, 0, SEEK_SET);
+    uint8_t *data = malloc(len);
+    fread(data, 1, len, f);
+    fclose(f);
+
+    PPMModel ppm;
+    ppm_init(&ppm);
+
+    MatchModel match;
+    match_init(&match);
+
+    WordModel word;
+    word_init(&word);
+
+    HighCtxModel hctx;
+    highctx_init(&hctx);
+
+    TweedieDenoiser *twd = malloc(sizeof(TweedieDenoiser));
+    tweedie_init(twd);
+
+    memset(delta_sum, 0, sizeof(delta_sum));
+    memset(delta_count, 0, sizeof(delta_count));
+
+    double probs[256], word_probs[256], hctx_probs[256];
+
+    for (size_t i = 0; i < len; i++) {
+        uint8_t byte = data[i];
+        double confidence;
+        int order;
+
+        ppm_predict(&ppm, probs, &confidence, &order);
+        my_clamp_normalize(probs);
+
+        int match_byte;
+        double match_conf;
+        match_predict(&match, &match_byte, &match_conf);
+        my_blend_match(probs, match_byte, match_conf);
+
+        double w_conf;
+        if (word_predict_cached(&word, word_probs, &w_conf))
+            my_blend_word(probs, word_probs, w_conf);
+
+        double hctx_conf;
+        if (highctx_predict(&hctx, hctx_probs, &hctx_conf))
+            my_blend_hctx(probs, hctx_probs, hctx_conf);
+
+        tweedie_denoise_instrumented(twd, probs, order, confidence);
+        my_clamp_normalize(probs);
+
+        tweedie_update(twd, byte);
+        match_update(&match, byte);
+        word_update(&word, byte);
+        highctx_update(&hctx, byte);
+        ppm_update(&ppm, byte);
+
+        if ((i + 1) % 50000 == 0)
+            fprintf(stderr, "\r  %5.1f%%", (i + 1) * 100.0 / len);
+    }
+    fprintf(stderr, "\r                    \r");
+
+    double gammas[] = {0.500, 0.200, 0.059, 0.015};
+    int    c_repr[] = {128, 512, 2048, 8192};
+
+    printf("File: %s (%zu bytes)\n\n", argv[1], len);
+    printf("%-8s %-8s", "gamma", "C_repr");
+    for (int s = 0; s < TWD_STEPS; s++)
+        printf("  step_%d  ", s);
+    printf("\n");
+
+    for (int b = 0; b < N_CONF_REPORT; b++) {
+        printf("%-8.3f %-8d", gammas[b], c_repr[b]);
+        for (int s = 0; s < TWD_STEPS; s++) {
+            if (delta_count[s][b] > 0)
+                printf("  %.4f  ", delta_sum[s][b] / delta_count[s][b]);
+            else
+                printf("  ---     ");
+        }
+        printf("\n");
+    }
+
+    free(twd);
+    free(data);
+    return 0;
+}

ppm.h

@@ -9,6 +9,7 @@
 #define PPM_MAX_ORDER 4
 #define PPM_NSYM 256
 #define PPM_PRIOR 0.5
+#define PPM_MAX_CAPACITY (1 << 19)  /* 524288 entries per table; ~3.2 GB total for 5 tables */
 
 /*
  * Hash table entry: maps a 64-bit context hash to a count array.
@@ -77,9 +78,10 @@ static inline PPMEntry *ppm_table_find(PPMTable *t, uint64_t key) {
 }
 
 static inline PPMEntry *ppm_table_insert(PPMTable *t, uint64_t key) {
-    /* Grow if > 60% full */
+    /* Grow if > 60% full, but respect capacity cap */
     if (t->used * 5 > t->capacity * 3) {
-        ppm_table_grow(t);
+        if (t->capacity < PPM_MAX_CAPACITY)
+            ppm_table_grow(t);
     }
     uint32_t mask = t->capacity - 1;
     uint32_t idx = (uint32_t)(key & mask);
@@ -87,6 +89,10 @@ static inline PPMEntry *ppm_table_insert(PPMTable *t, uint64_t key) {
         PPMEntry *e = &t->entries[idx];
         if (e->key == key) return e;  /* already exists */
         if (e->key == 0) {
+            /* At capacity and table is full: don't insert new entry */
+            if (t->capacity >= PPM_MAX_CAPACITY &&
+                t->used * 5 > t->capacity * 3)
+                return NULL;
             /* init new entry with prior */
             e->key = key;
             for (int i = 0; i < PPM_NSYM; i++)
@@ -135,39 +141,96 @@ static inline void ppm_free(PPMModel *m) {
 }
 
 /*
- * predict_with_confidence: fills probs[256] and returns confidence + order.
- * Matches Python: fallback from max_order down to 0, first context with total > 1.
- * If nothing found, returns uniform.
+ * Predict with PPMC-style exclusion.
+ *
+ * From highest order down, symbols observed at each order receive probability
+ * proportional to their real observation count.  Symbols already assigned
+ * probability at a higher order are *excluded* from lower-order distributions.
+ * Escape probability (Method C):  P_esc = d / (n + d)
+ *   where d = distinct observed symbols, n = total real observations.
  */
 static inline void ppm_predict(PPMModel *m, double *probs,
                                 double *out_confidence, int *out_order) {
+    int excluded[PPM_NSYM];
+    memset(excluded, 0, sizeof(excluded));
+    for (int i = 0; i < PPM_NSYM; i++) probs[i] = 0.0;
+
+    double remaining = 1.0;
+    int best_order = -1;
+    double best_conf = 0.0;
+
     for (int order = PPM_MAX_ORDER; order >= 0; order--) {
-        const uint8_t *ctx_start;
         int ctx_len = order;
-
         if (ctx_len > m->hist_len) continue;
-        ctx_start = m->history + m->hist_len - ctx_len;
 
+        const uint8_t *ctx_start = m->history + m->hist_len - ctx_len;
         uint64_t key = ppm_hash_context(ctx_start, ctx_len);
         PPMEntry *e = ppm_table_find(&m->tables[order], key);
         if (e == NULL) continue;
-        if (e->total <= 1.0) continue;
 
-        double inv_total = 1.0 / e->total;
-        for (int i = 0; i < PPM_NSYM; i++)
-            probs[i] = e->counts[i] * inv_total;
+        /* Count real observations for non-excluded symbols */
+        double n = 0.0;
+        int d = 0;
+        for (int s = 0; s < PPM_NSYM; s++) {
+            if (excluded[s]) continue;
+            double real = e->counts[s] - PPM_PRIOR;
+            if (real > 0.01) {
+                n += real;
+                d++;
+            }
+        }
+
+        if (d == 0) continue;
+
+        if (best_order < 0) {
+            best_order = order;
+            best_conf = e->total;
+        }
+
+        /* PPMC escape: d / (n + d) */
+        double p_esc  = (double)d / (n + d);
+        double p_nesc = 1.0 - p_esc;
+
+        for (int s = 0; s < PPM_NSYM; s++) {
+            if (excluded[s]) continue;
+            double real = e->counts[s] - PPM_PRIOR;
+            if (real > 0.01) {
+                probs[s] += remaining * p_nesc * (real / n);
+                excluded[s] = 1;
+            }
+        }
+
+        remaining *= p_esc;
+    }
+
+    /* Distribute remaining mass uniformly among non-excluded symbols */
+    int n_rem = 0;
+    for (int s = 0; s < PPM_NSYM; s++)
+        if (!excluded[s]) n_rem++;
 
-        *out_confidence = e->total;
-        *out_order = order;
-        return;
+    if (n_rem > 0) {
+        double per = remaining / n_rem;
+        for (int s = 0; s < PPM_NSYM; s++)
+            if (!excluded[s])
+                probs[s] += per;
+    } else {
+        double per = remaining / PPM_NSYM;
+        for (int s = 0; s < PPM_NSYM; s++)
+            probs[s] += per;
     }
 
-    /* uniform fallback */
-    double u = 1.0 / 256.0;
+    /* Ensure positive + normalize */
+    double sum = 0.0;
+    for (int i = 0; i < PPM_NSYM; i++) {
+        if (probs[i] < 1e-10) probs[i] = 1e-10;
+        sum += probs[i];
+    }
+    double inv = 1.0 / sum;
     for (int i = 0; i < PPM_NSYM; i++)
-        probs[i] = u;
-    *out_confidence = 0.0;
-    *out_order = -1;
+        probs[i] *= inv;
+
+    *out_confidence = best_conf;
+    *out_order = best_order;
 }
 
 /*
@@ -183,6 +246,7 @@ static inline void ppm_update(PPMModel *m, uint8_t symbol) {
         uint64_t key = ppm_hash_context(ctx_start, ctx_len);
 
         PPMEntry *e = ppm_table_insert(&m->tables[order], key);
+        if (e == NULL) continue;  /* table full, skip this context */
         e->counts[symbol] += 1.0;
         e->total += 1.0;
     }

ppm_excl.h

@@ -0,0 +1,198 @@
+#ifndef PPM_H
+#define PPM_H
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+
+#define PPM_MAX_ORDER 4
+#define PPM_NSYM 256
+#define PPM_PRIOR 0.5
+
+/*
+ * Hash table entry: maps a 64-bit context hash to a count array.
+ * counts[i] stores the (float) count for symbol i.
+ * total caches sum(counts).
+ * key == 0 means empty slot.
+ */
+typedef struct {
+    uint64_t key;          /* context hash (0 = empty) */
+    double counts[PPM_NSYM];
+    double total;
+} PPMEntry;
+
+typedef struct {
+    PPMEntry *entries;
+    uint32_t capacity;     /* power of 2 */
+    uint32_t used;
+} PPMTable;
+
+typedef struct {
+    PPMTable tables[PPM_MAX_ORDER + 1];  /* order 0..4 */
+    uint8_t *history;
+    int hist_len;
+    int hist_cap;
+} PPMModel;
+
+/* ── Hash helper ── */
+
+static inline uint64_t ppm_hash_context(const uint8_t *ctx, int len) {
+    /* We need a non-zero hash for all contexts including order-0 (empty).
+     * Use FNV-1a style. Order-0 empty context gets a fixed hash. */
+    if (len == 0) return 1;  /* special: order-0 empty context */
+    uint64_t h = 14695981039346656037ULL;
+    for (int i = 0; i < len; i++) {
+        h ^= ctx[i];
+        h *= 1099511628211ULL;
+    }
+    if (h == 0) h = 1;  /* reserve 0 for empty slot */
+    return h;
+}
+
+/* ── Table operations ── */
+
+static inline void ppm_table_init(PPMTable *t, uint32_t capacity) {
+    t->capacity = capacity;
+    t->used = 0;
+    t->entries = (PPMEntry *)calloc(capacity, sizeof(PPMEntry));
+}
+
+static inline void ppm_table_free(PPMTable *t) {
+    free(t->entries);
+    t->entries = NULL;
+}
+
+static inline void ppm_table_grow(PPMTable *t);
+
+static inline PPMEntry *ppm_table_find(PPMTable *t, uint64_t key) {
+    uint32_t mask = t->capacity - 1;
+    uint32_t idx = (uint32_t)(key & mask);
+    for (;;) {
+        PPMEntry *e = &t->entries[idx];
+        if (e->key == key) return e;
+        if (e->key == 0) return NULL;
+        idx = (idx + 1) & mask;
+    }
+}
+
+static inline PPMEntry *ppm_table_insert(PPMTable *t, uint64_t key) {
+    /* Grow if > 60% full */
+    if (t->used * 5 > t->capacity * 3) {
+        ppm_table_grow(t);
+    }
+    uint32_t mask = t->capacity - 1;
+    uint32_t idx = (uint32_t)(key & mask);
+    for (;;) {
+        PPMEntry *e = &t->entries[idx];
+        if (e->key == key) return e;  /* already exists */
+        if (e->key == 0) {
+            /* init new entry with prior */
+            e->key = key;
+            for (int i = 0; i < PPM_NSYM; i++)
+                e->counts[i] = PPM_PRIOR;
+            e->total = PPM_NSYM * PPM_PRIOR;
+            t->used++;
+            return e;
+        }
+        idx = (idx + 1) & mask;
+    }
+}
+
+static inline void ppm_table_grow(PPMTable *t) {
+    uint32_t old_cap = t->capacity;
+    PPMEntry *old = t->entries;
+    uint32_t new_cap = old_cap * 2;
+    t->entries = (PPMEntry *)calloc(new_cap, sizeof(PPMEntry));
+    t->capacity = new_cap;
+    t->used = 0;
+    for (uint32_t i = 0; i < old_cap; i++) {
+        if (old[i].key != 0) {
+            /* re-insert */
+            PPMEntry *ne = ppm_table_insert(t, old[i].key);
+            memcpy(ne->counts, old[i].counts, sizeof(old[i].counts));
+            ne->total = old[i].total;
+        }
+    }
+    free(old);
+}
+
+/* ── PPM Model ── */
+
+static inline void ppm_init(PPMModel *m) {
+    for (int o = 0; o <= PPM_MAX_ORDER; o++)
+        ppm_table_init(&m->tables[o], 1024);
+    m->hist_cap = 4096;
+    m->hist_len = 0;
+    m->history = (uint8_t *)malloc(m->hist_cap);
+}
+
+static inline void ppm_free(PPMModel *m) {
+    for (int o = 0; o <= PPM_MAX_ORDER; o++)
+        ppm_table_free(&m->tables[o]);
+    free(m->history);
+    m->history = NULL;
+}
+
+/*
+ * predict_with_confidence: fills probs[256] and returns confidence + order.
+ * Matches Python: fallback from max_order down to 0, first context with total > 1.
+ * If nothing found, returns uniform.
+ */
+static inline void ppm_predict(PPMModel *m, double *probs,
+                                double *out_confidence, int *out_order) {
+    for (int order = PPM_MAX_ORDER; order >= 0; order--) {
+        const uint8_t *ctx_start;
+        int ctx_len = order;
+
+        if (ctx_len > m->hist_len) continue;
+        ctx_start = m->history + m->hist_len - ctx_len;
+
+        uint64_t key = ppm_hash_context(ctx_start, ctx_len);
+        PPMEntry *e = ppm_table_find(&m->tables[order], key);
+        if (e == NULL) continue;
+        if (e->total <= 1.0) continue;
+
+        double inv_total = 1.0 / e->total;
+        for (int i = 0; i < PPM_NSYM; i++)
+            probs[i] = e->counts[i] * inv_total;
+
+        *out_confidence = e->total;
+        *out_order = order;
+        return;
+    }
+
+    /* uniform fallback */
+    double u = 1.0 / 256.0;
+    for (int i = 0; i < PPM_NSYM; i++)
+        probs[i] = u;
+    *out_confidence = 0.0;
+    *out_order = -1;
+}
+
+/*
+ * update: add symbol count to all orders (0..4) where context is available.
+ * Then append symbol to history.
+ */
+static inline void ppm_update(PPMModel *m, uint8_t symbol) {
+    for (int order = 0; order <= PPM_MAX_ORDER; order++) {
+        int ctx_len = order;
+        if (ctx_len > m->hist_len) continue;
+
+        const uint8_t *ctx_start = m->history + m->hist_len - ctx_len;
+        uint64_t key = ppm_hash_context(ctx_start, ctx_len);
+
+        PPMEntry *e = ppm_table_insert(&m->tables[order], key);
+        e->counts[symbol] += 1.0;
+        e->total += 1.0;
+    }
+
+    /* append to history */
+    if (m->hist_len >= m->hist_cap) {
+        m->hist_cap *= 2;
+        m->history = (uint8_t *)realloc(m->history, m->hist_cap);
+    }
+    m->history[m->hist_len++] = symbol;
+}
+
+#endif /* PPM_H */

tweedie.h

@@ -16,6 +16,10 @@
  * the additive correction δ = E[θ|p̂] - E[p̂] = hit_rate - avg_pred.
  * This δ equals σ²·s(p̂) — the full Tweedie correction term.
  *
+ * Variance-aware James-Stein shrinkage: each correction δ is shrunk toward
+ * zero based on SNR = δ²·N/var(error). When SNR < 4, the correction is
+ * attenuated proportionally, preventing noisy bucket estimates from hurting.
+ *
  * Binary tree decomposition: 256-way → 8 binary decisions (MSB to LSB).
  * Multi-step: K=3 denoising steps with independent score tables.
  * Calibration context: (step, bit_context, order, shape, confidence, prob_bin)
@@ -56,6 +60,7 @@ typedef struct {
     double sum_pred;   /* sum of predicted P(right) */
     double hits;       /* times true symbol went right */
     double total;      /* total observations */
+    double sum_sq_err; /* sum of (went_right - p_right)^2 */
 } TwdCalibEntry;
 
 typedef struct {
@@ -138,6 +143,7 @@ static inline void tweedie_init(TweedieDenoiser *td) {
                             td->table[t][b][o][s][c][p].sum_pred = center * TWD_PRIOR_WEIGHT;
                             td->table[t][b][o][s][c][p].hits     = center * TWD_PRIOR_WEIGHT;
                             td->table[t][b][o][s][c][p].total    = TWD_PRIOR_WEIGHT;
+                            td->table[t][b][o][s][c][p].sum_sq_err = TWD_PRIOR_WEIGHT * 0.25;
                         }
 }
 
@@ -214,6 +220,17 @@ static inline void tweedie_denoise(TweedieDenoiser *td, double *probs,
                 double emp_rate = e->hits / e->total;
                 double delta = emp_rate - avg_pred;
 
+                /* Variance-aware James-Stein shrinkage:
+                 * SNR = δ²·N / var(error). Shrink δ → 0 when SNR < 4. */
+                double var_err = e->sum_sq_err / e->total;
+                if (e->total > 10.0 && var_err > 1e-10) {
+                    double snr = delta * delta * e->total / var_err;
+                    double shrink = (snr > 4.0) ? 1.0 : snr / 4.0;
+                    delta *= shrink;
+                } else {
+                    delta = 0.0;
+                }
+
                 double p_right_corr = p_right + delta;
                 if (p_right_corr < 1e-8)       p_right_corr = 1e-8;
                 if (p_right_corr > 1.0 - 1e-8) p_right_corr = 1.0 - 1e-8;
@@ -269,6 +286,8 @@ static inline void tweedie_update(TweedieDenoiser *td, uint8_t true_symbol) {
             int bctx = td->cached_bctx[step][node_id];
 
             TwdCalibEntry *e = &td->table[step][bctx][og][sb][cb][pbin];
+            double err = (double)went_right - td->cached_p_right[step][node_id];
+            e->sum_sq_err += err * err;
             e->sum_pred += td->cached_p_right[step][node_id];
             e->total    += 1.0;
             if (went_right)