v85: no official harness exists; pathkeep retention NULL (retracts v84 +0.0009); 89.2% of restored basenames already present; 3 compressors registered; 373 tests

answer_retention_eval.py

@@ -39,6 +39,9 @@ COMPRESSORS = {
     "structural_reasoning": "structural_reasoning_compressor.py",
     "adaptive_structural": "adaptive_structural_compressor.py",
     "structural_lowcomp": "structural_lowcomp_compressor.py",
+    "structural_selective": "structural_selective_compressor.py",
+    "structural_selective_call": "structural_selective_call_compressor.py",
+    "structural_pathkeep": "structural_pathkeep_compressor.py",
 }
 STOP = {
     "that","this","with","from","were","have","been","which","when","what","the",

basename_novelty.json

@@ -0,0 +1,6 @@
+{
+  "restored_basenames": 1454,
+  "already_elsewhere_in_shipped_output": 1297,
+  "genuinely_new_tokens": 157,
+  "pct_already_present": 89.2
+}

basename_novelty.py

@@ -0,0 +1,36 @@
+import json, csv, importlib.util, re
+from pathlib import Path
+R=Path("/var/lib/octave/sn114/repo"); PL=Path("/var/lib/octave/sn114/external/SOMA-plugin")
+csv.field_size_limit(50_000_000)
+rows=list(csv.DictReader(open(R/"miner/plain_text_compression/sample_tasks/CoT-Compression-1/challenges.csv")))
+G1=chr(0xab); G2=chr(0xbb); BS=chr(92)
+CALL=re.compile(G1+'c'+BS+'d+[a-z]{2}(.*?)'+G2, re.DOTALL)
+PATHTOK=re.compile(r'/[\w./-]+')
+def load(fn):
+    s=importlib.util.spec_from_file_location("n_"+fn[:8], PL/fn)
+    m=importlib.util.module_from_spec(s); s.loader.exec_module(m); return m
+A=load('structural_cot_compressor.py'); B=load('structural_pathkeep_compressor.py')
+A.MAX_TOOL_CALL=60; B.MAX_TOOL_CALL=60
+restored=0; already_elsewhere=0; genuinely_new=0
+for r in rows:
+    raw=r['challenge_text']
+    ca=A.compress_content(raw); cb=B.compress_content(raw)
+    ba=CALL.findall(ca); bb=CALL.findall(cb)
+    if len(ba)!=len(bb): continue
+    for x,y in zip(ba,bb):
+        if x==y: continue
+        py=PATHTOK.findall(y)
+        if not py: continue
+        base=max(py,key=len).rsplit('/',1)[-1]
+        if not base or base in x: continue
+        restored+=1
+        # is that basename present ANYWHERE else in the shipped compressed output?
+        if base in ca: already_elsewhere+=1
+        else: genuinely_new+=1
+out={'restored_basenames':restored,'already_elsewhere_in_shipped_output':already_elsewhere,
+     'genuinely_new_tokens':genuinely_new,
+     'pct_already_present':round(100*already_elsewhere/restored,2) if restored else None}
+json.dump(out, open(R/"basename_novelty.json","w"), indent=2)
+print('  restored basenames:',restored)
+print('  already elsewhere in shipped output:',already_elsewhere,'(%s%%)'%out['pct_already_present'])
+print('  genuinely new to the document:',genuinely_new)

official_retention_v84.json

@@ -0,0 +1,131 @@
+{
+  "metric": "lexical answer-retention proxy (NOT the official LLM-judged score)",
+  "results": {
+    "performance": {
+      "compression_percent": 97.738,
+      "mean_answer_retention": 0.6023,
+      "micro_term_retention": 0.594,
+      "worst_question_retention": 0.0909,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.5887
+    },
+    "aggressive": {
+      "compression_percent": 99.388,
+      "mean_answer_retention": 0.1983,
+      "micro_term_retention": 0.1888,
+      "worst_question_retention": 0.0,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.1971
+    },
+    "code_focused": {
+      "compression_percent": 98.24,
+      "mean_answer_retention": 0.309,
+      "micro_term_retention": 0.2908,
+      "worst_question_retention": 0.0,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.3036
+    },
+    "hybrid": {
+      "compression_percent": 49.016,
+      "mean_answer_retention": 0.9619,
+      "micro_term_retention": 0.9627,
+      "worst_question_retention": 0.6154,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.4715
+    },
+    "adaptive": {
+      "compression_percent": 41.808,
+      "mean_answer_retention": 0.9619,
+      "micro_term_retention": 0.9627,
+      "worst_question_retention": 0.6154,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.4022
+    },
+    "thinking_strip": {
+      "compression_percent": 12.909,
+      "mean_answer_retention": 0.9982,
+      "micro_term_retention": 0.9983,
+      "worst_question_retention": 0.9091,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.1289
+    },
+    "dialogue": {
+      "compression_percent": 0.0,
+      "mean_answer_retention": 1.0,
+      "micro_term_retention": 1.0,
+      "worst_question_retention": 1.0,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.0
+    },
+    "structural": {
+      "compression_percent": 78.833,
+      "mean_answer_retention": 0.9628,
+      "micro_term_retention": 0.9621,
+      "worst_question_retention": 0.6364,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.759
+    },
+    "structural_reasoning": {
+      "compression_percent": 77.438,
+      "mean_answer_retention": 0.9643,
+      "micro_term_retention": 0.9633,
+      "worst_question_retention": 0.6364,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.7467
+    },
+    "adaptive_structural": {
+      "compression_percent": 86.49,
+      "mean_answer_retention": 0.9091,
+      "micro_term_retention": 0.9094,
+      "worst_question_retention": 0.5455,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.7863
+    },
+    "structural_lowcomp": {
+      "compression_percent": 64.129,
+      "mean_answer_retention": 0.9822,
+      "micro_term_retention": 0.9815,
+      "worst_question_retention": 0.7273,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.6299
+    },
+    "structural_selective": {
+      "compression_percent": 79.255,
+      "mean_answer_retention": 0.9628,
+      "micro_term_retention": 0.9621,
+      "worst_question_retention": 0.6364,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.763
+    },
+    "structural_selective_call": {
+      "compression_percent": 80.102,
+      "mean_answer_retention": 0.9587,
+      "micro_term_retention": 0.9586,
+      "worst_question_retention": 0.6364,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.7679
+    },
+    "structural_pathkeep": {
+      "compression_percent": 78.787,
+      "mean_answer_retention": 0.9628,
+      "micro_term_retention": 0.9621,
+      "worst_question_retention": 0.6364,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.7585
+    }
+  }
+}

pathkeep_official.json

@@ -0,0 +1,83 @@
+{
+  "harness_reality": {
+    "what_i_expected": "an official harness that produced the published 73.11%/0.9360 on 150 challenges",
+    "what_exists": "answer_retention_eval.py, which self-labels as a \"lexical answer-retention proxy (NOT the official LLM-judged score)\", has no --limit flag and defaults to a 5-task/49-question subset",
+    "conclusion": "there is no official harness in this repo; every retention number I have is a lexical proxy of some kind, and the announced comparison could not be run as announced"
+  },
+  "registry_edit": "answer_retention_eval.py COMPRESSORS gained structural_selective, structural_selective_call and structural_pathkeep; purely additive, no existing entry modified",
+  "official_proxy_results": {
+    "structural": {
+      "compression_percent": 78.833,
+      "mean_answer_retention": 0.9628,
+      "micro_term_retention": 0.9621,
+      "worst_question_retention": 0.6364,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.759
+    },
+    "structural_reasoning": {
+      "compression_percent": 77.438,
+      "mean_answer_retention": 0.9643,
+      "micro_term_retention": 0.9633,
+      "worst_question_retention": 0.6364,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.7467
+    },
+    "structural_lowcomp": {
+      "compression_percent": 64.129,
+      "mean_answer_retention": 0.9822,
+      "micro_term_retention": 0.9815,
+      "worst_question_retention": 0.7273,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.6299
+    },
+    "structural_selective": {
+      "compression_percent": 79.255,
+      "mean_answer_retention": 0.9628,
+      "micro_term_retention": 0.9621,
+      "worst_question_retention": 0.6364,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.763
+    },
+    "structural_selective_call": {
+      "compression_percent": 80.102,
+      "mean_answer_retention": 0.9587,
+      "micro_term_retention": 0.9586,
+      "worst_question_retention": 0.6364,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.7679
+    },
+    "structural_pathkeep": {
+      "compression_percent": 78.787,
+      "mean_answer_retention": 0.9628,
+      "micro_term_retention": 0.9621,
+      "worst_question_retention": 0.6364,
+      "tasks": 5,
+      "questions_scored": 49,
+      "retention_x_compression": 0.7585
+    }
+  },
+  "null_result": {
+    "finding": "pathkeep retention is IDENTICAL to shipped at 0.9628 mean / 0.9621 micro / 0.6364 worst; compression 78.787% vs 78.833%",
+    "retraction_from_v84": "I published \"term retention +0.0009, small but the correct sign\". The better-specified proxy says +0.0000. My ad-hoc metric produced a signal indistinguishable from its own noise and I reported its direction as meaningful",
+    "error_class": "eighth narration-class error, and the first where I over-read a delta rather than a range"
+  },
+  "explanation_tested": {
+    "hypothesis": "restored basenames are usually already elsewhere in the document, so a bag-of-terms proxy cannot see the change - stated before measuring",
+    "restored_basenames": 1454,
+    "already_elsewhere": 1297,
+    "pct_already_present": 89.2,
+    "genuinely_new": 157,
+    "verdict": "CONFIRMED - 89.2% already present, only 157 of 1454 restorations add a token the document lacked"
+  },
+  "decision": {
+    "pathkeep": "stays UNPROMOTED",
+    "sharper_reason": "not \"risk unquantified\" but \"benefit unobservable with available instruments\" - the mechanism claim is about LOCAL availability inside the span the agent is reading, and every proxy I own is position-blind",
+    "what_it_names": "this is a more useful blocker because it says exactly what an OpenRouter key would buy: a metric that can see position"
+  },
+  "revert_proofs": "14 checks incl. 2 live source mutations (unregister pathkeep, harness claiming to be official) and 12 report mutations covering the retraction, the tested explanation, post-hoc restatement, silent promotion and baseline drift"
+}

test_compressors.py

@@ -7504,5 +7504,105 @@ class TestPathKeepCompressor(unittest.TestCase):
         self.assertIn("NOT comparable", self.rep["metric_caveat"])
         self.assertIn("73.11", self.rep["metric_caveat"])
 
+class TestPathKeepAgainstOfficialProxy(unittest.TestCase):
+    """I announced I would run pathkeep through "the official harness path that produced
+    73.11%/0.9360". Two things came out of trying, and both are corrections.
+
+    THERE IS NO OFFICIAL HARNESS HERE. answer_retention_eval.py self-labels as a "lexical
+    answer-retention proxy (NOT the official LLM-judged score)", has no --limit flag, and
+    defaults to a 5-task/49-question subset. So it is a better-specified proxy than my ad-hoc
+    regex, not an official measurement, and the comparison I announced could not be run as
+    announced. Every retention number I hold is a lexical proxy of some kind.
+
+    THE RESULT IS NULL. pathkeep retention is IDENTICAL to shipped - 0.9628 mean, 0.9621
+    micro, 0.6364 worst - with compression 78.787% vs 78.833%.
+
+    RETRACTION FROM v84: I published "term retention +0.0009, small but the correct sign".
+    The better-specified proxy says +0.0000. My ad-hoc metric produced a signal
+    indistinguishable from its own noise and I reported its direction as meaningful. Eighth
+    narration-class error, and the first where I over-read a DELTA rather than a range.
+
+    EXPLANATION STATED FIRST, THEN TESTED: restored basenames are usually already elsewhere
+    in the document, so a bag-of-terms proxy cannot see the change. Measured: of 1454
+    restored basenames, 1297 (89.2%) already appear in the shipped compressed output; only
+    157 add a token the document lacked. CONFIRMED, and it predicts the null exactly.
+
+    DECISION: pathkeep stays unpromoted, for a sharper reason than before - not "risk
+    unquantified" but "benefit unobservable with available instruments". The mechanism claim
+    is about LOCAL availability inside the span the agent is reading; every proxy I own is
+    position-blind. That names what an OpenRouter key would buy.
+    """
+
+    P = Path("/var/lib/octave/sn114/repo/pathkeep_official.json")
+    AR = Path("/var/lib/octave/sn114/repo/answer_retention_eval.py")
+
+    @classmethod
+    def setUpClass(cls):
+        if not cls.P.exists():
+            raise unittest.SkipTest("pathkeep_official.json missing")
+        cls.rep = json.loads(cls.P.read_text())
+        cls.res = cls.rep["official_proxy_results"]
+
+    def test_harness_self_labels_as_a_proxy(self):
+        src = self.AR.read_text()
+        self.assertIn(
+            "NOT the official LLM-judged score", src,
+            "answer_retention_eval.py no longer disclaims being official; if it became "
+            "the real harness the whole proxy caveat needs revisiting")
+        self.assertIn("no --limit", self.rep["harness_reality"]["what_exists"])
+
+    def test_new_compressors_are_registered_live(self):
+        src = self.AR.read_text()
+        for name in ("structural_selective", "structural_selective_call",
+                     "structural_pathkeep"):
+            with self.subTest(name=name):
+                self.assertIn(
+                    name, src,
+                    "%s dropped out of the COMPRESSORS registry, so it is silently "
+                    "excluded from every sweep" % name)
+
+    def test_pathkeep_retention_is_identical_not_better(self):
+        ship = self.res["structural"]
+        pk = self.res["structural_pathkeep"]
+        for key in ("mean_answer_retention", "micro_term_retention",
+                    "worst_question_retention"):
+            with self.subTest(key=key):
+                self.assertEqual(
+                    pk[key], ship[key],
+                    "pathkeep %s now differs from shipped; the recorded null result and "
+                    "the v84 retraction would both need re-deriving" % key)
+
+    def test_v84_retention_claim_is_retracted(self):
+        r = self.rep["null_result"]["retraction_from_v84"]
+        self.assertIn("+0.0009", r)
+        self.assertIn("indistinguishable from its own noise", r)
+        self.assertIn("over-read a delta", self.rep["null_result"]["error_class"])
+
+    def test_basename_novelty_explains_the_null(self):
+        e = self.rep["explanation_tested"]
+        self.assertEqual(e["verdict"][:9], "CONFIRMED")
+        self.assertGreater(
+            e["pct_already_present"], 80.0,
+            "if most restored basenames were NOT already in the document, a lexical "
+            "proxy should have seen the change and the null would need another cause")
+        self.assertEqual(e["restored_basenames"],
+                         e["already_elsewhere"] + e["genuinely_new"])
+
+    def test_explanation_was_stated_before_measuring(self):
+        self.assertIn("stated before measuring",
+                      self.rep["explanation_tested"]["hypothesis"])
+
+    def test_decision_names_the_instrument_gap(self):
+        d = self.rep["decision"]
+        self.assertEqual(d["pathkeep"], "stays UNPROMOTED")
+        self.assertIn("position-blind", d["sharper_reason"])
+        self.assertIn("unobservable", d["sharper_reason"])
+
+    def test_registry_edit_was_additive(self):
+        self.assertIn("purely additive", self.rep["registry_edit"])
+        ship = self.res["structural"]
+        self.assertAlmostEqual(ship["compression_percent"], 78.833, places=3)
+        self.assertAlmostEqual(ship["mean_answer_retention"], 0.9628, places=4)
+
 if __name__ == "__main__":
     unittest.main(verbosity=2)