Update managed compression API

app.py

@@ -7,10 +7,235 @@ from typing import Any
 from fastapi import FastAPI, HTTPException
 
 CLASSIFIER_MODEL = "microsoft/deberta-v3-small"
+RULES_VERSION = "hf-space-rules-v0.1.0"
+LOW_SIGNAL_PATTERNS = [
+    re.compile(pattern, re.IGNORECASE)
+    for pattern in [
+        r"\blet me know if you (need|want|have)\b",
+        r"\bi hope this helps\b",
+        r"\bas an ai language model\b",
+        r"\bof course[,!]?\s*$",
+        r"\bcertainly[,!]?\s*$",
+        r"\bsure[,!]?\s*$",
+    ]
+]
 
 app = FastAPI(title="Touchdown Compression Classifier", version="0.1.0")
 
 
+def _find_spans(text: str, needle: str) -> list[tuple[int, int]]:
+    spans = []
+    cursor = 0
+    while needle:
+        index = text.find(needle, cursor)
+        if index < 0:
+            return spans
+        spans.append((index, index + len(needle)))
+        cursor = index + len(needle)
+    return spans
+
+
+def _regex_spans(text: str, pattern: str, flags: int) -> list[tuple[int, int]]:
+    return [(match.start(), match.end()) for match in re.finditer(pattern, text, flags)]
+
+
+def _json_line_spans(text: str) -> list[tuple[int, int]]:
+    spans = []
+    cursor = 0
+    for line in text.splitlines(keepends=True):
+        stripped = line.strip()
+        if stripped and stripped[0] in "[{":
+            try:
+                import json
+
+                json.loads(stripped)
+            except Exception:
+                pass
+            else:
+                offset = line.index(stripped[0])
+                spans.append((cursor + offset, cursor + offset + len(stripped)))
+        cursor += len(line)
+    return spans
+
+
+def _merge(spans: list[tuple[int, int]]) -> list[tuple[int, int]]:
+    if not spans:
+        return []
+    ordered = sorted(spans)
+    merged = [ordered[0]]
+    for start, end in ordered[1:]:
+        prev_start, prev_end = merged[-1]
+        if start <= prev_end:
+            merged[-1] = (prev_start, max(prev_end, end))
+        else:
+            merged.append((start, end))
+    return merged
+
+
+def _overlaps(spans: list[tuple[int, int]], start: int, end: int) -> bool:
+    return any(start < span_end and end > span_start for span_start, span_end in spans)
+
+
+def _is_subsequence(candidate: str, original: str) -> bool:
+    cursor = 0
+    for char in candidate:
+        cursor = original.find(char, cursor)
+        if cursor < 0:
+            return False
+        cursor += 1
+    return True
+
+
+def _protected_spans(
+    text: str,
+    protected_values: list[str],
+    *,
+    preserve_code: bool,
+    preserve_json: bool,
+) -> tuple[
+    list[tuple[int, int]],
+    list[tuple[int, int]],
+    list[tuple[int, int]],
+    list[tuple[int, int]],
+]:
+    spans = []
+    for value in protected_values:
+        spans.extend(_find_spans(text, value))
+    code_spans = (
+        _regex_spans(text, r"```.*?```", re.MULTILINE | re.DOTALL)
+        if preserve_code
+        else []
+    )
+    json_spans = (
+        _regex_spans(text, r"```(?:json|JSON)\s*\n.*?```", re.MULTILINE | re.DOTALL)
+        + _json_line_spans(text)
+        if preserve_json
+        else []
+    )
+    system_spans = (
+        _regex_spans(text, r"<system\b[^>]*>.*?</system>", re.MULTILINE | re.DOTALL)
+        + _regex_spans(text, r"^system\s*:\s*.*$", re.MULTILINE | re.IGNORECASE)
+    )
+    spans.extend(code_spans)
+    spans.extend(json_spans)
+    spans.extend(system_spans)
+    return _merge(spans), code_spans, json_spans, system_spans
+
+
+def _compress_text(payload: dict[str, Any]) -> dict[str, Any]:
+    text = payload.get("input")
+    if not isinstance(text, str):
+        raise HTTPException(status_code=400, detail="input must be a string")
+    settings = payload.get("compression_settings") or {}
+    if not isinstance(settings, dict):
+        raise HTTPException(status_code=400, detail="compression_settings must be an object")
+    protected_values = payload.get("protected_spans") or []
+    if not isinstance(protected_values, list) or not all(
+        isinstance(value, str) for value in protected_values
+    ):
+        raise HTTPException(status_code=400, detail="protected_spans must be strings")
+    try:
+        aggressiveness = float(settings.get("aggressiveness", 0.35))
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail="aggressiveness must be numeric") from exc
+    if not 0.0 <= aggressiveness <= 1.0:
+        raise HTTPException(status_code=400, detail="aggressiveness must be 0..1")
+
+    started = time.perf_counter()
+    preserve_code = bool(settings.get("preserve_code", True))
+    preserve_json = bool(settings.get("preserve_json", True))
+    protected, code_spans, json_spans, system_spans = _protected_spans(
+        text,
+        protected_values,
+        preserve_code=preserve_code,
+        preserve_json=preserve_json,
+    )
+    seen: set[str] = set()
+    drops: list[tuple[int, int, str, str]] = []
+    cursor = 0
+    previous_blank_kept = False
+    for line in text.splitlines(keepends=True):
+        start = cursor
+        end = cursor + len(line)
+        cursor = end
+        stripped = line.strip()
+        normalized = re.sub(r"\s+", " ", stripped).lower()
+        if _overlaps(protected, start, end):
+            if normalized:
+                seen.add(normalized)
+            previous_blank_kept = False
+            continue
+        reason = None
+        if not stripped and previous_blank_kept:
+            reason = "extra_blank"
+        elif len(normalized) > 30 and normalized in seen and aggressiveness >= 0.10:
+            reason = "duplicate_line"
+        elif aggressiveness >= 0.20 and any(
+            pattern.search(stripped) for pattern in LOW_SIGNAL_PATTERNS
+        ):
+            reason = "low_signal_phrase"
+        if reason:
+            drops.append((start, end, reason, stripped[:120]))
+            continue
+        previous_blank_kept = not stripped
+        if normalized:
+            seen.add(normalized)
+
+    chunks = []
+    drop_ranges = _merge([(start, end) for start, end, _, _ in drops])
+    cursor = 0
+    for start, end in drop_ranges:
+        chunks.append(text[cursor:start])
+        cursor = end
+    chunks.append(text[cursor:])
+    output = "".join(chunks)
+    before = max(1, round(len(text) / 4.0))
+    after = max(1, round(len(output) / 4.0))
+    saved = max(0, before - after)
+    missing = [value for value in protected_values if value and value not in output]
+    code_preserved = all(text[start:end] in output for start, end in code_spans)
+    json_preserved = all(text[start:end] in output for start, end in json_spans)
+    system_preserved = all(text[start:end] in output for start, end in system_spans)
+    decision = "reject" if (
+        missing or not code_preserved or not json_preserved or not system_preserved
+    ) else (
+        "high_confidence" if saved > 0 and aggressiveness <= 0.65 else "no_op"
+    )
+    return {
+        "output": output,
+        "original_input_tokens": before,
+        "output_tokens": after,
+        "tokens_saved": saved,
+        "compression_percentage": round(100.0 * saved / before, 1),
+        "receipt": {
+            "protected_spans_checked": len(protected_values),
+            "protected_spans_missing": len(missing),
+            "code_blocks_detected": len(code_spans),
+            "code_blocks_preserved": code_preserved,
+            "json_blocks_detected": len(json_spans),
+            "json_blocks_preserved": json_preserved,
+            "system_prompt_spans_detected": len(system_spans),
+            "system_prompts_preserved": system_preserved,
+            "decision": decision,
+            "compressor_latency_ms": round((time.perf_counter() - started) * 1000.0, 3),
+            "deletion_only": _is_subsequence(output, text),
+            "deterministic": True,
+            "rules_version": RULES_VERSION,
+            "classifier": {
+                "model": CLASSIFIER_MODEL,
+                "status": "tokenizer_loaded_no_trained_keep_drop_head",
+                "labels_received": 0,
+                "drop_spans_applied": 0,
+            },
+            "dropped_segments_count": len(drops),
+            "dropped_segments": [
+                {"reason": reason, "preview": preview}
+                for _, _, reason, preview in drops[:20]
+            ],
+        },
+    }
+
+
 def _tokens(text: str) -> list[dict[str, Any]]:
     started = time.perf_counter()
     try:
@@ -76,26 +301,4 @@ def classify(payload: dict[str, Any]) -> dict[str, Any]:
 
 @app.post("/v1/compress")
 def compress(payload: dict[str, Any]) -> dict[str, Any]:
-    # The Space can be called immediately, but managed compression logic lives
-    # in the package service. This endpoint is deliberately conservative until
-    # the package is vendored into the Space image.
-    text = payload.get("input")
-    if not isinstance(text, str):
-        raise HTTPException(status_code=400, detail="input must be a string")
-    started = time.perf_counter()
-    return {
-        "output": text,
-        "original_input_tokens": max(1, round(len(text) / 4.0)),
-        "output_tokens": max(1, round(len(text) / 4.0)),
-        "tokens_saved": 0,
-        "compression_percentage": 0.0,
-        "receipt": {
-            "protected_spans_checked": len(payload.get("protected_spans") or []),
-            "protected_spans_missing": 0,
-            "code_blocks_preserved": True,
-            "decision": "no_op_classifier_scaffold",
-            "compressor_latency_ms": round((time.perf_counter() - started) * 1000.0, 3),
-            "classifier_model": CLASSIFIER_MODEL,
-            "classifier_status": "tokenizer_loaded_no_trained_keep_drop_head",
-        },
-    }
+    return _compress_text(payload)