Upload folder using huggingface_hub

README.md

@@ -26,14 +26,16 @@ Live Space:
 
 - `https://wchen22-touchdown-compression-classifier.hf.space`
 - Verified 2026-06-11 with HF CLI: runtime stage `RUNNING`, hardware
-  `cpu-basic`, domain `READY`.
+  `cpu-basic`, domain `READY`, repo/runtime SHA
+  `b402ba63bf08ce65bd30da071256555382be4fe0`.
 - The deployed scaffold supports chunked ONNX artifact inference for long
   prompts. Use `hf spaces info wchen22/touchdown-compression-classifier --format
   json` for the current repo/runtime SHA.
 - Live smoke:
-  `python3 scripts/smoke_compression_api.py --base-url https://wchen22-touchdown-compression-classifier.hf.space --include-classify --include-batch --include-gzip`
+  `python3 scripts/smoke_compression_api.py --base-url https://wchen22-touchdown-compression-classifier.hf.space --include-classify --include-batch --include-messages --include-gzip`
   validates `/health`, `/v1/classify`, single `/v1/compress`, and managed
-  `inputs[]` batch, plus gzipped JSON request/response transport.
+  `inputs[]` batch, managed `messages[]`, plus gzipped JSON request/response
+  transport.
 - Full deployment receipt:
   `python3 scripts/verify_compression_space.py --expected-sha <sha> --out reports/generated/compression_space/hf_space_verification.json`
   validates HF runtime metadata, repo/runtime SHA agreement, API smoke, and
@@ -41,12 +43,16 @@ Live Space:
 - Fresh local receipts are written under
   `reports/generated/compression_space/`; run the full verifier with the
   current Space SHA to check runtime, API smoke, and remote/local file parity.
+  Current live receipt:
+  `reports/generated/compression_space/hf_space_verification_2026-06-11-idempotency-replay-health.json`.
 - Latest live result: `/v1/compress` saved 27/102 estimated tokens;
   managed `inputs[]` returned `input_count=2`, `succeeded=2`, `failed=0`,
+  managed `messages[]` returned `message_count=2` with system-role protection,
   gzip transport returned `response_content_encoding=gzip`, and `/v1/classify`
   returned KEEP-only DeBERTa tokenizer labels. Receipts include
-  removed-span/char totals, classifier DROP block reasons, and tool-schema
-  preservation counts when `tools` or `tool_schemas` are supplied.
+  removed-span/char totals, classifier DROP block reasons, tool-schema
+  preservation counts when `tools` or `tool_schemas` are supplied, and
+  `/health` idempotency TTL reporting.
   Matching `Idempotency-Key` retries replay the first in-memory response;
   payload conflicts return HTTP 409. This is per-process memory on the Space,
   not a durable distributed store.

app.py

@@ -29,6 +29,7 @@ GZIP_ENCODING = "gzip"
 GZIP_MAGIC = b"\x1f\x8b"
 DEFAULT_IDEMPOTENCY_TTL_SECONDS = 24 * 60 * 60
 IDEMPOTENCY_TTL_ENV = "TOUCHDOWN_IDEMPOTENCY_TTL_SECONDS"
+DEFAULT_PROTECTED_MESSAGE_ROLES = ("system", "developer")
 LOW_SIGNAL_PATTERNS = [
     re.compile(pattern, re.IGNORECASE)
     for pattern in [
@@ -315,7 +316,7 @@ def _store_idempotency_body(
 def _handle_compress_with_idempotency(payload: dict[str, Any]) -> dict[str, Any]:
     key = _idempotency_key_from_payload(payload)
     if not key:
-        return _handle_batch(payload) if "inputs" in payload else _compress_text(payload)
+        return _dispatch_compress(payload)
     route = "/v1/compress"
     fingerprint = _idempotency_fingerprint(route, payload)
     cached = _cached_idempotency_body(
@@ -325,7 +326,7 @@ def _handle_compress_with_idempotency(payload: dict[str, Any]) -> dict[str, Any]
     )
     if cached is not None:
         return cached
-    body = _handle_batch(payload) if "inputs" in payload else _compress_text(payload)
+    body = _dispatch_compress(payload)
     request_id = payload.get("request_id") if isinstance(payload.get("request_id"), str) else None
     return _store_idempotency_body(
         route=route,
@@ -650,6 +651,66 @@ def _receipt_id(payload: dict[str, Any]) -> str:
     return "tdcr_" + hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:24]
 
 
+def _stable_json_sha256(value: Any) -> str:
+    encoded = json.dumps(
+        value,
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(",", ":"),
+    )
+    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+
+
+def _aggregate_receipt_id(payload: dict[str, Any]) -> str:
+    encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+    return "tdcm_" + hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:24]
+
+
+def _string_set(
+    value: Any,
+    *,
+    default: tuple[str, ...] = (),
+    field_name: str,
+) -> set[str]:
+    if value is None:
+        return set(default)
+    if (
+        isinstance(value, list)
+        and all(isinstance(item, str) and item for item in value)
+    ):
+        return {item.lower() for item in value}
+    raise HTTPException(status_code=400, detail=f"{field_name} must be a list of strings")
+
+
+def _message_role(message: dict[str, Any], index: int) -> str:
+    role = message.get("role")
+    if not isinstance(role, str) or not role:
+        raise HTTPException(
+            status_code=400,
+            detail=f"messages[{index}].role must be a string",
+        )
+    return role.lower()
+
+
+def _message_decision(
+    *,
+    tokens_saved: int,
+    receipts: list[dict[str, Any]],
+) -> str:
+    decisions = [
+        receipt.get("decision")
+        for receipt in receipts
+        if isinstance(receipt, dict)
+    ]
+    if any(decision == "reject" for decision in decisions):
+        return "reject"
+    if any(decision == "needs_review" for decision in decisions):
+        return "needs_review"
+    if tokens_saved <= 0:
+        return "no_op"
+    return "high_confidence"
+
+
 def _correlation_payload(
     payload: dict[str, Any],
     *,
@@ -1085,6 +1146,183 @@ def _handle_batch(payload: dict[str, Any]) -> dict[str, Any]:
     return body
 
 
+def _handle_messages(payload: dict[str, Any]) -> dict[str, Any]:
+    if "input" in payload or "inputs" in payload:
+        raise HTTPException(status_code=400, detail="provide either messages, input, or inputs")
+    messages = payload.get("messages")
+    if not isinstance(messages, list) or not messages:
+        raise HTTPException(status_code=400, detail="messages must be a non-empty list")
+    if not all(isinstance(message, dict) for message in messages):
+        raise HTTPException(status_code=400, detail="messages entries must be objects")
+    settings = payload.get("compression_settings") or {}
+    if not isinstance(settings, dict):
+        raise HTTPException(status_code=400, detail="compression_settings must be an object")
+    protected_roles = _string_set(
+        settings.get("protected_roles"),
+        default=DEFAULT_PROTECTED_MESSAGE_ROLES,
+        field_name="compression_settings.protected_roles",
+    )
+    compress_roles = (
+        _string_set(
+            settings.get("compress_roles"),
+            field_name="compression_settings.compress_roles",
+        )
+        if "compress_roles" in settings else None
+    )
+    protected_values = payload.get("protected_spans") or []
+    if not isinstance(protected_values, list) or not all(
+        isinstance(value, str) for value in protected_values
+    ):
+        raise HTTPException(status_code=400, detail="protected_spans must be strings")
+    request_id = payload.get("request_id")
+    if request_id is not None and not isinstance(request_id, str):
+        raise HTTPException(status_code=400, detail="request_id must be a string")
+    idempotency_key = payload.get("idempotency_key")
+    if idempotency_key is not None and not isinstance(idempotency_key, str):
+        raise HTTPException(status_code=400, detail="idempotency_key must be a string")
+
+    output_messages: list[dict[str, Any]] = []
+    receipts: list[dict[str, Any]] = []
+    nested_receipts: list[dict[str, Any]] = []
+    receipt_ids: list[str] = []
+    original_tokens = 0
+    output_tokens = 0
+    compressed_message_count = 0
+    skipped_message_count = 0
+
+    for index, message in enumerate(messages):
+        role = _message_role(message, index)
+        content = message.get("content")
+        output_message = dict(message)
+        if not isinstance(content, str):
+            skipped_message_count += 1
+            receipts.append({
+                "index": index,
+                "role": role,
+                "status": "skipped",
+                "reason": "non_string_content",
+                "content_type": type(content).__name__,
+                "original_input_tokens": 0,
+                "output_tokens": 0,
+                "tokens_saved": 0,
+            })
+            output_messages.append(output_message)
+            continue
+
+        role_protected = role in protected_roles or (
+            compress_roles is not None and role not in compress_roles
+        )
+        item_protected = list(protected_values)
+        if role_protected and content:
+            item_protected.append(content)
+        result = _compress_text({
+            "input": content,
+            "compression_settings": settings,
+            "protected_spans": item_protected,
+            "tool_schemas": payload.get("tool_schemas", payload.get("tools")),
+            "request_id": request_id,
+            "idempotency_key": idempotency_key,
+        })
+        output_message["content"] = result["output"]
+        output_messages.append(output_message)
+        original_tokens += int(result["original_input_tokens"])
+        output_tokens += int(result["output_tokens"])
+        tokens_saved = int(result["tokens_saved"])
+        if tokens_saved > 0:
+            compressed_message_count += 1
+        receipt = result["receipt"]
+        nested_receipts.append(receipt)
+        receipt_ids.append(receipt["receipt_id"])
+        receipts.append({
+            "index": index,
+            "role": role,
+            "status": "ok",
+            "protected_by_role": role_protected,
+            "original_input_tokens": result["original_input_tokens"],
+            "output_tokens": result["output_tokens"],
+            "tokens_saved": tokens_saved,
+            "compression_percentage": result["compression_percentage"],
+            "receipt_id": receipt["receipt_id"],
+            "receipt": receipt,
+        })
+
+    tokens_saved_total = max(0, original_tokens - output_tokens)
+    compression_pct = (
+        round(100.0 * tokens_saved_total / original_tokens, 1)
+        if original_tokens else 0.0
+    )
+    decision = _message_decision(
+        tokens_saved=tokens_saved_total,
+        receipts=nested_receipts,
+    )
+    aggregate_receipt = {
+        "receipt_version": "message-compression-receipt-v0.1.0",
+        "receipt_id": _aggregate_receipt_id({
+            "input_sha256": _stable_json_sha256(messages),
+            "output_sha256": _stable_json_sha256(output_messages),
+            "receipt_ids": receipt_ids,
+            "tokens_saved": tokens_saved_total,
+            "compression_percentage": compression_pct,
+            "decision": decision,
+        }),
+        "request_id": request_id,
+        "idempotency_key": idempotency_key,
+        "message_count": len(messages),
+        "compressed_message_count": compressed_message_count,
+        "skipped_message_count": skipped_message_count,
+        "protected_roles": sorted(protected_roles),
+        "compress_roles": sorted(compress_roles) if compress_roles is not None else None,
+        "decision": decision,
+        "deletion_only": all(
+            receipt.get("deletion_only", True) for receipt in nested_receipts
+        ),
+        "deterministic": True,
+        "input_sha256": _stable_json_sha256(messages),
+        "output_sha256": _stable_json_sha256(output_messages),
+        "message_receipt_ids": receipt_ids,
+    }
+    compressed_prompt: dict[str, Any] = {
+        "messages": output_messages,
+        "protected_spans": protected_values,
+    }
+    if "tools" in payload:
+        compressed_prompt["tools"] = payload["tools"]
+    if "tool_schemas" in payload:
+        compressed_prompt["tool_schemas"] = payload["tool_schemas"]
+    body = {
+        "schema_version": API_SCHEMA_VERSION,
+        "status": "ok",
+        "endpoint": "/v1/compress",
+        "maturity": "measurement_only",
+        "messages": output_messages,
+        "compressed_messages": output_messages,
+        "compressed_prompts": [compressed_prompt],
+        "message_count": len(messages),
+        "compressed_message_count": compressed_message_count,
+        "skipped_message_count": skipped_message_count,
+        "original_input_tokens": original_tokens,
+        "output_tokens": output_tokens,
+        "tokens_saved": tokens_saved_total,
+        "compression_percentage": compression_pct,
+        "receipt_ids": receipt_ids,
+        "receipts": receipts,
+        "receipt": aggregate_receipt,
+    }
+    if request_id is not None:
+        body["request_id"] = request_id
+    if idempotency_key is not None:
+        body["idempotency_key"] = idempotency_key
+    return body
+
+
+def _dispatch_compress(payload: dict[str, Any]) -> dict[str, Any]:
+    if "inputs" in payload:
+        return _handle_batch(payload)
+    if "messages" in payload:
+        return _handle_messages(payload)
+    return _compress_text(payload)
+
+
 def _tokens(text: str) -> list[dict[str, Any]]:
     started = time.perf_counter()
     try: