Upload folder using huggingface_hub
Browse files
README.md
CHANGED
|
@@ -31,9 +31,9 @@ Live Space:
|
|
| 31 |
prompts. Use `hf spaces info wchen22/touchdown-compression-classifier --format
|
| 32 |
json` for the current repo/runtime SHA.
|
| 33 |
- Live smoke:
|
| 34 |
-
`python3 scripts/smoke_compression_api.py --base-url https://wchen22-touchdown-compression-classifier.hf.space --include-classify --include-batch`
|
| 35 |
validates `/health`, `/v1/classify`, single `/v1/compress`, and managed
|
| 36 |
-
`inputs[]` batch.
|
| 37 |
- Full deployment receipt:
|
| 38 |
`python3 scripts/verify_compression_space.py --expected-sha <sha> --out reports/generated/compression_space/hf_space_verification.json`
|
| 39 |
validates HF runtime metadata, repo/runtime SHA agreement, API smoke, and
|
|
@@ -46,6 +46,8 @@ Live Space:
|
|
| 46 |
and `/v1/classify` returned KEEP-only DeBERTa tokenizer labels. Receipts
|
| 47 |
include removed-span/char totals, classifier DROP block reasons, and
|
| 48 |
tool-schema preservation counts when `tools` or `tool_schemas` are supplied.
|
|
|
|
|
|
|
| 49 |
- `/v1/classify` is tokenizer/fallback KEEP-only until a trained KEEP/DROP head
|
| 50 |
is mounted. `/v1/compress` is rules-first deletion-only compression with
|
| 51 |
safety receipts. The Space app supports both single `input` requests and
|
|
|
|
| 31 |
prompts. Use `hf spaces info wchen22/touchdown-compression-classifier --format
|
| 32 |
json` for the current repo/runtime SHA.
|
| 33 |
- Live smoke:
|
| 34 |
+
`python3 scripts/smoke_compression_api.py --base-url https://wchen22-touchdown-compression-classifier.hf.space --include-classify --include-batch --include-gzip`
|
| 35 |
validates `/health`, `/v1/classify`, single `/v1/compress`, and managed
|
| 36 |
+
`inputs[]` batch, plus gzipped JSON request/response transport.
|
| 37 |
- Full deployment receipt:
|
| 38 |
`python3 scripts/verify_compression_space.py --expected-sha <sha> --out reports/generated/compression_space/hf_space_verification.json`
|
| 39 |
validates HF runtime metadata, repo/runtime SHA agreement, API smoke, and
|
|
|
|
| 46 |
and `/v1/classify` returned KEEP-only DeBERTa tokenizer labels. Receipts
|
| 47 |
include removed-span/char totals, classifier DROP block reasons, and
|
| 48 |
tool-schema preservation counts when `tools` or `tool_schemas` are supplied.
|
| 49 |
+
The HTTP surface accepts `Content-Encoding: gzip` request bodies and gzip
|
| 50 |
+
responses for `Accept-Encoding: gzip` or gzipped requests.
|
| 51 |
- `/v1/classify` is tokenizer/fallback KEEP-only until a trained KEEP/DROP head
|
| 52 |
is mounted. `/v1/compress` is rules-first deletion-only compression with
|
| 53 |
safety receipts. The Space app supports both single `input` requests and
|
app.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
|
|
|
| 3 |
import hashlib
|
| 4 |
import json
|
| 5 |
import math
|
|
@@ -11,7 +12,7 @@ from functools import lru_cache
|
|
| 11 |
from pathlib import Path
|
| 12 |
from typing import Any
|
| 13 |
|
| 14 |
-
from fastapi import FastAPI, HTTPException, Request
|
| 15 |
|
| 16 |
CLASSIFIER_MODEL = "microsoft/deberta-v3-small"
|
| 17 |
CLASSIFIER_ARTIFACT_DIR = os.environ.get("TOUCHDOWN_CLASSIFIER_ARTIFACT_DIR")
|
|
@@ -19,6 +20,9 @@ API_SCHEMA_VERSION = "0.1.0"
|
|
| 19 |
RULES_VERSION = "hf-space-rules-v0.1.0"
|
| 20 |
REQUEST_ID_HEADER = "X-Request-ID"
|
| 21 |
IDEMPOTENCY_KEY_HEADER = "Idempotency-Key"
|
|
|
|
|
|
|
|
|
|
| 22 |
LOW_SIGNAL_PATTERNS = [
|
| 23 |
re.compile(pattern, re.IGNORECASE)
|
| 24 |
for pattern in [
|
|
@@ -35,12 +39,129 @@ app = FastAPI(title="Touchdown Compression Classifier", version="0.1.0")
|
|
| 35 |
|
| 36 |
|
| 37 |
@app.middleware("http")
|
| 38 |
-
async def
|
| 39 |
request_id = request.headers.get(REQUEST_ID_HEADER) or f"tdreq_{uuid.uuid4().hex}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
request.state.request_id = request_id
|
| 41 |
response = await call_next(request)
|
| 42 |
response.headers[REQUEST_ID_HEADER] = request_id
|
| 43 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
|
| 45 |
|
| 46 |
@lru_cache(maxsize=1)
|
|
@@ -799,6 +920,12 @@ def health() -> dict[str, Any]:
|
|
| 799 |
"classifier_artifact_dir_configured": bool(CLASSIFIER_ARTIFACT_DIR),
|
| 800 |
"classifier_error": classifier_error,
|
| 801 |
"phase": "rules_api_first",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 802 |
}
|
| 803 |
|
| 804 |
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
+
import gzip
|
| 4 |
import hashlib
|
| 5 |
import json
|
| 6 |
import math
|
|
|
|
| 12 |
from pathlib import Path
|
| 13 |
from typing import Any
|
| 14 |
|
| 15 |
+
from fastapi import FastAPI, HTTPException, Request, Response
|
| 16 |
|
| 17 |
CLASSIFIER_MODEL = "microsoft/deberta-v3-small"
|
| 18 |
CLASSIFIER_ARTIFACT_DIR = os.environ.get("TOUCHDOWN_CLASSIFIER_ARTIFACT_DIR")
|
|
|
|
| 20 |
RULES_VERSION = "hf-space-rules-v0.1.0"
|
| 21 |
REQUEST_ID_HEADER = "X-Request-ID"
|
| 22 |
IDEMPOTENCY_KEY_HEADER = "Idempotency-Key"
|
| 23 |
+
CONTENT_ENCODING_HEADER = "Content-Encoding"
|
| 24 |
+
ACCEPT_ENCODING_HEADER = "Accept-Encoding"
|
| 25 |
+
GZIP_ENCODING = "gzip"
|
| 26 |
LOW_SIGNAL_PATTERNS = [
|
| 27 |
re.compile(pattern, re.IGNORECASE)
|
| 28 |
for pattern in [
|
|
|
|
| 39 |
|
| 40 |
|
| 41 |
@app.middleware("http")
|
| 42 |
+
async def transport_middleware(request: Request, call_next):
|
| 43 |
request_id = request.headers.get(REQUEST_ID_HEADER) or f"tdreq_{uuid.uuid4().hex}"
|
| 44 |
+
try:
|
| 45 |
+
body, request_was_gzip = _decode_request_body(
|
| 46 |
+
await request.body(),
|
| 47 |
+
request.headers.get(CONTENT_ENCODING_HEADER),
|
| 48 |
+
)
|
| 49 |
+
except UnsupportedContentEncoding as exc:
|
| 50 |
+
response = Response(
|
| 51 |
+
content=json.dumps({
|
| 52 |
+
"status": "error",
|
| 53 |
+
"error": str(exc),
|
| 54 |
+
"request_id": request_id,
|
| 55 |
+
}),
|
| 56 |
+
status_code=415,
|
| 57 |
+
media_type="application/json",
|
| 58 |
+
headers={REQUEST_ID_HEADER: request_id},
|
| 59 |
+
)
|
| 60 |
+
return await _gzip_response_if_needed(
|
| 61 |
+
request,
|
| 62 |
+
response,
|
| 63 |
+
request_was_gzip=False,
|
| 64 |
+
)
|
| 65 |
+
except ValueError as exc:
|
| 66 |
+
response = Response(
|
| 67 |
+
content=json.dumps({
|
| 68 |
+
"status": "error",
|
| 69 |
+
"error": str(exc),
|
| 70 |
+
"request_id": request_id,
|
| 71 |
+
}),
|
| 72 |
+
status_code=400,
|
| 73 |
+
media_type="application/json",
|
| 74 |
+
headers={REQUEST_ID_HEADER: request_id},
|
| 75 |
+
)
|
| 76 |
+
return await _gzip_response_if_needed(
|
| 77 |
+
request,
|
| 78 |
+
response,
|
| 79 |
+
request_was_gzip=False,
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
async def receive() -> dict[str, Any]:
|
| 83 |
+
return {"type": "http.request", "body": body, "more_body": False}
|
| 84 |
+
|
| 85 |
+
request = Request(request.scope, receive)
|
| 86 |
request.state.request_id = request_id
|
| 87 |
response = await call_next(request)
|
| 88 |
response.headers[REQUEST_ID_HEADER] = request_id
|
| 89 |
+
return await _gzip_response_if_needed(
|
| 90 |
+
request,
|
| 91 |
+
response,
|
| 92 |
+
request_was_gzip=request_was_gzip,
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
class UnsupportedContentEncoding(ValueError):
|
| 97 |
+
"""Raised when an HTTP request uses an unsupported content encoding."""
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _content_encoding(value: str | None) -> str:
|
| 101 |
+
return (value or "identity").split(";", 1)[0].strip().lower() or "identity"
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def _decode_request_body(raw: bytes, content_encoding: str | None) -> tuple[bytes, bool]:
|
| 105 |
+
encoding = _content_encoding(content_encoding)
|
| 106 |
+
if encoding == "identity":
|
| 107 |
+
return raw, False
|
| 108 |
+
if encoding != GZIP_ENCODING:
|
| 109 |
+
raise UnsupportedContentEncoding(
|
| 110 |
+
f"unsupported content encoding: {content_encoding}"
|
| 111 |
+
)
|
| 112 |
+
try:
|
| 113 |
+
return gzip.decompress(raw), True
|
| 114 |
+
except OSError as exc:
|
| 115 |
+
raise ValueError("invalid gzip request body") from exc
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def _accepts_gzip(accept_encoding: str | None) -> bool:
|
| 119 |
+
for part in (accept_encoding or "").split(","):
|
| 120 |
+
normalized = part.replace(" ", "").lower()
|
| 121 |
+
token = normalized.split(";", 1)[0]
|
| 122 |
+
if token in (GZIP_ENCODING, "*") and "q=0" not in normalized:
|
| 123 |
+
return True
|
| 124 |
+
return False
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _should_gzip_response(
|
| 128 |
+
*,
|
| 129 |
+
accept_encoding: str | None,
|
| 130 |
+
request_was_gzip: bool,
|
| 131 |
+
) -> bool:
|
| 132 |
+
return request_was_gzip or _accepts_gzip(accept_encoding)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
async def _gzip_response_if_needed(
|
| 136 |
+
request: Request,
|
| 137 |
+
response: Response,
|
| 138 |
+
*,
|
| 139 |
+
request_was_gzip: bool,
|
| 140 |
+
) -> Response:
|
| 141 |
+
should_gzip = _should_gzip_response(
|
| 142 |
+
accept_encoding=request.headers.get(ACCEPT_ENCODING_HEADER),
|
| 143 |
+
request_was_gzip=request_was_gzip,
|
| 144 |
+
)
|
| 145 |
+
if not should_gzip or response.headers.get(CONTENT_ENCODING_HEADER):
|
| 146 |
+
return response
|
| 147 |
+
|
| 148 |
+
if hasattr(response, "body_iterator"):
|
| 149 |
+
body = b""
|
| 150 |
+
async for chunk in response.body_iterator:
|
| 151 |
+
body += chunk
|
| 152 |
+
else:
|
| 153 |
+
body = getattr(response, "body", b"")
|
| 154 |
+
compressed = gzip.compress(body)
|
| 155 |
+
headers = dict(response.headers)
|
| 156 |
+
headers[CONTENT_ENCODING_HEADER] = GZIP_ENCODING
|
| 157 |
+
headers["Vary"] = ACCEPT_ENCODING_HEADER
|
| 158 |
+
headers["Content-Length"] = str(len(compressed))
|
| 159 |
+
return Response(
|
| 160 |
+
content=compressed,
|
| 161 |
+
status_code=response.status_code,
|
| 162 |
+
headers=headers,
|
| 163 |
+
background=getattr(response, "background", None),
|
| 164 |
+
)
|
| 165 |
|
| 166 |
|
| 167 |
@lru_cache(maxsize=1)
|
|
|
|
| 920 |
"classifier_artifact_dir_configured": bool(CLASSIFIER_ARTIFACT_DIR),
|
| 921 |
"classifier_error": classifier_error,
|
| 922 |
"phase": "rules_api_first",
|
| 923 |
+
"transport": {
|
| 924 |
+
"request_id_header": REQUEST_ID_HEADER,
|
| 925 |
+
"idempotency_key_header": IDEMPOTENCY_KEY_HEADER,
|
| 926 |
+
"gzip_request_bodies": True,
|
| 927 |
+
"gzip_responses": True,
|
| 928 |
+
},
|
| 929 |
}
|
| 930 |
|
| 931 |
|