n1th1sh commited on
Commit
ba2fd7e
·
verified ·
1 Parent(s): 2e95a57

Update docclass.py

Browse files
Files changed (1) hide show
  1. docclass.py +10 -3
docclass.py CHANGED
@@ -1623,8 +1623,13 @@ def _extract(actual_type: str, data: bytes) -> tuple[object, bool]:
1623
  return ex, False
1624
 
1625
 
1626
- def analyze_file(data: bytes, filename: str, model: Layer3DocType) -> dict:
1627
- """All three layers for one file — the handover's classify_file, bounded."""
 
 
 
 
 
1628
  analysis = analyze_bytes(data, filename=filename, limits=SCAN_LIMITS, keep_bytes=True)
1629
  identity = analysis.identity
1630
  ex, ocr_used = _extract(identity.actual_type, data)
@@ -1654,6 +1659,7 @@ def analyze_file(data: bytes, filename: str, model: Layer3DocType) -> dict:
1654
  "truncated": bool(getattr(ex, "truncated", False)),
1655
  "nested_entries": nested,
1656
  "nested_truncated": bool(analysis.container.truncated) or len(analysis.container.entries) > MAX_NESTED_CLASSIFIED,
 
1657
  }
1658
 
1659
 
@@ -1692,7 +1698,8 @@ def build_router(verify: Callable[[Request], None]) -> APIRouter:
1692
  t0 = time.time()
1693
  from starlette.concurrency import run_in_threadpool
1694
  try:
1695
- result = await run_in_threadpool(analyze_file, data, filename, model)
 
1696
  except HTTPException:
1697
  raise
1698
  except Exception as exc: # noqa: BLE001
 
1623
  return ex, False
1624
 
1625
 
1626
+ def analyze_file(data: bytes, filename: str, model: Layer3DocType, return_text: bool = False) -> dict:
1627
+ """All three layers for one file — the handover's classify_file, bounded.
1628
+
1629
+ return_text: also return the extracted / OCR'd text (at most MAX_TEXT_CHARS),
1630
+ so CLOAK can run its PII scan on images and scanned PDFs without a second OCR
1631
+ pass. Off by default: a caller that only wants the document type is not sent
1632
+ the contents back."""
1633
  analysis = analyze_bytes(data, filename=filename, limits=SCAN_LIMITS, keep_bytes=True)
1634
  identity = analysis.identity
1635
  ex, ocr_used = _extract(identity.actual_type, data)
 
1659
  "truncated": bool(getattr(ex, "truncated", False)),
1660
  "nested_entries": nested,
1661
  "nested_truncated": bool(analysis.container.truncated) or len(analysis.container.entries) > MAX_NESTED_CLASSIFIED,
1662
+ **({"text": (getattr(ex, "text", None) or "")[:MAX_TEXT_CHARS]} if return_text else {}),
1663
  }
1664
 
1665
 
 
1698
  t0 = time.time()
1699
  from starlette.concurrency import run_in_threadpool
1700
  try:
1701
+ return_text = request.headers.get("x-return-text", "") == "1"
1702
+ result = await run_in_threadpool(analyze_file, data, filename, model, return_text)
1703
  except HTTPException:
1704
  raise
1705
  except Exception as exc: # noqa: BLE001