File size: 5,465 Bytes
1ef1de7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | """Quality and confidence reporting for parsed documents."""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
from .schemas import Document
@dataclass(slots=True)
class QualityIssue:
severity: str
code: str
message: str
sample: str | None = None
def to_dict(self) -> dict[str, Any]:
return {"severity": self.severity, "code": self.code, "message": self.message, "sample": self.sample}
@dataclass(slots=True)
class DocumentQuality:
passed: bool
quality_score: float
metrics: dict[str, float | int | None]
issues: list[QualityIssue] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return {
"passed": self.passed,
"quality_score": self.quality_score,
"metrics": self.metrics,
"issues": [issue.to_dict() for issue in self.issues],
}
def evaluate_document_quality(
document: Document,
*,
min_line_confidence: float = 0.80,
min_quality_score: float = 0.70,
) -> DocumentQuality:
lines = [line for page in document.pages for line in page.text_lines]
blocks = [block for page in document.pages for block in page.blocks]
confidences = [line.confidence for line in lines if line.confidence is not None]
low_confidence = [line for line in lines if line.confidence is not None and line.confidence < min_line_confidence]
missing_confidence = [line for line in lines if line.confidence is None]
empty_lines = [line for line in lines if not line.text.strip()]
tables_without_cells = [table for table in document.tables if not table.cells]
blocks_without_lines = [block for block in blocks if block.block_type in {"text", "title", "table"} and not block.line_ids]
metrics: dict[str, float | int | None] = {
"page_count": len(document.pages),
"line_count": len(lines),
"block_count": len(blocks),
"table_count": len(document.tables),
"figure_count": len(document.figures),
"average_line_confidence": sum(confidences) / len(confidences) if confidences else None,
"minimum_line_confidence": min(confidences) if confidences else None,
"low_confidence_line_count": len(low_confidence),
"missing_confidence_line_count": len(missing_confidence),
"empty_text_line_count": len(empty_lines),
"tables_without_cells": len(tables_without_cells),
"blocks_without_lines": len(blocks_without_lines),
}
issues: list[QualityIssue] = []
if not document.pages:
issues.append(QualityIssue("error", "no_pages", "document contains no pages"))
if not lines:
issues.append(QualityIssue("error", "no_text_lines", "document contains no OCR text lines"))
if blocks and not document.text.strip():
issues.append(QualityIssue("error", "empty_document_text", "document blocks contain no text"))
if missing_confidence:
issues.append(
QualityIssue(
"warning",
"missing_line_confidence",
f"{len(missing_confidence)} text lines have no confidence score",
sample=missing_confidence[0].line_id,
)
)
if low_confidence:
issues.append(
QualityIssue(
"warning",
"low_line_confidence",
f"{len(low_confidence)} text lines are below confidence threshold {min_line_confidence}",
sample=low_confidence[0].line_id,
)
)
if tables_without_cells:
issues.append(
QualityIssue(
"warning",
"table_without_cells",
f"{len(tables_without_cells)} tables have no cells",
sample=tables_without_cells[0].table_id,
)
)
if blocks_without_lines:
issues.append(
QualityIssue(
"warning",
"block_without_lines",
f"{len(blocks_without_lines)} text/table blocks have no source line ids",
sample=blocks_without_lines[0].block_id,
)
)
score = _quality_score(metrics, len(lines))
if score < min_quality_score:
issues.append(QualityIssue("error", "quality_score_below_threshold", f"quality score {score:.4f} < required {min_quality_score:.4f}"))
passed = not any(issue.severity == "error" for issue in issues)
return DocumentQuality(passed=passed, quality_score=score, metrics=metrics, issues=issues)
def _quality_score(metrics: dict[str, float | int | None], line_count: int) -> float:
if line_count <= 0:
return 0.0
penalty = 0.0
low = int(metrics["low_confidence_line_count"] or 0)
missing = int(metrics["missing_confidence_line_count"] or 0)
empty = int(metrics["empty_text_line_count"] or 0)
tables_without_cells = int(metrics["tables_without_cells"] or 0)
blocks_without_lines = int(metrics["blocks_without_lines"] or 0)
penalty += low / line_count * 0.35
penalty += missing / line_count * 0.20
penalty += empty / line_count * 0.20
penalty += min(0.15, tables_without_cells * 0.05)
penalty += min(0.10, blocks_without_lines * 0.02)
confidence = metrics.get("average_line_confidence")
if isinstance(confidence, int | float):
penalty += max(0.0, 0.95 - float(confidence)) * 0.20
return round(max(0.0, min(1.0, 1.0 - penalty)), 6)
|