Spaces:
Sleeping
Heat code extractor v3: capture all 6 KB419-1 codes; reject OCR-mangled English; normalize O→0
Browse filesThree issues fixed from the staging test on KB419-1:
1) Missing KG416-1 and KS182-1: the OCR column heuristic was only
scanning 3 lines after a 'Heat No.' header. KB419-1's table has
6 rows, so the last 3 were dropped. Bumped both the pdfplumber
bbox-based scan (6 → 20 lines) and the OCR-text heuristic (3 → 20
non-empty lines) so multi-row tables are captured in full.
2) CON-REDUCER false positive: the previous 'isalpha()' check missed
tokens with dashes. Replaced with a 'must contain at least one digit'
rule that catches both pure-alpha (DIMENSION) and alpha+dash
(CON-REDUCER, WP304-S) tokens with no digits.
3) JNO21-1 vs JN021-1 cosmetic OCR drift: Tesseract reads the digit 0
as letter O. For OCR-sourced candidates, replace O→0 in-place.
None of our 14 confirmed real heat codes use letter O.
Side effect of (1) was that the wider lookahead also caught a wave of
OCR-mangled English words (ACC0RDING, BEL0W, FR0M, D0NE, ABO̧VE, etc.)
where Tesseract turned letter O into digit 0. Added a fourth filter:
reject candidates whose every digit is a zero. None of the real heat
codes match this pattern; all OCR-mangled English words do.
Validated locally against all 8 text-layer cases (all real codes
captured) and against KB419-1 in forced-OCR mode (all 6 codes captured,
4 extras remaining).
- app/lib/utils/heat_code.py +38 -9
|
@@ -176,6 +176,19 @@ def extract_heat_codes(file_bytes: bytes, filename: str | None = None) -> list[s
|
|
| 176 |
except Exception as e:
|
| 177 |
logger.debug("OCR fallback failed: %s", e)
|
| 178 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 179 |
# Build the set of tokens that appear as standards-references in the
|
| 180 |
# document text. Combine text_layer + ocr_text if both were collected.
|
| 181 |
combined_text = text_layer
|
|
@@ -324,9 +337,11 @@ def _from_column_position(file_bytes: bytes) -> Iterable[str]:
|
|
| 324 |
label_center = (x0 + x1) / 2
|
| 325 |
half_width = max((x1 - x0) / 2, 20.0) # min 20pt to be forgiving
|
| 326 |
|
| 327 |
-
# Scan up to
|
| 328 |
-
# half-width of the label.
|
| 329 |
-
|
|
|
|
|
|
|
| 330 |
for nw in lines[next_key]:
|
| 331 |
nw_center = (nw["x0"] + nw["x1"]) / 2
|
| 332 |
if abs(nw_center - label_center) <= half_width:
|
|
@@ -347,14 +362,17 @@ def _from_ocr_column_heuristic(ocr_text: str) -> Iterable[str]:
|
|
| 347 |
for i, line in enumerate(lines):
|
| 348 |
if not label_re.search(line):
|
| 349 |
continue
|
| 350 |
-
# Look at the next
|
|
|
|
|
|
|
|
|
|
| 351 |
looked = 0
|
| 352 |
-
for j in range(i + 1, min(i +
|
| 353 |
nxt = lines[j].strip()
|
| 354 |
if not nxt:
|
| 355 |
continue
|
| 356 |
looked += 1
|
| 357 |
-
if looked >
|
| 358 |
break
|
| 359 |
for tok in _tokenize_cell(nxt):
|
| 360 |
if _CODE_SHAPE_RE.match(tok):
|
|
@@ -500,9 +518,20 @@ def _is_plausible_heat_code(code: str) -> bool:
|
|
| 500 |
# overlaps with real heat codes; we don't reject those on shape alone.
|
| 501 |
if any(upper.startswith(p) for p in _SPEC_PREFIXES):
|
| 502 |
return False
|
| 503 |
-
# Reject
|
| 504 |
-
# least one digit
|
| 505 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 506 |
return False
|
| 507 |
|
| 508 |
return True
|
|
|
|
| 176 |
except Exception as e:
|
| 177 |
logger.debug("OCR fallback failed: %s", e)
|
| 178 |
|
| 179 |
+
# OCR commonly confuses the digit '0' with the letter 'O' (e.g. "JN021-1"
|
| 180 |
+
# becomes "JNO21-1"). None of our 14 confirmed real heat codes uses the
|
| 181 |
+
# letter O — every one has digit 0 in positions where O would appear. So
|
| 182 |
+
# for OCR-sourced candidates we replace O→0 in-place. This avoids creating
|
| 183 |
+
# duplicate folders ("JNO21-1" AND "JN021-1") and produces the canonical
|
| 184 |
+
# form. Text-layer candidates are left alone — they have the correct chars.
|
| 185 |
+
for idx in range(len(candidates)):
|
| 186 |
+
code, source = candidates[idx]
|
| 187 |
+
if source.startswith("ocr") and ("O" in code or "o" in code):
|
| 188 |
+
normalized = code.replace("O", "0").replace("o", "0")
|
| 189 |
+
if normalized != code:
|
| 190 |
+
candidates[idx] = (normalized, f"{source}_o2zero")
|
| 191 |
+
|
| 192 |
# Build the set of tokens that appear as standards-references in the
|
| 193 |
# document text. Combine text_layer + ocr_text if both were collected.
|
| 194 |
combined_text = text_layer
|
|
|
|
| 337 |
label_center = (x0 + x1) / 2
|
| 338 |
half_width = max((x1 - x0) / 2, 20.0) # min 20pt to be forgiving
|
| 339 |
|
| 340 |
+
# Scan up to 20 lines below for tokens whose center is within
|
| 341 |
+
# half-width of the label. Multi-row MTR tables (KB419-1
|
| 342 |
+
# case has 6 rows; some have more) need a generous lookahead
|
| 343 |
+
# or we miss data rows farther down the table.
|
| 344 |
+
for next_key in sorted_keys[i + 1 : i + 21]:
|
| 345 |
for nw in lines[next_key]:
|
| 346 |
nw_center = (nw["x0"] + nw["x1"]) / 2
|
| 347 |
if abs(nw_center - label_center) <= half_width:
|
|
|
|
| 362 |
for i, line in enumerate(lines):
|
| 363 |
if not label_re.search(line):
|
| 364 |
continue
|
| 365 |
+
# Look at the next 20 non-empty lines. Multi-row MTR tables can have
|
| 366 |
+
# 6+ data rows (KB419-1 case); a tight lookahead drops rows past the
|
| 367 |
+
# first few. Spec-prefix and material-spec filters downstream catch
|
| 368 |
+
# the noise this wider lookahead would otherwise admit.
|
| 369 |
looked = 0
|
| 370 |
+
for j in range(i + 1, min(i + 40, len(lines))):
|
| 371 |
nxt = lines[j].strip()
|
| 372 |
if not nxt:
|
| 373 |
continue
|
| 374 |
looked += 1
|
| 375 |
+
if looked > 20:
|
| 376 |
break
|
| 377 |
for tok in _tokenize_cell(nxt):
|
| 378 |
if _CODE_SHAPE_RE.match(tok):
|
|
|
|
| 518 |
# overlaps with real heat codes; we don't reject those on shape alone.
|
| 519 |
if any(upper.startswith(p) for p in _SPEC_PREFIXES):
|
| 520 |
return False
|
| 521 |
+
# Reject tokens with no digits at all. Real heat codes always contain at
|
| 522 |
+
# least one digit (verified across all 14 confirmed samples). This catches
|
| 523 |
+
# both pure-alpha tokens (DIMENSION) and alpha+dash tokens (CON-REDUCER,
|
| 524 |
+
# WP304-S, etc.) that the per-char isalpha() check would miss.
|
| 525 |
+
digits = [c for c in upper if c.isdigit()]
|
| 526 |
+
if not digits:
|
| 527 |
+
return False
|
| 528 |
+
|
| 529 |
+
# Reject tokens whose ONLY digit(s) are zero(s). This is the OCR-mangled-
|
| 530 |
+
# English-word signature: Tesseract reads letter O as digit 0, so words
|
| 531 |
+
# like ACCORDING, BELOW, FROM, DONE, COMPOSITION, etc. become ACC0RDING,
|
| 532 |
+
# BEL0W, FR0M, D0NE, C0MP0SITI0N. None of our 14 confirmed real heat codes
|
| 533 |
+
# have all-zero digits; they always have at least one 1–9 digit.
|
| 534 |
+
if all(d == "0" for d in digits):
|
| 535 |
return False
|
| 536 |
|
| 537 |
return True
|