Spaces:
Sleeping
Tighten heat_code extractor: reject standards numbers + material specs
Browse filesProduction extractor was returning 8 codes from 10720.pdf when only
2 are real heat codes. False positives were fragments of standards
references that the OCR fallback was grabbing because they appear
near 'Heat Code' headers in the doc text:
- 0103 from 'NACE MR 0103'
- 0175 from 'MR 0175'
- 2015 from 'ISO 9001:2015'
- 15156 from 'ISO 15156'
- 17945 from 'ISO 17945'
- A106B from '1/2 X 4 XH A106B SMLS TBE NIPPLE' (material spec)
Three fixes:
1) Expand year regex from 20[2-3]\d to 20[0-3]\d (catches 2015 etc).
2) Add material-spec pattern A\d{3}[A-Z] for A106B / A105N / A234B.
Distinct from real heat codes like A8633 (4 digits, no trailing
letter) so no regression on those.
3) New _collect_standards_numbers() scans the document text for
ISO/NACE/ASTM/ASME/EN/DIN/API/AISI/JIS/MSS/PED prefixes and
adds the trailing standard numbers to a reject set. The filter
rejects candidates that match any of those tokens.
Validated against all 10 sample MTRs β A8633, KB419-1 (six codes),
D460, SD55822, Y211213D40-1, 24L0604, VC9387 all still extract
correctly. Forced-OCR-path test on 10720.pdf now returns just
['10720', '114420'] β all 6 false positives killed.
- app/lib/utils/heat_code.py +86 -5
|
@@ -92,8 +92,38 @@ _REJECT_WORDS: frozenset[str] = frozenset({
|
|
| 92 |
# Spec number patterns to reject (e.g., A312, SA-105, A105-21).
|
| 93 |
_SPEC_NUM_RE = re.compile(r"^[A-Z]{1,4}-?\d{2,5}(-\d{1,3})?$", re.IGNORECASE)
|
| 94 |
|
| 95 |
-
# Year-like standalone tokens (
|
| 96 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
|
| 98 |
|
| 99 |
# βββ Public API ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -146,8 +176,19 @@ def extract_heat_codes(file_bytes: bytes, filename: str | None = None) -> list[s
|
|
| 146 |
except Exception as e:
|
| 147 |
logger.debug("OCR fallback failed: %s", e)
|
| 148 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
# Filter + dedupe (case-insensitive on code; preserve first-seen casing)
|
| 150 |
-
filtered = _filter_and_dedupe(candidates)
|
| 151 |
|
| 152 |
if logger.isEnabledFor(logging.DEBUG):
|
| 153 |
logger.debug(
|
|
@@ -375,10 +416,43 @@ def _tokenize_cell(cell_text: str) -> list[str]:
|
|
| 375 |
return [t.strip() for t in re.split(r"[\s,;|]+", cell_text) if t.strip()]
|
| 376 |
|
| 377 |
|
| 378 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 379 |
"""Apply rejection rules + dedupe case-insensitively, preserving order.
|
| 380 |
Also drops substring duplicates: if 'KB419' and 'KB419-1' are both present,
|
| 381 |
-
keep only the longer one (avoid over-counting when codes get split).
|
|
|
|
|
|
|
|
|
|
|
|
|
| 382 |
seen: dict[str, str] = {} # upper -> original-cased
|
| 383 |
|
| 384 |
for code, _source in candidates:
|
|
@@ -387,6 +461,8 @@ def _filter_and_dedupe(candidates: list[tuple[str, str]]) -> list[str]:
|
|
| 387 |
|
| 388 |
if upper in seen:
|
| 389 |
continue
|
|
|
|
|
|
|
| 390 |
if not _is_plausible_heat_code(code):
|
| 391 |
continue
|
| 392 |
seen[upper] = code
|
|
@@ -411,6 +487,11 @@ def _is_plausible_heat_code(code: str) -> bool:
|
|
| 411 |
return False
|
| 412 |
if _YEAR_RE.match(upper):
|
| 413 |
return False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 414 |
# Pipe-spec rejection (40S, 304L, 80SS-style schedule/grade designators).
|
| 415 |
if _PIPE_SPEC_RE.match(upper):
|
| 416 |
return False
|
|
|
|
| 92 |
# Spec number patterns to reject (e.g., A312, SA-105, A105-21).
|
| 93 |
_SPEC_NUM_RE = re.compile(r"^[A-Z]{1,4}-?\d{2,5}(-\d{1,3})?$", re.IGNORECASE)
|
| 94 |
|
| 95 |
+
# Year-like standalone tokens (2010-2039). Expanded from 2020-2030 because
|
| 96 |
+
# MTRs reference older standards years like "ISO 9001:2015" and Tesseract
|
| 97 |
+
# may misread small text producing tokens in this range.
|
| 98 |
+
_YEAR_RE = re.compile(r"^(20[0-3]\d)$")
|
| 99 |
+
|
| 100 |
+
# Material-spec pattern: "A106B", "A105N", "A234B" β letter A + 3 digits +
|
| 101 |
+
# single trailing letter. ASTM material grade specs. Distinct from real
|
| 102 |
+
# heat codes like "A8633" (letter + 4 digits, no trailing letter).
|
| 103 |
+
_MATERIAL_SPEC_RE = re.compile(r"^A\d{3}[A-Z]$", re.IGNORECASE)
|
| 104 |
+
|
| 105 |
+
# Standards-prefix patterns: anywhere in the document text, if a token
|
| 106 |
+
# follows one of these prefixes, that token is a standards reference, NOT
|
| 107 |
+
# a heat code. Examples: "ISO 15156", "NACE MR 0103", "ASTM A312", "EN 10204".
|
| 108 |
+
# The capture group is the standards number we want to reject.
|
| 109 |
+
_STANDARDS_REFERENCE_RES: tuple[re.Pattern[str], ...] = (
|
| 110 |
+
re.compile(r"\bISO\s+(\d{3,6})\b", re.IGNORECASE),
|
| 111 |
+
re.compile(r"\bNACE\s*MR\s*(\d{3,6})\b", re.IGNORECASE),
|
| 112 |
+
re.compile(r"\bMR\s*(\d{3,6})\b", re.IGNORECASE),
|
| 113 |
+
re.compile(r"\bASTM\s+([A-Z]?\d{3,5}[A-Z]?(?:-\d{1,3})?)\b", re.IGNORECASE),
|
| 114 |
+
re.compile(r"\bASME\s+(?:SA-?)?([A-Z]?\d{2,5}[A-Z]?(?:[.\-]\d{1,4})?)\b", re.IGNORECASE),
|
| 115 |
+
re.compile(r"\bEN\s+(\d{3,6}(?::\d{4})?)\b", re.IGNORECASE),
|
| 116 |
+
re.compile(r"\bDIN\s+(\d{3,6})\b", re.IGNORECASE),
|
| 117 |
+
re.compile(r"\bAPI\s+(\d+L?)\b", re.IGNORECASE),
|
| 118 |
+
re.compile(r"\bAISI\s+(\d{3,4}[A-Z]?)\b", re.IGNORECASE),
|
| 119 |
+
re.compile(r"\bJIS\s+(\w+)\b", re.IGNORECASE),
|
| 120 |
+
re.compile(r"\bMSS\s*SP\s*(\d{1,4})\b", re.IGNORECASE),
|
| 121 |
+
re.compile(r"\bPED\s+(\d{4}/\d{2}/[A-Z]+)\b", re.IGNORECASE),
|
| 122 |
+
# "ISO 9001:2015" β both numbers are bad
|
| 123 |
+
re.compile(r"(\d{4}):(\d{4})"),
|
| 124 |
+
# "A106/A106M-18" or "SA106/SA106M" β slash-separated material specs
|
| 125 |
+
re.compile(r"\b([A-Z]?\d{3,4}[A-Z]?(?:-\d{1,3})?)/([A-Z]?\d{3,4}[A-Z]?(?:-\d{1,3})?)\b"),
|
| 126 |
+
)
|
| 127 |
|
| 128 |
|
| 129 |
# βββ Public API ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 176 |
except Exception as e:
|
| 177 |
logger.debug("OCR fallback failed: %s", e)
|
| 178 |
|
| 179 |
+
# Build the set of tokens that appear as standards-references in the
|
| 180 |
+
# document text. Combine text_layer + ocr_text if both were collected.
|
| 181 |
+
combined_text = text_layer
|
| 182 |
+
if not candidates or len(text_layer) < 500:
|
| 183 |
+
# OCR fallback may have produced text we should also scan.
|
| 184 |
+
try:
|
| 185 |
+
combined_text = (text_layer or "") + "\n" + _ocr_text(file_bytes)
|
| 186 |
+
except Exception:
|
| 187 |
+
pass
|
| 188 |
+
standards_tokens = _collect_standards_numbers(combined_text)
|
| 189 |
+
|
| 190 |
# Filter + dedupe (case-insensitive on code; preserve first-seen casing)
|
| 191 |
+
filtered = _filter_and_dedupe(candidates, standards_tokens)
|
| 192 |
|
| 193 |
if logger.isEnabledFor(logging.DEBUG):
|
| 194 |
logger.debug(
|
|
|
|
| 416 |
return [t.strip() for t in re.split(r"[\s,;|]+", cell_text) if t.strip()]
|
| 417 |
|
| 418 |
|
| 419 |
+
def _collect_standards_numbers(doc_text: str) -> set[str]:
|
| 420 |
+
"""Scan document text for standards references (ISO 15156, NACE MR 0103,
|
| 421 |
+
ASTM A312, EN 10204, etc.) and return the set of standard-number tokens
|
| 422 |
+
that should never be treated as heat codes.
|
| 423 |
+
|
| 424 |
+
Returned tokens are uppercased and stripped of leading non-alphanumerics
|
| 425 |
+
so they can be compared directly against `code.upper()` in the filter.
|
| 426 |
+
"""
|
| 427 |
+
bad: set[str] = set()
|
| 428 |
+
if not doc_text:
|
| 429 |
+
return bad
|
| 430 |
+
for pat in _STANDARDS_REFERENCE_RES:
|
| 431 |
+
for m in pat.finditer(doc_text):
|
| 432 |
+
for grp in m.groups():
|
| 433 |
+
if not grp:
|
| 434 |
+
continue
|
| 435 |
+
tok = re.sub(r"^[^A-Za-z0-9]+", "", grp).upper()
|
| 436 |
+
if tok:
|
| 437 |
+
bad.add(tok)
|
| 438 |
+
# Also reject the no-dash form (e.g., "A106-18" β also drop "A106")
|
| 439 |
+
no_dash = tok.split("-")[0]
|
| 440 |
+
if no_dash != tok:
|
| 441 |
+
bad.add(no_dash)
|
| 442 |
+
return bad
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
def _filter_and_dedupe(
|
| 446 |
+
candidates: list[tuple[str, str]],
|
| 447 |
+
standards_tokens: set[str] | None = None,
|
| 448 |
+
) -> list[str]:
|
| 449 |
"""Apply rejection rules + dedupe case-insensitively, preserving order.
|
| 450 |
Also drops substring duplicates: if 'KB419' and 'KB419-1' are both present,
|
| 451 |
+
keep only the longer one (avoid over-counting when codes get split).
|
| 452 |
+
`standards_tokens` is an optional set of uppercased tokens that were
|
| 453 |
+
detected as standards references in the document text β these are rejected
|
| 454 |
+
regardless of their shape."""
|
| 455 |
+
standards_tokens = standards_tokens or set()
|
| 456 |
seen: dict[str, str] = {} # upper -> original-cased
|
| 457 |
|
| 458 |
for code, _source in candidates:
|
|
|
|
| 461 |
|
| 462 |
if upper in seen:
|
| 463 |
continue
|
| 464 |
+
if upper in standards_tokens:
|
| 465 |
+
continue
|
| 466 |
if not _is_plausible_heat_code(code):
|
| 467 |
continue
|
| 468 |
seen[upper] = code
|
|
|
|
| 487 |
return False
|
| 488 |
if _YEAR_RE.match(upper):
|
| 489 |
return False
|
| 490 |
+
# Material-spec pattern (A106B, A105N, A234B) β letter A + 3 digits +
|
| 491 |
+
# single trailing letter. Real heat codes like A8633 have 4 digits and
|
| 492 |
+
# no trailing letter, so they're not affected.
|
| 493 |
+
if _MATERIAL_SPEC_RE.match(upper):
|
| 494 |
+
return False
|
| 495 |
# Pipe-spec rejection (40S, 304L, 80SS-style schedule/grade designators).
|
| 496 |
if _PIPE_SPEC_RE.match(upper):
|
| 497 |
return False
|