Avinash Nalla commited on
Commit
cd10e70
Β·
1 Parent(s): 0737b39

Tighten heat_code extractor: reject standards numbers + material specs

Browse files

Production extractor was returning 8 codes from 10720.pdf when only
2 are real heat codes. False positives were fragments of standards
references that the OCR fallback was grabbing because they appear
near 'Heat Code' headers in the doc text:
- 0103 from 'NACE MR 0103'
- 0175 from 'MR 0175'
- 2015 from 'ISO 9001:2015'
- 15156 from 'ISO 15156'
- 17945 from 'ISO 17945'
- A106B from '1/2 X 4 XH A106B SMLS TBE NIPPLE' (material spec)

Three fixes:
1) Expand year regex from 20[2-3]\d to 20[0-3]\d (catches 2015 etc).
2) Add material-spec pattern A\d{3}[A-Z] for A106B / A105N / A234B.
Distinct from real heat codes like A8633 (4 digits, no trailing
letter) so no regression on those.
3) New _collect_standards_numbers() scans the document text for
ISO/NACE/ASTM/ASME/EN/DIN/API/AISI/JIS/MSS/PED prefixes and
adds the trailing standard numbers to a reject set. The filter
rejects candidates that match any of those tokens.

Validated against all 10 sample MTRs β€” A8633, KB419-1 (six codes),
D460, SD55822, Y211213D40-1, 24L0604, VC9387 all still extract
correctly. Forced-OCR-path test on 10720.pdf now returns just
['10720', '114420'] β€” all 6 false positives killed.

Files changed (1) hide show
  1. app/lib/utils/heat_code.py +86 -5
app/lib/utils/heat_code.py CHANGED
@@ -92,8 +92,38 @@ _REJECT_WORDS: frozenset[str] = frozenset({
92
  # Spec number patterns to reject (e.g., A312, SA-105, A105-21).
93
  _SPEC_NUM_RE = re.compile(r"^[A-Z]{1,4}-?\d{2,5}(-\d{1,3})?$", re.IGNORECASE)
94
 
95
- # Year-like standalone tokens (2020-2030).
96
- _YEAR_RE = re.compile(r"^(20[2-3]\d)$")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
 
98
 
99
  # ─── Public API ────────────────────────────────────────────────────────────────
@@ -146,8 +176,19 @@ def extract_heat_codes(file_bytes: bytes, filename: str | None = None) -> list[s
146
  except Exception as e:
147
  logger.debug("OCR fallback failed: %s", e)
148
 
 
 
 
 
 
 
 
 
 
 
 
149
  # Filter + dedupe (case-insensitive on code; preserve first-seen casing)
150
- filtered = _filter_and_dedupe(candidates)
151
 
152
  if logger.isEnabledFor(logging.DEBUG):
153
  logger.debug(
@@ -375,10 +416,43 @@ def _tokenize_cell(cell_text: str) -> list[str]:
375
  return [t.strip() for t in re.split(r"[\s,;|]+", cell_text) if t.strip()]
376
 
377
 
378
- def _filter_and_dedupe(candidates: list[tuple[str, str]]) -> list[str]:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
379
  """Apply rejection rules + dedupe case-insensitively, preserving order.
380
  Also drops substring duplicates: if 'KB419' and 'KB419-1' are both present,
381
- keep only the longer one (avoid over-counting when codes get split)."""
 
 
 
 
382
  seen: dict[str, str] = {} # upper -> original-cased
383
 
384
  for code, _source in candidates:
@@ -387,6 +461,8 @@ def _filter_and_dedupe(candidates: list[tuple[str, str]]) -> list[str]:
387
 
388
  if upper in seen:
389
  continue
 
 
390
  if not _is_plausible_heat_code(code):
391
  continue
392
  seen[upper] = code
@@ -411,6 +487,11 @@ def _is_plausible_heat_code(code: str) -> bool:
411
  return False
412
  if _YEAR_RE.match(upper):
413
  return False
 
 
 
 
 
414
  # Pipe-spec rejection (40S, 304L, 80SS-style schedule/grade designators).
415
  if _PIPE_SPEC_RE.match(upper):
416
  return False
 
92
  # Spec number patterns to reject (e.g., A312, SA-105, A105-21).
93
  _SPEC_NUM_RE = re.compile(r"^[A-Z]{1,4}-?\d{2,5}(-\d{1,3})?$", re.IGNORECASE)
94
 
95
+ # Year-like standalone tokens (2010-2039). Expanded from 2020-2030 because
96
+ # MTRs reference older standards years like "ISO 9001:2015" and Tesseract
97
+ # may misread small text producing tokens in this range.
98
+ _YEAR_RE = re.compile(r"^(20[0-3]\d)$")
99
+
100
+ # Material-spec pattern: "A106B", "A105N", "A234B" β€” letter A + 3 digits +
101
+ # single trailing letter. ASTM material grade specs. Distinct from real
102
+ # heat codes like "A8633" (letter + 4 digits, no trailing letter).
103
+ _MATERIAL_SPEC_RE = re.compile(r"^A\d{3}[A-Z]$", re.IGNORECASE)
104
+
105
+ # Standards-prefix patterns: anywhere in the document text, if a token
106
+ # follows one of these prefixes, that token is a standards reference, NOT
107
+ # a heat code. Examples: "ISO 15156", "NACE MR 0103", "ASTM A312", "EN 10204".
108
+ # The capture group is the standards number we want to reject.
109
+ _STANDARDS_REFERENCE_RES: tuple[re.Pattern[str], ...] = (
110
+ re.compile(r"\bISO\s+(\d{3,6})\b", re.IGNORECASE),
111
+ re.compile(r"\bNACE\s*MR\s*(\d{3,6})\b", re.IGNORECASE),
112
+ re.compile(r"\bMR\s*(\d{3,6})\b", re.IGNORECASE),
113
+ re.compile(r"\bASTM\s+([A-Z]?\d{3,5}[A-Z]?(?:-\d{1,3})?)\b", re.IGNORECASE),
114
+ re.compile(r"\bASME\s+(?:SA-?)?([A-Z]?\d{2,5}[A-Z]?(?:[.\-]\d{1,4})?)\b", re.IGNORECASE),
115
+ re.compile(r"\bEN\s+(\d{3,6}(?::\d{4})?)\b", re.IGNORECASE),
116
+ re.compile(r"\bDIN\s+(\d{3,6})\b", re.IGNORECASE),
117
+ re.compile(r"\bAPI\s+(\d+L?)\b", re.IGNORECASE),
118
+ re.compile(r"\bAISI\s+(\d{3,4}[A-Z]?)\b", re.IGNORECASE),
119
+ re.compile(r"\bJIS\s+(\w+)\b", re.IGNORECASE),
120
+ re.compile(r"\bMSS\s*SP\s*(\d{1,4})\b", re.IGNORECASE),
121
+ re.compile(r"\bPED\s+(\d{4}/\d{2}/[A-Z]+)\b", re.IGNORECASE),
122
+ # "ISO 9001:2015" β€” both numbers are bad
123
+ re.compile(r"(\d{4}):(\d{4})"),
124
+ # "A106/A106M-18" or "SA106/SA106M" β€” slash-separated material specs
125
+ re.compile(r"\b([A-Z]?\d{3,4}[A-Z]?(?:-\d{1,3})?)/([A-Z]?\d{3,4}[A-Z]?(?:-\d{1,3})?)\b"),
126
+ )
127
 
128
 
129
  # ─── Public API ────────────────────────────────────────────────────────────────
 
176
  except Exception as e:
177
  logger.debug("OCR fallback failed: %s", e)
178
 
179
+ # Build the set of tokens that appear as standards-references in the
180
+ # document text. Combine text_layer + ocr_text if both were collected.
181
+ combined_text = text_layer
182
+ if not candidates or len(text_layer) < 500:
183
+ # OCR fallback may have produced text we should also scan.
184
+ try:
185
+ combined_text = (text_layer or "") + "\n" + _ocr_text(file_bytes)
186
+ except Exception:
187
+ pass
188
+ standards_tokens = _collect_standards_numbers(combined_text)
189
+
190
  # Filter + dedupe (case-insensitive on code; preserve first-seen casing)
191
+ filtered = _filter_and_dedupe(candidates, standards_tokens)
192
 
193
  if logger.isEnabledFor(logging.DEBUG):
194
  logger.debug(
 
416
  return [t.strip() for t in re.split(r"[\s,;|]+", cell_text) if t.strip()]
417
 
418
 
419
+ def _collect_standards_numbers(doc_text: str) -> set[str]:
420
+ """Scan document text for standards references (ISO 15156, NACE MR 0103,
421
+ ASTM A312, EN 10204, etc.) and return the set of standard-number tokens
422
+ that should never be treated as heat codes.
423
+
424
+ Returned tokens are uppercased and stripped of leading non-alphanumerics
425
+ so they can be compared directly against `code.upper()` in the filter.
426
+ """
427
+ bad: set[str] = set()
428
+ if not doc_text:
429
+ return bad
430
+ for pat in _STANDARDS_REFERENCE_RES:
431
+ for m in pat.finditer(doc_text):
432
+ for grp in m.groups():
433
+ if not grp:
434
+ continue
435
+ tok = re.sub(r"^[^A-Za-z0-9]+", "", grp).upper()
436
+ if tok:
437
+ bad.add(tok)
438
+ # Also reject the no-dash form (e.g., "A106-18" β†’ also drop "A106")
439
+ no_dash = tok.split("-")[0]
440
+ if no_dash != tok:
441
+ bad.add(no_dash)
442
+ return bad
443
+
444
+
445
+ def _filter_and_dedupe(
446
+ candidates: list[tuple[str, str]],
447
+ standards_tokens: set[str] | None = None,
448
+ ) -> list[str]:
449
  """Apply rejection rules + dedupe case-insensitively, preserving order.
450
  Also drops substring duplicates: if 'KB419' and 'KB419-1' are both present,
451
+ keep only the longer one (avoid over-counting when codes get split).
452
+ `standards_tokens` is an optional set of uppercased tokens that were
453
+ detected as standards references in the document text β€” these are rejected
454
+ regardless of their shape."""
455
+ standards_tokens = standards_tokens or set()
456
  seen: dict[str, str] = {} # upper -> original-cased
457
 
458
  for code, _source in candidates:
 
461
 
462
  if upper in seen:
463
  continue
464
+ if upper in standards_tokens:
465
+ continue
466
  if not _is_plausible_heat_code(code):
467
  continue
468
  seen[upper] = code
 
487
  return False
488
  if _YEAR_RE.match(upper):
489
  return False
490
+ # Material-spec pattern (A106B, A105N, A234B) β€” letter A + 3 digits +
491
+ # single trailing letter. Real heat codes like A8633 have 4 digits and
492
+ # no trailing letter, so they're not affected.
493
+ if _MATERIAL_SPEC_RE.match(upper):
494
+ return False
495
  # Pipe-spec rejection (40S, 304L, 80SS-style schedule/grade designators).
496
  if _PIPE_SPEC_RE.match(upper):
497
  return False