rehan953 commited on
Commit
25e632e
·
verified ·
1 Parent(s): 60b8c65

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +26 -6
app.py CHANGED
@@ -3114,6 +3114,9 @@ def _dedupe_subset_transaction_tables_html(text: str) -> str:
3114
  - Earlier table is a strict subset of a later one → drop earlier (UCB-style
3115
  "Deposits (continued)" preview before the same rows appear in the full list).
3116
  - Identical row sets → drop the later table.
 
 
 
3117
  """
3118
  if not text or "<table" not in text.lower():
3119
  return text
@@ -3135,22 +3138,39 @@ def _dedupe_subset_transaction_tables_html(text: str) -> str:
3135
  for m in matches:
3136
  grids.append((m, _parse_grid(m.group(0))))
3137
 
3138
- hdr_ok = ["date", "description", "amount"]
3139
-
3140
  def _header_matches(grid):
3141
  if not grid or len(grid) < 2:
3142
  return False
3143
  h = [re.sub(r"\s+", " ", str(c or "").strip().lower()) for c in grid[0]]
3144
- return len(h) == 3 and h == hdr_ok
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3145
 
3146
  def _row_set(g):
3147
  s = set()
3148
  for r in g[1:]:
3149
  if not any(str(c or "").strip() for c in r):
3150
  continue
3151
- t = tuple(str(c or "").strip() for c in r)
3152
- if any(t):
3153
- s.add(t)
 
3154
  return s
3155
 
3156
  drop_idx = set()
 
3114
  - Earlier table is a strict subset of a later one → drop earlier (UCB-style
3115
  "Deposits (continued)" preview before the same rows appear in the full list).
3116
  - Identical row sets → drop the later table.
3117
+
3118
+ Row matching uses normalized keys so OCR variants (e.g. "0 27" vs "027", backticks)
3119
+ do not prevent duplicate tables from being detected.
3120
  """
3121
  if not text or "<table" not in text.lower():
3122
  return text
 
3138
  for m in matches:
3139
  grids.append((m, _parse_grid(m.group(0))))
3140
 
 
 
3141
  def _header_matches(grid):
3142
  if not grid or len(grid) < 2:
3143
  return False
3144
  h = [re.sub(r"\s+", " ", str(c or "").strip().lower()) for c in grid[0]]
3145
+ if len(h) != 3:
3146
+ return False
3147
+ d_ok = h[0] == "date"
3148
+ desc_ok = "description" in h[1]
3149
+ amt_ok = h[2] == "amount"
3150
+ return d_ok and desc_ok and amt_ok
3151
+
3152
+ def _norm_row_key(cells):
3153
+ """Stable key for same logical row despite minor OCR/HTML differences."""
3154
+ parts = [str(c or "").strip() for c in cells[:3]]
3155
+ while len(parts) < 3:
3156
+ parts.append("")
3157
+ date_s, desc_s, amt_s = parts[0], parts[1], parts[2]
3158
+ date_s = re.sub(r"\s+", "", date_s)
3159
+ amt_s = re.sub(r"[\s$,]", "", amt_s).lower().replace("−", "-")
3160
+ desc_s = html.unescape(desc_s)
3161
+ desc_s = desc_s.replace("`", "'").replace("’", "'")
3162
+ desc_s = re.sub(r"\s+", "", desc_s.lower())
3163
+ return (date_s, desc_s, amt_s)
3164
 
3165
  def _row_set(g):
3166
  s = set()
3167
  for r in g[1:]:
3168
  if not any(str(c or "").strip() for c in r):
3169
  continue
3170
+ cells = [str(c or "").strip() for c in r]
3171
+ if not any(cells):
3172
+ continue
3173
+ s.add(_norm_row_key(cells))
3174
  return s
3175
 
3176
  drop_idx = set()