Upload folder using huggingface_hub

app.py

@@ -101,6 +101,8 @@ def clean_name_text(s: str) -> str:
     s = s.strip(" -:,.")
     s = re.sub(r"\bSG0?(\d+)\b", r"SG\1", s, flags=re.I)
     s = re.sub(r"\b(RR)[\s\-]*2\b", r"RR-2", s, flags=re.I)
+    # fix common OCR mistakes for doctor prefixes
+    s = re.sub(r"\bOR\b", "DR", s)  # sometimes OCR turns 'DR' -> 'OR'
     return s.strip()
 
 # ---------------- image preprocessing ----------------
@@ -138,6 +140,8 @@ def image_to_tsv_cells(cv_img: Any) -> List[Dict[str, Any]]:
     n = len(o.get("text", []))
     for i in range(n):
         raw = o["text"][i]
+        if raw is None:
+            continue
         txt = str(raw).strip()
         if not txt:
             continue
@@ -145,22 +149,13 @@ def image_to_tsv_cells(cv_img: Any) -> List[Dict[str, Any]]:
             conf = float(o["conf"][i]) if o["conf"][i] not in (None, "", "-1") else -1.0
         except Exception:
             conf = -1.0
-        left = int(o["left"][i])
-        top = int(o["top"][i])
-        width = int(o["width"][i])
-        height = int(o["height"][i])
+        left = int(o.get("left", [0])[i])
+        top = int(o.get("top", [0])[i])
+        width = int(o.get("width", [0])[i])
+        height = int(o.get("height", [0])[i])
         center_y = top + height / 2.0
         center_x = left + width / 2.0
-        cells.append({
-            "text": txt,
-            "conf": conf,
-            "left": left,
-            "top": top,
-            "width": width,
-            "height": height,
-            "center_y": center_y,
-            "center_x": center_x
-        })
+        cells.append({"text": txt, "conf": conf, "left": left, "top": top, "width": width, "height": height, "center_y": center_y, "center_x": center_x})
     return cells
 
 # ---------------- grouping & merge helpers ----------------
@@ -192,6 +187,7 @@ def merge_multiline_names(rows: List[List[Dict[str, Any]]]) -> List[List[Dict[st
         row = rows[i]
         tokens = [c["text"] for c in row]
         has_num = any(is_numeric_token(t) for t in tokens)
+        # if row looks pure text and next row contains numbers but short left text tokens, merge
         if not has_num and i + 1 < len(rows):
             next_row = rows[i+1]
             next_tokens = [c["text"] for c in next_row]
@@ -210,12 +206,34 @@ def merge_multiline_names(rows: List[List[Dict[str, Any]]]) -> List[List[Dict[st
                 merged.append(sorted(merged_row, key=lambda cc: cc["left"]))
                 i += 2
                 continue
+        # Additional merge: If a row ends with a trailing token like a doctor's name line with single token and next row also text, merge (helps names split across 2+ lines)
+        if not has_num and i + 1 < len(rows):
+            next_row = rows[i+1]
+            next_tokens = [c["text"] for c in next_row]
+            next_has_num = any(is_numeric_token(t) for t in next_tokens)
+            if not next_has_num and len(tokens) <= 3 and len(next_tokens) <= 3:
+                # merge both textual lines into one (keeps relative left ordering by shifting)
+                merged_row = []
+                min_left = min((c["left"] for c in next_row + row), default=0)
+                offset = 10
+                for c in row + next_row:
+                    newc = c.copy()
+                    if newc["left"] > min_left:
+                        newc["left"] = newc["left"]
+                    else:
+                        newc["left"] = min_left - offset
+                    newc["center_x"] = newc["left"] + newc.get("width", 0) / 2.0
+                    merged_row.append(newc)
+                    offset += 5
+                merged.append(sorted(merged_row, key=lambda cc: cc["left"]))
+                i += 2
+                continue
         merged.append(row)
         i += 1
     return merged
 
 # ---------------- numeric column detection ----------------
-# >>> FIX START — replaced rigid 50px with adaptive clustering
+# >>> CHANGE: adaptive clustering (restored to conservative adaptive threshold)
 def detect_numeric_columns(cells: List[Dict[str, Any]], max_columns: int = 4) -> List[float]:
     xs = [c["center_x"] for c in cells if is_numeric_token(c["text"])]
     if not xs:
@@ -225,9 +243,10 @@ def detect_numeric_columns(cells: List[Dict[str, Any]], max_columns: int = 4) ->
         return [xs[0]]
     gaps = [xs[i+1] - xs[i] for i in range(len(xs)-1)]
     mean_gap = float(np.mean(gaps))
-    std_gap  = float(np.std(gaps)) if len(gaps) > 1 else 0
+    std_gap = float(np.std(gaps)) if len(gaps) > 1 else 0.0
     gap_thresh = max(30.0, mean_gap + 0.6 * std_gap)
-    clusters, curr = [], [xs[0]]
+    clusters = []
+    curr = [xs[0]]
     for i, g in enumerate(gaps):
         if g > gap_thresh and len(clusters) < (max_columns - 1):
             clusters.append(curr)
@@ -239,7 +258,6 @@ def detect_numeric_columns(cells: List[Dict[str, Any]], max_columns: int = 4) ->
     if len(centers) > max_columns:
         centers = centers[-max_columns:]
     return sorted(centers)
-# >>> FIX END
 
 def assign_token_to_column(token_x: float, column_centers: List[float]) -> Optional[int]:
     if not column_centers:
@@ -248,7 +266,6 @@ def assign_token_to_column(token_x: float, column_centers: List[float]) -> Optio
     return int(np.argmin(distances))
 
 # ---------------- parsing rows into items ----------------
-
 def parse_rows_with_columns(rows: List[List[Dict[str, Any]]], page_cells: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
     parsed_items = []
     rows = merge_multiline_names(rows)
@@ -258,42 +275,44 @@ def parse_rows_with_columns(rows: List[List[Dict[str, Any]]], page_cells: List[D
         tokens = [c["text"] for c in row]
         if not tokens:
             continue
+        joined_lower = " ".join(tokens).lower()
+        if FOOTER_KEYWORDS.search(joined_lower) and not any(is_numeric_token(t) for t in tokens):
+            continue
         if all(not is_numeric_token(t) for t in tokens):
             continue
 
-        # >>> FIX START — build numeric token list for inference
+        # gather numeric candidates (unique, filtered)
         numeric_values = []
         for t in tokens:
             if is_numeric_token(t):
                 v = normalize_num_str(t)
                 if v is not None:
                     numeric_values.append(float(v))
-        # >>> FIX END
+        # de-duplicate and sort descending (larger candidates first)
+        numeric_values = sorted(list({int(x) if float(x).is_integer() else x for x in numeric_values}), reverse=True)
 
         if column_centers:
             left_text_parts = []
             numeric_bucket_map = {i: [] for i in range(len(column_centers))}
-
             for c in row:
                 t = c["text"]
+                cx = c["center_x"]
                 if is_numeric_token(t):
-                    col_idx = assign_token_to_column(c["center_x"], column_centers)
+                    col_idx = assign_token_to_column(cx, column_centers)
                     if col_idx is None:
-                        numeric_bucket_map[len(column_centers)-1].append(t)
+                        numeric_bucket_map[len(column_centers) - 1].append(t)
                     else:
                         numeric_bucket_map[col_idx].append(t)
                 else:
                     left_text_parts.append(t)
-
             raw_name = " ".join(left_text_parts).strip()
-            name = clean_name_text(raw_name)
+            name = clean_name_text(raw_name) if raw_name else ""
 
             num_cols = len(column_centers)
             def get_bucket(idx):
                 vals = numeric_bucket_map.get(idx, [])
                 return vals[-1] if vals else None
 
-            # base extraction
             amount = normalize_num_str(get_bucket(num_cols - 1)) if num_cols >= 1 else None
             rate   = normalize_num_str(get_bucket(num_cols - 2)) if num_cols >= 2 else None
             qty    = normalize_num_str(get_bucket(num_cols - 3)) if num_cols >= 3 else None
@@ -302,78 +321,127 @@ def parse_rows_with_columns(rows: List[List[Dict[str, Any]]], page_cells: List[D
                 for t in reversed(tokens):
                     if is_numeric_token(t):
                         amount = normalize_num_str(t)
-                        break
+                        if amount is not None:
+                            break
 
-            # >>> FIX START — strong inference block
+            # >>> CHANGE: safer inference — skip tiny candidates like 1, enforce qty bounds, require close ratio
             if amount is not None and numeric_values:
-                # Look for: amount / candidate_rate ≈ integer
+                # Only accept candidate as rate if candidate >= 2 (or amount is tiny) and candidate < amount
                 for cand in numeric_values:
-                    if cand == 0 or cand == amount:
+                    try:
+                        cand_float = float(cand)
+                    except:
+                        continue
+                    if cand_float <= 1.0:
+                        continue
+                    if amount <= 5 and cand_float < 1.0:
+                        continue
+                    if cand_float >= amount:
+                        continue
+                    ratio = amount / cand_float if cand_float else None
+                    if ratio is None:
                         continue
-                    ratio = amount / cand
                     r = round(ratio)
-                    if 1 <= r <= 200 and abs(ratio - r) <= max(0.04*r, 0.2):
-                        rate = cand
-                        qty  = float(r)
-                        break
-            # >>> FIX END
-
-            # fallback inference
-            if (rate is None or rate == 0) and qty:
+                    if r < 1 or r > 200:
+                        continue
+                    # require relative closeness threshold (adaptive)
+                    if abs(ratio - r) <= max(0.03 * r, 0.15):
+                        # Accept only if qty reasonable (<=100)
+                        if r <= 100:
+                            rate = cand_float
+                            qty = float(r)
+                            break
+
+            # fallback compute rate if qty found but rate missing
+            if (rate is None or rate == 0) and qty and qty != 0 and amount is not None:
                 try:
-                    rate = amount / qty
-                except:
+                    candidate_rate = amount / qty
+                    # require candidate_rate > 1 (avoid tiny rates) and reasonable
+                    if candidate_rate >= 2:
+                        rate = candidate_rate
+                except Exception:
                     pass
 
+            # final defaults
             if qty is None:
                 qty = 1.0
 
-            # cleanup
-            try: amount = float(round(amount,2))
-            except: continue
-            try: rate = float(round(rate,2)) if rate else 0.0
-            except: rate = 0.0
-            try: qty = float(qty)
-            except: qty = 1.0
+            # normalize and sanity-check
+            try:
+                amount = float(round(amount, 2))
+            except Exception:
+                continue
+            try:
+                rate = float(round(rate, 2)) if rate is not None else 0.0
+            except Exception:
+                rate = 0.0
+            try:
+                qty = float(qty)
+            except Exception:
+                qty = 1.0
 
             parsed_items.append({
                 "item_name": name if name else "UNKNOWN",
                 "item_amount": amount,
-                "item_rate": rate,
-                "item_quantity": qty
+                "item_rate": rate if rate is not None else 0.0,
+                "item_quantity": qty if qty is not None else 1.0,
             })
 
         else:
-            numeric_idxs = [i for i,t in enumerate(tokens) if is_numeric_token(t)]
+            numeric_idxs = [i for i, t in enumerate(tokens) if is_numeric_token(t)]
             if not numeric_idxs:
                 continue
-
             last = numeric_idxs[-1]
-            amount = normalize_num_str(tokens[last])
-            if amount is None:
+            amt = normalize_num_str(tokens[last])
+            if amt is None:
+                continue
+            name = " ".join(tokens[:last]).strip()
+            if not name:
                 continue
+            rate = None; qty = None
 
-            name = clean_name_text(" ".join(tokens[:last]).strip())
-            rate = 0.0
-            qty = 1.0
+            # try to pick rate/qty from previous numeric tokens (right-to-left)
+            # and use the safer inference logic (ignore candidate == 1)
+            right_nums = []
+            for i in numeric_idxs:
+                v = normalize_num_str(tokens[i])
+                if v is not None:
+                    right_nums.append(float(v))
+            right_nums = sorted(list({int(x) if float(x).is_integer() else x for x in right_nums}), reverse=True)
+
+            # attempt direct mapping: last numeric = amount, previous maybe rate / qty
+            if len(right_nums) >= 2:
+                cand = right_nums[1]
+                if float(cand) > 1 and float(cand) < float(amt):
+                    # check ratio
+                    ratio = float(amt) / float(cand) if cand else None
+                    if ratio:
+                        r = round(ratio)
+                        if 1 <= r <= 200 and abs(ratio - r) <= max(0.03 * r, 0.15) and r <= 100:
+                            rate = float(cand)
+                            qty = float(r)
+            # fallback: conservative search like above
+            if rate is None and right_nums:
+                for cand in right_nums:
+                    if cand <= 1.0 or cand >= float(amt):
+                        continue
+                    ratio = float(amt) / float(cand)
+                    r = round(ratio)
+                    if 1 <= r <= 100 and abs(ratio - r) <= max(0.03 * r, 0.15):
+                        rate = float(cand)
+                        qty = float(r)
+                        break
 
-            # >>> FIX START — fallback inference also upgraded
-            for cand in numeric_values:
-                if cand == 0 or cand == amount:
-                    continue
-                ratio = amount / cand
-                r = round(ratio)
-                if 1 <= r <= 200 and abs(ratio - r) <= max(0.04*r, 0.2):
-                    rate = cand
-                    qty = float(r)
-                    break
-            # >>> FIX END
+            if qty is None:
+                qty = 1.0
+            if rate is None:
+                rate = 0.0
 
             parsed_items.append({
-                "item_name": name,
-                "item_amount": float(round(amount,2)),
-                "item_rate": float(round(rate,2)),
-                "item_quantity": float(qty)
+                "item_name": clean_name_text(name),
+                "item_amount": float(round(amt, 2)),
+                "item_rate": float(round(rate, 2)),
+                "item_quantity": float(qty),
             })
 
     return parsed_items
@@ -384,69 +452,144 @@ def dedupe_items(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
     out = []
     for it in items:
         nm = re.sub(r"\s+", " ", it["item_name"].lower()).strip()
-        key = (nm[:120], round(it["item_amount"], 2))
+        key = (nm[:120], round(float(it["item_amount"]), 2))
         if key in seen:
             continue
         seen.add(key)
         out.append(it)
     return out
 
-# ---------------- Gemini refinement (unchanged) ----------------
-def refine_with_gemini(page_items: List[Dict[str, Any]], page_text: str = ""):
+def detect_subtotals_and_totals(rows_texts: List[str]) -> Dict[str, Optional[float]]:
+    subtotal = None; final = None
+    for rt in rows_texts[::-1]:
+        if not rt or rt.strip() == "":
+            continue
+        if TOTAL_KEYWORDS.search(rt):
+            m = NUM_RE.search(rt)
+            if m:
+                v = normalize_num_str(m.group(0))
+                if v is None:
+                    continue
+                if re.search(r"sub", rt, re.I):
+                    if subtotal is None: subtotal = float(round(v, 2))
+                else:
+                    if final is None: final = float(round(v, 2))
+    return {"subtotal": subtotal, "final_total": final}
+
+# ---------------- Gemini refinement (deterministic) ----------------
+def refine_with_gemini(page_items: List[Dict[str, Any]], page_text: str = "") -> Tuple[List[Dict[str, Any]], Dict[str, int]]:
     zero_usage = {"total_tokens": 0, "input_tokens": 0, "output_tokens": 0}
     if not GEMINI_API_KEY or genai is None:
         return page_items, zero_usage
-
     try:
         safe_text = sanitize_ocr_text(page_text)
         system_prompt = (
-            "You are a strict bill-extraction cleaner. Return ONLY a JSON array."
+            "You are a strict bill-extraction cleaner. Return ONLY a JSON array (no explanation, no backticks). "
+            "Each entry must be an object with keys: item_name (string), item_amount (float), item_rate (float), item_quantity (float). "
+            "Do NOT include subtotal or total lines as items. Do not invent items; only clean/fix/normalize the given items."
         )
         user_prompt = (
             f"page_text='''{safe_text}'''\n"
             f"items = {json.dumps(page_items, ensure_ascii=False)}\n\n"
-            "Return only the cleaned JSON array."
+            "Example:\n"
+            "items = [{'item_name':'Consultation Charge | DR PREETHI','item_amount':300.0,'item_rate':0.0,'item_quantity':300.0},\n"
+            "         {'item_name':'Description Qty / Hrs Consultation Rate Discount Net Amt','item_amount':1950.0,'item_rate':1950.0,'item_quantity':1.0}]\n"
+            "=>\n"
+            "[{'item_name':'Consultation Charge | DR PREETHI MARY JOSEPH','item_amount':300.0,'item_rate':300.0,'item_quantity':1.0}]\n\n"
+            "Return only the cleaned JSON array of items."
         )
-
         model = genai.GenerativeModel(GEMINI_MODEL_NAME)
-        response = model.generate_content([
-            {"role": "system", "parts": [system_prompt]},
-            {"role": "user", "parts": [user_prompt]}
-        ], temperature=0.0)
-
+        response = model.generate_content(
+            [
+                {"role": "system", "parts": [system_prompt]},
+                {"role": "user", "parts": [user_prompt]},
+            ],
+            temperature=0.0,
+            max_output_tokens=1000,
+        )
         raw = response.text.strip()
         if raw.startswith("```"):
-            raw = raw.split("```")[1]
+            raw = re.sub(r"^```[a-zA-Z]*", "", raw)
+            raw = re.sub(r"```$", "", raw).strip()
         parsed = json.loads(raw)
-
         if isinstance(parsed, list):
             cleaned = []
             for obj in parsed:
                 try:
                     cleaned.append({
-                        "item_name": str(obj.get("item_name","")).strip(),
-                        "item_amount": float(obj.get("item_amount",0)),
-                        "item_rate": float(obj.get("item_rate",0)),
-                        "item_quantity": float(obj.get("item_quantity",1)),
+                        "item_name": str(obj.get("item_name", "")).strip(),
+                        "item_amount": float(obj.get("item_amount", 0.0)),
+                        "item_rate": float(obj.get("item_rate", 0.0) or 0.0),
+                        "item_quantity": float(obj.get("item_quantity", 1.0) or 1.0),
                     })
-                except:
+                except Exception:
                     continue
             return cleaned, zero_usage
-
         return page_items, zero_usage
-
     except Exception:
         return page_items, zero_usage
 
 # ---------------- header heuristics & final filter ----------------
-def final_item_filter(item, known_page_headers):
-    name = item["item_name"].lower()
-    amt = item["item_amount"]
-    if amt <= 0:
+def looks_like_header_text(txt: str, top_of_page: bool = False) -> bool:
+    if not txt:
+        return False
+    t = re.sub(r"\s+", " ", txt.strip().lower())
+    # exact phrase blacklist
+    if any(h == t for h in HEADER_PHRASES):
+        return True
+    hits = sum(1 for k in HEADER_KEYWORDS if k in t)
+    if hits >= 2:
+        return True
+    tokens = re.split(r"[\s\|,/:]+", t)
+    key_hit_count = sum(1 for tok in tokens if tok in HEADER_KEYWORDS)
+    if key_hit_count >= 3:
+        return True
+    if top_of_page and len(tokens) <= 10 and key_hit_count >= 2:
+        return True
+    if ("rate" in t or "net" in t) and "amt" in t and not any(ch.isdigit() for ch in t):
+        return True
+    if t.startswith("description") or t.startswith("qty") or t.startswith("qty /"):
+        return True
+    return False
+
+def final_item_filter(item: Dict[str, Any], known_page_headers: List[str] = [], other_item_names: List[str] = []) -> bool:
+    name = (item.get("item_name") or "").strip()
+    if not name:
+        return False
+    ln = name.lower()
+    # header exact detection
+    for h in known_page_headers:
+        if h and h.strip() and h.strip().lower() in ln:
+            return False
+    if FOOTER_KEYWORDS.search(ln):
+        return False
+    if item.get("item_amount", 0) > 1_000_000:
         return False
-    if FOOTER_KEYWORDS.search(name):
+    if len(name) <= 2 and not re.search(r"[a-zA-Z]", name):
         return False
-    if any(h in name for h in known_page_headers):
+    # avoid pure section headers (short & header words)
+    words = ln.split()
+    header_word_hits = sum(1 for k in HEADER_KEYWORDS if k in ln)
+    if header_word_hits >= 1 and len(words) <= 3:
+        # if page contains more detailed items with 'room'/'rent'/'nursing' etc, remove this generic header
+        lower_other = " ".join(other_item_names).lower()
+        if any(k in lower_other for k in ["room", "rent", "nursing", "ward", "surgeon", "anaes", "ot", "charges", "procedure", "radiology"]):
+            return False
+        # also if name is exactly one of the short header words, drop
+        if ln in ("charge", "charges", "services", "consultation", "room", "radiology", "surgery"):
+            return False
+    # drop non-informative labels even if they have amount (summary rows)
+    if len(words) <= 4 and re.search(r"\b(charges|services|room|radiolog|laborat|surgery|procedure|rent|nursing)\b", ln):
+        # try to detect if it's a summary (presence of other more specific items)
+        lower_other = " ".join(other_item_names).lower()
+        if any(tok in lower_other for tok in ["rent", "room", "ward", "nursing", "surgeon", "anaes", "ot"]):
+            return False
+    if float(item.get("item_amount", 0)) <= 0.0:
+        return False
+    # sanity check rate vs amount
+    rate = float(item.get("item_rate", 0) or 0)
+    amt = float(item.get("item_amount", 0) or 0)
+    if rate and rate > amt * 10 and amt < 10000:
         return False
     return True
 
@@ -455,42 +598,117 @@ def final_item_filter(item, known_page_headers):
 async def extract_bill_data(payload: BillRequest):
     doc_url = payload.document
     try:
-        resp = requests.get(doc_url, timeout=30)
+        headers = {"User-Agent": "Mozilla/5.0"}
+        resp = requests.get(doc_url, headers=headers, timeout=30)
+        if resp.status_code != 200:
+            raise RuntimeError(f"download failed status={resp.status_code}")
         file_bytes = resp.content
-    except:
-        return {"is_success": False, "data": {}}
+    except Exception:
+        return {"is_success": False, "token_usage": {"total_tokens": 0, "input_tokens": 0, "output_tokens": 0}, "data": {"pagewise_line_items": [], "total_item_count": 0}}
 
-    if doc_url.lower().endswith(".pdf"):
-        images = convert_from_bytes(file_bytes)
-    else:
-        images = [Image.open(BytesIO(file_bytes))]
+    images = []
+    clean_url = doc_url.split("?", 1)[0].lower()
+    try:
+        if clean_url.endswith(".pdf"):
+            images = convert_from_bytes(file_bytes)
+        elif any(clean_url.endswith(ext) for ext in [".png", ".jpg", ".jpeg", ".tiff", ".bmp"]):
+            images = [Image.open(BytesIO(file_bytes))]
+        else:
+            try:
+                images = convert_from_bytes(file_bytes)
+            except Exception:
+                images = []
+    except Exception:
+        images = []
 
     pagewise = []
-    total_items = 0
-
-    for idx, img in enumerate(images, start=1):
-        proc = preprocess_image(img)
-        cells = image_to_tsv_cells(proc)
-        rows = group_cells_into_rows(cells)
-
-        rows_text = [" ".join([c["text"] for c in r]) for r in rows]
-        parsed = parse_rows_with_columns(rows, cells)
-
-        pagewise.append({
-            "page_no": str(idx),
-            "page_type": "Bill Detail",
-            "bill_items": parsed
-        })
-        total_items += len(parsed)
-
-    return {
-        "is_success": True,
-        "data": {
-            "pagewise_line_items": pagewise,
-            "total_item_count": total_items
-        }
-    }
+    cumulative_token_usage = {"total_tokens": 0, "input_tokens": 0, "output_tokens": 0}
+
+    for idx, page_img in enumerate(images, start=1):
+        try:
+            proc = preprocess_image(page_img)
+            cells = image_to_tsv_cells(proc)
+            rows = group_cells_into_rows(cells, y_tolerance=12)
+            rows_texts = [" ".join([c["text"] for c in r]).strip() for r in rows]
+
+            # === HEADER PREFILTER: remove header-like rows anywhere on page ===
+            rows_filtered = []
+            for i, (r, rt) in enumerate(zip(rows, rows_texts)):
+                top_flag = (i < 6)
+                rt_norm = sanitize_ocr_text(rt).lower()
+                if looks_like_header_text(rt_norm, top_of_page=top_flag):
+                    continue
+                if any(h in rt_norm for h in HEADER_PHRASES):
+                    continue
+                rows_filtered.append(r)
+            # recompute row texts and a simple page_text
+            rows = rows_filtered
+            rows_texts = [" ".join([c["text"] for c in r]).strip() for r in rows]
+            page_text = sanitize_ocr_text(" ".join(rows_texts))
+
+            # detect page-level top headers (for final filtering)
+            top_headers = []
+            for i, rt in enumerate(rows_texts[:6]):
+                if looks_like_header_text(rt, top_of_page=(i < 4)):
+                    top_headers.append(rt.strip().lower())
+
+            parsed_items = parse_rows_with_columns(rows, cells)
+
+            # ALWAYS attempt Gemini refinement if available (deterministic settings)
+            refined_items, token_u = refine_with_gemini(parsed_items, page_text)
+            for k in cumulative_token_usage:
+                cumulative_token_usage[k] += token_u.get(k, 0)
+
+            # Prepare other_item_names for contextual filtering (helps remove generic section headers)
+            other_item_names = [it.get("item_name","") for it in refined_items]
+
+            # final cleaning & dedupe
+            cleaned = [p for p in refined_items if final_item_filter(p, known_page_headers=top_headers, other_item_names=other_item_names)]
+            cleaned = dedupe_items(cleaned)
+            cleaned = [p for p in cleaned if not looks_like_header_text(p["item_name"].lower())]
+
+            page_type = "Bill Detail"
+            page_txt = page_text.lower()
+            if any(x in page_txt for x in ["pharmacy", "medicine", "tablet"]):
+                page_type = "Pharmacy"
+            if "final bill" in page_txt or "grand total" in page_txt:
+                page_type = "Final Bill"
+
+            pagewise.append({"page_no": str(idx), "page_type": page_type, "bill_items": cleaned})
+        except Exception:
+            pagewise.append({"page_no": str(idx), "page_type": "Bill Detail", "bill_items": []})
+            continue
+
+    total_item_count = sum(len(p.get("bill_items", [])) for p in pagewise)
+    if not GEMINI_API_KEY or genai is None:
+        cumulative_token_usage["warning_no_gemini"] = 1
+
+    return {"is_success": True, "token_usage": cumulative_token_usage, "data": {"pagewise_line_items": pagewise, "total_item_count": total_item_count}}
+
+# ---------------- debug TSV ----------------
+@app.post("/debug-tsv")
+async def debug_tsv(payload: BillRequest):
+    doc_url = payload.document
+    try:
+        resp = requests.get(doc_url, timeout=20)
+        if resp.status_code != 200:
+            return {"error": "Download failed"}
+        file_bytes = resp.content
+    except Exception:
+        return {"error": "Download failed"}
+    clean_url = doc_url.split("?", 1)[0].lower()
+    if clean_url.endswith(".pdf"):
+        imgs = convert_from_bytes(file_bytes)
+        img = imgs[0]
+    else:
+        img = Image.open(BytesIO(file_bytes))
+    proc = preprocess_image(img)
+    cells = image_to_tsv_cells(proc)
+    return {"cells": cells}
 
 @app.get("/")
-def health():
-    return {"status": "ok"}
+def health_check():
+    msg = "Bill extraction API (final) live."
+    if not GEMINI_API_KEY or genai is None:
+        msg += " (No GEMINI_API_KEY/configured SDK — LLM refinement skipped.)"
+    return {"status": "ok", "message": msg, "hint": "POST /extract-bill-data with {'document':'<url>'}"}