Upload 4 files

README.md

@@ -1,147 +1,41 @@
 ---
-title: Bank Statement AI Mode Extractor
-emoji: 🧾
-colorFrom: green
-colorTo: blue
-sdk: docker
+title: Bank PDF To Transaction JSON
+eemoji: 🧾
+colorFrom: blue
+colorTo: green
+sdk: gradio
+sdk_version: 4.44.0
+app_file: app.py
 pinned: false
 ---
 
-# Bank Statement AI Mode Extractor
+# Free Bank Statement PDF → Transaction JSON
 
-Local FastAPI app that:
+Modified from the MinerU Space idea, but focused on bank transaction extraction.
 
-1. Accepts a PDF bank statement from the frontend.
-2. Converts every PDF page into a PNG image.
-3. Sends page images to Google AI Mode in parallel waves, one page per tab.
-4. Saves every batch JSON response.
-5. Combines all batch responses into one JSON file.
-
-Google AI Mode does not provide a stable public browser automation API, so this app uses Playwright Chromium. In Docker Spaces it runs as a headed browser inside `xvfb-run`.
-
-## Setup
-
-```powershell
-python -m venv .venv
-.\.venv\Scripts\Activate.ps1
-pip install -r requirements.txt
-python -m playwright install chromium
-```
-
-## Run
-
-```powershell
-uvicorn app.main:app --reload --port 8000
-```
-
-Open:
-
-```text
-http://127.0.0.1:8000
-```
-
-## Important
-
-- Hugging Face Spaces should be created with `sdk: docker`.
-- The frontend uploads only `pdf` and optional `pwd`.
-- The frontend polls `/api/jobs/{job_id}` for job status.
-- Public deployment may be blocked by Google AI Mode bot checks, regional availability, or account/login requirements.
-- If Google changes AI Mode controls, adjust the AI Mode URL or selectors in `app/google_ai_mode.py`.
-- Output files are stored under `runs/<job_id>/`.
-
-## Hugging Face Deployment
-
-Create a new Space:
-
-```text
-SDK: Docker
-Visibility: Public or Private
-```
-
-Upload this repository's files to the Space, including:
-
-```text
-Dockerfile
-README.md
-requirements.txt
-app.py
-app/
-static/
-```
-
-Do not upload local runtime folders:
-
-```text
-runs/
-browser-profile/
-browser-profile-debug-upload/
-browser-profile-visible-test/
-```
-
-The Dockerfile runs:
-
-```bash
-xvfb-run -a uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-7860}
-```
-
-Supported environment variables:
-
-```text
-API_ONLY=true
-PARALLEL_TABS=10
-BROWSER_HEADLESS=false
-BROWSER_CHANNEL=chromium
-BROWSER_PROXY_SERVER=http://host:port
-BROWSER_PROXY_USERNAME=optional
-BROWSER_PROXY_PASSWORD=optional
-```
-
-Only one proxy server is supported for normal network routing. Rotating proxy/IP logic is intentionally not included.
-
-Status response includes frontend-friendly counters:
+## Output schema
 
 ```json
 {
-  "status": "extracting",
-  "total_pages": 50,
-  "completed_pages": 20,
-  "total_batches": 5,
-  "completed_batches": 2,
-  "active_tabs": 10,
-  "current_pages": "21-30",
-  "page_statuses": {
-    "21": "queued"
-  },
-  "latest_screenshot": "server-side path"
+  "date": "",
+  "narration": "",
+  "voucher_type": "Payment/Receipt",
+  "amount": "",
+  "closing_balance": ""
 }
 ```
 
-Browser view screenshots:
+## Fast mode
 
-```bash
-curl -L "$SPACE_URL/api/jobs/<job_id>/browser-view/latest" -o latest.png
-curl -L "$SPACE_URL/api/jobs/<job_id>/browser-view/page/21" -o page-21.png
-```
+Uses `pdfplumber` first, which is fast and good for text/table PDFs.
 
-## API
+## MinerU fallback
 
-Create a job:
+The app has a checkbox for MinerU fallback, but MinerU is commented in `requirements.txt` because it is heavy. For GPU Space, uncomment:
 
-```bash
-curl -X POST "$SPACE_URL/api/jobs" \
-  -F "pdf=@statement.pdf" \
-  -F "pwd=optional-password"
+```txt
+mineru[pipeline,api]>=2.7.6
+gradio-pdf>=0.0.22
 ```
 
-Check status:
-
-```bash
-curl "$SPACE_URL/api/jobs/<job_id>"
-```
-
-Download:
-
-```bash
-curl -L "$SPACE_URL/api/jobs/<job_id>/download" -o result.json
-curl -L "$SPACE_URL/api/jobs/<job_id>/download.csv" -o result.csv
-```
+Then restart the Space.

app.py

@@ -1 +1,423 @@
-from app.main import app
+# Hugging Face Space: Fast PDF Bank Statement -> JSON
+# Modified from the idea of opendatalab/MinerU Space:
+# - original launches generic mineru-gradio
+# - this app is bank-focused and returns only transactions JSON
+# - fast path: pdfplumber text/table extraction
+# - fallback: optional MinerU CLI if installed/available
+
+import os
+import re
+import json
+import tempfile
+import subprocess
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
+
+import gradio as gr
+import pdfplumber
+
+os.environ.setdefault("GRADIO_SSR_MODE", "false")
+os.environ.setdefault("MINERU_MODEL_SOURCE", "local")
+os.environ.setdefault("MINERU_PDF_RENDER_TIMEOUT", "10")
+
+MONTHS = {
+    "jan": "01", "feb": "02", "mar": "03", "apr": "04", "may": "05", "jun": "06",
+    "jul": "07", "aug": "08", "sep": "09", "oct": "10", "nov": "11", "dec": "12",
+    "january": "01", "february": "02", "march": "03", "april": "04", "june": "06",
+    "july": "07", "august": "08", "september": "09", "october": "10", "november": "11", "december": "12",
+}
+
+SKIP_WORDS = [
+    "opening balance", "page total", "closing balance", "transaction details", "account statement",
+    "txn date", "value date", "narration", "debit", "credit", "balance", "branch name",
+    "account number", "ifsc", "customer id", "from date", "to date", "statement of account",
+]
+
+
+def clean_spaces(s: Any) -> str:
+    if s is None:
+        return ""
+    s = str(s).replace("\n", " ").replace("\t", " ")
+    s = re.sub(r"\s+", " ", s).strip()
+    return s
+
+
+def fix_number_spaces(s: str) -> str:
+    s = clean_spaces(s)
+    # 11962.9 6 -> 11962.96, 675725 6.04 -> 6757256.04
+    old = None
+    while old != s:
+        old = s
+        s = re.sub(r"(?<=\d)\s+(?=\d)", "", s)
+    return s
+
+
+def clean_amount(s: Any) -> str:
+    s = fix_number_spaces(str(s or ""))
+    s = re.sub(r"\b(Cr|Dr)\b", "", s, flags=re.I)
+    s = s.replace(",", "")
+    m = re.search(r"-?\d+(?:\.\d+)?", s)
+    if not m:
+        return ""
+    try:
+        return f"{float(m.group(0)):.2f}"
+    except Exception:
+        return m.group(0)
+
+
+def clean_balance(s: Any) -> str:
+    s = fix_number_spaces(str(s or ""))
+    sign = ""
+    msign = re.search(r"\b(Cr|Dr)\b", s, flags=re.I)
+    if msign:
+        sign = msign.group(1).title()
+    amt = clean_amount(s)
+    return f"{amt} {sign}".strip() if amt else ""
+
+
+def parse_date(s: str) -> str:
+    s = clean_spaces(s)
+    if not s:
+        return ""
+    # Apr/01/2024 or 01/Apr/2024
+    for fmt in ["%b/%d/%Y", "%d/%b/%Y", "%d-%m-%Y", "%d/%m/%Y", "%d-%m-%y", "%d/%m/%y", "%Y-%m-%d"]:
+        try:
+            return datetime.strptime(s, fmt).strftime("%Y-%m-%d")
+        except Exception:
+            pass
+    m = re.search(r"(\d{1,2})[-/ ]([A-Za-z]{3,9})[-/ ](\d{2,4})", s)
+    if m:
+        d, mon, y = m.groups()
+        mm = MONTHS.get(mon.lower()[:3])
+        if mm:
+            if len(y) == 2:
+                y = "20" + y
+            return f"{y}-{mm}-{int(d):02d}"
+    m = re.search(r"([A-Za-z]{3,9})[-/ ](\d{1,2})[-/ ](\d{2,4})", s)
+    if m:
+        mon, d, y = m.groups()
+        mm = MONTHS.get(mon.lower()[:3])
+        if mm:
+            if len(y) == 2:
+                y = "20" + y
+            return f"{y}-{mm}-{int(d):02d}"
+    return s
+
+
+def looks_date(s: str) -> bool:
+    s = clean_spaces(s)
+    return bool(re.search(r"\d{1,2}[-/]\d{1,2}[-/]\d{2,4}|\d{1,2}[-/ ][A-Za-z]{3,9}[-/ ]\d{2,4}|[A-Za-z]{3,9}[-/]\d{1,2}[-/]\d{2,4}", s))
+
+
+def is_skip_row(text: str) -> bool:
+    t = text.lower()
+    return any(w in t for w in SKIP_WORDS)
+
+
+def detect_bank_text(text: str) -> str:
+    t = text.lower()
+    if "vidarbha merchants" in t:
+        return "Vidarbha Merchants Urban Co-op Bank Ltd."
+    if "bank of baroda" in t:
+        return "Bank of Baroda"
+    if "hdfc bank" in t:
+        return "HDFC Bank"
+    if "icici bank" in t:
+        return "ICICI Bank"
+    if "state bank of india" in t or "sbi" in t:
+        return "State Bank of India"
+    return "Unknown"
+
+
+def extract_account_number(text: str) -> str:
+    patterns = [
+        r"Account Number\s*[:-]?\s*([0-9Xx* -]{6,})",
+        r"A/C Number\s*[:-]?\s*([0-9Xx* -]{6,})",
+        r"Account No\.?\s*[:-]?\s*([0-9Xx* -]{6,})",
+        r"A/C No\.?\s*[:-]?\s*([0-9Xx* -]{6,})",
+    ]
+    for p in patterns:
+        m = re.search(p, text, flags=re.I)
+        if m:
+            return re.sub(r"\s+", "", m.group(1)).strip()
+    return ""
+
+
+def make_txn(date: str, narration: str, debit: str, credit: str, balance: str) -> Optional[Dict[str, str]]:
+    narration = clean_spaces(narration)
+    if not date or not narration or is_skip_row(narration):
+        return None
+    debit_amt = clean_amount(debit)
+    credit_amt = clean_amount(credit)
+    if debit_amt:
+        voucher_type = "Payment"
+        amount = debit_amt
+    elif credit_amt:
+        voucher_type = "Receipt"
+        amount = credit_amt
+    else:
+        return None
+    return {
+        "date": parse_date(date),
+        "narration": narration,
+        "voucher_type": voucher_type,
+        "amount": amount,
+        "closing_balance": clean_balance(balance),
+    }
+
+
+def parse_structured_table(table: List[List[Any]]) -> List[Dict[str, str]]:
+    out: List[Dict[str, str]] = []
+    if not table:
+        return out
+
+    header_idx = -1
+    headers: List[str] = []
+    for i, row in enumerate(table[:8]):
+        cells = [clean_spaces(c).lower() for c in row]
+        joined = " ".join(cells)
+        if ("date" in joined and "balance" in joined) and ("debit" in joined or "withdraw" in joined) and ("credit" in joined or "deposit" in joined):
+            header_idx = i
+            headers = cells
+            break
+
+    if header_idx == -1:
+        return parse_loose_table(table)
+
+    def find_col(*names: str) -> int:
+        for name in names:
+            for idx, h in enumerate(headers):
+                if name in h:
+                    return idx
+        return -1
+
+    date_i = find_col("txn date", "date")
+    narr_i = find_col("narration", "particular", "description")
+    debit_i = find_col("debit", "withdraw")
+    credit_i = find_col("credit", "deposit")
+    bal_i = find_col("balance")
+
+    pending: Optional[Dict[str, str]] = None
+    for row in table[header_idx + 1:]:
+        row = [clean_spaces(c) for c in row]
+        if not any(row):
+            continue
+        row_text = " ".join(row)
+        if is_skip_row(row_text):
+            continue
+
+        date = row[date_i] if 0 <= date_i < len(row) else ""
+        narr = row[narr_i] if 0 <= narr_i < len(row) else ""
+        debit = row[debit_i] if 0 <= debit_i < len(row) else ""
+        credit = row[credit_i] if 0 <= credit_i < len(row) else ""
+        bal = row[bal_i] if 0 <= bal_i < len(row) else ""
+
+        if date and looks_date(date):
+            txn = make_txn(date, narr, debit, credit, bal)
+            if txn:
+                out.append(txn)
+                pending = out[-1]
+            else:
+                pending = None
+        elif pending and narr and not clean_amount(row_text):
+            pending["narration"] = clean_spaces(pending["narration"] + " " + narr)
+    return out
+
+
+def parse_loose_table(table: List[List[Any]]) -> List[Dict[str, str]]:
+    out: List[Dict[str, str]] = []
+    pending: Optional[Dict[str, str]] = None
+    for row in table:
+        cells = [clean_spaces(c) for c in row if clean_spaces(c)]
+        if not cells:
+            continue
+        text = " ".join(cells)
+        if is_skip_row(text):
+            continue
+        if looks_date(cells[0]) and len(cells) >= 3:
+            date = cells[0]
+            # heuristic: last is balance, amount is previous numeric, narration middle
+            balance = cells[-1]
+            nums = [(i, clean_amount(c)) for i, c in enumerate(cells)]
+            nums = [(i, a) for i, a in nums if a]
+            if len(nums) >= 2:
+                amount_i, amount = nums[-2]
+                bal = cells[nums[-1][0]]
+                narration = " ".join(cells[1:amount_i]) or cells[1]
+                # Most loose BOB examples have deposit-only amount; default Receipt.
+                txn = {
+                    "date": parse_date(date),
+                    "narration": narration,
+                    "voucher_type": "Receipt",
+                    "amount": amount,
+                    "closing_balance": clean_balance(bal),
+                }
+                out.append(txn)
+                pending = out[-1]
+        elif pending and not re.search(r"\bpage total\b", text, flags=re.I):
+            # continuation narration line
+            if not clean_amount(text) or text.upper().startswith(("UPI/", "IMPS/", "NEFT/", "RTGS/")):
+                pending["narration"] = clean_spaces(pending["narration"] + " " + text)
+    return out
+
+
+def extract_with_pdfplumber(pdf_path: str, max_pages: int = 30) -> Tuple[List[Dict[str, str]], str, str, int]:
+    transactions: List[Dict[str, str]] = []
+    all_text = []
+    pages_read = 0
+    with pdfplumber.open(pdf_path) as pdf:
+        for page in pdf.pages[:max_pages]:
+            pages_read += 1
+            txt = page.extract_text() or ""
+            all_text.append(txt)
+            tables = page.extract_tables() or []
+            for table in tables:
+                transactions.extend(parse_structured_table(table))
+
+            # fallback line parser if no tables on page
+            if not tables and txt:
+                transactions.extend(parse_text_lines(txt))
+
+    full_text = "\n".join(all_text)
+    return dedupe_transactions(transactions), detect_bank_text(full_text), extract_account_number(full_text), pages_read
+
+
+def parse_text_lines(text: str) -> List[Dict[str, str]]:
+    out: List[Dict[str, str]] = []
+    pending: Optional[Dict[str, str]] = None
+    for line in text.splitlines():
+        line = clean_spaces(line)
+        if not line or is_skip_row(line):
+            continue
+        if looks_date(line):
+            parts = line.split()
+            date_part = parts[0]
+            if not looks_date(date_part) and len(parts) > 1:
+                date_part = " ".join(parts[:3])
+            nums = list(re.finditer(r"(?:\d{1,3}(?:,\d{2,3})+|\d+)(?:\.\d+)?\s*(?:Cr|Dr)?", line, flags=re.I))
+            if len(nums) >= 2:
+                amount_m = nums[-2]
+                balance_m = nums[-1]
+                narr = line[len(date_part):amount_m.start()].strip()
+                debit_credit_part = line[amount_m.start():amount_m.end()]
+                # simple heuristic by Cr/Dr marker near amount
+                voucher = "Payment" if re.search(r"\bDr\b", debit_credit_part, re.I) else "Receipt"
+                txn = {
+                    "date": parse_date(date_part),
+                    "narration": clean_spaces(narr),
+                    "voucher_type": voucher,
+                    "amount": clean_amount(amount_m.group(0)),
+                    "closing_balance": clean_balance(balance_m.group(0)),
+                }
+                if txn["narration"] and txn["amount"]:
+                    out.append(txn)
+                    pending = out[-1]
+        elif pending:
+            pending["narration"] = clean_spaces(pending["narration"] + " " + line)
+    return out
+
+
+def dedupe_transactions(txns: List[Dict[str, str]]) -> List[Dict[str, str]]:
+    seen = set()
+    out = []
+    for t in txns:
+        key = (t.get("date"), t.get("narration"), t.get("voucher_type"), t.get("amount"), t.get("closing_balance"))
+        if key not in seen:
+            seen.add(key)
+            out.append(t)
+    return out
+
+
+def mineru_available() -> bool:
+    try:
+        r = subprocess.run(["bash", "-lc", "command -v mineru || command -v magic-pdf"], capture_output=True, text=True, timeout=3)
+        return r.returncode == 0 and bool(r.stdout.strip())
+    except Exception:
+        return False
+
+
+def extract_with_mineru_cli(pdf_path: str, max_pages: int = 20) -> Tuple[List[Dict[str, str]], str]:
+    """Optional fallback. MinerU CLI output names differ by version, so we scan output md/json/txt."""
+    if not mineru_available():
+        return [], "MinerU CLI not found. Fast pdfplumber path used only."
+    out_dir = tempfile.mkdtemp(prefix="mineru_out_")
+    cmd_candidates = [
+        f"mineru -p '{pdf_path}' -o '{out_dir}'",
+        f"magic-pdf -p '{pdf_path}' -o '{out_dir}' -m auto",
+    ]
+    last_err = ""
+    for cmd in cmd_candidates:
+        try:
+            r = subprocess.run(["bash", "-lc", cmd], capture_output=True, text=True, timeout=240)
+            if r.returncode == 0:
+                texts = []
+                for root, _, files in os.walk(out_dir):
+                    for fn in files:
+                        if fn.lower().endswith((".md", ".txt", ".json")):
+                            p = os.path.join(root, fn)
+                            try:
+                                texts.append(open(p, "r", encoding="utf-8", errors="ignore").read())
+                            except Exception:
+                                pass
+                parsed = parse_text_lines("\n".join(texts))
+                return dedupe_transactions(parsed), "MinerU fallback completed."
+            last_err = r.stderr[-1000:] or r.stdout[-1000:]
+        except Exception as e:
+            last_err = str(e)
+    return [], "MinerU fallback failed: " + last_err[:500]
+
+
+def extract_pdf(pdf_file, max_pages: int, use_mineru_fallback: bool):
+    if pdf_file is None:
+        return {"success": False, "error": "Please upload a PDF."}, ""
+    pdf_path = pdf_file.name if hasattr(pdf_file, "name") else str(pdf_file)
+    try:
+        txns, bank, acc, pages = extract_with_pdfplumber(pdf_path, max_pages=max_pages)
+        engine = "pdfplumber-fast"
+        note = ""
+        if use_mineru_fallback and len(txns) == 0:
+            miner_txns, note = extract_with_mineru_cli(pdf_path, max_pages=max_pages)
+            if miner_txns:
+                txns = miner_txns
+                engine = "mineru-fallback"
+
+        result = {
+            "success": True,
+            "engine": engine,
+            "pages_read": pages,
+            "bank_name": bank,
+            "account_number": acc,
+            "count": len(txns),
+            "transactions": txns,
+        }
+        if note:
+            result["note"] = note
+        json_text = json.dumps(result, ensure_ascii=False, indent=2)
+        out_path = os.path.join(tempfile.gettempdir(), "transactions.json")
+        with open(out_path, "w", encoding="utf-8") as f:
+            f.write(json_text)
+        return result, out_path
+    except Exception as e:
+        return {"success": False, "error": str(e)}, ""
+
+
+CSS = """
+.gradio-container {max-width: 1100px !important}
+#title {text-align:center}
+"""
+
+with gr.Blocks(css=CSS, title="Free Bank PDF to JSON") as demo:
+    gr.Markdown("# Free PDF Bank Statement → Transaction JSON", elem_id="title")
+    gr.Markdown("Fast testing app for Hugging Face. Output fields: `date`, `narration`, `voucher_type`, `amount`, `closing_balance` only.")
+    with gr.Row():
+        pdf = gr.File(label="Upload bank statement PDF", file_types=[".pdf"])
+        with gr.Column():
+            max_pages = gr.Slider(1, 100, value=30, step=1, label="Max pages for testing")
+            mineru = gr.Checkbox(value=False, label="Use MinerU fallback only if fast parser finds 0 transactions")
+            btn = gr.Button("Extract JSON", variant="primary")
+    output = gr.JSON(label="Result JSON")
+    download = gr.File(label="Download transactions.json")
+    btn.click(extract_pdf, inputs=[pdf, max_pages, mineru], outputs=[output, download])
+
+if __name__ == "__main__":
+    demo.queue(default_concurrency_limit=2).launch(server_name="0.0.0.0", server_port=int(os.environ.get("PORT", 7860)))

packages.txt

@@ -0,0 +1,3 @@
+fonts-noto-core
+fonts-noto-cjk
+poppler-utils

requirements.txt

@@ -1,6 +1,6 @@
-fastapi==0.115.6
-uvicorn[standard]==0.34.0
-python-multipart==0.0.20
-pymupdf==1.25.1
-Pillow>=12.0.0
-playwright>=1.56.0
+gradio>=4.44.0
+pdfplumber>=0.11.4
+pypdfium2>=4.30.0
+# Optional heavy fallback. Uncomment only on GPU Space / paid Space if you really want full MinerU installed.
+# mineru[pipeline,api]>=2.7.6
+# gradio-pdf>=0.0.22