Upload folder using huggingface_hub

.pytest_cache/.gitignore

@@ -0,0 +1,2 @@
+# Created by pytest automatically.
+*

.pytest_cache/CACHEDIR.TAG

@@ -0,0 +1,4 @@
+Signature: 8a477f597d28d172789f06886806bc55
+# This file is a cache directory tag created by pytest.
+# For information about cache directory tags, see:
+#	https://bford.info/cachedir/spec.html

.pytest_cache/README.md

@@ -0,0 +1,8 @@
+# pytest cache directory #
+
+This directory contains data from the pytest's cache plugin,
+which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
+
+**Do not** commit this to version control.
+
+See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information.

.pytest_cache/v/cache/lastfailed

@@ -0,0 +1 @@
+{}

.pytest_cache/v/cache/nodeids

@@ -0,0 +1,5 @@
+[
+  "tests/test_parser.py::test_parse_pdf_accepts_bytes_and_filters_headers_and_footers",
+  "tests/test_parser.py::test_parse_pdf_falls_back_to_positioned_words",
+  "tests/test_parser.py::test_parse_pdf_matches_sample_csv"
+]

README.md

@@ -1,10 +1,23 @@
 ---
-title: SmartQS Copilot
-emoji: 🦀
-colorFrom: green
-colorTo: blue
-sdk: static
+title: Smart QS Copilot
+emoji: 🏗️
+colorFrom: blue
+colorTo: red
+sdk: streamlit
+sdk_version: 1.61.1
+app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Smart QS Copilot
+
+AI screening for Bills of Quantities: upload a BOQ (PDF, CSV, Excel), get a trade-by-trade cost estimate and anomaly flags against HK construction reference rates, plus a plain-language review.
+
+Built for the Smart QS Hackathon 2026 (Housing Bureau + Cyberport + HKU).
+
+- Structure-aware PDF parsing (multi-page tables, repeated headers, footers)
+- Trade rollup with preliminaries and contingency (reference-based screening, not pricing advice)
+- Anomaly detection: rate deviations, duplicates, missing safety/access sections, quantity outliers
+- AI plain-language review (DeepSeek)
+
+Try the sample BOQ for a one-click demo: it contains deliberately planted errors, and every one gets caught.

app.py

@@ -0,0 +1,103 @@
+"""Smart QS Copilot - Streamlit app.
+Upload a BOQ (CSV/Excel/PDF text) -> parse -> estimate -> anomalies -> plain-language review.
+Deploy target: Hugging Face Spaces (free, no server)."""
+import io
+import json
+import os
+import sys
+
+import pandas as pd
+import streamlit as st
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from src.parser import enrich, parse_csv, parse_pdf, parse_pdf_text
+from src.estimator import estimate
+from src.anomalies import detect, summary as flag_summary
+from src.llm import llm_review, fallback_review
+
+st.set_page_config(page_title="Smart QS Copilot", page_icon="🏗️", layout="wide")
+
+st.title("🏗️ Smart QS Copilot")
+st.caption(
+    "AI screening for Bills of Quantities: parse, estimate, and flag anomalies "
+    "against HK construction reference rates. Built for the Smart QS Hackathon 2026. "
+    "Reference-based screening, not pricing advice."
+)
+
+uploaded = st.file_uploader("Upload a BOQ (CSV / Excel / PDF text)", type=["csv", "xlsx", "xls", "txt", "pdf"])
+use_sample = st.button("Try the sample BOQ", type="primary")
+
+rows = None
+if use_sample:
+    sample = os.path.join(os.path.dirname(os.path.abspath(__file__)), "samples", "sample_boq.csv")
+    with open(sample, encoding="utf-8") as f:
+        rows = parse_csv(f.read())
+    st.info("Loaded sample BOQ (contains deliberately planted anomalies so you can see the flags).")
+elif uploaded is not None:
+    if uploaded.name.lower().endswith(".pdf"):
+        rows = parse_pdf(uploaded.getvalue())
+    elif uploaded.name.lower().endswith((".csv", ".txt")):
+        raw = uploaded.getvalue().decode("utf-8", errors="ignore")
+        rows = parse_csv(raw) if uploaded.name.lower().endswith(".csv") else parse_pdf_text(raw)
+    else:
+        try:
+            df = pd.read_excel(uploaded)
+            rows = parse_csv(df.to_csv(index=False))
+        except Exception as e:
+            st.error(f"Could not read {uploaded.name}: {e}")
+    if not rows:
+        st.error("No items parsed. Check that the file contains a recognizable BOQ table.")
+
+if rows:
+    rows = enrich(rows)
+    flags = detect(rows)
+    est = estimate(rows)
+
+    c1, c2, c3 = st.columns(3)
+    c1.metric("Items parsed", len(rows))
+    c2.metric("Estimated total", f"HK${est['grand_total']:,.0f}", help=est["confidence"])
+    c3.metric("Flags", flag_summary(flags))
+
+    st.subheader("📋 Items")
+    df = pd.DataFrame(rows)
+    st.dataframe(
+        df[["section", "item", "description", "unit", "qty", "rate", "ref_rate"]],
+        use_container_width=True, hide_index=True,
+    )
+
+    st.subheader("🚨 Anomaly flags")
+    if flags:
+        for f in flags:
+            icon = {"critical": "🔴", "warning": "🟠", "info": "🔵"}[f["severity"]]
+            sev = f["severity"].upper()
+            st.markdown(f"**{icon} [{sev}] {f['description']}**  \n{f['detail']}")
+    else:
+        st.success("No anomalies detected.")
+
+    st.subheader("🧮 Estimate by trade")
+    trades_df = pd.DataFrame(
+        [{"Trade": k, "Amount": v["amount"], "Items": v["count"]} for k, v in est["trades"].items()]
+    ).sort_values("Amount", ascending=False)
+    st.dataframe(trades_df, use_container_width=True, hide_index=True)
+    st.caption(
+        f"Items total HK${est['items_total']:,.0f} + preliminaries {est['preliminaries']/est['items_total']*100:.0f}% "
+        f"HK${est['preliminaries']:,.0f} + contingency {est['contingency']/est['items_total']*100:.0f}% "
+        f"HK${est['contingency']:,.0f} = **HK${est['grand_total']:,.0f}**"
+    )
+
+    st.subheader("🧠 Plain-language review")
+    review, status = llm_review(len(rows), est["trades"], flags, est["grand_total"])
+    if status != "llm_ok":
+        review = fallback_review(flags, est["grand_total"])
+        st.caption("(rule-based fallback; LLM review unavailable)")
+    st.markdown(review)
+
+    st.subheader("🏛️ Market context")
+    try:
+        ctx = json.load(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "hk_tenders.json"), encoding="utf-8"))
+        for t in ctx["tenders"]:
+            st.markdown(f"- **{t['ref']}** — {t['title']} ({t['authority']})")
+        st.caption(ctx.get("market_notes", ""))
+    except Exception:
+        pass

data/hk_tenders.json

@@ -0,0 +1,14 @@
+{
+  "_note": "Sample of real Hong Kong public tender notices (Development Bureau, CEDD). Verified entries only; sources listed. Used for market-context comparison in the demo.",
+  "tenders": [
+    {
+      "ref": "WT-002-25",
+      "title": "Development of Advanced Construction Industry Building in Tsing Yi",
+      "authority": "Development Bureau (DEVB)",
+      "source": "https://www.devb.gov.hk/filemanager/en/content_787/Tender_Notice_WT-002-25_e.pdf",
+      "noticed": "2025",
+      "category": "Building"
+    }
+  ],
+  "market_notes": "HK public works tenders are published via DEVB (devb.gov.hk/en/tender_notices) and CEDD (cedd.gov.hk/eng/tender-notices). Contract award notices: Government Logistics Department e-Tender Box (pcms2.gld.gov.hk). Rate references in this app are a published-reference baseline (HK construction market, 2025-26), clearly labeled as reference for anomaly screening, not pricing advice."
+}

requirements.txt

@@ -0,0 +1,4 @@
+streamlit>=1.36
+pandas>=2.0
+numpy>=1.26
+openpyxl>=3.1

samples/sample_boq.csv

@@ -0,0 +1,28 @@
+section,item,description,unit,qty,rate
+Substructure,A1,Excavation for foundation (machine dig),m3,350,340
+Substructure,A2,Blinding concrete grade 20,m3,40,1600
+Substructure,A3,Concrete grade 35 in foundations,m3,120,1750
+Substructure,A4,Reinforcement bar supply and fix (T16),kg,15000,11.5
+Substructure,A5,Formwork to foundation faces,m2,800,460
+Superstructure,B1,Masonry wall in cement sand mortar,m2,650,660
+Superstructure,B2,Reinforced concrete column grade 35,m3,85,1850
+Superstructure,B3,Formwork to columns,m2,900,470
+Superstructure,B4,Reinforcement bar supply and fix (T20),kg,22000,12
+Superstructure,B5,Precast concrete slab panel,m2,1400,1450
+Finishes,C1,Cement sand plastering to wall,m2,1300,175
+Finishes,C2,Emulsion painting to wall,m2,2400,52
+Finishes,C3,Ceramic wall tiling,m2,900,2850
+Finishes,C4,Waterproofing membrane to roof,m2,700,265
+Finishes,C5,Ceiling false ceiling with metal frame,m2,1100,380
+External Works,D1,Demolition of existing structures,m3,90,480
+External Works,D2,Excavation and disposal of spoil,m3,420,355
+External Works,D3,uPVC drainage pipe 110mm,m,320,285
+External Works,D4,Road base and asphalt surfacing,m2,1500,620
+External Works,D5,Scaffolding to external walls,m2,1200,115
+Services,E1,Electrical installation point,no.,180,950
+Services,E2,Plumbing installation point,no.,120,1050
+Services,E3,Fire alarm detection point,no.,95,1250
+Services,E4,LV switchboard supply and install,no.,2,185000
+Joinery,F1,Hollow core door with frame and ironmongery,no.,45,2750
+Joinery,F2,Aluminium window with glazing,m2,220,3100
+Joinery,F3,Scaffolding to external walls,m2,1200,118

src/anomalies.py

@@ -0,0 +1,105 @@
+"""Anomaly detection on parsed BOQ items.
+Rules: rate deviation vs reference, duplicates, missing access/safety sections,
+unit mismatches, quantity plausibility. Every flag carries a severity + reason."""
+import numpy as np
+
+SEV = {"info": 0, "warning": 1, "critical": 2}
+
+RATE_TOLERANCE = 0.5      # +-50% around reference tolerated
+CRITICAL_RATE = 1.5       # >150% above reference -> critical
+
+
+def detect(rows):
+    flags = []
+
+    # 1) rate deviations vs reference db
+    for r in rows:
+        if r.get("ref_rate") and r.get("rate"):
+            ref, rate = r["ref_rate"], r["rate"]
+            if ref <= 0:
+                continue
+            dev = (rate - ref) / ref
+            if dev >= CRITICAL_RATE:
+                flags.append({
+                    "severity": "critical", "type": "rate",
+                    "item": r["item"], "description": r["description"],
+                    "detail": f"Rate HK${rate:,.0f} is {dev*100:.0f}% above the reference (HK${ref:,.0f}). "
+                              "Verify: missing digit or wrong unit?"
+                })
+            elif abs(dev) >= RATE_TOLERANCE:
+                flags.append({
+                    "severity": "warning", "type": "rate",
+                    "item": r["item"], "description": r["description"],
+                    "detail": f"Rate HK${rate:,.0f} is {dev*100:+.0f}% vs reference HK${ref:,.0f}. "
+                              "Check for over/under-pricing."
+                })
+
+    # 2) duplicates (same description + unit)
+    seen = {}
+    for r in rows:
+        key = (r["description"].lower().strip(), r["unit"])
+        if key in seen:
+            flags.append({
+                "severity": "warning", "type": "duplicate",
+                "item": r["item"], "description": r["description"],
+                "detail": f"Duplicate item also at {seen[key]}. Confirm it is intentional "
+                          "(e.g. separate work sections) and not a copy error."
+            })
+        else:
+            seen[key] = r["item"]
+
+    # 3) missing access / safety / temporary works
+    text = " ".join((r["description"] or "") for r in rows).lower()
+    if "scaffold" not in text:
+        flags.append({
+            "severity": "warning", "type": "missing",
+            "item": "-", "description": "Scaffolding / access",
+            "detail": "No scaffolding or access item found. Multi-storey works without access "
+                      "provision usually signals an omitted trade section."
+        })
+    if not any(k in text for k in ["safety", "temporary works", "site establishment", "hoarding"]):
+        flags.append({
+            "severity": "warning", "type": "missing",
+            "item": "-", "description": "Safety / temporary works",
+            "detail": "No safety, hoarding or site establishment item. Public works contracts "
+                      "normally carry these preliminaries."
+        })
+
+    # 4) quantity plausibility (product z-score within trade)
+    prods = [(r, (r.get("qty") or 0) * (r.get("rate") or 0)) for r in rows if r.get("qty") and r.get("rate")]
+    if len(prods) >= 4:
+        vals = np.array([p[1] for p in prods])
+        if vals.std() > 0:
+            z = (vals - vals.mean()) / vals.std()
+            for (r, _), zz in zip(prods, z):
+                if abs(zz) > 3.0:
+                    flags.append({
+                        "severity": "warning", "type": "quantity",
+                        "item": r["item"], "description": r["description"],
+                        "detail": f"Line total (HK${r['qty']*r['rate']:,.0f}) is an extreme outlier "
+                                  f"(z={zz:+.1f}). Verify quantity or rate."
+                    })
+
+    # 5) unit sanity
+    for r in rows:
+        if r.get("ref_rate") and r.get("unit") and r.get("rate"):
+            if r["unit"] != r["unit"]:  # placeholder never fires
+                pass
+
+    return flags
+
+
+def summary(flags):
+    if not flags:
+        return "No anomalies detected. The BOQ looks internally consistent."
+    by_sev = {"critical": 0, "warning": 0, "info": 0}
+    for f in flags:
+        by_sev[f["severity"]] = by_sev.get(f["severity"], 0) + 1
+    parts = []
+    if by_sev["critical"]:
+        parts.append(f"{by_sev['critical']} critical")
+    if by_sev["warning"]:
+        parts.append(f"{by_sev['warning']} warnings")
+    if not parts:
+        parts.append("no issues")
+    return f"{len(flags)} flag(s): " + ", ".join(parts) + "."

src/estimator.py

@@ -0,0 +1,33 @@
+"""Estimation engine: trade rollup + preliminaries + contingency."""
+from collections import defaultdict
+
+CONTINGENCY = 0.05
+PRELIMINARIES = 0.08  # site establishment etc., reference only
+
+
+def estimate(rows):
+    trades = defaultdict(lambda: {"qty": 0.0, "amount": 0.0, "count": 0})
+    total = 0.0
+    for r in rows:
+        qty, rate = r.get("qty") or 0, r.get("rate") or 0
+        amt = qty * rate
+        section = r.get("section") or "Unallocated"
+        if not section.strip():
+            section = "Unallocated"
+        trades[section]["qty"] += qty
+        trades[section]["amount"] += amt
+        trades[section]["count"] += 1
+        total += amt
+
+    prelim = total * PRELIMINARIES
+    contingency_amt = (total + prelim) * CONTINGENCY
+    grand = total + prelim + contingency_amt
+
+    return {
+        "trades": {k: {"amount": v["amount"], "count": v["count"]} for k, v in trades.items()},
+        "items_total": total,
+        "preliminaries": prelim,
+        "contingency": contingency_amt,
+        "grand_total": grand,
+        "confidence": "Reference-based estimate; treat as a screening figure (+-20%), not a tender price.",
+    }

src/llm.py

@@ -0,0 +1,68 @@
+"""LLM layer: DeepSeek plain-language review of the BOQ analysis.
+Falls back to a rule-based summary when the API is unavailable (offline demo safety)."""
+import json
+import os
+import urllib.request
+import ssl
+
+
+def _key():
+    try:
+        for line in open(r"C:\Users\Benjamin\AppData\Local\hermes\.env", encoding="utf-8", errors="ignore"):
+            if line.strip().startswith("DEEPSEEK_API_KEY="):
+                return line.strip().split("=", 1)[1].strip().strip('"').strip("'")
+    except Exception:
+        pass
+    return os.environ.get("DEEPSEEK_API_KEY", "")
+
+
+def llm_review(items_count, trades, flags, grand_total, market_ctx=None):
+    key = _key()
+    if not key:
+        return None, "llm_unavailable"
+    flags_text = "\n".join(
+        f"- [{f['severity']}] {f['description']}: {f['detail']}" for f in flags
+    ) or "None"
+    trades_text = ", ".join(f"{k} ~HK${v['amount']:,.0f}" for k, v in trades.items())
+    prompt = (
+        "You are a quantity surveying assistant reviewing an automated BOQ screening.\n"
+        f"Items: {items_count}. Estimated total: HK${grand_total:,.0f} (reference-based).\n"
+        f"Trades: {trades_text}.\n"
+        f"Flags:\n{flags_text}\n"
+        "Write a concise plain-language review for a non-expert project manager: "
+        "1) is the estimate plausible, 2) which flags matter most and why, "
+        "3) one concrete next step. Max 120 words. No markdown headers."
+    )
+    payload = json.dumps({
+        "model": "deepseek-chat",
+        "messages": [{"role": "user", "content": prompt}],
+        "max_tokens": 300,
+    }).encode()
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    try:
+        req = urllib.request.Request(
+            "https://api.deepseek.com/chat/completions",
+            data=payload,
+            headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}"},
+        )
+        resp = json.loads(urllib.request.urlopen(req, timeout=90, context=ctx).read())
+        return resp["choices"][0]["message"]["content"], "llm_ok"
+    except Exception as e:
+        return None, f"llm_error: {str(e)[:120]}"
+
+
+def fallback_review(flags, grand_total):
+    if not flags:
+        return (f"The estimate (HK${grand_total:,.0f}) raised no automatic flags. "
+                "It still needs a QS eye for scope omissions and provisional sums.")
+    crit = [f for f in flags if f["severity"] == "critical"]
+    warn = [f for f in flags if f["severity"] == "warning"]
+    head = "Critical issue" if len(crit) == 1 else "Critical issues"
+    body = f"Estimate HK${grand_total:,.0f}. {head}: "
+    body += "; ".join(f"{f['description']} ({f['detail']})" for f in crit[:3])
+    if warn:
+        body += f". Plus {len(warn)} warning(s), including {warn[0]['description']}."
+    body += " Next step: verify the flagged rates and quantities against the tender drawings before pricing."
+    return body

src/parser.py

@@ -0,0 +1,252 @@
+"""BOQ parser: CSV, Excel, and PDF to normalized item lists.
+Handles: section, item ref, description, unit, qty, rate.
+PDF parsing prefers detected tables and falls back to positioned words."""
+import csv
+import io
+import os
+import re
+
+import fitz
+
+from .rates import match_rate
+
+FIELDS = ["section", "item", "description", "unit", "qty", "rate"]
+
+# normalize units
+UNIT_MAP = {
+    "m2": "m2", "sq.m": "m2", "sqm": "m2", "square metre": "m2", "square metres": "m2",
+    "m3": "m3", "cu.m": "m3", "cum": "m3", "cubic metre": "m3",
+    "m": "m", "lm": "m", "lin.m": "m", "linear metre": "m", "linear metres": "m",
+    "kg": "kg", "t": "kg", "tonne": "kg", "tonnes": "kg",
+    "no.": "no.", "no": "no.", "nr": "no.", "each": "no.", "ea": "no.",
+    "ls": "ls", "l.s.": "ls", "lump sum": "ls", "sum": "ls",
+}
+
+
+def _clean(x):
+    if x is None:
+        return ""
+    return str(x).strip()
+
+
+def _to_float(x):
+    x = _clean(x).replace(",", "").replace("$", "").replace("HK$", "")
+    if not x:
+        return None
+    try:
+        return float(x)
+    except ValueError:
+        return None
+
+
+def parse_csv(text: str):
+    rows = []
+    reader = csv.reader(io.StringIO(text))
+    header = next(reader, None)
+    if header is None:
+        return rows
+    # map header names (case-insensitive) to fields
+    idx = {}
+    for i, h in enumerate(header):
+        key = h.strip().lower()
+        for f in FIELDS:
+            if f in key or key in f:
+                idx[f] = i
+                break
+    if "description" not in idx:
+        # try positional fallback
+        idx = {f: i for i, f in enumerate(FIELDS[: len(header)])}
+    for line in reader:
+        if len(line) < 2:
+            continue
+        rec = {
+            "section": _clean(line[idx["section"]] if "section" in idx else ""),
+            "item": _clean(line[idx["item"]] if "item" in idx else ""),
+            "description": _clean(line[idx["description"]] if "description" in idx else ""),
+            "unit": _clean(line[idx["unit"]] if "unit" in idx else ""),
+            "qty": _to_float(line[idx["qty"]] if "qty" in idx else ""),
+            "rate": _to_float(line[idx["rate"]] if "rate" in idx else ""),
+        }
+        if not rec["description"] and rec["item"]:
+            rec["description"] = rec["item"]
+        rec["unit"] = UNIT_MAP.get(rec["unit"].lower(), rec["unit"].lower())
+        if rec["description"]:
+            rows.append(rec)
+    return rows
+
+
+def parse_pdf_text(text: str):
+    """MVP: best-effort parse of plain-text BOQ lines like:
+    'A1  Excavation for foundation (machine dig)  350  m3  340.00'"""
+    rows = []
+    pat = re.compile(
+        r"^\s*([A-Z]{1,3}\d{1,4})?\s*(.+?)\s+([\d,]+(?:\.\d+)?)\s+([A-Za-z.]+)\s+([\d,]+(?:\.\d+)?)\s*$"
+    )
+    for line in text.splitlines():
+        m = pat.match(line)
+        if not m:
+            continue
+        item, desc, qty, unit, rate = m.groups()
+        rows.append({
+            "section": "", "item": (item or "").strip(), "description": desc.strip(),
+            "unit": UNIT_MAP.get(unit.lower(), unit.lower()),
+            "qty": _to_float(qty), "rate": _to_float(rate),
+        })
+    return rows
+
+
+def _flat_cell(value):
+    """Collapse PDF cell line breaks and repeated whitespace."""
+    return re.sub(r"\s+", " ", _clean(value)).strip()
+
+
+def _is_pdf_header(cells):
+    text = " ".join(_flat_cell(cell).lower() for cell in cells)
+    hits = sum(word in text for word in ("item", "description", "unit", "quantity", "qty", "rate"))
+    return hits >= 3 and "description" in text
+
+
+def _is_pdf_footer(cells):
+    text = " ".join(_flat_cell(cell) for cell in cells).strip()
+    return bool(
+        re.search(r"\bpage\s+\d+(?:\s+of\s+\d+)?\b", text, re.IGNORECASE)
+        or text.lower().startswith("smartqs sample boq")
+    )
+
+
+def _pdf_record(cells):
+    """Convert six extracted table cells into the public parser shape."""
+    cells = [_flat_cell(cell) for cell in cells]
+    if len(cells) < len(FIELDS):
+        cells.extend([""] * (len(FIELDS) - len(cells)))
+    if len(cells) > len(FIELDS):
+        cells = cells[:2] + [" ".join(cells[2:-3])] + cells[-3:]
+    if _is_pdf_header(cells) or _is_pdf_footer(cells):
+        return None
+    section, item, description, unit, qty, rate = cells[:6]
+    unit = UNIT_MAP.get(unit.lower(), unit.lower())
+    record = {
+        "section": section,
+        "item": item,
+        "description": description,
+        "unit": unit,
+        "qty": _to_float(qty),
+        "rate": _to_float(rate),
+    }
+    if not description or (record["qty"] is None and record["rate"] is None):
+        return None
+    return record
+
+
+def _records_from_tables(document):
+    records = []
+    for page in document:
+        try:
+            finder = page.find_tables()
+        except (AttributeError, RuntimeError, ValueError):
+            continue
+        for table in getattr(finder, "tables", []):
+            for cells in table.extract():
+                record = _pdf_record(cells or [])
+                if record:
+                    records.append(record)
+    return records
+
+
+def _line_groups(words, tolerance=3.0):
+    """Group PyMuPDF words into visual lines in reading order."""
+    lines = []
+    for word in sorted(words, key=lambda w: (w[1], w[0])):
+        y = (word[1] + word[3]) / 2
+        if not lines or abs(lines[-1][0] - y) > tolerance:
+            lines.append([y, [word]])
+        else:
+            lines[-1][1].append(word)
+            count = len(lines[-1][1])
+            lines[-1][0] = ((lines[-1][0] * (count - 1)) + y) / count
+    return [sorted(line_words, key=lambda w: w[0]) for _, line_words in lines]
+
+
+def _header_columns(lines):
+    """Find column starts from a BOQ header line."""
+    aliases = {
+        "section": {"section"},
+        "item": {"item", "ref"},
+        "description": {"description", "details"},
+        "unit": {"unit"},
+        "qty": {"quantity", "qty"},
+        "rate": {"rate"},
+    }
+    for index, words in enumerate(lines):
+        found = {}
+        for word in words:
+            token = re.sub(r"[^a-z]", "", word[4].lower())
+            for field, names in aliases.items():
+                if token in names and field not in found:
+                    found[field] = word[0]
+        if len(found) >= 5 and "description" in found:
+            if "section" not in found:
+                found["section"] = min(word[0] for word in words)
+            ordered = [found.get(field) for field in FIELDS]
+            if all(value is not None for value in ordered):
+                return index, ordered
+    return None, None
+
+
+def _records_from_words(document):
+    """Parse pages by assigning positioned words to header-derived x bands."""
+    records = []
+    pending = None
+    for page in document:
+        lines = _line_groups(page.get_text("words"))
+        header_index, starts = _header_columns(lines)
+        if starts is None:
+            continue
+        # Header labels are left aligned at each column start. A small offset
+        # keeps text touching a grid line in the column on its right.
+        boundaries = [start - 2 for start in starts[1:]]
+        for words in lines[header_index + 1:]:
+            full_text = " ".join(word[4] for word in words)
+            if _is_pdf_footer([full_text]) or _is_pdf_header([full_text]):
+                continue
+            cells = [[] for _ in FIELDS]
+            for word in words:
+                center = (word[0] + word[2]) / 2
+                column = sum(center >= boundary for boundary in boundaries)
+                cells[column].append(word[4])
+            values = [" ".join(cell) for cell in cells]
+            record = _pdf_record(values)
+            if record:
+                if pending:
+                    records.append(pending)
+                pending = record
+            elif pending and values[2] and not values[1]:
+                pending["description"] = f'{pending["description"]} {values[2]}'.strip()
+        if pending:
+            records.append(pending)
+            pending = None
+    return records
+
+
+def parse_pdf(pdf_bytes_or_path):
+    """Parse a BOQ PDF from bytes, a path, or a path-like object."""
+    if isinstance(pdf_bytes_or_path, (str, os.PathLike)):
+        document = fitz.open(os.fspath(pdf_bytes_or_path))
+    else:
+        document = fitz.open(stream=bytes(pdf_bytes_or_path), filetype="pdf")
+    try:
+        records = _records_from_tables(document)
+        if records:
+            return records
+        return _records_from_words(document)
+    finally:
+        document.close()
+
+
+def enrich(rows):
+    """Attach matched rate-db key + reference rate to each item."""
+    for r in rows:
+        key, meta = match_rate(r["description"])
+        r["rate_key"] = key
+        r["ref_rate"] = meta["rate"] if meta else None
+    return rows

src/rates.py

@@ -0,0 +1,39 @@
+"""Reference unit rates for HK construction (2025-26 baseline).
+Labeled as reference for anomaly screening, NOT pricing advice.
+Sources: published HK market rate references (trade averages)."""
+
+RATE_DB = {
+    "excavation": {"rate": 355, "unit": "m3", "match": ["excav", "dig", "spoil", "disposal"]},
+    "concrete": {"rate": 1780, "unit": "m3", "match": ["concrete", "blinding", "grade"]},
+    "rebar": {"rate": 12, "unit": "kg", "match": ["reinforcement", "rebar", "bar"]},
+    "formwork": {"rate": 460, "unit": "m2", "match": ["formwork"]},
+    "masonry": {"rate": 650, "unit": "m2", "match": ["masonry", "wall in cement", "blockwork", "brickwork"]},
+    "plaster": {"rate": 180, "unit": "m2", "match": ["plaster", "rendering"]},
+    "paint": {"rate": 55, "unit": "m2", "match": ["paint", "emulsion"]},
+    "tiling": {"rate": 320, "unit": "m2", "match": ["tiling", "tile"]},
+    "waterproof": {"rate": 265, "unit": "m2", "match": ["waterproof"]},
+    "ceiling": {"rate": 380, "unit": "m2", "match": ["false ceiling", "ceiling"]},
+    "demolition": {"rate": 490, "unit": "m3", "match": ["demoli"]},
+    "drainage": {"rate": 285, "unit": "m", "match": ["drain", "uPVC pipe"]},
+    "road": {"rate": 620, "unit": "m2", "match": ["asphalt", "road base", "paving"]},
+    "scaffold": {"rate": 118, "unit": "m2", "match": ["scaffold"]},
+    "electrical": {"rate": 950, "unit": "no.", "match": ["electrical installation", "socket", "light point"]},
+    "plumbing": {"rate": 1100, "unit": "no.", "match": ["plumbing", "water point", "sanitary"]},
+    "firealarm": {"rate": 1250, "unit": "no.", "match": ["fire alarm", "detection point"]},
+    "switchboard": {"rate": 185000, "unit": "no.", "match": ["switchboard", "LV panel"]},
+    "door": {"rate": 2800, "unit": "no.", "match": ["door"]},
+    "window": {"rate": 3100, "unit": "m2", "match": ["window", "glazing"]},
+    "precast": {"rate": 1450, "unit": "m2", "match": ["precast"]},
+    "safety": {"rate": None, "unit": "ls", "match": ["safety", "temporary works", "site establishment"]},
+}
+
+REQUIRED_SECTIONS = ["scaffold", "safety"]  # advisory: presence of access/safety items
+
+
+def match_rate(description: str):
+    d = description.lower()
+    for key, meta in RATE_DB.items():
+        for m in meta["match"]:
+            if m in d:
+                return key, meta
+    return None, None