src/finreport_agent/generic.py

from __future__ import annotations

import re
import statistics
from dataclasses import dataclass
from decimal import Decimal, InvalidOperation

from .models import (
    ExtractedDocument,
    ExtractedPage,
    GenericDocumentReport,
    GenericRow,
    GenericTable,
    Severity,
    SourceRef,
    ValidationFinding,
    ValidationResult,
    WordBox,
)


DATE_TOKEN = re.compile(r"^\d{1,2}[./]\d{1,2}[./]\d{2,4}$")
NOTE_TOKEN = re.compile(r"^\([\d,\s-]+\)$")
NUMBER_TOKEN = re.compile(
    r"^[+-]?(?:\d{1,3}(?:[.\s]\d{3})+|\d+)(?:,\d+)?%?$"
)
TITLE_KEYWORDS = (
    "TABLOSU",
    "BİLANÇO",
    "NAZIM HESAPLAR",
    "NAKİT AKIŞ",
    "ÖZKAYNAKLAR DEĞİŞİM",
)
HEADER_EXCLUSIONS = (
    "SUNUM PARA BİRİMİ",
    "FİNANSAL TABLO NİTELİĞİ",
    "DİPNOT REFERANSI",
    "CARİ DÖNEM",
    "ÖNCEKİ DÖNEM",
)


@dataclass
class _Line:
    y: float
    words: list[WordBox]

    @property
    def text(self) -> str:
        return " ".join(word.text for word in sorted(self.words, key=lambda item: item.x0))


@dataclass
class _Fragment:
    table_id: str
    title: str | None
    unit: str | None
    headers: list[str]
    rows: list[GenericRow]
    source_pages: list[int]
    confidence: float


def parse_generic_number(raw: str) -> Decimal | None:
    text = raw.strip().replace("\u00a0", "").replace(" ", "")
    if not text or DATE_TOKEN.match(text) or NOTE_TOKEN.match(text):
        return None
    percent = text.endswith("%")
    if percent:
        text = text[:-1]
    if not NUMBER_TOKEN.match(text):
        return None
    if "," in text:
        normalized = text.replace(".", "").replace(",", ".")
    elif re.match(r"^[+-]?\d{1,3}(?:\.\d{3})+$", text):
        normalized = text.replace(".", "")
    else:
        normalized = text
    try:
        value = Decimal(normalized)
    except InvalidOperation:
        return None
    return value / 100 if percent else value


def _line_groups(page: ExtractedPage) -> list[_Line]:
    heights = [word.y1 - word.y0 for word in page.words if word.y1 > word.y0]
    median_height = statistics.median(heights) if heights else 10
    tolerance = max(1.8, median_height * 0.22)
    lines: list[_Line] = []
    for word in sorted(page.words, key=lambda item: (item.y0, item.x0)):
        for line in lines[-4:]:
            if abs(line.y - word.y0) <= tolerance:
                line.words.append(word)
                break
        else:
            lines.append(_Line(y=word.y0, words=[word]))
    for line in lines:
        line.words.sort(key=lambda item: item.x0)
    return lines


def _cluster_positions(values: list[float], tolerance: float) -> list[tuple[float, int]]:
    clusters: list[list[float]] = []
    for value in sorted(values):
        if clusters and abs(value - statistics.mean(clusters[-1])) <= tolerance:
            clusters[-1].append(value)
        else:
            clusters.append([value])
    return [(statistics.mean(cluster), len(cluster)) for cluster in clusters]


def _value_columns(page: ExtractedPage, lines: list[_Line]) -> list[float]:
    right_edges = [
        word.x1
        for line in lines
        for word in line.words
        if word.x0 >= page.width * 0.22 and parse_generic_number(word.text) is not None
    ]
    tolerance = max(3.5, page.width * 0.0045)
    minimum_support = max(3, int(len(lines) * 0.035))
    clusters = _cluster_positions(right_edges, tolerance)
    supported = [center for center, count in clusters if count >= minimum_support]
    if len(supported) > 24:
        supported = sorted(
            clusters,
            key=lambda item: item[1],
            reverse=True,
        )[:24]
        supported = sorted(center for center, _ in supported)
    return supported


def _nearest_column(value: float, columns: list[float], tolerance: float) -> int | None:
    if not columns:
        return None
    distances = [abs(value - column) for column in columns]
    index = min(range(len(columns)), key=distances.__getitem__)
    return index if distances[index] <= tolerance else None


def _title_for_fragment(
    page: ExtractedPage,
    lines: list[_Line],
    first_anchor_y: float,
) -> str | None:
    candidates: list[tuple[int, float, str]] = []
    for line in lines:
        if line.y >= first_anchor_y - 20:
            break
        text = re.sub(r"\s+", " ", line.text).strip()
        upper = text.upper()
        if len(text) < 8 or any(upper.startswith(item) for item in HEADER_EXCLUSIONS):
            continue
        if DATE_TOKEN.search(text) or text.startswith("http"):
            continue
        keyword_score = sum(keyword in upper for keyword in TITLE_KEYWORDS)
        if keyword_score:
            candidates.append((keyword_score, -line.y, text))
    if not candidates:
        return None
    candidates.sort(reverse=True)
    return candidates[0][2][:140]


def _unit_for_page(lines: list[_Line]) -> str | None:
    for line in lines:
        if "SUNUM PARA BİRİMİ" in line.text.upper():
            match = re.search(r"(\d[\d.\s]*\s*[A-Z]{2,4})$", line.text)
            return match.group(1).strip() if match else line.text
    return None


def _header_for_column(
    lines: list[_Line],
    columns: list[float],
    column_index: int,
    first_anchor_y: float,
) -> str:
    left = (
        (columns[column_index - 1] + columns[column_index]) / 2
        if column_index
        else columns[column_index] - 45
    )
    right = (
        (columns[column_index] + columns[column_index + 1]) / 2
        if column_index + 1 < len(columns)
        else columns[column_index] + 45
    )
    lower_y = max(0, first_anchor_y - 185)
    parts: list[str] = []
    for line in lines:
        if not lower_y <= line.y < first_anchor_y - 8:
            continue
        words = [
            word.text
            for word in line.words
            if left <= (word.x0 + word.x1) / 2 < right
        ]
        if words:
            part = " ".join(words)
            if part not in parts:
                parts.append(part)
    header = " ".join(parts[-5:]).strip()
    return header[:120] or f"Değer {column_index + 1}"


def _source_ref(words: list[WordBox]) -> SourceRef:
    return SourceRef(
        page=words[0].page,
        bbox=(
            min(word.x0 for word in words),
            min(word.y0 for word in words),
            max(word.x1 for word in words),
            max(word.y1 for word in words),
        ),
        method=words[0].method,
        confidence=min(word.confidence for word in words),
    )


def _table_fragment(page: ExtractedPage) -> _Fragment | None:
    lines = _line_groups(page)
    columns = _value_columns(page, lines)
    if len(columns) < 2:
        return None

    assignment_tolerance = max(8.0, page.width * 0.012)
    anchors: list[tuple[float, _Line, dict[int, tuple[Decimal, WordBox]]]] = []
    for line in lines:
        assigned: dict[int, tuple[Decimal, WordBox]] = {}
        for word in line.words:
            parsed = parse_generic_number(word.text)
            if parsed is None:
                continue
            index = _nearest_column(word.x1, columns, assignment_tolerance)
            if index is not None:
                assigned[index] = (parsed, word)
        if len(assigned) >= 2:
            anchors.append((line.y, line, assigned))

    if len(anchors) < 3:
        return None

    first_anchor_y = anchors[0][0]
    headers = [
        _header_for_column(lines, columns, index, first_anchor_y)
        for index in range(len(columns))
    ]
    title = _title_for_fragment(page, lines, first_anchor_y)
    row_gaps = [
        current[0] - previous[0]
        for previous, current in zip(anchors, anchors[1:])
        if 2 < current[0] - previous[0] < 100
    ]
    typical_gap = statistics.median(row_gaps) if row_gaps else 20
    label_boundary = columns[0] - max(16, page.width * 0.018)
    staged_rows: list[tuple[GenericRow, float]] = []

    for index, (anchor_y, _, assigned) in enumerate(anchors):
        previous_y = anchors[index - 1][0] if index else anchor_y - typical_gap
        next_y = anchors[index + 1][0] if index + 1 < len(anchors) else anchor_y + typical_gap
        lower = (previous_y + anchor_y) / 2
        upper = (anchor_y + next_y) / 2
        band_words = [
            word
            for word in page.words
            if lower <= word.y0 < upper
        ]
        label_words = [word for word in band_words if word.x1 < label_boundary]
        note_words = [
            word
            for word in label_words
            if NOTE_TOKEN.match(word.text)
            and word.x0 >= page.width * 0.18
        ]
        note_ids = {id(word) for word in note_words}
        label_words = [word for word in label_words if id(word) not in note_ids]
        label_words.sort(key=lambda item: (item.y0, item.x0))
        label = " ".join(word.text for word in label_words)
        label = re.sub(r"\s+([),])", r"\1", label)
        label = re.sub(r"([(])\s+", r"\1", label)
        label = re.sub(r"\s+", " ", label).strip()
        label = re.sub(r"^Ayrılmış\)\s*", "", label, flags=re.IGNORECASE)
        if not label:
            continue
        values: list[Decimal | str | None] = [None] * len(columns)
        for column_index, (value, _) in assigned.items():
            values[column_index] = value
        source_words = label_words + note_words + [item[1] for item in assigned.values()]
        indent = min((word.x0 for word in label_words), default=0)
        staged_rows.append(
            (
                GenericRow(
                    label=label[:500],
                    note_reference=" ".join(word.text for word in note_words) or None,
                    values=values,
                    source=_source_ref(source_words),
                ),
                indent,
            )
        )

    if len(staged_rows) < 3:
        return None
    active_columns = [
        column_index
        for column_index in range(len(headers))
        if any(row.values[column_index] is not None for row, _ in staged_rows)
    ]
    if len(active_columns) >= 2 and len(active_columns) < len(headers):
        headers = [headers[column_index] for column_index in active_columns]
        for row, _ in staged_rows:
            row.values = [row.values[column_index] for column_index in active_columns]

    minimum_indent = min(indent for _, indent in staged_rows)
    indent_unit = max(10, page.width * 0.013)
    rows: list[GenericRow] = []
    for row, indent in staged_rows:
        row.hierarchy_level = min(8, max(0, round((indent - minimum_indent) / indent_unit)))
        rows.append(row)

    confidence = min(
        0.98,
        0.62 + min(0.22, len(rows) / 100) + min(0.12, len(headers) / 50),
    )
    return _Fragment(
        table_id=f"table_p{page.number}",
        title=title,
        unit=_unit_for_page(lines),
        headers=headers,
        rows=rows,
        source_pages=[page.number],
        confidence=confidence,
    )


def _key_value_fragment(page: ExtractedPage) -> _Fragment | None:
    lines = _line_groups(page)
    pairs: list[tuple[str, str, list[WordBox]]] = []
    for line in lines:
        if line.y > page.height * 0.38 or len(line.words) < 2:
            continue
        gaps = [
            (line.words[index + 1].x0 - line.words[index].x1, index)
            for index in range(len(line.words) - 1)
        ]
        gap, split = max(gaps)
        if gap < page.width * 0.07:
            continue
        left_words = line.words[: split + 1]
        right_words = line.words[split + 1 :]
        left = " ".join(word.text for word in left_words).strip()
        right = " ".join(word.text for word in right_words).strip()
        if 3 <= len(left) <= 80 and right:
            pairs.append((left, right, line.words))
    if len(pairs) < 2:
        return None
    rows = [
        GenericRow(
            label=left,
            values=[right],
            source=_source_ref(words),
        )
        for left, right, words in pairs[:20]
    ]
    title = None
    for line in lines:
        if line.y >= pairs[0][2][0].y0:
            break
        if len(line.text) >= 8 and not line.text.startswith("http"):
            title = line.text
    return _Fragment(
        table_id=f"kv_p{page.number}",
        title=title or f"Bilgi Tablosu - Sayfa {page.number}",
        unit=None,
        headers=["Değer"],
        rows=rows,
        source_pages=[page.number],
        confidence=0.72,
    )


def discover_fragments(document: ExtractedDocument) -> list[_Fragment]:
    fragments: list[_Fragment] = []
    for page in document.pages:
        fragment = _table_fragment(page)
        if fragment:
            if (
                fragments
                and fragment.title is None
                and fragments[-1].source_pages[-1] + 1 == page.number
                and len(fragments[-1].headers) == len(fragment.headers)
            ):
                fragments[-1].rows.extend(fragment.rows)
                fragments[-1].source_pages.append(page.number)
                fragments[-1].confidence = min(
                    fragments[-1].confidence,
                    fragment.confidence,
                )
            else:
                if (
                    fragments
                    and fragment.title is None
                    and fragments[-1].title
                    and fragments[-1].source_pages[-1] + 1 == page.number
                    and abs(len(fragments[-1].headers) - len(fragment.headers)) <= 4
                ):
                    fragment.title = f"{fragments[-1].title} - Devam"
                fragments.append(fragment)
            continue
        key_value = _key_value_fragment(page)
        if key_value:
            fragments.append(key_value)
    return fragments


def document_front_matter(document: ExtractedDocument) -> str:
    page = document.pages[0]
    lines = _line_groups(page)
    useful = [
        line.text
        for line in lines
        if not line.text.startswith("http")
        and "KAP'TA YAYINLANMA" not in line.text.upper()
    ]
    return "\n".join(useful[:12])


def build_generic_report(
    document: ExtractedDocument,
    *,
    fragments: list[_Fragment] | None = None,
    document_title: str,
    document_type: str,
    summary: str,
    planner_mode: str,
    table_metadata: dict[str, dict[str, str | bool]],
) -> GenericDocumentReport:
    fragments = fragments if fragments is not None else discover_fragments(document)
    tables: list[GenericTable] = []
    used_names: set[str] = set()
    for index, fragment in enumerate(fragments, start=1):
        metadata = table_metadata.get(fragment.table_id, {})
        if metadata.get("include") is False:
            continue
        fallback_title = fragment.title or f"Tablo {index}"
        sheet_name = str(metadata.get("sheet_name") or fallback_title).replace("_", " ")
        sheet_name = re.sub(r"[:\\/?*\[\]]", " ", sheet_name)
        sheet_name = (
            re.sub(r"\s+", " ", sheet_name).strip().strip("'")[:31].rstrip("'")
            or f"Tablo {index}"
        )
        base = sheet_name
        counter = 2
        while sheet_name.casefold() in used_names:
            suffix = f" {counter}"
            sheet_name = f"{base[:31-len(suffix)]}{suffix}"
            counter += 1
        used_names.add(sheet_name.casefold())
        header_counts: dict[str, int] = {}
        unique_headers: list[str] = []
        for header_index, raw_header in enumerate(fragment.headers, start=1):
            header = re.sub(r"\s+", " ", raw_header).strip() or f"Değer {header_index}"
            key = header.casefold()
            header_counts[key] = header_counts.get(key, 0) + 1
            if header_counts[key] > 1:
                header = f"{header} ({header_counts[key]})"
            unique_headers.append(header)
        tables.append(
            GenericTable(
                table_id=fragment.table_id,
                title=str(metadata.get("title") or fallback_title),
                sheet_name=sheet_name,
                semantic_type=str(metadata.get("semantic_type") or "financial_table"),
                description=str(metadata.get("description") or ""),
                unit=fragment.unit,
                headers=unique_headers,
                rows=fragment.rows,
                source_pages=fragment.source_pages,
                confidence=fragment.confidence,
            )
        )

    if not tables:
        raise RuntimeError("Belgede Excel'e aktarılabilecek tablo bulunamadı.")
    findings = generic_validation(tables)
    return GenericDocumentReport(
        document_title=document_title,
        document_type=document_type,
        summary=summary,
        source_filename=document.source_filename,
        source_sha256=document.sha256,
        planner_mode=planner_mode,
        tables=tables,
    )


def generic_validation(tables: list[GenericTable]) -> ValidationResult:
    findings: list[ValidationFinding] = []
    for table in tables:
        empty_labels = sum(not row.label.strip() for row in table.rows)
        if empty_labels:
            findings.append(
                ValidationFinding(
                    severity=Severity.WARNING,
                    code="EMPTY_ROW_LABELS",
                    message=f"{empty_labels} satırın açıklama alanı boş.",
                    sheet=table.sheet_name,
                )
            )
        findings.append(
            ValidationFinding(
                severity=Severity.INFO,
                code="TABLE_DISCOVERED",
                message=(
                    f"{len(table.rows)} satır ve {len(table.headers)} değer sütunu "
                    f"keşfedildi; kaynak sayfalar: {', '.join(map(str, table.source_pages))}."
                ),
                sheet=table.sheet_name,
            )
        )
    error_count = sum(item.severity == Severity.ERROR for item in findings)
    warning_count = sum(item.severity == Severity.WARNING for item in findings)
    info_count = sum(item.severity == Severity.INFO for item in findings)
    return ValidationResult(
        status=(
            "Başarısız"
            if error_count
            else "Uyarılı"
            if warning_count
            else "Tablolar Çıkarıldı"
        ),
        findings=findings,
        error_count=error_count,
        warning_count=warning_count,
        info_count=info_count,
    )