from __future__ import annotations import re import statistics from dataclasses import dataclass from decimal import Decimal, InvalidOperation from .models import ( ExtractedDocument, ExtractedPage, GenericDocumentReport, GenericRow, GenericTable, Severity, SourceRef, ValidationFinding, ValidationResult, WordBox, ) DATE_TOKEN = re.compile(r"^\d{1,2}[./]\d{1,2}[./]\d{2,4}$") NOTE_TOKEN = re.compile(r"^\([\d,\s-]+\)$") NUMBER_TOKEN = re.compile( r"^[+-]?(?:\d{1,3}(?:[.\s]\d{3})+|\d+)(?:,\d+)?%?$" ) TITLE_KEYWORDS = ( "TABLOSU", "BİLANÇO", "NAZIM HESAPLAR", "NAKİT AKIŞ", "ÖZKAYNAKLAR DEĞİŞİM", ) HEADER_EXCLUSIONS = ( "SUNUM PARA BİRİMİ", "FİNANSAL TABLO NİTELİĞİ", "DİPNOT REFERANSI", "CARİ DÖNEM", "ÖNCEKİ DÖNEM", ) @dataclass class _Line: y: float words: list[WordBox] @property def text(self) -> str: return " ".join(word.text for word in sorted(self.words, key=lambda item: item.x0)) @dataclass class _Fragment: table_id: str title: str | None unit: str | None headers: list[str] rows: list[GenericRow] source_pages: list[int] confidence: float def parse_generic_number(raw: str) -> Decimal | None: text = raw.strip().replace("\u00a0", "").replace(" ", "") if not text or DATE_TOKEN.match(text) or NOTE_TOKEN.match(text): return None percent = text.endswith("%") if percent: text = text[:-1] if not NUMBER_TOKEN.match(text): return None if "," in text: normalized = text.replace(".", "").replace(",", ".") elif re.match(r"^[+-]?\d{1,3}(?:\.\d{3})+$", text): normalized = text.replace(".", "") else: normalized = text try: value = Decimal(normalized) except InvalidOperation: return None return value / 100 if percent else value def _line_groups(page: ExtractedPage) -> list[_Line]: heights = [word.y1 - word.y0 for word in page.words if word.y1 > word.y0] median_height = statistics.median(heights) if heights else 10 tolerance = max(1.8, median_height * 0.22) lines: list[_Line] = [] for word in sorted(page.words, key=lambda item: (item.y0, item.x0)): for line in lines[-4:]: if abs(line.y - word.y0) <= tolerance: line.words.append(word) break else: lines.append(_Line(y=word.y0, words=[word])) for line in lines: line.words.sort(key=lambda item: item.x0) return lines def _cluster_positions(values: list[float], tolerance: float) -> list[tuple[float, int]]: clusters: list[list[float]] = [] for value in sorted(values): if clusters and abs(value - statistics.mean(clusters[-1])) <= tolerance: clusters[-1].append(value) else: clusters.append([value]) return [(statistics.mean(cluster), len(cluster)) for cluster in clusters] def _value_columns(page: ExtractedPage, lines: list[_Line]) -> list[float]: right_edges = [ word.x1 for line in lines for word in line.words if word.x0 >= page.width * 0.22 and parse_generic_number(word.text) is not None ] tolerance = max(3.5, page.width * 0.0045) minimum_support = max(3, int(len(lines) * 0.035)) clusters = _cluster_positions(right_edges, tolerance) supported = [center for center, count in clusters if count >= minimum_support] if len(supported) > 24: supported = sorted( clusters, key=lambda item: item[1], reverse=True, )[:24] supported = sorted(center for center, _ in supported) return supported def _nearest_column(value: float, columns: list[float], tolerance: float) -> int | None: if not columns: return None distances = [abs(value - column) for column in columns] index = min(range(len(columns)), key=distances.__getitem__) return index if distances[index] <= tolerance else None def _title_for_fragment( page: ExtractedPage, lines: list[_Line], first_anchor_y: float, ) -> str | None: candidates: list[tuple[int, float, str]] = [] for line in lines: if line.y >= first_anchor_y - 20: break text = re.sub(r"\s+", " ", line.text).strip() upper = text.upper() if len(text) < 8 or any(upper.startswith(item) for item in HEADER_EXCLUSIONS): continue if DATE_TOKEN.search(text) or text.startswith("http"): continue keyword_score = sum(keyword in upper for keyword in TITLE_KEYWORDS) if keyword_score: candidates.append((keyword_score, -line.y, text)) if not candidates: return None candidates.sort(reverse=True) return candidates[0][2][:140] def _unit_for_page(lines: list[_Line]) -> str | None: for line in lines: if "SUNUM PARA BİRİMİ" in line.text.upper(): match = re.search(r"(\d[\d.\s]*\s*[A-Z]{2,4})$", line.text) return match.group(1).strip() if match else line.text return None def _header_for_column( lines: list[_Line], columns: list[float], column_index: int, first_anchor_y: float, ) -> str: left = ( (columns[column_index - 1] + columns[column_index]) / 2 if column_index else columns[column_index] - 45 ) right = ( (columns[column_index] + columns[column_index + 1]) / 2 if column_index + 1 < len(columns) else columns[column_index] + 45 ) lower_y = max(0, first_anchor_y - 185) parts: list[str] = [] for line in lines: if not lower_y <= line.y < first_anchor_y - 8: continue words = [ word.text for word in line.words if left <= (word.x0 + word.x1) / 2 < right ] if words: part = " ".join(words) if part not in parts: parts.append(part) header = " ".join(parts[-5:]).strip() return header[:120] or f"Değer {column_index + 1}" def _source_ref(words: list[WordBox]) -> SourceRef: return SourceRef( page=words[0].page, bbox=( min(word.x0 for word in words), min(word.y0 for word in words), max(word.x1 for word in words), max(word.y1 for word in words), ), method=words[0].method, confidence=min(word.confidence for word in words), ) def _table_fragment(page: ExtractedPage) -> _Fragment | None: lines = _line_groups(page) columns = _value_columns(page, lines) if len(columns) < 2: return None assignment_tolerance = max(8.0, page.width * 0.012) anchors: list[tuple[float, _Line, dict[int, tuple[Decimal, WordBox]]]] = [] for line in lines: assigned: dict[int, tuple[Decimal, WordBox]] = {} for word in line.words: parsed = parse_generic_number(word.text) if parsed is None: continue index = _nearest_column(word.x1, columns, assignment_tolerance) if index is not None: assigned[index] = (parsed, word) if len(assigned) >= 2: anchors.append((line.y, line, assigned)) if len(anchors) < 3: return None first_anchor_y = anchors[0][0] headers = [ _header_for_column(lines, columns, index, first_anchor_y) for index in range(len(columns)) ] title = _title_for_fragment(page, lines, first_anchor_y) row_gaps = [ current[0] - previous[0] for previous, current in zip(anchors, anchors[1:]) if 2 < current[0] - previous[0] < 100 ] typical_gap = statistics.median(row_gaps) if row_gaps else 20 label_boundary = columns[0] - max(16, page.width * 0.018) staged_rows: list[tuple[GenericRow, float]] = [] for index, (anchor_y, _, assigned) in enumerate(anchors): previous_y = anchors[index - 1][0] if index else anchor_y - typical_gap next_y = anchors[index + 1][0] if index + 1 < len(anchors) else anchor_y + typical_gap lower = (previous_y + anchor_y) / 2 upper = (anchor_y + next_y) / 2 band_words = [ word for word in page.words if lower <= word.y0 < upper ] label_words = [word for word in band_words if word.x1 < label_boundary] note_words = [ word for word in label_words if NOTE_TOKEN.match(word.text) and word.x0 >= page.width * 0.18 ] note_ids = {id(word) for word in note_words} label_words = [word for word in label_words if id(word) not in note_ids] label_words.sort(key=lambda item: (item.y0, item.x0)) label = " ".join(word.text for word in label_words) label = re.sub(r"\s+([),])", r"\1", label) label = re.sub(r"([(])\s+", r"\1", label) label = re.sub(r"\s+", " ", label).strip() label = re.sub(r"^Ayrılmış\)\s*", "", label, flags=re.IGNORECASE) if not label: continue values: list[Decimal | str | None] = [None] * len(columns) for column_index, (value, _) in assigned.items(): values[column_index] = value source_words = label_words + note_words + [item[1] for item in assigned.values()] indent = min((word.x0 for word in label_words), default=0) staged_rows.append( ( GenericRow( label=label[:500], note_reference=" ".join(word.text for word in note_words) or None, values=values, source=_source_ref(source_words), ), indent, ) ) if len(staged_rows) < 3: return None active_columns = [ column_index for column_index in range(len(headers)) if any(row.values[column_index] is not None for row, _ in staged_rows) ] if len(active_columns) >= 2 and len(active_columns) < len(headers): headers = [headers[column_index] for column_index in active_columns] for row, _ in staged_rows: row.values = [row.values[column_index] for column_index in active_columns] minimum_indent = min(indent for _, indent in staged_rows) indent_unit = max(10, page.width * 0.013) rows: list[GenericRow] = [] for row, indent in staged_rows: row.hierarchy_level = min(8, max(0, round((indent - minimum_indent) / indent_unit))) rows.append(row) confidence = min( 0.98, 0.62 + min(0.22, len(rows) / 100) + min(0.12, len(headers) / 50), ) return _Fragment( table_id=f"table_p{page.number}", title=title, unit=_unit_for_page(lines), headers=headers, rows=rows, source_pages=[page.number], confidence=confidence, ) def _key_value_fragment(page: ExtractedPage) -> _Fragment | None: lines = _line_groups(page) pairs: list[tuple[str, str, list[WordBox]]] = [] for line in lines: if line.y > page.height * 0.38 or len(line.words) < 2: continue gaps = [ (line.words[index + 1].x0 - line.words[index].x1, index) for index in range(len(line.words) - 1) ] gap, split = max(gaps) if gap < page.width * 0.07: continue left_words = line.words[: split + 1] right_words = line.words[split + 1 :] left = " ".join(word.text for word in left_words).strip() right = " ".join(word.text for word in right_words).strip() if 3 <= len(left) <= 80 and right: pairs.append((left, right, line.words)) if len(pairs) < 2: return None rows = [ GenericRow( label=left, values=[right], source=_source_ref(words), ) for left, right, words in pairs[:20] ] title = None for line in lines: if line.y >= pairs[0][2][0].y0: break if len(line.text) >= 8 and not line.text.startswith("http"): title = line.text return _Fragment( table_id=f"kv_p{page.number}", title=title or f"Bilgi Tablosu - Sayfa {page.number}", unit=None, headers=["Değer"], rows=rows, source_pages=[page.number], confidence=0.72, ) def discover_fragments(document: ExtractedDocument) -> list[_Fragment]: fragments: list[_Fragment] = [] for page in document.pages: fragment = _table_fragment(page) if fragment: if ( fragments and fragment.title is None and fragments[-1].source_pages[-1] + 1 == page.number and len(fragments[-1].headers) == len(fragment.headers) ): fragments[-1].rows.extend(fragment.rows) fragments[-1].source_pages.append(page.number) fragments[-1].confidence = min( fragments[-1].confidence, fragment.confidence, ) else: if ( fragments and fragment.title is None and fragments[-1].title and fragments[-1].source_pages[-1] + 1 == page.number and abs(len(fragments[-1].headers) - len(fragment.headers)) <= 4 ): fragment.title = f"{fragments[-1].title} - Devam" fragments.append(fragment) continue key_value = _key_value_fragment(page) if key_value: fragments.append(key_value) return fragments def document_front_matter(document: ExtractedDocument) -> str: page = document.pages[0] lines = _line_groups(page) useful = [ line.text for line in lines if not line.text.startswith("http") and "KAP'TA YAYINLANMA" not in line.text.upper() ] return "\n".join(useful[:12]) def build_generic_report( document: ExtractedDocument, *, fragments: list[_Fragment] | None = None, document_title: str, document_type: str, summary: str, planner_mode: str, table_metadata: dict[str, dict[str, str | bool]], ) -> GenericDocumentReport: fragments = fragments if fragments is not None else discover_fragments(document) tables: list[GenericTable] = [] used_names: set[str] = set() for index, fragment in enumerate(fragments, start=1): metadata = table_metadata.get(fragment.table_id, {}) if metadata.get("include") is False: continue fallback_title = fragment.title or f"Tablo {index}" sheet_name = str(metadata.get("sheet_name") or fallback_title).replace("_", " ") sheet_name = re.sub(r"[:\\/?*\[\]]", " ", sheet_name) sheet_name = ( re.sub(r"\s+", " ", sheet_name).strip().strip("'")[:31].rstrip("'") or f"Tablo {index}" ) base = sheet_name counter = 2 while sheet_name.casefold() in used_names: suffix = f" {counter}" sheet_name = f"{base[:31-len(suffix)]}{suffix}" counter += 1 used_names.add(sheet_name.casefold()) header_counts: dict[str, int] = {} unique_headers: list[str] = [] for header_index, raw_header in enumerate(fragment.headers, start=1): header = re.sub(r"\s+", " ", raw_header).strip() or f"Değer {header_index}" key = header.casefold() header_counts[key] = header_counts.get(key, 0) + 1 if header_counts[key] > 1: header = f"{header} ({header_counts[key]})" unique_headers.append(header) tables.append( GenericTable( table_id=fragment.table_id, title=str(metadata.get("title") or fallback_title), sheet_name=sheet_name, semantic_type=str(metadata.get("semantic_type") or "financial_table"), description=str(metadata.get("description") or ""), unit=fragment.unit, headers=unique_headers, rows=fragment.rows, source_pages=fragment.source_pages, confidence=fragment.confidence, ) ) if not tables: raise RuntimeError("Belgede Excel'e aktarılabilecek tablo bulunamadı.") findings = generic_validation(tables) return GenericDocumentReport( document_title=document_title, document_type=document_type, summary=summary, source_filename=document.source_filename, source_sha256=document.sha256, planner_mode=planner_mode, tables=tables, ) def generic_validation(tables: list[GenericTable]) -> ValidationResult: findings: list[ValidationFinding] = [] for table in tables: empty_labels = sum(not row.label.strip() for row in table.rows) if empty_labels: findings.append( ValidationFinding( severity=Severity.WARNING, code="EMPTY_ROW_LABELS", message=f"{empty_labels} satırın açıklama alanı boş.", sheet=table.sheet_name, ) ) findings.append( ValidationFinding( severity=Severity.INFO, code="TABLE_DISCOVERED", message=( f"{len(table.rows)} satır ve {len(table.headers)} değer sütunu " f"keşfedildi; kaynak sayfalar: {', '.join(map(str, table.source_pages))}." ), sheet=table.sheet_name, ) ) error_count = sum(item.severity == Severity.ERROR for item in findings) warning_count = sum(item.severity == Severity.WARNING for item in findings) info_count = sum(item.severity == Severity.INFO for item in findings) return ValidationResult( status=( "Başarısız" if error_count else "Uyarılı" if warning_count else "Tablolar Çıkarıldı" ), findings=findings, error_count=error_count, warning_count=warning_count, info_count=info_count, )