from __future__ import annotations import json import re from pathlib import Path from typing import Any from utils import normalize_for_search, search_terms_match PROFILE_SCHEMA = "MCKF-DocumentSemanticProfiles-v1.0" def load_semantic_profiles(path: Path) -> dict[str, Any]: if not path.exists(): return {"schema": PROFILE_SCHEMA, "documents": {}} payload = json.loads(path.read_text(encoding="utf-8")) if not isinstance(payload, dict) or not isinstance(payload.get("documents", {}), dict): raise ValueError("document_semantic_profiles.json must contain a documents object") return payload def document_profile(profiles: dict[str, Any], document_id: str) -> dict[str, Any]: return dict((profiles.get("documents", {}) or {}).get(document_id, {}) or {}) def build_article_normative_metadata( article: dict[str, Any], document: dict[str, Any], profile: dict[str, Any] | None = None, ) -> dict[str, Any]: """Compile headings plus reviewed document knowledge into a semantic address.""" profile = profile or {} article_id = str(article.get("article_id", "") or "") override = dict((profile.get("article_overrides", {}) or {}).get(article_id, {}) or {}) heading_path = article.get("heading_path", []) or [] article_heading = _clean_heading(str(article.get("title", "") or "")) regulates = str( override.get("regulates", "") or article_heading or _fallback_regulates(article) or f"{article_id} yürürlük durumu" ).strip() display_heading = str( override.get("display_heading", "") or article_heading or regulates ).strip() domain_path = _unique( list(profile.get("domain_path", []) or []) + list(document.get("domain_tags", []) or []) + [item.get("title", "") for item in heading_path if item.get("title")] ) canonical_concepts = _unique( list(override.get("canonical_concepts", []) or []) + ([regulates] if regulates else []) ) query_aliases = _unique( list(override.get("query_aliases", []) or []) + canonical_concepts + ([article_heading] if article_heading else []) ) legal_effect_types = _unique( list(override.get("legal_effect_types", []) or []) + _infer_legal_effect_types(str(article.get("source_text", "") or ""), article_heading) ) subject_classes = _unique( list(override.get("subject_classes", []) or []) + _infer_subject_classes(str(article.get("source_text", "") or ""), article_heading) ) regulated_situations = _unique( list(override.get("regulated_situations", []) or []) + ([regulates] if regulates else []) ) exclusions = _unique(list(override.get("exclusions", []) or [])) legal_operations = _unique(list(override.get("legal_operations", []) or [])) competent_authorities = _unique(list(override.get("competent_authorities", []) or [])) operational_actors = _unique(list(override.get("operational_actors", []) or [])) normative_variables = _extract_profile_variables( " ".join( [article_heading] + [str(item.get("title", "") or "") for item in heading_path] + [str(article.get("source_text", "") or "")] ), profile, ) variable_values = [value for values in normative_variables.values() for value in values] variable_search_values = _profile_variable_search_values(normative_variables, profile) searchable_parts = ( domain_path + canonical_concepts + query_aliases + legal_effect_types + subject_classes + regulated_situations + legal_operations + competent_authorities + operational_actors + variable_values + variable_search_values ) canonical_address = " > ".join( _unique( [str(document.get("title", "") or "")] + [str(item.get("title", "") or "") for item in heading_path] + [article_id, regulates] ) ) return { "schema": "MCKF-NormativeAddress-v1.0", "document_id": document.get("document_id", ""), "article_id": article_id, "document_type": document.get("document_type", ""), "article_kind": _article_kind(article_id), "heading_path": heading_path, "article_heading": article_heading, "display_heading": display_heading, "domain_path": domain_path, "regulates": regulates, "canonical_concepts": canonical_concepts, "query_aliases": query_aliases, "subject_classes": subject_classes, "regulated_situations": regulated_situations, "legal_effect_types": legal_effect_types, "exclusions": exclusions, "legal_operations": legal_operations, "competent_authorities": competent_authorities, "operational_actors": operational_actors, "review_notes": list(override.get("review_notes", []) or []), # Human-authored, source-locked presentation semantics. These fields # are optional for ordinary retrieval, but they are the only material # the deterministic Knowledge Assistant may present as a canonical # summary without asking an LLM to interpret the provision. "approved_summary": str(override.get("approved_summary", "") or ""), "approved_points": list(override.get("approved_points", []) or []), "inventory_summary": str(override.get("inventory_summary", "") or ""), "topic_memberships": _unique(list(override.get("topic_memberships", []) or [])), "effective_from": str(override.get("effective_from", "") or ""), "effective_to": str(override.get("effective_to", "") or ""), "normative_variables": normative_variables, "canonical_address": canonical_address, "search_text": " ".join(_unique(searchable_parts)), "review_status": override.get("review_status", "derived_from_structure"), "provenance": { "heading_derived": bool(article_heading or heading_path), "document_profile": bool(override), "document_variable_schema": bool(profile.get("semantic_dimensions")), }, } def semantic_address_score(question: str, address: dict[str, Any]) -> float: """Score a query against a reviewed canonical address, not raw article text.""" query = normalize_for_search(question) if not query: return 0.0 if _matches_excluded_scope(query, address): return 0.0 aliases = [ normalize_for_search(str(value)) for value in ( list(address.get("query_aliases", []) or []) + list(address.get("canonical_concepts", []) or []) + list(address.get("regulated_situations", []) or []) ) if value ] if any(alias and alias in query for alias in aliases): return 1.0 query_terms = _content_terms(query) if not query_terms: return 0.0 address_terms = _content_terms(normalize_for_search(str(address.get("search_text", "") or ""))) if not address_terms: return 0.0 matched = sum(1 for term in query_terms if _term_matches(term, address_terms)) if len(query_terms) > 1 and matched < 2: return 0.0 return matched / len(query_terms) def semantic_address_text(address: dict[str, Any]) -> str: return str(address.get("search_text", "") or "") def semantic_address_focus_text(address: dict[str, Any]) -> str: """Return only article-specific address terms suitable for lexical indexes.""" provenance = address.get("provenance", {}) or {} if not (provenance.get("document_profile") or provenance.get("heading_derived")): return "" values = [address.get("article_heading", ""), address.get("regulates", "")] values += list(address.get("canonical_concepts", []) or []) values += list(address.get("query_aliases", []) or []) values += list(address.get("regulated_situations", []) or []) if provenance.get("document_profile"): values += list(address.get("legal_operations", []) or []) values += list(address.get("competent_authorities", []) or []) values += list(address.get("operational_actors", []) or []) return " ".join(_unique(values)) def _matches_excluded_scope(query: str, address: dict[str, Any]) -> bool: """Reject a reviewed address when the question primarily names an excluded concept.""" query_terms = _content_terms(query) if not query_terms: return False article_id = normalize_for_search(str(address.get("article_id", "") or "")) if article_id and article_id in query: return False positive_terms = _content_terms( normalize_for_search( " ".join( str(value) for value in ( [address.get("regulates", "")] + list(address.get("canonical_concepts", []) or []) + list(address.get("query_aliases", []) or []) + list(address.get("regulated_situations", []) or []) ) if value ) ) ) for exclusion in address.get("exclusions", []) or []: exclusion_terms = _content_terms(normalize_for_search(str(exclusion))) - positive_terms if len(exclusion_terms) < 2: continue matched = sum(1 for term in query_terms if _term_matches(term, exclusion_terms)) if ( matched >= 2 and matched / len(query_terms) >= 0.40 and matched / len(exclusion_terms) >= 0.50 ): return True return False def _clean_heading(value: str) -> str: value = re.sub(r"[.:]+\s*\d*\s*$", "", value).strip() return value if normalize_for_search(value) != "baslik bulunamadi" else "" def _fallback_regulates(article: dict[str, Any]) -> str: text = re.sub(r"\s+", " ", str(article.get("source_text", "") or "")).strip() article_id = str(article.get("article_id", "") or "Madde") normalized_source = normalize_for_search(text) if "mulga" in normalized_source and not _has_substantive_body(text): return f"{article_id} hükmünün mülga olma durumu" if "iptal" in normalized_source and not _has_substantive_body(text): return f"{article_id} hükmünün iptal durumu" text = re.sub(r"^(?:Ek |Geçici )?Madde\s+\w+\s*[-–]?\s*", "", text, flags=re.IGNORECASE) text = re.sub(r"^\([^)]*(?:Ek|Değişik|Mülga)[^)]*\)\s*", "", text, flags=re.IGNORECASE) text = re.sub(r"^\d{1,3}\s*$", "", text).strip() text = text.replace("T.C.", "T.C") sentence = re.split(r"(?<=[.!?])\s+", text, maxsplit=1)[0] return sentence[:220].rsplit(" ", 1)[0] if len(sentence) > 220 else sentence def _has_substantive_body(text: str) -> bool: body = re.sub(r"^(?:Ek |Geçici )?Madde\s+\w+\s*[-–]?\s*", "", text, flags=re.IGNORECASE) body = re.sub(r"^\([^)]*(?:Ek|Değişik|Mülga|İptal)[^)]*\)\s*", "", body, flags=re.IGNORECASE) body = re.sub(r"^\d{1,3}\s*$", "", body).strip() return len(normalize_for_search(body).split()) >= 4 def _infer_legal_effect_types(text: str, heading: str) -> list[str]: normalized = normalize_for_search(f"{heading} {text}") patterns = { "appointment": ("atanir", "atanır", "atanma", "secilir"), "authority_or_duty": ("gorev", "yetki", "sorumlu"), "status_restoration": ("yeniden ogren", "yeniden kayit", "ilisigi kesilen", "baslayabilirler"), "payment_obligation": ("odenir", "ucret", "ücret", "ödeme", "ödemeler", "katki payi"), "eligibility": ("yararlan", "hak kazan", "sartiyla"), "sanction": ("ceza", "iptal", "ilisigi kesilir", "ilişiği kesilir"), "establishment": ("kurulur", "acilir", "açılır", "teskil edilir", "teşkil edilir"), "definition": ("tanim", "tanımlanır", "ifade eder", "denir"), "repealed_or_annulled": ("mulga", "iptal"), } return [effect for effect, markers in patterns.items() if any(marker in normalized for marker in markers)] def _infer_subject_classes(text: str, heading: str) -> list[str]: normalized = normalize_for_search(f"{heading} {text[:600]}") subjects = { "student": ("ogrenci", "öğrenci"), "academic_staff": ("ogretim elemani", "öğretim elemanı", "ogretim uyesi", "öğretim üyesi", "arastirma gorevlisi"), "rector": ("rektor", "rektör",), "dean": ("dekan",), "university": ("universite", "üniversite",), "higher_education_institution": ("yuksekogretim kurumu",), } return [subject for subject, markers in subjects.items() if any(marker in normalized for marker in markers)] def _extract_profile_variables(text: str, profile: dict[str, Any]) -> dict[str, list[str]]: """Apply document-specific semantic dimensions without document-specific code.""" normalized = normalize_for_search(text) extracted: dict[str, list[str]] = {} for dimension, values in (profile.get("semantic_dimensions", {}) or {}).items(): matched: list[str] = [] for canonical, aliases in (values or {}).items(): markers = [canonical, *(aliases or [])] if any(normalize_for_search(str(marker)) in normalized for marker in markers if marker): matched.append(str(canonical)) if matched: extracted[str(dimension)] = _unique(matched) return extracted def _profile_variable_search_values( variables: dict[str, list[str]], profile: dict[str, Any], ) -> list[str]: """Attach institution-maintained aliases to matched canonical variables.""" dimensions = profile.get("semantic_dimensions", {}) or {} searchable: list[str] = [] for dimension, canonical_values in variables.items(): configured = dimensions.get(dimension, {}) or {} for canonical in canonical_values: searchable.append(canonical) searchable.extend(str(value) for value in configured.get(canonical, []) or []) return _unique(searchable) def _article_kind(article_id: str) -> str: normalized = normalize_for_search(article_id) if normalized.startswith("gecici madde"): return "geçici_madde" if normalized.startswith("ek madde"): return "ek_madde" return "madde" def _content_terms(text: str) -> set[str]: stopwords = { "hangi", "nedir", "nasil", "madde", "maddelerde", "duzenleniyor", "duzenlenir", "sayili", "kanun", "kanuna", "yonetmelik", "yonerge", "gore", "ile", "ve", "bir", } return {term for term in text.split() if len(term) >= 2 and term not in stopwords and not term.isdigit()} def _term_matches(term: str, candidates: set[str]) -> bool: return any(search_terms_match(term, candidate) for candidate in candidates) def _unique(values: list[Any]) -> list[str]: result: list[str] = [] seen: set[str] = set() for value in values: text = str(value or "").strip() key = normalize_for_search(text) if text and key not in seen: seen.add(key) result.append(text) return result