Update Anwendung
Browse files- src/__pycache__/answer_composer.cpython-311.pyc +0 -0
- src/__pycache__/app.cpython-311.pyc +0 -0
- src/__pycache__/llm_client_groq.cpython-311.pyc +0 -0
- src/__pycache__/orchestrator.cpython-311.pyc +0 -0
- src/__pycache__/retriever.cpython-311.pyc +0 -0
- src/answer_composer.py +966 -89
- src/app.py +1338 -146
- src/ask_workflow.png +0 -0
- src/llm_client_groq.py +71 -12
- src/orchestrator.py +1220 -0
- src/retriever.py +1119 -275
- src/visualize_graph.py +15 -0
- static/index.html +0 -0
- static/js/script.js +745 -0
- static/styles/style.css +882 -0
src/__pycache__/answer_composer.cpython-311.pyc
CHANGED
|
Binary files a/src/__pycache__/answer_composer.cpython-311.pyc and b/src/__pycache__/answer_composer.cpython-311.pyc differ
|
|
|
src/__pycache__/app.cpython-311.pyc
CHANGED
|
Binary files a/src/__pycache__/app.cpython-311.pyc and b/src/__pycache__/app.cpython-311.pyc differ
|
|
|
src/__pycache__/llm_client_groq.cpython-311.pyc
CHANGED
|
Binary files a/src/__pycache__/llm_client_groq.cpython-311.pyc and b/src/__pycache__/llm_client_groq.cpython-311.pyc differ
|
|
|
src/__pycache__/orchestrator.cpython-311.pyc
ADDED
|
Binary file (68.5 kB). View file
|
|
|
src/__pycache__/retriever.cpython-311.pyc
CHANGED
|
Binary files a/src/__pycache__/retriever.cpython-311.pyc and b/src/__pycache__/retriever.cpython-311.pyc differ
|
|
|
src/answer_composer.py
CHANGED
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
| 3 |
import re
|
| 4 |
from typing import Any, Dict, List, Optional, Tuple, Set
|
| 5 |
|
| 6 |
-
from
|
| 7 |
|
| 8 |
|
| 9 |
# ---------------------------------------------------------------------------
|
|
@@ -35,37 +35,208 @@ _ANSWER_REQUEST_PATTERNS = [
|
|
| 35 |
re.compile(r"\bgib\s+mir\s+die\s+antwort\b", re.I),
|
| 36 |
]
|
| 37 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
_DOCUMENT_TASK_PROMPT = """
|
| 39 |
-
Du
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
-
|
| 54 |
-
-
|
| 55 |
-
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
""".strip()
|
| 70 |
|
| 71 |
|
|
@@ -118,6 +289,9 @@ class AnswerComposer:
|
|
| 118 |
include_neighbor_hits_in_context: bool = True,
|
| 119 |
include_pure_neighbors_as_sources: bool = False,
|
| 120 |
append_sources_to_answer: bool = False,
|
|
|
|
|
|
|
|
|
|
| 121 |
):
|
| 122 |
self.llm = llm_client
|
| 123 |
self.max_context_chars = max_context_chars
|
|
@@ -130,6 +304,9 @@ class AnswerComposer:
|
|
| 130 |
self.include_neighbor_hits_in_context = include_neighbor_hits_in_context
|
| 131 |
self.include_pure_neighbors_as_sources = include_pure_neighbors_as_sources
|
| 132 |
self.append_sources_to_answer = append_sources_to_answer
|
|
|
|
|
|
|
|
|
|
| 133 |
|
| 134 |
# ------------------------------------------------------------------
|
| 135 |
# Meta-Hilfsmethoden
|
|
@@ -231,6 +408,37 @@ class AnswerComposer:
|
|
| 231 |
return str(start)
|
| 232 |
return "?"
|
| 233 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 234 |
@classmethod
|
| 235 |
def _container(cls, hit: Dict[str, Any]) -> str:
|
| 236 |
return str(cls._hit_value(hit, "container", "container_id", default="Unbekannt"))
|
|
@@ -243,6 +451,30 @@ class AnswerComposer:
|
|
| 243 |
def _chunk_index(cls, hit: Dict[str, Any]) -> Any:
|
| 244 |
return cls._hit_value(hit, "chunk_index", "chunk_index_in_section", default="?")
|
| 245 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 246 |
@classmethod
|
| 247 |
def _text(cls, hit: Dict[str, Any]) -> str:
|
| 248 |
return str(hit.get("text") or hit.get("document") or "").strip()
|
|
@@ -261,6 +493,96 @@ class AnswerComposer:
|
|
| 261 |
except (TypeError, ValueError):
|
| 262 |
return 0.0
|
| 263 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 264 |
@classmethod
|
| 265 |
def _source_key(cls, hit: Dict[str, Any]) -> Tuple[Any, ...]:
|
| 266 |
metadata = hit.get("metadata") or {}
|
|
@@ -308,44 +630,205 @@ class AnswerComposer:
|
|
| 308 |
|
| 309 |
return True
|
| 310 |
|
| 311 |
-
def _prepare_hits_for_context(
|
| 312 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 313 |
if not hits:
|
| 314 |
return []
|
| 315 |
|
| 316 |
-
filtered = [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 317 |
|
| 318 |
-
# Dedupe auf Chunk-Ebene.
|
| 319 |
-
deduped: Dict[Tuple[Any, ...], Dict[str, Any]] = {}
|
| 320 |
for hit in filtered:
|
|
|
|
| 321 |
key = self._source_key(hit)
|
| 322 |
-
|
| 323 |
-
if
|
|
|
|
|
|
|
|
|
|
| 324 |
deduped[key] = hit
|
| 325 |
|
| 326 |
prepared = list(deduped.values())
|
| 327 |
|
| 328 |
-
#
|
| 329 |
-
|
| 330 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 331 |
|
| 332 |
prepared.sort(
|
| 333 |
-
key=
|
| 334 |
-
1 if "explicit_section" in set(self._retrieval_kinds(h)) else 0,
|
| 335 |
-
self._score(h),
|
| 336 |
-
-int(self._chunk_index(h) if str(self._chunk_index(h)).isdigit() else 0),
|
| 337 |
-
),
|
| 338 |
reverse=True,
|
| 339 |
)
|
| 340 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 341 |
for hit in prepared:
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 345 |
continue
|
| 346 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 347 |
section_limited.append(hit)
|
| 348 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 349 |
return section_limited[: self.max_hits_for_context]
|
| 350 |
|
| 351 |
@classmethod
|
|
@@ -356,28 +839,34 @@ class AnswerComposer:
|
|
| 356 |
max_sources: int = 5,
|
| 357 |
include_pure_neighbors: bool = False,
|
| 358 |
allowed_container_ids: Optional[List[str]] = None,
|
|
|
|
| 359 |
) -> List[Dict[str, Any]]:
|
| 360 |
"""
|
| 361 |
-
Baut eine
|
| 362 |
|
| 363 |
Wichtig:
|
| 364 |
-
-
|
| 365 |
-
-
|
| 366 |
-
|
|
|
|
| 367 |
"""
|
| 368 |
allowed = set(allowed_container_ids or [])
|
| 369 |
-
|
| 370 |
-
sources: List[Dict[str, Any]] = []
|
| 371 |
|
| 372 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 373 |
|
| 374 |
-
for hit in
|
| 375 |
container = cls._container(hit)
|
| 376 |
if allowed and container not in allowed:
|
| 377 |
continue
|
| 378 |
|
| 379 |
-
|
| 380 |
-
|
|
|
|
| 381 |
continue
|
| 382 |
|
| 383 |
text = cls._text(hit)
|
|
@@ -385,27 +874,118 @@ class AnswerComposer:
|
|
| 385 |
continue
|
| 386 |
|
| 387 |
key = cls._source_display_key(hit)
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 391 |
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 395 |
"container": container,
|
| 396 |
"section": cls._section(hit),
|
|
|
|
|
|
|
| 397 |
"page_range": cls._page_range(hit),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 398 |
"path": cls._hit_value(hit, "path", "section_path", default=""),
|
| 399 |
"score": round(cls._score(hit), 4),
|
| 400 |
-
"retrieval_kinds":
|
| 401 |
"chunk_index": cls._chunk_index(hit),
|
|
|
|
|
|
|
| 402 |
}
|
| 403 |
-
|
| 404 |
|
| 405 |
-
if
|
| 406 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 407 |
|
| 408 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
|
| 410 |
@staticmethod
|
| 411 |
def format_sources_markdown(sources: List[Dict[str, Any]]) -> str:
|
|
@@ -416,8 +996,29 @@ class AnswerComposer:
|
|
| 416 |
for source in sources:
|
| 417 |
container = source.get("container", "Unbekannt")
|
| 418 |
section = source.get("section", "ohne Abschnitt")
|
|
|
|
|
|
|
| 419 |
pages = source.get("page_range", "?")
|
| 420 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 421 |
return "\n".join(lines)
|
| 422 |
|
| 423 |
# ------------------------------------------------------------------
|
|
@@ -430,33 +1031,60 @@ class AnswerComposer:
|
|
| 430 |
section = cls._section(hit)
|
| 431 |
page_range = cls._page_range(hit)
|
| 432 |
chunk_index = cls._chunk_index(hit)
|
| 433 |
-
|
| 434 |
-
kind_text = ",".join(kinds) if kinds else "retrieved"
|
| 435 |
score = cls._score(hit)
|
| 436 |
text = cls._text(hit)
|
| 437 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 438 |
return (
|
| 439 |
-
f"[Quelle {index}]
|
| 440 |
-
f"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 441 |
f"{text}"
|
| 442 |
)
|
| 443 |
|
| 444 |
-
def _build_rag_context(self, hits: List[Dict[str, Any]]) -> Tuple[str, List[Dict[str, Any]]]:
|
| 445 |
-
prepared_hits = self._prepare_hits_for_context(hits)
|
| 446 |
if not prepared_hits:
|
| 447 |
return "", []
|
| 448 |
|
| 449 |
parts: List[str] = []
|
| 450 |
total = 0
|
| 451 |
|
|
|
|
| 452 |
for i, hit in enumerate(prepared_hits, start=1):
|
| 453 |
-
|
|
|
|
|
|
|
| 454 |
if self.max_context_chars and total + len(block) + 2 > self.max_context_chars:
|
| 455 |
break
|
| 456 |
parts.append(block)
|
|
|
|
| 457 |
total += len(block) + 2
|
| 458 |
|
| 459 |
-
used_hits = prepared_hits[: len(parts)]
|
| 460 |
return "\n\n".join(parts), used_hits
|
| 461 |
|
| 462 |
@staticmethod
|
|
@@ -502,11 +1130,256 @@ class AnswerComposer:
|
|
| 502 |
# Entfernt nur einen finalen Abschnitt "Quellen:" oder "Fundstellen:",
|
| 503 |
# nicht aber Quellenmarker im Fließtext wie [Quelle 1].
|
| 504 |
pattern = re.compile(
|
| 505 |
-
r"\n{1,3}(Quellen|Fundstellen)\s*:\s*
|
| 506 |
flags=re.I,
|
| 507 |
)
|
| 508 |
return pattern.sub("", text).strip()
|
| 509 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 510 |
# ------------------------------------------------------------------
|
| 511 |
# Öffentliche API
|
| 512 |
# ------------------------------------------------------------------
|
|
@@ -538,7 +1411,7 @@ class AnswerComposer:
|
|
| 538 |
"none",
|
| 539 |
)
|
| 540 |
|
| 541 |
-
rag_context, used_hits = self._build_rag_context(hits)
|
| 542 |
if not rag_context.strip():
|
| 543 |
return (
|
| 544 |
"Die gefundenen Treffer enthalten keinen verwertbaren Text. "
|
|
@@ -551,14 +1424,16 @@ class AnswerComposer:
|
|
| 551 |
rag_context=rag_context,
|
| 552 |
memory=memory,
|
| 553 |
)
|
| 554 |
-
answer = self.
|
|
|
|
| 555 |
|
| 556 |
if self.append_sources_to_answer:
|
| 557 |
sources = self.build_sources(
|
| 558 |
used_hits,
|
| 559 |
-
max_sources=self.max_sources,
|
| 560 |
-
include_pure_neighbors=self.include_pure_neighbors_as_sources,
|
| 561 |
allowed_container_ids=list(self.allowed_container_ids) if self.allowed_container_ids else None,
|
|
|
|
| 562 |
)
|
| 563 |
sources_md = self.format_sources_markdown(sources)
|
| 564 |
if sources_md:
|
|
@@ -579,7 +1454,7 @@ class AnswerComposer:
|
|
| 579 |
answer, source_type = self._answer_meta(question, memory)
|
| 580 |
return answer, source_type, []
|
| 581 |
|
| 582 |
-
rag_context, used_hits = self._build_rag_context(hits)
|
| 583 |
if not rag_context.strip():
|
| 584 |
return (
|
| 585 |
"Die gefundenen Treffer enthalten keinen verwertbaren Text. "
|
|
@@ -593,12 +1468,14 @@ class AnswerComposer:
|
|
| 593 |
rag_context=rag_context,
|
| 594 |
memory=memory,
|
| 595 |
)
|
| 596 |
-
answer = self.
|
|
|
|
| 597 |
|
| 598 |
sources = self.build_sources(
|
| 599 |
used_hits,
|
| 600 |
-
max_sources=self.max_sources,
|
| 601 |
-
include_pure_neighbors=self.include_pure_neighbors_as_sources,
|
| 602 |
allowed_container_ids=list(self.allowed_container_ids) if self.allowed_container_ids else None,
|
|
|
|
| 603 |
)
|
| 604 |
return answer, "document", sources
|
|
|
|
| 3 |
import re
|
| 4 |
from typing import Any, Dict, List, Optional, Tuple, Set
|
| 5 |
|
| 6 |
+
from llm_client_groq import ConversationMemory, is_meta_question
|
| 7 |
|
| 8 |
|
| 9 |
# ---------------------------------------------------------------------------
|
|
|
|
| 35 |
re.compile(r"\bgib\s+mir\s+die\s+antwort\b", re.I),
|
| 36 |
]
|
| 37 |
|
| 38 |
+
_SOURCE_MARKER_RE = re.compile(r"\[Quelle\s+(\d+)\]", re.I)
|
| 39 |
+
_SOURCE_MARKER_MULTI_RE = re.compile(r"\[Quellen\s+([\d,\s]+)\]", re.I)
|
| 40 |
+
_GRANULAR_BUCHSTABE_REF_RE = re.compile(
|
| 41 |
+
r"(§\s*\d{1,3}[a-z]?\s+Abs\.?\s*\d+[a-z]?)\s+"
|
| 42 |
+
r"(?:Buchst\.?|Buchstabe)\s*([a-z])",
|
| 43 |
+
re.I,
|
| 44 |
+
)
|
| 45 |
+
_DUPLICATE_PAREN_BASE_REF_RE = re.compile(
|
| 46 |
+
r"\((§\s*\d{1,3}[a-z]?\s+Abs\.?\s*\d+[a-z]?)(?:\s+(?:Buchst\.?|Buchstabe)\s*[a-z])\),\s*\1",
|
| 47 |
+
re.I,
|
| 48 |
+
)
|
| 49 |
+
_DUPLICATE_BASE_REF_RE = re.compile(
|
| 50 |
+
r"(§\s*\d{1,3}[a-z]?\s+Abs\.?\s*\d+[a-z]?)(?:\s+(?:Buchst\.?|Buchstabe)\s*[a-z]),\s*\1",
|
| 51 |
+
re.I,
|
| 52 |
+
)
|
| 53 |
+
_SECTION_REF_RE = re.compile(r"§\s*(\d{1,3}[a-z]?)(?!\d)", re.I)
|
| 54 |
+
_TOKEN_RE = re.compile(r"[a-zäöüß0-9]{3,}", re.I)
|
| 55 |
+
_STOPWORDS = {
|
| 56 |
+
"und", "oder", "der", "die", "das", "den", "dem", "des", "ein", "eine",
|
| 57 |
+
"einer", "einem", "einen", "für", "mit", "nach", "aus", "bei", "von",
|
| 58 |
+
"zur", "zum", "ist", "sind", "wird", "werden", "wann", "was", "wie",
|
| 59 |
+
"welche", "welcher", "welches", "apotheke", "rahmenvertrag", "vertrag",
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
_DOCUMENT_TASK_PROMPT = """
|
| 63 |
+
Du bist ein juristischer Assistent.
|
| 64 |
+
|
| 65 |
+
Beantworte ausschließlich anhand der bereitgestellten Textstellen.
|
| 66 |
+
|
| 67 |
+
=========================
|
| 68 |
+
EVIDENCE FIRST
|
| 69 |
+
=========================
|
| 70 |
+
|
| 71 |
+
Jede einzelne Aussage muss unmittelbar auf den bereitgestellten Quellen beruhen.
|
| 72 |
+
|
| 73 |
+
Verwende niemals eigenes Wissen.
|
| 74 |
+
|
| 75 |
+
Erfinde niemals
|
| 76 |
+
|
| 77 |
+
- Paragraphen
|
| 78 |
+
- Fundstellen
|
| 79 |
+
- Definitionen
|
| 80 |
+
- Rechtsfolgen
|
| 81 |
+
- Ausnahmen
|
| 82 |
+
- Beispiele
|
| 83 |
+
- Vermutungen.
|
| 84 |
+
|
| 85 |
+
Wenn sich eine Aussage nicht unmittelbar aus den Quellen ergibt,
|
| 86 |
+
darf sie nicht ausgegeben werden.
|
| 87 |
+
|
| 88 |
+
=========================
|
| 89 |
+
LEGALDEFINITIONEN
|
| 90 |
+
=========================
|
| 91 |
+
|
| 92 |
+
Wenn nach der Bedeutung eines Begriffes gefragt wird:
|
| 93 |
+
|
| 94 |
+
1. Suche zuerst nach einer Legaldefinition.
|
| 95 |
+
|
| 96 |
+
2. Gib diese wortlautnah wieder.
|
| 97 |
+
|
| 98 |
+
3. Erst danach darfst du eine kurze Einordnung geben.
|
| 99 |
+
|
| 100 |
+
Eigene Definitionen sind unzulässig.
|
| 101 |
+
|
| 102 |
+
=========================
|
| 103 |
+
AUFZÄHLUNGEN
|
| 104 |
+
=========================
|
| 105 |
+
|
| 106 |
+
Wenn eine Norm
|
| 107 |
+
|
| 108 |
+
- Voraussetzungen
|
| 109 |
+
- Tatbestandsmerkmale
|
| 110 |
+
- Fallgruppen
|
| 111 |
+
- Alternativen
|
| 112 |
+
- Kataloge
|
| 113 |
+
|
| 114 |
+
enthält,
|
| 115 |
+
|
| 116 |
+
müssen sämtliche Punkte vollständig wiedergegeben werden.
|
| 117 |
+
|
| 118 |
+
Verwende niemals
|
| 119 |
+
|
| 120 |
+
- unter anderem
|
| 121 |
+
- beispielsweise
|
| 122 |
+
- insbesondere
|
| 123 |
+
|
| 124 |
+
wenn die Norm abschließend formuliert ist.
|
| 125 |
+
|
| 126 |
+
=========================
|
| 127 |
+
NICHT GEREGELT
|
| 128 |
+
=========================
|
| 129 |
+
|
| 130 |
+
Schreibe
|
| 131 |
+
|
| 132 |
+
"nicht geregelt"
|
| 133 |
+
|
| 134 |
+
nur wenn
|
| 135 |
+
|
| 136 |
+
- keine einschlägige Quelle vorhanden ist
|
| 137 |
+
|
| 138 |
+
UND
|
| 139 |
+
|
| 140 |
+
- keine Legaldefinition vorhanden ist
|
| 141 |
+
|
| 142 |
+
UND
|
| 143 |
+
|
| 144 |
+
- keine andere bereitgestellte Norm die Frage beantwortet.
|
| 145 |
+
|
| 146 |
+
=========================
|
| 147 |
+
VORHANDENE PARAGRAPHEN NIEMALS LEUGNEN
|
| 148 |
+
=========================
|
| 149 |
+
|
| 150 |
+
Wenn eine bereitgestellte Textstelle den gefragten Paragraphen enthält
|
| 151 |
+
(erkennbar am Feld "Abschnitt:" oder "Norm:" der Quelle, z. B. § 23),
|
| 152 |
+
dann IST dieser Paragraph vorhanden.
|
| 153 |
+
|
| 154 |
+
Sage in diesem Fall NIEMALS, der Paragraph sei
|
| 155 |
+
- "nicht vorhanden"
|
| 156 |
+
- "nicht enthalten"
|
| 157 |
+
- "nicht in den bereitgestellten Quellen"
|
| 158 |
+
- "nicht geregelt"
|
| 159 |
+
|
| 160 |
+
und schreibe NIEMALS "es gibt keine Regelung/Aussage/Information zu diesem
|
| 161 |
+
Paragraphen".
|
| 162 |
+
|
| 163 |
+
Gib stattdessen wieder, was die Textstelle tatsächlich sagt — auch wenn sie kurz
|
| 164 |
+
ist oder nur aus einer Verweisung besteht.
|
| 165 |
+
|
| 166 |
+
=========================
|
| 167 |
+
VERWEISUNGEN AUF EXTERNE NORMEN
|
| 168 |
+
=========================
|
| 169 |
+
|
| 170 |
+
Manche Paragraphen regeln einen Sachverhalt nicht selbst, sondern verweisen auf
|
| 171 |
+
eine externe Norm (z. B. "Der Apothekenabschlag richtet sich nach § 130 SGB V").
|
| 172 |
+
|
| 173 |
+
Wenn die bereitgestellte Textstelle nur auf eine externe Norm verweist:
|
| 174 |
+
|
| 175 |
+
- Gib den Paragraphen und seine Verweisung wieder, mit dem Quellenmarker des
|
| 176 |
+
Paragraphen (z. B. "§ 23 bestimmt, dass sich der Apothekenabschlag nach
|
| 177 |
+
§ 130 SGB V richtet [Quelle n]").
|
| 178 |
+
- Weise darauf hin, dass die Einzelheiten in der verwiesenen Norm geregelt sind
|
| 179 |
+
und diese Norm nicht Teil der bereitgestellten Texte ist.
|
| 180 |
+
|
| 181 |
+
Das ist eine gültige, vollständige Antwort und NICHT "nicht geregelt".
|
| 182 |
+
|
| 183 |
+
=========================
|
| 184 |
+
QUELLEN
|
| 185 |
+
=========================
|
| 186 |
+
|
| 187 |
+
Verwende ausschließlich vorhandene Marker.
|
| 188 |
+
|
| 189 |
+
Jede tragende Aussage erhält mindestens einen Quellenmarker.
|
| 190 |
+
|
| 191 |
+
Erfinde niemals Quellenmarker.
|
| 192 |
+
|
| 193 |
+
Erfinde niemals Paragraphen.
|
| 194 |
+
|
| 195 |
+
=========================
|
| 196 |
+
ANTWORTSCHEMA
|
| 197 |
+
=========================
|
| 198 |
+
|
| 199 |
+
Gliedere die Antwort mit genau diesen Abschnitts-Überschriften.
|
| 200 |
+
|
| 201 |
+
Jede Überschrift steht auf einer eigenen Zeile und endet mit einem Doppelpunkt.
|
| 202 |
+
Der zugehörige Inhalt folgt in der nächsten Zeile.
|
| 203 |
+
Zwischen den Abschnitten steht eine Leerzeile.
|
| 204 |
+
|
| 205 |
+
Verwende reinen Text, KEIN Markdown:
|
| 206 |
+
kein #, kein *, kein Fettdruck, keine Nummerierung der Überschriften.
|
| 207 |
+
|
| 208 |
+
Kurzantwort:
|
| 209 |
+
- Das Ergebnis in 1–3 Sätzen, mit Quellenmarker.
|
| 210 |
+
|
| 211 |
+
Maßgebliche Norm:
|
| 212 |
+
- Die einschlägige(n) Vorschrift(en), z. B. § 9 Abs. 3, mit Quellenmarker.
|
| 213 |
+
|
| 214 |
+
Wortlaut / Kriterien:
|
| 215 |
+
- Den maßgeblichen Normtext wortlautnah wiedergeben.
|
| 216 |
+
- Aufzählungen (Buchstaben a–z, Nummern, Absätze) VOLLSTÄNDIG wiedergeben.
|
| 217 |
+
|
| 218 |
+
Einordnung:
|
| 219 |
+
- Nur, wenn zum Verständnis erforderlich und durch die Quellen gedeckt.
|
| 220 |
+
- Andernfalls diesen Abschnitt inklusive Überschrift vollständig weglassen.
|
| 221 |
+
|
| 222 |
+
Regeln zum Schema:
|
| 223 |
+
|
| 224 |
+
- Abschnitte ohne Evidenz vollständig weglassen (inklusive Überschrift).
|
| 225 |
+
- Keine Ergebnisformel, kein Fazit, keine Zusammenfassung am Ende.
|
| 226 |
+
- Keine hypothetischen Ausnahmen.
|
| 227 |
+
- Keine freien Schlussfolgerungen.
|
| 228 |
+
|
| 229 |
+
=========================
|
| 230 |
+
STIL
|
| 231 |
+
=========================
|
| 232 |
+
|
| 233 |
+
Präzise.
|
| 234 |
+
|
| 235 |
+
Juristisch belastbar.
|
| 236 |
+
|
| 237 |
+
Keine Wiederholungen.
|
| 238 |
+
|
| 239 |
+
Keine Quellenliste am Ende.
|
| 240 |
""".strip()
|
| 241 |
|
| 242 |
|
|
|
|
| 289 |
include_neighbor_hits_in_context: bool = True,
|
| 290 |
include_pure_neighbors_as_sources: bool = False,
|
| 291 |
append_sources_to_answer: bool = False,
|
| 292 |
+
display_all_context_sources: bool = True,
|
| 293 |
+
validate_source_markers: bool = True,
|
| 294 |
+
prefer_direct_hits_over_neighbors: bool = True,
|
| 295 |
):
|
| 296 |
self.llm = llm_client
|
| 297 |
self.max_context_chars = max_context_chars
|
|
|
|
| 304 |
self.include_neighbor_hits_in_context = include_neighbor_hits_in_context
|
| 305 |
self.include_pure_neighbors_as_sources = include_pure_neighbors_as_sources
|
| 306 |
self.append_sources_to_answer = append_sources_to_answer
|
| 307 |
+
self.display_all_context_sources = display_all_context_sources
|
| 308 |
+
self.validate_source_markers = validate_source_markers
|
| 309 |
+
self.prefer_direct_hits_over_neighbors = prefer_direct_hits_over_neighbors
|
| 310 |
|
| 311 |
# ------------------------------------------------------------------
|
| 312 |
# Meta-Hilfsmethoden
|
|
|
|
| 408 |
return str(start)
|
| 409 |
return "?"
|
| 410 |
|
| 411 |
+
@staticmethod
|
| 412 |
+
def _int_or_none(value: Any) -> Optional[int]:
|
| 413 |
+
try:
|
| 414 |
+
return int(value)
|
| 415 |
+
except (TypeError, ValueError):
|
| 416 |
+
return None
|
| 417 |
+
|
| 418 |
+
@classmethod
|
| 419 |
+
def _page_bounds(cls, hit: Dict[str, Any]) -> Tuple[Optional[int], Optional[int]]:
|
| 420 |
+
start = cls._int_or_none(cls._hit_value(hit, "page_start", default=None))
|
| 421 |
+
end = cls._int_or_none(cls._hit_value(hit, "page_end", default=None))
|
| 422 |
+
if end is None:
|
| 423 |
+
end = start
|
| 424 |
+
return start, end
|
| 425 |
+
|
| 426 |
+
@classmethod
|
| 427 |
+
def _highlight_text(cls, hit: Dict[str, Any], *, max_chars: int = 2000) -> str:
|
| 428 |
+
"""Chunk-Text für die PDF-Fundstellen-Hervorhebung.
|
| 429 |
+
|
| 430 |
+
Die Ingest-Pipeline stellt jedem Chunk eine konstruierte Kontextzeile
|
| 431 |
+
wie "Vertrag · § 6 Titel" voran, die so nicht im PDF steht. Sie wird
|
| 432 |
+
entfernt, damit der Viewer nur echten Dokumenttext matchen muss.
|
| 433 |
+
"""
|
| 434 |
+
text = cls._text(hit)
|
| 435 |
+
if not text:
|
| 436 |
+
return ""
|
| 437 |
+
head, sep, rest = text.partition("\n\n")
|
| 438 |
+
if sep and " · " in head and len(head) <= 200 and rest.strip():
|
| 439 |
+
text = rest.strip()
|
| 440 |
+
return text[:max_chars]
|
| 441 |
+
|
| 442 |
@classmethod
|
| 443 |
def _container(cls, hit: Dict[str, Any]) -> str:
|
| 444 |
return str(cls._hit_value(hit, "container", "container_id", default="Unbekannt"))
|
|
|
|
| 451 |
def _chunk_index(cls, hit: Dict[str, Any]) -> Any:
|
| 452 |
return cls._hit_value(hit, "chunk_index", "chunk_index_in_section", default="?")
|
| 453 |
|
| 454 |
+
@classmethod
|
| 455 |
+
def _canonical_ref(cls, hit: Dict[str, Any]) -> str:
|
| 456 |
+
direct = cls._hit_value(hit, "canonical_ref", default="")
|
| 457 |
+
if direct:
|
| 458 |
+
return str(direct)
|
| 459 |
+
|
| 460 |
+
metadata = hit.get("metadata") or {}
|
| 461 |
+
paragraph = metadata.get("paragraph") or hit.get("paragraph") or cls._section(hit)
|
| 462 |
+
subsection = metadata.get("subsection") or hit.get("subsection")
|
| 463 |
+
sentence = metadata.get("sentence") or hit.get("sentence")
|
| 464 |
+
number = metadata.get("number") or hit.get("number")
|
| 465 |
+
letter = metadata.get("letter") or hit.get("letter")
|
| 466 |
+
|
| 467 |
+
parts: List[str] = [str(paragraph)] if paragraph else []
|
| 468 |
+
if subsection:
|
| 469 |
+
parts.append(f"Abs. {subsection}")
|
| 470 |
+
if sentence:
|
| 471 |
+
parts.append(f"Satz {sentence}")
|
| 472 |
+
if number:
|
| 473 |
+
parts.append(f"Nr. {number}")
|
| 474 |
+
if letter:
|
| 475 |
+
parts.append(f"Buchst. {str(letter).lower()}")
|
| 476 |
+
return " ".join(parts)
|
| 477 |
+
|
| 478 |
@classmethod
|
| 479 |
def _text(cls, hit: Dict[str, Any]) -> str:
|
| 480 |
return str(hit.get("text") or hit.get("document") or "").strip()
|
|
|
|
| 493 |
except (TypeError, ValueError):
|
| 494 |
return 0.0
|
| 495 |
|
| 496 |
+
@classmethod
|
| 497 |
+
def _source_number(cls, hit: Dict[str, Any]) -> Optional[int]:
|
| 498 |
+
value = cls._hit_value(hit, "source_number", default=None)
|
| 499 |
+
try:
|
| 500 |
+
return int(value) if value is not None else None
|
| 501 |
+
except (TypeError, ValueError):
|
| 502 |
+
return None
|
| 503 |
+
|
| 504 |
+
@classmethod
|
| 505 |
+
def _is_pure_neighbor(cls, hit: Dict[str, Any]) -> bool:
|
| 506 |
+
kinds = set(cls._retrieval_kinds(hit))
|
| 507 |
+
return bool(kinds) and kinds == {"neighbor"}
|
| 508 |
+
|
| 509 |
+
@classmethod
|
| 510 |
+
def _is_direct_evidence(cls, hit: Dict[str, Any]) -> bool:
|
| 511 |
+
kinds = set(cls._retrieval_kinds(hit))
|
| 512 |
+
if not kinds:
|
| 513 |
+
return True
|
| 514 |
+
return not cls._is_pure_neighbor(hit)
|
| 515 |
+
|
| 516 |
+
@staticmethod
|
| 517 |
+
def _query_tokens(question: str) -> Set[str]:
|
| 518 |
+
tokens = {t.lower() for t in _TOKEN_RE.findall(question or "")}
|
| 519 |
+
return {t for t in tokens if t not in _STOPWORDS and not t.isdigit()}
|
| 520 |
+
|
| 521 |
+
@staticmethod
|
| 522 |
+
def _section_refs_from_question(question: str) -> Set[str]:
|
| 523 |
+
return {f"§ {m.group(1)}" for m in _SECTION_REF_RE.finditer(question or "")}
|
| 524 |
+
|
| 525 |
+
@classmethod
|
| 526 |
+
def _lexical_overlap(cls, question: str, hit: Dict[str, Any]) -> int:
|
| 527 |
+
query_terms = cls._query_tokens(question)
|
| 528 |
+
if not query_terms:
|
| 529 |
+
return 0
|
| 530 |
+
haystack = " ".join(
|
| 531 |
+
[
|
| 532 |
+
cls._canonical_ref(hit),
|
| 533 |
+
cls._section(hit),
|
| 534 |
+
cls._text(hit)[:2500],
|
| 535 |
+
]
|
| 536 |
+
).lower()
|
| 537 |
+
return sum(1 for term in query_terms if term in haystack)
|
| 538 |
+
|
| 539 |
+
@classmethod
|
| 540 |
+
def _specificity_score(cls, hit: Dict[str, Any]) -> int:
|
| 541 |
+
ref = cls._canonical_ref(hit).lower()
|
| 542 |
+
score = 0
|
| 543 |
+
if "abs." in ref or "absatz" in ref:
|
| 544 |
+
score += 2
|
| 545 |
+
if "satz" in ref:
|
| 546 |
+
score += 1
|
| 547 |
+
if "nr." in ref or "buchst." in ref:
|
| 548 |
+
score += 1
|
| 549 |
+
if cls._hit_value(hit, "unit_type", default="") == "definition":
|
| 550 |
+
score += 1
|
| 551 |
+
if cls._hit_value(hit, "chunk_kind", default="") == "parent":
|
| 552 |
+
score += 1
|
| 553 |
+
return score
|
| 554 |
+
|
| 555 |
+
def _hit_relevance_key(self, hit: Dict[str, Any], question: str = "") -> Tuple[Any, ...]:
|
| 556 |
+
kinds = set(self._retrieval_kinds(hit))
|
| 557 |
+
q_sections = self._section_refs_from_question(question)
|
| 558 |
+
section = self._section(hit)
|
| 559 |
+
canonical = self._canonical_ref(hit)
|
| 560 |
+
|
| 561 |
+
explicit = int(bool({"explicit_section", "section_lookup", "exact_reference", "definition_lookup"} & kinds))
|
| 562 |
+
direct = int(self._is_direct_evidence(hit)) if self.prefer_direct_hits_over_neighbors else 0
|
| 563 |
+
section_match = int(bool(q_sections and (section in q_sections or any(ref in canonical for ref in q_sections))))
|
| 564 |
+
definition = int(self._hit_value(hit, "unit_type", default="") == "definition")
|
| 565 |
+
specificity = self._specificity_score(hit)
|
| 566 |
+
overlap = self._lexical_overlap(question, hit)
|
| 567 |
+
pure_neighbor_penalty = -1 if self._is_pure_neighbor(hit) else 0
|
| 568 |
+
chunk_idx = self._chunk_index(hit)
|
| 569 |
+
try:
|
| 570 |
+
chunk_sort = -int(chunk_idx)
|
| 571 |
+
except (TypeError, ValueError):
|
| 572 |
+
chunk_sort = 0
|
| 573 |
+
|
| 574 |
+
return (
|
| 575 |
+
definition * 100,
|
| 576 |
+
explicit * 50,
|
| 577 |
+
section_match * 20,
|
| 578 |
+
direct * 10,
|
| 579 |
+
specificity * 5,
|
| 580 |
+
overlap,
|
| 581 |
+
self._score(hit),
|
| 582 |
+
pure_neighbor_penalty,
|
| 583 |
+
chunk_sort,
|
| 584 |
+
)
|
| 585 |
+
|
| 586 |
@classmethod
|
| 587 |
def _source_key(cls, hit: Dict[str, Any]) -> Tuple[Any, ...]:
|
| 588 |
metadata = hit.get("metadata") or {}
|
|
|
|
| 630 |
|
| 631 |
return True
|
| 632 |
|
| 633 |
+
def _prepare_hits_for_context(
|
| 634 |
+
self,
|
| 635 |
+
hits: List[Dict[str, Any]],
|
| 636 |
+
question: str = "",
|
| 637 |
+
) -> List[Dict[str, Any]]:
|
| 638 |
+
"""
|
| 639 |
+
Filtert, priorisiert und begrenzt Treffer für den LLM-Kontext.
|
| 640 |
+
|
| 641 |
+
Priorität:
|
| 642 |
+
|
| 643 |
+
1. Legaldefinitionen
|
| 644 |
+
2. explizite Paragraphentreffer
|
| 645 |
+
3. direkte Evidenz
|
| 646 |
+
4. Parent-Chunks
|
| 647 |
+
5. hohe Textübereinstimmung
|
| 648 |
+
6. Retrieval-Score
|
| 649 |
+
"""
|
| 650 |
+
|
| 651 |
if not hits:
|
| 652 |
return []
|
| 653 |
|
| 654 |
+
filtered = [
|
| 655 |
+
hit
|
| 656 |
+
for hit in hits
|
| 657 |
+
if self._should_keep_for_context(hit)
|
| 658 |
+
]
|
| 659 |
+
|
| 660 |
+
#
|
| 661 |
+
# Deduplizieren
|
| 662 |
+
#
|
| 663 |
+
|
| 664 |
+
deduped = {}
|
| 665 |
|
|
|
|
|
|
|
| 666 |
for hit in filtered:
|
| 667 |
+
|
| 668 |
key = self._source_key(hit)
|
| 669 |
+
|
| 670 |
+
if (
|
| 671 |
+
key not in deduped
|
| 672 |
+
or self._score(hit) > self._score(deduped[key])
|
| 673 |
+
):
|
| 674 |
deduped[key] = hit
|
| 675 |
|
| 676 |
prepared = list(deduped.values())
|
| 677 |
|
| 678 |
+
#
|
| 679 |
+
# Priorisierung
|
| 680 |
+
#
|
| 681 |
+
|
| 682 |
+
# Paragraphen, die die Frage ausdrücklich nennt (z. B. "§ 23"), müssen
|
| 683 |
+
# zwingend an die Spitze des Kontexts. Sonst kann ein exakt passender,
|
| 684 |
+
# aber kurzer Chunk (etwa § 23 Apothekenabschlag) hinter semantisch
|
| 685 |
+
# ähnlichen Nachbarn landen, das LLM liest ihn nicht und behauptet
|
| 686 |
+
# fälschlich, der Paragraph sei "nicht geregelt/nicht vorhanden".
|
| 687 |
+
q_sections = self._section_refs_from_question(question)
|
| 688 |
+
|
| 689 |
+
def ranking(hit):
|
| 690 |
+
|
| 691 |
+
retrieval = set(self._retrieval_kinds(hit))
|
| 692 |
+
|
| 693 |
+
return (
|
| 694 |
+
|
| 695 |
+
#
|
| 696 |
+
# allerhöchste Priorität:
|
| 697 |
+
# exakt der in der Frage genannte Paragraph
|
| 698 |
+
#
|
| 699 |
+
|
| 700 |
+
bool(q_sections and self._section(hit) in q_sections),
|
| 701 |
+
|
| 702 |
+
#
|
| 703 |
+
# höchste Priorität
|
| 704 |
+
#
|
| 705 |
+
|
| 706 |
+
self._hit_value(hit, "unit_type", default="") == "definition",
|
| 707 |
+
|
| 708 |
+
#
|
| 709 |
+
# expliziter Paragraph
|
| 710 |
+
#
|
| 711 |
+
|
| 712 |
+
bool(
|
| 713 |
+
{
|
| 714 |
+
"explicit_section",
|
| 715 |
+
"section_lookup",
|
| 716 |
+
"definition_lookup",
|
| 717 |
+
"exact_reference",
|
| 718 |
+
}
|
| 719 |
+
& retrieval
|
| 720 |
+
),
|
| 721 |
+
|
| 722 |
+
#
|
| 723 |
+
# direkte Evidenz
|
| 724 |
+
#
|
| 725 |
+
|
| 726 |
+
self._is_direct_evidence(hit),
|
| 727 |
+
|
| 728 |
+
#
|
| 729 |
+
# Parent-Chunks bevorzugen
|
| 730 |
+
#
|
| 731 |
+
|
| 732 |
+
self._hit_value(
|
| 733 |
+
hit,
|
| 734 |
+
"chunk_kind",
|
| 735 |
+
default="",
|
| 736 |
+
)
|
| 737 |
+
== "parent",
|
| 738 |
+
|
| 739 |
+
#
|
| 740 |
+
# spezifische Fundstelle
|
| 741 |
+
#
|
| 742 |
+
|
| 743 |
+
self._specificity_score(hit),
|
| 744 |
+
|
| 745 |
+
#
|
| 746 |
+
# lexikalische Übereinstimmung
|
| 747 |
+
#
|
| 748 |
+
|
| 749 |
+
self._lexical_overlap(
|
| 750 |
+
question,
|
| 751 |
+
hit,
|
| 752 |
+
),
|
| 753 |
+
|
| 754 |
+
#
|
| 755 |
+
# Retrievalscore
|
| 756 |
+
#
|
| 757 |
+
|
| 758 |
+
self._score(hit),
|
| 759 |
+
|
| 760 |
+
#
|
| 761 |
+
# Neighbor zuletzt
|
| 762 |
+
#
|
| 763 |
+
|
| 764 |
+
not self._is_pure_neighbor(hit),
|
| 765 |
+
|
| 766 |
+
)
|
| 767 |
|
| 768 |
prepared.sort(
|
| 769 |
+
key=ranking,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 770 |
reverse=True,
|
| 771 |
)
|
| 772 |
|
| 773 |
+
#
|
| 774 |
+
# Maximal x Chunks pro Paragraph
|
| 775 |
+
#
|
| 776 |
+
|
| 777 |
+
section_limited = []
|
| 778 |
+
|
| 779 |
+
per_section_count = {}
|
| 780 |
+
|
| 781 |
for hit in prepared:
|
| 782 |
+
|
| 783 |
+
key = (
|
| 784 |
+
self._container(hit),
|
| 785 |
+
self._section(hit),
|
| 786 |
+
)
|
| 787 |
+
|
| 788 |
+
if (
|
| 789 |
+
per_section_count.get(key, 0)
|
| 790 |
+
>= self.max_chunks_per_section
|
| 791 |
+
):
|
| 792 |
continue
|
| 793 |
+
|
| 794 |
+
per_section_count[key] = (
|
| 795 |
+
per_section_count.get(key, 0)
|
| 796 |
+
+ 1
|
| 797 |
+
)
|
| 798 |
+
|
| 799 |
section_limited.append(hit)
|
| 800 |
|
| 801 |
+
#
|
| 802 |
+
# Sicherstellen,
|
| 803 |
+
# dass mindestens eine Definition enthalten bleibt
|
| 804 |
+
#
|
| 805 |
+
|
| 806 |
+
definitions = [
|
| 807 |
+
h
|
| 808 |
+
for h in prepared
|
| 809 |
+
if self._hit_value(
|
| 810 |
+
h,
|
| 811 |
+
"unit_type",
|
| 812 |
+
default="",
|
| 813 |
+
)
|
| 814 |
+
== "definition"
|
| 815 |
+
]
|
| 816 |
+
|
| 817 |
+
if definitions:
|
| 818 |
+
|
| 819 |
+
first_definition = definitions[0]
|
| 820 |
+
|
| 821 |
+
if first_definition not in section_limited:
|
| 822 |
+
|
| 823 |
+
section_limited.insert(
|
| 824 |
+
0,
|
| 825 |
+
first_definition,
|
| 826 |
+
)
|
| 827 |
+
|
| 828 |
+
#
|
| 829 |
+
# Kontext begrenzen
|
| 830 |
+
#
|
| 831 |
+
|
| 832 |
return section_limited[: self.max_hits_for_context]
|
| 833 |
|
| 834 |
@classmethod
|
|
|
|
| 839 |
max_sources: int = 5,
|
| 840 |
include_pure_neighbors: bool = False,
|
| 841 |
allowed_container_ids: Optional[List[str]] = None,
|
| 842 |
+
referenced_source_numbers: Optional[Set[int]] = None,
|
| 843 |
) -> List[Dict[str, Any]]:
|
| 844 |
"""
|
| 845 |
+
Baut eine kuratierte, nummern-stabile Quellenliste für die Anzeige.
|
| 846 |
|
| 847 |
Wichtig:
|
| 848 |
+
- Quellenmarker aus dem RAG-Kontext bleiben stabil erhalten.
|
| 849 |
+
- Mehrere Chunks derselben Section/Seiten werden gruppiert, aber die
|
| 850 |
+
zugehörigen [Quelle n]-Nummern bleiben sichtbar.
|
| 851 |
+
- Reine Neighbor-Treffer werden standardmäßig nicht als Quellen angezeigt.
|
| 852 |
"""
|
| 853 |
allowed = set(allowed_container_ids or [])
|
| 854 |
+
grouped: Dict[Tuple[Any, Any, Any], Dict[str, Any]] = {}
|
|
|
|
| 855 |
|
| 856 |
+
def order_key(hit: Dict[str, Any]) -> Tuple[int, float]:
|
| 857 |
+
number = cls._source_number(hit)
|
| 858 |
+
if number is not None:
|
| 859 |
+
return (number, 0.0)
|
| 860 |
+
return (10_000, -cls._score(hit))
|
| 861 |
|
| 862 |
+
for hit in sorted(hits, key=order_key):
|
| 863 |
container = cls._container(hit)
|
| 864 |
if allowed and container not in allowed:
|
| 865 |
continue
|
| 866 |
|
| 867 |
+
number = cls._source_number(hit)
|
| 868 |
+
is_referenced = bool(referenced_source_numbers and number in referenced_source_numbers)
|
| 869 |
+
if cls._is_pure_neighbor(hit) and not include_pure_neighbors and not is_referenced:
|
| 870 |
continue
|
| 871 |
|
| 872 |
text = cls._text(hit)
|
|
|
|
| 874 |
continue
|
| 875 |
|
| 876 |
key = cls._source_display_key(hit)
|
| 877 |
+
number = cls._source_number(hit)
|
| 878 |
+
canonical = cls._canonical_ref(hit)
|
| 879 |
+
kinds = cls._retrieval_kinds(hit)
|
| 880 |
+
|
| 881 |
+
# PDF-Locator: Datei, numerische Seiten und Hervorhebungstext pro
|
| 882 |
+
# Chunk, damit die UI die Fundstelle im PDF ansteuern kann.
|
| 883 |
+
page_start, page_end = cls._page_bounds(hit)
|
| 884 |
+
source_file = str(cls._hit_value(hit, "source_file", default="") or "")
|
| 885 |
+
doc_id = str(cls._hit_value(hit, "doc_id", default="") or "")
|
| 886 |
+
highlight = cls._highlight_text(hit)
|
| 887 |
+
highlight_entry = (
|
| 888 |
+
{"page_start": page_start, "page_end": page_end, "text": highlight}
|
| 889 |
+
if highlight
|
| 890 |
+
else None
|
| 891 |
+
)
|
| 892 |
|
| 893 |
+
item = grouped.get(key)
|
| 894 |
+
if item is None:
|
| 895 |
+
grouped[key] = {
|
| 896 |
+
"source_number": number,
|
| 897 |
+
"source_numbers": [number] if number is not None else [],
|
| 898 |
+
"source_marker": f"[Quelle {number}]" if number is not None else "",
|
| 899 |
+
"source_label": f"[Quelle {number}]" if number is not None else "",
|
| 900 |
"container": container,
|
| 901 |
"section": cls._section(hit),
|
| 902 |
+
"canonical_ref": canonical,
|
| 903 |
+
"canonical_refs": [canonical] if canonical else [],
|
| 904 |
"page_range": cls._page_range(hit),
|
| 905 |
+
"page_start": page_start,
|
| 906 |
+
"page_end": page_end,
|
| 907 |
+
"source_file": source_file,
|
| 908 |
+
"doc_id": doc_id,
|
| 909 |
+
"highlights": [highlight_entry] if highlight_entry else [],
|
| 910 |
"path": cls._hit_value(hit, "path", "section_path", default=""),
|
| 911 |
"score": round(cls._score(hit), 4),
|
| 912 |
+
"retrieval_kinds": list(dict.fromkeys(kinds)),
|
| 913 |
"chunk_index": cls._chunk_index(hit),
|
| 914 |
+
"display_title": "",
|
| 915 |
+
"display_label": "",
|
| 916 |
}
|
| 917 |
+
continue
|
| 918 |
|
| 919 |
+
if page_start is not None and (item.get("page_start") is None or page_start < item["page_start"]):
|
| 920 |
+
item["page_start"] = page_start
|
| 921 |
+
if page_end is not None and (item.get("page_end") is None or page_end > item["page_end"]):
|
| 922 |
+
item["page_end"] = page_end
|
| 923 |
+
if source_file and not item.get("source_file"):
|
| 924 |
+
item["source_file"] = source_file
|
| 925 |
+
if doc_id and not item.get("doc_id"):
|
| 926 |
+
item["doc_id"] = doc_id
|
| 927 |
+
if highlight_entry is not None:
|
| 928 |
+
existing_texts = {(h.get("text") or "")[:120] for h in item.get("highlights") or []}
|
| 929 |
+
if highlight[:120] not in existing_texts:
|
| 930 |
+
item.setdefault("highlights", []).append(highlight_entry)
|
| 931 |
+
|
| 932 |
+
if number is not None and number not in item["source_numbers"]:
|
| 933 |
+
item["source_numbers"].append(number)
|
| 934 |
+
item["source_numbers"].sort()
|
| 935 |
+
item["source_number"] = item["source_numbers"][0]
|
| 936 |
+
if len(item["source_numbers"]) == 1:
|
| 937 |
+
item["source_marker"] = f"[Quelle {item['source_numbers'][0]}]"
|
| 938 |
+
item["source_label"] = item["source_marker"]
|
| 939 |
+
else:
|
| 940 |
+
joined = ", ".join(str(n) for n in item["source_numbers"])
|
| 941 |
+
item["source_marker"] = f"[Quellen {joined}]"
|
| 942 |
+
item["source_label"] = item["source_marker"]
|
| 943 |
+
|
| 944 |
+
if canonical and canonical not in item["canonical_refs"]:
|
| 945 |
+
item["canonical_refs"].append(canonical)
|
| 946 |
+
# Behalte als Hauptfundstelle die spezifischste Referenz.
|
| 947 |
+
item["canonical_ref"] = max(item["canonical_refs"], key=lambda r: ("Abs." in r, "Satz" in r, len(r)))
|
| 948 |
+
|
| 949 |
+
item["score"] = max(float(item.get("score", 0.0)), round(cls._score(hit), 4))
|
| 950 |
+
for kind in kinds:
|
| 951 |
+
if kind not in item["retrieval_kinds"]:
|
| 952 |
+
item["retrieval_kinds"].append(kind)
|
| 953 |
+
|
| 954 |
+
sources = list(grouped.values())
|
| 955 |
|
| 956 |
+
for source in sources:
|
| 957 |
+
numbers = source.get("source_numbers") or []
|
| 958 |
+
if numbers:
|
| 959 |
+
if len(numbers) == 1:
|
| 960 |
+
label = f"[Quelle {numbers[0]}]"
|
| 961 |
+
else:
|
| 962 |
+
label = "[Quellen " + ", ".join(str(n) for n in numbers) + "]"
|
| 963 |
+
else:
|
| 964 |
+
label = ""
|
| 965 |
+
source["source_marker"] = label
|
| 966 |
+
source["source_label"] = label
|
| 967 |
+
canonical_refs = source.get("canonical_refs") or []
|
| 968 |
+
canonical = "; ".join(canonical_refs[:3]) if canonical_refs else source.get("canonical_ref")
|
| 969 |
+
if canonical and canonical != source.get("section"):
|
| 970 |
+
source["display_title"] = f"{label} {source.get('container')}::{source.get('section')} ({canonical}), Seiten {source.get('page_range')}".strip()
|
| 971 |
+
else:
|
| 972 |
+
source["display_title"] = f"{label} {source.get('container')}::{source.get('section')}, Seiten {source.get('page_range')}".strip()
|
| 973 |
+
source["display_label"] = source["display_title"]
|
| 974 |
+
|
| 975 |
+
sources.sort(key=lambda s: (s.get("source_numbers") or [10_000])[0])
|
| 976 |
+
|
| 977 |
+
referenced = set(referenced_source_numbers or set())
|
| 978 |
+
if referenced:
|
| 979 |
+
referenced_sources = [
|
| 980 |
+
s for s in sources
|
| 981 |
+
if referenced.intersection(set(s.get("source_numbers") or []))
|
| 982 |
+
]
|
| 983 |
+
other_sources = [s for s in sources if s not in referenced_sources]
|
| 984 |
+
# Never drop sources that are cited in the answer. The cap only limits extras.
|
| 985 |
+
budget = max(int(max_sources), len(referenced_sources))
|
| 986 |
+
return (referenced_sources + other_sources[: max(0, budget - len(referenced_sources))])
|
| 987 |
+
|
| 988 |
+
return sources[:max_sources]
|
| 989 |
|
| 990 |
@staticmethod
|
| 991 |
def format_sources_markdown(sources: List[Dict[str, Any]]) -> str:
|
|
|
|
| 996 |
for source in sources:
|
| 997 |
container = source.get("container", "Unbekannt")
|
| 998 |
section = source.get("section", "ohne Abschnitt")
|
| 999 |
+
canonical_refs = source.get("canonical_refs") or []
|
| 1000 |
+
canonical = "; ".join(canonical_refs[:3]) if canonical_refs else source.get("canonical_ref")
|
| 1001 |
pages = source.get("page_range", "?")
|
| 1002 |
+
marker = source.get("source_marker") or source.get("source_label") or ""
|
| 1003 |
+
if not marker:
|
| 1004 |
+
numbers = source.get("source_numbers") or []
|
| 1005 |
+
if numbers:
|
| 1006 |
+
if len(numbers) == 1:
|
| 1007 |
+
marker = f"[Quelle {numbers[0]}]"
|
| 1008 |
+
else:
|
| 1009 |
+
marker = "[Quellen " + ", ".join(str(n) for n in numbers) + "]"
|
| 1010 |
+
else:
|
| 1011 |
+
marker = "-"
|
| 1012 |
+
|
| 1013 |
+
role = ""
|
| 1014 |
+
kinds = set(source.get("retrieval_kinds") or [])
|
| 1015 |
+
if kinds == {"neighbor"}:
|
| 1016 |
+
role = " · Kontext/Nachbar"
|
| 1017 |
+
|
| 1018 |
+
if canonical and canonical != section:
|
| 1019 |
+
lines.append(f"- {marker} {container}::{section} ({canonical}), Seiten {pages}{role}")
|
| 1020 |
+
else:
|
| 1021 |
+
lines.append(f"- {marker} {container}::{section}, Seiten {pages}{role}")
|
| 1022 |
return "\n".join(lines)
|
| 1023 |
|
| 1024 |
# ------------------------------------------------------------------
|
|
|
|
| 1031 |
section = cls._section(hit)
|
| 1032 |
page_range = cls._page_range(hit)
|
| 1033 |
chunk_index = cls._chunk_index(hit)
|
| 1034 |
+
|
|
|
|
| 1035 |
score = cls._score(hit)
|
| 1036 |
text = cls._text(hit)
|
| 1037 |
|
| 1038 |
+
canonical_ref = cls._canonical_ref(hit)
|
| 1039 |
+
|
| 1040 |
+
role = (
|
| 1041 |
+
"Kontext/Nachbar"
|
| 1042 |
+
if cls._is_pure_neighbor(hit)
|
| 1043 |
+
else "Hauptquelle"
|
| 1044 |
+
)
|
| 1045 |
+
|
| 1046 |
+
retrieval = ", ".join(cls._retrieval_kinds(hit)) or "retrieved"
|
| 1047 |
+
|
| 1048 |
+
unit_type = cls._hit_value(hit, "unit_type", default="")
|
| 1049 |
+
|
| 1050 |
+
chunk_kind = cls._hit_value(hit, "chunk_kind", default="")
|
| 1051 |
+
|
| 1052 |
return (
|
| 1053 |
+
f"[Quelle {index}]\n"
|
| 1054 |
+
f"Container: {container}\n"
|
| 1055 |
+
f"Abschnitt: {section}\n"
|
| 1056 |
+
f"Norm: {canonical_ref}\n"
|
| 1057 |
+
f"Typ: {unit_type or '-'}\n"
|
| 1058 |
+
f"Chunk: {chunk_kind or '-'}\n"
|
| 1059 |
+
f"Rolle: {role}\n"
|
| 1060 |
+
f"Retrieval: {retrieval}\n"
|
| 1061 |
+
f"Seiten: {page_range}\n"
|
| 1062 |
+
f"Chunk-Index: {chunk_index}\n"
|
| 1063 |
+
f"Score: {score:.4f}\n"
|
| 1064 |
+
f"\n"
|
| 1065 |
+
f"Text:\n"
|
| 1066 |
f"{text}"
|
| 1067 |
)
|
| 1068 |
|
| 1069 |
+
def _build_rag_context(self, hits: List[Dict[str, Any]], question: str = "") -> Tuple[str, List[Dict[str, Any]]]:
|
| 1070 |
+
prepared_hits = self._prepare_hits_for_context(hits, question=question)
|
| 1071 |
if not prepared_hits:
|
| 1072 |
return "", []
|
| 1073 |
|
| 1074 |
parts: List[str] = []
|
| 1075 |
total = 0
|
| 1076 |
|
| 1077 |
+
used_hits: List[Dict[str, Any]] = []
|
| 1078 |
for i, hit in enumerate(prepared_hits, start=1):
|
| 1079 |
+
hit_with_number = dict(hit)
|
| 1080 |
+
hit_with_number["source_number"] = i
|
| 1081 |
+
block = self._format_source_block(i, hit_with_number)
|
| 1082 |
if self.max_context_chars and total + len(block) + 2 > self.max_context_chars:
|
| 1083 |
break
|
| 1084 |
parts.append(block)
|
| 1085 |
+
used_hits.append(hit_with_number)
|
| 1086 |
total += len(block) + 2
|
| 1087 |
|
|
|
|
| 1088 |
return "\n\n".join(parts), used_hits
|
| 1089 |
|
| 1090 |
@staticmethod
|
|
|
|
| 1130 |
# Entfernt nur einen finalen Abschnitt "Quellen:" oder "Fundstellen:",
|
| 1131 |
# nicht aber Quellenmarker im Fließtext wie [Quelle 1].
|
| 1132 |
pattern = re.compile(
|
| 1133 |
+
r"(?:\n{1,3}|(?<=\.)\s+|^)(Quellen|Fundstellen)\s*:\s*(?:\n|.)*$",
|
| 1134 |
flags=re.I,
|
| 1135 |
)
|
| 1136 |
return pattern.sub("", text).strip()
|
| 1137 |
|
| 1138 |
+
@staticmethod
|
| 1139 |
+
def _used_source_numbers(used_hits: List[Dict[str, Any]]) -> Set[int]:
|
| 1140 |
+
numbers: Set[int] = set()
|
| 1141 |
+
for hit in used_hits:
|
| 1142 |
+
value = hit.get("source_number")
|
| 1143 |
+
try:
|
| 1144 |
+
if value is not None:
|
| 1145 |
+
numbers.add(int(value))
|
| 1146 |
+
except (TypeError, ValueError):
|
| 1147 |
+
continue
|
| 1148 |
+
return numbers
|
| 1149 |
+
|
| 1150 |
+
@staticmethod
|
| 1151 |
+
def _legal_ref_norm(text: str) -> str:
|
| 1152 |
+
"""Normalize legal references for conservative support checks."""
|
| 1153 |
+
s = (text or "").lower()
|
| 1154 |
+
s = s.replace("§§", "§")
|
| 1155 |
+
s = re.sub(r"\babsatz\b", "abs", s)
|
| 1156 |
+
s = re.sub(r"\babs\.\b", "abs", s)
|
| 1157 |
+
s = re.sub(r"\bbuchstabe\b", "buchst", s)
|
| 1158 |
+
s = re.sub(r"\bbuchst\.\b", "buchst", s)
|
| 1159 |
+
s = re.sub(r"\s+", " ", s)
|
| 1160 |
+
return s.strip(" .,:;()[]")
|
| 1161 |
+
|
| 1162 |
+
@classmethod
|
| 1163 |
+
def _supported_legal_ref_blobs(cls, used_hits: List[Dict[str, Any]]) -> Set[str]:
|
| 1164 |
+
"""Build normalized blobs used to decide whether a granular ref is grounded."""
|
| 1165 |
+
blobs: Set[str] = set()
|
| 1166 |
+
for hit in used_hits:
|
| 1167 |
+
bits = [
|
| 1168 |
+
cls._canonical_ref(hit),
|
| 1169 |
+
cls._section(hit),
|
| 1170 |
+
str(cls._hit_value(hit, "paragraph", default="")),
|
| 1171 |
+
str(cls._hit_value(hit, "subsection", default="")),
|
| 1172 |
+
str(cls._hit_value(hit, "letter", default="")),
|
| 1173 |
+
cls._text(hit)[:3000],
|
| 1174 |
+
]
|
| 1175 |
+
blob = cls._legal_ref_norm(" ".join(bits))
|
| 1176 |
+
if blob:
|
| 1177 |
+
blobs.add(blob)
|
| 1178 |
+
return blobs
|
| 1179 |
+
|
| 1180 |
+
@classmethod
|
| 1181 |
+
def _strip_unsupported_granular_refs(cls, answer: str, used_hits: List[Dict[str, Any]]) -> str:
|
| 1182 |
+
"""Remove unsupported Buchst.-precision while preserving supported paragraph/Abs. refs.
|
| 1183 |
+
|
| 1184 |
+
This is intentionally conservative. It does not delete the base norm; it only
|
| 1185 |
+
downgrades e.g. "§ 6 Abs. 1 Buchst. a" to "§ 6 Abs. 1" when that granular
|
| 1186 |
+
letter reference is not present in any canonical ref or source text.
|
| 1187 |
+
"""
|
| 1188 |
+
text = answer or ""
|
| 1189 |
+
if not text:
|
| 1190 |
+
return text
|
| 1191 |
+
|
| 1192 |
+
# First fix common self-contradictory patterns produced by LLMs.
|
| 1193 |
+
text = _DUPLICATE_PAREN_BASE_REF_RE.sub(r"(\1)", text)
|
| 1194 |
+
text = _DUPLICATE_BASE_REF_RE.sub(r"\1", text)
|
| 1195 |
+
supported_blobs = cls._supported_legal_ref_blobs(used_hits)
|
| 1196 |
+
|
| 1197 |
+
def is_supported(full: str, base: str, letter: str) -> bool:
|
| 1198 |
+
full_norm = cls._legal_ref_norm(full)
|
| 1199 |
+
# Support exact canonical/textual occurrences, including variants.
|
| 1200 |
+
variants = {
|
| 1201 |
+
full_norm,
|
| 1202 |
+
cls._legal_ref_norm(f"{base} Buchst. {letter}"),
|
| 1203 |
+
cls._legal_ref_norm(f"{base} Buchstabe {letter}"),
|
| 1204 |
+
cls._legal_ref_norm(f"{base} {letter})"),
|
| 1205 |
+
}
|
| 1206 |
+
return any(any(v and v in blob for blob in supported_blobs) for v in variants)
|
| 1207 |
+
|
| 1208 |
+
def repl(match: re.Match[str]) -> str:
|
| 1209 |
+
full = match.group(0)
|
| 1210 |
+
base = re.sub(r"\s+", " ", match.group(1)).strip()
|
| 1211 |
+
letter = match.group(2).lower()
|
| 1212 |
+
return full if is_supported(full, base, letter) else base
|
| 1213 |
+
|
| 1214 |
+
text = _GRANULAR_BUCHSTABE_REF_RE.sub(repl, text)
|
| 1215 |
+
text = re.sub(r"\(\s*(§\s*\d{1,3}[a-z]?\s+Abs\.?\s*\d+[a-z]?)\s*\),\s*\1", r"(\1)", text, flags=re.I)
|
| 1216 |
+
return text
|
| 1217 |
+
|
| 1218 |
+
@classmethod
|
| 1219 |
+
def _referenced_source_numbers(cls, answer: str) -> Set[int]:
|
| 1220 |
+
numbers: Set[int] = set()
|
| 1221 |
+
for m in _SOURCE_MARKER_RE.finditer(answer or ""):
|
| 1222 |
+
try:
|
| 1223 |
+
numbers.add(int(m.group(1)))
|
| 1224 |
+
except (TypeError, ValueError):
|
| 1225 |
+
pass
|
| 1226 |
+
for m in _SOURCE_MARKER_MULTI_RE.finditer(answer or ""):
|
| 1227 |
+
for part in re.split(r"[,\s]+", m.group(1)):
|
| 1228 |
+
if not part:
|
| 1229 |
+
continue
|
| 1230 |
+
try:
|
| 1231 |
+
numbers.add(int(part))
|
| 1232 |
+
except (TypeError, ValueError):
|
| 1233 |
+
pass
|
| 1234 |
+
return numbers
|
| 1235 |
+
|
| 1236 |
+
@classmethod
|
| 1237 |
+
def _postprocess_document_answer(
|
| 1238 |
+
cls,
|
| 1239 |
+
answer: str,
|
| 1240 |
+
used_hits: List[Dict[str, Any]],
|
| 1241 |
+
*,
|
| 1242 |
+
validate_markers: bool = True,
|
| 1243 |
+
) -> str:
|
| 1244 |
+
"""
|
| 1245 |
+
Bereitet die LLM-Antwort nach.
|
| 1246 |
+
|
| 1247 |
+
Schritte:
|
| 1248 |
+
1. Entfernt frei erzeugte Quellenlisten.
|
| 1249 |
+
2. Entfernt ungültige Quellenmarker.
|
| 1250 |
+
3. Entfernt nicht belegte Buchstaben-/Untergliederungsreferenzen.
|
| 1251 |
+
4. Entfernt typische Halluzinationsformulierungen.
|
| 1252 |
+
5. Bereinigt Formatierung.
|
| 1253 |
+
"""
|
| 1254 |
+
|
| 1255 |
+
text = cls._strip_model_generated_sources(answer)
|
| 1256 |
+
|
| 1257 |
+
if validate_markers:
|
| 1258 |
+
text = cls._strip_invalid_source_markers(text, used_hits)
|
| 1259 |
+
|
| 1260 |
+
text = cls._strip_unsupported_granular_refs(text, used_hits)
|
| 1261 |
+
|
| 1262 |
+
# -------------------------------------------------------------
|
| 1263 |
+
# Halluzinationsschutz
|
| 1264 |
+
# -------------------------------------------------------------
|
| 1265 |
+
|
| 1266 |
+
hallucination_patterns = [
|
| 1267 |
+
|
| 1268 |
+
# erfundene Ergebnisblöcke
|
| 1269 |
+
r"(?im)^Ergebnis:\s*",
|
| 1270 |
+
|
| 1271 |
+
# pauschale Aussagen
|
| 1272 |
+
r"(?im)^Es bestehen keine Ausnahmen\.?\s*$",
|
| 1273 |
+
r"(?im)^Es sind keine Ausnahmen geregelt\.?\s*$",
|
| 1274 |
+
r"(?im)^Weitere Ausnahmen sind nicht vorgesehen\.?\s*$",
|
| 1275 |
+
|
| 1276 |
+
# spekulative Aussagen
|
| 1277 |
+
r"(?im)^Im Einzelfall kann.*$",
|
| 1278 |
+
r"(?im)^Die Krankenkasse kann im Einzelfall.*$",
|
| 1279 |
+
|
| 1280 |
+
# unbelegte Schlussformeln
|
| 1281 |
+
r"(?im)^Zusammenfassend gilt.*$",
|
| 1282 |
+
]
|
| 1283 |
+
|
| 1284 |
+
for pattern in hallucination_patterns:
|
| 1285 |
+
text = re.sub(pattern, "", text)
|
| 1286 |
+
|
| 1287 |
+
# -------------------------------------------------------------
|
| 1288 |
+
# Leere Überschriften entfernen
|
| 1289 |
+
#
|
| 1290 |
+
# Wichtig: Nur *tatsächlich* leere Abschnitts-Überschriften entfernen.
|
| 1291 |
+
# Eine Überschrift mit Inhalt in der Folgezeile (z. B. "Einordnung:\n
|
| 1292 |
+
# Die Kriterien sind abschließend.") muss erhalten bleiben. Die frühere
|
| 1293 |
+
# Regex `^Einordnung:\s*$` hätte auch das Label mit Inhalt entfernt, weil
|
| 1294 |
+
# `$` im MULTILINE-Modus bereits vor dem Zeilenumbruch greift.
|
| 1295 |
+
# -------------------------------------------------------------
|
| 1296 |
+
|
| 1297 |
+
text = cls._strip_empty_section_headings(text)
|
| 1298 |
+
|
| 1299 |
+
# -------------------------------------------------------------
|
| 1300 |
+
# Leerraum bereinigen
|
| 1301 |
+
# -------------------------------------------------------------
|
| 1302 |
+
|
| 1303 |
+
text = re.sub(r"[ \t]{2,}", " ", text)
|
| 1304 |
+
|
| 1305 |
+
text = re.sub(r"\n{3,}", "\n\n", text)
|
| 1306 |
+
|
| 1307 |
+
text = text.strip()
|
| 1308 |
+
|
| 1309 |
+
return text
|
| 1310 |
+
|
| 1311 |
+
# Abschnitts-Überschriften, die als leere Reste entfernt werden dürfen.
|
| 1312 |
+
# Enthält die Labels des aktuellen Schemas sowie Alt-Reste.
|
| 1313 |
+
_SECTION_HEADINGS = (
|
| 1314 |
+
"Kurzantwort",
|
| 1315 |
+
"Maßgebliche Norm",
|
| 1316 |
+
"Maßgebliche Norm(en)",
|
| 1317 |
+
"Wortlaut / Kriterien",
|
| 1318 |
+
"Wortlaut",
|
| 1319 |
+
"Einordnung",
|
| 1320 |
+
"Ausnahmen",
|
| 1321 |
+
"Ergebnis",
|
| 1322 |
+
"Heilungen",
|
| 1323 |
+
"Retaxationsgrenzen",
|
| 1324 |
+
)
|
| 1325 |
+
|
| 1326 |
+
@classmethod
|
| 1327 |
+
def _strip_empty_section_headings(cls, text: str) -> str:
|
| 1328 |
+
"""Entfernt Abschnitts-Überschriften, denen kein Inhalt folgt.
|
| 1329 |
+
|
| 1330 |
+
Eine Überschrift gilt als leer, wenn bis zur nächsten Überschrift oder
|
| 1331 |
+
bis zum Textende keine inhaltliche Zeile folgt. Überschriften mit Inhalt
|
| 1332 |
+
bleiben unangetastet.
|
| 1333 |
+
"""
|
| 1334 |
+
if not text:
|
| 1335 |
+
return text
|
| 1336 |
+
|
| 1337 |
+
label_re = re.compile(
|
| 1338 |
+
r"^(?:#+\s*)?(?:"
|
| 1339 |
+
+ "|".join(re.escape(h) for h in cls._SECTION_HEADINGS)
|
| 1340 |
+
+ r"):?\s*$",
|
| 1341 |
+
re.I,
|
| 1342 |
+
)
|
| 1343 |
+
|
| 1344 |
+
lines = text.split("\n")
|
| 1345 |
+
keep = [True] * len(lines)
|
| 1346 |
+
|
| 1347 |
+
for i, line in enumerate(lines):
|
| 1348 |
+
if not label_re.match(line.strip()):
|
| 1349 |
+
continue
|
| 1350 |
+
|
| 1351 |
+
# Nächste nicht-leere Zeile suchen.
|
| 1352 |
+
j = i + 1
|
| 1353 |
+
while j < len(lines) and not lines[j].strip():
|
| 1354 |
+
j += 1
|
| 1355 |
+
|
| 1356 |
+
# Leer, wenn Textende erreicht ist oder direkt die nächste
|
| 1357 |
+
# Überschrift folgt.
|
| 1358 |
+
if j >= len(lines) or label_re.match(lines[j].strip()):
|
| 1359 |
+
keep[i] = False
|
| 1360 |
+
|
| 1361 |
+
return "\n".join(line for idx, line in enumerate(lines) if keep[idx])
|
| 1362 |
+
|
| 1363 |
+
@classmethod
|
| 1364 |
+
def _strip_invalid_source_markers(cls, answer: str, used_hits: List[Dict[str, Any]]) -> str:
|
| 1365 |
+
"""Entfernt Marker wie [Quelle 9], wenn Quelle 9 nicht im Kontext stand."""
|
| 1366 |
+
valid_numbers = cls._used_source_numbers(used_hits)
|
| 1367 |
+
if not valid_numbers:
|
| 1368 |
+
return _SOURCE_MARKER_RE.sub("", answer or "").strip()
|
| 1369 |
+
|
| 1370 |
+
def repl(match: re.Match[str]) -> str:
|
| 1371 |
+
try:
|
| 1372 |
+
number = int(match.group(1))
|
| 1373 |
+
except (TypeError, ValueError):
|
| 1374 |
+
return ""
|
| 1375 |
+
return match.group(0) if number in valid_numbers else ""
|
| 1376 |
+
|
| 1377 |
+
cleaned = _SOURCE_MARKER_RE.sub(repl, answer or "")
|
| 1378 |
+
# Nur überflüssige Leerzeichen vor Satzzeichen bereinigen; Zeilenumbrüche und Schema bleiben erhalten.
|
| 1379 |
+
cleaned = re.sub(r"[ ]{2,}", " ", cleaned)
|
| 1380 |
+
cleaned = re.sub(r"\s+([,.;:])", r"\1", cleaned)
|
| 1381 |
+
return cleaned.strip()
|
| 1382 |
+
|
| 1383 |
# ------------------------------------------------------------------
|
| 1384 |
# Öffentliche API
|
| 1385 |
# ------------------------------------------------------------------
|
|
|
|
| 1411 |
"none",
|
| 1412 |
)
|
| 1413 |
|
| 1414 |
+
rag_context, used_hits = self._build_rag_context(hits, question=question)
|
| 1415 |
if not rag_context.strip():
|
| 1416 |
return (
|
| 1417 |
"Die gefundenen Treffer enthalten keinen verwertbaren Text. "
|
|
|
|
| 1424 |
rag_context=rag_context,
|
| 1425 |
memory=memory,
|
| 1426 |
)
|
| 1427 |
+
answer = self._postprocess_document_answer(answer, used_hits, validate_markers=self.validate_source_markers)
|
| 1428 |
+
referenced_numbers = self._referenced_source_numbers(answer)
|
| 1429 |
|
| 1430 |
if self.append_sources_to_answer:
|
| 1431 |
sources = self.build_sources(
|
| 1432 |
used_hits,
|
| 1433 |
+
max_sources=(len(used_hits) if self.display_all_context_sources else self.max_sources),
|
| 1434 |
+
include_pure_neighbors=(self.include_pure_neighbors_as_sources or self.display_all_context_sources),
|
| 1435 |
allowed_container_ids=list(self.allowed_container_ids) if self.allowed_container_ids else None,
|
| 1436 |
+
referenced_source_numbers=referenced_numbers,
|
| 1437 |
)
|
| 1438 |
sources_md = self.format_sources_markdown(sources)
|
| 1439 |
if sources_md:
|
|
|
|
| 1454 |
answer, source_type = self._answer_meta(question, memory)
|
| 1455 |
return answer, source_type, []
|
| 1456 |
|
| 1457 |
+
rag_context, used_hits = self._build_rag_context(hits, question=question)
|
| 1458 |
if not rag_context.strip():
|
| 1459 |
return (
|
| 1460 |
"Die gefundenen Treffer enthalten keinen verwertbaren Text. "
|
|
|
|
| 1468 |
rag_context=rag_context,
|
| 1469 |
memory=memory,
|
| 1470 |
)
|
| 1471 |
+
answer = self._postprocess_document_answer(answer, used_hits, validate_markers=self.validate_source_markers)
|
| 1472 |
+
referenced_numbers = self._referenced_source_numbers(answer)
|
| 1473 |
|
| 1474 |
sources = self.build_sources(
|
| 1475 |
used_hits,
|
| 1476 |
+
max_sources=(len(used_hits) if self.display_all_context_sources else self.max_sources),
|
| 1477 |
+
include_pure_neighbors=(self.include_pure_neighbors_as_sources or self.display_all_context_sources),
|
| 1478 |
allowed_container_ids=list(self.allowed_container_ids) if self.allowed_container_ids else None,
|
| 1479 |
+
referenced_source_numbers=referenced_numbers,
|
| 1480 |
)
|
| 1481 |
return answer, "document", sources
|
src/app.py
CHANGED
|
@@ -7,7 +7,7 @@ from huggingface_hub import snapshot_download
|
|
| 7 |
from dataclasses import dataclass
|
| 8 |
from pathlib import Path
|
| 9 |
from threading import Lock
|
| 10 |
-
from typing import Any, Dict, List, Optional,
|
| 11 |
from uuid import uuid4
|
| 12 |
|
| 13 |
from fastapi import FastAPI, HTTPException, Request, Response
|
|
@@ -16,15 +16,22 @@ from fastapi.responses import FileResponse
|
|
| 16 |
from fastapi.staticfiles import StaticFiles
|
| 17 |
from pydantic import BaseModel, Field
|
| 18 |
|
| 19 |
-
from
|
| 20 |
-
from
|
| 21 |
DEFAULT_SYSTEM_PROMPT,
|
| 22 |
ConversationMemory,
|
| 23 |
GroqClient,
|
| 24 |
classify_question,
|
| 25 |
is_meta_question,
|
| 26 |
)
|
| 27 |
-
from
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
|
| 29 |
|
| 30 |
# -----------------------------------------------------------------------------
|
|
@@ -36,16 +43,26 @@ logger = logging.getLogger(__name__)
|
|
| 36 |
APP_TITLE = os.getenv("APP_TITLE", "Juristischer RAG-Prototyp")
|
| 37 |
APP_VERSION = os.getenv("APP_VERSION", "1.1")
|
| 38 |
|
| 39 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
APP_DIR = Path(__file__).resolve().parent
|
| 41 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
|
| 43 |
# ---- HF Dataset -> lokale DB nach /data ziehen (Spaces) ----
|
|
|
|
| 44 |
HF_DATASET_REPO = os.getenv("HF_DATASET_REPO", "AlixJabda/bav-ki-db")
|
| 45 |
DATA_DIR = Path(os.getenv("DATA_DIR", "/data"))
|
| 46 |
CHROMA_DATA_DIR = DATA_DIR / "chroma_db"
|
| 47 |
|
| 48 |
-
# Optional Token
|
| 49 |
HF_TOKEN = os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACE_HUB_TOKEN")
|
| 50 |
|
| 51 |
# Nur einmal laden, wenn noch nicht vorhanden
|
|
@@ -66,43 +83,102 @@ except Exception as exc:
|
|
| 66 |
def _resolve_project_path(env_name: str, default_relative: str | Path) -> str:
|
| 67 |
raw = os.getenv(env_name)
|
| 68 |
path = Path(raw) if raw else Path(default_relative)
|
|
|
|
| 69 |
if not path.is_absolute():
|
| 70 |
path = BASE_DIR / path
|
|
|
|
| 71 |
return str(path.resolve())
|
| 72 |
|
| 73 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
STATIC_DIR = _resolve_project_path("STATIC_DIR", "static")
|
| 75 |
-
INDEX_FILE = _resolve_project_path("INDEX_FILE", Path(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
|
| 77 |
SESSION_COOKIE = os.getenv("SESSION_COOKIE", "session_id")
|
| 78 |
SESSION_COOKIE_SECURE = os.getenv("SESSION_COOKIE_SECURE", "false").lower() == "true"
|
| 79 |
SESSION_COOKIE_MAX_AGE = int(os.getenv("SESSION_COOKIE_MAX_AGE", str(60 * 60 * 8)))
|
| 80 |
|
|
|
|
| 81 |
# Keine lokalen Windows-Pfade im Deployment.
|
| 82 |
-
# Standard:
|
| 83 |
-
#
|
| 84 |
-
|
|
|
|
|
|
|
|
|
|
| 85 |
if not Path(CHROMA_PERSIST_DIR).is_absolute():
|
| 86 |
CHROMA_PERSIST_DIR = str((BASE_DIR / CHROMA_PERSIST_DIR).resolve())
|
|
|
|
| 87 |
CHROMA_COLLECTION = os.getenv("CHROMA_COLLECTION", "rv129")
|
| 88 |
|
| 89 |
-
|
|
|
|
|
|
|
| 90 |
try:
|
|
|
|
|
|
|
|
|
|
| 91 |
print("DEBUG effective CHROMA_PERSIST_DIR:", CHROMA_PERSIST_DIR)
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
except Exception as e:
|
| 97 |
print("DEBUG path check failed:", repr(e))
|
|
|
|
| 98 |
# --- /DEBUG ---
|
| 99 |
|
|
|
|
| 100 |
DEFAULT_CONTAINER_ID = os.getenv("DEFAULT_CONTAINER_ID", "Vertrag")
|
| 101 |
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
|
| 107 |
GROQ_MODEL = os.getenv("GROQ_MODEL", "llama-3.3-70b-versatile")
|
| 108 |
GROQ_TEMPERATURE = float(os.getenv("GROQ_TEMPERATURE", "0.05"))
|
|
@@ -116,12 +192,24 @@ DEFAULT_MAX_SOURCES = int(os.getenv("DEFAULT_MAX_SOURCES", "5"))
|
|
| 116 |
DEFAULT_RAG_CONTEXT_CHARS = int(os.getenv("DEFAULT_RAG_CONTEXT_CHARS", "12000"))
|
| 117 |
DEFAULT_MIN_SCORE = float(os.getenv("DEFAULT_MIN_SCORE", "0.0"))
|
| 118 |
|
|
|
|
| 119 |
# Wenn True, werden nur Quellen zurückgegeben, die das Modell als [Quelle n]
|
| 120 |
# in der Antwort erwähnt. Wenn das Modell keine Marker verwendet, fällt die App
|
| 121 |
# auf die kuratierte Quellenliste zurück.
|
| 122 |
SOURCE_SYNC_TO_ANSWER_MARKERS = os.getenv("SOURCE_SYNC_TO_ANSWER_MARKERS", "true").lower() == "true"
|
| 123 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
_raw_origins = os.getenv("CORS_ORIGINS", "*")
|
|
|
|
| 125 |
CORS_ORIGINS = (
|
| 126 |
["*"]
|
| 127 |
if _raw_origins.strip() == "*"
|
|
@@ -130,7 +218,7 @@ CORS_ORIGINS = (
|
|
| 130 |
|
| 131 |
|
| 132 |
# -----------------------------------------------------------------------------
|
| 133 |
-
# App
|
| 134 |
# -----------------------------------------------------------------------------
|
| 135 |
|
| 136 |
app = FastAPI(
|
|
@@ -139,9 +227,6 @@ app = FastAPI(
|
|
| 139 |
version=APP_VERSION,
|
| 140 |
)
|
| 141 |
|
| 142 |
-
if os.path.isdir(STATIC_DIR):
|
| 143 |
-
app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
|
| 144 |
-
|
| 145 |
app.add_middleware(
|
| 146 |
CORSMiddleware,
|
| 147 |
allow_origins=CORS_ORIGINS,
|
|
@@ -150,6 +235,15 @@ app.add_middleware(
|
|
| 150 |
allow_headers=["*"],
|
| 151 |
)
|
| 152 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
|
| 154 |
# -----------------------------------------------------------------------------
|
| 155 |
# Session State
|
|
@@ -225,6 +319,9 @@ def _build_retriever() -> LegalRetriever:
|
|
| 225 |
collection=CHROMA_COLLECTION,
|
| 226 |
model_name=EMBEDDING_MODEL,
|
| 227 |
default_container_id=DEFAULT_CONTAINER_ID,
|
|
|
|
|
|
|
|
|
|
| 228 |
)
|
| 229 |
except TypeError:
|
| 230 |
instance = LegalRetriever(
|
|
@@ -280,7 +377,14 @@ class Question(BaseModel):
|
|
| 280 |
|
| 281 |
include_neighbors: bool = Field(default=True)
|
| 282 |
include_explicit_sections: bool = Field(default=True)
|
| 283 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 284 |
debug: bool = Field(default=False)
|
| 285 |
|
| 286 |
|
|
@@ -307,28 +411,98 @@ def _build_llm(session: SessionState) -> GroqClient:
|
|
| 307 |
|
| 308 |
|
| 309 |
def _build_composer(llm: GroqClient, payload: Question) -> AnswerComposer:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 310 |
allowed = [DEFAULT_CONTAINER_ID] if payload.restrict_to_default_container else None
|
| 311 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 312 |
try:
|
| 313 |
-
return AnswerComposer(
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
max_chunks_per_section=4,
|
| 320 |
-
max_sources=payload.max_sources or DEFAULT_MAX_SOURCES,
|
| 321 |
-
min_score_for_context=DEFAULT_MIN_SCORE,
|
| 322 |
-
include_neighbor_hits_in_context=True,
|
| 323 |
-
include_pure_neighbors_as_sources=False,
|
| 324 |
-
append_sources_to_answer=False,
|
| 325 |
)
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 331 |
)
|
|
|
|
|
|
|
| 332 |
|
| 333 |
|
| 334 |
def _retrieve_hits(payload: Question) -> List[Dict[str, Any]]:
|
|
@@ -387,33 +561,213 @@ def _page_value(hit: Dict[str, Any]) -> Optional[str]:
|
|
| 387 |
return None
|
| 388 |
|
| 389 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 390 |
def _source_from_hit(hit: Dict[str, Any], *, source_number: Optional[int] = None) -> Dict[str, Any]:
|
| 391 |
"""
|
| 392 |
Einheitliche Quellennormalisierung.
|
| 393 |
|
| 394 |
-
Unterstützt
|
| 395 |
-
|
|
|
|
|
|
|
| 396 |
"""
|
| 397 |
metadata = hit.get("metadata") or {}
|
| 398 |
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 405 |
"chunk_index": _coalesce(
|
| 406 |
hit.get("chunk_index"),
|
| 407 |
hit.get("chunk_index_in_section"),
|
|
|
|
| 408 |
metadata.get("chunk_index_in_section"),
|
| 409 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 410 |
"score": hit.get("score"),
|
| 411 |
"rank_score": hit.get("rank_score"),
|
| 412 |
-
"retrieval_kinds": hit.get("retrieval_kinds", []),
|
|
|
|
|
|
|
| 413 |
}
|
| 414 |
|
| 415 |
-
|
| 416 |
-
|
|
|
|
|
|
|
| 417 |
|
| 418 |
return source
|
| 419 |
|
|
@@ -424,7 +778,7 @@ def _extract_answer_source_numbers(answer: str) -> List[int]:
|
|
| 424 |
[Quelle 1], [Quelle 2], ...
|
| 425 |
"""
|
| 426 |
nums: List[int] = []
|
| 427 |
-
for m in re.finditer(r"\[Quelle\s+(\d+)\]", answer or "", flags=re.I):
|
| 428 |
try:
|
| 429 |
nums.append(int(m.group(1)))
|
| 430 |
except ValueError:
|
|
@@ -432,21 +786,117 @@ def _extract_answer_source_numbers(answer: str) -> List[int]:
|
|
| 432 |
return list(dict.fromkeys(nums))
|
| 433 |
|
| 434 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 435 |
def _dedupe_sources(sources: Iterable[Dict[str, Any]], *, max_sources: int) -> List[Dict[str, Any]]:
|
| 436 |
-
|
| 437 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 438 |
|
| 439 |
for raw in sources:
|
| 440 |
src = _source_from_hit(raw)
|
| 441 |
-
key = (src.get("container"), src.get("section"), src.get("pages"), src.get("chunk_index"))
|
| 442 |
-
if key in seen:
|
| 443 |
-
continue
|
| 444 |
-
seen.add(key)
|
| 445 |
-
out.append(src)
|
| 446 |
-
if len(out) >= max_sources:
|
| 447 |
-
break
|
| 448 |
|
| 449 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 450 |
|
| 451 |
|
| 452 |
def _build_sources_from_composer_or_hits(
|
|
@@ -461,7 +911,7 @@ def _build_sources_from_composer_or_hits(
|
|
| 461 |
try:
|
| 462 |
raw_sources = composer.build_sources(
|
| 463 |
hits,
|
| 464 |
-
max_sources=max_sources,
|
| 465 |
include_pure_neighbors=False,
|
| 466 |
allowed_container_ids=allowed,
|
| 467 |
)
|
|
@@ -470,12 +920,138 @@ def _build_sources_from_composer_or_hits(
|
|
| 470 |
try:
|
| 471 |
raw_sources = composer.build_sources(hits, max_sources=max_sources)
|
| 472 |
return _dedupe_sources(raw_sources, max_sources=max_sources)
|
| 473 |
-
except Exception:
|
| 474 |
-
|
| 475 |
|
| 476 |
return _dedupe_sources(hits, max_sources=max_sources)
|
| 477 |
|
| 478 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 479 |
def _normalize_returned_sources(
|
| 480 |
*,
|
| 481 |
answer: str,
|
|
@@ -486,36 +1062,121 @@ def _normalize_returned_sources(
|
|
| 486 |
"""
|
| 487 |
Finale API-Quellenlogik.
|
| 488 |
|
| 489 |
-
|
| 490 |
-
-
|
| 491 |
-
-
|
| 492 |
-
|
| 493 |
-
angezeigten Quellen in derselben Reihenfolge aus den Hits abgeleitet.
|
| 494 |
-
- Falls keine Marker genannt wurden, wird die kuratierte Composer-Quellenliste
|
| 495 |
-
genutzt.
|
| 496 |
"""
|
| 497 |
max_sources = payload.max_sources or DEFAULT_MAX_SOURCES
|
| 498 |
-
|
| 499 |
cited_numbers = _extract_answer_source_numbers(answer)
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
deduped = _dedupe_sources(cited_sources, max_sources=max_sources)
|
| 508 |
-
if deduped:
|
| 509 |
-
return deduped
|
| 510 |
-
|
| 511 |
-
# Fallback: Composer-Sources normalisieren.
|
| 512 |
-
normalized = _dedupe_sources(sources, max_sources=max_sources)
|
| 513 |
if normalized:
|
| 514 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 515 |
|
| 516 |
return _dedupe_sources(hits, max_sources=max_sources)
|
| 517 |
|
| 518 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 519 |
def _debug_hit(hit: Dict[str, Any]) -> Dict[str, Any]:
|
| 520 |
text = (hit.get("text") or hit.get("document") or "").strip()
|
| 521 |
return {
|
|
@@ -534,91 +1195,621 @@ def _collection_count() -> Any:
|
|
| 534 |
|
| 535 |
|
| 536 |
# -----------------------------------------------------------------------------
|
| 537 |
-
#
|
| 538 |
# -----------------------------------------------------------------------------
|
| 539 |
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 545 |
|
| 546 |
-
session: SessionState = request.state.session
|
| 547 |
-
memory = session.get_memory()
|
| 548 |
|
| 549 |
-
|
| 550 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 551 |
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 556 |
else:
|
| 557 |
-
|
|
|
|
| 558 |
|
| 559 |
-
if hasattr(composer, "compose_with_sources"):
|
| 560 |
try:
|
| 561 |
-
answer, answer_type, raw_sources = composer.compose_with_sources(
|
| 562 |
-
question,
|
| 563 |
-
hits,
|
| 564 |
-
memory=memory,
|
| 565 |
)
|
| 566 |
except TypeError:
|
| 567 |
-
answer, answer_type = composer.compose(question, hits, memory=memory)
|
| 568 |
-
raw_sources = _build_sources_from_composer_or_hits(composer, hits, payload)
|
| 569 |
else:
|
| 570 |
-
answer, answer_type = composer.compose(question, hits, memory=memory)
|
| 571 |
-
raw_sources = _build_sources_from_composer_or_hits(composer, hits, payload)
|
| 572 |
|
| 573 |
-
|
| 574 |
-
|
| 575 |
-
|
|
|
|
| 576 |
sources=raw_sources,
|
| 577 |
-
hits=hits,
|
| 578 |
-
payload=payload,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 579 |
)
|
| 580 |
else:
|
| 581 |
-
sources = []
|
| 582 |
|
| 583 |
-
memory.add_turn(
|
| 584 |
-
user_message=question,
|
| 585 |
-
assistant_message=answer,
|
| 586 |
-
question_kind=classify_question(question),
|
| 587 |
)
|
|
|
|
|
|
|
|
|
|
| 588 |
|
| 589 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 590 |
"question": question,
|
| 591 |
-
"answer": answer,
|
| 592 |
-
"answer_type": answer_type,
|
| 593 |
"session_id": request.state.session_id,
|
| 594 |
"factual_question_index": memory.factual_question_count,
|
| 595 |
"total_turns": memory.total_turns,
|
| 596 |
-
"sources": sources,
|
|
|
|
|
|
|
| 597 |
}
|
| 598 |
|
| 599 |
if payload.debug:
|
| 600 |
-
response_body["debug"] =
|
| 601 |
-
"is_meta_question": is_meta_question(question),
|
| 602 |
-
"question_kind": classify_question(question),
|
| 603 |
-
"retrieved_hit_count": len(hits),
|
| 604 |
-
"answer_source_numbers": _extract_answer_source_numbers(answer),
|
| 605 |
-
"hits": [_debug_hit(hit) for hit in hits],
|
| 606 |
-
"top_k": payload.top_k or DEFAULT_TOP_K,
|
| 607 |
-
"fetch_k": payload.fetch_k or DEFAULT_FETCH_K,
|
| 608 |
-
"max_final_results": payload.max_final_results or DEFAULT_MAX_FINAL_RESULTS,
|
| 609 |
-
"max_sources": payload.max_sources or DEFAULT_MAX_SOURCES,
|
| 610 |
-
"include_neighbors": payload.include_neighbors,
|
| 611 |
-
"include_explicit_sections": payload.include_explicit_sections,
|
| 612 |
-
"restrict_to_default_container": payload.restrict_to_default_container,
|
| 613 |
-
"source_sync_to_answer_markers": SOURCE_SYNC_TO_ANSWER_MARKERS,
|
| 614 |
-
"chroma_persist_dir": CHROMA_PERSIST_DIR,
|
| 615 |
-
"chroma_collection": CHROMA_COLLECTION,
|
| 616 |
-
"chroma_count": _collection_count(),
|
| 617 |
-
}
|
| 618 |
|
| 619 |
return response_body
|
| 620 |
|
| 621 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 622 |
@app.get("/health")
|
| 623 |
def health():
|
| 624 |
return {
|
|
@@ -632,6 +1823,8 @@ def health():
|
|
| 632 |
"embedding_model": EMBEDDING_MODEL,
|
| 633 |
"groq_model": GROQ_MODEL,
|
| 634 |
"source_sync_to_answer_markers": SOURCE_SYNC_TO_ANSWER_MARKERS,
|
|
|
|
|
|
|
| 635 |
}
|
| 636 |
|
| 637 |
|
|
@@ -753,4 +1946,3 @@ def index():
|
|
| 753 |
detail=f"UI-Datei nicht gefunden: {INDEX_FILE}",
|
| 754 |
)
|
| 755 |
return FileResponse(INDEX_FILE)
|
| 756 |
-
|
|
|
|
| 7 |
from dataclasses import dataclass
|
| 8 |
from pathlib import Path
|
| 9 |
from threading import Lock
|
| 10 |
+
from typing import Any, Callable, Dict, Iterable, List, Literal, Optional, TypedDict
|
| 11 |
from uuid import uuid4
|
| 12 |
|
| 13 |
from fastapi import FastAPI, HTTPException, Request, Response
|
|
|
|
| 16 |
from fastapi.staticfiles import StaticFiles
|
| 17 |
from pydantic import BaseModel, Field
|
| 18 |
|
| 19 |
+
from retriever import LegalRetriever
|
| 20 |
+
from llm_client_groq import (
|
| 21 |
DEFAULT_SYSTEM_PROMPT,
|
| 22 |
ConversationMemory,
|
| 23 |
GroqClient,
|
| 24 |
classify_question,
|
| 25 |
is_meta_question,
|
| 26 |
)
|
| 27 |
+
from answer_composer import AnswerComposer
|
| 28 |
+
from orchestrator import LegalAnswerOrchestrator, OrchestratorOptions
|
| 29 |
+
|
| 30 |
+
try:
|
| 31 |
+
from langgraph.graph import StateGraph, END
|
| 32 |
+
except ImportError: # LangGraph ist nur für Workflow-Visualisierung erforderlich.
|
| 33 |
+
StateGraph = None # type: ignore[assignment]
|
| 34 |
+
END = "__end__" # type: ignore[assignment]
|
| 35 |
|
| 36 |
|
| 37 |
# -----------------------------------------------------------------------------
|
|
|
|
| 43 |
APP_TITLE = os.getenv("APP_TITLE", "Juristischer RAG-Prototyp")
|
| 44 |
APP_VERSION = os.getenv("APP_VERSION", "1.1")
|
| 45 |
|
| 46 |
+
# Robuste Pfad-Ermittlung:
|
| 47 |
+
# Funktioniert sowohl für:
|
| 48 |
+
# repo/app.py
|
| 49 |
+
# als auch für:
|
| 50 |
+
# repo/src/app.py
|
| 51 |
APP_DIR = Path(__file__).resolve().parent
|
| 52 |
+
|
| 53 |
+
if (APP_DIR / "static").exists():
|
| 54 |
+
BASE_DIR = APP_DIR
|
| 55 |
+
else:
|
| 56 |
+
BASE_DIR = APP_DIR.parent
|
| 57 |
+
|
| 58 |
|
| 59 |
# ---- HF Dataset -> lokale DB nach /data ziehen (Spaces) ----
|
| 60 |
+
|
| 61 |
HF_DATASET_REPO = os.getenv("HF_DATASET_REPO", "AlixJabda/bav-ki-db")
|
| 62 |
DATA_DIR = Path(os.getenv("DATA_DIR", "/data"))
|
| 63 |
CHROMA_DATA_DIR = DATA_DIR / "chroma_db"
|
| 64 |
|
| 65 |
+
# Optional Token, wenn Dataset private ist
|
| 66 |
HF_TOKEN = os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACE_HUB_TOKEN")
|
| 67 |
|
| 68 |
# Nur einmal laden, wenn noch nicht vorhanden
|
|
|
|
| 83 |
def _resolve_project_path(env_name: str, default_relative: str | Path) -> str:
|
| 84 |
raw = os.getenv(env_name)
|
| 85 |
path = Path(raw) if raw else Path(default_relative)
|
| 86 |
+
|
| 87 |
if not path.is_absolute():
|
| 88 |
path = BASE_DIR / path
|
| 89 |
+
|
| 90 |
return str(path.resolve())
|
| 91 |
|
| 92 |
|
| 93 |
+
# Deine aktuelle Struktur:
|
| 94 |
+
#
|
| 95 |
+
# static/
|
| 96 |
+
# ├── index.html
|
| 97 |
+
# ├── js/
|
| 98 |
+
# │ └── script.js
|
| 99 |
+
# ├── styles/
|
| 100 |
+
# │ └── style.css
|
| 101 |
+
# └── berliner_apotheker_verein_cover.jpg
|
| 102 |
+
|
| 103 |
STATIC_DIR = _resolve_project_path("STATIC_DIR", "static")
|
| 104 |
+
INDEX_FILE = _resolve_project_path("INDEX_FILE", Path("static") / "index.html")
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
# --- PDF-Quelldokumente für die interaktive Fundstellen-Anzeige ---
|
| 108 |
+
#
|
| 109 |
+
# Suchreihenfolge:
|
| 110 |
+
# 1. PDF_DIR (Env, explizit)
|
| 111 |
+
# 2. DATA_DIR/pdfs -> HF-Dataset-Snapshot auf Spaces (dort pdfs/ ablegen)
|
| 112 |
+
# 3. BASE_DIR/data/pdfs bzw. BASE_DIR/data -> im Docker-Image mitgelieferte PDFs
|
| 113 |
+
_PDF_ENV_DIR = os.getenv("PDF_DIR")
|
| 114 |
+
|
| 115 |
+
PDF_SEARCH_DIRS: List[Path] = [
|
| 116 |
+
path
|
| 117 |
+
for path in [
|
| 118 |
+
Path(_PDF_ENV_DIR).resolve() if _PDF_ENV_DIR else None,
|
| 119 |
+
DATA_DIR / "pdfs",
|
| 120 |
+
BASE_DIR / "data" / "pdfs",
|
| 121 |
+
BASE_DIR / "data",
|
| 122 |
+
]
|
| 123 |
+
if path is not None
|
| 124 |
+
]
|
| 125 |
+
|
| 126 |
+
_PDF_NAME_RE = re.compile(r"^[\w.\- ]+\.pdf$", re.I)
|
| 127 |
+
|
| 128 |
|
| 129 |
SESSION_COOKIE = os.getenv("SESSION_COOKIE", "session_id")
|
| 130 |
SESSION_COOKIE_SECURE = os.getenv("SESSION_COOKIE_SECURE", "false").lower() == "true"
|
| 131 |
SESSION_COOKIE_MAX_AGE = int(os.getenv("SESSION_COOKIE_MAX_AGE", str(60 * 60 * 8)))
|
| 132 |
|
| 133 |
+
|
| 134 |
# Keine lokalen Windows-Pfade im Deployment.
|
| 135 |
+
# Standard: /data/chroma_db für Spaces; per Env überschreibbar.
|
| 136 |
+
# Wichtig: Die Chroma-DB liegt im Unterordner chroma_db (dorthin lädt der
|
| 137 |
+
# HF-Snapshot, siehe CHROMA_DATA_DIR). DATA_DIR selbst enthält nur eine leere
|
| 138 |
+
# chroma.sqlite3 -> "verfügbare Collections=[]".
|
| 139 |
+
CHROMA_PERSIST_DIR = os.getenv("CHROMA_PERSIST_DIR", str(CHROMA_DATA_DIR))
|
| 140 |
+
|
| 141 |
if not Path(CHROMA_PERSIST_DIR).is_absolute():
|
| 142 |
CHROMA_PERSIST_DIR = str((BASE_DIR / CHROMA_PERSIST_DIR).resolve())
|
| 143 |
+
|
| 144 |
CHROMA_COLLECTION = os.getenv("CHROMA_COLLECTION", "rv129")
|
| 145 |
|
| 146 |
+
|
| 147 |
+
# --- DEBUG: effektive Pfade prüfen, nur temporär ---
|
| 148 |
+
|
| 149 |
try:
|
| 150 |
+
print("DEBUG BASE_DIR:", BASE_DIR)
|
| 151 |
+
print("DEBUG STATIC_DIR:", STATIC_DIR)
|
| 152 |
+
print("DEBUG INDEX_FILE:", INDEX_FILE)
|
| 153 |
print("DEBUG effective CHROMA_PERSIST_DIR:", CHROMA_PERSIST_DIR)
|
| 154 |
+
|
| 155 |
+
if Path("/data").exists():
|
| 156 |
+
print("DEBUG /data listing:", os.listdir("/data")[:50])
|
| 157 |
+
print("DEBUG exists /data/chroma.sqlite3:", (Path("/data") / "chroma.sqlite3").exists())
|
| 158 |
+
|
| 159 |
+
print(
|
| 160 |
+
"DEBUG exists CHROMA_PERSIST_DIR/chroma.sqlite3:",
|
| 161 |
+
(Path(CHROMA_PERSIST_DIR) / "chroma.sqlite3").exists(),
|
| 162 |
+
)
|
| 163 |
except Exception as e:
|
| 164 |
print("DEBUG path check failed:", repr(e))
|
| 165 |
+
|
| 166 |
# --- /DEBUG ---
|
| 167 |
|
| 168 |
+
|
| 169 |
DEFAULT_CONTAINER_ID = os.getenv("DEFAULT_CONTAINER_ID", "Vertrag")
|
| 170 |
|
| 171 |
+
# "auto" liest das Embedding-Modell aus der Chroma-Collection-Metadata, die die
|
| 172 |
+
# Ingest-Pipeline schreibt. Damit kann die Query-Seite nie mit einem anderen
|
| 173 |
+
# Modell arbeiten als der Index (die Ursache stiller Ranking-Ausfälle bzw.
|
| 174 |
+
# harter Dimensionsfehler nach einem Modellwechsel wie MiniLM -> e5-base).
|
| 175 |
+
EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "auto")
|
| 176 |
+
|
| 177 |
+
# Cross-Encoder-Reranking der fusionierten Kandidaten. Hob das Hybrid-Retrieval
|
| 178 |
+
# im Golden-Eval (configs/eval_cases.json im Ingest-Repo) von 17/18 auf 18/18.
|
| 179 |
+
ENABLE_RERANKER = os.getenv("ENABLE_RERANKER", "true").lower() == "true"
|
| 180 |
+
RERANKER_MODEL = os.getenv("RERANKER_MODEL", "cross-encoder/mmarco-mMiniLMv2-L12-H384-v1")
|
| 181 |
+
RERANKER_CANDIDATES = int(os.getenv("RERANKER_CANDIDATES", "20"))
|
| 182 |
|
| 183 |
GROQ_MODEL = os.getenv("GROQ_MODEL", "llama-3.3-70b-versatile")
|
| 184 |
GROQ_TEMPERATURE = float(os.getenv("GROQ_TEMPERATURE", "0.05"))
|
|
|
|
| 192 |
DEFAULT_RAG_CONTEXT_CHARS = int(os.getenv("DEFAULT_RAG_CONTEXT_CHARS", "12000"))
|
| 193 |
DEFAULT_MIN_SCORE = float(os.getenv("DEFAULT_MIN_SCORE", "0.0"))
|
| 194 |
|
| 195 |
+
|
| 196 |
# Wenn True, werden nur Quellen zurückgegeben, die das Modell als [Quelle n]
|
| 197 |
# in der Antwort erwähnt. Wenn das Modell keine Marker verwendet, fällt die App
|
| 198 |
# auf die kuratierte Quellenliste zurück.
|
| 199 |
SOURCE_SYNC_TO_ANSWER_MARKERS = os.getenv("SOURCE_SYNC_TO_ANSWER_MARKERS", "true").lower() == "true"
|
| 200 |
|
| 201 |
+
|
| 202 |
+
# --- Ask-Graph-Orchestrierung ---
|
| 203 |
+
# Organisiert die bestehende /ask-Funktionalität als expliziten Graphen.
|
| 204 |
+
# Wichtig: Es werden keine neuen fachlichen Routen eingeführt.
|
| 205 |
+
|
| 206 |
+
ASK_GRAPH_ENABLED = os.getenv("ASK_GRAPH_ENABLED", "true").lower() == "true"
|
| 207 |
+
ORCHESTRATOR_ENABLED = os.getenv("ORCHESTRATOR_ENABLED", "true").lower() == "true"
|
| 208 |
+
MAX_ASK_GRAPH_STEPS = int(os.getenv("MAX_ASK_GRAPH_STEPS", "10"))
|
| 209 |
+
|
| 210 |
+
|
| 211 |
_raw_origins = os.getenv("CORS_ORIGINS", "*")
|
| 212 |
+
|
| 213 |
CORS_ORIGINS = (
|
| 214 |
["*"]
|
| 215 |
if _raw_origins.strip() == "*"
|
|
|
|
| 218 |
|
| 219 |
|
| 220 |
# -----------------------------------------------------------------------------
|
| 221 |
+
# FastAPI-App
|
| 222 |
# -----------------------------------------------------------------------------
|
| 223 |
|
| 224 |
app = FastAPI(
|
|
|
|
| 227 |
version=APP_VERSION,
|
| 228 |
)
|
| 229 |
|
|
|
|
|
|
|
|
|
|
| 230 |
app.add_middleware(
|
| 231 |
CORSMiddleware,
|
| 232 |
allow_origins=CORS_ORIGINS,
|
|
|
|
| 235 |
allow_headers=["*"],
|
| 236 |
)
|
| 237 |
|
| 238 |
+
# Static-Dateien ausliefern:
|
| 239 |
+
#
|
| 240 |
+
# /static/index.html
|
| 241 |
+
# /static/styles/style.css
|
| 242 |
+
# /static/js/script.js
|
| 243 |
+
# /static/berliner_apotheker_verein_cover.jpg
|
| 244 |
+
if os.path.isdir(STATIC_DIR):
|
| 245 |
+
app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
|
| 246 |
+
|
| 247 |
|
| 248 |
# -----------------------------------------------------------------------------
|
| 249 |
# Session State
|
|
|
|
| 319 |
collection=CHROMA_COLLECTION,
|
| 320 |
model_name=EMBEDDING_MODEL,
|
| 321 |
default_container_id=DEFAULT_CONTAINER_ID,
|
| 322 |
+
enable_reranker=ENABLE_RERANKER,
|
| 323 |
+
reranker_model=RERANKER_MODEL,
|
| 324 |
+
reranker_candidates=RERANKER_CANDIDATES,
|
| 325 |
)
|
| 326 |
except TypeError:
|
| 327 |
instance = LegalRetriever(
|
|
|
|
| 377 |
|
| 378 |
include_neighbors: bool = Field(default=True)
|
| 379 |
include_explicit_sections: bool = Field(default=True)
|
| 380 |
+
# False: Anlagen und Anhänge (z. B. Anlage 11, Dienstleistungs-Anhänge)
|
| 381 |
+
# werden mitdurchsucht. Die Priorisierung des Hauptvertrags übernimmt das
|
| 382 |
+
# Ranking/Reranking, nicht ein harter Container-Filter, der früher ganze
|
| 383 |
+
# Fragenklassen (Beitritt, pharmazeutische Dienstleistungen) blind machte.
|
| 384 |
+
restrict_to_default_container: bool = Field(default=False)
|
| 385 |
+
verify_negative_answer: bool = Field(default=True)
|
| 386 |
+
allow_clarification: bool = Field(default=True)
|
| 387 |
+
enrich_citations: bool = Field(default=True)
|
| 388 |
debug: bool = Field(default=False)
|
| 389 |
|
| 390 |
|
|
|
|
| 411 |
|
| 412 |
|
| 413 |
def _build_composer(llm: GroqClient, payload: Question) -> AnswerComposer:
|
| 414 |
+
"""Baut den AnswerComposer im finalen Legal-RAG-Modus.
|
| 415 |
+
|
| 416 |
+
Wichtig: Neue Composer-Versionen liefern nummernstabile, gruppierte Quellen
|
| 417 |
+
mit Feldern wie display_title, source_numbers und canonical_refs. Diese
|
| 418 |
+
Optionen werden hier bewusst explizit gesetzt, damit die API-/UI-Schicht
|
| 419 |
+
nicht wieder auf die alte path/pages-Anzeige zurückfällt.
|
| 420 |
+
"""
|
| 421 |
allowed = [DEFAULT_CONTAINER_ID] if payload.restrict_to_default_container else None
|
| 422 |
|
| 423 |
+
kwargs: Dict[str, Any] = {
|
| 424 |
+
"max_context_chars": DEFAULT_RAG_CONTEXT_CHARS,
|
| 425 |
+
"pass_memory_to_llm_for_documents": False,
|
| 426 |
+
"allowed_container_ids": allowed,
|
| 427 |
+
"max_hits_for_context": payload.max_final_results or DEFAULT_MAX_FINAL_RESULTS,
|
| 428 |
+
"max_chunks_per_section": 4,
|
| 429 |
+
"max_sources": payload.max_sources or DEFAULT_MAX_SOURCES,
|
| 430 |
+
"min_score_for_context": DEFAULT_MIN_SCORE,
|
| 431 |
+
"include_neighbor_hits_in_context": True,
|
| 432 |
+
"include_pure_neighbors_as_sources": False,
|
| 433 |
+
"append_sources_to_answer": False,
|
| 434 |
+
# Neue Qualitätsoptionen des finalen Composer.
|
| 435 |
+
"display_all_context_sources": True,
|
| 436 |
+
"validate_source_markers": True,
|
| 437 |
+
"prefer_direct_hits_over_neighbors": True,
|
| 438 |
+
}
|
| 439 |
+
|
| 440 |
try:
|
| 441 |
+
return AnswerComposer(llm, **kwargs)
|
| 442 |
+
except TypeError as exc:
|
| 443 |
+
logger.warning(
|
| 444 |
+
"AnswerComposer unterstützt nicht alle finalen Optionen; "
|
| 445 |
+
"falle auf kompatible Minimalinitialisierung zurück: %s",
|
| 446 |
+
exc,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 447 |
)
|
| 448 |
+
legacy_keys = {
|
| 449 |
+
"max_context_chars",
|
| 450 |
+
"pass_memory_to_llm_for_documents",
|
| 451 |
+
"allowed_container_ids",
|
| 452 |
+
"max_hits_for_context",
|
| 453 |
+
"max_chunks_per_section",
|
| 454 |
+
"max_sources",
|
| 455 |
+
"min_score_for_context",
|
| 456 |
+
"include_neighbor_hits_in_context",
|
| 457 |
+
"include_pure_neighbors_as_sources",
|
| 458 |
+
"append_sources_to_answer",
|
| 459 |
+
}
|
| 460 |
+
legacy_kwargs = {key: value for key, value in kwargs.items() if key in legacy_keys}
|
| 461 |
+
try:
|
| 462 |
+
return AnswerComposer(llm, **legacy_kwargs)
|
| 463 |
+
except TypeError as exc2:
|
| 464 |
+
logger.warning(
|
| 465 |
+
"AnswerComposer unterstützt nur Minimalparameter; "
|
| 466 |
+
"Quellenqualität kann eingeschränkt sein: %s",
|
| 467 |
+
exc2,
|
| 468 |
+
)
|
| 469 |
+
return AnswerComposer(
|
| 470 |
+
llm,
|
| 471 |
+
max_context_chars=DEFAULT_RAG_CONTEXT_CHARS,
|
| 472 |
+
pass_memory_to_llm_for_documents=False,
|
| 473 |
+
)
|
| 474 |
+
|
| 475 |
+
|
| 476 |
+
def _build_orchestrator(
|
| 477 |
+
retriever_instance: LegalRetriever,
|
| 478 |
+
composer: AnswerComposer,
|
| 479 |
+
payload: Question,
|
| 480 |
+
) -> LegalAnswerOrchestrator:
|
| 481 |
+
options = OrchestratorOptions(
|
| 482 |
+
top_k=payload.top_k or DEFAULT_TOP_K,
|
| 483 |
+
fetch_k=payload.fetch_k or DEFAULT_FETCH_K,
|
| 484 |
+
max_final_results=payload.max_final_results or DEFAULT_MAX_FINAL_RESULTS,
|
| 485 |
+
max_sources=payload.max_sources or DEFAULT_MAX_SOURCES,
|
| 486 |
+
include_neighbors=payload.include_neighbors,
|
| 487 |
+
include_explicit_sections=payload.include_explicit_sections,
|
| 488 |
+
restrict_to_default_container=payload.restrict_to_default_container,
|
| 489 |
+
default_container_id=DEFAULT_CONTAINER_ID,
|
| 490 |
+
min_score=DEFAULT_MIN_SCORE,
|
| 491 |
+
enable_negative_recheck=payload.verify_negative_answer,
|
| 492 |
+
enable_clarification=payload.allow_clarification,
|
| 493 |
+
enrich_citations_with_canonical_refs=payload.enrich_citations,
|
| 494 |
+
debug=payload.debug,
|
| 495 |
+
)
|
| 496 |
+
try:
|
| 497 |
+
return LegalAnswerOrchestrator(
|
| 498 |
+
retriever_instance,
|
| 499 |
+
composer,
|
| 500 |
+
options=options,
|
| 501 |
+
question_classifier=classify_question,
|
| 502 |
+
meta_detector=is_meta_question,
|
| 503 |
)
|
| 504 |
+
except TypeError:
|
| 505 |
+
return LegalAnswerOrchestrator(retriever_instance, composer, options=options)
|
| 506 |
|
| 507 |
|
| 508 |
def _retrieve_hits(payload: Question) -> List[Dict[str, Any]]:
|
|
|
|
| 561 |
return None
|
| 562 |
|
| 563 |
|
| 564 |
+
def _list_value(*values: Any) -> List[Any]:
|
| 565 |
+
"""Gibt die erste nicht-leere Listen-/Skalarangabe als Liste zurück."""
|
| 566 |
+
for value in values:
|
| 567 |
+
if value is None or value == "":
|
| 568 |
+
continue
|
| 569 |
+
if isinstance(value, (list, tuple, set)):
|
| 570 |
+
return [item for item in value if item is not None and item != ""]
|
| 571 |
+
return [value]
|
| 572 |
+
return []
|
| 573 |
+
|
| 574 |
+
|
| 575 |
+
def _int_list(*values: Any) -> List[int]:
|
| 576 |
+
out: List[int] = []
|
| 577 |
+
for value in _list_value(*values):
|
| 578 |
+
try:
|
| 579 |
+
number = int(value)
|
| 580 |
+
except (TypeError, ValueError):
|
| 581 |
+
continue
|
| 582 |
+
if number not in out:
|
| 583 |
+
out.append(number)
|
| 584 |
+
return sorted(out)
|
| 585 |
+
|
| 586 |
+
|
| 587 |
+
def _int_or_none(value: Any) -> Optional[int]:
|
| 588 |
+
try:
|
| 589 |
+
return int(value)
|
| 590 |
+
except (TypeError, ValueError):
|
| 591 |
+
return None
|
| 592 |
+
|
| 593 |
+
|
| 594 |
+
def _highlight_text_from_hit(hit: Dict[str, Any], *, max_chars: int = 2000) -> str:
|
| 595 |
+
"""Chunk-Text für die PDF-Hervorhebung, ohne konstruierten Kontext-Header.
|
| 596 |
+
|
| 597 |
+
Die Ingest-Pipeline stellt jedem Chunk eine Zeile "Container · § x Titel"
|
| 598 |
+
voran, die im PDF nicht existiert und daher nicht gematcht werden kann.
|
| 599 |
+
"""
|
| 600 |
+
text = str(hit.get("text") or hit.get("document") or "").strip()
|
| 601 |
+
if not text:
|
| 602 |
+
return ""
|
| 603 |
+
head, sep, rest = text.partition("\n\n")
|
| 604 |
+
if sep and " · " in head and len(head) <= 200 and rest.strip():
|
| 605 |
+
text = rest.strip()
|
| 606 |
+
return text[:max_chars]
|
| 607 |
+
|
| 608 |
+
|
| 609 |
+
def _highlights_from_hit(hit: Dict[str, Any]) -> List[Dict[str, Any]]:
|
| 610 |
+
"""Normalisiert die Hervorhebungsliste einer Quelle bzw. eines Raw-Hits."""
|
| 611 |
+
metadata = hit.get("metadata") or {}
|
| 612 |
+
|
| 613 |
+
existing = hit.get("highlights")
|
| 614 |
+
if isinstance(existing, list) and existing:
|
| 615 |
+
out = []
|
| 616 |
+
for entry in existing:
|
| 617 |
+
if not isinstance(entry, dict):
|
| 618 |
+
continue
|
| 619 |
+
text = str(entry.get("text") or "").strip()
|
| 620 |
+
if not text:
|
| 621 |
+
continue
|
| 622 |
+
out.append(
|
| 623 |
+
{
|
| 624 |
+
"page_start": _int_or_none(entry.get("page_start")),
|
| 625 |
+
"page_end": _int_or_none(entry.get("page_end")) or _int_or_none(entry.get("page_start")),
|
| 626 |
+
"text": text[:2000],
|
| 627 |
+
}
|
| 628 |
+
)
|
| 629 |
+
return out
|
| 630 |
+
|
| 631 |
+
text = _highlight_text_from_hit(hit)
|
| 632 |
+
if not text:
|
| 633 |
+
return []
|
| 634 |
+
|
| 635 |
+
page_start = _int_or_none(_coalesce(hit.get("page_start"), metadata.get("page_start")))
|
| 636 |
+
page_end = _int_or_none(_coalesce(hit.get("page_end"), metadata.get("page_end"))) or page_start
|
| 637 |
+
return [{"page_start": page_start, "page_end": page_end, "text": text}]
|
| 638 |
+
|
| 639 |
+
|
| 640 |
+
def _canonical_refs_from_hit(hit: Dict[str, Any]) -> List[str]:
|
| 641 |
+
metadata = hit.get("metadata") or {}
|
| 642 |
+
refs = [
|
| 643 |
+
str(ref).strip()
|
| 644 |
+
for ref in _list_value(hit.get("canonical_refs"), metadata.get("canonical_refs"))
|
| 645 |
+
if str(ref).strip()
|
| 646 |
+
]
|
| 647 |
+
|
| 648 |
+
single = _coalesce(hit.get("canonical_ref"), metadata.get("canonical_ref"))
|
| 649 |
+
if single is not None and str(single).strip() and str(single).strip() not in refs:
|
| 650 |
+
refs.append(str(single).strip())
|
| 651 |
+
|
| 652 |
+
return refs
|
| 653 |
+
|
| 654 |
+
|
| 655 |
+
def _source_marker(numbers: List[int]) -> str:
|
| 656 |
+
if not numbers:
|
| 657 |
+
return ""
|
| 658 |
+
if len(numbers) == 1:
|
| 659 |
+
return f"[Quelle {numbers[0]}]"
|
| 660 |
+
return "[Quellen " + ", ".join(str(n) for n in numbers) + "]"
|
| 661 |
+
|
| 662 |
+
|
| 663 |
+
def _format_source_display_title(source: Dict[str, Any]) -> str:
|
| 664 |
+
"""Erzeugt eine UI-fertige, nummernstabile Quellenanzeige."""
|
| 665 |
+
marker = _coalesce(source.get("source_label"), source.get("source_marker")) or _source_marker(
|
| 666 |
+
_int_list(source.get("source_numbers"), source.get("source_number"))
|
| 667 |
+
)
|
| 668 |
+
|
| 669 |
+
container = _coalesce(source.get("container"), "Unbekannt")
|
| 670 |
+
section = _coalesce(source.get("section"), "ohne Abschnitt")
|
| 671 |
+
path = _coalesce(source.get("path"), source.get("section_path"))
|
| 672 |
+
if not path:
|
| 673 |
+
path = f"{container}::{section}"
|
| 674 |
+
|
| 675 |
+
pages = _coalesce(source.get("page_range"), source.get("pages"), "?")
|
| 676 |
+
canonical_refs = [
|
| 677 |
+
str(ref).strip()
|
| 678 |
+
for ref in _list_value(source.get("canonical_refs"), source.get("canonical_ref"))
|
| 679 |
+
if str(ref).strip()
|
| 680 |
+
]
|
| 681 |
+
|
| 682 |
+
# Keine redundante Anzeige "§ 6 (§ 6)".
|
| 683 |
+
canonical_refs = [ref for ref in dict.fromkeys(canonical_refs) if ref != section]
|
| 684 |
+
ref_part = f" ({'; '.join(canonical_refs[:4])})" if canonical_refs else ""
|
| 685 |
+
|
| 686 |
+
role = ""
|
| 687 |
+
kinds = set(source.get("retrieval_kinds") or [])
|
| 688 |
+
if kinds == {"neighbor"}:
|
| 689 |
+
role = " · Kontext/Nachbar"
|
| 690 |
+
|
| 691 |
+
return f"{marker} {path}{ref_part}, Seiten {pages}{role}".strip()
|
| 692 |
+
|
| 693 |
+
|
| 694 |
def _source_from_hit(hit: Dict[str, Any], *, source_number: Optional[int] = None) -> Dict[str, Any]:
|
| 695 |
"""
|
| 696 |
Einheitliche Quellennormalisierung.
|
| 697 |
|
| 698 |
+
Unterstützt Raw-Retriever-Hits und bereits kuratierte Composer-Quellen.
|
| 699 |
+
Anders als die ältere Version bewahrt sie Composer-Felder wie
|
| 700 |
+
display_title, source_numbers und canonical_refs, damit die UI nicht wieder
|
| 701 |
+
auf bloße path/pages-Ausgaben zurückfällt.
|
| 702 |
"""
|
| 703 |
metadata = hit.get("metadata") or {}
|
| 704 |
|
| 705 |
+
explicit_number = source_number if source_number is not None else _coalesce(
|
| 706 |
+
hit.get("source_number"),
|
| 707 |
+
metadata.get("source_number"),
|
| 708 |
+
)
|
| 709 |
+
source_numbers = _int_list(
|
| 710 |
+
hit.get("source_numbers"),
|
| 711 |
+
metadata.get("source_numbers"),
|
| 712 |
+
explicit_number,
|
| 713 |
+
)
|
| 714 |
+
|
| 715 |
+
pages = _page_value(hit)
|
| 716 |
+
container = _coalesce(hit.get("container"), hit.get("container_id"), metadata.get("container_id"))
|
| 717 |
+
section = _coalesce(hit.get("section"), hit.get("section_id"), metadata.get("section_id"))
|
| 718 |
+
canonical_refs = _canonical_refs_from_hit(hit)
|
| 719 |
+
canonical_ref = _coalesce(hit.get("canonical_ref"), metadata.get("canonical_ref"))
|
| 720 |
+
if canonical_ref is None and canonical_refs:
|
| 721 |
+
canonical_ref = canonical_refs[0]
|
| 722 |
+
|
| 723 |
+
path = _coalesce(
|
| 724 |
+
hit.get("path"),
|
| 725 |
+
hit.get("section_path"),
|
| 726 |
+
metadata.get("path"),
|
| 727 |
+
metadata.get("section_path"),
|
| 728 |
+
)
|
| 729 |
+
if not path and container and section:
|
| 730 |
+
path = f"{container}::{section}"
|
| 731 |
+
|
| 732 |
+
source: Dict[str, Any] = {
|
| 733 |
+
"source_number": explicit_number,
|
| 734 |
+
"source_numbers": source_numbers,
|
| 735 |
+
"source_marker": _coalesce(hit.get("source_marker"), metadata.get("source_marker"), _source_marker(source_numbers)),
|
| 736 |
+
"source_label": _coalesce(hit.get("source_label"), metadata.get("source_label"), _source_marker(source_numbers)),
|
| 737 |
+
"container": container,
|
| 738 |
+
"section": section,
|
| 739 |
+
"canonical_ref": canonical_ref,
|
| 740 |
+
"canonical_refs": canonical_refs,
|
| 741 |
+
"path": path,
|
| 742 |
+
"pages": pages,
|
| 743 |
+
"page_range": pages,
|
| 744 |
"chunk_index": _coalesce(
|
| 745 |
hit.get("chunk_index"),
|
| 746 |
hit.get("chunk_index_in_section"),
|
| 747 |
+
metadata.get("chunk_index"),
|
| 748 |
metadata.get("chunk_index_in_section"),
|
| 749 |
),
|
| 750 |
+
# PDF-Locator für die interaktive Fundstellen-Anzeige in der UI.
|
| 751 |
+
"page_start": _int_or_none(_coalesce(hit.get("page_start"), metadata.get("page_start"))),
|
| 752 |
+
"page_end": _int_or_none(_coalesce(hit.get("page_end"), metadata.get("page_end"), hit.get("page_start"), metadata.get("page_start"))),
|
| 753 |
+
"source_file": _coalesce(hit.get("source_file"), metadata.get("source_file")),
|
| 754 |
+
"doc_id": _coalesce(hit.get("doc_id"), metadata.get("doc_id")),
|
| 755 |
+
"highlights": _highlights_from_hit(hit),
|
| 756 |
+
"legal_unit_id": _coalesce(hit.get("legal_unit_id"), metadata.get("legal_unit_id")),
|
| 757 |
+
"parent_unit_id": _coalesce(hit.get("parent_unit_id"), metadata.get("parent_unit_id")),
|
| 758 |
+
"chunk_kind": _coalesce(hit.get("chunk_kind"), metadata.get("chunk_kind")),
|
| 759 |
+
"unit_type": _coalesce(hit.get("unit_type"), metadata.get("unit_type")),
|
| 760 |
"score": hit.get("score"),
|
| 761 |
"rank_score": hit.get("rank_score"),
|
| 762 |
+
"retrieval_kinds": hit.get("retrieval_kinds", metadata.get("retrieval_kinds", [])),
|
| 763 |
+
"display_label": _coalesce(hit.get("display_label"), metadata.get("display_label")),
|
| 764 |
+
"display_title": _coalesce(hit.get("display_title"), metadata.get("display_title")),
|
| 765 |
}
|
| 766 |
|
| 767 |
+
if not source["display_title"]:
|
| 768 |
+
source["display_title"] = _format_source_display_title(source)
|
| 769 |
+
if not source["display_label"]:
|
| 770 |
+
source["display_label"] = source["display_title"]
|
| 771 |
|
| 772 |
return source
|
| 773 |
|
|
|
|
| 778 |
[Quelle 1], [Quelle 2], ...
|
| 779 |
"""
|
| 780 |
nums: List[int] = []
|
| 781 |
+
for m in re.finditer(r"\[Quelle\s+(\d+)(?:[^\]]*)\]", answer or "", flags=re.I):
|
| 782 |
try:
|
| 783 |
nums.append(int(m.group(1)))
|
| 784 |
except ValueError:
|
|
|
|
| 786 |
return list(dict.fromkeys(nums))
|
| 787 |
|
| 788 |
|
| 789 |
+
def _merge_source(existing: Dict[str, Any], incoming: Dict[str, Any]) -> Dict[str, Any]:
|
| 790 |
+
"""Führt gruppierbare Quellen zusammen, ohne Nummern/Fundstellen zu verlieren."""
|
| 791 |
+
merged = dict(existing)
|
| 792 |
+
|
| 793 |
+
numbers = _int_list(existing.get("source_numbers"), existing.get("source_number"), incoming.get("source_numbers"), incoming.get("source_number"))
|
| 794 |
+
refs = [
|
| 795 |
+
str(ref).strip()
|
| 796 |
+
for ref in _list_value(existing.get("canonical_refs"), existing.get("canonical_ref"), incoming.get("canonical_refs"), incoming.get("canonical_ref"))
|
| 797 |
+
if str(ref).strip()
|
| 798 |
+
]
|
| 799 |
+
refs = list(dict.fromkeys(refs))
|
| 800 |
+
|
| 801 |
+
kinds = list(dict.fromkeys(
|
| 802 |
+
list(existing.get("retrieval_kinds") or []) + list(incoming.get("retrieval_kinds") or [])
|
| 803 |
+
))
|
| 804 |
+
|
| 805 |
+
merged["source_numbers"] = numbers
|
| 806 |
+
merged["source_number"] = numbers[0] if numbers else _coalesce(existing.get("source_number"), incoming.get("source_number"))
|
| 807 |
+
merged["source_marker"] = _source_marker(numbers) or _coalesce(existing.get("source_marker"), incoming.get("source_marker"))
|
| 808 |
+
merged["source_label"] = merged["source_marker"]
|
| 809 |
+
merged["canonical_refs"] = refs
|
| 810 |
+
merged["canonical_ref"] = _coalesce(existing.get("canonical_ref"), incoming.get("canonical_ref"), refs[0] if refs else None)
|
| 811 |
+
merged["retrieval_kinds"] = kinds
|
| 812 |
+
|
| 813 |
+
# Behalte den höchsten Score, ohne None/Strings zu erzwingen.
|
| 814 |
+
for key in ("score", "rank_score"):
|
| 815 |
+
try:
|
| 816 |
+
old_score = float(existing.get(key) or 0.0)
|
| 817 |
+
new_score = float(incoming.get(key) or 0.0)
|
| 818 |
+
merged[key] = max(old_score, new_score)
|
| 819 |
+
except (TypeError, ValueError):
|
| 820 |
+
merged[key] = _coalesce(existing.get(key), incoming.get(key))
|
| 821 |
+
|
| 822 |
+
# Bei mehreren Chunks derselben Section ist chunk_index für die Anzeige nicht
|
| 823 |
+
# mehr sinnvoll eindeutig. Nur behalten, wenn identisch.
|
| 824 |
+
if existing.get("chunk_index") != incoming.get("chunk_index"):
|
| 825 |
+
merged["chunk_index"] = None
|
| 826 |
+
|
| 827 |
+
# PDF-Locator zusammenführen: Seitenbereich aufspannen, Datei behalten,
|
| 828 |
+
# Hervorhebungen beider Quellen sammeln.
|
| 829 |
+
starts = [p for p in (_int_or_none(existing.get("page_start")), _int_or_none(incoming.get("page_start"))) if p is not None]
|
| 830 |
+
ends = [p for p in (_int_or_none(existing.get("page_end")), _int_or_none(incoming.get("page_end"))) if p is not None]
|
| 831 |
+
merged["page_start"] = min(starts) if starts else None
|
| 832 |
+
merged["page_end"] = max(ends) if ends else merged["page_start"]
|
| 833 |
+
merged["source_file"] = _coalesce(existing.get("source_file"), incoming.get("source_file"))
|
| 834 |
+
merged["doc_id"] = _coalesce(existing.get("doc_id"), incoming.get("doc_id"))
|
| 835 |
+
|
| 836 |
+
highlights: List[Dict[str, Any]] = []
|
| 837 |
+
seen_highlights: set = set()
|
| 838 |
+
for entry in list(existing.get("highlights") or []) + list(incoming.get("highlights") or []):
|
| 839 |
+
if not isinstance(entry, dict):
|
| 840 |
+
continue
|
| 841 |
+
text = str(entry.get("text") or "").strip()
|
| 842 |
+
if not text:
|
| 843 |
+
continue
|
| 844 |
+
dedupe_key = (entry.get("page_start"), text[:120])
|
| 845 |
+
if dedupe_key in seen_highlights:
|
| 846 |
+
continue
|
| 847 |
+
seen_highlights.add(dedupe_key)
|
| 848 |
+
highlights.append(entry)
|
| 849 |
+
merged["highlights"] = highlights[:8]
|
| 850 |
+
|
| 851 |
+
merged["display_title"] = _format_source_display_title(merged)
|
| 852 |
+
merged["display_label"] = merged["display_title"]
|
| 853 |
+
return merged
|
| 854 |
+
|
| 855 |
+
|
| 856 |
+
def _source_dedupe_key(src: Dict[str, Any]) -> tuple[Any, ...]:
|
| 857 |
+
"""Dedupe-Key, der Composer-Gruppierungen respektiert."""
|
| 858 |
+
numbers = tuple(_int_list(src.get("source_numbers"), src.get("source_number")))
|
| 859 |
+
if numbers:
|
| 860 |
+
return ("numbers", numbers)
|
| 861 |
+
|
| 862 |
+
display_title = src.get("display_title")
|
| 863 |
+
if display_title:
|
| 864 |
+
return ("display", display_title)
|
| 865 |
+
|
| 866 |
+
canonical_refs = tuple(src.get("canonical_refs") or [])
|
| 867 |
+
return (
|
| 868 |
+
"location",
|
| 869 |
+
src.get("container"),
|
| 870 |
+
src.get("section"),
|
| 871 |
+
src.get("pages") or src.get("page_range"),
|
| 872 |
+
canonical_refs,
|
| 873 |
+
)
|
| 874 |
+
|
| 875 |
+
|
| 876 |
def _dedupe_sources(sources: Iterable[Dict[str, Any]], *, max_sources: int) -> List[Dict[str, Any]]:
|
| 877 |
+
"""Dedupliziert Quellen, ohne Composer-Felder zu verlieren.
|
| 878 |
+
|
| 879 |
+
Die alte Variante normalisierte jede Quelle auf path/pages/chunk_index zurück
|
| 880 |
+
und zerstörte dadurch display_title, source_numbers und canonical_refs.
|
| 881 |
+
"""
|
| 882 |
+
grouped: Dict[tuple[Any, ...], Dict[str, Any]] = {}
|
| 883 |
+
order: List[tuple[Any, ...]] = []
|
| 884 |
|
| 885 |
for raw in sources:
|
| 886 |
src = _source_from_hit(raw)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 887 |
|
| 888 |
+
# Wenn keine Nummern vorhanden sind, gruppiere gleiche Norm/Seiten, nicht
|
| 889 |
+
# einzelne Chunk-Indizes. Das verhindert doppelte "Vertrag::§ 6"-Zeilen.
|
| 890 |
+
key = _source_dedupe_key(src)
|
| 891 |
+
if key not in grouped:
|
| 892 |
+
grouped[key] = src
|
| 893 |
+
order.append(key)
|
| 894 |
+
else:
|
| 895 |
+
grouped[key] = _merge_source(grouped[key], src)
|
| 896 |
+
|
| 897 |
+
out = [grouped[key] for key in order]
|
| 898 |
+
out.sort(key=lambda s: (_int_list(s.get("source_numbers"), s.get("source_number")) or [10_000])[0])
|
| 899 |
+
return out[:max_sources]
|
| 900 |
|
| 901 |
|
| 902 |
def _build_sources_from_composer_or_hits(
|
|
|
|
| 911 |
try:
|
| 912 |
raw_sources = composer.build_sources(
|
| 913 |
hits,
|
| 914 |
+
max_sources=max(max_sources, min(len(hits), DEFAULT_MAX_FINAL_RESULTS)),
|
| 915 |
include_pure_neighbors=False,
|
| 916 |
allowed_container_ids=allowed,
|
| 917 |
)
|
|
|
|
| 920 |
try:
|
| 921 |
raw_sources = composer.build_sources(hits, max_sources=max_sources)
|
| 922 |
return _dedupe_sources(raw_sources, max_sources=max_sources)
|
| 923 |
+
except Exception as exc:
|
| 924 |
+
logger.warning("Composer build_sources fallback failed: %s", exc)
|
| 925 |
|
| 926 |
return _dedupe_sources(hits, max_sources=max_sources)
|
| 927 |
|
| 928 |
|
| 929 |
+
def _sources_cover_cited_numbers(sources: List[Dict[str, Any]], cited_numbers: List[int]) -> bool:
|
| 930 |
+
if not cited_numbers:
|
| 931 |
+
return True
|
| 932 |
+
covered = set()
|
| 933 |
+
for source in sources:
|
| 934 |
+
covered.update(_int_list(source.get("source_numbers"), source.get("source_number")))
|
| 935 |
+
return set(cited_numbers).issubset(covered)
|
| 936 |
+
|
| 937 |
+
|
| 938 |
+
def _sources_for_cited_numbers_from_raw_hits(
|
| 939 |
+
cited_numbers: List[int],
|
| 940 |
+
hits: List[Dict[str, Any]],
|
| 941 |
+
*,
|
| 942 |
+
max_sources: int,
|
| 943 |
+
) -> List[Dict[str, Any]]:
|
| 944 |
+
"""Letzter Fallback: mappt [Quelle n] auf Treffer n.
|
| 945 |
+
|
| 946 |
+
Dieser Pfad ist nur ein Sicherheitsnetz. Bevorzugt werden Composer-Sources,
|
| 947 |
+
weil der Composer den Kontext sortiert/dedupliziert und die Nummern korrekt
|
| 948 |
+
kennt.
|
| 949 |
+
"""
|
| 950 |
+
cited_sources: List[Dict[str, Any]] = []
|
| 951 |
+
for num in cited_numbers:
|
| 952 |
+
idx = num - 1
|
| 953 |
+
if 0 <= idx < len(hits):
|
| 954 |
+
cited_sources.append(_source_from_hit(hits[idx], source_number=num))
|
| 955 |
+
return _dedupe_sources(cited_sources, max_sources=max_sources)
|
| 956 |
+
|
| 957 |
+
|
| 958 |
+
_FINE_REF_RE = re.compile(
|
| 959 |
+
r"§\s*(\d{1,3}[a-z]?)\s+Abs\.\s*(\d+[a-z]?)\s+Buchst\.\s*([a-z])",
|
| 960 |
+
flags=re.I,
|
| 961 |
+
)
|
| 962 |
+
|
| 963 |
+
|
| 964 |
+
def _evidence_blob(sources: List[Dict[str, Any]], hits: List[Dict[str, Any]]) -> str:
|
| 965 |
+
parts: List[str] = []
|
| 966 |
+
for item in list(sources or []) + list(hits or []):
|
| 967 |
+
metadata = item.get("metadata") or {}
|
| 968 |
+
parts.extend(
|
| 969 |
+
str(value)
|
| 970 |
+
for value in [
|
| 971 |
+
item.get("canonical_ref"),
|
| 972 |
+
item.get("canonical_refs"),
|
| 973 |
+
item.get("section"),
|
| 974 |
+
item.get("section_id"),
|
| 975 |
+
item.get("path"),
|
| 976 |
+
item.get("text"),
|
| 977 |
+
item.get("document"),
|
| 978 |
+
metadata.get("canonical_ref"),
|
| 979 |
+
metadata.get("section_id"),
|
| 980 |
+
metadata.get("text"),
|
| 981 |
+
]
|
| 982 |
+
if value
|
| 983 |
+
)
|
| 984 |
+
return " ".join(parts).lower()
|
| 985 |
+
|
| 986 |
+
|
| 987 |
+
def _sanitize_unsupported_fine_references(
|
| 988 |
+
answer: str,
|
| 989 |
+
*,
|
| 990 |
+
sources: List[Dict[str, Any]],
|
| 991 |
+
hits: List[Dict[str, Any]],
|
| 992 |
+
) -> str:
|
| 993 |
+
"""Entschärft erfundene Feinfundstellen wie '§ 6 Abs. 1 Buchst. a'.
|
| 994 |
+
|
| 995 |
+
Wenn die genaue Buchstabenfundstelle nicht im Kontext/Metadaten belegt ist,
|
| 996 |
+
wird auf die belastbarere Absatzfundstelle zurückgeführt.
|
| 997 |
+
"""
|
| 998 |
+
if not answer:
|
| 999 |
+
return answer
|
| 1000 |
+
|
| 1001 |
+
evidence = _evidence_blob(sources, hits)
|
| 1002 |
+
|
| 1003 |
+
def repl(match: re.Match[str]) -> str:
|
| 1004 |
+
para, abs_no, letter = match.group(1), match.group(2), match.group(3).lower()
|
| 1005 |
+
exact_patterns = [
|
| 1006 |
+
f"§ {para} abs. {abs_no} buchst. {letter}",
|
| 1007 |
+
f"§{para} abs. {abs_no} buchst. {letter}",
|
| 1008 |
+
f"§ {para} absatz {abs_no} buchstabe {letter}",
|
| 1009 |
+
f"§ {para} abs. {abs_no} lit. {letter}",
|
| 1010 |
+
]
|
| 1011 |
+
if any(pattern in evidence for pattern in exact_patterns):
|
| 1012 |
+
return match.group(0)
|
| 1013 |
+
return f"§ {para} Abs. {abs_no}"
|
| 1014 |
+
|
| 1015 |
+
cleaned = _FINE_REF_RE.sub(repl, answer)
|
| 1016 |
+
|
| 1017 |
+
# Entfernt häufige Dopplung: "(§ 6 Abs. 1), § 6 Abs. 1"
|
| 1018 |
+
cleaned = re.sub(
|
| 1019 |
+
r"\((§\s*\d{1,3}[a-z]?\s+Abs\.\s*\d+[a-z]?)\),\s*\1",
|
| 1020 |
+
r"(\1)",
|
| 1021 |
+
cleaned,
|
| 1022 |
+
flags=re.I,
|
| 1023 |
+
)
|
| 1024 |
+
return cleaned
|
| 1025 |
+
|
| 1026 |
+
|
| 1027 |
+
def _postprocess_answer(
|
| 1028 |
+
composer: AnswerComposer,
|
| 1029 |
+
answer: str,
|
| 1030 |
+
*,
|
| 1031 |
+
sources: List[Dict[str, Any]],
|
| 1032 |
+
hits: List[Dict[str, Any]],
|
| 1033 |
+
) -> str:
|
| 1034 |
+
"""Zentraler letzter Antwort-Postprocessor für Composer- und Orchestratorpfad."""
|
| 1035 |
+
text = answer or ""
|
| 1036 |
+
|
| 1037 |
+
if hasattr(composer, "_strip_model_generated_sources"):
|
| 1038 |
+
try:
|
| 1039 |
+
text = composer._strip_model_generated_sources(text) # type: ignore[attr-defined]
|
| 1040 |
+
except Exception as exc:
|
| 1041 |
+
logger.debug("Composer source-strip postprocessing skipped: %s", exc)
|
| 1042 |
+
|
| 1043 |
+
if hasattr(composer, "_strip_invalid_source_markers"):
|
| 1044 |
+
try:
|
| 1045 |
+
# Erwartet meist used_hits/source_number-Strukturen; sources sind dafür
|
| 1046 |
+
# besser geeignet als Raw-Hits.
|
| 1047 |
+
text = composer._strip_invalid_source_markers(text, sources or hits) # type: ignore[attr-defined]
|
| 1048 |
+
except Exception as exc:
|
| 1049 |
+
logger.debug("Composer marker postprocessing skipped: %s", exc)
|
| 1050 |
+
|
| 1051 |
+
text = _sanitize_unsupported_fine_references(text, sources=sources, hits=hits)
|
| 1052 |
+
return text.strip()
|
| 1053 |
+
|
| 1054 |
+
|
| 1055 |
def _normalize_returned_sources(
|
| 1056 |
*,
|
| 1057 |
answer: str,
|
|
|
|
| 1062 |
"""
|
| 1063 |
Finale API-Quellenlogik.
|
| 1064 |
|
| 1065 |
+
Wichtigste Regel:
|
| 1066 |
+
Composer-/Orchestrator-Sources sind der primäre Wahrheitsanker für [Quelle n].
|
| 1067 |
+
Raw-Hits werden nur als Fallback verwendet. Dadurch bleiben display_title,
|
| 1068 |
+
source_numbers und canonical_refs erhalten.
|
|
|
|
|
|
|
|
|
|
| 1069 |
"""
|
| 1070 |
max_sources = payload.max_sources or DEFAULT_MAX_SOURCES
|
|
|
|
| 1071 |
cited_numbers = _extract_answer_source_numbers(answer)
|
| 1072 |
+
|
| 1073 |
+
normalized = _dedupe_sources(
|
| 1074 |
+
sources or [],
|
| 1075 |
+
max_sources=max(max_sources, len(cited_numbers), DEFAULT_MAX_SOURCES),
|
| 1076 |
+
)
|
| 1077 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1078 |
if normalized:
|
| 1079 |
+
if SOURCE_SYNC_TO_ANSWER_MARKERS and cited_numbers:
|
| 1080 |
+
# Zeige mindestens alle zitierten Quellen. Wenn Composer-Sources die
|
| 1081 |
+
# zitierten Nummern abdecken, keine Rückabbildung auf Raw-Hits.
|
| 1082 |
+
if _sources_cover_cited_numbers(normalized, cited_numbers):
|
| 1083 |
+
cited_set = set(cited_numbers)
|
| 1084 |
+
cited_first = [
|
| 1085 |
+
src for src in normalized
|
| 1086 |
+
if cited_set.intersection(_int_list(src.get("source_numbers"), src.get("source_number")))
|
| 1087 |
+
]
|
| 1088 |
+
rest = [src for src in normalized if src not in cited_first]
|
| 1089 |
+
return (cited_first + rest)[:max(max_sources, len(cited_first))]
|
| 1090 |
+
fallback = _sources_for_cited_numbers_from_raw_hits(cited_numbers, hits, max_sources=max_sources)
|
| 1091 |
+
if fallback:
|
| 1092 |
+
return fallback
|
| 1093 |
+
return normalized[:max_sources]
|
| 1094 |
+
|
| 1095 |
+
if SOURCE_SYNC_TO_ANSWER_MARKERS and cited_numbers and hits:
|
| 1096 |
+
fallback = _sources_for_cited_numbers_from_raw_hits(cited_numbers, hits, max_sources=max_sources)
|
| 1097 |
+
if fallback:
|
| 1098 |
+
return fallback
|
| 1099 |
|
| 1100 |
return _dedupe_sources(hits, max_sources=max_sources)
|
| 1101 |
|
| 1102 |
|
| 1103 |
+
_INLINE_MARKER_RE = re.compile(r"\[Quellen?\s+\d+(?:[^\]]*)\]", flags=re.I)
|
| 1104 |
+
|
| 1105 |
+
|
| 1106 |
+
def _remap_inline_source_markers(answer: str, old_to_new: Dict[int, int]) -> str:
|
| 1107 |
+
"""Schreibt Inline-Marker [Quelle n]/[Quellen n, m] auf die neuen Anzeigenummern um.
|
| 1108 |
+
|
| 1109 |
+
Unbekannte Nummern (kein Mapping vorhanden) bleiben unverändert stehen; sie
|
| 1110 |
+
werden an anderer Stelle bereits als ungültige Marker entfernt.
|
| 1111 |
+
"""
|
| 1112 |
+
if not answer or not old_to_new:
|
| 1113 |
+
return answer
|
| 1114 |
+
|
| 1115 |
+
def repl(match: re.Match[str]) -> str:
|
| 1116 |
+
mapped: List[int] = []
|
| 1117 |
+
for raw in re.findall(r"\d+", match.group(0)):
|
| 1118 |
+
try:
|
| 1119 |
+
new = old_to_new.get(int(raw))
|
| 1120 |
+
except (TypeError, ValueError):
|
| 1121 |
+
new = None
|
| 1122 |
+
if new is not None and new not in mapped:
|
| 1123 |
+
mapped.append(new)
|
| 1124 |
+
if not mapped:
|
| 1125 |
+
return match.group(0)
|
| 1126 |
+
mapped.sort()
|
| 1127 |
+
return _source_marker(mapped)
|
| 1128 |
+
|
| 1129 |
+
return _INLINE_MARKER_RE.sub(repl, answer)
|
| 1130 |
+
|
| 1131 |
+
|
| 1132 |
+
def _renumber_sources_for_display(
|
| 1133 |
+
answer: str,
|
| 1134 |
+
sources: List[Dict[str, Any]],
|
| 1135 |
+
) -> Tuple[str, List[Dict[str, Any]]]:
|
| 1136 |
+
"""Vergibt stabile, fortlaufende Anzeigenummern (1..n) für die finalen Quellen.
|
| 1137 |
+
|
| 1138 |
+
Hintergrund: Die ursprünglichen [Quelle n]-Nummern sind Retrieval-Ränge aus
|
| 1139 |
+
dem RAG-Kontext. Dadurch beginnt die Anzeige nicht bei 1 und enthält Lücken
|
| 1140 |
+
(z. B. "[Quellen 3, 8]"). Für die UI werden die tatsächlich angezeigten
|
| 1141 |
+
Quellen in ihrer Anzeige-Reihenfolge auf 1..n abgebildet und die
|
| 1142 |
+
Inline-Marker im Antworttext konsistent mitgezogen. Mehrere Alt-Nummern
|
| 1143 |
+
derselben Quelle (gruppierte Chunks) fallen dabei auf eine Anzeigenummer
|
| 1144 |
+
zusammen.
|
| 1145 |
+
"""
|
| 1146 |
+
if not sources:
|
| 1147 |
+
return answer, sources
|
| 1148 |
+
|
| 1149 |
+
old_to_new: Dict[int, int] = {}
|
| 1150 |
+
next_number = 1
|
| 1151 |
+
|
| 1152 |
+
for source in sources:
|
| 1153 |
+
old_numbers = _int_list(source.get("source_numbers"), source.get("source_number"))
|
| 1154 |
+
if not old_numbers:
|
| 1155 |
+
continue
|
| 1156 |
+
|
| 1157 |
+
assigned: Optional[int] = None
|
| 1158 |
+
for old in old_numbers:
|
| 1159 |
+
if old in old_to_new:
|
| 1160 |
+
assigned = old_to_new[old]
|
| 1161 |
+
break
|
| 1162 |
+
if assigned is None:
|
| 1163 |
+
assigned = next_number
|
| 1164 |
+
next_number += 1
|
| 1165 |
+
for old in old_numbers:
|
| 1166 |
+
old_to_new.setdefault(old, assigned)
|
| 1167 |
+
|
| 1168 |
+
marker = f"[Quelle {assigned}]"
|
| 1169 |
+
source["source_number"] = assigned
|
| 1170 |
+
source["source_numbers"] = [assigned]
|
| 1171 |
+
source["source_marker"] = marker
|
| 1172 |
+
source["source_label"] = marker
|
| 1173 |
+
source["display_title"] = _format_source_display_title(source)
|
| 1174 |
+
source["display_label"] = source["display_title"]
|
| 1175 |
+
|
| 1176 |
+
new_answer = _remap_inline_source_markers(answer, old_to_new)
|
| 1177 |
+
return new_answer, sources
|
| 1178 |
+
|
| 1179 |
+
|
| 1180 |
def _debug_hit(hit: Dict[str, Any]) -> Dict[str, Any]:
|
| 1181 |
text = (hit.get("text") or hit.get("document") or "").strip()
|
| 1182 |
return {
|
|
|
|
| 1195 |
|
| 1196 |
|
| 1197 |
# -----------------------------------------------------------------------------
|
| 1198 |
+
# Ask Graph Workflow
|
| 1199 |
# -----------------------------------------------------------------------------
|
| 1200 |
|
| 1201 |
+
AskRoute = Literal["meta", "non_meta", "legacy", "clarification"]
|
| 1202 |
+
AskNodeName = Literal[
|
| 1203 |
+
"__start__",
|
| 1204 |
+
"meta_decision",
|
| 1205 |
+
"retrieve_hits",
|
| 1206 |
+
"compose_answer",
|
| 1207 |
+
"orchestrate_answer",
|
| 1208 |
+
"normalize_sources",
|
| 1209 |
+
"update_memory",
|
| 1210 |
+
"build_response",
|
| 1211 |
+
"__end__",
|
| 1212 |
+
]
|
| 1213 |
|
|
|
|
|
|
|
| 1214 |
|
| 1215 |
+
@dataclass
|
| 1216 |
+
class AskGraphStep:
|
| 1217 |
+
"""Rückgabe eines Graph-Knotens."""
|
| 1218 |
+
|
| 1219 |
+
next_node: AskNodeName
|
| 1220 |
+
reason: str = ""
|
| 1221 |
+
|
| 1222 |
+
|
| 1223 |
+
@dataclass
|
| 1224 |
+
class AskGraphTraceEntry:
|
| 1225 |
+
node: str
|
| 1226 |
+
next_node: str
|
| 1227 |
+
reason: str = ""
|
| 1228 |
+
route: Optional[str] = None
|
| 1229 |
+
hit_count: int = 0
|
| 1230 |
+
answer_type: str = "unknown"
|
| 1231 |
+
|
| 1232 |
+
|
| 1233 |
+
@dataclass
|
| 1234 |
+
class AskGraphContext:
|
| 1235 |
+
"""
|
| 1236 |
+
Gemeinsamer Zustand des /ask-Graphen.
|
| 1237 |
+
|
| 1238 |
+
Dieser Graph kapselt ausschließlich die bereits vorhandene Funktionalität:
|
| 1239 |
+
Meta-Fragen werden ohne Retrieval beantwortet, alle anderen Fragen laufen
|
| 1240 |
+
durch Retrieval, AnswerComposer, Quellennormalisierung, Memory-Update und
|
| 1241 |
+
Response-Aufbau.
|
| 1242 |
+
"""
|
| 1243 |
+
|
| 1244 |
+
payload: Question
|
| 1245 |
+
request: Request
|
| 1246 |
+
session: SessionState
|
| 1247 |
+
memory: ConversationMemory
|
| 1248 |
+
llm: GroqClient
|
| 1249 |
+
composer: AnswerComposer
|
| 1250 |
+
question: str
|
| 1251 |
+
orchestrator: Optional[LegalAnswerOrchestrator] = None
|
| 1252 |
+
|
| 1253 |
+
route: Optional[AskRoute] = None
|
| 1254 |
+
hits: Optional[List[Dict[str, Any]]] = None
|
| 1255 |
+
raw_sources: Optional[List[Dict[str, Any]]] = None
|
| 1256 |
+
sources: Optional[List[Dict[str, Any]]] = None
|
| 1257 |
+
answer: str = ""
|
| 1258 |
+
answer_type: str = "unknown"
|
| 1259 |
+
response_body: Optional[Dict[str, Any]] = None
|
| 1260 |
+
orchestrator_debug: Optional[Dict[str, Any]] = None
|
| 1261 |
+
needs_clarification: bool = False
|
| 1262 |
+
clarification_question: Optional[str] = None
|
| 1263 |
+
trace: Optional[List[AskGraphTraceEntry]] = None
|
| 1264 |
+
|
| 1265 |
+
def __post_init__(self) -> None:
|
| 1266 |
+
if self.hits is None:
|
| 1267 |
+
self.hits = []
|
| 1268 |
+
if self.raw_sources is None:
|
| 1269 |
+
self.raw_sources = []
|
| 1270 |
+
if self.sources is None:
|
| 1271 |
+
self.sources = []
|
| 1272 |
+
if self.trace is None:
|
| 1273 |
+
self.trace = []
|
| 1274 |
+
|
| 1275 |
+
|
| 1276 |
+
def _ask_graph_meta_decision(ctx: AskGraphContext) -> AskGraphStep:
|
| 1277 |
+
"""Routet Meta-Fragen, Orchestrator-Fragen und Legacy-Fallback sauber."""
|
| 1278 |
+
if is_meta_question(ctx.question):
|
| 1279 |
+
ctx.route = "meta"
|
| 1280 |
+
return AskGraphStep("compose_answer", "meta question without retrieval")
|
| 1281 |
+
|
| 1282 |
+
if ORCHESTRATOR_ENABLED and ctx.orchestrator is not None:
|
| 1283 |
+
ctx.route = "non_meta"
|
| 1284 |
+
return AskGraphStep("orchestrate_answer", "regular question via legal orchestrator")
|
| 1285 |
+
|
| 1286 |
+
ctx.route = "legacy"
|
| 1287 |
+
return AskGraphStep("retrieve_hits", "orchestrator unavailable or disabled; legacy retrieval")
|
| 1288 |
+
|
| 1289 |
+
|
| 1290 |
+
def _ask_graph_retrieve_hits(ctx: AskGraphContext) -> AskGraphStep:
|
| 1291 |
+
"""Entspricht dem bisherigen Aufruf von _retrieve_hits(payload)."""
|
| 1292 |
+
ctx.hits = _retrieve_hits(ctx.payload)
|
| 1293 |
+
return AskGraphStep("compose_answer", f"retrieved {len(ctx.hits)} hits")
|
| 1294 |
+
|
| 1295 |
+
|
| 1296 |
+
def _ask_graph_orchestrate_answer(ctx: AskGraphContext) -> AskGraphStep:
|
| 1297 |
+
"""Fachlicher Orchestrator-Knoten: Retrieval, Recheck, Antwort, Audit."""
|
| 1298 |
+
if ctx.orchestrator is None:
|
| 1299 |
+
ctx.route = "legacy"
|
| 1300 |
+
return AskGraphStep("retrieve_hits", "orchestrator unavailable; falling back to legacy retrieval")
|
| 1301 |
+
|
| 1302 |
+
result = ctx.orchestrator.run(ctx.question, memory=ctx.memory)
|
| 1303 |
+
ctx.answer = result.answer
|
| 1304 |
+
ctx.answer_type = result.answer_type
|
| 1305 |
+
ctx.hits = result.hits
|
| 1306 |
+
ctx.raw_sources = result.raw_sources
|
| 1307 |
+
ctx.sources = result.sources
|
| 1308 |
+
ctx.needs_clarification = result.needs_clarification
|
| 1309 |
+
ctx.clarification_question = result.clarification_question
|
| 1310 |
+
ctx.orchestrator_debug = result.to_debug_dict()
|
| 1311 |
+
if result.needs_clarification:
|
| 1312 |
+
ctx.route = "clarification"
|
| 1313 |
+
return AskGraphStep("normalize_sources", "answer orchestrated and audited")
|
| 1314 |
+
|
| 1315 |
+
|
| 1316 |
+
def _ask_graph_compose_answer(ctx: AskGraphContext) -> AskGraphStep:
|
| 1317 |
+
"""
|
| 1318 |
+
Entspricht der bisherigen Antwortgenerierung im /ask-Endpoint.
|
| 1319 |
+
|
| 1320 |
+
Meta-Fragen werden mit leerem Kontext komponiert. Nicht-Meta-Fragen nutzen
|
| 1321 |
+
weiterhin compose_with_sources, sofern vorhanden, sonst compose plus
|
| 1322 |
+
Quellen-Fallback.
|
| 1323 |
+
"""
|
| 1324 |
+
if ctx.route == "meta":
|
| 1325 |
+
ctx.answer, ctx.answer_type = ctx.composer.compose(ctx.question, [], memory=ctx.memory)
|
| 1326 |
+
ctx.raw_sources = []
|
| 1327 |
+
return AskGraphStep("normalize_sources", "meta answer composed")
|
| 1328 |
+
|
| 1329 |
+
if hasattr(ctx.composer, "compose_with_sources"):
|
| 1330 |
+
try:
|
| 1331 |
+
ctx.answer, ctx.answer_type, raw_sources = ctx.composer.compose_with_sources(
|
| 1332 |
+
ctx.question,
|
| 1333 |
+
ctx.hits,
|
| 1334 |
+
memory=ctx.memory,
|
| 1335 |
+
)
|
| 1336 |
+
except TypeError:
|
| 1337 |
+
ctx.answer, ctx.answer_type = ctx.composer.compose(ctx.question, ctx.hits, memory=ctx.memory)
|
| 1338 |
+
raw_sources = _build_sources_from_composer_or_hits(ctx.composer, ctx.hits, ctx.payload)
|
| 1339 |
+
else:
|
| 1340 |
+
ctx.answer, ctx.answer_type = ctx.composer.compose(ctx.question, ctx.hits, memory=ctx.memory)
|
| 1341 |
+
raw_sources = _build_sources_from_composer_or_hits(ctx.composer, ctx.hits, ctx.payload)
|
| 1342 |
+
|
| 1343 |
+
ctx.raw_sources = raw_sources
|
| 1344 |
+
return AskGraphStep("normalize_sources", "answer composed")
|
| 1345 |
+
|
| 1346 |
+
|
| 1347 |
+
def _ask_graph_normalize_sources(ctx: AskGraphContext) -> AskGraphStep:
|
| 1348 |
+
"""Normalisiert Quellen und führt finalen Antwort-Postprocessing-Schritt aus.
|
| 1349 |
+
|
| 1350 |
+
Der Orchestratorpfad darf den Composer-Postprocessor nicht umgehen. Deshalb
|
| 1351 |
+
werden hier Quellen zuerst nummernstabil normalisiert, danach wird die
|
| 1352 |
+
Antwort gegen diese Quellen nachbearbeitet und anschließend erneut leicht
|
| 1353 |
+
synchronisiert.
|
| 1354 |
+
"""
|
| 1355 |
+
if ctx.answer_type == "document":
|
| 1356 |
+
raw_input_sources = ctx.raw_sources or ctx.sources or []
|
| 1357 |
+
ctx.sources = _normalize_returned_sources(
|
| 1358 |
+
answer=ctx.answer,
|
| 1359 |
+
sources=raw_input_sources,
|
| 1360 |
+
hits=ctx.hits,
|
| 1361 |
+
payload=ctx.payload,
|
| 1362 |
+
)
|
| 1363 |
+
ctx.answer = _postprocess_answer(
|
| 1364 |
+
ctx.composer,
|
| 1365 |
+
ctx.answer,
|
| 1366 |
+
sources=ctx.sources,
|
| 1367 |
+
hits=ctx.hits or [],
|
| 1368 |
+
)
|
| 1369 |
+
ctx.sources = _normalize_returned_sources(
|
| 1370 |
+
answer=ctx.answer,
|
| 1371 |
+
sources=ctx.sources or raw_input_sources,
|
| 1372 |
+
hits=ctx.hits,
|
| 1373 |
+
payload=ctx.payload,
|
| 1374 |
+
)
|
| 1375 |
+
# Anzeige-Nummern stabil auf 1..n abbilden (statt Retrieval-Ränge wie
|
| 1376 |
+
# "[Quellen 3, 8]"). Muss als letzter Schritt laufen, damit Antworttext
|
| 1377 |
+
# und Quellenliste dieselben Nummern zeigen.
|
| 1378 |
+
ctx.answer, ctx.sources = _renumber_sources_for_display(ctx.answer, ctx.sources)
|
| 1379 |
+
else:
|
| 1380 |
+
ctx.sources = []
|
| 1381 |
+
|
| 1382 |
+
return AskGraphStep("update_memory", "sources normalized and answer postprocessed")
|
| 1383 |
+
|
| 1384 |
+
|
| 1385 |
+
def _ask_graph_update_memory(ctx: AskGraphContext) -> AskGraphStep:
|
| 1386 |
+
"""Entspricht dem bisherigen memory.add_turn(...)."""
|
| 1387 |
+
ctx.memory.add_turn(
|
| 1388 |
+
user_message=ctx.question,
|
| 1389 |
+
assistant_message=ctx.answer,
|
| 1390 |
+
question_kind=classify_question(ctx.question),
|
| 1391 |
+
)
|
| 1392 |
+
return AskGraphStep("build_response", "conversation memory updated")
|
| 1393 |
+
|
| 1394 |
+
|
| 1395 |
+
_PAGE_RANGE_RE = re.compile(r"(\d+)\s*[–\-]\s*(\d+)|^(\d+)$")
|
| 1396 |
+
|
| 1397 |
+
|
| 1398 |
+
def _attach_pdf_locators(sources: List[Dict[str, Any]], hits: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
| 1399 |
+
"""Sicherheitsnetz: fehlende PDF-Locator aus Hits bzw. page_range ableiten.
|
| 1400 |
+
|
| 1401 |
+
Composer/Orchestrator liefern die Felder normalerweise bereits mit. Ältere
|
| 1402 |
+
Pfade (Legacy-Fallbacks) können sie verlieren; da der Korpus derzeit aus
|
| 1403 |
+
einem Dokument besteht, ist source_file aus einem beliebigen Hit belastbar.
|
| 1404 |
+
"""
|
| 1405 |
+
fallback_file = None
|
| 1406 |
+
fallback_doc_id = None
|
| 1407 |
+
for hit in hits or []:
|
| 1408 |
+
metadata = hit.get("metadata") or {}
|
| 1409 |
+
fallback_file = fallback_file or _coalesce(hit.get("source_file"), metadata.get("source_file"))
|
| 1410 |
+
fallback_doc_id = fallback_doc_id or _coalesce(hit.get("doc_id"), metadata.get("doc_id"))
|
| 1411 |
+
if fallback_file and fallback_doc_id:
|
| 1412 |
+
break
|
| 1413 |
+
|
| 1414 |
+
for source in sources or []:
|
| 1415 |
+
if not source.get("source_file") and fallback_file:
|
| 1416 |
+
source["source_file"] = fallback_file
|
| 1417 |
+
if not source.get("doc_id") and fallback_doc_id:
|
| 1418 |
+
source["doc_id"] = fallback_doc_id
|
| 1419 |
+
|
| 1420 |
+
if source.get("page_start") is None:
|
| 1421 |
+
match = _PAGE_RANGE_RE.search(str(source.get("page_range") or source.get("pages") or ""))
|
| 1422 |
+
if match:
|
| 1423 |
+
start = match.group(1) or match.group(3)
|
| 1424 |
+
end = match.group(2) or start
|
| 1425 |
+
source["page_start"] = _int_or_none(start)
|
| 1426 |
+
source["page_end"] = _int_or_none(end)
|
| 1427 |
+
if source.get("page_end") is None:
|
| 1428 |
+
source["page_end"] = source.get("page_start")
|
| 1429 |
+
|
| 1430 |
+
if not source.get("highlights"):
|
| 1431 |
+
source["highlights"] = []
|
| 1432 |
+
|
| 1433 |
+
return sources
|
| 1434 |
+
|
| 1435 |
+
|
| 1436 |
+
def _ask_graph_build_response(ctx: AskGraphContext) -> AskGraphStep:
|
| 1437 |
+
"""Baut exakt die bisherige API-Response-Struktur."""
|
| 1438 |
+
_attach_pdf_locators(ctx.sources or [], ctx.hits or [])
|
| 1439 |
+
ctx.response_body = {
|
| 1440 |
+
"question": ctx.question,
|
| 1441 |
+
"answer": ctx.answer,
|
| 1442 |
+
"answer_type": ctx.answer_type,
|
| 1443 |
+
"session_id": ctx.request.state.session_id,
|
| 1444 |
+
"factual_question_index": ctx.memory.factual_question_count,
|
| 1445 |
+
"total_turns": ctx.memory.total_turns,
|
| 1446 |
+
"sources": ctx.sources or [],
|
| 1447 |
+
"needs_clarification": ctx.needs_clarification,
|
| 1448 |
+
"clarification_question": ctx.clarification_question,
|
| 1449 |
+
}
|
| 1450 |
+
return AskGraphStep("__end__", "response built")
|
| 1451 |
+
|
| 1452 |
+
|
| 1453 |
+
ASK_GRAPH_NODES: Dict[AskNodeName, Callable[[AskGraphContext], AskGraphStep]] = {
|
| 1454 |
+
"meta_decision": _ask_graph_meta_decision,
|
| 1455 |
+
"retrieve_hits": _ask_graph_retrieve_hits,
|
| 1456 |
+
"compose_answer": _ask_graph_compose_answer,
|
| 1457 |
+
"orchestrate_answer": _ask_graph_orchestrate_answer,
|
| 1458 |
+
"normalize_sources": _ask_graph_normalize_sources,
|
| 1459 |
+
"update_memory": _ask_graph_update_memory,
|
| 1460 |
+
"build_response": _ask_graph_build_response,
|
| 1461 |
+
}
|
| 1462 |
+
|
| 1463 |
+
ASK_GRAPH: Dict[str, List[Dict[str, str]]] = {
|
| 1464 |
+
"__start__": [{"to": "meta_decision", "label": ""}],
|
| 1465 |
+
"meta_decision": [
|
| 1466 |
+
{"to": "compose_answer", "label": "meta"},
|
| 1467 |
+
{"to": "orchestrate_answer", "label": "non_meta"},
|
| 1468 |
+
{"to": "retrieve_hits", "label": "legacy"},
|
| 1469 |
+
],
|
| 1470 |
+
"retrieve_hits": [{"to": "compose_answer", "label": ""}],
|
| 1471 |
+
"orchestrate_answer": [{"to": "normalize_sources", "label": ""}],
|
| 1472 |
+
"compose_answer": [{"to": "normalize_sources", "label": ""}],
|
| 1473 |
+
"normalize_sources": [{"to": "update_memory", "label": ""}],
|
| 1474 |
+
"update_memory": [{"to": "build_response", "label": ""}],
|
| 1475 |
+
"build_response": [{"to": "__end__", "label": ""}],
|
| 1476 |
+
}
|
| 1477 |
+
|
| 1478 |
+
|
| 1479 |
+
class AskGraphVizState(TypedDict, total=False):
|
| 1480 |
+
"""Minimaler LangGraph-State nur für Visualisierung/Rendering."""
|
| 1481 |
+
|
| 1482 |
+
route: str
|
| 1483 |
+
|
| 1484 |
+
|
| 1485 |
+
def _ask_langgraph_passthrough(state: AskGraphVizState) -> AskGraphVizState:
|
| 1486 |
+
"""Dummy-Node für LangGraph-Rendering; die echte Logik bleibt in run_ask_graph."""
|
| 1487 |
+
return state
|
| 1488 |
+
|
| 1489 |
+
|
| 1490 |
+
def _make_ask_langgraph_router(source: str, labels: List[str]):
|
| 1491 |
+
"""
|
| 1492 |
+
Erzeugt einen Router für LangGraph-Visualisierung.
|
| 1493 |
+
|
| 1494 |
+
Für das Rendering ist nur die Mapping-Struktur wichtig. Falls der Graph
|
| 1495 |
+
doch testweise ausgeführt wird, kann pro Node über '<node>_route' geroutet
|
| 1496 |
+
werden; sonst wird der erste Label-Zweig genutzt.
|
| 1497 |
+
"""
|
| 1498 |
+
default_label = labels[0]
|
| 1499 |
+
|
| 1500 |
+
def _router(state: AskGraphVizState) -> str:
|
| 1501 |
+
return state.get(f"{source}_route", state.get("route", default_label))
|
| 1502 |
+
|
| 1503 |
+
return _router
|
| 1504 |
+
|
| 1505 |
|
| 1506 |
+
def build_ask_workflow():
|
| 1507 |
+
"""
|
| 1508 |
+
Baut den LangGraph-Workflow aus ASK_GRAPH.
|
| 1509 |
+
|
| 1510 |
+
Diese Funktion ist die stabile Schnittstelle für visualize_graph.py:
|
| 1511 |
+
from app import build_ask_workflow
|
| 1512 |
+
workflow = build_ask_workflow()
|
| 1513 |
+
workflow.get_graph(xray=True).draw_mermaid_png()
|
| 1514 |
+
|
| 1515 |
+
Wenn ASK_GRAPH in app.py geändert wird, übernimmt die Visualisierung diese
|
| 1516 |
+
Änderung automatisch, ohne dass visualize_graph.py angepasst werden muss.
|
| 1517 |
+
"""
|
| 1518 |
+
if StateGraph is None:
|
| 1519 |
+
raise RuntimeError(
|
| 1520 |
+
"LangGraph ist nicht installiert. Bitte ausführen: pip install -U langgraph langchain-core"
|
| 1521 |
+
)
|
| 1522 |
+
|
| 1523 |
+
workflow = StateGraph(AskGraphVizState)
|
| 1524 |
+
|
| 1525 |
+
node_names = set()
|
| 1526 |
+
for source, edges in ASK_GRAPH.items():
|
| 1527 |
+
if source not in {"__start__", "__end__"}:
|
| 1528 |
+
node_names.add(source)
|
| 1529 |
+
for edge in edges:
|
| 1530 |
+
target = edge["to"]
|
| 1531 |
+
if target not in {"__start__", "__end__"}:
|
| 1532 |
+
node_names.add(target)
|
| 1533 |
+
|
| 1534 |
+
for node_name in sorted(node_names):
|
| 1535 |
+
workflow.add_node(node_name, _ask_langgraph_passthrough)
|
| 1536 |
+
|
| 1537 |
+
start_edges = ASK_GRAPH.get("__start__") or []
|
| 1538 |
+
if not start_edges:
|
| 1539 |
+
raise RuntimeError("ASK_GRAPH benötigt eine __start__-Kante.")
|
| 1540 |
+
workflow.set_entry_point(start_edges[0]["to"])
|
| 1541 |
+
|
| 1542 |
+
for source, edges in ASK_GRAPH.items():
|
| 1543 |
+
if source in {"__start__", "__end__"}:
|
| 1544 |
+
continue
|
| 1545 |
+
|
| 1546 |
+
labelled_edges = [edge for edge in edges if edge.get("label")]
|
| 1547 |
+
plain_edges = [edge for edge in edges if not edge.get("label")]
|
| 1548 |
+
|
| 1549 |
+
if labelled_edges:
|
| 1550 |
+
labels = [edge["label"] for edge in labelled_edges]
|
| 1551 |
+
workflow.add_conditional_edges(
|
| 1552 |
+
source,
|
| 1553 |
+
_make_ask_langgraph_router(source, labels),
|
| 1554 |
+
{edge["label"]: (END if edge["to"] == "__end__" else edge["to"]) for edge in labelled_edges},
|
| 1555 |
+
)
|
| 1556 |
+
|
| 1557 |
+
for edge in plain_edges:
|
| 1558 |
+
target = END if edge["to"] == "__end__" else edge["to"]
|
| 1559 |
+
workflow.add_edge(source, target)
|
| 1560 |
+
|
| 1561 |
+
return workflow.compile()
|
| 1562 |
+
|
| 1563 |
+
|
| 1564 |
+
def get_ask_workflow():
|
| 1565 |
+
"""Alias für Visualisierungsskripte, die eine Getter-Funktion bevorzugen."""
|
| 1566 |
+
return build_ask_workflow()
|
| 1567 |
+
|
| 1568 |
+
|
| 1569 |
+
# Backwards-/Convenience-Aliase für Visualisierungsskripte.
|
| 1570 |
+
# Wichtig: FastAPI bleibt weiterhin in der Variable `app`; der LangGraph darf
|
| 1571 |
+
# deshalb nicht ebenfalls `app` heißen.
|
| 1572 |
+
try:
|
| 1573 |
+
ASK_WORKFLOW = build_ask_workflow()
|
| 1574 |
+
except Exception:
|
| 1575 |
+
ASK_WORKFLOW = None
|
| 1576 |
+
|
| 1577 |
+
AGENT_GRAPH = ASK_GRAPH
|
| 1578 |
+
ASK_LANGGRAPH_APP = ASK_WORKFLOW
|
| 1579 |
+
|
| 1580 |
+
|
| 1581 |
+
def _ask_graph_mermaid() -> str:
|
| 1582 |
+
lines = ["flowchart TD"]
|
| 1583 |
+
for source, edges in ASK_GRAPH.items():
|
| 1584 |
+
for edge in edges:
|
| 1585 |
+
target = edge["to"]
|
| 1586 |
+
label = edge.get("label") or ""
|
| 1587 |
+
if label:
|
| 1588 |
+
lines.append(f' {source}["{source}"] -- "{label}" --> {target}["{target}"]')
|
| 1589 |
+
else:
|
| 1590 |
+
lines.append(f' {source}["{source}"] --> {target}["{target}"]')
|
| 1591 |
+
return "\n".join(lines)
|
| 1592 |
+
|
| 1593 |
+
|
| 1594 |
+
def run_ask_graph(ctx: AskGraphContext) -> AskGraphContext:
|
| 1595 |
+
"""Führt den /ask-Graphen aus, ohne fachliche Zusatzpfade einzuführen."""
|
| 1596 |
+
current: AskNodeName = "meta_decision"
|
| 1597 |
+
|
| 1598 |
+
for _ in range(MAX_ASK_GRAPH_STEPS):
|
| 1599 |
+
node = ASK_GRAPH_NODES.get(current)
|
| 1600 |
+
if node is None:
|
| 1601 |
+
raise HTTPException(status_code=500, detail=f"Unbekannter Ask-Graph-Knoten: {current}")
|
| 1602 |
+
|
| 1603 |
+
step = node(ctx)
|
| 1604 |
+
ctx.trace.append(
|
| 1605 |
+
AskGraphTraceEntry(
|
| 1606 |
+
node=current,
|
| 1607 |
+
next_node=step.next_node,
|
| 1608 |
+
reason=step.reason,
|
| 1609 |
+
route=ctx.route,
|
| 1610 |
+
hit_count=len(ctx.hits or []),
|
| 1611 |
+
answer_type=ctx.answer_type,
|
| 1612 |
+
)
|
| 1613 |
+
)
|
| 1614 |
+
|
| 1615 |
+
if step.next_node == "__end__":
|
| 1616 |
+
return ctx
|
| 1617 |
+
|
| 1618 |
+
current = step.next_node
|
| 1619 |
+
|
| 1620 |
+
raise HTTPException(
|
| 1621 |
+
status_code=500,
|
| 1622 |
+
detail=f"Ask-Graph nach {MAX_ASK_GRAPH_STEPS} Schritten abgebrochen.",
|
| 1623 |
+
)
|
| 1624 |
+
|
| 1625 |
+
|
| 1626 |
+
def _legacy_answer_flow(ctx: AskGraphContext) -> AskGraphContext:
|
| 1627 |
+
"""
|
| 1628 |
+
Fallback auf die bisherige monolithische /ask-Logik.
|
| 1629 |
+
|
| 1630 |
+
Dieser Pfad bleibt absichtlich funktionsgleich zum Graphen und dient nur
|
| 1631 |
+
als schneller Rollback über ASK_GRAPH_ENABLED=false.
|
| 1632 |
+
"""
|
| 1633 |
+
if is_meta_question(ctx.question):
|
| 1634 |
+
ctx.route = "meta"
|
| 1635 |
+
ctx.hits = []
|
| 1636 |
+
ctx.answer, ctx.answer_type = ctx.composer.compose(ctx.question, [], memory=ctx.memory)
|
| 1637 |
+
ctx.sources = []
|
| 1638 |
else:
|
| 1639 |
+
ctx.route = "non_meta"
|
| 1640 |
+
ctx.hits = _retrieve_hits(ctx.payload)
|
| 1641 |
|
| 1642 |
+
if hasattr(ctx.composer, "compose_with_sources"):
|
| 1643 |
try:
|
| 1644 |
+
ctx.answer, ctx.answer_type, raw_sources = ctx.composer.compose_with_sources(
|
| 1645 |
+
ctx.question,
|
| 1646 |
+
ctx.hits,
|
| 1647 |
+
memory=ctx.memory,
|
| 1648 |
)
|
| 1649 |
except TypeError:
|
| 1650 |
+
ctx.answer, ctx.answer_type = ctx.composer.compose(ctx.question, ctx.hits, memory=ctx.memory)
|
| 1651 |
+
raw_sources = _build_sources_from_composer_or_hits(ctx.composer, ctx.hits, ctx.payload)
|
| 1652 |
else:
|
| 1653 |
+
ctx.answer, ctx.answer_type = ctx.composer.compose(ctx.question, ctx.hits, memory=ctx.memory)
|
| 1654 |
+
raw_sources = _build_sources_from_composer_or_hits(ctx.composer, ctx.hits, ctx.payload)
|
| 1655 |
|
| 1656 |
+
ctx.raw_sources = raw_sources
|
| 1657 |
+
if ctx.answer_type == "document":
|
| 1658 |
+
ctx.sources = _normalize_returned_sources(
|
| 1659 |
+
answer=ctx.answer,
|
| 1660 |
sources=raw_sources,
|
| 1661 |
+
hits=ctx.hits,
|
| 1662 |
+
payload=ctx.payload,
|
| 1663 |
+
)
|
| 1664 |
+
ctx.answer = _postprocess_answer(
|
| 1665 |
+
ctx.composer,
|
| 1666 |
+
ctx.answer,
|
| 1667 |
+
sources=ctx.sources,
|
| 1668 |
+
hits=ctx.hits or [],
|
| 1669 |
+
)
|
| 1670 |
+
ctx.sources = _normalize_returned_sources(
|
| 1671 |
+
answer=ctx.answer,
|
| 1672 |
+
sources=ctx.sources or raw_sources,
|
| 1673 |
+
hits=ctx.hits,
|
| 1674 |
+
payload=ctx.payload,
|
| 1675 |
)
|
| 1676 |
else:
|
| 1677 |
+
ctx.sources = []
|
| 1678 |
|
| 1679 |
+
ctx.memory.add_turn(
|
| 1680 |
+
user_message=ctx.question,
|
| 1681 |
+
assistant_message=ctx.answer,
|
| 1682 |
+
question_kind=classify_question(ctx.question),
|
| 1683 |
)
|
| 1684 |
+
_ask_graph_build_response(ctx)
|
| 1685 |
+
return ctx
|
| 1686 |
+
|
| 1687 |
|
| 1688 |
+
def _ask_graph_debug_payload(ctx: AskGraphContext) -> Dict[str, Any]:
|
| 1689 |
+
return {
|
| 1690 |
+
"ask_graph_enabled": ASK_GRAPH_ENABLED,
|
| 1691 |
+
"orchestrator_enabled": ORCHESTRATOR_ENABLED,
|
| 1692 |
+
"orchestrator": ctx.orchestrator_debug,
|
| 1693 |
+
"workflow_trace": [entry.__dict__ for entry in (ctx.trace or [])],
|
| 1694 |
+
"workflow_graph": ASK_GRAPH,
|
| 1695 |
+
"workflow_mermaid": _ask_graph_mermaid(),
|
| 1696 |
+
"route": ctx.route,
|
| 1697 |
+
"is_meta_question": is_meta_question(ctx.question),
|
| 1698 |
+
"question_kind": classify_question(ctx.question),
|
| 1699 |
+
"retrieved_hit_count": len(ctx.hits or []),
|
| 1700 |
+
"answer_source_numbers": _extract_answer_source_numbers(ctx.answer),
|
| 1701 |
+
"raw_sources": ctx.raw_sources or [],
|
| 1702 |
+
"normalized_sources": ctx.sources or [],
|
| 1703 |
+
"hits": [_debug_hit(hit) for hit in (ctx.hits or [])],
|
| 1704 |
+
"top_k": ctx.payload.top_k or DEFAULT_TOP_K,
|
| 1705 |
+
"fetch_k": ctx.payload.fetch_k or DEFAULT_FETCH_K,
|
| 1706 |
+
"max_final_results": ctx.payload.max_final_results or DEFAULT_MAX_FINAL_RESULTS,
|
| 1707 |
+
"max_sources": ctx.payload.max_sources or DEFAULT_MAX_SOURCES,
|
| 1708 |
+
"include_neighbors": ctx.payload.include_neighbors,
|
| 1709 |
+
"include_explicit_sections": ctx.payload.include_explicit_sections,
|
| 1710 |
+
"restrict_to_default_container": ctx.payload.restrict_to_default_container,
|
| 1711 |
+
"source_sync_to_answer_markers": SOURCE_SYNC_TO_ANSWER_MARKERS,
|
| 1712 |
+
"chroma_persist_dir": CHROMA_PERSIST_DIR,
|
| 1713 |
+
"chroma_collection": CHROMA_COLLECTION,
|
| 1714 |
+
"chroma_count": _collection_count(),
|
| 1715 |
+
}
|
| 1716 |
+
|
| 1717 |
+
|
| 1718 |
+
# -----------------------------------------------------------------------------
|
| 1719 |
+
# API Endpoints
|
| 1720 |
+
# -----------------------------------------------------------------------------
|
| 1721 |
+
|
| 1722 |
+
@app.post("/ask")
|
| 1723 |
+
def ask(payload: Question, request: Request):
|
| 1724 |
+
question = _clean_question(payload.question)
|
| 1725 |
+
if not question:
|
| 1726 |
+
raise HTTPException(status_code=422, detail="question darf nicht leer sein.")
|
| 1727 |
+
|
| 1728 |
+
session: SessionState = request.state.session
|
| 1729 |
+
memory = session.get_memory()
|
| 1730 |
+
|
| 1731 |
+
llm = _build_llm(session)
|
| 1732 |
+
composer = _build_composer(llm, payload)
|
| 1733 |
+
orchestrator = _build_orchestrator(get_retriever(), composer, payload) if ORCHESTRATOR_ENABLED else None
|
| 1734 |
+
|
| 1735 |
+
ctx = AskGraphContext(
|
| 1736 |
+
payload=payload,
|
| 1737 |
+
request=request,
|
| 1738 |
+
session=session,
|
| 1739 |
+
memory=memory,
|
| 1740 |
+
llm=llm,
|
| 1741 |
+
composer=composer,
|
| 1742 |
+
question=question,
|
| 1743 |
+
orchestrator=orchestrator,
|
| 1744 |
+
)
|
| 1745 |
+
|
| 1746 |
+
ctx = run_ask_graph(ctx) if ASK_GRAPH_ENABLED else _legacy_answer_flow(ctx)
|
| 1747 |
+
|
| 1748 |
+
response_body: Dict[str, Any] = ctx.response_body or {
|
| 1749 |
"question": question,
|
| 1750 |
+
"answer": ctx.answer,
|
| 1751 |
+
"answer_type": ctx.answer_type,
|
| 1752 |
"session_id": request.state.session_id,
|
| 1753 |
"factual_question_index": memory.factual_question_count,
|
| 1754 |
"total_turns": memory.total_turns,
|
| 1755 |
+
"sources": ctx.sources or [],
|
| 1756 |
+
"needs_clarification": ctx.needs_clarification,
|
| 1757 |
+
"clarification_question": ctx.clarification_question,
|
| 1758 |
}
|
| 1759 |
|
| 1760 |
if payload.debug:
|
| 1761 |
+
response_body["debug"] = _ask_graph_debug_payload(ctx)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1762 |
|
| 1763 |
return response_body
|
| 1764 |
|
| 1765 |
|
| 1766 |
+
@app.get("/pdf/{filename}")
|
| 1767 |
+
def get_pdf(filename: str):
|
| 1768 |
+
"""Liefert ein Quell-PDF für den eingebetteten Fundstellen-Viewer aus.
|
| 1769 |
+
|
| 1770 |
+
Es werden nur einfache PDF-Dateinamen akzeptiert (kein Pfadanteil), und
|
| 1771 |
+
die Datei muss in einem der konfigurierten PDF-Verzeichnisse liegen.
|
| 1772 |
+
"""
|
| 1773 |
+
name = os.path.basename(filename or "").strip()
|
| 1774 |
+
if not name or not _PDF_NAME_RE.match(name):
|
| 1775 |
+
raise HTTPException(status_code=400, detail="Ungültiger PDF-Dateiname.")
|
| 1776 |
+
|
| 1777 |
+
for directory in PDF_SEARCH_DIRS:
|
| 1778 |
+
candidate = (directory / name).resolve()
|
| 1779 |
+
try:
|
| 1780 |
+
candidate.relative_to(directory.resolve())
|
| 1781 |
+
except (ValueError, OSError):
|
| 1782 |
+
continue
|
| 1783 |
+
if candidate.is_file():
|
| 1784 |
+
return FileResponse(
|
| 1785 |
+
str(candidate),
|
| 1786 |
+
media_type="application/pdf",
|
| 1787 |
+
headers={
|
| 1788 |
+
"Content-Disposition": f'inline; filename="{name}"',
|
| 1789 |
+
"Cache-Control": "public, max-age=3600",
|
| 1790 |
+
},
|
| 1791 |
+
)
|
| 1792 |
+
|
| 1793 |
+
raise HTTPException(status_code=404, detail=f"PDF nicht gefunden: {name}")
|
| 1794 |
+
|
| 1795 |
+
|
| 1796 |
+
@app.get("/debug/workflow")
|
| 1797 |
+
def debug_workflow():
|
| 1798 |
+
return {
|
| 1799 |
+
"ok": True,
|
| 1800 |
+
"ask_graph_enabled": ASK_GRAPH_ENABLED,
|
| 1801 |
+
"max_ask_graph_steps": MAX_ASK_GRAPH_STEPS,
|
| 1802 |
+
"langgraph_available": StateGraph is not None,
|
| 1803 |
+
"graph": ASK_GRAPH,
|
| 1804 |
+
"mermaid": _ask_graph_mermaid(),
|
| 1805 |
+
}
|
| 1806 |
+
|
| 1807 |
+
|
| 1808 |
+
@app.get("/debug/workflow/mermaid")
|
| 1809 |
+
def debug_workflow_mermaid():
|
| 1810 |
+
return Response(_ask_graph_mermaid(), media_type="text/plain")
|
| 1811 |
+
|
| 1812 |
+
|
| 1813 |
@app.get("/health")
|
| 1814 |
def health():
|
| 1815 |
return {
|
|
|
|
| 1823 |
"embedding_model": EMBEDDING_MODEL,
|
| 1824 |
"groq_model": GROQ_MODEL,
|
| 1825 |
"source_sync_to_answer_markers": SOURCE_SYNC_TO_ANSWER_MARKERS,
|
| 1826 |
+
"ask_graph_enabled": ASK_GRAPH_ENABLED,
|
| 1827 |
+
"orchestrator_enabled": ORCHESTRATOR_ENABLED,
|
| 1828 |
}
|
| 1829 |
|
| 1830 |
|
|
|
|
| 1946 |
detail=f"UI-Datei nicht gefunden: {INDEX_FILE}",
|
| 1947 |
)
|
| 1948 |
return FileResponse(INDEX_FILE)
|
|
|
src/ask_workflow.png
ADDED
|
src/llm_client_groq.py
CHANGED
|
@@ -4,11 +4,15 @@ import os
|
|
| 4 |
import time
|
| 5 |
import random
|
| 6 |
import re
|
|
|
|
| 7 |
from typing import Optional, List, Dict, Any
|
| 8 |
|
| 9 |
from groq import Groq
|
| 10 |
|
| 11 |
|
|
|
|
|
|
|
|
|
|
| 12 |
# ---------------------------------------------------------------------------
|
| 13 |
# Frageklassifikation: Single Source of Truth
|
| 14 |
# ---------------------------------------------------------------------------
|
|
@@ -454,31 +458,86 @@ class GroqClient:
|
|
| 454 |
|
| 455 |
@staticmethod
|
| 456 |
def _looks_like_augmented_document_prompt(prompt: str) -> bool:
|
| 457 |
-
"""Erkennt, ob AnswerComposer bereits fachliche Instruktionen enthält.
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
|
| 461 |
-
|
| 462 |
-
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 466 |
)
|
|
|
|
| 467 |
|
| 468 |
@staticmethod
|
| 469 |
def _source_marker_count(rag_context: str) -> int:
|
| 470 |
return len(set(re.findall(r"\[Quelle\s+\d+\]", rag_context or "")))
|
| 471 |
|
| 472 |
def _call_chat_completion(self, messages: List[Dict[str, str]], **kwargs: Any) -> str:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 473 |
last_err: Optional[Exception] = None
|
| 474 |
|
| 475 |
for attempt in range(self.max_retries + 1):
|
| 476 |
try:
|
| 477 |
response = self.client.chat.completions.create(
|
| 478 |
-
model=
|
| 479 |
messages=messages,
|
| 480 |
-
temperature=
|
| 481 |
-
max_tokens=
|
| 482 |
)
|
| 483 |
content = response.choices[0].message.content
|
| 484 |
return content or ""
|
|
|
|
| 4 |
import time
|
| 5 |
import random
|
| 6 |
import re
|
| 7 |
+
import logging
|
| 8 |
from typing import Optional, List, Dict, Any
|
| 9 |
|
| 10 |
from groq import Groq
|
| 11 |
|
| 12 |
|
| 13 |
+
logger = logging.getLogger(__name__)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
# ---------------------------------------------------------------------------
|
| 17 |
# Frageklassifikation: Single Source of Truth
|
| 18 |
# ---------------------------------------------------------------------------
|
|
|
|
| 458 |
|
| 459 |
@staticmethod
|
| 460 |
def _looks_like_augmented_document_prompt(prompt: str) -> bool:
|
| 461 |
+
"""Erkennt, ob AnswerComposer bereits fachliche Instruktionen enthält.
|
| 462 |
+
|
| 463 |
+
Wichtig: Wenn diese Erkennung fehlschlägt, hängt der Client seine
|
| 464 |
+
eigenen LEGAL_TASK_INSTRUCTIONS zusätzlich an und erzeugt zwei
|
| 465 |
+
widersprüchliche Anweisungssets im selben Prompt (z. B. "Ergebnisformel"
|
| 466 |
+
vs. "Keine Ergebnisformel"). Der Composer-Prompt nutzt Block-Header in
|
| 467 |
+
Großbuchstaben, daher wird case-insensitiv geprüft.
|
| 468 |
+
"""
|
| 469 |
+
p = (prompt or "")
|
| 470 |
+
if "Nutzerfrage:" not in p:
|
| 471 |
+
return False
|
| 472 |
+
lowered = p.lower()
|
| 473 |
+
markers = (
|
| 474 |
+
"du beantwortest eine juristische sachfrage",
|
| 475 |
+
"antwortschema",
|
| 476 |
+
"verbindliche regeln:",
|
| 477 |
+
"evidence first",
|
| 478 |
)
|
| 479 |
+
return any(marker in lowered for marker in markers)
|
| 480 |
|
| 481 |
@staticmethod
|
| 482 |
def _source_marker_count(rag_context: str) -> int:
|
| 483 |
return len(set(re.findall(r"\[Quelle\s+\d+\]", rag_context or "")))
|
| 484 |
|
| 485 |
def _call_chat_completion(self, messages: List[Dict[str, str]], **kwargs: Any) -> str:
|
| 486 |
+
"""Call Groq with a locked model.
|
| 487 |
+
|
| 488 |
+
Production safety rule:
|
| 489 |
+
- The billed Groq model is always `self.model`, i.e. the model configured
|
| 490 |
+
when `GroqClient` is instantiated, typically from `GROQ_MODEL` in app.py.
|
| 491 |
+
- Per-call model overrides through kwargs are rejected instead of being
|
| 492 |
+
silently accepted. This prevents accidental billing of other models such
|
| 493 |
+
as `openai/gpt-oss-120b` or `qwen/qwen3-32b`.
|
| 494 |
+
- `service_tier` overrides are also rejected here. The app should not
|
| 495 |
+
switch Groq billing tiers from arbitrary downstream calls.
|
| 496 |
+
"""
|
| 497 |
+
blocked_model = kwargs.pop("model", None)
|
| 498 |
+
if blocked_model is not None and str(blocked_model) != str(self.model):
|
| 499 |
+
raise RuntimeError(
|
| 500 |
+
"Blocked Groq model override: "
|
| 501 |
+
f"requested={blocked_model!r}, configured={self.model!r}. "
|
| 502 |
+
"Only the configured model may be used for billing."
|
| 503 |
+
)
|
| 504 |
+
|
| 505 |
+
blocked_service_tier = kwargs.pop("service_tier", None)
|
| 506 |
+
if blocked_service_tier is not None:
|
| 507 |
+
raise RuntimeError(
|
| 508 |
+
"Blocked Groq service_tier override: "
|
| 509 |
+
f"requested={blocked_service_tier!r}. "
|
| 510 |
+
"Billing-tier selection must not be changed per request."
|
| 511 |
+
)
|
| 512 |
+
|
| 513 |
+
actual_model = self.model
|
| 514 |
+
temperature = kwargs.pop("temperature", self.temperature)
|
| 515 |
+
max_tokens = kwargs.pop("max_tokens", self.max_tokens)
|
| 516 |
+
|
| 517 |
+
if kwargs:
|
| 518 |
+
logger.warning(
|
| 519 |
+
"Ignoring unsupported Groq call kwargs",
|
| 520 |
+
extra={"ignored_kwargs": sorted(kwargs.keys())},
|
| 521 |
+
)
|
| 522 |
+
|
| 523 |
+
logger.info(
|
| 524 |
+
"calling Groq chat completion",
|
| 525 |
+
extra={
|
| 526 |
+
"groq_model": actual_model,
|
| 527 |
+
"temperature": temperature,
|
| 528 |
+
"max_tokens": max_tokens,
|
| 529 |
+
},
|
| 530 |
+
)
|
| 531 |
+
|
| 532 |
last_err: Optional[Exception] = None
|
| 533 |
|
| 534 |
for attempt in range(self.max_retries + 1):
|
| 535 |
try:
|
| 536 |
response = self.client.chat.completions.create(
|
| 537 |
+
model=actual_model,
|
| 538 |
messages=messages,
|
| 539 |
+
temperature=temperature,
|
| 540 |
+
max_tokens=max_tokens,
|
| 541 |
)
|
| 542 |
content = response.choices[0].message.content
|
| 543 |
return content or ""
|
src/orchestrator.py
ADDED
|
@@ -0,0 +1,1220 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import re
|
| 4 |
+
from dataclasses import asdict, dataclass, field
|
| 5 |
+
from typing import Any, Dict, List, Optional, Sequence, Tuple, TYPE_CHECKING
|
| 6 |
+
|
| 7 |
+
from llm_client_groq import classify_question, is_meta_question
|
| 8 |
+
|
| 9 |
+
if TYPE_CHECKING:
|
| 10 |
+
from llm_client_groq import ConversationMemory
|
| 11 |
+
else:
|
| 12 |
+
ConversationMemory = Any # type: ignore[assignment]
|
| 13 |
+
|
| 14 |
+
# ---------------------------------------------------------------------------
|
| 15 |
+
# Regexes and lightweight legal intent detection
|
| 16 |
+
# ---------------------------------------------------------------------------
|
| 17 |
+
|
| 18 |
+
NORM_REF_RE = re.compile(
|
| 19 |
+
r"§{1,2}\s*(?P<section>\d+[a-zA-Z]?)"
|
| 20 |
+
r"(?:\s*(?:-|–|bis)\s*(?P<section_to>\d+[a-zA-Z]?))?"
|
| 21 |
+
r"(?:\s*(?:Abs\.|Absatz)\s*(?P<subsection>\d+[a-zA-Z]?))?"
|
| 22 |
+
r"(?:\s*Satz\s*(?P<sentence>\d+[a-zA-Z]?))?"
|
| 23 |
+
r"(?:\s*(?:Nr\.|Nummer)\s*(?P<number>\d+[a-zA-Z]?))?"
|
| 24 |
+
r"(?:\s*(?:Buchst\.|Buchstabe|lit\.)\s*(?P<letter>[a-zA-Z]))?",
|
| 25 |
+
re.I,
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
SOURCE_MARKER_RE = re.compile(r"\[Quelle\s+(\d+)(?:[^\]]*)\]", re.I)
|
| 29 |
+
SOURCE_MARKER_WITH_OPTIONAL_REF_RE = re.compile(
|
| 30 |
+
r"\[Quelle\s+(\d+)(?:[^\]]*)\]"
|
| 31 |
+
r"(?:\s*\(§[^)]{1,120}\))?",
|
| 32 |
+
re.I,
|
| 33 |
+
)
|
| 34 |
+
ORPHAN_LEGAL_REF_PAREN_RE = re.compile(r"\(§\s*\d{1,3}[a-z]?(?:\s+[^)]{0,80})?\)", re.I)
|
| 35 |
+
NEGATIVE_ANSWER_RE = re.compile(
|
| 36 |
+
r"\b(keine\s+(relevante\s+)?textstelle|nicht\s+(geregelt|enthalten|auffindbar)|"
|
| 37 |
+
r"keine\s+(regelung|informationen|aussage)|nicht\s+belastbar\s+ableitbar|"
|
| 38 |
+
r"kann\s+.*?nicht\s+(festgestellt|beantwortet)\s+werden)\b",
|
| 39 |
+
re.I,
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
# Fine-grained citation fragments generated by the LLM must be verified against
|
| 43 |
+
# actual source metadata. In practice, models often over-specialise references
|
| 44 |
+
# such as "§ 6 Abs. 1 Buchst. a" although the underlying chunk only supports
|
| 45 |
+
# "§ 6 Abs. 1". The orchestrator therefore downgrades unsupported fine refs
|
| 46 |
+
# instead of passing them through as if they were verified citations.
|
| 47 |
+
FINE_BUCHST_REF_RE = re.compile(
|
| 48 |
+
r"§\s*(?P<section>\d{1,3}[a-z]?)\s+"
|
| 49 |
+
r"(?:Abs\.|Absatz)\s*(?P<subsection>\d{1,3}[a-z]?)\s+"
|
| 50 |
+
r"(?:Buchst\.|Buchstabe|lit\.)\s*(?P<letter>[a-z])",
|
| 51 |
+
re.I,
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
CITATION_PAREN_RE = re.compile(
|
| 55 |
+
r"(?P<marker>\[Quelle\s+\d+(?:[^\]]*)\])\s*"
|
| 56 |
+
r"\((?P<ref>§[^)]{1,120})\)",
|
| 57 |
+
re.I,
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
DANGLING_CITATION_GRAMMAR_REPLACEMENTS: Tuple[Tuple[re.Pattern[str], str], ...] = (
|
| 61 |
+
# Remove orphan legal-ref parentheses that remain after invalid source markers
|
| 62 |
+
# were stripped, e.g. "[Quelle 1] (§ 6) und (§ 6)".
|
| 63 |
+
(re.compile(r"(\[Quelle\s+\d+(?:[^\]]*)\](?:\s*\(§[^)]{1,120}\))?)\s+und\s+\(§[^)]{1,120}\)", re.I), r"\1"),
|
| 64 |
+
(re.compile(r"(\[Quelle\s+\d+(?:[^\]]*)\](?:\s*\(§[^)]{1,120}\))?)\s*,\s*\(§[^)]{1,120}\)", re.I), r"\1"),
|
| 65 |
+
(re.compile(r"\s+(?:und|oder)\s+\(§[^)]{1,120}\)", re.I), r""),
|
| 66 |
+
(re.compile(r"\s+und\s+(beschrieben|genannt|geregelt|dargelegt|aufgeführt)\b", re.I), r" \1"),
|
| 67 |
+
(re.compile(r"\bwie\s+in\s+([^.;:\n]{1,160}?)\s+und\s+(beschrieben|genannt|geregelt|dargelegt)\b", re.I), r"wie in \1 \2"),
|
| 68 |
+
(re.compile(r"\bdie\s+in\s+den\s+Quellen\s+([^.;:\n]{1,160}?)\s+und\s+genannt\s+sind", re.I), r"die in \1 genannt sind"),
|
| 69 |
+
(re.compile(r"\bdie\s+in\s+([^.;:\n]{1,160}?)\s+und\s+genannt\s+sind", re.I), r"die in \1 genannt sind"),
|
| 70 |
+
(re.compile(r"\bund\s+weiteren\s+Quellen\s+(dargelegt|beschrieben|genannt)\s+(sind|ist)", re.I), r"\1 \2"),
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
DEFINITION_RE = re.compile(
|
| 74 |
+
r"\b(was\s+(versteht|bedeutet)|wie\s+definiert|definition|legaldefinition|"
|
| 75 |
+
r"begriff|unter\s+[„\"']?[^?]+[”\"']?\s+versteht)\b",
|
| 76 |
+
re.I,
|
| 77 |
+
)
|
| 78 |
+
ENUMERATION_RE = re.compile(
|
| 79 |
+
r"\b(welche|alle|sämtliche|liste|nennt|kriterien|voraussetzungen|tatbestandsmerkmale|"
|
| 80 |
+
r"bestandteile|anforderungen|fälle|maßnahmen|regelt\s+§)\b",
|
| 81 |
+
re.I,
|
| 82 |
+
)
|
| 83 |
+
COMPARISON_RE = re.compile(r"\b(unterschied|vergleiche|vergleich|gegenüber|vs\.?|versus)\b", re.I)
|
| 84 |
+
CLARIFICATION_RISK_RE = re.compile(
|
| 85 |
+
r"\b(das|dies|diese|der\s+fall|so\s+ein\s+fall|dort|hierbei|teilnahme|anspruch|"
|
| 86 |
+
r"abrechnung|pflicht|folge|konsequenz)\b",
|
| 87 |
+
re.I,
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
BUILTIN_QUERY_EXPANSIONS: Dict[str, List[str]] = {
|
| 91 |
+
"nicht verfügbar": ["Nichtverfügbarkeit", "lieferfähig", "nicht lieferbar", "Lieferengpass", "Verfügbarkeit"],
|
| 92 |
+
"beitritt": ["teilnehmen", "Teilnahme", "Mitgliedsverband", "DAV", "Erklärung", "beitreten"],
|
| 93 |
+
"auseinzelung": ["Teilmenge", "Auseinzelung", "einzelne Einheit", "Packung", "Entnahme"],
|
| 94 |
+
"pharmazeutische dienstleistungen": ["pharmazeutische Dienstleistungen", "Anlage 11", "Anspruchsvoraussetzungen", "Vergütung", "Abrechnung"],
|
| 95 |
+
"wunscharzneimittel": ["Wunscharzneimittel", "Kostenerstattung", "anderes Fertigarzneimittel", "§§ 11 bis 14"],
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
# ---------------------------------------------------------------------------
|
| 100 |
+
# Public dataclasses
|
| 101 |
+
# ---------------------------------------------------------------------------
|
| 102 |
+
|
| 103 |
+
@dataclass(slots=True, frozen=True)
|
| 104 |
+
class NormReference:
|
| 105 |
+
section: str
|
| 106 |
+
section_to: str | None = None
|
| 107 |
+
subsection: str | None = None
|
| 108 |
+
sentence: str | None = None
|
| 109 |
+
number: str | None = None
|
| 110 |
+
letter: str | None = None
|
| 111 |
+
|
| 112 |
+
@property
|
| 113 |
+
def section_id(self) -> str:
|
| 114 |
+
return f"§ {self.section}"
|
| 115 |
+
|
| 116 |
+
@property
|
| 117 |
+
def canonical_ref(self) -> str:
|
| 118 |
+
parts = [self.section_id]
|
| 119 |
+
if self.subsection:
|
| 120 |
+
parts.append(f"Abs. {self.subsection}")
|
| 121 |
+
if self.sentence:
|
| 122 |
+
parts.append(f"Satz {self.sentence}")
|
| 123 |
+
if self.number:
|
| 124 |
+
parts.append(f"Nr. {self.number}")
|
| 125 |
+
if self.letter:
|
| 126 |
+
parts.append(f"Buchst. {self.letter.lower()}")
|
| 127 |
+
return " ".join(parts)
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
@dataclass(slots=True)
|
| 131 |
+
class QuestionAnalysis:
|
| 132 |
+
question: str
|
| 133 |
+
question_kind: str
|
| 134 |
+
intent: str
|
| 135 |
+
norm_references: List[NormReference] = field(default_factory=list)
|
| 136 |
+
query_terms: List[str] = field(default_factory=list)
|
| 137 |
+
expanded_terms: List[str] = field(default_factory=list)
|
| 138 |
+
needs_clarification: bool = False
|
| 139 |
+
clarification_question: str | None = None
|
| 140 |
+
reasons: List[str] = field(default_factory=list)
|
| 141 |
+
|
| 142 |
+
def to_dict(self) -> Dict[str, Any]:
|
| 143 |
+
return {
|
| 144 |
+
"question": self.question,
|
| 145 |
+
"question_kind": self.question_kind,
|
| 146 |
+
"intent": self.intent,
|
| 147 |
+
"norm_references": [asdict(ref) | {"canonical_ref": ref.canonical_ref} for ref in self.norm_references],
|
| 148 |
+
"query_terms": list(self.query_terms),
|
| 149 |
+
"expanded_terms": list(self.expanded_terms),
|
| 150 |
+
"needs_clarification": self.needs_clarification,
|
| 151 |
+
"clarification_question": self.clarification_question,
|
| 152 |
+
"reasons": list(self.reasons),
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
@dataclass(slots=True)
|
| 157 |
+
class RetrievalAssessment:
|
| 158 |
+
hit_count: int = 0
|
| 159 |
+
direct_hit_count: int = 0
|
| 160 |
+
exact_norm_hit_count: int = 0
|
| 161 |
+
definition_hit_count: int = 0
|
| 162 |
+
parent_context_count: int = 0
|
| 163 |
+
neighbor_count: int = 0
|
| 164 |
+
containers: List[str] = field(default_factory=list)
|
| 165 |
+
sections: List[str] = field(default_factory=list)
|
| 166 |
+
canonical_refs: List[str] = field(default_factory=list)
|
| 167 |
+
low_confidence: bool = False
|
| 168 |
+
negative_recheck_performed: bool = False
|
| 169 |
+
negative_recheck_added_hits: int = 0
|
| 170 |
+
reasons: List[str] = field(default_factory=list)
|
| 171 |
+
|
| 172 |
+
def to_dict(self) -> Dict[str, Any]:
|
| 173 |
+
return asdict(self)
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
@dataclass(slots=True)
|
| 177 |
+
class AnswerAudit:
|
| 178 |
+
cited_source_numbers: List[int] = field(default_factory=list)
|
| 179 |
+
invalid_source_numbers: List[int] = field(default_factory=list)
|
| 180 |
+
cited_canonical_refs: List[str] = field(default_factory=list)
|
| 181 |
+
answer_basis: str = "unknown" # explicit | derived | negative | insufficient | unknown
|
| 182 |
+
negative_answer_detected: bool = False
|
| 183 |
+
completeness_warnings: List[str] = field(default_factory=list)
|
| 184 |
+
citation_warnings: List[str] = field(default_factory=list)
|
| 185 |
+
auto_fixes: List[str] = field(default_factory=list)
|
| 186 |
+
recommended_action: str = "accept" # accept | recheck | clarify | caution
|
| 187 |
+
confidence: float = 0.5
|
| 188 |
+
|
| 189 |
+
def to_dict(self) -> Dict[str, Any]:
|
| 190 |
+
return asdict(self)
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
@dataclass(slots=True)
|
| 194 |
+
class OrchestratorOptions:
|
| 195 |
+
top_k: int = 8
|
| 196 |
+
fetch_k: int = 24
|
| 197 |
+
max_final_results: int = 10
|
| 198 |
+
max_sources: int = 5
|
| 199 |
+
include_neighbors: bool = True
|
| 200 |
+
include_explicit_sections: bool = True
|
| 201 |
+
# False: das Ranking priorisiert den Hauptvertrag; ein harter Filter machte
|
| 202 |
+
# Anlagen/Anhänge (Anlage 11, Dienstleistungs-Anhänge) unauffindbar.
|
| 203 |
+
restrict_to_default_container: bool = False
|
| 204 |
+
default_container_id: str = "Vertrag"
|
| 205 |
+
min_score: float | None = None
|
| 206 |
+
enable_negative_recheck: bool = True
|
| 207 |
+
enable_clarification: bool = True
|
| 208 |
+
enable_answer_audit: bool = True
|
| 209 |
+
enrich_citations_with_canonical_refs: bool = True
|
| 210 |
+
debug: bool = False
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
@dataclass(slots=True)
|
| 214 |
+
class OrchestratorResult:
|
| 215 |
+
answer: str
|
| 216 |
+
answer_type: str
|
| 217 |
+
hits: List[Dict[str, Any]] = field(default_factory=list)
|
| 218 |
+
raw_sources: List[Dict[str, Any]] = field(default_factory=list)
|
| 219 |
+
sources: List[Dict[str, Any]] = field(default_factory=list)
|
| 220 |
+
analysis: QuestionAnalysis | None = None
|
| 221 |
+
retrieval_assessment: RetrievalAssessment | None = None
|
| 222 |
+
answer_audit: AnswerAudit | None = None
|
| 223 |
+
needs_clarification: bool = False
|
| 224 |
+
clarification_question: str | None = None
|
| 225 |
+
debug: Dict[str, Any] = field(default_factory=dict)
|
| 226 |
+
|
| 227 |
+
def to_debug_dict(self) -> Dict[str, Any]:
|
| 228 |
+
return {
|
| 229 |
+
"analysis": self.analysis.to_dict() if self.analysis else None,
|
| 230 |
+
"retrieval_assessment": self.retrieval_assessment.to_dict() if self.retrieval_assessment else None,
|
| 231 |
+
"answer_audit": self.answer_audit.to_dict() if self.answer_audit else None,
|
| 232 |
+
"needs_clarification": self.needs_clarification,
|
| 233 |
+
"clarification_question": self.clarification_question,
|
| 234 |
+
"debug": dict(self.debug),
|
| 235 |
+
}
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
# ---------------------------------------------------------------------------
|
| 239 |
+
# Helper functions
|
| 240 |
+
# ---------------------------------------------------------------------------
|
| 241 |
+
|
| 242 |
+
def _normalize(text: str) -> str:
|
| 243 |
+
return " ".join((text or "").strip().split())
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def _lower(text: str) -> str:
|
| 247 |
+
return _normalize(text).lower()
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def _hit_meta(hit: Dict[str, Any]) -> Dict[str, Any]:
|
| 251 |
+
meta = hit.get("metadata") or {}
|
| 252 |
+
return meta if isinstance(meta, dict) else {}
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def _hit_value(hit: Dict[str, Any], *keys: str, default: Any = None) -> Any:
|
| 256 |
+
meta = _hit_meta(hit)
|
| 257 |
+
for key in keys:
|
| 258 |
+
value = hit.get(key)
|
| 259 |
+
if value is not None and value != "":
|
| 260 |
+
return value
|
| 261 |
+
for key in keys:
|
| 262 |
+
value = meta.get(key)
|
| 263 |
+
if value is not None and value != "":
|
| 264 |
+
return value
|
| 265 |
+
return default
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def _hit_text(hit: Dict[str, Any]) -> str:
|
| 269 |
+
return str(hit.get("text") or hit.get("document") or "").strip()
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def _hit_kinds(hit: Dict[str, Any]) -> List[str]:
|
| 273 |
+
kinds = hit.get("retrieval_kinds") or []
|
| 274 |
+
if isinstance(kinds, str):
|
| 275 |
+
return [kinds]
|
| 276 |
+
return [str(k) for k in kinds]
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def _hit_score(hit: Dict[str, Any]) -> float:
|
| 280 |
+
try:
|
| 281 |
+
return float(hit.get("rank_score", hit.get("score", 0.0)) or 0.0)
|
| 282 |
+
except (TypeError, ValueError):
|
| 283 |
+
return 0.0
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def _source_key(hit: Dict[str, Any]) -> Tuple[Any, ...]:
|
| 287 |
+
meta = _hit_meta(hit)
|
| 288 |
+
text_hash = meta.get("text_hash") or hit.get("text_hash")
|
| 289 |
+
if text_hash:
|
| 290 |
+
return ("hash", text_hash)
|
| 291 |
+
legal_unit_id = meta.get("legal_unit_id") or hit.get("legal_unit_id")
|
| 292 |
+
if legal_unit_id:
|
| 293 |
+
return ("legal_unit", legal_unit_id)
|
| 294 |
+
return (
|
| 295 |
+
_hit_value(hit, "container", "container_id", default=""),
|
| 296 |
+
_hit_value(hit, "section", "section_id", default=""),
|
| 297 |
+
_hit_value(hit, "chunk_index", "chunk_index_in_section", default=""),
|
| 298 |
+
_hit_text(hit)[:160],
|
| 299 |
+
)
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def _canonical_ref_from_hit(hit: Dict[str, Any]) -> str:
|
| 303 |
+
meta = _hit_meta(hit)
|
| 304 |
+
direct = _hit_value(hit, "canonical_ref", default=None)
|
| 305 |
+
if direct:
|
| 306 |
+
return str(direct)
|
| 307 |
+
|
| 308 |
+
paragraph = meta.get("paragraph") or hit.get("paragraph") or _hit_value(hit, "section", "section_id", default="")
|
| 309 |
+
subsection = meta.get("subsection") or hit.get("subsection")
|
| 310 |
+
sentence = meta.get("sentence") or hit.get("sentence")
|
| 311 |
+
number = meta.get("number") or hit.get("number")
|
| 312 |
+
letter = meta.get("letter") or hit.get("letter")
|
| 313 |
+
|
| 314 |
+
parts = [str(paragraph)] if paragraph else []
|
| 315 |
+
if subsection:
|
| 316 |
+
parts.append(f"Abs. {subsection}")
|
| 317 |
+
if sentence:
|
| 318 |
+
parts.append(f"Satz {sentence}")
|
| 319 |
+
if number:
|
| 320 |
+
parts.append(f"Nr. {number}")
|
| 321 |
+
if letter:
|
| 322 |
+
parts.append(f"Buchst. {str(letter).lower()}")
|
| 323 |
+
return " ".join(parts)
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
def _dedupe_hits(hits: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
| 327 |
+
merged: Dict[Tuple[Any, ...], Dict[str, Any]] = {}
|
| 328 |
+
for hit in hits:
|
| 329 |
+
key = _source_key(hit)
|
| 330 |
+
old = merged.get(key)
|
| 331 |
+
if old is None or _hit_score(hit) > _hit_score(old):
|
| 332 |
+
merged[key] = dict(hit)
|
| 333 |
+
else:
|
| 334 |
+
old_kinds = set(_hit_kinds(old))
|
| 335 |
+
old_kinds.update(_hit_kinds(hit))
|
| 336 |
+
old["retrieval_kinds"] = sorted(old_kinds)
|
| 337 |
+
out = list(merged.values())
|
| 338 |
+
out.sort(key=lambda h: (_hit_score(h), "parent_context" in set(_hit_kinds(h))), reverse=True)
|
| 339 |
+
return out
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
def extract_norm_references(question: str, *, max_range: int = 30) -> List[NormReference]:
|
| 343 |
+
refs: List[NormReference] = []
|
| 344 |
+
for match in NORM_REF_RE.finditer(question or ""):
|
| 345 |
+
start = match.group("section")
|
| 346 |
+
end = match.group("section_to")
|
| 347 |
+
if end and start.isdigit() and end.isdigit():
|
| 348 |
+
start_i, end_i = int(start), int(end)
|
| 349 |
+
if start_i <= end_i and (end_i - start_i) <= max_range:
|
| 350 |
+
for no in range(start_i, end_i + 1):
|
| 351 |
+
refs.append(NormReference(section=str(no)))
|
| 352 |
+
continue
|
| 353 |
+
refs.append(
|
| 354 |
+
NormReference(
|
| 355 |
+
section=start,
|
| 356 |
+
section_to=end,
|
| 357 |
+
subsection=match.group("subsection"),
|
| 358 |
+
sentence=match.group("sentence"),
|
| 359 |
+
number=match.group("number"),
|
| 360 |
+
letter=match.group("letter"),
|
| 361 |
+
)
|
| 362 |
+
)
|
| 363 |
+
seen: set[str] = set()
|
| 364 |
+
unique: List[NormReference] = []
|
| 365 |
+
for ref in refs:
|
| 366 |
+
key = ref.canonical_ref
|
| 367 |
+
if key not in seen:
|
| 368 |
+
seen.add(key)
|
| 369 |
+
unique.append(ref)
|
| 370 |
+
return unique
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
def expand_query_terms(question: str) -> List[str]:
|
| 374 |
+
q = _lower(question)
|
| 375 |
+
terms: List[str] = []
|
| 376 |
+
for key, values in BUILTIN_QUERY_EXPANSIONS.items():
|
| 377 |
+
if key in q or any(v.lower() in q for v in values):
|
| 378 |
+
terms.extend([key, *values])
|
| 379 |
+
# Extract quoted terms as high-value legal-definition candidates.
|
| 380 |
+
for quoted in re.findall(r"[„\"']([^„”\"']{2,80})[”\"']", question or ""):
|
| 381 |
+
terms.append(quoted.strip())
|
| 382 |
+
return list(dict.fromkeys(t for t in terms if t))
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
# ---------------------------------------------------------------------------
|
| 386 |
+
# Orchestrator
|
| 387 |
+
# ---------------------------------------------------------------------------
|
| 388 |
+
|
| 389 |
+
class LegalAnswerOrchestrator:
|
| 390 |
+
"""
|
| 391 |
+
Fachliche Ablaufsteuerung für die juristische RAG-Antwort.
|
| 392 |
+
|
| 393 |
+
Diese Klasse ist bewusst unabhängig von FastAPI. `app.py` bleibt die
|
| 394 |
+
Aussteuerungs- und API-Schicht und kann diesen Orchestrator als einen
|
| 395 |
+
Knoten im Ask-Graph verwenden.
|
| 396 |
+
|
| 397 |
+
Erwartete externe Komponenten:
|
| 398 |
+
- retriever: besitzt idealerweise `query(...)` und optional `verify_negative_result(...)`
|
| 399 |
+
- composer: besitzt `compose_with_sources(...)` oder `compose(...)`
|
| 400 |
+
"""
|
| 401 |
+
|
| 402 |
+
def __init__(self, retriever: Any, composer: Any, *, options: OrchestratorOptions | None = None):
|
| 403 |
+
self.retriever = retriever
|
| 404 |
+
self.composer = composer
|
| 405 |
+
self.options = options or OrchestratorOptions()
|
| 406 |
+
|
| 407 |
+
# ------------------------------------------------------------------
|
| 408 |
+
# Analysis and planning
|
| 409 |
+
# ------------------------------------------------------------------
|
| 410 |
+
|
| 411 |
+
def analyze_question(self, question: str) -> QuestionAnalysis:
|
| 412 |
+
q = _normalize(question)
|
| 413 |
+
lower = q.lower()
|
| 414 |
+
refs = extract_norm_references(q)
|
| 415 |
+
expanded = expand_query_terms(q)
|
| 416 |
+
reasons: List[str] = []
|
| 417 |
+
|
| 418 |
+
if is_meta_question(q):
|
| 419 |
+
intent = "meta"
|
| 420 |
+
reasons.append("meta_question")
|
| 421 |
+
elif DEFINITION_RE.search(q):
|
| 422 |
+
intent = "definition"
|
| 423 |
+
reasons.append("definition_pattern")
|
| 424 |
+
elif refs and ENUMERATION_RE.search(q):
|
| 425 |
+
intent = "norm_enumeration"
|
| 426 |
+
reasons.append("explicit_norm_and_enumeration_pattern")
|
| 427 |
+
elif refs:
|
| 428 |
+
intent = "explicit_norm"
|
| 429 |
+
reasons.append("explicit_norm_reference")
|
| 430 |
+
elif COMPARISON_RE.search(q):
|
| 431 |
+
intent = "comparison"
|
| 432 |
+
reasons.append("comparison_pattern")
|
| 433 |
+
elif ENUMERATION_RE.search(q):
|
| 434 |
+
intent = "enumeration"
|
| 435 |
+
reasons.append("enumeration_pattern")
|
| 436 |
+
else:
|
| 437 |
+
intent = "legal" if classify_question(q) == "legal" else classify_question(q)
|
| 438 |
+
reasons.append("classified_by_llm_client_rules")
|
| 439 |
+
|
| 440 |
+
# Conservative clarification detection: only ask when no explicit norm is
|
| 441 |
+
# present and the wording is likely underspecified.
|
| 442 |
+
needs_clarification = False
|
| 443 |
+
clarification_question: str | None = None
|
| 444 |
+
if self.options.enable_clarification and not refs and CLARIFICATION_RISK_RE.search(lower):
|
| 445 |
+
if len(expanded) == 0 and len(q.split()) <= 12:
|
| 446 |
+
needs_clarification = True
|
| 447 |
+
clarification_question = (
|
| 448 |
+
"Meinst du eine konkrete Regelung im Rahmenvertrag, eine bestimmte Anlage "
|
| 449 |
+
"oder die rechtliche Folge für einen bestimmten Sachverhalt?"
|
| 450 |
+
)
|
| 451 |
+
reasons.append("underspecified_without_norm_reference")
|
| 452 |
+
|
| 453 |
+
return QuestionAnalysis(
|
| 454 |
+
question=q,
|
| 455 |
+
question_kind=classify_question(q),
|
| 456 |
+
intent=intent,
|
| 457 |
+
norm_references=refs,
|
| 458 |
+
query_terms=[r.canonical_ref for r in refs],
|
| 459 |
+
expanded_terms=expanded,
|
| 460 |
+
needs_clarification=needs_clarification,
|
| 461 |
+
clarification_question=clarification_question,
|
| 462 |
+
reasons=reasons,
|
| 463 |
+
)
|
| 464 |
+
|
| 465 |
+
def _retriever_query(self, question: str, analysis: QuestionAnalysis, options: OrchestratorOptions) -> List[Dict[str, Any]]:
|
| 466 |
+
where = {"container_id": options.default_container_id} if options.restrict_to_default_container else None
|
| 467 |
+
explicit_sections = [ref.section_id for ref in analysis.norm_references] or None
|
| 468 |
+
|
| 469 |
+
kwargs: Dict[str, Any] = {
|
| 470 |
+
"question": question,
|
| 471 |
+
"top_k": options.top_k,
|
| 472 |
+
"where": where,
|
| 473 |
+
"fetch_k": options.fetch_k,
|
| 474 |
+
"include_explicit_sections": options.include_explicit_sections,
|
| 475 |
+
"explicit_sections": explicit_sections,
|
| 476 |
+
"include_neighbors": options.include_neighbors,
|
| 477 |
+
"max_final_results": options.max_final_results,
|
| 478 |
+
"restrict_to_default_container": options.restrict_to_default_container,
|
| 479 |
+
}
|
| 480 |
+
if options.min_score is not None:
|
| 481 |
+
kwargs["min_score"] = options.min_score
|
| 482 |
+
|
| 483 |
+
# Newer standalone retriever may accept these flags; older one will not.
|
| 484 |
+
if analysis.intent == "definition":
|
| 485 |
+
kwargs["enable_definition_lookup"] = True
|
| 486 |
+
kwargs["verify_negative_answer"] = False
|
| 487 |
+
|
| 488 |
+
try:
|
| 489 |
+
return list(self.retriever.query(**kwargs) or [])
|
| 490 |
+
except TypeError:
|
| 491 |
+
# Backward-compatible fallback for older retrievers.
|
| 492 |
+
kwargs.pop("enable_definition_lookup", None)
|
| 493 |
+
kwargs.pop("verify_negative_answer", None)
|
| 494 |
+
try:
|
| 495 |
+
return list(self.retriever.query(**kwargs) or [])
|
| 496 |
+
except TypeError:
|
| 497 |
+
try:
|
| 498 |
+
return list(self.retriever.query(question=question, top_k=options.top_k, where=where) or [])
|
| 499 |
+
except TypeError:
|
| 500 |
+
return list(self.retriever.query(question, top_k=options.top_k) or [])
|
| 501 |
+
|
| 502 |
+
def assess_retrieval(self, hits: Sequence[Dict[str, Any]], analysis: QuestionAnalysis) -> RetrievalAssessment:
|
| 503 |
+
kinds = [kind for hit in hits for kind in _hit_kinds(hit)]
|
| 504 |
+
containers = list(dict.fromkeys(str(_hit_value(h, "container", "container_id", default="")) for h in hits if _hit_value(h, "container", "container_id", default="")))
|
| 505 |
+
sections = list(dict.fromkeys(str(_hit_value(h, "section", "section_id", default="")) for h in hits if _hit_value(h, "section", "section_id", default="")))
|
| 506 |
+
canonical_refs = list(dict.fromkeys(ref for ref in (_canonical_ref_from_hit(h) for h in hits) if ref))
|
| 507 |
+
|
| 508 |
+
direct_hit_count = sum(1 for k in kinds if k not in {"neighbor", "parent_context"})
|
| 509 |
+
exact_count = sum(1 for k in kinds if k in {"exact_norm", "explicit_section", "section_lookup"})
|
| 510 |
+
definition_count = sum(1 for h in hits if bool(_hit_value(h, "is_definition", default=False)) or "definition" in set(_hit_kinds(h)))
|
| 511 |
+
parent_count = sum(1 for k in kinds if k == "parent_context")
|
| 512 |
+
neighbor_count = sum(1 for k in kinds if k == "neighbor")
|
| 513 |
+
|
| 514 |
+
low_confidence = False
|
| 515 |
+
reasons: List[str] = []
|
| 516 |
+
if not hits:
|
| 517 |
+
low_confidence = True
|
| 518 |
+
reasons.append("no_hits")
|
| 519 |
+
if analysis.norm_references and exact_count == 0:
|
| 520 |
+
low_confidence = True
|
| 521 |
+
reasons.append("explicit_norm_without_exact_hit")
|
| 522 |
+
if analysis.intent == "definition" and definition_count == 0:
|
| 523 |
+
low_confidence = True
|
| 524 |
+
reasons.append("definition_question_without_definition_hit")
|
| 525 |
+
if direct_hit_count == 0 and hits:
|
| 526 |
+
low_confidence = True
|
| 527 |
+
reasons.append("only_context_hits")
|
| 528 |
+
|
| 529 |
+
return RetrievalAssessment(
|
| 530 |
+
hit_count=len(hits),
|
| 531 |
+
direct_hit_count=direct_hit_count,
|
| 532 |
+
exact_norm_hit_count=exact_count,
|
| 533 |
+
definition_hit_count=definition_count,
|
| 534 |
+
parent_context_count=parent_count,
|
| 535 |
+
neighbor_count=neighbor_count,
|
| 536 |
+
containers=containers,
|
| 537 |
+
sections=sections,
|
| 538 |
+
canonical_refs=canonical_refs[:20],
|
| 539 |
+
low_confidence=low_confidence,
|
| 540 |
+
reasons=reasons,
|
| 541 |
+
)
|
| 542 |
+
|
| 543 |
+
def _negative_recheck(
|
| 544 |
+
self,
|
| 545 |
+
question: str,
|
| 546 |
+
analysis: QuestionAnalysis,
|
| 547 |
+
hits: List[Dict[str, Any]],
|
| 548 |
+
assessment: RetrievalAssessment,
|
| 549 |
+
options: OrchestratorOptions,
|
| 550 |
+
) -> Tuple[List[Dict[str, Any]], RetrievalAssessment]:
|
| 551 |
+
if not options.enable_negative_recheck or not assessment.low_confidence:
|
| 552 |
+
return hits, assessment
|
| 553 |
+
|
| 554 |
+
before = len(hits)
|
| 555 |
+
recheck_hits: List[Dict[str, Any]] = []
|
| 556 |
+
|
| 557 |
+
if hasattr(self.retriever, "verify_negative_result"):
|
| 558 |
+
try:
|
| 559 |
+
check = self.retriever.verify_negative_result(question)
|
| 560 |
+
if isinstance(check, dict):
|
| 561 |
+
recheck_hits = list(check.get("results") or check.get("hits") or [])
|
| 562 |
+
elif isinstance(check, list):
|
| 563 |
+
recheck_hits = list(check)
|
| 564 |
+
except TypeError:
|
| 565 |
+
try:
|
| 566 |
+
check = self.retriever.verify_negative_result(question=question)
|
| 567 |
+
if isinstance(check, dict):
|
| 568 |
+
recheck_hits = list(check.get("results") or check.get("hits") or [])
|
| 569 |
+
except Exception:
|
| 570 |
+
recheck_hits = []
|
| 571 |
+
except Exception:
|
| 572 |
+
recheck_hits = []
|
| 573 |
+
|
| 574 |
+
if not recheck_hits:
|
| 575 |
+
# Manual broad fallback: all containers, no restrictive score if possible.
|
| 576 |
+
broad_options = OrchestratorOptions(**asdict(options))
|
| 577 |
+
broad_options.restrict_to_default_container = False
|
| 578 |
+
broad_options.min_score = None
|
| 579 |
+
broad_options.include_neighbors = True
|
| 580 |
+
broad_options.include_explicit_sections = True
|
| 581 |
+
broad_options.fetch_k = max(options.fetch_k, options.top_k * 6)
|
| 582 |
+
broad_options.max_final_results = max(options.max_final_results, 16)
|
| 583 |
+
recheck_hits = self._retriever_query(question, analysis, broad_options)
|
| 584 |
+
|
| 585 |
+
combined = _dedupe_hits([*hits, *recheck_hits])
|
| 586 |
+
assessment = self.assess_retrieval(combined, analysis)
|
| 587 |
+
assessment.negative_recheck_performed = True
|
| 588 |
+
assessment.negative_recheck_added_hits = max(0, len(combined) - before)
|
| 589 |
+
if assessment.negative_recheck_added_hits:
|
| 590 |
+
assessment.reasons.append("negative_recheck_added_hits")
|
| 591 |
+
return combined, assessment
|
| 592 |
+
|
| 593 |
+
# ------------------------------------------------------------------
|
| 594 |
+
# Composition and audit
|
| 595 |
+
# ------------------------------------------------------------------
|
| 596 |
+
|
| 597 |
+
def _compose(self, question: str, hits: List[Dict[str, Any]], memory: Optional[ConversationMemory]) -> Tuple[str, str, List[Dict[str, Any]]]:
|
| 598 |
+
if hasattr(self.composer, "compose_with_sources"):
|
| 599 |
+
try:
|
| 600 |
+
return self.composer.compose_with_sources(question, hits, memory=memory)
|
| 601 |
+
except TypeError:
|
| 602 |
+
pass
|
| 603 |
+
answer, answer_type = self.composer.compose(question, hits, memory=memory)
|
| 604 |
+
sources: List[Dict[str, Any]] = []
|
| 605 |
+
if hasattr(self.composer, "build_sources"):
|
| 606 |
+
try:
|
| 607 |
+
sources = self.composer.build_sources(hits)
|
| 608 |
+
except Exception:
|
| 609 |
+
sources = []
|
| 610 |
+
return answer, answer_type, sources
|
| 611 |
+
|
| 612 |
+
@staticmethod
|
| 613 |
+
def _source_numbers_from_source(source: Dict[str, Any], *, fallback: int | None = None) -> List[int]:
|
| 614 |
+
"""Return all source numbers represented by a Composer/API source item."""
|
| 615 |
+
numbers: List[int] = []
|
| 616 |
+
|
| 617 |
+
raw_numbers = source.get("source_numbers")
|
| 618 |
+
if raw_numbers is not None:
|
| 619 |
+
if not isinstance(raw_numbers, (list, tuple, set)):
|
| 620 |
+
raw_numbers = [raw_numbers]
|
| 621 |
+
for value in raw_numbers:
|
| 622 |
+
try:
|
| 623 |
+
number = int(value)
|
| 624 |
+
except (TypeError, ValueError):
|
| 625 |
+
continue
|
| 626 |
+
if number not in numbers:
|
| 627 |
+
numbers.append(number)
|
| 628 |
+
|
| 629 |
+
raw_single = source.get("source_number")
|
| 630 |
+
if raw_single is not None:
|
| 631 |
+
try:
|
| 632 |
+
number = int(raw_single)
|
| 633 |
+
if number not in numbers:
|
| 634 |
+
numbers.append(number)
|
| 635 |
+
except (TypeError, ValueError):
|
| 636 |
+
pass
|
| 637 |
+
|
| 638 |
+
if not numbers and fallback is not None:
|
| 639 |
+
numbers.append(fallback)
|
| 640 |
+
|
| 641 |
+
return sorted(numbers)
|
| 642 |
+
|
| 643 |
+
@staticmethod
|
| 644 |
+
def _canonical_refs_from_source(source: Dict[str, Any]) -> List[str]:
|
| 645 |
+
refs: List[str] = []
|
| 646 |
+
|
| 647 |
+
raw_refs = source.get("canonical_refs")
|
| 648 |
+
if raw_refs is not None:
|
| 649 |
+
if not isinstance(raw_refs, (list, tuple, set)):
|
| 650 |
+
raw_refs = [raw_refs]
|
| 651 |
+
for value in raw_refs:
|
| 652 |
+
ref = _normalize(str(value or ""))
|
| 653 |
+
if ref and ref not in refs:
|
| 654 |
+
refs.append(ref)
|
| 655 |
+
|
| 656 |
+
direct = _normalize(str(source.get("canonical_ref") or ""))
|
| 657 |
+
if direct and direct not in refs:
|
| 658 |
+
refs.append(direct)
|
| 659 |
+
|
| 660 |
+
fallback = _normalize(_canonical_ref_from_hit(source))
|
| 661 |
+
if fallback and fallback not in refs:
|
| 662 |
+
refs.append(fallback)
|
| 663 |
+
|
| 664 |
+
return refs
|
| 665 |
+
|
| 666 |
+
@classmethod
|
| 667 |
+
def _source_number_to_ref(cls, sources: Sequence[Dict[str, Any]], hits: Sequence[Dict[str, Any]]) -> Dict[int, str]:
|
| 668 |
+
"""Map every visible [Quelle n] number to the best known canonical ref.
|
| 669 |
+
|
| 670 |
+
Composer sources may group multiple source numbers into one displayed
|
| 671 |
+
source item. The previous implementation only considered
|
| 672 |
+
``source_number`` and silently lost grouped numbers such as
|
| 673 |
+
``source_numbers=[1, 3]``.
|
| 674 |
+
"""
|
| 675 |
+
mapping: Dict[int, str] = {}
|
| 676 |
+
|
| 677 |
+
for idx, source in enumerate(sources, start=1):
|
| 678 |
+
numbers = cls._source_numbers_from_source(source, fallback=idx)
|
| 679 |
+
refs = cls._canonical_refs_from_source(source)
|
| 680 |
+
if not refs:
|
| 681 |
+
continue
|
| 682 |
+
|
| 683 |
+
# Prefer a concise, stable ref for inline enrichment. Fine-grained
|
| 684 |
+
# refs remain available in the separate source list; inline enrichment
|
| 685 |
+
# must not invent suspicious Buchst.-level citations.
|
| 686 |
+
ref = refs[0]
|
| 687 |
+
for candidate in refs:
|
| 688 |
+
if "Buchst." not in candidate and "Buchstabe" not in candidate:
|
| 689 |
+
ref = candidate
|
| 690 |
+
break
|
| 691 |
+
|
| 692 |
+
for number in numbers:
|
| 693 |
+
mapping.setdefault(number, ref)
|
| 694 |
+
|
| 695 |
+
# Fallback: use retrieval hit order only when Composer sources did not
|
| 696 |
+
# provide any number mapping. This keeps older stacks usable while not
|
| 697 |
+
# overriding Composer numbering.
|
| 698 |
+
if not mapping:
|
| 699 |
+
for idx, hit in enumerate(hits, start=1):
|
| 700 |
+
ref = _canonical_ref_from_hit(hit)
|
| 701 |
+
if ref:
|
| 702 |
+
mapping[idx] = ref
|
| 703 |
+
|
| 704 |
+
return mapping
|
| 705 |
+
|
| 706 |
+
@staticmethod
|
| 707 |
+
def _answer_has_inline_ref_after_marker(answer: str, marker_end: int) -> bool:
|
| 708 |
+
return bool(re.match(r"\s*\(", (answer or "")[marker_end : marker_end + 8]))
|
| 709 |
+
|
| 710 |
+
@classmethod
|
| 711 |
+
def _enrich_answer_citations(cls, answer: str, source_to_ref: Dict[int, str]) -> str:
|
| 712 |
+
"""Conservatively add canonical refs to bare source markers.
|
| 713 |
+
|
| 714 |
+
Inline enrichment is intentionally restrained. Generic section-only
|
| 715 |
+
references such as ``§ 6`` add little value and caused noisy outputs
|
| 716 |
+
like ``[Quelle 1] (§ 6) und (§ 6)``. More specific refs may still be
|
| 717 |
+
added when the marker is bare and the model did not already provide a
|
| 718 |
+
parenthetical.
|
| 719 |
+
"""
|
| 720 |
+
if not answer or not source_to_ref:
|
| 721 |
+
return answer
|
| 722 |
+
|
| 723 |
+
def repl(match: re.Match[str]) -> str:
|
| 724 |
+
source_no = int(match.group(1))
|
| 725 |
+
ref = source_to_ref.get(source_no)
|
| 726 |
+
marker = match.group(0)
|
| 727 |
+
if not ref:
|
| 728 |
+
return marker
|
| 729 |
+
if cls._answer_has_inline_ref_after_marker(answer, match.end()):
|
| 730 |
+
return marker
|
| 731 |
+
if not re.search(r"\b(?:Abs\.|Satz|Nr\.|Buchst\.)\b", ref, flags=re.I):
|
| 732 |
+
return marker
|
| 733 |
+
return f"{marker} ({ref})"
|
| 734 |
+
|
| 735 |
+
return SOURCE_MARKER_RE.sub(repl, answer)
|
| 736 |
+
|
| 737 |
+
@staticmethod
|
| 738 |
+
def _extract_source_numbers(answer: str) -> List[int]:
|
| 739 |
+
out: List[int] = []
|
| 740 |
+
for match in SOURCE_MARKER_RE.finditer(answer or ""):
|
| 741 |
+
try:
|
| 742 |
+
out.append(int(match.group(1)))
|
| 743 |
+
except ValueError:
|
| 744 |
+
continue
|
| 745 |
+
return list(dict.fromkeys(out))
|
| 746 |
+
|
| 747 |
+
@staticmethod
|
| 748 |
+
def _normalize_legal_ref(ref: str) -> str:
|
| 749 |
+
s = _normalize(ref)
|
| 750 |
+
s = re.sub(r"§\s*", "§ ", s)
|
| 751 |
+
s = re.sub(r"\bAbsatz\b", "Abs.", s, flags=re.I)
|
| 752 |
+
s = re.sub(r"\bBuchstabe\b|\blit\.", "Buchst.", s, flags=re.I)
|
| 753 |
+
s = re.sub(r"\s+", " ", s).strip(" .,;:")
|
| 754 |
+
return s.lower()
|
| 755 |
+
|
| 756 |
+
@classmethod
|
| 757 |
+
def _supported_ref_set(cls, sources: Sequence[Dict[str, Any]], hits: Sequence[Dict[str, Any]]) -> set[str]:
|
| 758 |
+
refs: set[str] = set()
|
| 759 |
+
for source in sources:
|
| 760 |
+
for ref in cls._canonical_refs_from_source(source):
|
| 761 |
+
refs.add(cls._normalize_legal_ref(ref))
|
| 762 |
+
for hit in hits:
|
| 763 |
+
ref = _canonical_ref_from_hit(hit)
|
| 764 |
+
if ref:
|
| 765 |
+
refs.add(cls._normalize_legal_ref(ref))
|
| 766 |
+
return refs
|
| 767 |
+
|
| 768 |
+
@classmethod
|
| 769 |
+
def _downgrade_unsupported_fine_refs(
|
| 770 |
+
cls,
|
| 771 |
+
answer: str,
|
| 772 |
+
sources: Sequence[Dict[str, Any]],
|
| 773 |
+
hits: Sequence[Dict[str, Any]],
|
| 774 |
+
) -> Tuple[str, List[str]]:
|
| 775 |
+
"""Downgrade unsupported Buchst.-level references to Absatz level."""
|
| 776 |
+
fixes: List[str] = []
|
| 777 |
+
if not answer:
|
| 778 |
+
return answer, fixes
|
| 779 |
+
|
| 780 |
+
supported = cls._supported_ref_set(sources, hits)
|
| 781 |
+
|
| 782 |
+
def repl(match: re.Match[str]) -> str:
|
| 783 |
+
full = match.group(0)
|
| 784 |
+
norm_full = cls._normalize_legal_ref(full)
|
| 785 |
+
if norm_full in supported:
|
| 786 |
+
return full
|
| 787 |
+
downgraded = f"§ {match.group('section')} Abs. {match.group('subsection')}"
|
| 788 |
+
fixes.append(f"downgraded_unsupported_fine_ref:{full}->{downgraded}")
|
| 789 |
+
return downgraded
|
| 790 |
+
|
| 791 |
+
cleaned = FINE_BUCHST_REF_RE.sub(repl, answer)
|
| 792 |
+
|
| 793 |
+
# Collapse duplicates introduced by downgrading, e.g.
|
| 794 |
+
# "[Quelle 1] (§ 6 Abs. 1), § 6 Abs. 1".
|
| 795 |
+
cleaned = re.sub(
|
| 796 |
+
r"(\[Quelle\s+\d+(?:[^\]]*)\]\s*\((§\s*\d{1,3}[a-z]?\s+Abs\.\s*\d{1,3}[a-z]?)\))\s*,\s*\2",
|
| 797 |
+
r"\1",
|
| 798 |
+
cleaned,
|
| 799 |
+
flags=re.I,
|
| 800 |
+
)
|
| 801 |
+
cleaned = re.sub(
|
| 802 |
+
r"(\((§\s*\d{1,3}[a-z]?\s+Abs\.\s*\d{1,3}[a-z]?)\))\s*,\s*\2",
|
| 803 |
+
r"\1",
|
| 804 |
+
cleaned,
|
| 805 |
+
flags=re.I,
|
| 806 |
+
)
|
| 807 |
+
return cleaned, fixes
|
| 808 |
+
|
| 809 |
+
@classmethod
|
| 810 |
+
def _valid_source_numbers(cls, sources: Sequence[Dict[str, Any]], hits: Sequence[Dict[str, Any]]) -> set[int]:
|
| 811 |
+
valid: set[int] = set()
|
| 812 |
+
for idx, source in enumerate(sources, start=1):
|
| 813 |
+
valid.update(cls._source_numbers_from_source(source, fallback=idx))
|
| 814 |
+
if not valid:
|
| 815 |
+
valid.update(range(1, len(hits) + 1))
|
| 816 |
+
return valid
|
| 817 |
+
|
| 818 |
+
@classmethod
|
| 819 |
+
def _strip_invalid_source_markers_safely(
|
| 820 |
+
cls,
|
| 821 |
+
answer: str,
|
| 822 |
+
sources: Sequence[Dict[str, Any]],
|
| 823 |
+
hits: Sequence[Dict[str, Any]],
|
| 824 |
+
) -> Tuple[str, List[str]]:
|
| 825 |
+
"""Remove invalid source markers including attached parenthetical refs.
|
| 826 |
+
|
| 827 |
+
Previous versions removed only ``[Quelle n]`` and left the enrichment
|
| 828 |
+
tail behind. That produced broken fragments like ``und (§ 6)``. This
|
| 829 |
+
method removes the whole citation atom for invalid markers and then runs
|
| 830 |
+
a grammar cleanup pass.
|
| 831 |
+
"""
|
| 832 |
+
fixes: List[str] = []
|
| 833 |
+
if not answer:
|
| 834 |
+
return answer, fixes
|
| 835 |
+
|
| 836 |
+
valid = cls._valid_source_numbers(sources, hits)
|
| 837 |
+
|
| 838 |
+
def repl(match: re.Match[str]) -> str:
|
| 839 |
+
try:
|
| 840 |
+
number = int(match.group(1))
|
| 841 |
+
except (TypeError, ValueError):
|
| 842 |
+
fixes.append("removed_malformed_source_marker")
|
| 843 |
+
return ""
|
| 844 |
+
if number in valid:
|
| 845 |
+
return match.group(0)
|
| 846 |
+
fixes.append(f"removed_invalid_source_marker_with_ref:{number}")
|
| 847 |
+
return ""
|
| 848 |
+
|
| 849 |
+
cleaned = SOURCE_MARKER_WITH_OPTIONAL_REF_RE.sub(repl, answer)
|
| 850 |
+
cleaned = cls._cleanup_citation_grammar(cleaned)
|
| 851 |
+
return cleaned.strip(), fixes
|
| 852 |
+
|
| 853 |
+
@staticmethod
|
| 854 |
+
def _cleanup_citation_grammar(answer: str) -> str:
|
| 855 |
+
"""Repair grammar after citation cleanup without changing legal content."""
|
| 856 |
+
cleaned = answer or ""
|
| 857 |
+
|
| 858 |
+
# Collapse model/enrichment duplicates such as:
|
| 859 |
+
# [Quelle 1] (§ 6), § 6 Abs. 1 -> [Quelle 1] (§ 6 Abs. 1)
|
| 860 |
+
cleaned = re.sub(
|
| 861 |
+
r"(\[Quelle\s+\d+(?:[^\]]*)\])\s*\(§\s*(\d{1,3}[a-z]?)\)\s*,\s*(§\s*\2\s+Abs\.\s*\d{1,3}[a-z]?)",
|
| 862 |
+
r"\1 (\3)",
|
| 863 |
+
cleaned,
|
| 864 |
+
flags=re.I,
|
| 865 |
+
)
|
| 866 |
+
|
| 867 |
+
for pattern, replacement in DANGLING_CITATION_GRAMMAR_REPLACEMENTS:
|
| 868 |
+
cleaned = pattern.sub(replacement, cleaned)
|
| 869 |
+
|
| 870 |
+
# Remove duplicate adjacent identical parenthetical legal refs.
|
| 871 |
+
cleaned = re.sub(
|
| 872 |
+
r"(\(§[^)]{1,120}\))\s*(?:,|und)\s*\1",
|
| 873 |
+
r"\1",
|
| 874 |
+
cleaned,
|
| 875 |
+
flags=re.I,
|
| 876 |
+
)
|
| 877 |
+
|
| 878 |
+
# Clean common connective leftovers.
|
| 879 |
+
cleaned = re.sub(r"\s+und\s+([,.;:])", r"\1", cleaned, flags=re.I)
|
| 880 |
+
cleaned = re.sub(r"\s+und\s*(?=\.)", "", cleaned, flags=re.I)
|
| 881 |
+
cleaned = re.sub(r"\(\s*\)", "", cleaned)
|
| 882 |
+
cleaned = re.sub(r"\s+([,.;:])", r"\1", cleaned)
|
| 883 |
+
cleaned = re.sub(r"[ \t]{2,}", " ", cleaned)
|
| 884 |
+
cleaned = re.sub(r"\n[ \t]+", "\n", cleaned)
|
| 885 |
+
return cleaned.strip()
|
| 886 |
+
|
| 887 |
+
@classmethod
|
| 888 |
+
def _ensure_cited_sources_visible(
|
| 889 |
+
cls,
|
| 890 |
+
answer: str,
|
| 891 |
+
sources: List[Dict[str, Any]],
|
| 892 |
+
hits: List[Dict[str, Any]],
|
| 893 |
+
) -> List[Dict[str, Any]]:
|
| 894 |
+
"""Ensure every cited [Quelle n] is represented in returned sources."""
|
| 895 |
+
cited = cls._extract_source_numbers(answer)
|
| 896 |
+
if not cited:
|
| 897 |
+
return sources
|
| 898 |
+
|
| 899 |
+
visible = cls._valid_source_numbers(sources, [])
|
| 900 |
+
missing = [n for n in cited if n not in visible]
|
| 901 |
+
if not missing:
|
| 902 |
+
return sources
|
| 903 |
+
|
| 904 |
+
out = list(sources)
|
| 905 |
+
for number in missing:
|
| 906 |
+
idx = number - 1
|
| 907 |
+
if 0 <= idx < len(hits):
|
| 908 |
+
source = dict(hits[idx])
|
| 909 |
+
source["source_number"] = number
|
| 910 |
+
source["source_numbers"] = [number]
|
| 911 |
+
if "canonical_ref" not in source or not source.get("canonical_ref"):
|
| 912 |
+
source["canonical_ref"] = _canonical_ref_from_hit(source)
|
| 913 |
+
out.append(source)
|
| 914 |
+
return out
|
| 915 |
+
|
| 916 |
+
@classmethod
|
| 917 |
+
def _filter_sources_to_cited_sources(
|
| 918 |
+
cls,
|
| 919 |
+
answer: str,
|
| 920 |
+
sources: List[Dict[str, Any]],
|
| 921 |
+
hits: List[Dict[str, Any]],
|
| 922 |
+
*,
|
| 923 |
+
max_sources: int,
|
| 924 |
+
) -> List[Dict[str, Any]]:
|
| 925 |
+
"""Return only sources that are actually cited, with hit fallback.
|
| 926 |
+
|
| 927 |
+
For answer quality, visible sources should not include unrelated
|
| 928 |
+
retrieval leftovers when the answer cites only a small subset. If the
|
| 929 |
+
model cites no source markers, keep the curated source list.
|
| 930 |
+
"""
|
| 931 |
+
cited = cls._extract_source_numbers(answer)
|
| 932 |
+
if not cited:
|
| 933 |
+
return list(sources)[:max_sources]
|
| 934 |
+
|
| 935 |
+
by_number: Dict[int, Dict[str, Any]] = {}
|
| 936 |
+
for idx, source in enumerate(sources, start=1):
|
| 937 |
+
for number in cls._source_numbers_from_source(source, fallback=idx):
|
| 938 |
+
by_number.setdefault(number, source)
|
| 939 |
+
|
| 940 |
+
out: List[Dict[str, Any]] = []
|
| 941 |
+
seen_ids: set[int] = set()
|
| 942 |
+
for number in cited:
|
| 943 |
+
source = by_number.get(number)
|
| 944 |
+
if source is None:
|
| 945 |
+
hit_idx = number - 1
|
| 946 |
+
if 0 <= hit_idx < len(hits):
|
| 947 |
+
source = dict(hits[hit_idx])
|
| 948 |
+
source["source_number"] = number
|
| 949 |
+
source["source_numbers"] = [number]
|
| 950 |
+
source.setdefault("canonical_ref", _canonical_ref_from_hit(source))
|
| 951 |
+
if source is None:
|
| 952 |
+
continue
|
| 953 |
+
ident = id(source)
|
| 954 |
+
if ident in seen_ids:
|
| 955 |
+
continue
|
| 956 |
+
seen_ids.add(ident)
|
| 957 |
+
out.append(source)
|
| 958 |
+
if len(out) >= max_sources:
|
| 959 |
+
break
|
| 960 |
+
|
| 961 |
+
return out or list(sources)[:max_sources]
|
| 962 |
+
|
| 963 |
+
@classmethod
|
| 964 |
+
def _postprocess_answer(
|
| 965 |
+
cls,
|
| 966 |
+
answer: str,
|
| 967 |
+
sources: List[Dict[str, Any]],
|
| 968 |
+
hits: List[Dict[str, Any]],
|
| 969 |
+
) -> Tuple[str, List[str]]:
|
| 970 |
+
fixes: List[str] = []
|
| 971 |
+
cleaned, ref_fixes = cls._downgrade_unsupported_fine_refs(answer, sources, hits)
|
| 972 |
+
fixes.extend(ref_fixes)
|
| 973 |
+
cleaned, marker_fixes = cls._strip_invalid_source_markers_safely(cleaned, sources, hits)
|
| 974 |
+
fixes.extend(marker_fixes)
|
| 975 |
+
cleaned = cls._cleanup_citation_grammar(cleaned)
|
| 976 |
+
return cleaned, fixes
|
| 977 |
+
|
| 978 |
+
@staticmethod
|
| 979 |
+
def _detect_answer_basis(answer: str) -> str:
|
| 980 |
+
a = _lower(answer)
|
| 981 |
+
if NEGATIVE_ANSWER_RE.search(answer or ""):
|
| 982 |
+
return "negative"
|
| 983 |
+
if "ausdrücklich geregelt" in a:
|
| 984 |
+
return "explicit"
|
| 985 |
+
if "systematisch" in a or "ableitbar" in a or "auslegung" in a:
|
| 986 |
+
return "derived"
|
| 987 |
+
if "nicht belastbar" in a or "keine belastbare" in a:
|
| 988 |
+
return "insufficient"
|
| 989 |
+
return "unknown"
|
| 990 |
+
|
| 991 |
+
@staticmethod
|
| 992 |
+
def _expected_list_labels_from_hits(hits: Sequence[Dict[str, Any]]) -> List[str]:
|
| 993 |
+
text = "\n".join(_hit_text(h) for h in hits[:8])
|
| 994 |
+
labels = set()
|
| 995 |
+
for m in re.finditer(r"(?:^|\n|\s)([a-z])\)\s+", text):
|
| 996 |
+
labels.add(f"{m.group(1).lower()})")
|
| 997 |
+
for m in re.finditer(r"(?:^|\n|\s)(\d{1,2})[.)]\s+", text):
|
| 998 |
+
labels.add(f"{m.group(1)}")
|
| 999 |
+
ordered_letters = [f"{chr(i)})" for i in range(ord("a"), ord("z") + 1) if f"{chr(i)})" in labels]
|
| 1000 |
+
ordered_nums = sorted([x for x in labels if x.isdigit()], key=lambda n: int(n))
|
| 1001 |
+
return ordered_letters + ordered_nums
|
| 1002 |
+
|
| 1003 |
+
def audit_answer(
|
| 1004 |
+
self,
|
| 1005 |
+
question: str,
|
| 1006 |
+
answer: str,
|
| 1007 |
+
hits: List[Dict[str, Any]],
|
| 1008 |
+
sources: List[Dict[str, Any]],
|
| 1009 |
+
analysis: QuestionAnalysis,
|
| 1010 |
+
assessment: RetrievalAssessment,
|
| 1011 |
+
) -> AnswerAudit:
|
| 1012 |
+
cited = self._extract_source_numbers(answer)
|
| 1013 |
+
valid_numbers = self._valid_source_numbers(sources, hits)
|
| 1014 |
+
|
| 1015 |
+
invalid = [n for n in cited if n not in valid_numbers]
|
| 1016 |
+
basis = self._detect_answer_basis(answer)
|
| 1017 |
+
negative = basis == "negative"
|
| 1018 |
+
completeness_warnings: List[str] = []
|
| 1019 |
+
citation_warnings: List[str] = []
|
| 1020 |
+
|
| 1021 |
+
if not cited and hits and basis != "negative":
|
| 1022 |
+
citation_warnings.append("answer_contains_no_source_markers")
|
| 1023 |
+
if invalid:
|
| 1024 |
+
citation_warnings.append("answer_contains_invalid_source_markers")
|
| 1025 |
+
if analysis.norm_references and not any(ref.section_id in " ".join(assessment.sections + assessment.canonical_refs) for ref in analysis.norm_references):
|
| 1026 |
+
completeness_warnings.append("explicit_norm_not_reflected_in_retrieved_context")
|
| 1027 |
+
if analysis.intent in {"enumeration", "norm_enumeration"}:
|
| 1028 |
+
labels = self._expected_list_labels_from_hits(hits)
|
| 1029 |
+
if len(labels) >= 3:
|
| 1030 |
+
answer_lower = answer.lower()
|
| 1031 |
+
missing_labels = [label for label in labels if label not in answer_lower]
|
| 1032 |
+
# Do not force exact labels in prose answers, but flag likely omissions.
|
| 1033 |
+
if len(missing_labels) >= max(2, len(labels) // 2):
|
| 1034 |
+
completeness_warnings.append(
|
| 1035 |
+
"possible_incomplete_enumeration: expected list markers " + ", ".join(labels[:12])
|
| 1036 |
+
)
|
| 1037 |
+
if negative and assessment.hit_count > 0 and not assessment.low_confidence:
|
| 1038 |
+
completeness_warnings.append("negative_answer_despite_available_context")
|
| 1039 |
+
|
| 1040 |
+
confidence = 0.55
|
| 1041 |
+
if assessment.hit_count:
|
| 1042 |
+
confidence += 0.15
|
| 1043 |
+
if assessment.exact_norm_hit_count and analysis.norm_references:
|
| 1044 |
+
confidence += 0.15
|
| 1045 |
+
if assessment.definition_hit_count and analysis.intent == "definition":
|
| 1046 |
+
confidence += 0.15
|
| 1047 |
+
if citation_warnings:
|
| 1048 |
+
confidence -= 0.15
|
| 1049 |
+
if completeness_warnings:
|
| 1050 |
+
confidence -= 0.15
|
| 1051 |
+
if negative and assessment.low_confidence:
|
| 1052 |
+
confidence -= 0.10
|
| 1053 |
+
confidence = round(max(0.0, min(confidence, 0.98)), 2)
|
| 1054 |
+
|
| 1055 |
+
recommended_action = "accept"
|
| 1056 |
+
if analysis.needs_clarification:
|
| 1057 |
+
recommended_action = "clarify"
|
| 1058 |
+
elif negative and assessment.low_confidence:
|
| 1059 |
+
recommended_action = "recheck"
|
| 1060 |
+
elif citation_warnings or completeness_warnings:
|
| 1061 |
+
recommended_action = "caution"
|
| 1062 |
+
|
| 1063 |
+
refs_by_no = self._source_number_to_ref(sources, hits)
|
| 1064 |
+
cited_refs = [refs_by_no[n] for n in cited if n in refs_by_no]
|
| 1065 |
+
|
| 1066 |
+
return AnswerAudit(
|
| 1067 |
+
cited_source_numbers=cited,
|
| 1068 |
+
invalid_source_numbers=invalid,
|
| 1069 |
+
cited_canonical_refs=cited_refs,
|
| 1070 |
+
answer_basis=basis,
|
| 1071 |
+
negative_answer_detected=negative,
|
| 1072 |
+
completeness_warnings=completeness_warnings,
|
| 1073 |
+
citation_warnings=citation_warnings,
|
| 1074 |
+
recommended_action=recommended_action,
|
| 1075 |
+
confidence=confidence,
|
| 1076 |
+
)
|
| 1077 |
+
|
| 1078 |
+
def _maybe_recompose_after_negative_answer(
|
| 1079 |
+
self,
|
| 1080 |
+
question: str,
|
| 1081 |
+
answer: str,
|
| 1082 |
+
hits: List[Dict[str, Any]],
|
| 1083 |
+
sources: List[Dict[str, Any]],
|
| 1084 |
+
analysis: QuestionAnalysis,
|
| 1085 |
+
assessment: RetrievalAssessment,
|
| 1086 |
+
memory: Optional[ConversationMemory],
|
| 1087 |
+
options: OrchestratorOptions,
|
| 1088 |
+
) -> Tuple[str, str, List[Dict[str, Any]], List[Dict[str, Any]], RetrievalAssessment]:
|
| 1089 |
+
if not options.enable_negative_recheck or not NEGATIVE_ANSWER_RE.search(answer or ""):
|
| 1090 |
+
return answer, "document", sources, hits, assessment
|
| 1091 |
+
|
| 1092 |
+
old_count = len(hits)
|
| 1093 |
+
hits2, assessment2 = self._negative_recheck(question, analysis, hits, assessment, options)
|
| 1094 |
+
if len(hits2) <= old_count:
|
| 1095 |
+
return answer, "document", sources, hits, assessment2
|
| 1096 |
+
|
| 1097 |
+
answer2, answer_type2, sources2 = self._compose(question, hits2, memory)
|
| 1098 |
+
return answer2, answer_type2, sources2, hits2, assessment2
|
| 1099 |
+
|
| 1100 |
+
# ------------------------------------------------------------------
|
| 1101 |
+
# Public flow
|
| 1102 |
+
# ------------------------------------------------------------------
|
| 1103 |
+
|
| 1104 |
+
def run(
|
| 1105 |
+
self,
|
| 1106 |
+
question: str,
|
| 1107 |
+
*,
|
| 1108 |
+
memory: Optional[ConversationMemory] = None,
|
| 1109 |
+
options: OrchestratorOptions | None = None,
|
| 1110 |
+
) -> OrchestratorResult:
|
| 1111 |
+
options = options or self.options
|
| 1112 |
+
question = _normalize(question)
|
| 1113 |
+
analysis = self.analyze_question(question)
|
| 1114 |
+
|
| 1115 |
+
if analysis.intent == "meta":
|
| 1116 |
+
answer, answer_type = self.composer.compose(question, [], memory=memory)
|
| 1117 |
+
return OrchestratorResult(
|
| 1118 |
+
answer=answer,
|
| 1119 |
+
answer_type=answer_type,
|
| 1120 |
+
analysis=analysis,
|
| 1121 |
+
retrieval_assessment=RetrievalAssessment(),
|
| 1122 |
+
answer_audit=AnswerAudit(answer_basis="meta", confidence=1.0),
|
| 1123 |
+
)
|
| 1124 |
+
|
| 1125 |
+
if analysis.needs_clarification:
|
| 1126 |
+
answer = analysis.clarification_question or "Bitte präzisiere deine Frage."
|
| 1127 |
+
return OrchestratorResult(
|
| 1128 |
+
answer=answer,
|
| 1129 |
+
answer_type="clarification",
|
| 1130 |
+
analysis=analysis,
|
| 1131 |
+
retrieval_assessment=RetrievalAssessment(low_confidence=True, reasons=["clarification_required"]),
|
| 1132 |
+
answer_audit=AnswerAudit(answer_basis="clarification", recommended_action="clarify", confidence=0.4),
|
| 1133 |
+
needs_clarification=True,
|
| 1134 |
+
clarification_question=answer,
|
| 1135 |
+
)
|
| 1136 |
+
|
| 1137 |
+
hits = self._retriever_query(question, analysis, options)
|
| 1138 |
+
hits = _dedupe_hits(hits)
|
| 1139 |
+
assessment = self.assess_retrieval(hits, analysis)
|
| 1140 |
+
hits, assessment = self._negative_recheck(question, analysis, hits, assessment, options)
|
| 1141 |
+
|
| 1142 |
+
if not hits:
|
| 1143 |
+
answer = (
|
| 1144 |
+
"Ich habe im verfügbaren Vertragskorpus keine belastbare Textstelle gefunden. "
|
| 1145 |
+
"Das bedeutet nicht zwingend, dass der Sachverhalt rechtlich nicht geregelt ist; "
|
| 1146 |
+
"es heißt zunächst nur, dass die relevante Regelung im aktuellen Retrieval-Kontext "
|
| 1147 |
+
"nicht auffindbar war."
|
| 1148 |
+
)
|
| 1149 |
+
audit = AnswerAudit(
|
| 1150 |
+
answer_basis="insufficient",
|
| 1151 |
+
negative_answer_detected=True,
|
| 1152 |
+
recommended_action="caution",
|
| 1153 |
+
confidence=0.25,
|
| 1154 |
+
completeness_warnings=["no_retrieval_hits_after_recheck"],
|
| 1155 |
+
)
|
| 1156 |
+
return OrchestratorResult(
|
| 1157 |
+
answer=answer,
|
| 1158 |
+
answer_type="none",
|
| 1159 |
+
hits=[],
|
| 1160 |
+
raw_sources=[],
|
| 1161 |
+
sources=[],
|
| 1162 |
+
analysis=analysis,
|
| 1163 |
+
retrieval_assessment=assessment,
|
| 1164 |
+
answer_audit=audit,
|
| 1165 |
+
)
|
| 1166 |
+
|
| 1167 |
+
answer, answer_type, sources = self._compose(question, hits, memory)
|
| 1168 |
+
answer, answer_type, sources, hits, assessment = self._maybe_recompose_after_negative_answer(
|
| 1169 |
+
question, answer, hits, sources, analysis, assessment, memory, options
|
| 1170 |
+
)
|
| 1171 |
+
|
| 1172 |
+
# Keep source visibility and answer postprocessing inside the same
|
| 1173 |
+
# numbering universe as the Composer. This prevents the API layer from
|
| 1174 |
+
# deleting markers such as [Quelle 3] while the answer still contains
|
| 1175 |
+
# dangling grammar fragments.
|
| 1176 |
+
sources = self._ensure_cited_sources_visible(answer, sources, hits)
|
| 1177 |
+
|
| 1178 |
+
refs_by_no = self._source_number_to_ref(sources, hits)
|
| 1179 |
+
if options.enrich_citations_with_canonical_refs:
|
| 1180 |
+
answer = self._enrich_answer_citations(answer, refs_by_no)
|
| 1181 |
+
|
| 1182 |
+
sources = self._ensure_cited_sources_visible(answer, sources, hits)
|
| 1183 |
+
answer, auto_fixes = self._postprocess_answer(answer, sources, hits)
|
| 1184 |
+
sources = self._ensure_cited_sources_visible(answer, sources, hits)
|
| 1185 |
+
sources = self._filter_sources_to_cited_sources(
|
| 1186 |
+
answer,
|
| 1187 |
+
sources,
|
| 1188 |
+
hits,
|
| 1189 |
+
max_sources=options.max_sources,
|
| 1190 |
+
)
|
| 1191 |
+
|
| 1192 |
+
audit = self.audit_answer(question, answer, hits, sources, analysis, assessment) if options.enable_answer_audit else AnswerAudit()
|
| 1193 |
+
audit.auto_fixes.extend(auto_fixes)
|
| 1194 |
+
|
| 1195 |
+
debug: Dict[str, Any] = {}
|
| 1196 |
+
if options.debug:
|
| 1197 |
+
debug = {
|
| 1198 |
+
"source_number_to_canonical_ref": refs_by_no,
|
| 1199 |
+
"retrieved_hit_count": len(hits),
|
| 1200 |
+
"source_count": len(sources),
|
| 1201 |
+
"answer_auto_fixes": auto_fixes,
|
| 1202 |
+
"returned_source_numbers": [
|
| 1203 |
+
self._source_numbers_from_source(source, fallback=idx)
|
| 1204 |
+
for idx, source in enumerate(sources, start=1)
|
| 1205 |
+
],
|
| 1206 |
+
"cited_source_numbers_after_postprocess": self._extract_source_numbers(answer),
|
| 1207 |
+
}
|
| 1208 |
+
|
| 1209 |
+
return OrchestratorResult(
|
| 1210 |
+
answer=answer,
|
| 1211 |
+
answer_type=answer_type,
|
| 1212 |
+
hits=hits,
|
| 1213 |
+
raw_sources=sources,
|
| 1214 |
+
sources=sources,
|
| 1215 |
+
analysis=analysis,
|
| 1216 |
+
retrieval_assessment=assessment,
|
| 1217 |
+
answer_audit=audit,
|
| 1218 |
+
needs_clarification=False,
|
| 1219 |
+
debug=debug,
|
| 1220 |
+
)
|
src/retriever.py
CHANGED
|
@@ -1,9 +1,11 @@
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
import logging
|
|
|
|
| 4 |
import re
|
| 5 |
-
from collections import defaultdict
|
| 6 |
-
from
|
|
|
|
| 7 |
|
| 8 |
import chromadb
|
| 9 |
from sentence_transformers import SentenceTransformer
|
|
@@ -11,26 +13,80 @@ from sentence_transformers import SentenceTransformer
|
|
| 11 |
logger = logging.getLogger(__name__)
|
| 12 |
|
| 13 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
class LegalRetriever:
|
| 15 |
"""
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
-
|
| 28 |
-
-
|
| 29 |
-
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
- section_path
|
| 31 |
- page_start, page_end
|
| 32 |
- chunk_index_in_section
|
| 33 |
-
- text_hash optional
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
"""
|
| 35 |
|
| 36 |
SECTION_REF_RE = re.compile(
|
|
@@ -39,17 +95,121 @@ class LegalRetriever:
|
|
| 39 |
re.IGNORECASE,
|
| 40 |
)
|
| 41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
def __init__(
|
| 43 |
self,
|
| 44 |
persist_dir: str,
|
| 45 |
collection: str,
|
| 46 |
-
model_name: str,
|
| 47 |
*,
|
| 48 |
default_container_id: str = "Vertrag",
|
| 49 |
normalize_embeddings: bool = True,
|
| 50 |
-
default_to_contract: bool =
|
| 51 |
min_score: float = 0.20,
|
| 52 |
verbose_startup: bool = True,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
):
|
| 54 |
self.persist_dir = persist_dir
|
| 55 |
self.collection_name = collection
|
|
@@ -57,29 +217,7 @@ class LegalRetriever:
|
|
| 57 |
self.normalize_embeddings = normalize_embeddings
|
| 58 |
self.default_to_contract = default_to_contract
|
| 59 |
self.min_score = float(min_score)
|
| 60 |
-
|
| 61 |
-
import os
|
| 62 |
-
from pathlib import Path
|
| 63 |
-
|
| 64 |
-
# --- DEBUG: Space-Dateisystem prüfen ---
|
| 65 |
-
try:
|
| 66 |
-
print("DEBUG /data exists:", os.path.exists("/data"))
|
| 67 |
-
if os.path.exists("/data"):
|
| 68 |
-
print("DEBUG /data listing:", os.listdir("/data")[:50])
|
| 69 |
-
|
| 70 |
-
print("DEBUG persist_dir:", persist_dir)
|
| 71 |
-
print("DEBUG persist_dir exists:", Path(persist_dir).exists())
|
| 72 |
-
if Path(persist_dir).exists():
|
| 73 |
-
print("DEBUG persist_dir listing:", [p.name for p in Path(persist_dir).iterdir()][:50])
|
| 74 |
-
print("DEBUG chroma.sqlite3 exists:", (Path(persist_dir) / "chroma.sqlite3").exists())
|
| 75 |
-
|
| 76 |
-
# Optional: falls du vermutest, dass es irgendwo tiefer liegt
|
| 77 |
-
if os.path.exists("/data"):
|
| 78 |
-
hits = list(Path("/data").rglob("chroma.sqlite3"))
|
| 79 |
-
print("DEBUG found chroma.sqlite3 files:", [str(p) for p in hits][:10])
|
| 80 |
-
except Exception as e:
|
| 81 |
-
print("DEBUG filesystem check failed:", repr(e))
|
| 82 |
-
# --- /DEBUG ---
|
| 83 |
|
| 84 |
self.client = chromadb.PersistentClient(path=persist_dir)
|
| 85 |
|
|
@@ -93,11 +231,59 @@ class LegalRetriever:
|
|
| 93 |
f"verfügbare Collections={available!r}"
|
| 94 |
) from exc
|
| 95 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
if verbose_startup:
|
| 97 |
self._log_startup_diagnostics()
|
| 98 |
|
|
|
|
| 99 |
self.embedder = SentenceTransformer(model_name)
|
| 100 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
# ------------------------------------------------------------------
|
| 102 |
# Diagnostics
|
| 103 |
# ------------------------------------------------------------------
|
|
@@ -119,10 +305,7 @@ class LegalRetriever:
|
|
| 119 |
)
|
| 120 |
|
| 121 |
def diagnostics(self, *, sample: int = 3) -> Dict[str, Any]:
|
| 122 |
-
"""
|
| 123 |
-
Gibt eine kompakte Laufzeitdiagnose zurück.
|
| 124 |
-
Nützlich, um alte Collections oder falsche Pfade zu erkennen.
|
| 125 |
-
"""
|
| 126 |
collections = [c.name for c in self.client.list_collections()]
|
| 127 |
result: Dict[str, Any] = {
|
| 128 |
"persist_dir": self.persist_dir,
|
|
@@ -134,7 +317,10 @@ class LegalRetriever:
|
|
| 134 |
try:
|
| 135 |
res = self.col.get(limit=sample, include=["metadatas"])
|
| 136 |
result["metadata_sample"] = res.get("metadatas", [])
|
| 137 |
-
|
|
|
|
|
|
|
|
|
|
| 138 |
result["metadata_sample_error"] = repr(exc)
|
| 139 |
|
| 140 |
return result
|
|
@@ -145,13 +331,9 @@ class LegalRetriever:
|
|
| 145 |
@staticmethod
|
| 146 |
def _build_where(filters: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
| 147 |
"""
|
| 148 |
-
|
| 149 |
|
| 150 |
-
Chroma
|
| 151 |
-
Beispiele:
|
| 152 |
-
{"container_id": "Vertrag"}
|
| 153 |
-
{"$and": [{"container_id": "Vertrag"}, {"section_id": "§ 6"}]}
|
| 154 |
-
{"section_id": {"$in": ["§ 6", "§6", "6"]}}
|
| 155 |
"""
|
| 156 |
if not filters:
|
| 157 |
return None
|
|
@@ -186,6 +368,15 @@ class LegalRetriever:
|
|
| 186 |
return parts[0]
|
| 187 |
return {"$and": parts}
|
| 188 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 189 |
def _effective_where(
|
| 190 |
self,
|
| 191 |
where: Optional[Dict[str, Any]],
|
|
@@ -193,8 +384,8 @@ class LegalRetriever:
|
|
| 193 |
restrict_to_default_container: Optional[bool],
|
| 194 |
) -> Optional[Dict[str, Any]]:
|
| 195 |
"""
|
| 196 |
-
|
| 197 |
-
|
| 198 |
"""
|
| 199 |
if where:
|
| 200 |
return where
|
|
@@ -206,17 +397,13 @@ class LegalRetriever:
|
|
| 206 |
return None
|
| 207 |
|
| 208 |
# ------------------------------------------------------------------
|
| 209 |
-
# Paragraph references
|
| 210 |
# ------------------------------------------------------------------
|
| 211 |
@classmethod
|
| 212 |
def _parse_section_refs(cls, text: str, *, max_range: int = 30) -> List[str]:
|
| 213 |
"""
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
§§ 7-14
|
| 217 |
-
§§ 7 bis 14
|
| 218 |
-
|
| 219 |
-
Gibt normalisierte section_ids wie ["§ 6", "§ 7", ...] zurück.
|
| 220 |
"""
|
| 221 |
if not text:
|
| 222 |
return []
|
|
@@ -240,6 +427,35 @@ class LegalRetriever:
|
|
| 240 |
|
| 241 |
return list(dict.fromkeys(found))
|
| 242 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
@staticmethod
|
| 244 |
def _section_variants(section: str) -> List[str]:
|
| 245 |
if not section:
|
|
@@ -254,6 +470,38 @@ class LegalRetriever:
|
|
| 254 |
|
| 255 |
return list(dict.fromkeys(variants))
|
| 256 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 257 |
# ------------------------------------------------------------------
|
| 258 |
# Public API
|
| 259 |
# ------------------------------------------------------------------
|
|
@@ -267,21 +515,38 @@ class LegalRetriever:
|
|
| 267 |
include_explicit_sections: bool = True,
|
| 268 |
explicit_sections: Optional[List[str]] = None,
|
| 269 |
explicit_section_container_ids: Optional[List[str]] = None,
|
| 270 |
-
max_chunks_per_explicit_section: int =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 271 |
include_neighbors: bool = True,
|
| 272 |
neighbor_window: int = 1,
|
| 273 |
-
max_final_results: Optional[int] =
|
| 274 |
min_score: Optional[float] = None,
|
| 275 |
restrict_to_default_container: Optional[bool] = None,
|
| 276 |
-
max_chunks_per_section: int =
|
|
|
|
|
|
|
|
|
|
| 277 |
) -> List[Dict[str, Any]]:
|
| 278 |
"""
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
-
|
| 283 |
-
-
|
| 284 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 285 |
"""
|
| 286 |
if not question or not question.strip():
|
| 287 |
return []
|
|
@@ -290,9 +555,13 @@ class LegalRetriever:
|
|
| 290 |
where,
|
| 291 |
restrict_to_default_container=restrict_to_default_container,
|
| 292 |
)
|
| 293 |
-
|
| 294 |
threshold = self.min_score if min_score is None else float(min_score)
|
|
|
|
|
|
|
| 295 |
|
|
|
|
|
|
|
|
|
|
| 296 |
semantic_results = self._semantic_query(
|
| 297 |
question=question,
|
| 298 |
top_k=top_k,
|
|
@@ -300,43 +569,290 @@ class LegalRetriever:
|
|
| 300 |
fetch_k=fetch_k,
|
| 301 |
)
|
| 302 |
semantic_results = [hit for hit in semantic_results if float(hit.get("score", 0.0)) >= threshold]
|
| 303 |
-
|
| 304 |
-
all_results: List[Dict[str, Any]] = []
|
| 305 |
all_results.extend(semantic_results)
|
| 306 |
|
| 307 |
-
#
|
| 308 |
if include_explicit_sections:
|
|
|
|
| 309 |
sections = list(explicit_sections or [])
|
| 310 |
sections.extend(self._parse_section_refs(question))
|
| 311 |
sections = list(dict.fromkeys(sections))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 312 |
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
base_where=effective_where,
|
| 319 |
-
|
|
|
|
| 320 |
)
|
| 321 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 322 |
|
| 323 |
-
#
|
| 324 |
if include_neighbors and neighbor_window > 0:
|
| 325 |
neighbor_results: List[Dict[str, Any]] = []
|
| 326 |
-
|
|
|
|
| 327 |
neighbor_results.extend(
|
| 328 |
self._get_neighbors(hit, window=neighbor_window, base_where=effective_where)
|
| 329 |
)
|
| 330 |
all_results.extend(neighbor_results)
|
| 331 |
|
| 332 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 333 |
merged = self._limit_chunks_per_section(merged, max_chunks_per_section=max_chunks_per_section)
|
| 334 |
|
| 335 |
if max_final_results is not None:
|
| 336 |
merged = merged[:max_final_results]
|
| 337 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 338 |
return merged
|
| 339 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 340 |
def _semantic_query(
|
| 341 |
self,
|
| 342 |
*,
|
|
@@ -347,16 +863,14 @@ class LegalRetriever:
|
|
| 347 |
) -> List[Dict[str, Any]]:
|
| 348 |
n_results = fetch_k if fetch_k is not None else max(top_k * 4, top_k)
|
| 349 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 350 |
if hasattr(self.embedder, "encode_query"):
|
| 351 |
-
q_emb = self.embedder.encode_query(
|
| 352 |
-
[question],
|
| 353 |
-
normalize_embeddings=self.normalize_embeddings,
|
| 354 |
-
)[0].tolist()
|
| 355 |
else:
|
| 356 |
-
q_emb = self.embedder.encode(
|
| 357 |
-
[question],
|
| 358 |
-
normalize_embeddings=self.normalize_embeddings,
|
| 359 |
-
)[0].tolist()
|
| 360 |
|
| 361 |
chroma_where = self._build_where(where)
|
| 362 |
|
|
@@ -369,9 +883,58 @@ class LegalRetriever:
|
|
| 369 |
|
| 370 |
return self._format(res, retrieval_kind="semantic")[:n_results]
|
| 371 |
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 375 |
def _get_explicit_sections(
|
| 376 |
self,
|
| 377 |
*,
|
|
@@ -379,6 +942,7 @@ class LegalRetriever:
|
|
| 379 |
container_ids: List[str],
|
| 380 |
base_where: Optional[Dict[str, Any]] = None,
|
| 381 |
max_chunks_per_section: int = 4,
|
|
|
|
| 382 |
) -> List[Dict[str, Any]]:
|
| 383 |
out: List[Dict[str, Any]] = []
|
| 384 |
|
|
@@ -388,31 +952,124 @@ class LegalRetriever:
|
|
| 388 |
|
| 389 |
where = self._where_and(
|
| 390 |
self._build_where(base_where),
|
| 391 |
-
{
|
| 392 |
-
"container_id": container_id,
|
| 393 |
-
"section_id": {"$in": variants},
|
| 394 |
-
},
|
| 395 |
)
|
| 396 |
|
| 397 |
try:
|
| 398 |
-
res = self.col.get(
|
| 399 |
-
|
| 400 |
-
include=["documents", "metadatas"],
|
| 401 |
-
)
|
| 402 |
-
except Exception as exc:
|
| 403 |
logger.debug("explicit section lookup failed", exc_info=exc)
|
| 404 |
continue
|
| 405 |
|
| 406 |
-
formatted = self._format_get(
|
| 407 |
-
res,
|
| 408 |
-
retrieval_kind="explicit_section",
|
| 409 |
-
score=1.0,
|
| 410 |
-
)
|
| 411 |
formatted.sort(key=lambda x: x.get("chunk_index", 0))
|
| 412 |
out.extend(formatted[:max_chunks_per_section])
|
| 413 |
|
| 414 |
return out
|
| 415 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 416 |
def _get_neighbors(
|
| 417 |
self,
|
| 418 |
hit: Dict[str, Any],
|
|
@@ -432,10 +1089,7 @@ class LegalRetriever:
|
|
| 432 |
except (TypeError, ValueError):
|
| 433 |
return []
|
| 434 |
|
| 435 |
-
neighbor_indices = [
|
| 436 |
-
i for i in range(idx - window, idx + window + 1)
|
| 437 |
-
if i >= 0 and i != idx
|
| 438 |
-
]
|
| 439 |
if not neighbor_indices:
|
| 440 |
return []
|
| 441 |
|
|
@@ -449,11 +1103,8 @@ class LegalRetriever:
|
|
| 449 |
)
|
| 450 |
|
| 451 |
try:
|
| 452 |
-
res = self.col.get(
|
| 453 |
-
|
| 454 |
-
include=["documents", "metadatas"],
|
| 455 |
-
)
|
| 456 |
-
except Exception as exc:
|
| 457 |
logger.debug("neighbor lookup failed", exc_info=exc)
|
| 458 |
return []
|
| 459 |
|
|
@@ -465,94 +1116,100 @@ class LegalRetriever:
|
|
| 465 |
formatted.sort(key=lambda x: x.get("chunk_index", 0))
|
| 466 |
return formatted
|
| 467 |
|
| 468 |
-
|
| 469 |
-
# Convenience wrappers
|
| 470 |
-
# ------------------------------------------------------------------
|
| 471 |
-
def query_paragraph(
|
| 472 |
self,
|
| 473 |
-
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
**kwargs: Any,
|
| 478 |
) -> List[Dict[str, Any]]:
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
top_k=top_k,
|
| 482 |
-
where={
|
| 483 |
-
"container_id": container,
|
| 484 |
-
"section_id": paragraph,
|
| 485 |
-
},
|
| 486 |
-
include_explicit_sections=False,
|
| 487 |
-
**kwargs,
|
| 488 |
-
)
|
| 489 |
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
top_k: int = 6,
|
| 496 |
-
**kwargs: Any,
|
| 497 |
-
) -> List[Dict[str, Any]]:
|
| 498 |
-
where: Dict[str, Any] = {
|
| 499 |
-
"container_id": f"Anlage {anlage_nr}",
|
| 500 |
-
}
|
| 501 |
-
if paragraph:
|
| 502 |
-
where["section_id"] = paragraph
|
| 503 |
|
| 504 |
-
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
|
| 509 |
-
|
| 510 |
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
anlage_nr: int,
|
| 515 |
-
top_k: int = 6,
|
| 516 |
-
**kwargs: Any,
|
| 517 |
-
) -> List[Dict[str, Any]]:
|
| 518 |
-
return self.query(
|
| 519 |
-
question,
|
| 520 |
-
top_k=top_k,
|
| 521 |
-
where={
|
| 522 |
-
"container_type": "anhang",
|
| 523 |
-
"container_id": f"Anhang zu Anlage {anlage_nr}",
|
| 524 |
-
},
|
| 525 |
-
**kwargs,
|
| 526 |
-
)
|
| 527 |
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
section: str,
|
| 531 |
-
*,
|
| 532 |
-
container: str = "Vertrag",
|
| 533 |
-
max_chunks: int = 20,
|
| 534 |
-
) -> List[Dict[str, Any]]:
|
| 535 |
-
variants = self._section_variants(section)
|
| 536 |
-
where = {
|
| 537 |
-
"container_id": container,
|
| 538 |
-
"section_id": {"$in": variants},
|
| 539 |
-
}
|
| 540 |
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
)
|
| 546 |
-
|
| 547 |
-
|
|
|
|
|
|
|
|
|
|
| 548 |
|
| 549 |
-
|
| 550 |
-
res,
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 556 |
|
| 557 |
# ------------------------------------------------------------------
|
| 558 |
# Context and source formatting
|
|
@@ -563,10 +1220,9 @@ class LegalRetriever:
|
|
| 563 |
*,
|
| 564 |
max_chars: int = 12000,
|
| 565 |
include_neighbors: bool = True,
|
|
|
|
| 566 |
) -> str:
|
| 567 |
-
"""
|
| 568 |
-
Baut aus Retriever-Treffern einen stabilen RAG-Kontext für den LLM-Client.
|
| 569 |
-
"""
|
| 570 |
if not results:
|
| 571 |
return ""
|
| 572 |
|
|
@@ -578,13 +1234,23 @@ class LegalRetriever:
|
|
| 578 |
if not include_neighbors and kinds == {"neighbor"}:
|
| 579 |
continue
|
| 580 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 581 |
source = (
|
| 582 |
f"[Quelle {i}: {hit.get('container', 'Unbekannt')}::"
|
| 583 |
-
f"{hit.get('
|
| 584 |
-
f"Seiten {hit.get('page_range', '?')}, "
|
| 585 |
f"Chunk {hit.get('chunk_index', '?')}, "
|
| 586 |
-
f"Typ {','.join(hit.get('retrieval_kinds', []))}
|
| 587 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 588 |
text = (hit.get("text") or "").strip()
|
| 589 |
block = f"{source}\n{text}"
|
| 590 |
|
|
@@ -604,14 +1270,12 @@ class LegalRetriever:
|
|
| 604 |
include_neighbor_only: bool = False,
|
| 605 |
) -> List[Dict[str, Any]]:
|
| 606 |
"""
|
| 607 |
-
|
| 608 |
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
- dedupliziert nach Container, Section und Seitenbereich,
|
| 612 |
-
- begrenzt die Anzahl der angezeigten Quellen.
|
| 613 |
"""
|
| 614 |
-
seen: Set[Tuple[Any, Any, Any]] = set()
|
| 615 |
sources: List[Dict[str, Any]] = []
|
| 616 |
|
| 617 |
for hit in results:
|
|
@@ -619,11 +1283,9 @@ class LegalRetriever:
|
|
| 619 |
if kinds == {"neighbor"} and not include_neighbor_only:
|
| 620 |
continue
|
| 621 |
|
| 622 |
-
|
| 623 |
-
|
| 624 |
-
|
| 625 |
-
hit.get("page_range"),
|
| 626 |
-
)
|
| 627 |
|
| 628 |
if key in seen:
|
| 629 |
continue
|
|
@@ -634,6 +1296,7 @@ class LegalRetriever:
|
|
| 634 |
{
|
| 635 |
"container": hit.get("container"),
|
| 636 |
"section": hit.get("section"),
|
|
|
|
| 637 |
"path": hit.get("path"),
|
| 638 |
"page_range": hit.get("page_range"),
|
| 639 |
"page_start": hit.get("page_start"),
|
|
@@ -641,8 +1304,11 @@ class LegalRetriever:
|
|
| 641 |
"score": hit.get("score"),
|
| 642 |
"rank_score": hit.get("rank_score"),
|
| 643 |
"retrieval_kinds": hit.get("retrieval_kinds", []),
|
| 644 |
-
"
|
| 645 |
-
"
|
|
|
|
|
|
|
|
|
|
| 646 |
}
|
| 647 |
)
|
| 648 |
|
|
@@ -652,11 +1318,7 @@ class LegalRetriever:
|
|
| 652 |
return sources
|
| 653 |
|
| 654 |
@staticmethod
|
| 655 |
-
def format_sources_markdown(
|
| 656 |
-
sources: List[Dict[str, Any]],
|
| 657 |
-
*,
|
| 658 |
-
title: str = "Quellen",
|
| 659 |
-
) -> str:
|
| 660 |
if not sources:
|
| 661 |
return f"{title}: Keine Quellen gefunden."
|
| 662 |
|
|
@@ -666,10 +1328,11 @@ class LegalRetriever:
|
|
| 666 |
section = source.get("section") or "ohne Abschnitt"
|
| 667 |
pages = source.get("page_range") or "?"
|
| 668 |
canonical = source.get("canonical_ref")
|
|
|
|
| 669 |
if canonical and canonical != section:
|
| 670 |
lines.append(f"- {container}::{section} ({canonical}), Seiten {pages}")
|
| 671 |
else:
|
| 672 |
-
lines.append(f"- {container}::{
|
| 673 |
|
| 674 |
return "\n".join(lines)
|
| 675 |
|
|
@@ -696,12 +1359,7 @@ class LegalRetriever:
|
|
| 696 |
return out
|
| 697 |
|
| 698 |
@staticmethod
|
| 699 |
-
def _format_get(
|
| 700 |
-
res: Dict[str, Any],
|
| 701 |
-
*,
|
| 702 |
-
retrieval_kind: str,
|
| 703 |
-
score: float,
|
| 704 |
-
) -> List[Dict[str, Any]]:
|
| 705 |
docs = res.get("documents") or []
|
| 706 |
metas = res.get("metadatas") or []
|
| 707 |
|
|
@@ -720,8 +1378,8 @@ class LegalRetriever:
|
|
| 720 |
@staticmethod
|
| 721 |
def _similarity_from_distance(dist: Any) -> float:
|
| 722 |
"""
|
| 723 |
-
|
| 724 |
-
|
| 725 |
"""
|
| 726 |
try:
|
| 727 |
similarity = 1.0 - float(dist)
|
|
@@ -731,13 +1389,7 @@ class LegalRetriever:
|
|
| 731 |
return round(max(min(similarity, 1.0), -1.0), 4)
|
| 732 |
|
| 733 |
@staticmethod
|
| 734 |
-
def _normalize_hit(
|
| 735 |
-
*,
|
| 736 |
-
doc: str,
|
| 737 |
-
meta: Dict[str, Any],
|
| 738 |
-
score: float,
|
| 739 |
-
retrieval_kind: str,
|
| 740 |
-
) -> Dict[str, Any]:
|
| 741 |
page_start = meta.get("page_start")
|
| 742 |
page_end = meta.get("page_end", page_start)
|
| 743 |
page_range = "?"
|
|
@@ -753,13 +1405,16 @@ class LegalRetriever:
|
|
| 753 |
except (TypeError, ValueError):
|
| 754 |
chunk_index = 0
|
| 755 |
|
|
|
|
|
|
|
| 756 |
hit = {
|
| 757 |
"score": round(float(score), 4),
|
| 758 |
"rank_score": round(float(score), 4),
|
| 759 |
"text": doc or "",
|
| 760 |
"container": meta.get("container_id", "Unbekannt"),
|
| 761 |
"container_type": meta.get("container_type"),
|
| 762 |
-
"section": meta.get("section_id"
|
|
|
|
| 763 |
"path": meta.get("section_path", ""),
|
| 764 |
"page_range": page_range,
|
| 765 |
"page_start": page_start,
|
|
@@ -771,33 +1426,47 @@ class LegalRetriever:
|
|
| 771 |
hit["source_key"] = LegalRetriever._source_key(hit)
|
| 772 |
return hit
|
| 773 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 774 |
@staticmethod
|
| 775 |
def _source_key(hit: Dict[str, Any]) -> Tuple[Any, Any, Any, Any]:
|
| 776 |
meta = hit.get("metadata") or {}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 777 |
text_hash = meta.get("text_hash")
|
| 778 |
if text_hash:
|
| 779 |
-
return (
|
| 780 |
-
hit.get("container"),
|
| 781 |
-
hit.get("section"),
|
| 782 |
-
hit.get("chunk_index"),
|
| 783 |
-
text_hash,
|
| 784 |
-
)
|
| 785 |
|
| 786 |
-
return (
|
| 787 |
-
hit.get("container"),
|
| 788 |
-
hit.get("section"),
|
| 789 |
-
hit.get("chunk_index"),
|
| 790 |
-
(hit.get("text") or "")[:120],
|
| 791 |
-
)
|
| 792 |
|
| 793 |
@staticmethod
|
| 794 |
-
def _dedupe_and_rank(
|
| 795 |
-
|
| 796 |
-
|
| 797 |
-
|
| 798 |
-
|
| 799 |
-
|
| 800 |
-
"""
|
| 801 |
merged: Dict[Tuple[Any, Any, Any, Any], Dict[str, Any]] = {}
|
| 802 |
|
| 803 |
for hit in results:
|
|
@@ -808,14 +1477,8 @@ class LegalRetriever:
|
|
| 808 |
merged[key] = dict(hit)
|
| 809 |
continue
|
| 810 |
|
| 811 |
-
existing["score"] = max(
|
| 812 |
-
|
| 813 |
-
float(hit.get("score", 0.0)),
|
| 814 |
-
)
|
| 815 |
-
existing["rank_score"] = max(
|
| 816 |
-
float(existing.get("rank_score", 0.0)),
|
| 817 |
-
float(hit.get("rank_score", 0.0)),
|
| 818 |
-
)
|
| 819 |
|
| 820 |
kinds: Set[str] = set(existing.get("retrieval_kinds", []))
|
| 821 |
kinds.update(hit.get("retrieval_kinds", []))
|
|
@@ -823,27 +1486,56 @@ class LegalRetriever:
|
|
| 823 |
|
| 824 |
ranked = list(merged.values())
|
| 825 |
|
|
|
|
|
|
|
|
|
|
| 826 |
for hit in ranked:
|
|
|
|
| 827 |
kinds = set(hit.get("retrieval_kinds", []))
|
| 828 |
boost = 0.0
|
| 829 |
|
|
|
|
|
|
|
| 830 |
if "explicit_section" in kinds or "section_lookup" in kinds:
|
| 831 |
-
boost += 0.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 832 |
if "semantic" in kinds:
|
| 833 |
boost += 0.04
|
| 834 |
if kinds == {"neighbor"}:
|
| 835 |
-
boost -= 0.
|
| 836 |
|
| 837 |
-
# Hauptvertrag leicht bevorzugen.
|
| 838 |
if hit.get("container") == "Vertrag":
|
| 839 |
boost += 0.01
|
| 840 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 841 |
hit["rank_score"] = round(float(hit.get("score", 0.0)) + boost, 4)
|
| 842 |
|
| 843 |
ranked.sort(
|
| 844 |
key=lambda x: (
|
| 845 |
x.get("rank_score", 0.0),
|
| 846 |
x.get("score", 0.0),
|
|
|
|
| 847 |
-int(x.get("chunk_index", 0)),
|
| 848 |
),
|
| 849 |
reverse=True,
|
|
@@ -852,11 +1544,7 @@ class LegalRetriever:
|
|
| 852 |
return ranked
|
| 853 |
|
| 854 |
@staticmethod
|
| 855 |
-
def _limit_chunks_per_section(
|
| 856 |
-
results: List[Dict[str, Any]],
|
| 857 |
-
*,
|
| 858 |
-
max_chunks_per_section: int,
|
| 859 |
-
) -> List[Dict[str, Any]]:
|
| 860 |
if max_chunks_per_section <= 0:
|
| 861 |
return results
|
| 862 |
|
|
@@ -864,10 +1552,166 @@ class LegalRetriever:
|
|
| 864 |
limited: List[Dict[str, Any]] = []
|
| 865 |
|
| 866 |
for hit in results:
|
| 867 |
-
|
|
|
|
|
|
|
| 868 |
if counts[key] >= max_chunks_per_section:
|
| 869 |
continue
|
| 870 |
counts[key] += 1
|
| 871 |
limited.append(hit)
|
| 872 |
|
| 873 |
return limited
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
from __future__ import annotations
|
| 2 |
|
| 3 |
import logging
|
| 4 |
+
import math
|
| 5 |
import re
|
| 6 |
+
from collections import Counter, defaultdict
|
| 7 |
+
from dataclasses import dataclass
|
| 8 |
+
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
|
| 9 |
|
| 10 |
import chromadb
|
| 11 |
from sentence_transformers import SentenceTransformer
|
|
|
|
| 13 |
logger = logging.getLogger(__name__)
|
| 14 |
|
| 15 |
|
| 16 |
+
@dataclass(frozen=True, slots=True)
|
| 17 |
+
class NormReference:
|
| 18 |
+
"""Structured representation of a legal reference found in a user question."""
|
| 19 |
+
|
| 20 |
+
paragraph: str
|
| 21 |
+
subsection: str | None = None
|
| 22 |
+
sentence: str | None = None
|
| 23 |
+
number: str | None = None
|
| 24 |
+
letter: str | None = None
|
| 25 |
+
|
| 26 |
+
@property
|
| 27 |
+
def section_id(self) -> str:
|
| 28 |
+
return self.paragraph
|
| 29 |
+
|
| 30 |
+
@property
|
| 31 |
+
def canonical_ref(self) -> str:
|
| 32 |
+
parts = [self.paragraph]
|
| 33 |
+
if self.subsection:
|
| 34 |
+
parts.append(f"Abs. {self.subsection}")
|
| 35 |
+
if self.sentence:
|
| 36 |
+
parts.append(f"Satz {self.sentence}")
|
| 37 |
+
if self.number:
|
| 38 |
+
parts.append(f"Nr. {self.number}")
|
| 39 |
+
if self.letter:
|
| 40 |
+
parts.append(f"Buchst. {self.letter}")
|
| 41 |
+
return " ".join(parts)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
class LegalRetriever:
|
| 45 |
"""
|
| 46 |
+
Standalone production-oriented retriever for legal RAG applications on Chroma.
|
| 47 |
+
|
| 48 |
+
This class intentionally does not import from an ingestion package. It can live in a
|
| 49 |
+
separate backend/chatbot folder and only depends on Chroma metadata produced during
|
| 50 |
+
ingestion.
|
| 51 |
+
|
| 52 |
+
It is backward-compatible with the previous retriever API while adding support for
|
| 53 |
+
richer legal metadata generated by a parent-child legal chunker.
|
| 54 |
+
|
| 55 |
+
Core capabilities:
|
| 56 |
+
- default filtering to the main contract,
|
| 57 |
+
- semantic/dense retrieval,
|
| 58 |
+
- exact norm lookup for § / Abs. / Satz / Nr. / Buchst.,
|
| 59 |
+
- explicit section lookup for old collections,
|
| 60 |
+
- definition lookup via is_definition / defined_terms / § 2 fallback,
|
| 61 |
+
- BM25-like lexical fallback,
|
| 62 |
+
- optional neighbor chunk expansion,
|
| 63 |
+
- parent-context expansion via parent_unit_id,
|
| 64 |
+
- result fusion, deduplication and ranking,
|
| 65 |
+
- negative-answer verification before saying something is not regulated,
|
| 66 |
+
- source and context formatting for downstream LLM calls.
|
| 67 |
+
|
| 68 |
+
Expected legacy metadata per chunk:
|
| 69 |
+
- container_id, e.g. "Vertrag", "Anlage 4", "Anhang zu Anlage 11"
|
| 70 |
+
- container_type, e.g. "vertrag", "anlage", "anhang"
|
| 71 |
+
- section_id, e.g. "§ 6"
|
| 72 |
- section_path
|
| 73 |
- page_start, page_end
|
| 74 |
- chunk_index_in_section
|
| 75 |
+
- text_hash optional
|
| 76 |
+
|
| 77 |
+
Additional metadata supported from the optimized ingestion/chunking layer:
|
| 78 |
+
- chunk_kind: "parent" | "child" | ...
|
| 79 |
+
- legal_unit_id
|
| 80 |
+
- parent_unit_id
|
| 81 |
+
- canonical_ref
|
| 82 |
+
- paragraph
|
| 83 |
+
- subsection
|
| 84 |
+
- sentence
|
| 85 |
+
- number
|
| 86 |
+
- letter
|
| 87 |
+
- unit_type
|
| 88 |
+
- is_definition
|
| 89 |
+
- defined_terms
|
| 90 |
"""
|
| 91 |
|
| 92 |
SECTION_REF_RE = re.compile(
|
|
|
|
| 95 |
re.IGNORECASE,
|
| 96 |
)
|
| 97 |
|
| 98 |
+
NORM_REF_RE = re.compile(
|
| 99 |
+
r"§{1,2}\s*(?P<para>\d+[a-zA-Z]?)"
|
| 100 |
+
r"(?:\s*(?:Abs\.?|Absatz)\s*(?P<abs>\d+[a-zA-Z]?))?"
|
| 101 |
+
r"(?:\s*Satz\s*(?P<satz>\d+[a-zA-Z]?))?"
|
| 102 |
+
r"(?:\s*(?:Nr\.?|Nummer)\s*(?P<nr>\d+[a-zA-Z]?))?"
|
| 103 |
+
r"(?:\s*(?:Buchst\.?|Buchstabe|lit\.?)\s*(?P<letter>[a-zA-Z]))?",
|
| 104 |
+
re.IGNORECASE,
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
QUOTED_TERM_RE = re.compile(r"[„\"']([^„“\"']{2,120})[“\"']")
|
| 108 |
+
|
| 109 |
+
STOPWORDS = {
|
| 110 |
+
"aber",
|
| 111 |
+
"alle",
|
| 112 |
+
"alles",
|
| 113 |
+
"als",
|
| 114 |
+
"also",
|
| 115 |
+
"am",
|
| 116 |
+
"an",
|
| 117 |
+
"auch",
|
| 118 |
+
"auf",
|
| 119 |
+
"aus",
|
| 120 |
+
"bei",
|
| 121 |
+
"bis",
|
| 122 |
+
"da",
|
| 123 |
+
"das",
|
| 124 |
+
"dass",
|
| 125 |
+
"dem",
|
| 126 |
+
"den",
|
| 127 |
+
"der",
|
| 128 |
+
"des",
|
| 129 |
+
"die",
|
| 130 |
+
"dies",
|
| 131 |
+
"diese",
|
| 132 |
+
"dieser",
|
| 133 |
+
"dieses",
|
| 134 |
+
"ein",
|
| 135 |
+
"eine",
|
| 136 |
+
"einem",
|
| 137 |
+
"einen",
|
| 138 |
+
"einer",
|
| 139 |
+
"eines",
|
| 140 |
+
"er",
|
| 141 |
+
"es",
|
| 142 |
+
"für",
|
| 143 |
+
"gilt",
|
| 144 |
+
"hat",
|
| 145 |
+
"im",
|
| 146 |
+
"in",
|
| 147 |
+
"ist",
|
| 148 |
+
"kann",
|
| 149 |
+
"mit",
|
| 150 |
+
"nach",
|
| 151 |
+
"oder",
|
| 152 |
+
"rahmenvertrag",
|
| 153 |
+
"regelt",
|
| 154 |
+
"sagt",
|
| 155 |
+
"sind",
|
| 156 |
+
"unter",
|
| 157 |
+
"und",
|
| 158 |
+
"von",
|
| 159 |
+
"wann",
|
| 160 |
+
"was",
|
| 161 |
+
"welche",
|
| 162 |
+
"welchen",
|
| 163 |
+
"welcher",
|
| 164 |
+
"welches",
|
| 165 |
+
"wenn",
|
| 166 |
+
"wer",
|
| 167 |
+
"wie",
|
| 168 |
+
"wird",
|
| 169 |
+
"wo",
|
| 170 |
+
"zum",
|
| 171 |
+
"zur",
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
QUERY_EXPANSIONS: Dict[str, List[str]] = {
|
| 175 |
+
"nicht verfügbar": [
|
| 176 |
+
"nicht lieferbar",
|
| 177 |
+
"nicht vorrätig",
|
| 178 |
+
"Lieferengpass",
|
| 179 |
+
"Verfügbarkeit",
|
| 180 |
+
"lieferfähig",
|
| 181 |
+
"lieferbar",
|
| 182 |
+
],
|
| 183 |
+
"nicht lieferbar": ["nicht verfügbar", "Lieferengpass", "lieferfähig", "Verfügbarkeit"],
|
| 184 |
+
"lieferengpass": ["nicht verfügbar", "nicht lieferbar", "Verfügbarkeit", "lieferfähig"],
|
| 185 |
+
"auseinzelung": ["Teilmenge", "Auseinzelung", "aus Packungen entnehmen", "§ 16"],
|
| 186 |
+
"teilmenge": ["Auseinzelung", "Teilmenge", "§ 16"],
|
| 187 |
+
"beitritt": ["beitreten", "teilnehmen", "Mitgliedsverband", "DAV", "Erklärung", "§ 4"],
|
| 188 |
+
"teilnahme": ["Beitritt", "teilnehmen", "Mitgliedsverband", "DAV", "§ 4"],
|
| 189 |
+
"wunscharzneimittel": ["Wunscharzneimittel", "Kostenerstattung", "anderes Fertigarzneimittel", "§ 15"],
|
| 190 |
+
"pharmazeutische dienstleistungen": ["pharmazeutische Dienstleistung", "Anlage 11", "§ 33"],
|
| 191 |
+
"biosimilar": ["Biosimilar", "biotechnologisch", "Referenzarzneimittel"],
|
| 192 |
+
"bioidentical": ["Bioidentical", "Ausgangsstoff", "Herstellungsprozess"],
|
| 193 |
+
"importarzneimittel": ["Importarzneimittel", "Parallelimport", "Reimport", "Referenzarzneimittel"],
|
| 194 |
+
"rabattvertrag": ["Rabattvertrag", "rabattbegünstigt", "§ 11", "§ 130a"],
|
| 195 |
+
}
|
| 196 |
+
|
| 197 |
def __init__(
|
| 198 |
self,
|
| 199 |
persist_dir: str,
|
| 200 |
collection: str,
|
| 201 |
+
model_name: str = "auto",
|
| 202 |
*,
|
| 203 |
default_container_id: str = "Vertrag",
|
| 204 |
normalize_embeddings: bool = True,
|
| 205 |
+
default_to_contract: bool = False,
|
| 206 |
min_score: float = 0.20,
|
| 207 |
verbose_startup: bool = True,
|
| 208 |
+
lexical_scan_limit: int = 5000,
|
| 209 |
+
query_prefix: str | None = None,
|
| 210 |
+
enable_reranker: bool = False,
|
| 211 |
+
reranker_model: str = "cross-encoder/mmarco-mMiniLMv2-L12-H384-v1",
|
| 212 |
+
reranker_candidates: int = 20,
|
| 213 |
):
|
| 214 |
self.persist_dir = persist_dir
|
| 215 |
self.collection_name = collection
|
|
|
|
| 217 |
self.normalize_embeddings = normalize_embeddings
|
| 218 |
self.default_to_contract = default_to_contract
|
| 219 |
self.min_score = float(min_score)
|
| 220 |
+
self.lexical_scan_limit = int(max(100, lexical_scan_limit))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
|
| 222 |
self.client = chromadb.PersistentClient(path=persist_dir)
|
| 223 |
|
|
|
|
| 231 |
f"verfügbare Collections={available!r}"
|
| 232 |
) from exc
|
| 233 |
|
| 234 |
+
# ------------------------------------------------------------------
|
| 235 |
+
# Model/collection contract: the ingestion pipeline writes the exact
|
| 236 |
+
# embedding model and dimension into the collection metadata. Reading
|
| 237 |
+
# it here removes the classic failure mode where the query side embeds
|
| 238 |
+
# with a different model than the index (silently broken ranking or a
|
| 239 |
+
# hard dimension mismatch deep inside Chroma).
|
| 240 |
+
# ------------------------------------------------------------------
|
| 241 |
+
collection_meta = dict(getattr(self.col, "metadata", None) or {})
|
| 242 |
+
indexed_model = str(collection_meta.get("embedding_model") or "").strip()
|
| 243 |
+
|
| 244 |
+
requested = (model_name or "").strip()
|
| 245 |
+
if not requested or requested.lower() in {"auto", "collection"}:
|
| 246 |
+
if not indexed_model:
|
| 247 |
+
raise RuntimeError(
|
| 248 |
+
"EMBEDDING_MODEL='auto' verlangt, dass die Collection-Metadata "
|
| 249 |
+
"'embedding_model' enthält. Diese Collection wurde offenbar mit "
|
| 250 |
+
"einer älteren Ingest-Version gebaut; setze EMBEDDING_MODEL explizit."
|
| 251 |
+
)
|
| 252 |
+
model_name = indexed_model
|
| 253 |
+
elif indexed_model and indexed_model != requested:
|
| 254 |
+
raise RuntimeError(
|
| 255 |
+
"Embedding-Modell passt nicht zur Collection: "
|
| 256 |
+
f"konfiguriert={requested!r}, Collection wurde indexiert mit {indexed_model!r}. "
|
| 257 |
+
"Entweder EMBEDDING_MODEL='auto' setzen oder die Collection neu ingestieren."
|
| 258 |
+
)
|
| 259 |
+
|
| 260 |
if verbose_startup:
|
| 261 |
self._log_startup_diagnostics()
|
| 262 |
|
| 263 |
+
self.model_name = model_name
|
| 264 |
self.embedder = SentenceTransformer(model_name)
|
| 265 |
|
| 266 |
+
expected_dim = collection_meta.get("embedding_dim")
|
| 267 |
+
actual_dim = self.embedder.get_sentence_embedding_dimension()
|
| 268 |
+
if expected_dim and actual_dim and int(expected_dim) > 0 and int(expected_dim) != int(actual_dim):
|
| 269 |
+
raise RuntimeError(
|
| 270 |
+
"Embedding-Dimension passt nicht zur Collection: "
|
| 271 |
+
f"Modell {model_name!r} liefert {actual_dim}, Collection erwartet {expected_dim}."
|
| 272 |
+
)
|
| 273 |
+
|
| 274 |
+
# E5-style models are trained with asymmetric prefixes. Their
|
| 275 |
+
# sentence-transformers configs register EMPTY prompts, so
|
| 276 |
+
# encode_query() does NOT add the prefix automatically — it must be
|
| 277 |
+
# applied here.
|
| 278 |
+
if query_prefix is None:
|
| 279 |
+
query_prefix = "query: " if "e5" in model_name.lower() else ""
|
| 280 |
+
self.query_prefix = query_prefix
|
| 281 |
+
|
| 282 |
+
self.enable_reranker = bool(enable_reranker)
|
| 283 |
+
self.reranker_model = reranker_model
|
| 284 |
+
self.reranker_candidates = int(max(1, reranker_candidates))
|
| 285 |
+
self._reranker: Any = None
|
| 286 |
+
|
| 287 |
# ------------------------------------------------------------------
|
| 288 |
# Diagnostics
|
| 289 |
# ------------------------------------------------------------------
|
|
|
|
| 305 |
)
|
| 306 |
|
| 307 |
def diagnostics(self, *, sample: int = 3) -> Dict[str, Any]:
|
| 308 |
+
"""Return compact runtime diagnostics to catch stale paths or collections."""
|
|
|
|
|
|
|
|
|
|
| 309 |
collections = [c.name for c in self.client.list_collections()]
|
| 310 |
result: Dict[str, Any] = {
|
| 311 |
"persist_dir": self.persist_dir,
|
|
|
|
| 317 |
try:
|
| 318 |
res = self.col.get(limit=sample, include=["metadatas"])
|
| 319 |
result["metadata_sample"] = res.get("metadatas", [])
|
| 320 |
+
result["metadata_keys"] = sorted(
|
| 321 |
+
{key for meta in result["metadata_sample"] if isinstance(meta, dict) for key in meta.keys()}
|
| 322 |
+
)
|
| 323 |
+
except Exception as exc: # noqa: BLE001 - diagnostics must not crash the app.
|
| 324 |
result["metadata_sample_error"] = repr(exc)
|
| 325 |
|
| 326 |
return result
|
|
|
|
| 331 |
@staticmethod
|
| 332 |
def _build_where(filters: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
| 333 |
"""
|
| 334 |
+
Build a Chroma-compatible where filter.
|
| 335 |
|
| 336 |
+
Chroma permits only one top-level logical operator for compound filters.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 337 |
"""
|
| 338 |
if not filters:
|
| 339 |
return None
|
|
|
|
| 368 |
return parts[0]
|
| 369 |
return {"$and": parts}
|
| 370 |
|
| 371 |
+
@staticmethod
|
| 372 |
+
def _where_or(*conditions: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
| 373 |
+
parts: List[Dict[str, Any]] = [condition for condition in conditions if condition]
|
| 374 |
+
if not parts:
|
| 375 |
+
return None
|
| 376 |
+
if len(parts) == 1:
|
| 377 |
+
return parts[0]
|
| 378 |
+
return {"$or": parts}
|
| 379 |
+
|
| 380 |
def _effective_where(
|
| 381 |
self,
|
| 382 |
where: Optional[Dict[str, Any]],
|
|
|
|
| 384 |
restrict_to_default_container: Optional[bool],
|
| 385 |
) -> Optional[Dict[str, Any]]:
|
| 386 |
"""
|
| 387 |
+
By default, search the main contract. To search all containers intentionally,
|
| 388 |
+
call query(..., restrict_to_default_container=False).
|
| 389 |
"""
|
| 390 |
if where:
|
| 391 |
return where
|
|
|
|
| 397 |
return None
|
| 398 |
|
| 399 |
# ------------------------------------------------------------------
|
| 400 |
+
# Paragraph and norm references
|
| 401 |
# ------------------------------------------------------------------
|
| 402 |
@classmethod
|
| 403 |
def _parse_section_refs(cls, text: str, *, max_range: int = 30) -> List[str]:
|
| 404 |
"""
|
| 405 |
+
Recognize paragraph references such as § 6, §§ 7-14, §§ 7 bis 14.
|
| 406 |
+
Returns normalized section_ids such as ["§ 6", "§ 7", ...].
|
|
|
|
|
|
|
|
|
|
|
|
|
| 407 |
"""
|
| 408 |
if not text:
|
| 409 |
return []
|
|
|
|
| 427 |
|
| 428 |
return list(dict.fromkeys(found))
|
| 429 |
|
| 430 |
+
@classmethod
|
| 431 |
+
def _parse_norm_refs(cls, text: str) -> List[NormReference]:
|
| 432 |
+
"""Recognize structured references including Abs., Satz, Nr. and Buchst."""
|
| 433 |
+
if not text:
|
| 434 |
+
return []
|
| 435 |
+
|
| 436 |
+
refs: List[NormReference] = []
|
| 437 |
+
for match in cls.NORM_REF_RE.finditer(text):
|
| 438 |
+
para = match.group("para")
|
| 439 |
+
if not para:
|
| 440 |
+
continue
|
| 441 |
+
ref = NormReference(
|
| 442 |
+
paragraph=f"§ {para}",
|
| 443 |
+
subsection=match.group("abs"),
|
| 444 |
+
sentence=match.group("satz"),
|
| 445 |
+
number=match.group("nr"),
|
| 446 |
+
letter=(match.group("letter") or "").lower() or None,
|
| 447 |
+
)
|
| 448 |
+
refs.append(ref)
|
| 449 |
+
|
| 450 |
+
seen: Set[str] = set()
|
| 451 |
+
unique: List[NormReference] = []
|
| 452 |
+
for ref in refs:
|
| 453 |
+
key = ref.canonical_ref.lower()
|
| 454 |
+
if key not in seen:
|
| 455 |
+
unique.append(ref)
|
| 456 |
+
seen.add(key)
|
| 457 |
+
return unique
|
| 458 |
+
|
| 459 |
@staticmethod
|
| 460 |
def _section_variants(section: str) -> List[str]:
|
| 461 |
if not section:
|
|
|
|
| 470 |
|
| 471 |
return list(dict.fromkeys(variants))
|
| 472 |
|
| 473 |
+
@staticmethod
|
| 474 |
+
def _paragraph_variants(paragraph: str) -> List[str]:
|
| 475 |
+
return LegalRetriever._section_variants(paragraph)
|
| 476 |
+
|
| 477 |
+
@staticmethod
|
| 478 |
+
def _subsection_variants(subsection: str | None) -> List[str]:
|
| 479 |
+
if not subsection:
|
| 480 |
+
return []
|
| 481 |
+
s = str(subsection).strip()
|
| 482 |
+
return list(dict.fromkeys([s, f"Abs. {s}", f"Absatz {s}", f"({s})"]))
|
| 483 |
+
|
| 484 |
+
@staticmethod
|
| 485 |
+
def _sentence_variants(sentence: str | None) -> List[str]:
|
| 486 |
+
if not sentence:
|
| 487 |
+
return []
|
| 488 |
+
s = str(sentence).strip()
|
| 489 |
+
return list(dict.fromkeys([s, f"Satz {s}"]))
|
| 490 |
+
|
| 491 |
+
@staticmethod
|
| 492 |
+
def _number_variants(number: str | None) -> List[str]:
|
| 493 |
+
if not number:
|
| 494 |
+
return []
|
| 495 |
+
s = str(number).strip()
|
| 496 |
+
return list(dict.fromkeys([s, f"Nr. {s}", f"Nummer {s}"]))
|
| 497 |
+
|
| 498 |
+
@staticmethod
|
| 499 |
+
def _letter_variants(letter: str | None) -> List[str]:
|
| 500 |
+
if not letter:
|
| 501 |
+
return []
|
| 502 |
+
s = str(letter).strip().lower()
|
| 503 |
+
return list(dict.fromkeys([s, f"Buchst. {s}", f"Buchstabe {s}", f"{s})"]))
|
| 504 |
+
|
| 505 |
# ------------------------------------------------------------------
|
| 506 |
# Public API
|
| 507 |
# ------------------------------------------------------------------
|
|
|
|
| 515 |
include_explicit_sections: bool = True,
|
| 516 |
explicit_sections: Optional[List[str]] = None,
|
| 517 |
explicit_section_container_ids: Optional[List[str]] = None,
|
| 518 |
+
max_chunks_per_explicit_section: int = 6,
|
| 519 |
+
include_definitions: bool = True,
|
| 520 |
+
definition_k: int = 8,
|
| 521 |
+
include_lexical: bool = True,
|
| 522 |
+
lexical_k: int = 12,
|
| 523 |
+
lexical_scan_limit: Optional[int] = None,
|
| 524 |
+
include_parent_context: bool = True,
|
| 525 |
+
max_parent_contexts: int = 8,
|
| 526 |
include_neighbors: bool = True,
|
| 527 |
neighbor_window: int = 1,
|
| 528 |
+
max_final_results: Optional[int] = 12,
|
| 529 |
min_score: Optional[float] = None,
|
| 530 |
restrict_to_default_container: Optional[bool] = None,
|
| 531 |
+
max_chunks_per_section: int = 5,
|
| 532 |
+
expand_query: bool = True,
|
| 533 |
+
verify_negative_answer: bool = False,
|
| 534 |
+
rerank: Optional[bool] = None,
|
| 535 |
) -> List[Dict[str, Any]]:
|
| 536 |
"""
|
| 537 |
+
Hybrid legal retrieval.
|
| 538 |
+
|
| 539 |
+
The old behavior is preserved and expanded:
|
| 540 |
+
- semantic query,
|
| 541 |
+
- exact norm/section lookup,
|
| 542 |
+
- optional neighbors,
|
| 543 |
+
- section limits.
|
| 544 |
+
|
| 545 |
+
New paths:
|
| 546 |
+
- definitions,
|
| 547 |
+
- lexical fallback,
|
| 548 |
+
- parent context expansion,
|
| 549 |
+
- negative answer verification.
|
| 550 |
"""
|
| 551 |
if not question or not question.strip():
|
| 552 |
return []
|
|
|
|
| 555 |
where,
|
| 556 |
restrict_to_default_container=restrict_to_default_container,
|
| 557 |
)
|
|
|
|
| 558 |
threshold = self.min_score if min_score is None else float(min_score)
|
| 559 |
+
intent = self.classify_query_intent(question)
|
| 560 |
+
expanded_terms = self.expand_query_terms(question) if expand_query else []
|
| 561 |
|
| 562 |
+
all_results: List[Dict[str, Any]] = []
|
| 563 |
+
|
| 564 |
+
# 1) Semantic dense retrieval.
|
| 565 |
semantic_results = self._semantic_query(
|
| 566 |
question=question,
|
| 567 |
top_k=top_k,
|
|
|
|
| 569 |
fetch_k=fetch_k,
|
| 570 |
)
|
| 571 |
semantic_results = [hit for hit in semantic_results if float(hit.get("score", 0.0)) >= threshold]
|
|
|
|
|
|
|
| 572 |
all_results.extend(semantic_results)
|
| 573 |
|
| 574 |
+
# 2) Explicit norm / section lookup.
|
| 575 |
if include_explicit_sections:
|
| 576 |
+
norm_refs = self._parse_norm_refs(question)
|
| 577 |
sections = list(explicit_sections or [])
|
| 578 |
sections.extend(self._parse_section_refs(question))
|
| 579 |
sections = list(dict.fromkeys(sections))
|
| 580 |
+
containers = explicit_section_container_ids or [self.default_container_id]
|
| 581 |
+
|
| 582 |
+
if norm_refs:
|
| 583 |
+
all_results.extend(
|
| 584 |
+
self._get_explicit_norms(
|
| 585 |
+
refs=norm_refs,
|
| 586 |
+
container_ids=containers,
|
| 587 |
+
base_where=effective_where,
|
| 588 |
+
max_chunks_per_ref=max_chunks_per_explicit_section,
|
| 589 |
+
)
|
| 590 |
+
)
|
| 591 |
+
elif sections:
|
| 592 |
+
all_results.extend(
|
| 593 |
+
self._get_explicit_sections(
|
| 594 |
+
sections=sections,
|
| 595 |
+
container_ids=containers,
|
| 596 |
+
base_where=effective_where,
|
| 597 |
+
max_chunks_per_section=max_chunks_per_explicit_section,
|
| 598 |
+
)
|
| 599 |
+
)
|
| 600 |
|
| 601 |
+
# 3) Definition lookup for definition-like questions or quoted terms.
|
| 602 |
+
if include_definitions and (intent == "definition" or self._extract_definition_terms(question)):
|
| 603 |
+
all_results.extend(
|
| 604 |
+
self._definition_search(
|
| 605 |
+
question=question,
|
| 606 |
base_where=effective_where,
|
| 607 |
+
top_k=definition_k,
|
| 608 |
+
expanded_terms=expanded_terms,
|
| 609 |
)
|
| 610 |
+
)
|
| 611 |
+
|
| 612 |
+
# 4) Lexical fallback. This is intentionally independent from embeddings.
|
| 613 |
+
if include_lexical:
|
| 614 |
+
all_results.extend(
|
| 615 |
+
self._lexical_search(
|
| 616 |
+
question=question,
|
| 617 |
+
base_where=effective_where,
|
| 618 |
+
top_k=lexical_k,
|
| 619 |
+
expanded_terms=expanded_terms,
|
| 620 |
+
scan_limit=lexical_scan_limit or self.lexical_scan_limit,
|
| 621 |
+
)
|
| 622 |
+
)
|
| 623 |
|
| 624 |
+
# 5) Neighbor chunks as context. Useful for old collections without parent-child chunks.
|
| 625 |
if include_neighbors and neighbor_window > 0:
|
| 626 |
neighbor_results: List[Dict[str, Any]] = []
|
| 627 |
+
seed_hits = self._dedupe_and_rank(all_results)[: max(top_k, 1)]
|
| 628 |
+
for hit in seed_hits:
|
| 629 |
neighbor_results.extend(
|
| 630 |
self._get_neighbors(hit, window=neighbor_window, base_where=effective_where)
|
| 631 |
)
|
| 632 |
all_results.extend(neighbor_results)
|
| 633 |
|
| 634 |
+
# 6) Parent context expansion. Useful for new parent-child legal chunks.
|
| 635 |
+
if include_parent_context:
|
| 636 |
+
parent_results = self._expand_parent_context(
|
| 637 |
+
all_results,
|
| 638 |
+
base_where=effective_where,
|
| 639 |
+
max_parent_contexts=max_parent_contexts,
|
| 640 |
+
)
|
| 641 |
+
all_results.extend(parent_results)
|
| 642 |
+
|
| 643 |
+
merged = self._dedupe_and_rank(all_results, question=question, intent=intent)
|
| 644 |
+
|
| 645 |
+
use_reranker = self.enable_reranker if rerank is None else bool(rerank)
|
| 646 |
+
if use_reranker:
|
| 647 |
+
merged = self._rerank(question, merged)
|
| 648 |
+
|
| 649 |
merged = self._limit_chunks_per_section(merged, max_chunks_per_section=max_chunks_per_section)
|
| 650 |
|
| 651 |
if max_final_results is not None:
|
| 652 |
merged = merged[:max_final_results]
|
| 653 |
|
| 654 |
+
# 7) Optional negative-answer verification. If the normal search returns weak/no results,
|
| 655 |
+
# run a broad second pass across all containers before the calling layer says "not regulated".
|
| 656 |
+
if verify_negative_answer and self._looks_like_negative_risk(question, merged):
|
| 657 |
+
check = self.verify_negative_result(question, max_results=max_final_results or 12)
|
| 658 |
+
if not check.get("safe_to_answer_negative") and check.get("results"):
|
| 659 |
+
merged = check["results"]
|
| 660 |
+
|
| 661 |
return merged
|
| 662 |
|
| 663 |
+
def query_paragraph(
|
| 664 |
+
self,
|
| 665 |
+
question: str,
|
| 666 |
+
paragraph: str,
|
| 667 |
+
container: str = "Vertrag",
|
| 668 |
+
top_k: int = 6,
|
| 669 |
+
**kwargs: Any,
|
| 670 |
+
) -> List[Dict[str, Any]]:
|
| 671 |
+
return self.query(
|
| 672 |
+
question,
|
| 673 |
+
top_k=top_k,
|
| 674 |
+
where={
|
| 675 |
+
"container_id": container,
|
| 676 |
+
"section_id": paragraph,
|
| 677 |
+
},
|
| 678 |
+
include_explicit_sections=False,
|
| 679 |
+
**kwargs,
|
| 680 |
+
)
|
| 681 |
+
|
| 682 |
+
def query_anlage(
|
| 683 |
+
self,
|
| 684 |
+
question: str,
|
| 685 |
+
anlage_nr: int,
|
| 686 |
+
paragraph: Optional[str] = None,
|
| 687 |
+
top_k: int = 6,
|
| 688 |
+
**kwargs: Any,
|
| 689 |
+
) -> List[Dict[str, Any]]:
|
| 690 |
+
where: Dict[str, Any] = {"container_id": f"Anlage {anlage_nr}"}
|
| 691 |
+
if paragraph:
|
| 692 |
+
where["section_id"] = paragraph
|
| 693 |
+
|
| 694 |
+
return self.query(question, top_k=top_k, where=where, **kwargs)
|
| 695 |
+
|
| 696 |
+
def query_anhang(
|
| 697 |
+
self,
|
| 698 |
+
question: str,
|
| 699 |
+
anlage_nr: int,
|
| 700 |
+
top_k: int = 6,
|
| 701 |
+
**kwargs: Any,
|
| 702 |
+
) -> List[Dict[str, Any]]:
|
| 703 |
+
return self.query(
|
| 704 |
+
question,
|
| 705 |
+
top_k=top_k,
|
| 706 |
+
where={
|
| 707 |
+
"container_type": "anhang",
|
| 708 |
+
"container_id": f"Anhang zu Anlage {anlage_nr}",
|
| 709 |
+
},
|
| 710 |
+
**kwargs,
|
| 711 |
+
)
|
| 712 |
+
|
| 713 |
+
def get_section(
|
| 714 |
+
self,
|
| 715 |
+
section: str,
|
| 716 |
+
*,
|
| 717 |
+
container: str = "Vertrag",
|
| 718 |
+
max_chunks: int = 20,
|
| 719 |
+
include_parent_context: bool = False,
|
| 720 |
+
) -> List[Dict[str, Any]]:
|
| 721 |
+
variants = self._section_variants(section)
|
| 722 |
+
where = {"container_id": container, "section_id": {"$in": variants}}
|
| 723 |
+
|
| 724 |
+
try:
|
| 725 |
+
res = self.col.get(where=self._build_where(where), include=["documents", "metadatas"])
|
| 726 |
+
except Exception:
|
| 727 |
+
return []
|
| 728 |
+
|
| 729 |
+
formatted = self._format_get(res, retrieval_kind="section_lookup", score=1.0)
|
| 730 |
+
formatted.sort(key=lambda x: x.get("chunk_index", 0))
|
| 731 |
+
out = formatted[:max_chunks]
|
| 732 |
+
|
| 733 |
+
if include_parent_context:
|
| 734 |
+
out.extend(self._expand_parent_context(out, base_where={"container_id": container}))
|
| 735 |
+
out = self._dedupe_and_rank(out)
|
| 736 |
+
|
| 737 |
+
return out[:max_chunks]
|
| 738 |
+
|
| 739 |
+
def verify_negative_result(
|
| 740 |
+
self,
|
| 741 |
+
question: str,
|
| 742 |
+
*,
|
| 743 |
+
max_results: int = 12,
|
| 744 |
+
strong_score: float = 0.55,
|
| 745 |
+
) -> Dict[str, Any]:
|
| 746 |
+
"""
|
| 747 |
+
Run a broad second-pass search before the answer layer says that something
|
| 748 |
+
is not regulated or not found.
|
| 749 |
+
|
| 750 |
+
Returns:
|
| 751 |
+
{
|
| 752 |
+
"safe_to_answer_negative": bool,
|
| 753 |
+
"reason": str,
|
| 754 |
+
"results": list[dict],
|
| 755 |
+
"strong_result_count": int,
|
| 756 |
+
}
|
| 757 |
+
"""
|
| 758 |
+
results = self.query(
|
| 759 |
+
question,
|
| 760 |
+
top_k=max(10, max_results),
|
| 761 |
+
fetch_k=max(40, max_results * 4),
|
| 762 |
+
include_explicit_sections=True,
|
| 763 |
+
include_definitions=True,
|
| 764 |
+
include_lexical=True,
|
| 765 |
+
lexical_k=max(20, max_results * 2),
|
| 766 |
+
include_parent_context=True,
|
| 767 |
+
include_neighbors=True,
|
| 768 |
+
neighbor_window=1,
|
| 769 |
+
restrict_to_default_container=False,
|
| 770 |
+
min_score=0.0,
|
| 771 |
+
max_final_results=max_results,
|
| 772 |
+
max_chunks_per_section=6,
|
| 773 |
+
verify_negative_answer=False,
|
| 774 |
+
)
|
| 775 |
+
|
| 776 |
+
strong_kinds = {"exact_norm", "explicit_section", "definition", "lexical", "parent_context"}
|
| 777 |
+
strong = [
|
| 778 |
+
hit
|
| 779 |
+
for hit in results
|
| 780 |
+
if float(hit.get("rank_score", hit.get("score", 0.0))) >= strong_score
|
| 781 |
+
or bool(strong_kinds.intersection(set(hit.get("retrieval_kinds", []))))
|
| 782 |
+
]
|
| 783 |
+
|
| 784 |
+
if strong:
|
| 785 |
+
return {
|
| 786 |
+
"safe_to_answer_negative": False,
|
| 787 |
+
"reason": "Der breite Kontrollabruf hat potenziell relevante Regelungen gefunden.",
|
| 788 |
+
"results": results,
|
| 789 |
+
"strong_result_count": len(strong),
|
| 790 |
+
}
|
| 791 |
+
|
| 792 |
+
return {
|
| 793 |
+
"safe_to_answer_negative": True,
|
| 794 |
+
"reason": "Auch der breite Kontrollabruf hat keine belastbaren Treffer gefunden.",
|
| 795 |
+
"results": results,
|
| 796 |
+
"strong_result_count": 0,
|
| 797 |
+
}
|
| 798 |
+
|
| 799 |
+
# ------------------------------------------------------------------
|
| 800 |
+
# Intent and query expansion
|
| 801 |
+
# ------------------------------------------------------------------
|
| 802 |
+
@classmethod
|
| 803 |
+
def classify_query_intent(cls, question: str) -> str:
|
| 804 |
+
q = cls._norm_text(question)
|
| 805 |
+
if re.search(r"\b(was\s+versteht|wie\s+definiert|definition|legaldefinition|begriff|bedeutet)\b", q):
|
| 806 |
+
return "definition"
|
| 807 |
+
if cls._parse_norm_refs(question):
|
| 808 |
+
return "norm_lookup"
|
| 809 |
+
if re.search(r"\b(welche|voraussetzungen|kriterien|tatbestandsmerkmale|nennt|liste|auflistung)\b", q):
|
| 810 |
+
return "enumeration"
|
| 811 |
+
if re.search(r"\b(nicht\s+geregelt|keine\s+regelung|steht\s+nicht|nicht\s+enthalten)\b", q):
|
| 812 |
+
return "negative_check"
|
| 813 |
+
return "semantic"
|
| 814 |
+
|
| 815 |
+
@classmethod
|
| 816 |
+
def expand_query_terms(cls, question: str) -> List[str]:
|
| 817 |
+
q = cls._norm_text(question)
|
| 818 |
+
expansions: List[str] = []
|
| 819 |
+
for key, values in cls.QUERY_EXPANSIONS.items():
|
| 820 |
+
if key in q:
|
| 821 |
+
expansions.extend(values)
|
| 822 |
+
# Add quoted terms verbatim because legal definition questions often quote exact terms.
|
| 823 |
+
expansions.extend(cls._extract_definition_terms(question))
|
| 824 |
+
return list(dict.fromkeys([x.strip() for x in expansions if x and x.strip()]))
|
| 825 |
+
|
| 826 |
+
@classmethod
|
| 827 |
+
def _extract_definition_terms(cls, question: str) -> List[str]:
|
| 828 |
+
terms: List[str] = []
|
| 829 |
+
for match in cls.QUOTED_TERM_RE.finditer(question or ""):
|
| 830 |
+
term = match.group(1).strip()
|
| 831 |
+
if term:
|
| 832 |
+
terms.append(term)
|
| 833 |
+
|
| 834 |
+
q = (question or "").strip()
|
| 835 |
+
patterns = [
|
| 836 |
+
r"unter\s+(.+?)(?:\?|$)",
|
| 837 |
+
r"begriff\s+(.+?)(?:\?|$)",
|
| 838 |
+
r"bedeutet\s+(.+?)(?:\?|$)",
|
| 839 |
+
r"definiert\s+(?:der\s+vertrag\s+|der\s+rahmenvertrag\s+)?(?:ein\s+|eine\s+|einen\s+|das\s+|den\s+|die\s+)?(.+?)(?:\?|$)",
|
| 840 |
+
]
|
| 841 |
+
for pattern in patterns:
|
| 842 |
+
m = re.search(pattern, q, flags=re.I)
|
| 843 |
+
if not m:
|
| 844 |
+
continue
|
| 845 |
+
raw = m.group(1)
|
| 846 |
+
raw = re.sub(r"\b(im|in|nach|des|der|die|das|ein|eine|einen|rahmenvertrag|vertrag)\b", " ", raw, flags=re.I)
|
| 847 |
+
raw = re.sub(r"\s+", " ", raw).strip(" .,:;!?\"'„“")
|
| 848 |
+
if 2 <= len(raw) <= 80:
|
| 849 |
+
terms.append(raw)
|
| 850 |
+
|
| 851 |
+
return list(dict.fromkeys(terms))
|
| 852 |
+
|
| 853 |
+
# ------------------------------------------------------------------
|
| 854 |
+
# Retrieval paths
|
| 855 |
+
# ------------------------------------------------------------------
|
| 856 |
def _semantic_query(
|
| 857 |
self,
|
| 858 |
*,
|
|
|
|
| 863 |
) -> List[Dict[str, Any]]:
|
| 864 |
n_results = fetch_k if fetch_k is not None else max(top_k * 4, top_k)
|
| 865 |
|
| 866 |
+
query_text = question
|
| 867 |
+
if self.query_prefix and not question.startswith(self.query_prefix):
|
| 868 |
+
query_text = f"{self.query_prefix}{question}"
|
| 869 |
+
|
| 870 |
if hasattr(self.embedder, "encode_query"):
|
| 871 |
+
q_emb = self.embedder.encode_query([query_text], normalize_embeddings=self.normalize_embeddings)[0].tolist()
|
|
|
|
|
|
|
|
|
|
| 872 |
else:
|
| 873 |
+
q_emb = self.embedder.encode([query_text], normalize_embeddings=self.normalize_embeddings)[0].tolist()
|
|
|
|
|
|
|
|
|
|
| 874 |
|
| 875 |
chroma_where = self._build_where(where)
|
| 876 |
|
|
|
|
| 883 |
|
| 884 |
return self._format(res, retrieval_kind="semantic")[:n_results]
|
| 885 |
|
| 886 |
+
def _get_explicit_norms(
|
| 887 |
+
self,
|
| 888 |
+
*,
|
| 889 |
+
refs: List[NormReference],
|
| 890 |
+
container_ids: List[str],
|
| 891 |
+
base_where: Optional[Dict[str, Any]] = None,
|
| 892 |
+
max_chunks_per_ref: int = 6,
|
| 893 |
+
) -> List[Dict[str, Any]]:
|
| 894 |
+
out: List[Dict[str, Any]] = []
|
| 895 |
+
|
| 896 |
+
for container_id in container_ids:
|
| 897 |
+
for ref in refs:
|
| 898 |
+
# New metadata path: paragraph / subsection / sentence / number / letter.
|
| 899 |
+
metadata_filter: Dict[str, Any] = {"container_id": container_id}
|
| 900 |
+
metadata_filter["paragraph"] = {"$in": self._paragraph_variants(ref.paragraph)}
|
| 901 |
+
if ref.subsection:
|
| 902 |
+
metadata_filter["subsection"] = {"$in": self._subsection_variants(ref.subsection)}
|
| 903 |
+
if ref.sentence:
|
| 904 |
+
metadata_filter["sentence"] = {"$in": self._sentence_variants(ref.sentence)}
|
| 905 |
+
if ref.number:
|
| 906 |
+
metadata_filter["number"] = {"$in": self._number_variants(ref.number)}
|
| 907 |
+
if ref.letter:
|
| 908 |
+
metadata_filter["letter"] = {"$in": self._letter_variants(ref.letter)}
|
| 909 |
+
|
| 910 |
+
where = self._where_and(self._build_where(base_where), metadata_filter)
|
| 911 |
+
formatted: List[Dict[str, Any]] = []
|
| 912 |
+
|
| 913 |
+
try:
|
| 914 |
+
res = self.col.get(where=where, include=["documents", "metadatas"])
|
| 915 |
+
formatted = self._format_get(res, retrieval_kind="exact_norm", score=1.0)
|
| 916 |
+
except Exception as exc: # noqa: BLE001 - old collections may not have new metadata fields.
|
| 917 |
+
logger.debug("structured exact norm lookup failed", exc_info=exc)
|
| 918 |
+
|
| 919 |
+
# Legacy fallback: section_id only, then filter by canonical/text if Abs./Satz/etc. were specified.
|
| 920 |
+
if not formatted:
|
| 921 |
+
formatted = self._get_explicit_sections(
|
| 922 |
+
sections=[ref.section_id],
|
| 923 |
+
container_ids=[container_id],
|
| 924 |
+
base_where=base_where,
|
| 925 |
+
max_chunks_per_section=max_chunks_per_ref * 2,
|
| 926 |
+
retrieval_kind="exact_norm",
|
| 927 |
+
)
|
| 928 |
+
if ref.subsection or ref.sentence or ref.number or ref.letter:
|
| 929 |
+
narrowed = [hit for hit in formatted if self._hit_matches_norm_ref(hit, ref)]
|
| 930 |
+
if narrowed:
|
| 931 |
+
formatted = narrowed
|
| 932 |
+
|
| 933 |
+
formatted.sort(key=lambda hit: self._exact_norm_sort_key(hit, ref), reverse=True)
|
| 934 |
+
out.extend(formatted[:max_chunks_per_ref])
|
| 935 |
+
|
| 936 |
+
return out
|
| 937 |
+
|
| 938 |
def _get_explicit_sections(
|
| 939 |
self,
|
| 940 |
*,
|
|
|
|
| 942 |
container_ids: List[str],
|
| 943 |
base_where: Optional[Dict[str, Any]] = None,
|
| 944 |
max_chunks_per_section: int = 4,
|
| 945 |
+
retrieval_kind: str = "explicit_section",
|
| 946 |
) -> List[Dict[str, Any]]:
|
| 947 |
out: List[Dict[str, Any]] = []
|
| 948 |
|
|
|
|
| 952 |
|
| 953 |
where = self._where_and(
|
| 954 |
self._build_where(base_where),
|
| 955 |
+
{"container_id": container_id, "section_id": {"$in": variants}},
|
|
|
|
|
|
|
|
|
|
| 956 |
)
|
| 957 |
|
| 958 |
try:
|
| 959 |
+
res = self.col.get(where=where, include=["documents", "metadatas"])
|
| 960 |
+
except Exception as exc: # noqa: BLE001
|
|
|
|
|
|
|
|
|
|
| 961 |
logger.debug("explicit section lookup failed", exc_info=exc)
|
| 962 |
continue
|
| 963 |
|
| 964 |
+
formatted = self._format_get(res, retrieval_kind=retrieval_kind, score=1.0)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 965 |
formatted.sort(key=lambda x: x.get("chunk_index", 0))
|
| 966 |
out.extend(formatted[:max_chunks_per_section])
|
| 967 |
|
| 968 |
return out
|
| 969 |
|
| 970 |
+
def _definition_search(
|
| 971 |
+
self,
|
| 972 |
+
*,
|
| 973 |
+
question: str,
|
| 974 |
+
base_where: Optional[Dict[str, Any]],
|
| 975 |
+
top_k: int,
|
| 976 |
+
expanded_terms: Optional[List[str]] = None,
|
| 977 |
+
) -> List[Dict[str, Any]]:
|
| 978 |
+
terms = self._extract_definition_terms(question)
|
| 979 |
+
terms.extend(expanded_terms or [])
|
| 980 |
+
if not terms:
|
| 981 |
+
terms = self._content_terms(question)[:5]
|
| 982 |
+
|
| 983 |
+
terms = list(dict.fromkeys([term for term in terms if term]))
|
| 984 |
+
candidates: List[Dict[str, Any]] = []
|
| 985 |
+
|
| 986 |
+
# Preferred path: new metadata field is_definition=True.
|
| 987 |
+
where = self._where_and(self._build_where(base_where), {"is_definition": True})
|
| 988 |
+
try:
|
| 989 |
+
res = self.col.get(where=where, include=["documents", "metadatas"], limit=self.lexical_scan_limit)
|
| 990 |
+
candidates.extend(self._format_get(res, retrieval_kind="definition", score=0.85))
|
| 991 |
+
except Exception as exc: # noqa: BLE001
|
| 992 |
+
logger.debug("definition metadata lookup failed", exc_info=exc)
|
| 993 |
+
|
| 994 |
+
# Fallback: § 2 usually contains definitions in this contract.
|
| 995 |
+
if not candidates:
|
| 996 |
+
containers = [self.default_container_id]
|
| 997 |
+
candidates.extend(
|
| 998 |
+
self._get_explicit_sections(
|
| 999 |
+
sections=["§ 2"],
|
| 1000 |
+
container_ids=containers,
|
| 1001 |
+
base_where=base_where,
|
| 1002 |
+
max_chunks_per_section=80,
|
| 1003 |
+
retrieval_kind="definition",
|
| 1004 |
+
)
|
| 1005 |
+
)
|
| 1006 |
+
|
| 1007 |
+
scored: List[Dict[str, Any]] = []
|
| 1008 |
+
for hit in candidates:
|
| 1009 |
+
score = self._definition_score(hit, terms)
|
| 1010 |
+
if score <= 0:
|
| 1011 |
+
continue
|
| 1012 |
+
new_hit = dict(hit)
|
| 1013 |
+
new_hit["score"] = round(min(score, 1.0), 4)
|
| 1014 |
+
new_hit["rank_score"] = new_hit["score"]
|
| 1015 |
+
kinds = set(new_hit.get("retrieval_kinds", []))
|
| 1016 |
+
kinds.add("definition")
|
| 1017 |
+
new_hit["retrieval_kinds"] = sorted(kinds)
|
| 1018 |
+
scored.append(new_hit)
|
| 1019 |
+
|
| 1020 |
+
scored.sort(key=lambda h: (h.get("rank_score", 0.0), h.get("score", 0.0)), reverse=True)
|
| 1021 |
+
return scored[:top_k]
|
| 1022 |
+
|
| 1023 |
+
def _lexical_search(
|
| 1024 |
+
self,
|
| 1025 |
+
*,
|
| 1026 |
+
question: str,
|
| 1027 |
+
base_where: Optional[Dict[str, Any]],
|
| 1028 |
+
top_k: int,
|
| 1029 |
+
expanded_terms: Optional[List[str]] = None,
|
| 1030 |
+
scan_limit: int,
|
| 1031 |
+
) -> List[Dict[str, Any]]:
|
| 1032 |
+
tokens = self._content_terms(question)
|
| 1033 |
+
phrases = list(expanded_terms or [])
|
| 1034 |
+
for term in self._extract_definition_terms(question):
|
| 1035 |
+
if term not in phrases:
|
| 1036 |
+
phrases.append(term)
|
| 1037 |
+
|
| 1038 |
+
if not tokens and not phrases:
|
| 1039 |
+
return []
|
| 1040 |
+
|
| 1041 |
+
try:
|
| 1042 |
+
res = self.col.get(
|
| 1043 |
+
where=self._build_where(base_where),
|
| 1044 |
+
include=["documents", "metadatas"],
|
| 1045 |
+
limit=max(scan_limit, top_k),
|
| 1046 |
+
)
|
| 1047 |
+
except Exception as exc: # noqa: BLE001
|
| 1048 |
+
logger.debug("lexical scan failed", exc_info=exc)
|
| 1049 |
+
return []
|
| 1050 |
+
|
| 1051 |
+
candidates = self._format_get(res, retrieval_kind="lexical", score=0.0)
|
| 1052 |
+
scored: List[Dict[str, Any]] = []
|
| 1053 |
+
for hit in candidates:
|
| 1054 |
+
score = self._lexical_score(hit, tokens=tokens, phrases=phrases)
|
| 1055 |
+
if score <= 0:
|
| 1056 |
+
continue
|
| 1057 |
+
new_hit = dict(hit)
|
| 1058 |
+
new_hit["score"] = round(score, 4)
|
| 1059 |
+
new_hit["rank_score"] = round(score, 4)
|
| 1060 |
+
new_hit["retrieval_kinds"] = sorted(set(new_hit.get("retrieval_kinds", [])) | {"lexical"})
|
| 1061 |
+
scored.append(new_hit)
|
| 1062 |
+
|
| 1063 |
+
scored.sort(
|
| 1064 |
+
key=lambda hit: (
|
| 1065 |
+
hit.get("rank_score", 0.0),
|
| 1066 |
+
hit.get("score", 0.0),
|
| 1067 |
+
hit.get("metadata", {}).get("is_definition") is True,
|
| 1068 |
+
),
|
| 1069 |
+
reverse=True,
|
| 1070 |
+
)
|
| 1071 |
+
return scored[:top_k]
|
| 1072 |
+
|
| 1073 |
def _get_neighbors(
|
| 1074 |
self,
|
| 1075 |
hit: Dict[str, Any],
|
|
|
|
| 1089 |
except (TypeError, ValueError):
|
| 1090 |
return []
|
| 1091 |
|
| 1092 |
+
neighbor_indices = [i for i in range(idx - window, idx + window + 1) if i >= 0 and i != idx]
|
|
|
|
|
|
|
|
|
|
| 1093 |
if not neighbor_indices:
|
| 1094 |
return []
|
| 1095 |
|
|
|
|
| 1103 |
)
|
| 1104 |
|
| 1105 |
try:
|
| 1106 |
+
res = self.col.get(where=where, include=["documents", "metadatas"])
|
| 1107 |
+
except Exception as exc: # noqa: BLE001
|
|
|
|
|
|
|
|
|
|
| 1108 |
logger.debug("neighbor lookup failed", exc_info=exc)
|
| 1109 |
return []
|
| 1110 |
|
|
|
|
| 1116 |
formatted.sort(key=lambda x: x.get("chunk_index", 0))
|
| 1117 |
return formatted
|
| 1118 |
|
| 1119 |
+
def _expand_parent_context(
|
|
|
|
|
|
|
|
|
|
| 1120 |
self,
|
| 1121 |
+
hits: List[Dict[str, Any]],
|
| 1122 |
+
*,
|
| 1123 |
+
base_where: Optional[Dict[str, Any]] = None,
|
| 1124 |
+
max_parent_contexts: int = 8,
|
|
|
|
| 1125 |
) -> List[Dict[str, Any]]:
|
| 1126 |
+
parent_ids: List[str] = []
|
| 1127 |
+
score_by_parent: Dict[str, float] = defaultdict(float)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1128 |
|
| 1129 |
+
for hit in hits:
|
| 1130 |
+
meta = hit.get("metadata") or {}
|
| 1131 |
+
parent_id = meta.get("parent_unit_id") or meta.get("parent_id")
|
| 1132 |
+
legal_id = meta.get("legal_unit_id")
|
| 1133 |
+
chunk_kind = str(meta.get("chunk_kind") or "").lower()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1134 |
|
| 1135 |
+
if not parent_id:
|
| 1136 |
+
continue
|
| 1137 |
+
if legal_id and str(parent_id) == str(legal_id):
|
| 1138 |
+
continue
|
| 1139 |
+
if chunk_kind == "parent":
|
| 1140 |
+
continue
|
| 1141 |
|
| 1142 |
+
parent = str(parent_id)
|
| 1143 |
+
parent_ids.append(parent)
|
| 1144 |
+
score_by_parent[parent] = max(score_by_parent[parent], float(hit.get("rank_score", hit.get("score", 0.0))))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1145 |
|
| 1146 |
+
parent_ids = list(dict.fromkeys(parent_ids))[:max_parent_contexts]
|
| 1147 |
+
out: List[Dict[str, Any]] = []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1148 |
|
| 1149 |
+
for parent_id in parent_ids:
|
| 1150 |
+
where = self._where_and(
|
| 1151 |
+
self._build_where(base_where),
|
| 1152 |
+
{"legal_unit_id": parent_id},
|
| 1153 |
)
|
| 1154 |
+
try:
|
| 1155 |
+
res = self.col.get(where=where, include=["documents", "metadatas"], limit=4)
|
| 1156 |
+
except Exception as exc: # noqa: BLE001
|
| 1157 |
+
logger.debug("parent context lookup failed", exc_info=exc)
|
| 1158 |
+
continue
|
| 1159 |
|
| 1160 |
+
score = min(score_by_parent.get(parent_id, 0.75) + 0.05, 1.0)
|
| 1161 |
+
formatted = self._format_get(res, retrieval_kind="parent_context", score=score)
|
| 1162 |
+
for hit in formatted:
|
| 1163 |
+
hit["retrieval_kinds"] = sorted(set(hit.get("retrieval_kinds", [])) | {"parent_context"})
|
| 1164 |
+
out.extend(formatted)
|
| 1165 |
+
|
| 1166 |
+
return out
|
| 1167 |
+
|
| 1168 |
+
# ------------------------------------------------------------------
|
| 1169 |
+
# Cross-encoder reranking
|
| 1170 |
+
# ------------------------------------------------------------------
|
| 1171 |
+
def _get_reranker(self) -> Any:
|
| 1172 |
+
if self._reranker is None:
|
| 1173 |
+
from sentence_transformers import CrossEncoder
|
| 1174 |
+
|
| 1175 |
+
logger.info("loading cross-encoder reranker", extra={"model": self.reranker_model})
|
| 1176 |
+
self._reranker = CrossEncoder(self.reranker_model, max_length=512)
|
| 1177 |
+
return self._reranker
|
| 1178 |
+
|
| 1179 |
+
def _rerank(self, question: str, hits: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
| 1180 |
+
"""Re-order the fused candidate pool with a cross-encoder.
|
| 1181 |
+
|
| 1182 |
+
The bi-encoder retrieves candidates cheaply; the cross-encoder reads
|
| 1183 |
+
query and passage together and orders them far more precisely. In the
|
| 1184 |
+
golden evaluation this lifted hybrid retrieval from 17/18 to 18/18 and
|
| 1185 |
+
stabilizes which hit becomes [Quelle 1] for the LLM.
|
| 1186 |
+
"""
|
| 1187 |
+
if not hits:
|
| 1188 |
+
return hits
|
| 1189 |
+
|
| 1190 |
+
candidates = hits[: self.reranker_candidates]
|
| 1191 |
+
rest = hits[self.reranker_candidates :]
|
| 1192 |
+
pairs = [(question, str(hit.get("text") or "")) for hit in candidates]
|
| 1193 |
+
|
| 1194 |
+
try:
|
| 1195 |
+
scores = self._get_reranker().predict(pairs, batch_size=16, show_progress_bar=False)
|
| 1196 |
+
except Exception as exc: # noqa: BLE001 - reranking must never break retrieval.
|
| 1197 |
+
logger.warning("reranker failed; keeping fused order", exc_info=exc)
|
| 1198 |
+
return hits
|
| 1199 |
+
|
| 1200 |
+
reranked: List[Dict[str, Any]] = []
|
| 1201 |
+
for hit, score in zip(candidates, scores):
|
| 1202 |
+
enriched = dict(hit)
|
| 1203 |
+
enriched["rerank_score"] = round(float(score), 4)
|
| 1204 |
+
# Downstream layers (Orchestrator, Composer) sortieren nach
|
| 1205 |
+
# rank_score. Ohne diese Übernahme würde die Cross-Encoder-Ordnung
|
| 1206 |
+
# dort wieder durch die alte Fusion-Reihenfolge ersetzt. Sigmoid
|
| 1207 |
+
# bildet die Logits monoton auf (0, 1) ab.
|
| 1208 |
+
enriched["rank_score"] = round(1.0 / (1.0 + math.exp(-float(score))), 4)
|
| 1209 |
+
reranked.append(enriched)
|
| 1210 |
+
|
| 1211 |
+
reranked.sort(key=lambda h: h["rerank_score"], reverse=True)
|
| 1212 |
+
return reranked + rest
|
| 1213 |
|
| 1214 |
# ------------------------------------------------------------------
|
| 1215 |
# Context and source formatting
|
|
|
|
| 1220 |
*,
|
| 1221 |
max_chars: int = 12000,
|
| 1222 |
include_neighbors: bool = True,
|
| 1223 |
+
include_metadata: bool = True,
|
| 1224 |
) -> str:
|
| 1225 |
+
"""Build stable RAG context blocks for the downstream LLM client."""
|
|
|
|
|
|
|
| 1226 |
if not results:
|
| 1227 |
return ""
|
| 1228 |
|
|
|
|
| 1234 |
if not include_neighbors and kinds == {"neighbor"}:
|
| 1235 |
continue
|
| 1236 |
|
| 1237 |
+
meta = hit.get("metadata") or {}
|
| 1238 |
+
canonical = meta.get("canonical_ref") or hit.get("canonical_ref") or hit.get("section", "ohne Abschnitt")
|
| 1239 |
+
chunk_kind = meta.get("chunk_kind") or hit.get("chunk_kind") or ""
|
| 1240 |
+
unit_type = meta.get("unit_type") or ""
|
| 1241 |
+
definition = "Definition" if meta.get("is_definition") is True else ""
|
| 1242 |
+
details = ", ".join([x for x in [str(chunk_kind), str(unit_type), definition] if x])
|
| 1243 |
+
|
| 1244 |
source = (
|
| 1245 |
f"[Quelle {i}: {hit.get('container', 'Unbekannt')}::"
|
| 1246 |
+
f"{canonical}, Seiten {hit.get('page_range', '?')}, "
|
|
|
|
| 1247 |
f"Chunk {hit.get('chunk_index', '?')}, "
|
| 1248 |
+
f"Typ {','.join(hit.get('retrieval_kinds', []))}"
|
| 1249 |
)
|
| 1250 |
+
if include_metadata and details:
|
| 1251 |
+
source += f", Metadaten {details}"
|
| 1252 |
+
source += "]"
|
| 1253 |
+
|
| 1254 |
text = (hit.get("text") or "").strip()
|
| 1255 |
block = f"{source}\n{text}"
|
| 1256 |
|
|
|
|
| 1270 |
include_neighbor_only: bool = False,
|
| 1271 |
) -> List[Dict[str, Any]]:
|
| 1272 |
"""
|
| 1273 |
+
Build a clean, deduplicated source list.
|
| 1274 |
|
| 1275 |
+
Neighbor-only hits are excluded by default because they are usually context,
|
| 1276 |
+
not the primary legal basis.
|
|
|
|
|
|
|
| 1277 |
"""
|
| 1278 |
+
seen: Set[Tuple[Any, Any, Any, Any]] = set()
|
| 1279 |
sources: List[Dict[str, Any]] = []
|
| 1280 |
|
| 1281 |
for hit in results:
|
|
|
|
| 1283 |
if kinds == {"neighbor"} and not include_neighbor_only:
|
| 1284 |
continue
|
| 1285 |
|
| 1286 |
+
meta = hit.get("metadata") or {}
|
| 1287 |
+
canonical = meta.get("canonical_ref") or hit.get("canonical_ref") or hit.get("section")
|
| 1288 |
+
key = (hit.get("container"), canonical, hit.get("page_range"), meta.get("legal_unit_id"))
|
|
|
|
|
|
|
| 1289 |
|
| 1290 |
if key in seen:
|
| 1291 |
continue
|
|
|
|
| 1296 |
{
|
| 1297 |
"container": hit.get("container"),
|
| 1298 |
"section": hit.get("section"),
|
| 1299 |
+
"canonical_ref": canonical,
|
| 1300 |
"path": hit.get("path"),
|
| 1301 |
"page_range": hit.get("page_range"),
|
| 1302 |
"page_start": hit.get("page_start"),
|
|
|
|
| 1304 |
"score": hit.get("score"),
|
| 1305 |
"rank_score": hit.get("rank_score"),
|
| 1306 |
"retrieval_kinds": hit.get("retrieval_kinds", []),
|
| 1307 |
+
"section_title": meta.get("section_title"),
|
| 1308 |
+
"chunk_kind": meta.get("chunk_kind"),
|
| 1309 |
+
"unit_type": meta.get("unit_type"),
|
| 1310 |
+
"is_definition": meta.get("is_definition"),
|
| 1311 |
+
"defined_terms": meta.get("defined_terms"),
|
| 1312 |
}
|
| 1313 |
)
|
| 1314 |
|
|
|
|
| 1318 |
return sources
|
| 1319 |
|
| 1320 |
@staticmethod
|
| 1321 |
+
def format_sources_markdown(sources: List[Dict[str, Any]], *, title: str = "Quellen") -> str:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1322 |
if not sources:
|
| 1323 |
return f"{title}: Keine Quellen gefunden."
|
| 1324 |
|
|
|
|
| 1328 |
section = source.get("section") or "ohne Abschnitt"
|
| 1329 |
pages = source.get("page_range") or "?"
|
| 1330 |
canonical = source.get("canonical_ref")
|
| 1331 |
+
label = canonical or section
|
| 1332 |
if canonical and canonical != section:
|
| 1333 |
lines.append(f"- {container}::{section} ({canonical}), Seiten {pages}")
|
| 1334 |
else:
|
| 1335 |
+
lines.append(f"- {container}::{label}, Seiten {pages}")
|
| 1336 |
|
| 1337 |
return "\n".join(lines)
|
| 1338 |
|
|
|
|
| 1359 |
return out
|
| 1360 |
|
| 1361 |
@staticmethod
|
| 1362 |
+
def _format_get(res: Dict[str, Any], *, retrieval_kind: str, score: float) -> List[Dict[str, Any]]:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1363 |
docs = res.get("documents") or []
|
| 1364 |
metas = res.get("metadatas") or []
|
| 1365 |
|
|
|
|
| 1378 |
@staticmethod
|
| 1379 |
def _similarity_from_distance(dist: Any) -> float:
|
| 1380 |
"""
|
| 1381 |
+
For cosine space in Chroma, distance is typically 1 - cosine_similarity.
|
| 1382 |
+
For other metrics this remains only an approximate rank signal.
|
| 1383 |
"""
|
| 1384 |
try:
|
| 1385 |
similarity = 1.0 - float(dist)
|
|
|
|
| 1389 |
return round(max(min(similarity, 1.0), -1.0), 4)
|
| 1390 |
|
| 1391 |
@staticmethod
|
| 1392 |
+
def _normalize_hit(*, doc: str, meta: Dict[str, Any], score: float, retrieval_kind: str) -> Dict[str, Any]:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1393 |
page_start = meta.get("page_start")
|
| 1394 |
page_end = meta.get("page_end", page_start)
|
| 1395 |
page_range = "?"
|
|
|
|
| 1405 |
except (TypeError, ValueError):
|
| 1406 |
chunk_index = 0
|
| 1407 |
|
| 1408 |
+
canonical_ref = meta.get("canonical_ref") or LegalRetriever._canonical_from_metadata(meta) or meta.get("section_id")
|
| 1409 |
+
|
| 1410 |
hit = {
|
| 1411 |
"score": round(float(score), 4),
|
| 1412 |
"rank_score": round(float(score), 4),
|
| 1413 |
"text": doc or "",
|
| 1414 |
"container": meta.get("container_id", "Unbekannt"),
|
| 1415 |
"container_type": meta.get("container_type"),
|
| 1416 |
+
"section": meta.get("section_id") or meta.get("paragraph") or "ohne Abschnitt",
|
| 1417 |
+
"canonical_ref": canonical_ref,
|
| 1418 |
"path": meta.get("section_path", ""),
|
| 1419 |
"page_range": page_range,
|
| 1420 |
"page_start": page_start,
|
|
|
|
| 1426 |
hit["source_key"] = LegalRetriever._source_key(hit)
|
| 1427 |
return hit
|
| 1428 |
|
| 1429 |
+
@staticmethod
|
| 1430 |
+
def _canonical_from_metadata(meta: Dict[str, Any]) -> str | None:
|
| 1431 |
+
paragraph = meta.get("paragraph") or meta.get("section_id")
|
| 1432 |
+
if not paragraph:
|
| 1433 |
+
return None
|
| 1434 |
+
parts = [str(paragraph)]
|
| 1435 |
+
if meta.get("subsection"):
|
| 1436 |
+
sub = str(meta["subsection"])
|
| 1437 |
+
parts.append(sub if sub.lower().startswith("abs") else f"Abs. {sub}")
|
| 1438 |
+
if meta.get("sentence"):
|
| 1439 |
+
sent = str(meta["sentence"])
|
| 1440 |
+
parts.append(sent if sent.lower().startswith("satz") else f"Satz {sent}")
|
| 1441 |
+
if meta.get("number"):
|
| 1442 |
+
num = str(meta["number"])
|
| 1443 |
+
parts.append(num if num.lower().startswith(("nr", "nummer")) else f"Nr. {num}")
|
| 1444 |
+
if meta.get("letter"):
|
| 1445 |
+
letter = str(meta["letter"]).lower().replace(")", "")
|
| 1446 |
+
parts.append(letter if letter.lower().startswith("buchst") else f"Buchst. {letter}")
|
| 1447 |
+
return " ".join(parts)
|
| 1448 |
+
|
| 1449 |
@staticmethod
|
| 1450 |
def _source_key(hit: Dict[str, Any]) -> Tuple[Any, Any, Any, Any]:
|
| 1451 |
meta = hit.get("metadata") or {}
|
| 1452 |
+
legal_unit_id = meta.get("legal_unit_id")
|
| 1453 |
+
if legal_unit_id:
|
| 1454 |
+
return (hit.get("container"), legal_unit_id, meta.get("chunk_kind"), meta.get("text_hash"))
|
| 1455 |
+
|
| 1456 |
text_hash = meta.get("text_hash")
|
| 1457 |
if text_hash:
|
| 1458 |
+
return (hit.get("container"), hit.get("section"), hit.get("chunk_index"), text_hash)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1459 |
|
| 1460 |
+
return (hit.get("container"), hit.get("section"), hit.get("chunk_index"), (hit.get("text") or "")[:120])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1461 |
|
| 1462 |
@staticmethod
|
| 1463 |
+
def _dedupe_and_rank(
|
| 1464 |
+
results: List[Dict[str, Any]],
|
| 1465 |
+
*,
|
| 1466 |
+
question: str | None = None,
|
| 1467 |
+
intent: str | None = None,
|
| 1468 |
+
) -> List[Dict[str, Any]]:
|
| 1469 |
+
"""Deduplicate hits and rank legal-specific retrieval kinds above generic context."""
|
| 1470 |
merged: Dict[Tuple[Any, Any, Any, Any], Dict[str, Any]] = {}
|
| 1471 |
|
| 1472 |
for hit in results:
|
|
|
|
| 1477 |
merged[key] = dict(hit)
|
| 1478 |
continue
|
| 1479 |
|
| 1480 |
+
existing["score"] = max(float(existing.get("score", 0.0)), float(hit.get("score", 0.0)))
|
| 1481 |
+
existing["rank_score"] = max(float(existing.get("rank_score", 0.0)), float(hit.get("rank_score", 0.0)))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1482 |
|
| 1483 |
kinds: Set[str] = set(existing.get("retrieval_kinds", []))
|
| 1484 |
kinds.update(hit.get("retrieval_kinds", []))
|
|
|
|
| 1486 |
|
| 1487 |
ranked = list(merged.values())
|
| 1488 |
|
| 1489 |
+
target_refs = LegalRetriever._parse_norm_refs(question or "")
|
| 1490 |
+
target_terms = LegalRetriever._extract_definition_terms(question or "") if question else []
|
| 1491 |
+
|
| 1492 |
for hit in ranked:
|
| 1493 |
+
meta = hit.get("metadata") or {}
|
| 1494 |
kinds = set(hit.get("retrieval_kinds", []))
|
| 1495 |
boost = 0.0
|
| 1496 |
|
| 1497 |
+
if "exact_norm" in kinds:
|
| 1498 |
+
boost += 0.16
|
| 1499 |
if "explicit_section" in kinds or "section_lookup" in kinds:
|
| 1500 |
+
boost += 0.10
|
| 1501 |
+
if "definition" in kinds:
|
| 1502 |
+
boost += 0.13
|
| 1503 |
+
if "parent_context" in kinds:
|
| 1504 |
+
boost += 0.09
|
| 1505 |
+
if "lexical" in kinds:
|
| 1506 |
+
boost += 0.06
|
| 1507 |
if "semantic" in kinds:
|
| 1508 |
boost += 0.04
|
| 1509 |
if kinds == {"neighbor"}:
|
| 1510 |
+
boost -= 0.05
|
| 1511 |
|
|
|
|
| 1512 |
if hit.get("container") == "Vertrag":
|
| 1513 |
boost += 0.01
|
| 1514 |
|
| 1515 |
+
if meta.get("chunk_kind") == "parent":
|
| 1516 |
+
boost += 0.03
|
| 1517 |
+
if meta.get("is_definition") is True and intent == "definition":
|
| 1518 |
+
boost += 0.10
|
| 1519 |
+
|
| 1520 |
+
canonical = str(meta.get("canonical_ref") or hit.get("canonical_ref") or "").lower()
|
| 1521 |
+
for ref in target_refs:
|
| 1522 |
+
if ref.canonical_ref.lower() in canonical or ref.section_id.lower() in canonical:
|
| 1523 |
+
boost += 0.12
|
| 1524 |
+
break
|
| 1525 |
+
|
| 1526 |
+
meta_text = LegalRetriever._metadata_text(meta).lower()
|
| 1527 |
+
for term in target_terms:
|
| 1528 |
+
if term.lower() in meta_text:
|
| 1529 |
+
boost += 0.08
|
| 1530 |
+
break
|
| 1531 |
+
|
| 1532 |
hit["rank_score"] = round(float(hit.get("score", 0.0)) + boost, 4)
|
| 1533 |
|
| 1534 |
ranked.sort(
|
| 1535 |
key=lambda x: (
|
| 1536 |
x.get("rank_score", 0.0),
|
| 1537 |
x.get("score", 0.0),
|
| 1538 |
+
1 if (x.get("metadata") or {}).get("chunk_kind") == "parent" else 0,
|
| 1539 |
-int(x.get("chunk_index", 0)),
|
| 1540 |
),
|
| 1541 |
reverse=True,
|
|
|
|
| 1544 |
return ranked
|
| 1545 |
|
| 1546 |
@staticmethod
|
| 1547 |
+
def _limit_chunks_per_section(results: List[Dict[str, Any]], *, max_chunks_per_section: int) -> List[Dict[str, Any]]:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1548 |
if max_chunks_per_section <= 0:
|
| 1549 |
return results
|
| 1550 |
|
|
|
|
| 1552 |
limited: List[Dict[str, Any]] = []
|
| 1553 |
|
| 1554 |
for hit in results:
|
| 1555 |
+
meta = hit.get("metadata") or {}
|
| 1556 |
+
# Parent chunks are important context; count by canonical parent rather than raw section only.
|
| 1557 |
+
key = (hit.get("container"), hit.get("section"), meta.get("chunk_kind"))
|
| 1558 |
if counts[key] >= max_chunks_per_section:
|
| 1559 |
continue
|
| 1560 |
counts[key] += 1
|
| 1561 |
limited.append(hit)
|
| 1562 |
|
| 1563 |
return limited
|
| 1564 |
+
|
| 1565 |
+
# ------------------------------------------------------------------
|
| 1566 |
+
# Scoring helpers
|
| 1567 |
+
# ------------------------------------------------------------------
|
| 1568 |
+
@staticmethod
|
| 1569 |
+
def _hit_matches_norm_ref(hit: Dict[str, Any], ref: NormReference) -> bool:
|
| 1570 |
+
meta = hit.get("metadata") or {}
|
| 1571 |
+
haystack = " ".join(
|
| 1572 |
+
[
|
| 1573 |
+
str(meta.get("canonical_ref") or ""),
|
| 1574 |
+
str(hit.get("canonical_ref") or ""),
|
| 1575 |
+
str(meta.get("paragraph") or ""),
|
| 1576 |
+
str(meta.get("subsection") or ""),
|
| 1577 |
+
str(meta.get("sentence") or ""),
|
| 1578 |
+
str(meta.get("number") or ""),
|
| 1579 |
+
str(meta.get("letter") or ""),
|
| 1580 |
+
hit.get("text") or "",
|
| 1581 |
+
]
|
| 1582 |
+
).lower()
|
| 1583 |
+
|
| 1584 |
+
if ref.section_id.lower() not in haystack and ref.section_id.replace(" ", "").lower() not in haystack.replace(" ", ""):
|
| 1585 |
+
return False
|
| 1586 |
+
if ref.subsection and f"abs. {ref.subsection}".lower() not in haystack and f"({ref.subsection})" not in haystack:
|
| 1587 |
+
return False
|
| 1588 |
+
if ref.sentence and f"satz {ref.sentence}".lower() not in haystack:
|
| 1589 |
+
return False
|
| 1590 |
+
if ref.number and f"nr. {ref.number}".lower() not in haystack and f"nummer {ref.number}".lower() not in haystack:
|
| 1591 |
+
return False
|
| 1592 |
+
if ref.letter and f"buchst. {ref.letter}".lower() not in haystack and f"{ref.letter})" not in haystack:
|
| 1593 |
+
return False
|
| 1594 |
+
return True
|
| 1595 |
+
|
| 1596 |
+
@staticmethod
|
| 1597 |
+
def _exact_norm_sort_key(hit: Dict[str, Any], ref: NormReference) -> Tuple[int, int, float, int]:
|
| 1598 |
+
meta = hit.get("metadata") or {}
|
| 1599 |
+
canonical = str(meta.get("canonical_ref") or hit.get("canonical_ref") or "").lower()
|
| 1600 |
+
exact = 1 if ref.canonical_ref.lower() in canonical else 0
|
| 1601 |
+
parent = 1 if str(meta.get("chunk_kind") or "").lower() == "parent" else 0
|
| 1602 |
+
score = float(hit.get("score", 0.0))
|
| 1603 |
+
# Lower chunk index first for paragraph fallbacks.
|
| 1604 |
+
idx = -int(hit.get("chunk_index", 0))
|
| 1605 |
+
return exact, parent, score, idx
|
| 1606 |
+
|
| 1607 |
+
@classmethod
|
| 1608 |
+
def _definition_score(cls, hit: Dict[str, Any], terms: List[str]) -> float:
|
| 1609 |
+
meta = hit.get("metadata") or {}
|
| 1610 |
+
text = f"{hit.get('text') or ''}\n{cls._metadata_text(meta)}"
|
| 1611 |
+
haystack = cls._norm_text(text)
|
| 1612 |
+
score = 0.0
|
| 1613 |
+
|
| 1614 |
+
if meta.get("is_definition") is True:
|
| 1615 |
+
score += 0.45
|
| 1616 |
+
if str(meta.get("unit_type") or "").lower() == "definition":
|
| 1617 |
+
score += 0.35
|
| 1618 |
+
if str(meta.get("section_id") or meta.get("paragraph") or "").replace(" ", "") in {"§2", "2"}:
|
| 1619 |
+
score += 0.12
|
| 1620 |
+
|
| 1621 |
+
for term in terms:
|
| 1622 |
+
t = cls._norm_text(term)
|
| 1623 |
+
if not t:
|
| 1624 |
+
continue
|
| 1625 |
+
if t in haystack:
|
| 1626 |
+
score += 0.35
|
| 1627 |
+
else:
|
| 1628 |
+
token_hits = sum(1 for token in cls._tokenize(t) if token in haystack)
|
| 1629 |
+
if token_hits:
|
| 1630 |
+
score += min(0.18, 0.06 * token_hits)
|
| 1631 |
+
|
| 1632 |
+
return min(score, 1.0)
|
| 1633 |
+
|
| 1634 |
+
@classmethod
|
| 1635 |
+
def _lexical_score(cls, hit: Dict[str, Any], *, tokens: List[str], phrases: List[str]) -> float:
|
| 1636 |
+
meta = hit.get("metadata") or {}
|
| 1637 |
+
haystack_raw = f"{hit.get('text') or ''}\n{cls._metadata_text(meta)}"
|
| 1638 |
+
haystack = cls._norm_text(haystack_raw)
|
| 1639 |
+
if not haystack:
|
| 1640 |
+
return 0.0
|
| 1641 |
+
|
| 1642 |
+
raw = 0.0
|
| 1643 |
+
token_counts = Counter(cls._tokenize(haystack))
|
| 1644 |
+
for token in tokens:
|
| 1645 |
+
freq = token_counts.get(token, 0)
|
| 1646 |
+
if freq:
|
| 1647 |
+
raw += 1.0 + min(freq - 1, 3) * 0.2
|
| 1648 |
+
|
| 1649 |
+
for phrase in phrases:
|
| 1650 |
+
p = cls._norm_text(phrase)
|
| 1651 |
+
if p and p in haystack:
|
| 1652 |
+
raw += 3.0 if " " in p else 1.4
|
| 1653 |
+
|
| 1654 |
+
# Legal metadata signals.
|
| 1655 |
+
if meta.get("is_definition") is True:
|
| 1656 |
+
raw += 0.5
|
| 1657 |
+
if meta.get("canonical_ref") and any(token in cls._norm_text(str(meta.get("canonical_ref"))) for token in tokens):
|
| 1658 |
+
raw += 0.5
|
| 1659 |
+
|
| 1660 |
+
denom = max(len(tokens) + len(phrases) * 1.5, 4.0)
|
| 1661 |
+
score = raw / denom
|
| 1662 |
+
return round(min(score, 0.99), 4)
|
| 1663 |
+
|
| 1664 |
+
@classmethod
|
| 1665 |
+
def _content_terms(cls, text: str) -> List[str]:
|
| 1666 |
+
tokens = cls._tokenize(cls._norm_text(text))
|
| 1667 |
+
return list(dict.fromkeys([t for t in tokens if len(t) >= 3 and t not in cls.STOPWORDS]))
|
| 1668 |
+
|
| 1669 |
+
@staticmethod
|
| 1670 |
+
def _metadata_text(meta: Dict[str, Any]) -> str:
|
| 1671 |
+
keys = [
|
| 1672 |
+
"canonical_ref",
|
| 1673 |
+
"paragraph",
|
| 1674 |
+
"subsection",
|
| 1675 |
+
"sentence",
|
| 1676 |
+
"number",
|
| 1677 |
+
"letter",
|
| 1678 |
+
"unit_type",
|
| 1679 |
+
"defined_terms",
|
| 1680 |
+
"section_title",
|
| 1681 |
+
"section_id",
|
| 1682 |
+
"section_path",
|
| 1683 |
+
"container_id",
|
| 1684 |
+
]
|
| 1685 |
+
values: List[str] = []
|
| 1686 |
+
for key in keys:
|
| 1687 |
+
value = meta.get(key)
|
| 1688 |
+
if value is None:
|
| 1689 |
+
continue
|
| 1690 |
+
if isinstance(value, (list, tuple, set)):
|
| 1691 |
+
values.extend(str(v) for v in value)
|
| 1692 |
+
else:
|
| 1693 |
+
values.append(str(value))
|
| 1694 |
+
return " ".join(values)
|
| 1695 |
+
|
| 1696 |
+
@staticmethod
|
| 1697 |
+
def _looks_like_negative_risk(question: str, results: List[Dict[str, Any]]) -> bool:
|
| 1698 |
+
if not results:
|
| 1699 |
+
return True
|
| 1700 |
+
q = LegalRetriever._norm_text(question)
|
| 1701 |
+
if re.search(r"\b(nicht\s+geregelt|keine\s+regelung|steht\s+nicht|nicht\s+enthalten|nicht\s+gefunden)\b", q):
|
| 1702 |
+
return True
|
| 1703 |
+
strongest = max(float(hit.get("rank_score", hit.get("score", 0.0))) for hit in results)
|
| 1704 |
+
return strongest < 0.35
|
| 1705 |
+
|
| 1706 |
+
@staticmethod
|
| 1707 |
+
def _norm_text(text: str) -> str:
|
| 1708 |
+
s = str(text or "").lower()
|
| 1709 |
+
s = s.replace("§§", "§")
|
| 1710 |
+
s = re.sub(r"[\u00a0\t\r\n]+", " ", s)
|
| 1711 |
+
s = re.sub(r"\s+", " ", s)
|
| 1712 |
+
return s.strip()
|
| 1713 |
+
|
| 1714 |
+
@staticmethod
|
| 1715 |
+
def _tokenize(text: str) -> List[str]:
|
| 1716 |
+
return re.findall(r"[a-zäöüß0-9]{2,}", text.lower())
|
| 1717 |
+
|
src/visualize_graph.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
|
| 3 |
+
from app import build_ask_workflow
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
OUTPUT_FILE = Path("ask_workflow.png")
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
if __name__ == "__main__":
|
| 10 |
+
workflow = build_ask_workflow()
|
| 11 |
+
|
| 12 |
+
png_bytes = workflow.get_graph(xray=True).draw_mermaid_png()
|
| 13 |
+
OUTPUT_FILE.write_bytes(png_bytes)
|
| 14 |
+
|
| 15 |
+
print(f"PNG generiert: {OUTPUT_FILE.resolve()}")
|
static/index.html
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
static/js/script.js
ADDED
|
@@ -0,0 +1,745 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
const messagesEl = document.getElementById("messages");
|
| 2 |
+
const questionInput = document.getElementById("question");
|
| 3 |
+
const askBtn = document.getElementById("askBtn");
|
| 4 |
+
|
| 5 |
+
// ----- Helpers -----
|
| 6 |
+
|
| 7 |
+
function safeText(value, fallback = "") {
|
| 8 |
+
if (value === null || value === undefined) return fallback;
|
| 9 |
+
return String(value);
|
| 10 |
+
}
|
| 11 |
+
|
| 12 |
+
function normalizeArray(value) {
|
| 13 |
+
if (!value) return [];
|
| 14 |
+
return Array.isArray(value) ? value : [value];
|
| 15 |
+
}
|
| 16 |
+
|
| 17 |
+
function createEl(tag, className, text) {
|
| 18 |
+
const el = document.createElement(tag);
|
| 19 |
+
if (className) el.className = className;
|
| 20 |
+
if (text !== undefined) el.textContent = text;
|
| 21 |
+
return el;
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
function scrollToBottom() {
|
| 25 |
+
window.scrollTo({
|
| 26 |
+
top: document.body.scrollHeight,
|
| 27 |
+
behavior: "smooth"
|
| 28 |
+
});
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
function setAskLoading(isLoading) {
|
| 32 |
+
if (!askBtn) return;
|
| 33 |
+
askBtn.disabled = isLoading;
|
| 34 |
+
askBtn.textContent = isLoading ? "Antwort wird erstellt …" : "Frage stellen";
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
// ----- Source rendering -----
|
| 38 |
+
|
| 39 |
+
function buildSourceMarker(source) {
|
| 40 |
+
if (source.source_label) return source.source_label;
|
| 41 |
+
if (source.source_marker) return source.source_marker;
|
| 42 |
+
|
| 43 |
+
const numbers = normalizeArray(source.source_numbers)
|
| 44 |
+
.map(Number)
|
| 45 |
+
.filter(Number.isFinite)
|
| 46 |
+
.sort((a, b) => a - b);
|
| 47 |
+
|
| 48 |
+
if (numbers.length === 1) {
|
| 49 |
+
return `[Quelle ${numbers[0]}]`;
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
if (numbers.length > 1) {
|
| 53 |
+
return `[Quellen ${numbers.join(", ")}]`;
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
if (source.source_number !== null && source.source_number !== undefined) {
|
| 57 |
+
const n = Number(source.source_number);
|
| 58 |
+
if (Number.isFinite(n)) return `[Quelle ${n}]`;
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
return "";
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
function buildCanonicalRefs(source) {
|
| 65 |
+
const refs = normalizeArray(source.canonical_refs)
|
| 66 |
+
.map(String)
|
| 67 |
+
.map(s => s.trim())
|
| 68 |
+
.filter(Boolean);
|
| 69 |
+
|
| 70 |
+
if (refs.length) {
|
| 71 |
+
return refs.slice(0, 4).join("; ");
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
const singleRef = safeText(source.canonical_ref).trim();
|
| 75 |
+
const section = safeText(source.section).trim();
|
| 76 |
+
|
| 77 |
+
if (singleRef && singleRef !== section) {
|
| 78 |
+
return singleRef;
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
return "";
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
function renderSource(source) {
|
| 85 |
+
// Best case: Backend/Composer already provides a fully curated display title.
|
| 86 |
+
if (source.display_title) {
|
| 87 |
+
return safeText(source.display_title);
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
const marker = buildSourceMarker(source);
|
| 91 |
+
|
| 92 |
+
const container = safeText(source.container, "Unbekannt").trim();
|
| 93 |
+
const section = safeText(source.section, "ohne Abschnitt").trim();
|
| 94 |
+
|
| 95 |
+
const path =
|
| 96 |
+
safeText(source.path).trim() ||
|
| 97 |
+
safeText(source.section_path).trim() ||
|
| 98 |
+
`${container}::${section}`;
|
| 99 |
+
|
| 100 |
+
const pageRange =
|
| 101 |
+
safeText(source.page_range).trim() ||
|
| 102 |
+
safeText(source.pages).trim() ||
|
| 103 |
+
"?";
|
| 104 |
+
|
| 105 |
+
const canonicalRefs = buildCanonicalRefs(source);
|
| 106 |
+
const refPart = canonicalRefs ? ` (${canonicalRefs})` : "";
|
| 107 |
+
|
| 108 |
+
const role =
|
| 109 |
+
Array.isArray(source.retrieval_kinds) &&
|
| 110 |
+
source.retrieval_kinds.length === 1 &&
|
| 111 |
+
source.retrieval_kinds[0] === "neighbor"
|
| 112 |
+
? " · Kontext/Nachbar"
|
| 113 |
+
: "";
|
| 114 |
+
|
| 115 |
+
return `${marker} ${path}${refPart}, Seiten ${pageRange}${role}`.trim();
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
function sourceNumbersOf(source) {
|
| 119 |
+
const numbers = normalizeArray(source.source_numbers)
|
| 120 |
+
.map(Number)
|
| 121 |
+
.filter(Number.isFinite);
|
| 122 |
+
|
| 123 |
+
const single = Number(source.source_number);
|
| 124 |
+
if (Number.isFinite(single) && !numbers.includes(single)) {
|
| 125 |
+
numbers.push(single);
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
return numbers.sort((a, b) => a - b);
|
| 129 |
+
}
|
| 130 |
+
|
| 131 |
+
function buildSourceNumberMap(sources) {
|
| 132 |
+
const map = new Map();
|
| 133 |
+
for (const source of normalizeArray(sources)) {
|
| 134 |
+
for (const n of sourceNumbersOf(source)) {
|
| 135 |
+
if (!map.has(n)) map.set(n, source);
|
| 136 |
+
}
|
| 137 |
+
}
|
| 138 |
+
return map;
|
| 139 |
+
}
|
| 140 |
+
|
| 141 |
+
function isSourceOpenable(source) {
|
| 142 |
+
return Boolean(source && source.source_file);
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
function appendSources(parent, sources) {
|
| 146 |
+
if (!Array.isArray(sources) || sources.length === 0) return;
|
| 147 |
+
|
| 148 |
+
const srcWrap = createEl("div", "sources");
|
| 149 |
+
const strong = createEl("strong", null, "Quellen:");
|
| 150 |
+
const ul = document.createElement("ul");
|
| 151 |
+
|
| 152 |
+
const seen = new Set();
|
| 153 |
+
|
| 154 |
+
for (const source of sources) {
|
| 155 |
+
const rendered = renderSource(source);
|
| 156 |
+
|
| 157 |
+
if (!rendered || seen.has(rendered)) continue;
|
| 158 |
+
seen.add(rendered);
|
| 159 |
+
|
| 160 |
+
const li = createEl("li", "source-item");
|
| 161 |
+
|
| 162 |
+
if (isSourceOpenable(source)) {
|
| 163 |
+
const btn = createEl("button", "source-link", rendered);
|
| 164 |
+
btn.type = "button";
|
| 165 |
+
btn.title = "Fundstelle im PDF anzeigen";
|
| 166 |
+
btn.addEventListener("click", () => openSourceInViewer(source));
|
| 167 |
+
|
| 168 |
+
const icon = createEl("span", "source-link-icon", "📄");
|
| 169 |
+
icon.setAttribute("aria-hidden", "true");
|
| 170 |
+
btn.prepend(icon);
|
| 171 |
+
|
| 172 |
+
li.appendChild(btn);
|
| 173 |
+
} else {
|
| 174 |
+
li.textContent = rendered;
|
| 175 |
+
}
|
| 176 |
+
|
| 177 |
+
ul.appendChild(li);
|
| 178 |
+
}
|
| 179 |
+
|
| 180 |
+
if (!ul.children.length) return;
|
| 181 |
+
|
| 182 |
+
srcWrap.appendChild(strong);
|
| 183 |
+
srcWrap.appendChild(ul);
|
| 184 |
+
parent.appendChild(srcWrap);
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
// ----- Message rendering -----
|
| 188 |
+
|
| 189 |
+
// Erfasst [Quelle 3], [Quellen 1, 2] usw. – Zusatztext in der Klammer bleibt erhalten.
|
| 190 |
+
const INLINE_MARKER_RE = /\[Quellen?\s+\d+(?:[^\]]*)\]/g;
|
| 191 |
+
|
| 192 |
+
function renderAnswerText(container, answerText, sources) {
|
| 193 |
+
const text = safeText(answerText);
|
| 194 |
+
const bySourceNumber = buildSourceNumberMap(sources);
|
| 195 |
+
|
| 196 |
+
let lastIndex = 0;
|
| 197 |
+
|
| 198 |
+
for (const match of text.matchAll(INLINE_MARKER_RE)) {
|
| 199 |
+
if (match.index > lastIndex) {
|
| 200 |
+
container.appendChild(document.createTextNode(text.slice(lastIndex, match.index)));
|
| 201 |
+
}
|
| 202 |
+
|
| 203 |
+
const marker = match[0];
|
| 204 |
+
const numbers = (marker.match(/\d+/g) || []).map(Number);
|
| 205 |
+
const source = numbers.map(n => bySourceNumber.get(n)).find(Boolean);
|
| 206 |
+
|
| 207 |
+
if (source && isSourceOpenable(source)) {
|
| 208 |
+
const btn = createEl("button", "source-ref", marker);
|
| 209 |
+
btn.type = "button";
|
| 210 |
+
btn.title = "Fundstelle im PDF anzeigen";
|
| 211 |
+
btn.addEventListener("click", () => openSourceInViewer(source));
|
| 212 |
+
container.appendChild(btn);
|
| 213 |
+
} else {
|
| 214 |
+
container.appendChild(document.createTextNode(marker));
|
| 215 |
+
}
|
| 216 |
+
|
| 217 |
+
lastIndex = match.index + marker.length;
|
| 218 |
+
}
|
| 219 |
+
|
| 220 |
+
if (lastIndex < text.length) {
|
| 221 |
+
container.appendChild(document.createTextNode(text.slice(lastIndex)));
|
| 222 |
+
}
|
| 223 |
+
}
|
| 224 |
+
|
| 225 |
+
function appendQuestion(text) {
|
| 226 |
+
const wrap = createEl("div", "msg question");
|
| 227 |
+
wrap.textContent = "Frage:\n" + safeText(text);
|
| 228 |
+
messagesEl.appendChild(wrap);
|
| 229 |
+
}
|
| 230 |
+
|
| 231 |
+
function appendAnswer(answerText, sources) {
|
| 232 |
+
const wrap = createEl("div", "msg answer");
|
| 233 |
+
|
| 234 |
+
const answerDiv = createEl("div", "answer-text");
|
| 235 |
+
renderAnswerText(answerDiv, answerText, sources);
|
| 236 |
+
|
| 237 |
+
wrap.appendChild(answerDiv);
|
| 238 |
+
appendSources(wrap, sources);
|
| 239 |
+
|
| 240 |
+
messagesEl.appendChild(wrap);
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
function appendError(message) {
|
| 244 |
+
const wrap = createEl("div", "msg error");
|
| 245 |
+
wrap.textContent = safeText(message, "Es ist ein Fehler aufgetreten.");
|
| 246 |
+
messagesEl.appendChild(wrap);
|
| 247 |
+
}
|
| 248 |
+
|
| 249 |
+
// ----- Ask flow -----
|
| 250 |
+
|
| 251 |
+
async function send() {
|
| 252 |
+
const q = questionInput.value.trim();
|
| 253 |
+
if (!q) return;
|
| 254 |
+
|
| 255 |
+
appendQuestion(q);
|
| 256 |
+
questionInput.value = "";
|
| 257 |
+
setAskLoading(true);
|
| 258 |
+
|
| 259 |
+
try {
|
| 260 |
+
const res = await fetch("/ask", {
|
| 261 |
+
method: "POST",
|
| 262 |
+
headers: {
|
| 263 |
+
"Content-Type": "application/json"
|
| 264 |
+
},
|
| 265 |
+
body: JSON.stringify({ question: q })
|
| 266 |
+
});
|
| 267 |
+
|
| 268 |
+
let data = {};
|
| 269 |
+
try {
|
| 270 |
+
data = await res.json();
|
| 271 |
+
} catch {
|
| 272 |
+
throw new Error("Serverantwort war kein gültiges JSON.");
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
if (!res.ok) {
|
| 276 |
+
throw new Error(data.error || data.detail || `Serverfehler ${res.status}`);
|
| 277 |
+
}
|
| 278 |
+
|
| 279 |
+
appendAnswer(data.answer || "", data.sources || []);
|
| 280 |
+
} catch (err) {
|
| 281 |
+
appendError(
|
| 282 |
+
"Die Anfrage konnte nicht verarbeitet werden.\n" +
|
| 283 |
+
safeText(err && err.message ? err.message : err)
|
| 284 |
+
);
|
| 285 |
+
} finally {
|
| 286 |
+
setAskLoading(false);
|
| 287 |
+
scrollToBottom();
|
| 288 |
+
questionInput.focus();
|
| 289 |
+
}
|
| 290 |
+
}
|
| 291 |
+
|
| 292 |
+
// Optional: Enter sendet, Shift+Enter erzeugt Zeilenumbruch.
|
| 293 |
+
questionInput.addEventListener("keydown", event => {
|
| 294 |
+
if (event.key === "Enter" && !event.shiftKey) {
|
| 295 |
+
event.preventDefault();
|
| 296 |
+
send();
|
| 297 |
+
}
|
| 298 |
+
});
|
| 299 |
+
|
| 300 |
+
// ----- Prompt Modal -----
|
| 301 |
+
|
| 302 |
+
const backdrop = document.getElementById("promptModalBackdrop");
|
| 303 |
+
const promptTa = document.getElementById("systemPromptTextarea");
|
| 304 |
+
const promptStatus = document.getElementById("promptStatus");
|
| 305 |
+
|
| 306 |
+
let defaultPromptCache = "";
|
| 307 |
+
|
| 308 |
+
function setStatus(msg, ok = true) {
|
| 309 |
+
promptStatus.style.color = ok ? "#0a6" : "#b00";
|
| 310 |
+
promptStatus.textContent = msg || "";
|
| 311 |
+
}
|
| 312 |
+
|
| 313 |
+
function backdropClick(e) {
|
| 314 |
+
if (e.target === backdrop) {
|
| 315 |
+
closePromptModal();
|
| 316 |
+
}
|
| 317 |
+
}
|
| 318 |
+
|
| 319 |
+
async function openPromptModal() {
|
| 320 |
+
setStatus("");
|
| 321 |
+
backdrop.style.display = "flex";
|
| 322 |
+
|
| 323 |
+
try {
|
| 324 |
+
const res = await fetch("/system-prompt");
|
| 325 |
+
const data = await res.json();
|
| 326 |
+
|
| 327 |
+
if (!res.ok) {
|
| 328 |
+
throw new Error(data.error || data.detail || `Serverfehler ${res.status}`);
|
| 329 |
+
}
|
| 330 |
+
|
| 331 |
+
promptTa.value = data.system_prompt || "";
|
| 332 |
+
defaultPromptCache = data.default_system_prompt || "";
|
| 333 |
+
} catch (err) {
|
| 334 |
+
setStatus("Konnte System-Prompt nicht laden.", false);
|
| 335 |
+
}
|
| 336 |
+
}
|
| 337 |
+
|
| 338 |
+
function closePromptModal() {
|
| 339 |
+
backdrop.style.display = "none";
|
| 340 |
+
setStatus("");
|
| 341 |
+
}
|
| 342 |
+
|
| 343 |
+
async function savePrompt() {
|
| 344 |
+
setStatus("");
|
| 345 |
+
|
| 346 |
+
const newPrompt = (promptTa.value || "").trim();
|
| 347 |
+
|
| 348 |
+
if (!newPrompt) {
|
| 349 |
+
setStatus("Prompt ist leer – bitte Text einfügen.", false);
|
| 350 |
+
return;
|
| 351 |
+
}
|
| 352 |
+
|
| 353 |
+
try {
|
| 354 |
+
const res = await fetch("/system-prompt", {
|
| 355 |
+
method: "POST",
|
| 356 |
+
headers: {
|
| 357 |
+
"Content-Type": "application/json"
|
| 358 |
+
},
|
| 359 |
+
body: JSON.stringify({
|
| 360 |
+
system_prompt: newPrompt
|
| 361 |
+
})
|
| 362 |
+
});
|
| 363 |
+
|
| 364 |
+
const data = await res.json();
|
| 365 |
+
|
| 366 |
+
if (!res.ok) {
|
| 367 |
+
throw new Error(data.error || data.detail || `Serverfehler ${res.status}`);
|
| 368 |
+
}
|
| 369 |
+
|
| 370 |
+
promptTa.value = data.system_prompt || newPrompt;
|
| 371 |
+
setStatus("Gespeichert.");
|
| 372 |
+
} catch (err) {
|
| 373 |
+
setStatus("Speichern fehlgeschlagen.", false);
|
| 374 |
+
}
|
| 375 |
+
}
|
| 376 |
+
|
| 377 |
+
async function resetPrompt() {
|
| 378 |
+
if (!defaultPromptCache) {
|
| 379 |
+
await openPromptModal();
|
| 380 |
+
}
|
| 381 |
+
|
| 382 |
+
promptTa.value = defaultPromptCache || "";
|
| 383 |
+
await savePrompt();
|
| 384 |
+
}
|
| 385 |
+
|
| 386 |
+
// =============================================================================
|
| 387 |
+
// PDF-Fundstellen-Viewer
|
| 388 |
+
//
|
| 389 |
+
// Öffnet das Quell-PDF in einem Side-Panel, springt zur Seite der Fundstelle
|
| 390 |
+
// (page_start aus den Quellen-Metadaten) und hebt den Chunk-Text über den
|
| 391 |
+
// PDF.js-Text-Layer hervor. Das Matching ist bewusst fuzzy (normalisierter
|
| 392 |
+
// Text, satzweise), weil PDF-Extraktion und Ingest-Text leicht abweichen.
|
| 393 |
+
// =============================================================================
|
| 394 |
+
|
| 395 |
+
const pdfPanel = document.getElementById("pdfPanel");
|
| 396 |
+
const pdfViewportEl = document.getElementById("pdfViewport");
|
| 397 |
+
const pdfPageWrap = document.getElementById("pdfPageWrap");
|
| 398 |
+
const pdfCanvas = document.getElementById("pdfCanvas");
|
| 399 |
+
const pdfTextLayerEl = document.getElementById("pdfTextLayer");
|
| 400 |
+
const pdfStatusEl = document.getElementById("pdfStatus");
|
| 401 |
+
const pdfTitleEl = document.getElementById("pdfPanelTitle");
|
| 402 |
+
const pdfSubtitleEl = document.getElementById("pdfPanelSubtitle");
|
| 403 |
+
const pdfPageIndicator = document.getElementById("pdfPageIndicator");
|
| 404 |
+
const pdfPrevBtn = document.getElementById("pdfPrevBtn");
|
| 405 |
+
const pdfNextBtn = document.getElementById("pdfNextBtn");
|
| 406 |
+
const pdfJumpBtn = document.getElementById("pdfJumpBtn");
|
| 407 |
+
const pdfCloseBtn = document.getElementById("pdfCloseBtn");
|
| 408 |
+
const pdfOpenTab = document.getElementById("pdfOpenTab");
|
| 409 |
+
|
| 410 |
+
if (window.pdfjsLib) {
|
| 411 |
+
pdfjsLib.GlobalWorkerOptions.workerSrc =
|
| 412 |
+
"https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/pdf.worker.min.js";
|
| 413 |
+
}
|
| 414 |
+
|
| 415 |
+
const pdfState = {
|
| 416 |
+
doc: null,
|
| 417 |
+
url: null,
|
| 418 |
+
pageCount: 0,
|
| 419 |
+
currentPage: 1,
|
| 420 |
+
source: null,
|
| 421 |
+
highlightPages: [],
|
| 422 |
+
renderSeq: 0
|
| 423 |
+
};
|
| 424 |
+
|
| 425 |
+
function pdfUrlFor(source) {
|
| 426 |
+
return "/pdf/" + encodeURIComponent(safeText(source.source_file));
|
| 427 |
+
}
|
| 428 |
+
|
| 429 |
+
function setPdfStatus(message, isError = false) {
|
| 430 |
+
pdfStatusEl.textContent = safeText(message);
|
| 431 |
+
pdfStatusEl.classList.toggle("error", Boolean(isError));
|
| 432 |
+
pdfStatusEl.style.display = message ? "block" : "none";
|
| 433 |
+
}
|
| 434 |
+
|
| 435 |
+
function setPdfTitles(source) {
|
| 436 |
+
const refs = normalizeArray(source.canonical_refs).map(String).filter(Boolean);
|
| 437 |
+
const ref = refs.length ? refs.slice(0, 3).join("; ") : safeText(source.canonical_ref);
|
| 438 |
+
const path = safeText(source.path) || safeText(source.section);
|
| 439 |
+
|
| 440 |
+
pdfTitleEl.textContent = ref || path || "Fundstelle";
|
| 441 |
+
|
| 442 |
+
const parts = [];
|
| 443 |
+
if (path && path !== ref) parts.push(path);
|
| 444 |
+
if (source.source_file) parts.push(safeText(source.source_file));
|
| 445 |
+
const pageInfo = pdfPageRangeLabel(source);
|
| 446 |
+
if (pageInfo) parts.push(pageInfo);
|
| 447 |
+
pdfSubtitleEl.textContent = parts.join(" · ");
|
| 448 |
+
}
|
| 449 |
+
|
| 450 |
+
function pdfPageRangeLabel(source) {
|
| 451 |
+
const start = Number(source.page_start);
|
| 452 |
+
const end = Number(source.page_end);
|
| 453 |
+
if (!Number.isFinite(start)) return "";
|
| 454 |
+
if (Number.isFinite(end) && end !== start) return `Seiten ${start}–${end}`;
|
| 455 |
+
return `Seite ${start}`;
|
| 456 |
+
}
|
| 457 |
+
|
| 458 |
+
function collectHighlightPages(source) {
|
| 459 |
+
const pages = new Set();
|
| 460 |
+
|
| 461 |
+
for (const entry of normalizeArray(source.highlights)) {
|
| 462 |
+
const start = Number(entry && entry.page_start);
|
| 463 |
+
const end = Number(entry && entry.page_end);
|
| 464 |
+
if (Number.isFinite(start)) {
|
| 465 |
+
const last = Number.isFinite(end) ? end : start;
|
| 466 |
+
for (let p = start; p <= last && p - start < 20; p++) pages.add(p);
|
| 467 |
+
}
|
| 468 |
+
}
|
| 469 |
+
|
| 470 |
+
const srcStart = Number(source.page_start);
|
| 471 |
+
const srcEnd = Number(source.page_end);
|
| 472 |
+
if (Number.isFinite(srcStart)) {
|
| 473 |
+
const last = Number.isFinite(srcEnd) ? srcEnd : srcStart;
|
| 474 |
+
for (let p = srcStart; p <= last && p - srcStart < 20; p++) pages.add(p);
|
| 475 |
+
}
|
| 476 |
+
|
| 477 |
+
return [...pages].sort((a, b) => a - b);
|
| 478 |
+
}
|
| 479 |
+
|
| 480 |
+
function clampPdfPage(page) {
|
| 481 |
+
const n = Number(page);
|
| 482 |
+
if (!Number.isFinite(n)) return 1;
|
| 483 |
+
return Math.min(Math.max(1, Math.round(n)), pdfState.pageCount || 1);
|
| 484 |
+
}
|
| 485 |
+
|
| 486 |
+
function openPdfPanel() {
|
| 487 |
+
pdfPanel.classList.add("open");
|
| 488 |
+
pdfPanel.setAttribute("aria-hidden", "false");
|
| 489 |
+
}
|
| 490 |
+
|
| 491 |
+
function closePdfPanel() {
|
| 492 |
+
pdfPanel.classList.remove("open");
|
| 493 |
+
pdfPanel.setAttribute("aria-hidden", "true");
|
| 494 |
+
pdfState.renderSeq++;
|
| 495 |
+
}
|
| 496 |
+
|
| 497 |
+
function updatePdfNav() {
|
| 498 |
+
pdfPageIndicator.textContent = pdfState.pageCount
|
| 499 |
+
? `Seite ${pdfState.currentPage} / ${pdfState.pageCount}`
|
| 500 |
+
: "–";
|
| 501 |
+
pdfPrevBtn.disabled = pdfState.currentPage <= 1;
|
| 502 |
+
pdfNextBtn.disabled = pdfState.currentPage >= pdfState.pageCount;
|
| 503 |
+
pdfJumpBtn.style.display = pdfState.highlightPages.length ? "" : "none";
|
| 504 |
+
}
|
| 505 |
+
|
| 506 |
+
async function openSourceInViewer(source) {
|
| 507 |
+
if (!source || !source.source_file) return;
|
| 508 |
+
|
| 509 |
+
// Fallback ohne PDF.js (z. B. CDN blockiert) oder ohne Panel-Markup
|
| 510 |
+
// (veraltetes, gecachtes index.html): nativer Browser-Viewer im neuen Tab.
|
| 511 |
+
if (!window.pdfjsLib || !pdfPanel) {
|
| 512 |
+
const page = Number(source.page_start);
|
| 513 |
+
const anchor = Number.isFinite(page) ? `#page=${page}` : "";
|
| 514 |
+
window.open(pdfUrlFor(source) + anchor, "_blank", "noopener");
|
| 515 |
+
return;
|
| 516 |
+
}
|
| 517 |
+
|
| 518 |
+
openPdfPanel();
|
| 519 |
+
setPdfTitles(source);
|
| 520 |
+
|
| 521 |
+
const url = pdfUrlFor(source);
|
| 522 |
+
pdfOpenTab.href = url;
|
| 523 |
+
|
| 524 |
+
try {
|
| 525 |
+
if (!pdfState.doc || pdfState.url !== url) {
|
| 526 |
+
setPdfStatus("PDF wird geladen …");
|
| 527 |
+
pdfState.doc = await pdfjsLib.getDocument({ url }).promise;
|
| 528 |
+
pdfState.url = url;
|
| 529 |
+
pdfState.pageCount = pdfState.doc.numPages;
|
| 530 |
+
}
|
| 531 |
+
|
| 532 |
+
pdfState.source = source;
|
| 533 |
+
pdfState.highlightPages = collectHighlightPages(source);
|
| 534 |
+
|
| 535 |
+
setPdfStatus("");
|
| 536 |
+
const target = clampPdfPage(
|
| 537 |
+
Number.isFinite(Number(source.page_start))
|
| 538 |
+
? Number(source.page_start)
|
| 539 |
+
: (pdfState.highlightPages[0] || 1)
|
| 540 |
+
);
|
| 541 |
+
await renderPdfPage(target);
|
| 542 |
+
} catch (err) {
|
| 543 |
+
setPdfStatus(
|
| 544 |
+
"PDF konnte nicht geladen werden: " + safeText(err && err.message ? err.message : err),
|
| 545 |
+
true
|
| 546 |
+
);
|
| 547 |
+
}
|
| 548 |
+
}
|
| 549 |
+
|
| 550 |
+
async function renderPdfPage(pageNo) {
|
| 551 |
+
if (!pdfState.doc) return;
|
| 552 |
+
|
| 553 |
+
const seq = ++pdfState.renderSeq;
|
| 554 |
+
pdfState.currentPage = clampPdfPage(pageNo);
|
| 555 |
+
updatePdfNav();
|
| 556 |
+
|
| 557 |
+
const page = await pdfState.doc.getPage(pdfState.currentPage);
|
| 558 |
+
if (seq !== pdfState.renderSeq) return;
|
| 559 |
+
|
| 560 |
+
const availableWidth = Math.max(280, pdfViewportEl.clientWidth - 36);
|
| 561 |
+
const baseViewport = page.getViewport({ scale: 1 });
|
| 562 |
+
const scale = Math.min(2.5, availableWidth / baseViewport.width);
|
| 563 |
+
const viewport = page.getViewport({ scale });
|
| 564 |
+
|
| 565 |
+
const outputScale = window.devicePixelRatio || 1;
|
| 566 |
+
pdfCanvas.width = Math.floor(viewport.width * outputScale);
|
| 567 |
+
pdfCanvas.height = Math.floor(viewport.height * outputScale);
|
| 568 |
+
pdfCanvas.style.width = `${viewport.width}px`;
|
| 569 |
+
pdfCanvas.style.height = `${viewport.height}px`;
|
| 570 |
+
pdfPageWrap.style.width = `${viewport.width}px`;
|
| 571 |
+
pdfPageWrap.style.height = `${viewport.height}px`;
|
| 572 |
+
|
| 573 |
+
const renderTask = page.render({
|
| 574 |
+
canvasContext: pdfCanvas.getContext("2d"),
|
| 575 |
+
viewport,
|
| 576 |
+
transform: outputScale !== 1 ? [outputScale, 0, 0, outputScale, 0, 0] : null
|
| 577 |
+
});
|
| 578 |
+
|
| 579 |
+
const textContent = await page.getTextContent();
|
| 580 |
+
await renderTask.promise;
|
| 581 |
+
if (seq !== pdfState.renderSeq) return;
|
| 582 |
+
|
| 583 |
+
pdfTextLayerEl.innerHTML = "";
|
| 584 |
+
pdfTextLayerEl.style.width = `${viewport.width}px`;
|
| 585 |
+
pdfTextLayerEl.style.height = `${viewport.height}px`;
|
| 586 |
+
pdfTextLayerEl.style.setProperty("--scale-factor", String(viewport.scale));
|
| 587 |
+
|
| 588 |
+
const textDivs = [];
|
| 589 |
+
await pdfjsLib.renderTextLayer({
|
| 590 |
+
textContent,
|
| 591 |
+
container: pdfTextLayerEl,
|
| 592 |
+
viewport,
|
| 593 |
+
textDivs
|
| 594 |
+
}).promise;
|
| 595 |
+
if (seq !== pdfState.renderSeq) return;
|
| 596 |
+
|
| 597 |
+
const matched = applyPdfHighlights(pdfState.currentPage, textContent.items, textDivs);
|
| 598 |
+
|
| 599 |
+
if (matched) {
|
| 600 |
+
setPdfStatus("");
|
| 601 |
+
const first = pdfTextLayerEl.querySelector(".pdf-hl");
|
| 602 |
+
if (first) {
|
| 603 |
+
first.scrollIntoView({ block: "center", behavior: "smooth" });
|
| 604 |
+
}
|
| 605 |
+
} else if (pdfState.highlightPages.includes(pdfState.currentPage)) {
|
| 606 |
+
setPdfStatus(
|
| 607 |
+
"Fundstelle liegt auf dieser Seite – die genaue Textstelle konnte nicht automatisch markiert werden."
|
| 608 |
+
);
|
| 609 |
+
} else {
|
| 610 |
+
setPdfStatus("");
|
| 611 |
+
}
|
| 612 |
+
}
|
| 613 |
+
|
| 614 |
+
// ----- Text-Matching für die Hervorhebung -----
|
| 615 |
+
|
| 616 |
+
function pdfNormText(value) {
|
| 617 |
+
return safeText(value)
|
| 618 |
+
.toLowerCase()
|
| 619 |
+
.replace(//g, "")
|
| 620 |
+
.replace(/[^\p{L}\p{N}§]+/gu, " ")
|
| 621 |
+
.replace(/\s+/g, " ")
|
| 622 |
+
.trim();
|
| 623 |
+
}
|
| 624 |
+
|
| 625 |
+
function pdfSegmentsOf(text) {
|
| 626 |
+
const out = [];
|
| 627 |
+
for (const block of safeText(text).split(/\n{2,}/)) {
|
| 628 |
+
const flat = block.replace(/\s+/g, " ").trim();
|
| 629 |
+
if (!flat) continue;
|
| 630 |
+
|
| 631 |
+
const sentences = flat.split(/(?<=[.;:!?])\s+/);
|
| 632 |
+
let pushed = false;
|
| 633 |
+
for (const sentence of sentences) {
|
| 634 |
+
const s = sentence.trim();
|
| 635 |
+
if (s.length >= 12) {
|
| 636 |
+
out.push(s);
|
| 637 |
+
pushed = true;
|
| 638 |
+
}
|
| 639 |
+
}
|
| 640 |
+
if (!pushed && flat.length >= 12) out.push(flat);
|
| 641 |
+
}
|
| 642 |
+
return out;
|
| 643 |
+
}
|
| 644 |
+
|
| 645 |
+
function applyPdfHighlights(pageNo, items, textDivs) {
|
| 646 |
+
const source = pdfState.source;
|
| 647 |
+
if (!source) return false;
|
| 648 |
+
|
| 649 |
+
const entries = normalizeArray(source.highlights).filter(entry => {
|
| 650 |
+
if (!entry || !entry.text) return false;
|
| 651 |
+
const start = Number(entry.page_start);
|
| 652 |
+
const end = Number(entry.page_end);
|
| 653 |
+
if (!Number.isFinite(start)) return true; // ohne Seitenangabe: überall versuchen
|
| 654 |
+
return pageNo >= start && pageNo <= (Number.isFinite(end) ? end : start);
|
| 655 |
+
});
|
| 656 |
+
if (!entries.length) return false;
|
| 657 |
+
|
| 658 |
+
// Seitentext mit Index-Map über die Text-Layer-Items aufbauen.
|
| 659 |
+
// Leere Items (reine Whitespace-Fragmente) dürfen keine zusätzlichen
|
| 660 |
+
// Leerzeichen erzeugen, sonst scheitert das Substring-Matching.
|
| 661 |
+
let pageStr = "";
|
| 662 |
+
const spans = [];
|
| 663 |
+
items.forEach((item, i) => {
|
| 664 |
+
const t = pdfNormText(item.str);
|
| 665 |
+
if (!t) {
|
| 666 |
+
spans.push({ start: pageStr.length, end: pageStr.length, i });
|
| 667 |
+
return;
|
| 668 |
+
}
|
| 669 |
+
spans.push({ start: pageStr.length, end: pageStr.length + t.length, i });
|
| 670 |
+
pageStr += t + " ";
|
| 671 |
+
});
|
| 672 |
+
|
| 673 |
+
const markRange = (a, b) => {
|
| 674 |
+
let any = false;
|
| 675 |
+
for (const span of spans) {
|
| 676 |
+
if (span.end <= span.start) continue;
|
| 677 |
+
if (span.end > a && span.start < b) {
|
| 678 |
+
const div = textDivs[span.i];
|
| 679 |
+
if (div && div.textContent.trim()) {
|
| 680 |
+
div.classList.add("pdf-hl");
|
| 681 |
+
any = true;
|
| 682 |
+
}
|
| 683 |
+
}
|
| 684 |
+
}
|
| 685 |
+
return any;
|
| 686 |
+
};
|
| 687 |
+
|
| 688 |
+
let matchedAny = false;
|
| 689 |
+
|
| 690 |
+
for (const entry of entries) {
|
| 691 |
+
for (const segment of pdfSegmentsOf(entry.text)) {
|
| 692 |
+
const needle = pdfNormText(segment);
|
| 693 |
+
if (needle.length < 8) continue;
|
| 694 |
+
|
| 695 |
+
const idx = pageStr.indexOf(needle);
|
| 696 |
+
if (idx >= 0) {
|
| 697 |
+
matchedAny = markRange(idx, idx + needle.length) || matchedAny;
|
| 698 |
+
continue;
|
| 699 |
+
}
|
| 700 |
+
|
| 701 |
+
// Fallback für längere Segmente: Anfangs-, Mittel- und End-Sonden, damit
|
| 702 |
+
// Silbentrennung/Extraktionsunterschiede nicht das ganze Segment kosten.
|
| 703 |
+
if (needle.length > 40) {
|
| 704 |
+
const mid = Math.floor(needle.length / 2);
|
| 705 |
+
for (const probe of [needle.slice(0, 40), needle.slice(mid, mid + 40), needle.slice(-40)]) {
|
| 706 |
+
const trimmed = probe.trim();
|
| 707 |
+
if (trimmed.length < 16) continue;
|
| 708 |
+
const j = pageStr.indexOf(trimmed);
|
| 709 |
+
if (j >= 0) {
|
| 710 |
+
matchedAny = markRange(j, j + trimmed.length) || matchedAny;
|
| 711 |
+
}
|
| 712 |
+
}
|
| 713 |
+
}
|
| 714 |
+
}
|
| 715 |
+
}
|
| 716 |
+
|
| 717 |
+
return matchedAny;
|
| 718 |
+
}
|
| 719 |
+
|
| 720 |
+
// ----- Panel-Bedienung -----
|
| 721 |
+
// Guard: Bei veraltetem, gecachtem index.html fehlen die Panel-Elemente;
|
| 722 |
+
// dann werden keine Listener registriert (openSourceInViewer nutzt den
|
| 723 |
+
// Neuer-Tab-Fallback) und der restliche Chat bleibt voll funktionsfähig.
|
| 724 |
+
|
| 725 |
+
if (pdfPanel && pdfPrevBtn && pdfNextBtn && pdfJumpBtn && pdfCloseBtn) {
|
| 726 |
+
pdfPrevBtn.addEventListener("click", () => {
|
| 727 |
+
if (pdfState.currentPage > 1) renderPdfPage(pdfState.currentPage - 1);
|
| 728 |
+
});
|
| 729 |
+
|
| 730 |
+
pdfNextBtn.addEventListener("click", () => {
|
| 731 |
+
if (pdfState.currentPage < pdfState.pageCount) renderPdfPage(pdfState.currentPage + 1);
|
| 732 |
+
});
|
| 733 |
+
|
| 734 |
+
pdfJumpBtn.addEventListener("click", () => {
|
| 735 |
+
if (pdfState.highlightPages.length) renderPdfPage(pdfState.highlightPages[0]);
|
| 736 |
+
});
|
| 737 |
+
|
| 738 |
+
pdfCloseBtn.addEventListener("click", closePdfPanel);
|
| 739 |
+
|
| 740 |
+
document.addEventListener("keydown", event => {
|
| 741 |
+
if (event.key === "Escape" && pdfPanel.classList.contains("open")) {
|
| 742 |
+
closePdfPanel();
|
| 743 |
+
}
|
| 744 |
+
});
|
| 745 |
+
}
|
static/styles/style.css
ADDED
|
@@ -0,0 +1,882 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
:root {
|
| 2 |
+
--bg: #f5f6f8;
|
| 3 |
+
--surface: rgba(255, 255, 255, 0.72);
|
| 4 |
+
--surface-strong: rgba(255, 255, 255, 0.88);
|
| 5 |
+
--surface-soft: rgba(255, 255, 255, 0.52);
|
| 6 |
+
|
| 7 |
+
--text: #111827;
|
| 8 |
+
--text-muted: #6b7280;
|
| 9 |
+
|
| 10 |
+
--border: rgba(17, 24, 39, 0.08);
|
| 11 |
+
--border-light: rgba(255, 255, 255, 0.65);
|
| 12 |
+
|
| 13 |
+
--shadow-soft: 0 18px 50px rgba(15, 23, 42, 0.08);
|
| 14 |
+
--shadow-card: 0 24px 70px rgba(15, 23, 42, 0.10);
|
| 15 |
+
|
| 16 |
+
--accent: #111827;
|
| 17 |
+
--accent-hover: #1f2937;
|
| 18 |
+
|
| 19 |
+
--radius-xl: 28px;
|
| 20 |
+
--radius-lg: 20px;
|
| 21 |
+
--radius-md: 14px;
|
| 22 |
+
|
| 23 |
+
--transition: 180ms ease;
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
* {
|
| 27 |
+
box-sizing: border-box;
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
html {
|
| 31 |
+
min-height: 100%;
|
| 32 |
+
scroll-behavior: smooth;
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
body {
|
| 36 |
+
min-height: 100vh;
|
| 37 |
+
margin: 0;
|
| 38 |
+
padding: 0;
|
| 39 |
+
overflow-x: hidden;
|
| 40 |
+
|
| 41 |
+
font-family:
|
| 42 |
+
-apple-system,
|
| 43 |
+
BlinkMacSystemFont,
|
| 44 |
+
"SF Pro Display",
|
| 45 |
+
"SF Pro Text",
|
| 46 |
+
"Segoe UI",
|
| 47 |
+
Roboto,
|
| 48 |
+
Arial,
|
| 49 |
+
sans-serif;
|
| 50 |
+
|
| 51 |
+
color: var(--text);
|
| 52 |
+
|
| 53 |
+
background:
|
| 54 |
+
radial-gradient(circle at 18% 4%, rgba(191, 219, 254, 0.48), transparent 34%),
|
| 55 |
+
radial-gradient(circle at 84% 3%, rgba(221, 214, 254, 0.42), transparent 32%),
|
| 56 |
+
radial-gradient(circle at 64% 76%, rgba(207, 250, 254, 0.34), transparent 40%),
|
| 57 |
+
linear-gradient(180deg, #f8fafc 0%, #f3f6fb 52%, #f7f8fb 100%);
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
/* Dezentes Premium-Grid */
|
| 61 |
+
body::before {
|
| 62 |
+
content: "";
|
| 63 |
+
position: fixed;
|
| 64 |
+
inset: 0;
|
| 65 |
+
pointer-events: none;
|
| 66 |
+
|
| 67 |
+
background-image:
|
| 68 |
+
linear-gradient(rgba(255, 255, 255, 0.30) 1px, transparent 1px),
|
| 69 |
+
linear-gradient(90deg, rgba(255, 255, 255, 0.30) 1px, transparent 1px);
|
| 70 |
+
background-size: 58px 58px;
|
| 71 |
+
|
| 72 |
+
opacity: 0.55;
|
| 73 |
+
|
| 74 |
+
mask-image: linear-gradient(
|
| 75 |
+
to bottom,
|
| 76 |
+
rgba(0, 0, 0, 0.48),
|
| 77 |
+
rgba(0, 0, 0, 0.28) 45%,
|
| 78 |
+
transparent 82%
|
| 79 |
+
);
|
| 80 |
+
}
|
| 81 |
+
|
| 82 |
+
/* Header */
|
| 83 |
+
|
| 84 |
+
.header {
|
| 85 |
+
max-width: 900px;
|
| 86 |
+
display: flex;
|
| 87 |
+
justify-content: center;
|
| 88 |
+
align-items: center;
|
| 89 |
+
|
| 90 |
+
margin: 30px auto 18px auto;
|
| 91 |
+
padding: 14px 24px;
|
| 92 |
+
|
| 93 |
+
border-radius: 999px;
|
| 94 |
+
|
| 95 |
+
background: rgba(255, 255, 255, 0.28);
|
| 96 |
+
border: 1px solid rgba(255, 255, 255, 0.55);
|
| 97 |
+
|
| 98 |
+
box-shadow: 0 14px 44px rgba(15, 23, 42, 0.06);
|
| 99 |
+
|
| 100 |
+
backdrop-filter: blur(18px) saturate(140%);
|
| 101 |
+
-webkit-backdrop-filter: blur(18px) saturate(140%);
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
.logo {
|
| 105 |
+
max-height: 72px;
|
| 106 |
+
width: auto;
|
| 107 |
+
display: block;
|
| 108 |
+
background: transparent;
|
| 109 |
+
filter: drop-shadow(0 8px 18px rgba(15, 23, 42, 0.06));
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
/* Hauptkarte */
|
| 113 |
+
|
| 114 |
+
.chat {
|
| 115 |
+
position: relative;
|
| 116 |
+
|
| 117 |
+
width: min(900px, calc(100% - 32px));
|
| 118 |
+
margin: 28px auto 54px auto;
|
| 119 |
+
padding: 34px 34px 32px;
|
| 120 |
+
|
| 121 |
+
border-radius: var(--radius-xl);
|
| 122 |
+
border: 1px solid var(--border-light);
|
| 123 |
+
|
| 124 |
+
background: rgba(255, 255, 255, 0.72);
|
| 125 |
+
|
| 126 |
+
box-shadow: var(--shadow-card);
|
| 127 |
+
|
| 128 |
+
backdrop-filter: blur(26px) saturate(150%);
|
| 129 |
+
-webkit-backdrop-filter: blur(26px) saturate(150%);
|
| 130 |
+
|
| 131 |
+
overflow: hidden;
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
/* Sehr dezenter Lichtverlauf innerhalb der Karte */
|
| 135 |
+
.chat::before {
|
| 136 |
+
content: "";
|
| 137 |
+
position: absolute;
|
| 138 |
+
inset: 0;
|
| 139 |
+
pointer-events: none;
|
| 140 |
+
|
| 141 |
+
background: linear-gradient(
|
| 142 |
+
135deg,
|
| 143 |
+
rgba(255, 255, 255, 0.48),
|
| 144 |
+
rgba(255, 255, 255, 0.08)
|
| 145 |
+
);
|
| 146 |
+
}
|
| 147 |
+
|
| 148 |
+
.chat > * {
|
| 149 |
+
position: relative;
|
| 150 |
+
z-index: 1;
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
h2 {
|
| 154 |
+
margin: 0 0 24px 0;
|
| 155 |
+
|
| 156 |
+
font-size: clamp(1.65rem, 3vw, 2.35rem);
|
| 157 |
+
line-height: 1.08;
|
| 158 |
+
letter-spacing: -0.045em;
|
| 159 |
+
font-weight: 760;
|
| 160 |
+
|
| 161 |
+
color: #111827;
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
h2::after {
|
| 165 |
+
content: "KI-gestützte juristische Assistenz";
|
| 166 |
+
display: block;
|
| 167 |
+
|
| 168 |
+
margin-top: 10px;
|
| 169 |
+
|
| 170 |
+
font-size: 0.92rem;
|
| 171 |
+
line-height: 1.4;
|
| 172 |
+
letter-spacing: 0;
|
| 173 |
+
font-weight: 500;
|
| 174 |
+
|
| 175 |
+
color: var(--text-muted);
|
| 176 |
+
}
|
| 177 |
+
|
| 178 |
+
/* Nachrichten */
|
| 179 |
+
|
| 180 |
+
.messages {
|
| 181 |
+
display: flex;
|
| 182 |
+
flex-direction: column;
|
| 183 |
+
gap: 14px;
|
| 184 |
+
|
| 185 |
+
margin-bottom: 22px;
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
.msg {
|
| 189 |
+
margin-bottom: 0;
|
| 190 |
+
animation: messageIn 220ms ease both;
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
.question {
|
| 194 |
+
align-self: flex-end;
|
| 195 |
+
|
| 196 |
+
max-width: 82%;
|
| 197 |
+
padding: 14px 16px;
|
| 198 |
+
|
| 199 |
+
border-radius: 20px 20px 6px 20px;
|
| 200 |
+
|
| 201 |
+
color: #ffffff;
|
| 202 |
+
font-weight: 650;
|
| 203 |
+
line-height: 1.5;
|
| 204 |
+
white-space: pre-wrap;
|
| 205 |
+
|
| 206 |
+
background: #1f2937;
|
| 207 |
+
|
| 208 |
+
box-shadow: 0 12px 30px rgba(15, 23, 42, 0.16);
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
.answer {
|
| 212 |
+
align-self: flex-start;
|
| 213 |
+
|
| 214 |
+
max-width: 92%;
|
| 215 |
+
padding: 17px 18px;
|
| 216 |
+
|
| 217 |
+
white-space: pre-wrap;
|
| 218 |
+
line-height: 1.65;
|
| 219 |
+
|
| 220 |
+
border-radius: 20px 20px 20px 6px;
|
| 221 |
+
border: 1px solid rgba(17, 24, 39, 0.06);
|
| 222 |
+
|
| 223 |
+
background: rgba(255, 255, 255, 0.74);
|
| 224 |
+
|
| 225 |
+
box-shadow: 0 12px 32px rgba(15, 23, 42, 0.06);
|
| 226 |
+
|
| 227 |
+
backdrop-filter: blur(16px) saturate(140%);
|
| 228 |
+
-webkit-backdrop-filter: blur(16px) saturate(140%);
|
| 229 |
+
}
|
| 230 |
+
|
| 231 |
+
/* Eingabe */
|
| 232 |
+
|
| 233 |
+
textarea {
|
| 234 |
+
width: 100%;
|
| 235 |
+
min-height: 104px;
|
| 236 |
+
|
| 237 |
+
margin-bottom: 14px;
|
| 238 |
+
padding: 17px 18px;
|
| 239 |
+
|
| 240 |
+
resize: vertical;
|
| 241 |
+
|
| 242 |
+
border: 1px solid rgba(17, 24, 39, 0.09);
|
| 243 |
+
border-radius: 20px;
|
| 244 |
+
outline: none;
|
| 245 |
+
|
| 246 |
+
color: var(--text);
|
| 247 |
+
font: inherit;
|
| 248 |
+
line-height: 1.55;
|
| 249 |
+
|
| 250 |
+
background: rgba(255, 255, 255, 0.70);
|
| 251 |
+
|
| 252 |
+
box-shadow:
|
| 253 |
+
inset 0 1px 0 rgba(255, 255, 255, 0.75),
|
| 254 |
+
0 10px 28px rgba(15, 23, 42, 0.045);
|
| 255 |
+
|
| 256 |
+
backdrop-filter: blur(14px) saturate(135%);
|
| 257 |
+
-webkit-backdrop-filter: blur(14px) saturate(135%);
|
| 258 |
+
|
| 259 |
+
transition:
|
| 260 |
+
border-color var(--transition),
|
| 261 |
+
box-shadow var(--transition),
|
| 262 |
+
background var(--transition);
|
| 263 |
+
}
|
| 264 |
+
|
| 265 |
+
textarea::placeholder {
|
| 266 |
+
color: rgba(100, 116, 139, 0.72);
|
| 267 |
+
}
|
| 268 |
+
|
| 269 |
+
textarea:focus {
|
| 270 |
+
border-color: rgba(17, 24, 39, 0.18);
|
| 271 |
+
background: rgba(255, 255, 255, 0.88);
|
| 272 |
+
|
| 273 |
+
box-shadow:
|
| 274 |
+
0 0 0 4px rgba(17, 24, 39, 0.045),
|
| 275 |
+
0 16px 38px rgba(15, 23, 42, 0.075);
|
| 276 |
+
}
|
| 277 |
+
|
| 278 |
+
/* Buttons */
|
| 279 |
+
|
| 280 |
+
.row {
|
| 281 |
+
display: flex;
|
| 282 |
+
gap: 12px;
|
| 283 |
+
align-items: center;
|
| 284 |
+
flex-wrap: wrap;
|
| 285 |
+
}
|
| 286 |
+
|
| 287 |
+
button {
|
| 288 |
+
appearance: none;
|
| 289 |
+
|
| 290 |
+
border: 0;
|
| 291 |
+
border-radius: 999px;
|
| 292 |
+
|
| 293 |
+
padding: 12px 20px;
|
| 294 |
+
|
| 295 |
+
cursor: pointer;
|
| 296 |
+
|
| 297 |
+
font: inherit;
|
| 298 |
+
font-weight: 650;
|
| 299 |
+
letter-spacing: -0.01em;
|
| 300 |
+
|
| 301 |
+
transition:
|
| 302 |
+
transform var(--transition),
|
| 303 |
+
box-shadow var(--transition),
|
| 304 |
+
background var(--transition),
|
| 305 |
+
color var(--transition),
|
| 306 |
+
opacity var(--transition);
|
| 307 |
+
}
|
| 308 |
+
|
| 309 |
+
button:hover {
|
| 310 |
+
transform: translateY(-1px);
|
| 311 |
+
}
|
| 312 |
+
|
| 313 |
+
button:active {
|
| 314 |
+
transform: translateY(0) scale(0.985);
|
| 315 |
+
}
|
| 316 |
+
|
| 317 |
+
button:focus-visible {
|
| 318 |
+
outline: 4px solid rgba(17, 24, 39, 0.10);
|
| 319 |
+
outline-offset: 3px;
|
| 320 |
+
}
|
| 321 |
+
|
| 322 |
+
#askBtn {
|
| 323 |
+
color: #ffffff;
|
| 324 |
+
background: #1f2937;
|
| 325 |
+
box-shadow: 0 14px 30px rgba(15, 23, 42, 0.18);
|
| 326 |
+
}
|
| 327 |
+
|
| 328 |
+
#askBtn:hover {
|
| 329 |
+
background: #111827;
|
| 330 |
+
box-shadow: 0 18px 36px rgba(15, 23, 42, 0.22);
|
| 331 |
+
}
|
| 332 |
+
|
| 333 |
+
#promptBtn,
|
| 334 |
+
.modal-actions button {
|
| 335 |
+
color: #111827;
|
| 336 |
+
|
| 337 |
+
background: rgba(255, 255, 255, 0.68);
|
| 338 |
+
|
| 339 |
+
border: 1px solid rgba(17, 24, 39, 0.075);
|
| 340 |
+
|
| 341 |
+
box-shadow:
|
| 342 |
+
inset 0 1px 0 rgba(255, 255, 255, 0.78),
|
| 343 |
+
0 10px 24px rgba(15, 23, 42, 0.055);
|
| 344 |
+
|
| 345 |
+
backdrop-filter: blur(14px) saturate(135%);
|
| 346 |
+
-webkit-backdrop-filter: blur(14px) saturate(135%);
|
| 347 |
+
}
|
| 348 |
+
|
| 349 |
+
#promptBtn:hover,
|
| 350 |
+
.modal-actions button:hover {
|
| 351 |
+
background: rgba(255, 255, 255, 0.86);
|
| 352 |
+
|
| 353 |
+
box-shadow:
|
| 354 |
+
inset 0 1px 0 rgba(255, 255, 255, 0.86),
|
| 355 |
+
0 14px 30px rgba(15, 23, 42, 0.08);
|
| 356 |
+
}
|
| 357 |
+
|
| 358 |
+
/* Quellen */
|
| 359 |
+
|
| 360 |
+
.sources {
|
| 361 |
+
margin-top: 14px;
|
| 362 |
+
padding-top: 12px;
|
| 363 |
+
|
| 364 |
+
font-size: 0.9em;
|
| 365 |
+
color: var(--text-muted);
|
| 366 |
+
|
| 367 |
+
border-top: 1px solid rgba(17, 24, 39, 0.075);
|
| 368 |
+
}
|
| 369 |
+
|
| 370 |
+
.sources strong {
|
| 371 |
+
color: #374151;
|
| 372 |
+
}
|
| 373 |
+
|
| 374 |
+
.sources ul {
|
| 375 |
+
margin: 8px 0 0 18px;
|
| 376 |
+
padding: 0;
|
| 377 |
+
}
|
| 378 |
+
|
| 379 |
+
.sources li {
|
| 380 |
+
margin: 4px 0;
|
| 381 |
+
}
|
| 382 |
+
|
| 383 |
+
/* Klickbare Quellen */
|
| 384 |
+
|
| 385 |
+
.source-link {
|
| 386 |
+
display: inline-flex;
|
| 387 |
+
align-items: center;
|
| 388 |
+
gap: 8px;
|
| 389 |
+
|
| 390 |
+
width: auto;
|
| 391 |
+
padding: 6px 12px;
|
| 392 |
+
|
| 393 |
+
border: 1px solid rgba(17, 24, 39, 0.10);
|
| 394 |
+
border-radius: 12px;
|
| 395 |
+
|
| 396 |
+
font: inherit;
|
| 397 |
+
font-size: 1em;
|
| 398 |
+
font-weight: 500;
|
| 399 |
+
text-align: left;
|
| 400 |
+
color: #1d4ed8;
|
| 401 |
+
|
| 402 |
+
background: rgba(255, 255, 255, 0.55);
|
| 403 |
+
box-shadow: none;
|
| 404 |
+
|
| 405 |
+
cursor: pointer;
|
| 406 |
+
}
|
| 407 |
+
|
| 408 |
+
.source-link:hover {
|
| 409 |
+
color: #1e40af;
|
| 410 |
+
border-color: rgba(29, 78, 216, 0.35);
|
| 411 |
+
background: rgba(219, 234, 254, 0.55);
|
| 412 |
+
transform: none;
|
| 413 |
+
}
|
| 414 |
+
|
| 415 |
+
.source-link-icon {
|
| 416 |
+
flex: none;
|
| 417 |
+
font-size: 0.95em;
|
| 418 |
+
}
|
| 419 |
+
|
| 420 |
+
/* Inline-Quellenmarker im Antworttext */
|
| 421 |
+
|
| 422 |
+
.source-ref {
|
| 423 |
+
display: inline;
|
| 424 |
+
padding: 1px 6px;
|
| 425 |
+
margin: 0 1px;
|
| 426 |
+
|
| 427 |
+
border: 1px solid rgba(29, 78, 216, 0.22);
|
| 428 |
+
border-radius: 8px;
|
| 429 |
+
|
| 430 |
+
font: inherit;
|
| 431 |
+
font-size: 0.88em;
|
| 432 |
+
font-weight: 600;
|
| 433 |
+
color: #1d4ed8;
|
| 434 |
+
|
| 435 |
+
background: rgba(219, 234, 254, 0.45);
|
| 436 |
+
box-shadow: none;
|
| 437 |
+
|
| 438 |
+
cursor: pointer;
|
| 439 |
+
white-space: nowrap;
|
| 440 |
+
}
|
| 441 |
+
|
| 442 |
+
.source-ref:hover {
|
| 443 |
+
color: #1e40af;
|
| 444 |
+
background: rgba(191, 219, 254, 0.75);
|
| 445 |
+
border-color: rgba(29, 78, 216, 0.45);
|
| 446 |
+
transform: none;
|
| 447 |
+
}
|
| 448 |
+
|
| 449 |
+
/* PDF-Fundstellen-Viewer (Side-Panel) */
|
| 450 |
+
|
| 451 |
+
.pdf-panel {
|
| 452 |
+
position: fixed;
|
| 453 |
+
top: 0;
|
| 454 |
+
right: 0;
|
| 455 |
+
bottom: 0;
|
| 456 |
+
|
| 457 |
+
width: min(760px, 100vw);
|
| 458 |
+
|
| 459 |
+
display: flex;
|
| 460 |
+
flex-direction: column;
|
| 461 |
+
|
| 462 |
+
z-index: 9000;
|
| 463 |
+
|
| 464 |
+
border-left: 1px solid rgba(255, 255, 255, 0.65);
|
| 465 |
+
|
| 466 |
+
background: rgba(248, 250, 252, 0.92);
|
| 467 |
+
|
| 468 |
+
box-shadow: -24px 0 70px rgba(15, 23, 42, 0.22);
|
| 469 |
+
|
| 470 |
+
backdrop-filter: blur(26px) saturate(150%);
|
| 471 |
+
-webkit-backdrop-filter: blur(26px) saturate(150%);
|
| 472 |
+
|
| 473 |
+
transform: translateX(105%);
|
| 474 |
+
transition: transform 260ms ease;
|
| 475 |
+
}
|
| 476 |
+
|
| 477 |
+
.pdf-panel.open {
|
| 478 |
+
transform: translateX(0);
|
| 479 |
+
}
|
| 480 |
+
|
| 481 |
+
.pdf-panel-header {
|
| 482 |
+
display: flex;
|
| 483 |
+
align-items: center;
|
| 484 |
+
justify-content: space-between;
|
| 485 |
+
gap: 12px;
|
| 486 |
+
|
| 487 |
+
padding: 14px 18px;
|
| 488 |
+
|
| 489 |
+
border-bottom: 1px solid rgba(17, 24, 39, 0.08);
|
| 490 |
+
|
| 491 |
+
background: rgba(255, 255, 255, 0.66);
|
| 492 |
+
}
|
| 493 |
+
|
| 494 |
+
.pdf-panel-titles {
|
| 495 |
+
min-width: 0;
|
| 496 |
+
}
|
| 497 |
+
|
| 498 |
+
.pdf-panel-title {
|
| 499 |
+
font-weight: 700;
|
| 500 |
+
letter-spacing: -0.02em;
|
| 501 |
+
color: #111827;
|
| 502 |
+
|
| 503 |
+
white-space: nowrap;
|
| 504 |
+
overflow: hidden;
|
| 505 |
+
text-overflow: ellipsis;
|
| 506 |
+
}
|
| 507 |
+
|
| 508 |
+
.pdf-panel-subtitle {
|
| 509 |
+
margin-top: 2px;
|
| 510 |
+
|
| 511 |
+
font-size: 0.82em;
|
| 512 |
+
color: var(--text-muted);
|
| 513 |
+
|
| 514 |
+
white-space: nowrap;
|
| 515 |
+
overflow: hidden;
|
| 516 |
+
text-overflow: ellipsis;
|
| 517 |
+
}
|
| 518 |
+
|
| 519 |
+
.pdf-panel-controls {
|
| 520 |
+
display: flex;
|
| 521 |
+
align-items: center;
|
| 522 |
+
gap: 6px;
|
| 523 |
+
flex: none;
|
| 524 |
+
}
|
| 525 |
+
|
| 526 |
+
.pdf-ctrl {
|
| 527 |
+
display: inline-flex;
|
| 528 |
+
align-items: center;
|
| 529 |
+
justify-content: center;
|
| 530 |
+
|
| 531 |
+
min-width: 34px;
|
| 532 |
+
height: 34px;
|
| 533 |
+
padding: 0 10px;
|
| 534 |
+
|
| 535 |
+
border: 1px solid rgba(17, 24, 39, 0.10);
|
| 536 |
+
border-radius: 10px;
|
| 537 |
+
|
| 538 |
+
font: inherit;
|
| 539 |
+
font-size: 0.95em;
|
| 540 |
+
font-weight: 600;
|
| 541 |
+
color: #111827;
|
| 542 |
+
text-decoration: none;
|
| 543 |
+
|
| 544 |
+
background: rgba(255, 255, 255, 0.72);
|
| 545 |
+
box-shadow: none;
|
| 546 |
+
|
| 547 |
+
cursor: pointer;
|
| 548 |
+
}
|
| 549 |
+
|
| 550 |
+
.pdf-ctrl:hover {
|
| 551 |
+
background: rgba(255, 255, 255, 0.95);
|
| 552 |
+
transform: none;
|
| 553 |
+
}
|
| 554 |
+
|
| 555 |
+
.pdf-ctrl:disabled {
|
| 556 |
+
opacity: 0.4;
|
| 557 |
+
cursor: default;
|
| 558 |
+
}
|
| 559 |
+
|
| 560 |
+
.pdf-jump {
|
| 561 |
+
color: #92400e;
|
| 562 |
+
border-color: rgba(217, 119, 6, 0.35);
|
| 563 |
+
background: rgba(254, 243, 199, 0.75);
|
| 564 |
+
}
|
| 565 |
+
|
| 566 |
+
.pdf-jump:hover {
|
| 567 |
+
background: rgba(253, 230, 138, 0.9);
|
| 568 |
+
}
|
| 569 |
+
|
| 570 |
+
.pdf-close {
|
| 571 |
+
font-size: 1.25em;
|
| 572 |
+
}
|
| 573 |
+
|
| 574 |
+
.pdf-page-indicator {
|
| 575 |
+
padding: 0 4px;
|
| 576 |
+
|
| 577 |
+
font-size: 0.85em;
|
| 578 |
+
font-weight: 600;
|
| 579 |
+
color: var(--text-muted);
|
| 580 |
+
|
| 581 |
+
white-space: nowrap;
|
| 582 |
+
}
|
| 583 |
+
|
| 584 |
+
.pdf-viewport {
|
| 585 |
+
flex: 1;
|
| 586 |
+
overflow: auto;
|
| 587 |
+
|
| 588 |
+
padding: 18px;
|
| 589 |
+
}
|
| 590 |
+
|
| 591 |
+
.pdf-status {
|
| 592 |
+
display: none;
|
| 593 |
+
|
| 594 |
+
margin: 0 auto 12px auto;
|
| 595 |
+
padding: 10px 14px;
|
| 596 |
+
|
| 597 |
+
max-width: 640px;
|
| 598 |
+
|
| 599 |
+
border: 1px solid rgba(217, 119, 6, 0.25);
|
| 600 |
+
border-radius: 12px;
|
| 601 |
+
|
| 602 |
+
font-size: 0.88em;
|
| 603 |
+
color: #92400e;
|
| 604 |
+
|
| 605 |
+
background: rgba(254, 243, 199, 0.8);
|
| 606 |
+
}
|
| 607 |
+
|
| 608 |
+
.pdf-status.error {
|
| 609 |
+
color: #b91c1c;
|
| 610 |
+
border-color: rgba(185, 28, 28, 0.25);
|
| 611 |
+
background: rgba(254, 226, 226, 0.85);
|
| 612 |
+
}
|
| 613 |
+
|
| 614 |
+
.pdf-page-wrap {
|
| 615 |
+
position: relative;
|
| 616 |
+
margin: 0 auto;
|
| 617 |
+
|
| 618 |
+
background: #ffffff;
|
| 619 |
+
|
| 620 |
+
box-shadow: 0 14px 44px rgba(15, 23, 42, 0.16);
|
| 621 |
+
border-radius: 4px;
|
| 622 |
+
}
|
| 623 |
+
|
| 624 |
+
.pdf-page-wrap canvas {
|
| 625 |
+
display: block;
|
| 626 |
+
border-radius: 4px;
|
| 627 |
+
}
|
| 628 |
+
|
| 629 |
+
/* PDF.js-Text-Layer (unsichtbarer, selektierbarer Text über dem Canvas) */
|
| 630 |
+
|
| 631 |
+
.textLayer {
|
| 632 |
+
position: absolute;
|
| 633 |
+
inset: 0;
|
| 634 |
+
|
| 635 |
+
overflow: hidden;
|
| 636 |
+
|
| 637 |
+
line-height: 1;
|
| 638 |
+
text-align: initial;
|
| 639 |
+
|
| 640 |
+
forced-color-adjust: none;
|
| 641 |
+
}
|
| 642 |
+
|
| 643 |
+
.textLayer span,
|
| 644 |
+
.textLayer br {
|
| 645 |
+
position: absolute;
|
| 646 |
+
|
| 647 |
+
color: transparent;
|
| 648 |
+
white-space: pre;
|
| 649 |
+
|
| 650 |
+
cursor: text;
|
| 651 |
+
|
| 652 |
+
transform-origin: 0% 0%;
|
| 653 |
+
}
|
| 654 |
+
|
| 655 |
+
/* Hervorhebung der Fundstelle */
|
| 656 |
+
|
| 657 |
+
.textLayer span.pdf-hl {
|
| 658 |
+
background: rgba(255, 213, 64, 0.85);
|
| 659 |
+
mix-blend-mode: multiply;
|
| 660 |
+
|
| 661 |
+
border-radius: 3px;
|
| 662 |
+
box-shadow: 0 0 0 2px rgba(255, 213, 64, 0.85);
|
| 663 |
+
}
|
| 664 |
+
|
| 665 |
+
/* Modal */
|
| 666 |
+
|
| 667 |
+
.modal-backdrop {
|
| 668 |
+
position: fixed;
|
| 669 |
+
inset: 0;
|
| 670 |
+
|
| 671 |
+
display: none;
|
| 672 |
+
align-items: center;
|
| 673 |
+
justify-content: center;
|
| 674 |
+
|
| 675 |
+
padding: 24px;
|
| 676 |
+
|
| 677 |
+
z-index: 9999;
|
| 678 |
+
|
| 679 |
+
background: rgba(15, 23, 42, 0.34);
|
| 680 |
+
|
| 681 |
+
backdrop-filter: blur(14px) saturate(140%);
|
| 682 |
+
-webkit-backdrop-filter: blur(14px) saturate(140%);
|
| 683 |
+
|
| 684 |
+
animation: fadeIn 180ms ease both;
|
| 685 |
+
}
|
| 686 |
+
|
| 687 |
+
.modal {
|
| 688 |
+
width: min(900px, 100%);
|
| 689 |
+
|
| 690 |
+
padding: 24px;
|
| 691 |
+
|
| 692 |
+
border-radius: var(--radius-xl);
|
| 693 |
+
border: 1px solid rgba(255, 255, 255, 0.68);
|
| 694 |
+
|
| 695 |
+
background: rgba(255, 255, 255, 0.82);
|
| 696 |
+
|
| 697 |
+
box-shadow: 0 30px 90px rgba(15, 23, 42, 0.22);
|
| 698 |
+
|
| 699 |
+
backdrop-filter: blur(28px) saturate(150%);
|
| 700 |
+
-webkit-backdrop-filter: blur(28px) saturate(150%);
|
| 701 |
+
|
| 702 |
+
animation: modalIn 220ms ease both;
|
| 703 |
+
}
|
| 704 |
+
|
| 705 |
+
.modal h3 {
|
| 706 |
+
margin: 0 0 14px 0;
|
| 707 |
+
|
| 708 |
+
font-size: 1.35rem;
|
| 709 |
+
letter-spacing: -0.035em;
|
| 710 |
+
|
| 711 |
+
color: #111827;
|
| 712 |
+
}
|
| 713 |
+
|
| 714 |
+
.modal textarea {
|
| 715 |
+
min-height: 180px;
|
| 716 |
+
margin-bottom: 14px;
|
| 717 |
+
font-family: inherit;
|
| 718 |
+
}
|
| 719 |
+
|
| 720 |
+
.modal-actions {
|
| 721 |
+
display: flex;
|
| 722 |
+
gap: 10px;
|
| 723 |
+
justify-content: flex-end;
|
| 724 |
+
flex-wrap: wrap;
|
| 725 |
+
}
|
| 726 |
+
|
| 727 |
+
.hint {
|
| 728 |
+
margin-top: 12px;
|
| 729 |
+
|
| 730 |
+
font-size: 0.9em;
|
| 731 |
+
line-height: 1.5;
|
| 732 |
+
|
| 733 |
+
color: var(--text-muted);
|
| 734 |
+
}
|
| 735 |
+
|
| 736 |
+
.status {
|
| 737 |
+
min-height: 1.2em;
|
| 738 |
+
|
| 739 |
+
margin-right: auto;
|
| 740 |
+
|
| 741 |
+
align-self: center;
|
| 742 |
+
|
| 743 |
+
font-size: 0.9em;
|
| 744 |
+
font-weight: 600;
|
| 745 |
+
|
| 746 |
+
color: #059669;
|
| 747 |
+
}
|
| 748 |
+
|
| 749 |
+
/* Scrollbar */
|
| 750 |
+
|
| 751 |
+
::-webkit-scrollbar {
|
| 752 |
+
width: 12px;
|
| 753 |
+
}
|
| 754 |
+
|
| 755 |
+
::-webkit-scrollbar-track {
|
| 756 |
+
background: transparent;
|
| 757 |
+
}
|
| 758 |
+
|
| 759 |
+
::-webkit-scrollbar-thumb {
|
| 760 |
+
background: rgba(15, 23, 42, 0.16);
|
| 761 |
+
border: 4px solid transparent;
|
| 762 |
+
border-radius: 999px;
|
| 763 |
+
background-clip: content-box;
|
| 764 |
+
}
|
| 765 |
+
|
| 766 |
+
::-webkit-scrollbar-thumb:hover {
|
| 767 |
+
background: rgba(15, 23, 42, 0.25);
|
| 768 |
+
border: 4px solid transparent;
|
| 769 |
+
background-clip: content-box;
|
| 770 |
+
}
|
| 771 |
+
|
| 772 |
+
/* Animationen */
|
| 773 |
+
|
| 774 |
+
@keyframes messageIn {
|
| 775 |
+
from {
|
| 776 |
+
opacity: 0;
|
| 777 |
+
transform: translateY(6px);
|
| 778 |
+
}
|
| 779 |
+
|
| 780 |
+
to {
|
| 781 |
+
opacity: 1;
|
| 782 |
+
transform: translateY(0);
|
| 783 |
+
}
|
| 784 |
+
}
|
| 785 |
+
|
| 786 |
+
@keyframes fadeIn {
|
| 787 |
+
from {
|
| 788 |
+
opacity: 0;
|
| 789 |
+
}
|
| 790 |
+
|
| 791 |
+
to {
|
| 792 |
+
opacity: 1;
|
| 793 |
+
}
|
| 794 |
+
}
|
| 795 |
+
|
| 796 |
+
@keyframes modalIn {
|
| 797 |
+
from {
|
| 798 |
+
opacity: 0;
|
| 799 |
+
transform: translateY(10px) scale(0.98);
|
| 800 |
+
}
|
| 801 |
+
|
| 802 |
+
to {
|
| 803 |
+
opacity: 1;
|
| 804 |
+
transform: translateY(0) scale(1);
|
| 805 |
+
}
|
| 806 |
+
}
|
| 807 |
+
|
| 808 |
+
/* Responsive */
|
| 809 |
+
|
| 810 |
+
@media (max-width: 720px) {
|
| 811 |
+
.header {
|
| 812 |
+
width: min(100% - 20px, 900px);
|
| 813 |
+
margin-top: 18px;
|
| 814 |
+
padding: 14px 18px;
|
| 815 |
+
}
|
| 816 |
+
|
| 817 |
+
.logo {
|
| 818 |
+
max-height: 58px;
|
| 819 |
+
}
|
| 820 |
+
|
| 821 |
+
.chat {
|
| 822 |
+
width: min(100% - 20px, 900px);
|
| 823 |
+
margin: 18px auto 28px auto;
|
| 824 |
+
padding: 22px;
|
| 825 |
+
border-radius: 24px;
|
| 826 |
+
}
|
| 827 |
+
|
| 828 |
+
.question,
|
| 829 |
+
.answer {
|
| 830 |
+
max-width: 100%;
|
| 831 |
+
}
|
| 832 |
+
|
| 833 |
+
.row {
|
| 834 |
+
align-items: stretch;
|
| 835 |
+
}
|
| 836 |
+
|
| 837 |
+
button {
|
| 838 |
+
width: 100%;
|
| 839 |
+
}
|
| 840 |
+
|
| 841 |
+
.modal {
|
| 842 |
+
padding: 20px;
|
| 843 |
+
border-radius: 24px;
|
| 844 |
+
}
|
| 845 |
+
|
| 846 |
+
.modal-actions {
|
| 847 |
+
flex-direction: column;
|
| 848 |
+
}
|
| 849 |
+
|
| 850 |
+
.status {
|
| 851 |
+
margin-right: 0;
|
| 852 |
+
}
|
| 853 |
+
|
| 854 |
+
/* Viewer-Bedienelemente behalten ihre kompakte Breite */
|
| 855 |
+
.pdf-ctrl,
|
| 856 |
+
.source-ref,
|
| 857 |
+
.source-link {
|
| 858 |
+
width: auto;
|
| 859 |
+
}
|
| 860 |
+
|
| 861 |
+
.pdf-panel {
|
| 862 |
+
width: 100vw;
|
| 863 |
+
}
|
| 864 |
+
|
| 865 |
+
.pdf-panel-header {
|
| 866 |
+
flex-wrap: wrap;
|
| 867 |
+
}
|
| 868 |
+
|
| 869 |
+
.pdf-viewport {
|
| 870 |
+
padding: 10px;
|
| 871 |
+
}
|
| 872 |
+
}
|
| 873 |
+
|
| 874 |
+
@media (prefers-reduced-motion: reduce) {
|
| 875 |
+
*,
|
| 876 |
+
*::before,
|
| 877 |
+
*::after {
|
| 878 |
+
animation: none !important;
|
| 879 |
+
scroll-behavior: auto !important;
|
| 880 |
+
transition: none !important;
|
| 881 |
+
}
|
| 882 |
+
}
|