| from __future__ import annotations
|
|
|
| from collections import deque
|
| from dataclasses import dataclass, field
|
| import json
|
| import math
|
| import os
|
| from pathlib import Path
|
| import re
|
| import tempfile
|
| import time
|
| import uuid
|
| from typing import Any, Dict, Iterable, List, Mapping, Sequence
|
|
|
| from core.session_memory import SessionMemoryExtractor
|
| from experiments.replacement.memory_graph import (
|
| SessionMemoryGraphV2,
|
| SessionMemoryEdgeV2,
|
| SessionMemoryRecordV2,
|
| SQLiteSessionMemoryStore,
|
| _clean_text,
|
| _estimate_tokens,
|
| _normalize,
|
| _public_query_subject,
|
| _public_slot_root,
|
| _public_subject_signature,
|
| stable_slot_key,
|
| _tokenize,
|
| guess_slot_key,
|
| infer_category_hints,
|
| )
|
| from experiments.replacement.node_memory import (
|
| _call_with_supported_kwargs,
|
| DEFAULT_MATRIX_EVENT_TOP_K,
|
| LoadedNodeMemoryScorer,
|
| MEMORY_ROUTER_LAYERS,
|
| build_default_path_templates,
|
| extract_question_features,
|
| )
|
| from experiments.replacement.public_event_signature import compute_public_event_signature
|
| from experiments.replacement.memory_profiles import TMCRAProfile
|
| from experiments.replacement.typed_tunnel_augmentation import (
|
| annotate_memory_record,
|
| merge_typed_metadata,
|
| typed_edge_tags_between,
|
| typed_tunnel_signature_text,
|
| )
|
| from experiments.replacement.profile_layer import infer_profile_query_intent, is_profile_layer_record, profile_query_score_delta
|
| from experiments.replacement import injection_planner as injection_planner_runtime
|
| from experiments.replacement.temporal_modeling_types import TemporalFrame, TemporalQueryPlan
|
| from experiments.replacement.temporal_organizer import TemporalOrganizer
|
| from experiments.replacement.temporal_query_planner import TemporalQueryPlanner
|
| from experiments.replacement.temporal_router_runtime import LoadedTemporalRouter
|
| from experiments.replacement.timeline_evidence_pack import TimelineEvidencePackBuilder
|
| from experiments.replacement.timeline_state_layer import TimelineStateLayer
|
|
|
| from .base import MemoryAdapter, MemoryHit, MemoryRetrieval
|
|
|
|
|
| def _dedupe(items: Iterable[Any], *, max_items: int | None = None) -> List[str]:
|
| values: List[str] = []
|
| seen = set()
|
| for item in items:
|
| text = _clean_text(item)
|
| if not text:
|
| continue
|
| key = _normalize(text)
|
| if key in seen:
|
| continue
|
| seen.add(key)
|
| values.append(text)
|
| if max_items is not None and len(values) >= max_items:
|
| break
|
| return values
|
|
|
|
|
| def _apply_typed_tunnel_annotations(records: List[SessionMemoryRecordV2], *, source_text: str = "") -> List[SessionMemoryRecordV2]:
|
| for record in records:
|
| annotate_memory_record(record, source_text=source_text)
|
| return records
|
|
|
|
|
| def _estimate_tokens_from_hits(hits: Sequence[MemoryHit]) -> int:
|
| total = 0
|
| for hit in hits:
|
| total += _estimate_tokens(hit.value)
|
| total += sum(_estimate_tokens(anchor) for anchor in hit.anchors)
|
| return total
|
|
|
|
|
| def _float_env(name: str, default: float) -> float:
|
| raw = _clean_text(os.getenv(name, ""))
|
| if not raw:
|
| return float(default)
|
| try:
|
| return float(raw)
|
| except ValueError:
|
| return float(default)
|
|
|
|
|
| _GRAPH_PROMPT_MAX_CHARS = 12_000
|
| _GRAPH_PROMPT_MAX_HITS = 8
|
| _GRAPH_PROMPT_MAX_ACTIVE_SLOTS = 12
|
| _GRAPH_PROMPT_MAX_RELATIONS = 10
|
| _HYBRID_SELECTED_EVENT_FLOOR = 8
|
| _HYBRID_SELECTED_PATH_CAP = 6
|
| _HYBRID_TEMPORAL_PATH_CAP = 2
|
| _HYBRID_PROFILE_PATH_CAP = 3
|
| _MEMORY_ROUTER_GUIDED_MODES = {"guided", "route", "routing", "enforce"}
|
| _MEMORY_ROUTER_FORCE_MODES = {"force", "forced"}
|
| _MEMORY_ROUTER_DISABLED_MODES = {"", "off", "disabled", "none", "false", "0"}
|
| _MEMORY_ROUTER_OBSERVE_MODES = {"observe", "observer", "telemetry", "shadow"}
|
| _MEMORY_ROUTER_DEFAULT_THRESHOLD = 0.55
|
| _MEMORY_ROUTER_DEFAULT_MARGIN = 0.08
|
| _INJECTION_PLANNER_GUIDED_MODES = {"guided", "route", "routing", "enforce"}
|
| _INJECTION_PLANNER_FORCE_MODES = {"force", "forced"}
|
| _INJECTION_PLANNER_DISABLED_MODES = {"", "off", "disabled", "none", "false", "0"}
|
| _INJECTION_PLANNER_OBSERVE_MODES = {"observe", "observer", "telemetry", "shadow"}
|
| _TEMPORAL_LAYER_DISABLED_MODES = {"off", "disabled", "none", "false", "0"}
|
| _TEMPORAL_ROUTER_DEFAULT_WRITER_MIN_CONFIDENCE = 0.72
|
| _TEMPORAL_ROUTER_DEFAULT_QUERY_MIN_CONFIDENCE = 0.85
|
| _TEMPORAL_ROUTER_DEFAULT_QUERY_INTENT_MIN_CONFIDENCE = 0.60
|
| _EMBEDDER_INDEX_DISABLED_MODES = {"", "off", "disabled", "none", "false", "0"}
|
| _EMBEDDER_INDEX_BGE_M3_MODES = {"bge", "bge_m3", "bge-m3", "baai_bge_m3", "baai/bge-m3"}
|
| _EMBEDDER_INDEX_VERSION = "write_hash_sparse_v1"
|
| _EMBEDDER_INDEX_BGE_M3_VERSION = "write_bge_m3_dense_sparse_v1"
|
| _EMBEDDER_MODEL_CACHE: Dict[str, Any] = {}
|
| _HYBRID_SYMBOLIC_STOPWORDS = {
|
| "a",
|
| "an",
|
| "and",
|
| "are",
|
| "as",
|
| "at",
|
| "be",
|
| "did",
|
| "do",
|
| "does",
|
| "event",
|
| "for",
|
| "from",
|
| "had",
|
| "has",
|
| "have",
|
| "how",
|
| "in",
|
| "is",
|
| "of",
|
| "on",
|
| "or",
|
| "should",
|
| "the",
|
| "to",
|
| "turn",
|
| "was",
|
| "were",
|
| "what",
|
| "when",
|
| "where",
|
| "which",
|
| "who",
|
| "why",
|
| "will",
|
| "with",
|
| }
|
|
|
|
|
| def _coerce_memory_router_scores(payload: Mapping[str, Any]) -> Dict[str, float]:
|
| raw_scores = dict(payload.get("memory_router_scores", {}) or {})
|
| scores: Dict[str, float] = {}
|
| for layer in MEMORY_ROUTER_LAYERS:
|
| try:
|
| scores[layer] = float(raw_scores.get(layer, 0.0))
|
| except (TypeError, ValueError):
|
| scores[layer] = 0.0
|
| return scores
|
|
|
|
|
| def _memory_router_decision(
|
| payload: Mapping[str, Any],
|
| *,
|
| mode: str,
|
| threshold: float,
|
| margin: float,
|
| ) -> Dict[str, Any]:
|
| normalized_mode = _normalize(mode)
|
| scores = _coerce_memory_router_scores(payload)
|
| has_scores = bool(scores) and any(layer in dict(payload.get("memory_router_scores", {}) or {}) for layer in MEMORY_ROUTER_LAYERS)
|
| if not has_scores:
|
| return {
|
| "memory_router_enabled": False,
|
| "memory_router_guided": False,
|
| "memory_router_reason": "no_router_scores",
|
| "memory_router_scores": {},
|
| "memory_router_top_layers": [],
|
| "memory_router_active_layers": [],
|
| "memory_router_score_spread": 0.0,
|
| "memory_router_confidence": 0.0,
|
| }
|
| ranked_layers = [
|
| layer
|
| for layer, _ in sorted(
|
| scores.items(),
|
| key=lambda item: (-float(item[1]), item[0]),
|
| )
|
| ]
|
| score_values = list(scores.values())
|
| score_spread = max(score_values) - min(score_values)
|
| confidence = max(abs(float(score) - 0.5) for score in score_values)
|
| resolved_threshold = max(0.0, min(1.0, float(threshold or _MEMORY_ROUTER_DEFAULT_THRESHOLD)))
|
| resolved_margin = max(0.0, min(0.5, float(margin or _MEMORY_ROUTER_DEFAULT_MARGIN)))
|
| active_layers = [
|
| layer
|
| for layer in ranked_layers
|
| if float(scores.get(layer, 0.0)) >= resolved_threshold
|
| ]
|
| confident = bool(active_layers) and (score_spread >= resolved_margin or confidence >= resolved_margin)
|
| guided_requested = normalized_mode in _MEMORY_ROUTER_GUIDED_MODES
|
| forced = normalized_mode in _MEMORY_ROUTER_FORCE_MODES
|
| guided = bool(forced or (guided_requested and confident))
|
| if forced and not active_layers and ranked_layers:
|
| active_layers = [ranked_layers[0]]
|
| if active_layers and "event" not in active_layers:
|
| active_layers = ["event", *active_layers]
|
| reason = "observe"
|
| if normalized_mode in _MEMORY_ROUTER_DISABLED_MODES:
|
| reason = "disabled"
|
| elif normalized_mode in _MEMORY_ROUTER_OBSERVE_MODES:
|
| reason = "observe"
|
| elif forced:
|
| reason = "forced"
|
| elif guided_requested and not confident:
|
| reason = "low_confidence"
|
| elif guided:
|
| reason = "guided"
|
| return {
|
| "memory_router_enabled": True,
|
| "memory_router_guided": bool(guided and normalized_mode not in _MEMORY_ROUTER_DISABLED_MODES),
|
| "memory_router_reason": reason,
|
| "memory_router_scores": scores,
|
| "memory_router_top_layers": ranked_layers,
|
| "memory_router_active_layers": active_layers,
|
| "memory_router_score_spread": round(float(score_spread), 6),
|
| "memory_router_confidence": round(float(confidence), 6),
|
| "memory_router_threshold": round(float(resolved_threshold), 6),
|
| "memory_router_margin": round(float(resolved_margin), 6),
|
| "memory_router_mode": normalized_mode or "observe",
|
| }
|
|
|
|
|
| def _memory_router_allows(decision: Mapping[str, Any], *layers: str) -> bool:
|
| if not bool(decision.get("memory_router_guided")):
|
| return True
|
| requested = {_normalize(layer) for layer in layers if _normalize(layer)}
|
| if not requested:
|
| return True
|
| if "event" in requested:
|
| return True
|
| active = {_normalize(layer) for layer in list(decision.get("memory_router_active_layers", []) or [])}
|
| return bool(active & requested)
|
|
|
|
|
| def _hybrid_symbolic_tokens(value: Any) -> List[str]:
|
| return [
|
| token
|
| for token in _tokenize(value)
|
| if token and token not in _HYBRID_SYMBOLIC_STOPWORDS and not re.fullmatch(r"\d+", token)
|
| ]
|
|
|
|
|
| _PATH_UTILITY_STOPWORDS = set(_HYBRID_SYMBOLIC_STOPWORDS) | {
|
| "actually",
|
| "also",
|
| "bit",
|
| "feel",
|
| "feels",
|
| "get",
|
| "guess",
|
| "i",
|
| "im",
|
| "it",
|
| "its",
|
| "just",
|
| "kind",
|
| "like",
|
| "maybe",
|
| "me",
|
| "more",
|
| "much",
|
| "my",
|
| "really",
|
| "something",
|
| "still",
|
| "think",
|
| "thats",
|
| "there",
|
| "this",
|
| "way",
|
| "would",
|
| }
|
|
|
|
|
| def _path_utility_tokens(value: Any) -> List[str]:
|
| return [
|
| token
|
| for token in _hybrid_symbolic_tokens(value)
|
| if token and token not in _PATH_UTILITY_STOPWORDS
|
| ]
|
|
|
|
|
| _PROFILE_QUERY_ALIAS_GROUPS: tuple[set[str], ...] = (
|
| {
|
| "accessory",
|
| "accessories",
|
| "bag",
|
| "camera",
|
| "cameras",
|
| "equipment",
|
| "flash",
|
| "gear",
|
| "lens",
|
| "lenses",
|
| "photo",
|
| "photography",
|
| "sony",
|
| "tripod",
|
| },
|
| {
|
| "app",
|
| "apps",
|
| "dashboard",
|
| "interface",
|
| "layout",
|
| "panel",
|
| "software",
|
| "tool",
|
| "tools",
|
| "ui",
|
| "workflow",
|
| },
|
| {
|
| "diet",
|
| "drink",
|
| "food",
|
| "meal",
|
| "restaurant",
|
| "snack",
|
| "taste",
|
| },
|
| {
|
| "background",
|
| "career",
|
| "job",
|
| "occupation",
|
| "position",
|
| "previous",
|
| "profession",
|
| "role",
|
| "worked",
|
| "work",
|
| },
|
| )
|
|
|
|
|
| _PROFILE_QUERY_GENERIC_TOKENS = set(_PATH_UTILITY_STOPWORDS) | {
|
| "able",
|
| "about",
|
| "any",
|
| "anything",
|
| "based",
|
| "best",
|
| "can",
|
| "complement",
|
| "could",
|
| "current",
|
| "give",
|
| "help",
|
| "information",
|
| "looking",
|
| "make",
|
| "need",
|
| "please",
|
| "recommend",
|
| "recommendation",
|
| "recommendations",
|
| "should",
|
| "some",
|
| "suggest",
|
| "suggestion",
|
| "suggestions",
|
| "tell",
|
| "that",
|
| "using",
|
| "you",
|
| }
|
|
|
|
|
| def _profile_query_expanded_tokens(value: Any) -> set[str]:
|
| tokens = set(_path_utility_tokens(value))
|
| expanded = set(tokens)
|
| for group in _PROFILE_QUERY_ALIAS_GROUPS:
|
| if tokens & group:
|
| expanded.update(group)
|
| return expanded
|
|
|
|
|
| def _profile_specific_tokens(tokens: Iterable[Any]) -> set[str]:
|
| return {
|
| _normalize(token)
|
| for token in tokens
|
| if _clean_text(token)
|
| and _normalize(token) not in _PROFILE_QUERY_GENERIC_TOKENS
|
| and not re.fullmatch(r"\d+", _normalize(token))
|
| }
|
|
|
|
|
| def _profile_hit_match_score(query_tokens: set[str], expanded_query_tokens: set[str], hit: MemoryHit) -> tuple[float, List[str], List[str]]:
|
| metadata = dict(hit.metadata or {})
|
| payload_parts: List[Any] = [
|
| hit.category,
|
| hit.relation,
|
| hit.slot_key,
|
| hit.value,
|
| *list(hit.anchors or []),
|
| metadata.get("profile_summary", ""),
|
| metadata.get("profile_value", ""),
|
| metadata.get("profile_type", ""),
|
| metadata.get("profile_domain", ""),
|
| metadata.get("profile_domain_label", ""),
|
| metadata.get("semantic_slot", ""),
|
| metadata.get("subject", ""),
|
| metadata.get("extracted_subject", ""),
|
| metadata.get("profile_cluster_domains", []),
|
| metadata.get("profile_cluster_types", []),
|
| metadata.get("profile_cluster_route_terms", []),
|
| metadata.get("profile_route_terms", []),
|
| metadata.get("profile_support_values", []),
|
| ]
|
| payload_text = " ".join(str(item) for item in payload_parts)
|
| record_raw_tokens = set(_path_utility_tokens(payload_text))
|
| record_tokens = _profile_query_expanded_tokens(payload_text)
|
| specific_query_tokens = _profile_specific_tokens(query_tokens)
|
| raw_overlap_tokens = sorted(specific_query_tokens & _profile_specific_tokens(record_raw_tokens))
|
| expanded_overlap_tokens = sorted(_profile_specific_tokens(expanded_query_tokens) & _profile_specific_tokens(record_tokens))
|
| overlap_tokens = raw_overlap_tokens or expanded_overlap_tokens
|
| overlap_ratio = float(len(overlap_tokens)) / float(max(1, len(specific_query_tokens)))
|
| profile_type = _normalize(metadata.get("profile_type", ""))
|
| source_kind = _normalize(hit.source_kind)
|
| source_bonus = 0.0
|
| if source_kind in {"public_dialog_preference", "public_dialog_goal", "public_dialog_profile"}:
|
| source_bonus += 0.08
|
| if bool(metadata.get("profile_candidate_status") == "consolidated"):
|
| source_bonus += 0.04
|
| if bool(metadata.get("profile_cluster_node")):
|
| source_bonus -= 0.16
|
| type_bonus = 0.0
|
| if profile_type in {"preference", "goal", "setup", "usage_context"} and (
|
| {"recommend", "suggest", "suited", "accessory", "accessories", "gear", "equipment", "setup", "current"} & query_tokens
|
| ):
|
| type_bonus += 0.10
|
| if profile_type in {"setup", "usage_context"} and {"current", "setup", "profile"} & query_tokens:
|
| type_bonus += 0.08
|
| raw_bonus = 0.18 * len(raw_overlap_tokens)
|
| expanded_bonus = 0.08 * max(0, len(expanded_overlap_tokens) - len(raw_overlap_tokens))
|
| score = raw_bonus + expanded_bonus + (0.58 * overlap_ratio) + source_bonus + type_bonus
|
| return round(score, 6), overlap_tokens, raw_overlap_tokens
|
|
|
|
|
| def _bounded_event_id_union(*groups: Iterable[Any], max_items: int) -> List[str]:
|
| return _dedupe((item for group in groups for item in group), max_items=max(1, int(max_items)))
|
|
|
|
|
| def _symbolic_recall_event_ids(
|
| query: str,
|
| runtime_graph: Mapping[str, Any],
|
| *,
|
| grouped_hits: Mapping[str, Sequence[MemoryHit]],
|
| limit: int,
|
| ) -> List[str]:
|
| question_features = extract_question_features(query)
|
| query_tokens = set(_hybrid_symbolic_tokens(question_features.get("question_anchor_tokens", []) or query))
|
| if not query_tokens:
|
| return []
|
| nodes_by_id = {
|
| _clean_text(node.get("id", "")): dict(node)
|
| for node in list(runtime_graph.get("nodes", []) or [])
|
| if _clean_text(node.get("id", ""))
|
| }
|
| event_payloads: Dict[str, List[str]] = {}
|
| for node_id, node in nodes_by_id.items():
|
| if _clean_text(node.get("type", "")) == "event":
|
| metadata = dict(node.get("metadata", {}) or {})
|
| teacher_fields = dict(node.get("teacher_fields", {}) or {})
|
| event_payloads.setdefault(node_id, []).extend(
|
| [
|
| node.get("text", ""),
|
| node.get("speaker", ""),
|
| node.get("slot_key", ""),
|
| node.get("target_status", ""),
|
| node.get("profile_type", ""),
|
| node.get("profile_value", ""),
|
| *teacher_fields.values(),
|
| *metadata.values(),
|
| ]
|
| )
|
| for path in list(runtime_graph.get("paths", []) or []):
|
| event_id = _clean_text(path.get("event_id", ""))
|
| support_node = nodes_by_id.get(_clean_text(path.get("target", "")), {})
|
| if event_id and support_node:
|
| event_payloads.setdefault(event_id, []).append(support_node.get("text", ""))
|
| for event_id, group_hits in grouped_hits.items():
|
| payloads = event_payloads.setdefault(_clean_text(event_id), [])
|
| for hit in list(group_hits or []):
|
| metadata = dict(hit.metadata or {})
|
| payloads.extend([hit.value, hit.slot_key, hit.category, hit.relation, *hit.anchors, *metadata.values()])
|
|
|
| scored_events: List[tuple[str, float]] = []
|
| for event_id, payloads in event_payloads.items():
|
| event_tokens = set(_hybrid_symbolic_tokens(payloads))
|
| overlap = query_tokens & event_tokens
|
| if not overlap:
|
| continue
|
| overlap_ratio = float(len(overlap)) / float(max(1, len(query_tokens)))
|
| event_node = nodes_by_id.get(event_id, {})
|
| turn_index = int(event_node.get("turn_index", 0) or 0)
|
| scored_events.append((event_id, (len(overlap) * 4.0) + overlap_ratio + min(turn_index, 1000) * 0.000001))
|
| return [
|
| event_id
|
| for event_id, _ in sorted(scored_events, key=lambda item: (-float(item[1]), item[0]))
|
| ][: max(1, int(limit))]
|
|
|
|
|
| _EMBEDDER_INDEX_METADATA_TEXT_KEYS = (
|
| "raw_text",
|
| "source_turn_text",
|
| "source_span",
|
| "event_phrase",
|
| "event_summary",
|
| "profile_value",
|
| "profile_summary",
|
| "profile_type",
|
| "profile_domain",
|
| "profile_domain_label",
|
| "semantic_slot",
|
| "target_status",
|
| "subject",
|
| "subject_signature",
|
| "canonical_slot_key",
|
| "resource_key",
|
| "resolved_date",
|
| "resolved_time_value",
|
| "time_value",
|
| "time_display_value",
|
| "time_granularity",
|
| "speaker",
|
| "session_name",
|
| "topic_label",
|
| "topic_bucket_id",
|
| "origin_query",
|
| "writeback_class",
|
| "depth_layer",
|
| "memory_chain_depth_layer",
|
| )
|
| _EMBEDDER_INDEX_METADATA_LIST_KEYS = (
|
| "topic_keywords",
|
| "profile_route_terms",
|
| "profile_cluster_route_terms",
|
| "profile_support_values",
|
| "evidence_anchors",
|
| "support_memory_ids",
|
| "support_fact_refs",
|
| "support_path_refs",
|
| )
|
|
|
|
|
| def _embedder_index_enabled(mode: Any) -> bool:
|
| return _normalize(mode) not in _EMBEDDER_INDEX_DISABLED_MODES
|
|
|
|
|
| def _embedder_index_uses_bge_m3(mode: Any) -> bool:
|
| return _normalize(mode).replace("-", "_") in {item.replace("-", "_") for item in _EMBEDDER_INDEX_BGE_M3_MODES}
|
|
|
|
|
| def _embedder_index_version_for_mode(mode: Any) -> str:
|
| return _EMBEDDER_INDEX_BGE_M3_VERSION if _embedder_index_uses_bge_m3(mode) else _EMBEDDER_INDEX_VERSION
|
|
|
|
|
| def _embedder_index_text_items(value: Any, *, max_items: int = 64) -> List[str]:
|
| items: List[str] = []
|
|
|
| def visit(item: Any) -> None:
|
| if len(items) >= max_items:
|
| return
|
| if item is None:
|
| return
|
| if isinstance(item, Mapping):
|
| for key, nested in list(item.items()):
|
| if len(items) >= max_items:
|
| break
|
| key_text = _clean_text(key)
|
| if isinstance(nested, (str, int, float, bool)):
|
| value_text = _clean_text(nested)[:800]
|
| if value_text:
|
| items.append(f"{key_text} {value_text}".strip())
|
| else:
|
| visit(nested)
|
| return
|
| if isinstance(item, (list, tuple, set)):
|
| for nested in list(item):
|
| if len(items) >= max_items:
|
| break
|
| visit(nested)
|
| return
|
| text = _clean_text(item)[:800]
|
| if text:
|
| items.append(text)
|
|
|
| visit(value)
|
| return items[:max_items]
|
|
|
|
|
| def _embedder_index_term_weights(value: Any, *, max_terms: int = 96) -> Dict[str, float]:
|
| text = _clean_text(value)
|
| if not text:
|
| return {}
|
| counts: Dict[str, float] = {}
|
| for token in _path_utility_tokens(text):
|
| token = _normalize(token)
|
| if not token or len(token) > 64:
|
| continue
|
| counts[token] = counts.get(token, 0.0) + 1.0
|
| normalized_text = _normalize(text)
|
| cjk_chars = [char for char in normalized_text if "\u4e00" <= char <= "\u9fff"]
|
| for width, weight in ((2, 1.35), (3, 1.15)):
|
| if len(cjk_chars) < width:
|
| continue
|
| for index in range(0, len(cjk_chars) - width + 1):
|
| gram = "".join(cjk_chars[index : index + width])
|
| if gram:
|
| counts[gram] = counts.get(gram, 0.0) + weight
|
| if not counts:
|
| return {}
|
| ranked = sorted(counts.items(), key=lambda item: (-float(item[1]), item[0]))[: max(1, int(max_terms or 1))]
|
| norm = math.sqrt(sum(float(weight) * float(weight) for _, weight in ranked)) or 1.0
|
| return {term: round(float(weight) / norm, 6) for term, weight in ranked}
|
|
|
|
|
| def _embedder_dense_vectors_for_texts(texts: Sequence[str], *, mode: str) -> tuple[List[List[float]], Dict[str, Any]]:
|
| normalized_mode = _normalize(mode)
|
| if not _embedder_index_uses_bge_m3(normalized_mode):
|
| return [[] for _ in texts], {"write_embedder_dense_enabled": False}
|
| clean_texts = [_clean_text(text) for text in texts]
|
| metadata: Dict[str, Any] = {
|
| "write_embedder_dense_enabled": False,
|
| "write_embedder_dense_backend": "bge_m3_transformers",
|
| "write_embedder_dense_model": _clean_text(os.getenv("TMCRA_EMBEDDER_MODEL_PATH", "")) or "BAAI/bge-m3",
|
| }
|
| if not any(clean_texts):
|
| metadata["write_embedder_dense_error"] = "empty_texts"
|
| return [[] for _ in texts], metadata
|
| try:
|
| import torch
|
| from transformers import AutoModel, AutoTokenizer
|
| except Exception as exc:
|
| metadata["write_embedder_dense_error"] = f"dependency_unavailable:{exc.__class__.__name__}"
|
| return [[] for _ in texts], metadata
|
| model_name = metadata["write_embedder_dense_model"]
|
| device = _clean_text(os.getenv("TMCRA_EMBEDDER_DEVICE", ""))
|
| if not device:
|
| device = "cuda" if bool(getattr(torch, "cuda", None) and torch.cuda.is_available()) else "cpu"
|
| try:
|
| max_length = max(64, int(os.getenv("TMCRA_EMBEDDER_MODEL_MAX_LENGTH", "512") or 512))
|
| except (TypeError, ValueError):
|
| max_length = 512
|
| cache_key = f"bge_m3::{model_name}::{device}::{max_length}"
|
| try:
|
| cached = _EMBEDDER_MODEL_CACHE.get(cache_key)
|
| if cached is None:
|
| tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=False) |
| model = AutoModel.from_pretrained(model_name, trust_remote_code=False) |
| model.to(device)
|
| model.eval()
|
| cached = (tokenizer, model)
|
| _EMBEDDER_MODEL_CACHE[cache_key] = cached
|
| tokenizer, model = cached
|
| encoded = tokenizer(
|
| clean_texts,
|
| padding=True,
|
| truncation=True,
|
| max_length=max_length,
|
| return_tensors="pt",
|
| )
|
| encoded = {key: value.to(device) for key, value in encoded.items()}
|
| with torch.no_grad():
|
| output = model(**encoded)
|
| hidden = output.last_hidden_state
|
| mask = encoded.get("attention_mask")
|
| if mask is not None:
|
| mask = mask.unsqueeze(-1).expand(hidden.size()).float()
|
| pooled = torch.sum(hidden * mask, dim=1) / torch.clamp(mask.sum(dim=1), min=1e-9)
|
| else:
|
| pooled = hidden[:, 0]
|
| pooled = torch.nn.functional.normalize(pooled, p=2, dim=1)
|
| vectors = [
|
| [round(float(value), 6) for value in row.detach().cpu().tolist()]
|
| for row in pooled
|
| ]
|
| metadata.update(
|
| {
|
| "write_embedder_dense_enabled": True,
|
| "write_embedder_dense_device": device,
|
| "write_embedder_dense_dim": int(len(vectors[0]) if vectors else 0),
|
| "write_embedder_dense_max_length": int(max_length),
|
| }
|
| )
|
| return vectors, metadata
|
| except Exception as exc:
|
| metadata["write_embedder_dense_error"] = f"{exc.__class__.__name__}:{_clean_text(exc)[:240]}"
|
| return [[] for _ in texts], metadata
|
|
|
|
|
| def _prewarm_embedder_dense_if_requested(*, mode: str) -> Dict[str, Any]:
|
| flag = _normalize(os.getenv("TMCRA_EMBEDDER_PREWARM", ""))
|
| if flag in _EMBEDDER_INDEX_DISABLED_MODES or flag not in {"1", "true", "yes", "on", "auto"}:
|
| return {"embedder_prewarm_enabled": False}
|
| normalized_mode = _normalize(mode)
|
| if not _embedder_index_uses_bge_m3(normalized_mode):
|
| return {
|
| "embedder_prewarm_enabled": False,
|
| "embedder_prewarm_reason": "mode_not_dense",
|
| "embedder_prewarm_mode": normalized_mode or "off",
|
| }
|
| warmup_text = _clean_text(os.getenv("TMCRA_EMBEDDER_PREWARM_TEXT", "")) or "tmcra memory retrieval warmup"
|
| vectors, metadata = _embedder_dense_vectors_for_texts([warmup_text], mode=normalized_mode)
|
| return {
|
| "embedder_prewarm_enabled": bool(vectors and vectors[0]),
|
| "embedder_prewarm_mode": normalized_mode,
|
| "embedder_prewarm_dense_enabled": bool(metadata.get("write_embedder_dense_enabled")),
|
| "embedder_prewarm_dense_device": metadata.get("write_embedder_dense_device", ""),
|
| "embedder_prewarm_dense_error": metadata.get("write_embedder_dense_error", ""),
|
| }
|
|
|
|
|
| def _embedder_index_record_text(record: SessionMemoryRecordV2, *, turn_text: str = "") -> str:
|
| metadata = dict(record.metadata or {})
|
| parts: List[str] = [
|
| record.category,
|
| record.slot_key,
|
| record.value,
|
| record.value,
|
| record.value,
|
| record.relation,
|
| *list(record.anchor_concepts or []),
|
| *list(record.anchor_concepts or []),
|
| *list(record.evidence_anchors or []),
|
| ]
|
| has_record_evidence_text = any(
|
| _clean_text(metadata.get(key, ""))
|
| for key in ("raw_text", "source_turn_text", "source_span", "event_phrase", "profile_value", "profile_summary")
|
| )
|
| if turn_text and not has_record_evidence_text:
|
| parts.append(_clean_text(turn_text)[:1000])
|
| for key in _EMBEDDER_INDEX_METADATA_TEXT_KEYS:
|
| value = metadata.get(key)
|
| if value:
|
| parts.append(f"{key} {' '.join(_embedder_index_text_items(value, max_items=12))}".strip())
|
| for key in _EMBEDDER_INDEX_METADATA_LIST_KEYS:
|
| value = metadata.get(key)
|
| if value:
|
| parts.append(f"{key} {' '.join(_embedder_index_text_items(value, max_items=24))}".strip())
|
| return _clean_text(" ".join(_clean_text(part) for part in parts if _clean_text(part)))
|
|
|
|
|
| def _apply_write_embedder_index_to_graph(
|
| graph: SessionMemoryGraphV2,
|
| *,
|
| stored_ids: Sequence[str],
|
| turn_text: str,
|
| turn_index: int,
|
| mode: str,
|
| max_terms: int,
|
| ) -> Dict[str, Any]:
|
| normalized_mode = _normalize(mode)
|
| index_version = _embedder_index_version_for_mode(normalized_mode)
|
| metadata: Dict[str, Any] = {
|
| "write_embedder_index_enabled": False,
|
| "write_embedder_index_mode": normalized_mode or "off",
|
| "write_embedder_index_version": index_version,
|
| "write_embedder_index_record_count": 0,
|
| }
|
| if not _embedder_index_enabled(normalized_mode) or not stored_ids:
|
| return metadata
|
| indexed_ids: List[str] = []
|
| index_rows: List[tuple[SessionMemoryRecordV2, str, Dict[str, float]]] = []
|
| for memory_id in _dedupe(stored_ids):
|
| record = getattr(graph, "records_by_id", {}).get(memory_id)
|
| if record is None:
|
| continue
|
| index_text = _embedder_index_record_text(record, turn_text=turn_text)
|
| terms = _embedder_index_term_weights(
|
| index_text,
|
| max_terms=max_terms,
|
| )
|
| if not terms:
|
| continue
|
| index_rows.append((record, index_text, terms))
|
| dense_vectors, dense_metadata = _embedder_dense_vectors_for_texts(
|
| [index_text for _, index_text, _ in index_rows],
|
| mode=normalized_mode,
|
| )
|
| metadata.update(dense_metadata)
|
| for row_index, (record, _, terms) in enumerate(index_rows):
|
| dense_vector = dense_vectors[row_index] if row_index < len(dense_vectors) else []
|
| record_metadata = dict(record.metadata or {})
|
| record_metadata.update(
|
| {
|
| "write_embedder_index_enabled": True,
|
| "write_embedder_index_mode": normalized_mode,
|
| "write_embedder_index_version": index_version,
|
| "write_embedder_index_source_turn": int(turn_index),
|
| "write_embedder_index_term_count": int(len(terms)),
|
| "write_embedder_index_terms": dict(terms),
|
| "write_embedder_index_top_terms": list(terms.keys())[:24],
|
| }
|
| )
|
| if dense_vector:
|
| record_metadata.update(
|
| {
|
| "write_embedder_dense_enabled": True,
|
| "write_embedder_dense_backend": dense_metadata.get("write_embedder_dense_backend", ""),
|
| "write_embedder_dense_model": dense_metadata.get("write_embedder_dense_model", ""),
|
| "write_embedder_dense_dim": int(len(dense_vector)),
|
| "write_embedder_dense_vector": list(dense_vector),
|
| }
|
| )
|
| elif _embedder_index_uses_bge_m3(normalized_mode):
|
| record_metadata.update(
|
| {
|
| "write_embedder_dense_enabled": False,
|
| "write_embedder_dense_error": dense_metadata.get("write_embedder_dense_error", "dense_vector_unavailable"),
|
| }
|
| )
|
| record.metadata = record_metadata
|
| indexed_ids.append(record.memory_id)
|
| metadata.update(
|
| {
|
| "write_embedder_index_enabled": bool(indexed_ids),
|
| "write_embedder_index_record_count": int(len(indexed_ids)),
|
| "write_embedder_index_record_ids": list(indexed_ids[:24]),
|
| }
|
| )
|
| return metadata
|
|
|
|
|
| def _embedder_index_recall_event_ids(
|
| query: str,
|
| *,
|
| grouped_hits: Mapping[str, Sequence[MemoryHit]],
|
| mode: str,
|
| limit: int,
|
| max_terms: int,
|
| ) -> Dict[str, Any]:
|
| normalized_mode = _normalize(mode)
|
| index_version = _embedder_index_version_for_mode(normalized_mode)
|
| metadata: Dict[str, Any] = {
|
| "embedder_index_recall_enabled": False,
|
| "embedder_index_recall_mode": normalized_mode or "off",
|
| "embedder_index_recall_version": index_version,
|
| "embedder_index_event_ids": [],
|
| "embedder_index_event_scores": {},
|
| "embedder_index_record_count": 0,
|
| }
|
| if not _embedder_index_enabled(normalized_mode):
|
| return {"event_ids": [], "metadata": metadata}
|
| query_terms = _embedder_index_term_weights(query, max_terms=max_terms)
|
| query_vectors, query_dense_metadata = _embedder_dense_vectors_for_texts([query], mode=normalized_mode)
|
| query_vector = query_vectors[0] if query_vectors else []
|
| metadata.update(
|
| {
|
| "embedder_dense_recall_enabled": bool(query_vector),
|
| "embedder_dense_recall_backend": query_dense_metadata.get("write_embedder_dense_backend", ""),
|
| "embedder_dense_recall_model": query_dense_metadata.get("write_embedder_dense_model", ""),
|
| "embedder_dense_recall_error": query_dense_metadata.get("write_embedder_dense_error", ""),
|
| }
|
| )
|
| if not query_terms and not query_vector:
|
| metadata["embedder_index_recall_reason"] = "empty_query_terms"
|
| return {"event_ids": [], "metadata": metadata}
|
| scored_events: List[tuple[str, float, int]] = []
|
| indexed_record_count = 0
|
| dense_record_count = 0
|
| for event_id, group_hits in grouped_hits.items():
|
| event_score = 0.0
|
| event_turn = 0
|
| for hit in list(group_hits or []):
|
| hit_metadata = dict(hit.metadata or {})
|
| raw_terms = hit_metadata.get("write_embedder_index_terms")
|
| raw_vector = hit_metadata.get("write_embedder_dense_vector")
|
| if not isinstance(raw_terms, Mapping) and not isinstance(raw_vector, list):
|
| continue
|
| indexed_record_count += int(isinstance(raw_terms, Mapping))
|
| dense_record_count += int(isinstance(raw_vector, list) and bool(raw_vector))
|
| sparse_score = 0.0
|
| if isinstance(raw_terms, Mapping):
|
| for term, query_weight in query_terms.items():
|
| try:
|
| sparse_score += float(query_weight) * float(raw_terms.get(term, 0.0) or 0.0)
|
| except (TypeError, ValueError):
|
| continue
|
| dense_score = 0.0
|
| if query_vector and isinstance(raw_vector, list) and raw_vector:
|
| for query_value, record_value in zip(query_vector, raw_vector):
|
| try:
|
| dense_score += float(query_value) * float(record_value)
|
| except (TypeError, ValueError):
|
| continue
|
| score = max(float(sparse_score), float(dense_score) + (0.15 * float(sparse_score) if dense_score > 0.0 else 0.0))
|
| if score <= 0.0:
|
| continue
|
| event_score = max(event_score, float(score))
|
| event_turn = max(event_turn, int(hit.turn_index or 0))
|
| if event_score > 0.0:
|
| scored_events.append((_clean_text(event_id), round(event_score, 6), event_turn))
|
| scored_events.sort(key=lambda item: (-float(item[1]), -int(item[2]), item[0]))
|
| selected = scored_events[: max(1, int(limit or 1))]
|
| event_ids = [event_id for event_id, _, _ in selected if event_id]
|
| event_scores = {event_id: score for event_id, score, _ in selected if event_id}
|
| metadata.update(
|
| {
|
| "embedder_index_recall_enabled": True,
|
| "embedder_index_event_ids": list(event_ids),
|
| "embedder_index_event_scores": dict(event_scores),
|
| "embedder_index_query_terms": list(query_terms.keys())[:32],
|
| "embedder_index_record_count": int(indexed_record_count),
|
| "embedder_dense_record_count": int(dense_record_count),
|
| }
|
| )
|
| return {"event_ids": event_ids, "metadata": metadata}
|
|
|
|
|
| _IDENTIFIER_GENERIC_TOKENS = {
|
| "api",
|
| "agent",
|
| "code",
|
| "codename",
|
| "context",
|
| "debug",
|
| "goal",
|
| "memory",
|
| "model",
|
| "name",
|
| "project",
|
| "retrieval",
|
| "runtime",
|
| "session",
|
| "target",
|
| "test",
|
| "turn",
|
| }
|
| _IDENTIFIER_REQUEST_RE = re.compile(
|
| r"\b(code\s*name|codename|identifier|alias|project\s+name|model\s+name|api\s+name)\b|"
|
| r"(代号|编号|标识符|名称|名字|别名|项目名|模型名|接口名)",
|
| flags=re.IGNORECASE,
|
| )
|
| _IDENTIFIER_TOKEN_RE = re.compile(r"\b[A-Za-z][A-Za-z0-9_-]{2,63}\b")
|
|
|
|
|
| def _query_identifier_tokens(query: Any) -> List[str]:
|
| text = _clean_text(query)
|
| tokens: List[str] = []
|
| for match in _IDENTIFIER_TOKEN_RE.finditer(text):
|
| token = match.group(0)
|
| lowered = token.lower()
|
| if lowered in _IDENTIFIER_GENERIC_TOKENS:
|
| continue
|
| has_inner_upper = any(ch.isupper() for ch in token[1:])
|
| has_digit = any(ch.isdigit() for ch in token)
|
| has_joiner = "-" in token or "_" in token
|
| if has_inner_upper or has_digit or has_joiner:
|
| tokens.append(token)
|
| return _dedupe(tokens, max_items=8)
|
|
|
|
|
| def _query_requests_identifier_fact(query: Any) -> bool:
|
| return bool(_IDENTIFIER_REQUEST_RE.search(_clean_text(query)))
|
|
|
|
|
| def _hit_text_for_identifier_match(hit: MemoryHit) -> str:
|
| metadata = dict(hit.metadata or {})
|
| values = [
|
| hit.memory_id,
|
| hit.category,
|
| hit.value,
|
| hit.relation,
|
| hit.slot_key,
|
| hit.source_kind,
|
| *list(hit.anchors or []),
|
| metadata.get("raw_text", ""),
|
| metadata.get("source_turn_text", ""),
|
| metadata.get("source_span", ""),
|
| metadata.get("event_phrase", ""),
|
| metadata.get("profile_value", ""),
|
| metadata.get("target_status", ""),
|
| ]
|
| return " ".join(_clean_text(value) for value in values if _clean_text(value))
|
|
|
|
|
| def _copy_hit_with_identifier_boost(hit: MemoryHit, *, score: float, reasons: Sequence[str], matched_tokens: Sequence[str]) -> MemoryHit:
|
| metadata = dict(hit.metadata or {})
|
| metadata.update(
|
| {
|
| "identifier_protected": True,
|
| "identifier_protected_score": round(float(score), 6),
|
| "identifier_protected_reasons": list(reasons)[:8],
|
| "identifier_protected_matched_tokens": list(matched_tokens)[:8],
|
| "identifier_protected_original_score": round(float(hit.score), 6),
|
| }
|
| )
|
| return MemoryHit(
|
| memory_id=hit.memory_id,
|
| category=hit.category,
|
| value=hit.value,
|
| relation=hit.relation,
|
| anchors=list(hit.anchors),
|
| score=max(float(hit.score), round(1.0 + float(score), 6)),
|
| source_kind=hit.source_kind,
|
| slot_key=hit.slot_key,
|
| state=hit.state,
|
| turn_index=int(hit.turn_index),
|
| metadata=metadata,
|
| )
|
|
|
|
|
| def _identifier_protected_hits(
|
| *,
|
| query: str,
|
| final_hits: Sequence[MemoryHit],
|
| candidate_hits: Sequence[MemoryHit],
|
| top_k: int,
|
| ) -> Dict[str, Any]:
|
| identifier_tokens = _query_identifier_tokens(query)
|
| identifier_request = _query_requests_identifier_fact(query)
|
| if not identifier_tokens and not identifier_request:
|
| return {"enabled": False, "hits": list(final_hits), "promoted_hits": [], "metadata": {"identifier_protected_enabled": False}}
|
|
|
| query_text = _clean_text(query)
|
| pool: Dict[str, MemoryHit] = {}
|
| for hit in list(final_hits) + list(candidate_hits):
|
| key = hit.memory_id or f"{_hit_event_id(hit)}::{hit.slot_key}::{hit.value[:80]}"
|
| if key and key not in pool:
|
| pool[key] = hit
|
|
|
| scored: List[tuple[float, MemoryHit, List[str], List[str]]] = []
|
| for hit in pool.values():
|
| hit_text = _hit_text_for_identifier_match(hit)
|
| hit_lower = hit_text.lower()
|
| matched_tokens = [token for token in identifier_tokens if token.lower() in hit_lower]
|
| reasons: List[str] = []
|
| score = 0.0
|
| if matched_tokens:
|
| score += 12.0 + float(len(matched_tokens))
|
| reasons.append("exact_identifier_match")
|
| if identifier_request:
|
| if any(term in hit_lower for term in ("codename", "code name", "identifier", "alias", "project_codename")):
|
| score += 8.0
|
| reasons.append("identifier_field_match")
|
| if any(term in hit_text for term in ("代号", "编号", "标识符", "名称", "名字", "别名")):
|
| score += 8.0
|
| reasons.append("identifier_cjk_field_match")
|
| if "项目" in query_text and "项目" in hit_text:
|
| score += 1.5
|
| reasons.append("project_term_match")
|
| if score <= 0:
|
| continue
|
| scored.append((score, hit, matched_tokens, reasons))
|
|
|
| scored.sort(key=lambda item: (item[0], float(item[1].score), int(item[1].turn_index or 0)), reverse=True)
|
| promoted = [
|
| _copy_hit_with_identifier_boost(hit, score=score, reasons=reasons, matched_tokens=matched_tokens)
|
| for score, hit, matched_tokens, reasons in scored[:2]
|
| ]
|
| if not promoted:
|
| return {
|
| "enabled": True,
|
| "hits": list(final_hits),
|
| "promoted_hits": [],
|
| "metadata": {
|
| "identifier_protected_enabled": True,
|
| "identifier_query_tokens": identifier_tokens,
|
| "identifier_request": bool(identifier_request),
|
| "identifier_promoted_count": 0,
|
| },
|
| }
|
|
|
| promoted_keys = {hit.memory_id or f"{_hit_event_id(hit)}::{hit.slot_key}::{hit.value[:80]}" for hit in promoted}
|
| merged = list(promoted)
|
| for hit in final_hits:
|
| key = hit.memory_id or f"{_hit_event_id(hit)}::{hit.slot_key}::{hit.value[:80]}"
|
| if key in promoted_keys:
|
| continue
|
| merged.append(hit)
|
| if len(merged) >= max(1, int(top_k)):
|
| break
|
| return {
|
| "enabled": True,
|
| "hits": merged[: max(1, int(top_k))],
|
| "promoted_hits": promoted,
|
| "metadata": {
|
| "identifier_protected_enabled": True,
|
| "identifier_query_tokens": identifier_tokens,
|
| "identifier_request": bool(identifier_request),
|
| "identifier_promoted_count": len(promoted),
|
| "identifier_promoted_ids": [hit.memory_id for hit in promoted],
|
| },
|
| }
|
|
|
|
|
| def _trim_prompt_text(value: Any, *, max_chars: int = 220) -> str:
|
| text = _clean_text(value)
|
| if len(text) <= max_chars:
|
| return text
|
| return f"{text[: max(0, max_chars - 3)].rstrip()}..."
|
|
|
|
|
| def _prompt_hit_payload(hit: MemoryHit) -> Dict[str, Any]:
|
| return {
|
| "memory_id": hit.memory_id,
|
| "slot_key": hit.slot_key,
|
| "category": hit.category,
|
| "value": _trim_prompt_text(hit.value),
|
| "relation": hit.relation,
|
| "anchors": [_trim_prompt_text(anchor, max_chars=80) for anchor in list(hit.anchors)[:4]],
|
| "score": round(float(hit.score), 6),
|
| "state": hit.state,
|
| "turn_index": int(hit.turn_index),
|
| "source_kind": hit.source_kind,
|
| }
|
|
|
|
|
| def _prompt_record_payload(record: SessionMemoryRecordV2) -> Dict[str, Any]:
|
| metadata = dict(record.metadata or {})
|
| return {
|
| "slot_key": record.slot_key,
|
| "category": record.category,
|
| "value": _trim_prompt_text(record.value),
|
| "state": record.state,
|
| "turn_index": int(record.turn_index),
|
| "anchors": [_trim_prompt_text(anchor, max_chars=80) for anchor in list(record.anchor_concepts)[:4]],
|
| "memory_role": _clean_text(metadata.get("memory_role", "")),
|
| "authority": _clean_text(metadata.get("authority", "")),
|
| }
|
|
|
|
|
| def _graph_prompt_state_summary(graph: SessionMemoryGraphV2, retrieval: MemoryRetrieval) -> Dict[str, Any]:
|
| active_slots: List[Dict[str, Any]] = []
|
| for slot_key, record_id in list(graph.slot_heads.items())[:_GRAPH_PROMPT_MAX_ACTIVE_SLOTS]:
|
| record = graph.records_by_id.get(record_id)
|
| if record is None:
|
| continue
|
| active_slots.append(_prompt_record_payload(record))
|
| top_hits = [_prompt_hit_payload(hit) for hit in list(retrieval.hits)[:_GRAPH_PROMPT_MAX_HITS]]
|
| relation_preview = [
|
| {
|
| "from": _trim_prompt_text(item.get("from", ""), max_chars=72),
|
| "to": _trim_prompt_text(item.get("to", ""), max_chars=72),
|
| "relation": _clean_text(item.get("relation", "")),
|
| }
|
| for item in list(retrieval.relations)[:_GRAPH_PROMPT_MAX_RELATIONS]
|
| ]
|
| summary = {
|
| "records": len(graph.records_by_id),
|
| "active_slots": len(graph.slot_heads),
|
| "turn_index": int(graph.turn_index),
|
| "noise_turn_count": int(graph.noise_turn_count),
|
| "answer_support_events": len(graph.answer_support_log),
|
| "top_hits": top_hits,
|
| "active_slot_records": active_slots,
|
| "relation_preview": relation_preview,
|
| "context_truncated": False,
|
| "truncation_reason": "",
|
| }
|
| truncated = False
|
| truncation_reason = ""
|
| while len(json.dumps(summary, ensure_ascii=False)) > _GRAPH_PROMPT_MAX_CHARS:
|
| if len(summary["top_hits"]) > 1:
|
| summary["top_hits"] = summary["top_hits"][:-1]
|
| truncated = True
|
| truncation_reason = "trimmed_top_hits"
|
| continue
|
| if len(summary["active_slot_records"]) > 1:
|
| summary["active_slot_records"] = summary["active_slot_records"][:-1]
|
| truncated = True
|
| truncation_reason = "trimmed_active_slots"
|
| continue
|
| if len(summary["relation_preview"]) > 2:
|
| summary["relation_preview"] = summary["relation_preview"][:-1]
|
| truncated = True
|
| truncation_reason = "trimmed_relations"
|
| continue
|
| break
|
| summary["context_truncated"] = truncated
|
| summary["truncation_reason"] = truncation_reason
|
| return summary
|
|
|
|
|
| def _state_stats(*, storage_bytes: int, retrieval_context_tokens: int, total_state_tokens: int, **extra: Any) -> Dict[str, Any]:
|
| return {
|
| **extra,
|
| "storage_bytes": int(storage_bytes),
|
| "context_token_estimate": int(retrieval_context_tokens),
|
| "retrieval_context_token_estimate": int(retrieval_context_tokens),
|
| "total_state_token_estimate": int(total_state_tokens),
|
| }
|
|
|
|
|
| def _relation_hit(hit: MemoryHit, *, weight_bias: float = 0.0) -> Dict[str, Any]:
|
| if not hit.anchors:
|
| return {}
|
| anchor = hit.anchors[0]
|
| if not anchor or anchor == hit.value:
|
| return {}
|
| return {
|
| "from": anchor,
|
| "to": hit.value,
|
| "relation": hit.relation,
|
| "weight": round(max(0.25, min(0.98, 0.42 + hit.score * 0.4 + weight_bias)), 6),
|
| "source_kind": hit.source_kind,
|
| "memory_id": hit.memory_id,
|
| }
|
|
|
|
|
| def _raw_hit_to_memory_hit(payload: Dict[str, Any]) -> MemoryHit:
|
| metadata = dict(payload.get("metadata", {}) or {})
|
| if payload.get("supersedes"):
|
| metadata["supersedes"] = list(payload.get("supersedes", []) or [])
|
| slot_key = stable_slot_key(
|
| category=str(payload.get("category", "")),
|
| value=str(payload.get("value", "")),
|
| anchors=[str(anchor) for anchor in payload.get("anchor_concepts", payload.get("anchors", [])) or [] if _clean_text(anchor)],
|
| slot_key=str(payload.get("slot_key", metadata.get("slot", ""))),
|
| relation=str(payload.get("relation", "related_to")),
|
| metadata=metadata,
|
| )
|
| return MemoryHit(
|
| memory_id=str(payload.get("memory_id", "")),
|
| category=str(payload.get("category", "")),
|
| value=str(payload.get("value", "")),
|
| relation=str(payload.get("relation", "related_to")),
|
| anchors=[str(anchor) for anchor in payload.get("anchor_concepts", payload.get("anchors", [])) or [] if _clean_text(anchor)],
|
| score=float(payload.get("score", payload.get("relevance", 0.0)) or 0.0),
|
| source_kind=str(payload.get("source_kind", "memory")),
|
| slot_key=slot_key,
|
| state=str(payload.get("state", payload.get("metadata", {}).get("state", "active")) or "active"),
|
| turn_index=int(payload.get("turn_index", 0) or 0),
|
| metadata=metadata,
|
| )
|
|
|
|
|
| def _restore_hit_scores(hits: List[MemoryHit], scored_lookup: Dict[str, MemoryHit]) -> List[MemoryHit]:
|
| restored: List[MemoryHit] = []
|
| for hit in hits:
|
| scored = scored_lookup.get(hit.memory_id)
|
| if scored:
|
| hit.score = max(float(hit.score), float(scored.score))
|
| if not hit.anchors and scored.anchors:
|
| hit.anchors = list(scored.anchors)
|
| restored.append(hit)
|
| return restored
|
|
|
|
|
| def _current_subject_query(query: str) -> bool:
|
| lowered = _normalize(query)
|
| return bool(
|
| _public_query_subject(query)
|
| and any(marker in lowered for marker in ("right now", "current", "currently", "active", "now", "当前", "现在"))
|
| )
|
|
|
|
|
| def _record_subject_signatures(record: SessionMemoryRecordV2) -> set[str]:
|
| metadata = dict(record.metadata or {})
|
| signatures = {
|
| _normalize(metadata.get("subject_signature", "")).replace("-", "_"),
|
| _public_subject_signature(metadata.get("subject", "")),
|
| }
|
| canonical_slot_key = _clean_text(metadata.get("canonical_slot_key", "") or record.slot_key)
|
| if ".subject." in canonical_slot_key:
|
| signatures.add(_public_subject_signature(canonical_slot_key.split(".subject.", 1)[-1]))
|
| if ".subject." in record.slot_key:
|
| signatures.add(_public_subject_signature(record.slot_key.split(".subject.", 1)[-1]))
|
| signatures.discard("")
|
| return signatures
|
|
|
|
|
| def _current_subject_protected_hits(
|
| *,
|
| query: str,
|
| graph: SessionMemoryGraphV2,
|
| final_hits: Sequence[MemoryHit],
|
| top_k: int,
|
| ) -> Dict[str, Any]:
|
| if not _current_subject_query(query):
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {"current_subject_resolver_enabled": False},
|
| }
|
| subject = _public_query_subject(query)
|
| subject_signature = _public_subject_signature(subject)
|
| if not subject_signature:
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {
|
| "current_subject_resolver_enabled": True,
|
| "current_subject_resolver_reason": "no_subject_signature",
|
| },
|
| }
|
| def _candidate_record(record: SessionMemoryRecordV2 | None) -> bool:
|
| return bool(
|
| record is not None
|
| and _clean_text(record.source_kind).startswith("public_dialog")
|
| and _normalize(record.category) != "question"
|
| and subject_signature in _record_subject_signatures(record)
|
| )
|
|
|
| slot_head_candidates = [
|
| record
|
| for slot_key, memory_id in graph.slot_heads.items()
|
| for record in [graph.records_by_id.get(memory_id)]
|
| if _candidate_record(record)
|
| and subject_signature in _record_subject_signatures(record)
|
| ]
|
| candidates = slot_head_candidates or [
|
| record
|
| for record in graph.records_by_id.values()
|
| if record.state == "active" and _candidate_record(record)
|
| ]
|
| candidates.sort(
|
| key=lambda record: (
|
| int(
|
| _normalize((record.metadata or {}).get("target_status", "")) == "current"
|
| or _normalize(record.relation) == "current_subject_value"
|
| ),
|
| int(record.turn_index),
|
| float(record.confidence),
|
| float(record.salience),
|
| ),
|
| reverse=True,
|
| )
|
| promoted: List[MemoryHit] = []
|
| for index, record in enumerate(candidates[: max(1, min(2, int(top_k or 1)))], start=1):
|
| metadata = dict(record.metadata or {})
|
| metadata.update(
|
| {
|
| "current_subject_resolver": True,
|
| "current_subject_resolver_rank": index,
|
| "current_subject_query_subject": subject,
|
| "current_subject_query_signature": subject_signature,
|
| "public_subject_match": True,
|
| "public_subject_overlap": 1.0,
|
| }
|
| )
|
| promoted.append(
|
| MemoryHit(
|
| memory_id=record.memory_id,
|
| category=record.category,
|
| value=record.value,
|
| relation=record.relation,
|
| anchors=list(record.anchor_concepts),
|
| score=max(float(record.confidence), float(record.salience), 1.75),
|
| source_kind=record.source_kind,
|
| slot_key=record.slot_key,
|
| state=record.state,
|
| turn_index=int(record.turn_index),
|
| metadata=metadata,
|
| )
|
| )
|
| if not promoted:
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {
|
| "current_subject_resolver_enabled": True,
|
| "current_subject_resolver_reason": "no_active_subject_head",
|
| "current_subject_query_subject": subject,
|
| "current_subject_query_signature": subject_signature,
|
| },
|
| }
|
| promoted_ids = {hit.memory_id for hit in promoted}
|
| merged_tail: List[MemoryHit] = []
|
| for hit in final_hits:
|
| if hit.memory_id in promoted_ids:
|
| continue
|
| metadata = dict(hit.metadata or {})
|
| hit_state = _normalize(hit.state)
|
| same_subject = subject_signature in {
|
| _normalize(metadata.get("subject_signature", "")).replace("-", "_"),
|
| _public_subject_signature(metadata.get("subject", "")),
|
| _public_subject_signature(hit.slot_key.split(".subject.", 1)[-1]) if ".subject." in hit.slot_key else "",
|
| _public_subject_signature(_clean_text(metadata.get("canonical_slot_key", "")).split(".subject.", 1)[-1])
|
| if ".subject." in _clean_text(metadata.get("canonical_slot_key", ""))
|
| else "",
|
| }
|
| if same_subject and hit_state in {"superseded", "evidence", "historical", "stale", "false"}:
|
| continue
|
| merged_tail.append(hit)
|
| merged = [*promoted, *merged_tail]
|
| limit = max(int(top_k or 1), len(promoted))
|
| return {
|
| "hits": merged[:limit],
|
| "metadata": {
|
| "current_subject_resolver_enabled": True,
|
| "current_subject_resolver_reason": "promoted_active_subject_head",
|
| "current_subject_query_subject": subject,
|
| "current_subject_query_signature": subject_signature,
|
| "current_subject_promoted_memory_ids": [hit.memory_id for hit in promoted],
|
| },
|
| }
|
|
|
|
|
| def _depth_chain_protected_hits(
|
| *,
|
| query: str,
|
| graph: SessionMemoryGraphV2,
|
| final_hits: Sequence[MemoryHit],
|
| top_k: int,
|
| ) -> Dict[str, Any]:
|
| seed_memory_ids = [hit.memory_id for hit in final_hits if hit.memory_id]
|
| chain = graph.depth_chain_for_query(
|
| query,
|
| seed_memory_ids=seed_memory_ids,
|
| top_k=max(3, min(8, int(top_k or 1))),
|
| )
|
| if not chain.get("enabled") or not chain.get("nodes"):
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {
|
| "memory_chain_enabled": bool(chain.get("enabled", False)),
|
| "memory_chain_reason": _clean_text(chain.get("reason", "")),
|
| "memory_chain_node_count": 0,
|
| "memory_chain_edge_count": 0,
|
| "memory_chain": chain,
|
| },
|
| }
|
| seen = {hit.memory_id for hit in final_hits if hit.memory_id}
|
| chain_hits: List[MemoryHit] = []
|
| for rank, node in enumerate(list(chain.get("nodes", []) or []), start=1):
|
| if not isinstance(node, Mapping):
|
| continue
|
| memory_id = _clean_text(node.get("memory_id", ""))
|
| if not memory_id or memory_id in seen:
|
| continue
|
| payload = dict(node)
|
| metadata = dict(payload.get("metadata", {}) or {})
|
| metadata.update(
|
| {
|
| "memory_chain_protected": True,
|
| "memory_chain_rank": int(rank),
|
| "memory_chain_subject_signature": _clean_text(chain.get("subject_signature", "")),
|
| "memory_chain_depth_layer": _clean_text(metadata.get("depth_layer", "")) or "core_view",
|
| }
|
| )
|
| payload["metadata"] = metadata
|
| payload["score"] = max(float(payload.get("score", 0.0) or 0.0), 0.62 - (rank * 0.01))
|
| chain_hits.append(_raw_hit_to_memory_hit(payload))
|
| seen.add(memory_id)
|
| limit = max(int(top_k or 1), min(12, int(top_k or 1) + max(0, len(chain_hits))))
|
| merged = [*list(final_hits), *chain_hits]
|
| return {
|
| "hits": merged[:limit],
|
| "metadata": {
|
| "memory_chain_enabled": True,
|
| "memory_chain_reason": _clean_text(chain.get("reason", "")),
|
| "memory_chain_subject_signature": _clean_text(chain.get("subject_signature", "")),
|
| "memory_chain_node_count": int(chain.get("node_count", 0) or 0),
|
| "memory_chain_edge_count": int(chain.get("edge_count", 0) or 0),
|
| "memory_chain_depth_layers": list(chain.get("depth_layers", []) or []),
|
| "memory_chain": chain,
|
| },
|
| }
|
|
|
|
|
| def _is_public_dialog_hit(hit: MemoryHit) -> bool:
|
| return _clean_text(hit.source_kind).startswith("public_dialog")
|
|
|
|
|
| def _normalized_runtime_signature(prefix: str, value: str) -> str:
|
| normalized = _normalize(value)
|
| if not normalized:
|
| return ""
|
| return f"{prefix}{normalized.replace('|', '_').replace(':', '_')}"
|
|
|
|
|
| def _runtime_event_key(hit: MemoryHit) -> str:
|
| metadata = dict(hit.metadata or {})
|
| explicit = _clean_text(metadata.get("event_id", ""))
|
| if explicit:
|
| return explicit
|
| dia_id = _clean_text(metadata.get("dia_id", ""))
|
| if dia_id:
|
| return f"event::{dia_id}"
|
| if not _is_public_dialog_hit(hit):
|
| state_signature = _clean_text(metadata.get("state_signature", ""))
|
| if state_signature:
|
| return _normalized_runtime_signature("event::state::", state_signature)
|
| memory_signature = _clean_text(metadata.get("memory_signature", ""))
|
| if memory_signature:
|
| return _normalized_runtime_signature("event::memory::", memory_signature)
|
| slot_root = _public_slot_root(_clean_text(hit.slot_key))
|
| if slot_root:
|
| return slot_root
|
| return _clean_text(hit.memory_id)
|
|
|
|
|
| def _runtime_event_turn_index_from_id(event_id: str) -> int:
|
| text = _clean_text(event_id)
|
| if not text:
|
| return 0
|
| match = re.search(r"(?::|_)(\d+)$", text)
|
| if match:
|
| return int(match.group(1))
|
| matches = re.findall(r"\d+", text)
|
| return int(matches[-1]) if matches else 0
|
|
|
|
|
| def _representative_event_hit(group_hits: Sequence[MemoryHit], *, query: str = "") -> MemoryHit | None:
|
| semantic_source_kinds = {
|
| "public_dialog_fact",
|
| "public_dialog_preference",
|
| "public_dialog_goal",
|
| "public_dialog_constraint",
|
| "public_dialog_status",
|
| "public_dialog_profile",
|
| "replacement_memory",
|
| "session_memory",
|
| }
|
| query_tokens = set(_path_utility_tokens(query))
|
|
|
| def rank(hit: MemoryHit) -> tuple[bool, float, bool, bool, bool, bool, float]:
|
| metadata = dict(hit.metadata or {})
|
| source_kind = _clean_text(hit.source_kind)
|
| text_parts: List[Any] = [hit.value, hit.slot_key, hit.category]
|
| if source_kind != "public_dialog_turn":
|
| text_parts.extend([metadata.get("source_turn_text", ""), metadata.get("raw_text", "")])
|
| text = " ".join(
|
| _clean_text(item)
|
| for item in text_parts
|
| if _clean_text(item)
|
| )
|
| hit_tokens = set(_path_utility_tokens(text))
|
| overlap = len(query_tokens & hit_tokens) if query_tokens else 0
|
| has_number = bool(re.search(r"\b\d+\b", text))
|
| is_semantic = source_kind in semantic_source_kinds or (
|
| source_kind != "public_dialog_turn" and bool(_clean_text(metadata.get("memory_writer_role", "")))
|
| )
|
| direct_semantic_answer = bool(is_semantic and has_number and overlap >= 2)
|
| query_score = float(overlap) + (0.75 if has_number and overlap else 0.0)
|
| return (
|
| direct_semantic_answer,
|
| query_score,
|
| is_semantic,
|
| source_kind == "public_dialog_event",
|
| source_kind == "public_dialog_turn",
|
| source_kind in {"replacement_memory", "session_memory"},
|
| float(hit.score),
|
| )
|
|
|
| ordered = sorted(
|
| list(group_hits),
|
| key=rank,
|
| reverse=True,
|
| )
|
| return ordered[0] if ordered else None
|
|
|
|
|
| def _event_record_hits_from_graph(graph: SessionMemoryGraphV2, event_id: str) -> List[MemoryHit]:
|
| normalized_event_id = _clean_text(event_id)
|
| if not normalized_event_id:
|
| return []
|
| hits: List[MemoryHit] = []
|
| for record in graph.records_by_id.values():
|
| metadata = dict(record.metadata or {})
|
| if _clean_text(metadata.get("event_id", "")) != normalized_event_id:
|
| continue
|
| state = _normalize(record.state)
|
| if state not in {"active", "parallel_active", "evidence"}:
|
| continue
|
| hits.append(
|
| MemoryHit(
|
| memory_id=record.memory_id,
|
| category=record.category,
|
| value=record.value,
|
| relation=record.relation,
|
| anchors=list(record.anchor_concepts),
|
| score=max(float(record.confidence), float(record.salience), 0.01),
|
| source_kind=record.source_kind,
|
| slot_key=record.slot_key,
|
| state=record.state,
|
| turn_index=int(record.turn_index),
|
| metadata=metadata,
|
| )
|
| )
|
| return hits
|
|
|
| def _hit_matches_path_support(path_type: str, hit: MemoryHit) -> bool:
|
| metadata = dict(hit.metadata or {})
|
| source_kind = _clean_text(hit.source_kind)
|
| category = _clean_text(hit.category)
|
| relation = _clean_text(hit.relation)
|
| if path_type == "speaker_event_time":
|
| return bool(
|
| source_kind == "public_dialog_time"
|
| or _clean_text(metadata.get("resolved_time_value", ""))
|
| or _clean_text(metadata.get("resolved_date", ""))
|
| or _clean_text(metadata.get("time_value", ""))
|
| or _clean_text(metadata.get("time_display_value", ""))
|
| or _clean_text(metadata.get("time_granularity", ""))
|
| or relation == "event_date"
|
| or category in {"time", "event_time"}
|
| )
|
| if path_type == "speaker_event_profile":
|
| semantic_slot = _clean_text(metadata.get("semantic_slot", "")) or _clean_text(metadata.get("profile_type", ""))
|
| return bool(
|
| source_kind == "public_dialog_profile"
|
| or semantic_slot in {"identity", "research_topic", "education", "occupation", "profile"}
|
| or _clean_text(metadata.get("profile_value", ""))
|
| or category == "profile"
|
| )
|
| if path_type == "speaker_event_status":
|
| return bool(
|
| _clean_text(metadata.get("target_status", ""))
|
| or category in {"status", "stage_state"}
|
| or relation == "status_of"
|
| )
|
| if path_type == "speaker_event_source_turn":
|
| return bool(
|
| source_kind in {"public_dialog_turn", "public_dialog_text", "public_dialog_auxiliary_evidence"}
|
| or _clean_text(metadata.get("raw_text", ""))
|
| or _clean_text(metadata.get("origin_query", ""))
|
| or _clean_text(metadata.get("source_turn_text", ""))
|
| or not _is_public_dialog_hit(hit)
|
| )
|
| return False
|
|
|
|
|
| def _support_hit_for_path(path_type: str, group_hits: Sequence[MemoryHit]) -> MemoryHit | None:
|
| matching_hits = [hit for hit in group_hits if _hit_matches_path_support(path_type, hit)]
|
| if matching_hits:
|
| matching_hits.sort(key=lambda item: (float(item.score), int(item.turn_index)), reverse=True)
|
| return matching_hits[0]
|
| representative = _representative_event_hit(group_hits)
|
| return representative
|
|
|
|
|
| def _path_support_node_id(path: Dict[str, Any]) -> str:
|
| node_ids = list(path.get("node_ids", []) or [])
|
| if len(node_ids) < 3:
|
| return ""
|
| return _clean_text(node_ids[2])
|
|
|
|
|
| def _event_ids_from_hits(hits: Sequence[MemoryHit]) -> List[str]:
|
| return _dedupe(
|
| _clean_text(dict(hit.metadata or {}).get("event_id", ""))
|
| for hit in hits
|
| if _clean_text(dict(hit.metadata or {}).get("event_id", ""))
|
| )
|
|
|
|
|
| def _dia_ids_from_hits(hits: Sequence[MemoryHit]) -> List[str]:
|
| return _dedupe(
|
| _clean_text(dict(hit.metadata or {}).get("dia_id", ""))
|
| for hit in hits
|
| if _clean_text(dict(hit.metadata or {}).get("dia_id", ""))
|
| )
|
|
|
|
|
| def _final_hit_role_priority(hit: MemoryHit) -> int:
|
| metadata = dict(hit.metadata or {})
|
| source_kind = _clean_text(hit.source_kind)
|
| semantic_source_kinds = {
|
| "public_dialog_fact",
|
| "public_dialog_preference",
|
| "public_dialog_goal",
|
| "public_dialog_constraint",
|
| "public_dialog_status",
|
| "public_dialog_profile",
|
| "public_dialog_profile_cluster",
|
| "replacement_memory",
|
| "session_memory",
|
| }
|
| if source_kind in semantic_source_kinds or (
|
| source_kind != "public_dialog_turn" and bool(_clean_text(metadata.get("memory_writer_role", "")))
|
| ):
|
| return 0
|
| if bool(metadata.get("profile_first_source_support")):
|
| return 0
|
| role = _clean_text(metadata.get("evidence_snippet_role", ""))
|
| if role == "selected_path_support":
|
| return 1
|
| if role == "selected_event_representative":
|
| return 2
|
| if role == "selected_path_event":
|
| return 3
|
| return 4
|
|
|
|
|
| def _coverage_preserving_final_hits(
|
| final_hits: Sequence[MemoryHit],
|
| *,
|
| selected_event_ids: Sequence[str],
|
| top_k: int,
|
| ) -> List[MemoryHit]:
|
| """Keep selected-event coverage before filling the remaining prompt budget.
|
|
|
| Learned selection can emit both path-support and event-representative snippets
|
| for the same event. A pure score sort can then drop another selected event at
|
| the top-k boundary, which hides recall/rerank successes from the answer head.
|
| """
|
|
|
| budget = max(1, int(top_k or 1))
|
| hits = list(final_hits)
|
| selected_order = [
|
| _clean_text(event_id)
|
| for event_id in selected_event_ids
|
| if _clean_text(event_id)
|
| ][:budget]
|
| if not selected_order:
|
| return sorted(hits, key=lambda item: float(item.score), reverse=True)[:budget]
|
| hits_by_event: Dict[str, List[MemoryHit]] = {}
|
| for hit in hits:
|
| event_id = _clean_text(dict(hit.metadata or {}).get("event_id", ""))
|
| if event_id:
|
| hits_by_event.setdefault(event_id, []).append(hit)
|
| selected: List[MemoryHit] = []
|
| used_memory_ids = set()
|
| for event_id in selected_order:
|
| candidates = [hit for hit in hits_by_event.get(event_id, []) if hit.memory_id not in used_memory_ids]
|
| if not candidates:
|
| continue
|
| candidates.sort(key=lambda item: (_final_hit_role_priority(item), -float(item.score), item.memory_id))
|
| chosen = candidates[0]
|
| selected.append(chosen)
|
| used_memory_ids.add(chosen.memory_id)
|
| if len(selected) >= budget:
|
| return selected
|
| remaining = [
|
| hit
|
| for hit in hits
|
| if hit.memory_id not in used_memory_ids
|
| ]
|
| remaining.sort(key=lambda item: (-float(item.score), _final_hit_role_priority(item), item.memory_id))
|
| for hit in remaining:
|
| selected.append(hit)
|
| if len(selected) >= budget:
|
| break
|
| return selected[:budget]
|
|
|
|
|
| def _dominant_answer_type(question_analysis: Dict[str, Any], answer_type_scores: Dict[str, Any]) -> str:
|
| normalized_scores = {
|
| _clean_text(answer_type): float(value or 0.0)
|
| for answer_type, value in dict(answer_type_scores or {}).items()
|
| if _clean_text(answer_type)
|
| }
|
| if bool(question_analysis.get("is_temporal", False)) and normalized_scores.get("time", 0.0) >= (normalized_scores.get("abstain", 0.0) - 0.05):
|
| return "time"
|
| if not bool(question_analysis.get("is_temporal", False)) and normalized_scores:
|
| non_time_scores = {
|
| answer_type: score
|
| for answer_type, score in normalized_scores.items()
|
| if answer_type not in {"time", "abstain"}
|
| }
|
| if non_time_scores:
|
| return max(non_time_scores.items(), key=lambda item: (float(item[1]), item[0]))[0]
|
| if not normalized_scores:
|
| return "time" if bool(question_analysis.get("is_temporal", False)) else "event_text"
|
| return max(normalized_scores.items(), key=lambda item: (float(item[1]), item[0]))[0]
|
|
|
|
|
| def _answer_type_preferred_path_types(question_analysis: Dict[str, Any], answer_type_scores: Dict[str, Any]) -> List[str]:
|
| dominant_answer_type = _dominant_answer_type(question_analysis, answer_type_scores)
|
| if dominant_answer_type == "time" or bool(question_analysis.get("is_temporal", False)):
|
| return ["speaker_event_time", "speaker_event_source_turn", "speaker_event_status", "speaker_event_profile"]
|
| if dominant_answer_type == "profile":
|
| return ["speaker_event_profile", "speaker_event_source_turn", "speaker_event_status", "speaker_event_time"]
|
| if dominant_answer_type == "multi_evidence":
|
| return ["speaker_event_source_turn", "speaker_event_time", "speaker_event_profile", "speaker_event_status"]
|
| return ["speaker_event_source_turn", "speaker_event_time", "speaker_event_profile", "speaker_event_status"]
|
|
|
|
|
| def _reconciled_focused_answer_type(
|
| question_analysis: Dict[str, Any],
|
| answer_type_scores: Dict[str, Any],
|
| model_answer_type: str,
|
| ) -> str:
|
| model_type = _clean_text(model_answer_type)
|
| dominant_type = _dominant_answer_type(question_analysis, answer_type_scores)
|
| if bool(question_analysis.get("is_temporal", False)) and model_type not in {"", "time", "abstain"}:
|
| return "time"
|
| if model_type == "time" and not bool(question_analysis.get("is_temporal", False)):
|
| return dominant_type if dominant_type != "time" else "event_text"
|
| return model_type or dominant_type
|
|
|
|
|
| def _path_type_is_focus_compatible(path_type: str, *, focused_answer_type: str, question_analysis: Dict[str, Any]) -> bool:
|
| normalized_path_type = _clean_text(path_type)
|
| normalized_answer_type = _clean_text(focused_answer_type)
|
| if normalized_answer_type == "time" or bool(question_analysis.get("is_temporal", False)):
|
| return normalized_path_type in {"speaker_event_time", "speaker_event_source_turn"}
|
| if normalized_answer_type == "profile":
|
| return normalized_path_type in {"speaker_event_profile", "speaker_event_source_turn"}
|
| return True
|
|
|
|
|
| def _rank_focus_compatible_path_ids(
|
| *,
|
| runtime_paths: Mapping[str, Dict[str, Any]],
|
| selected_event_ids: Sequence[str],
|
| path_scores: Mapping[str, Any],
|
| event_scores: Mapping[str, Any],
|
| temporal_scores: Mapping[str, Any],
|
| question_analysis: Dict[str, Any],
|
| answer_type_scores: Dict[str, Any],
|
| focused_answer_type: str,
|
| ) -> List[str]:
|
| selected_event_id_set = {_clean_text(event_id) for event_id in selected_event_ids if _clean_text(event_id)}
|
| preferred_types = _answer_type_preferred_path_types(question_analysis, answer_type_scores)
|
| if _clean_text(focused_answer_type) == "profile" and "speaker_event_profile" not in preferred_types:
|
| preferred_types = ["speaker_event_profile", *preferred_types]
|
| if (_clean_text(focused_answer_type) == "time" or bool(question_analysis.get("is_temporal", False))) and "speaker_event_time" not in preferred_types:
|
| preferred_types = ["speaker_event_time", "speaker_event_source_turn", *preferred_types]
|
| ranked: List[str] = []
|
| seen = set()
|
| for preferred_type in preferred_types:
|
| candidates: List[tuple[str, float]] = []
|
| for path_id, path in runtime_paths.items():
|
| if _clean_text(path.get("type", "")) != preferred_type:
|
| continue
|
| event_id = _clean_text(path.get("event_id", ""))
|
| if selected_event_id_set and event_id not in selected_event_id_set:
|
| continue
|
| if not _path_type_is_focus_compatible(preferred_type, focused_answer_type=focused_answer_type, question_analysis=question_analysis):
|
| continue
|
| support_node_id = _path_support_node_id(path)
|
| score = (
|
| float(event_scores.get(event_id, 0.0) or 0.0)
|
| + (0.20 * float(path_scores.get(path_id, 0.0) or 0.0))
|
| + (0.15 * float(temporal_scores.get(support_node_id, 0.0) or 0.0))
|
| )
|
| candidates.append((path_id, score))
|
| for path_id, _ in sorted(candidates, key=lambda item: (-float(item[1]), item[0])):
|
| if path_id in seen:
|
| continue
|
| seen.add(path_id)
|
| ranked.append(path_id)
|
| if ranked and (_clean_text(focused_answer_type) == "time" or bool(question_analysis.get("is_temporal", False))):
|
| if preferred_type in {"speaker_event_time", "speaker_event_source_turn"}:
|
| break
|
| return ranked
|
|
|
|
|
| def _repair_selected_paths_for_focus(
|
| selected_path_ids: Sequence[str],
|
| *,
|
| runtime_paths: Mapping[str, Dict[str, Any]],
|
| selected_event_ids: Sequence[str],
|
| path_scores: Mapping[str, Any],
|
| event_scores: Mapping[str, Any],
|
| temporal_scores: Mapping[str, Any],
|
| question_analysis: Dict[str, Any],
|
| answer_type_scores: Dict[str, Any],
|
| focused_answer_type: str,
|
| limit: int,
|
| ) -> tuple[List[str], bool, str]:
|
| normalized_selected = [_clean_text(path_id) for path_id in selected_path_ids if _clean_text(path_id)]
|
| if not normalized_selected:
|
| return [], False, ""
|
| incompatible = [
|
| path_id
|
| for path_id in normalized_selected
|
| if not _path_type_is_focus_compatible(
|
| _clean_text(runtime_paths.get(path_id, {}).get("type", "")),
|
| focused_answer_type=focused_answer_type,
|
| question_analysis=question_analysis,
|
| )
|
| ]
|
| if not incompatible:
|
| return normalized_selected, False, ""
|
| compatible_ranked = _rank_focus_compatible_path_ids(
|
| runtime_paths=runtime_paths,
|
| selected_event_ids=selected_event_ids,
|
| path_scores=path_scores,
|
| event_scores=event_scores,
|
| temporal_scores=temporal_scores,
|
| question_analysis=question_analysis,
|
| answer_type_scores=answer_type_scores,
|
| focused_answer_type=focused_answer_type,
|
| )
|
| if not compatible_ranked:
|
| return normalized_selected, False, ""
|
| repaired = _dedupe(
|
| [
|
| *[path_id for path_id in normalized_selected if path_id not in incompatible],
|
| *compatible_ranked,
|
| ]
|
| )[: max(1, limit)]
|
| if repaired == normalized_selected:
|
| return normalized_selected, False, ""
|
| return repaired, True, "replaced_focus_incompatible_model_paths"
|
|
|
|
|
| def _runtime_node_by_id(runtime_graph: Mapping[str, Any]) -> Dict[str, Dict[str, Any]]:
|
| return {
|
| _clean_text(node.get("id", "")): dict(node)
|
| for node in list(runtime_graph.get("nodes", []) or [])
|
| if _clean_text(node.get("id", ""))
|
| }
|
|
|
|
|
| def _runtime_event_subject_signature(runtime_nodes: Mapping[str, Dict[str, Any]], event_id: str) -> str:
|
| node = dict(runtime_nodes.get(_clean_text(event_id), {}) or {})
|
| metadata = dict(node.get("metadata", {}) or {})
|
| return _clean_text(node.get("subject_signature", "")) or _clean_text(metadata.get("subject_signature", ""))
|
|
|
|
|
| def _path_utility_candidate_text(
|
| path: Mapping[str, Any],
|
| *,
|
| runtime_nodes: Mapping[str, Dict[str, Any]],
|
| grouped_hits: Mapping[str, Sequence[MemoryHit]],
|
| ) -> str:
|
| event_id = _clean_text(path.get("event_id", ""))
|
| support_node_id = _path_support_node_id(dict(path))
|
| node_texts = [
|
| _clean_text(dict(runtime_nodes.get(event_id, {}) or {}).get("text", "")),
|
| _clean_text(dict(runtime_nodes.get(support_node_id, {}) or {}).get("text", "")),
|
| ]
|
| path_type = _clean_text(path.get("type", ""))
|
| support_hit = _support_hit_for_path(path_type, grouped_hits.get(event_id, []))
|
| event_hit = _representative_event_hit(grouped_hits.get(event_id, []))
|
| hit_texts = [
|
| _clean_text(support_hit.value if support_hit is not None else ""),
|
| _clean_text(event_hit.value if event_hit is not None else ""),
|
| _runtime_source_turn_text(support_hit or event_hit, speaker=""),
|
| ]
|
| return " ".join(_dedupe([*node_texts, *hit_texts], max_items=6))
|
|
|
|
|
| def _path_utility_gate(
|
| candidate_path_ids: Sequence[str],
|
| *,
|
| query: str,
|
| runtime_graph: Mapping[str, Any],
|
| runtime_paths: Mapping[str, Dict[str, Any]],
|
| grouped_hits: Mapping[str, Sequence[MemoryHit]],
|
| selected_path_ids: Sequence[str],
|
| selected_event_ids_from_model: Sequence[str],
|
| path_scores: Mapping[str, Any],
|
| path_tunnel_support_scores: Mapping[str, Any],
|
| question_analysis: Dict[str, Any],
|
| focused_answer_type: str,
|
| score_threshold: float,
|
| limit: int,
|
| ) -> Dict[str, Any]:
|
| runtime_nodes = _runtime_node_by_id(runtime_graph)
|
| query_tokens = set(_path_utility_tokens(query))
|
| anchor_event_ids = _dedupe(
|
| [
|
| *[
|
| _clean_text(runtime_paths.get(path_id, {}).get("event_id", ""))
|
| for path_id in selected_path_ids
|
| if _clean_text(runtime_paths.get(path_id, {}).get("event_id", ""))
|
| ],
|
| *[_clean_text(event_id) for event_id in selected_event_ids_from_model if _clean_text(event_id)],
|
| ]
|
| )
|
| anchor_subject_signatures = {
|
| signature
|
| for signature in (
|
| _runtime_event_subject_signature(runtime_nodes, event_id)
|
| for event_id in anchor_event_ids
|
| )
|
| if signature
|
| }
|
| direct_path_ids: List[str] = []
|
| contrast_path_ids: List[str] = []
|
| latent_path_ids: List[str] = []
|
| noise_path_ids: List[str] = []
|
| utility_scores: Dict[str, float] = {}
|
| utility_roles: Dict[str, str] = {}
|
| utility_reasons: Dict[str, str] = {}
|
| utility_overlap_tokens: Dict[str, List[str]] = {}
|
| for path_id in _dedupe(candidate_path_ids):
|
| path = dict(runtime_paths.get(path_id, {}) or {})
|
| if not path:
|
| continue
|
| path_type = _clean_text(path.get("type", ""))
|
| event_id = _clean_text(path.get("event_id", ""))
|
| candidate_text = _path_utility_candidate_text(
|
| path,
|
| runtime_nodes=runtime_nodes,
|
| grouped_hits=grouped_hits,
|
| )
|
| candidate_tokens = set(_path_utility_tokens(candidate_text))
|
| overlap_tokens = sorted(query_tokens & candidate_tokens)
|
| overlap_ratio = float(len(overlap_tokens)) / float(max(1, min(len(query_tokens), len(candidate_tokens))))
|
| support_score = float(path_tunnel_support_scores.get(path_id, 0.0) or 0.0)
|
| decision_score = float(path_scores.get(path_id, support_score) or 0.0)
|
| path_score = max(support_score, decision_score)
|
| subject_signature = _runtime_event_subject_signature(runtime_nodes, event_id)
|
| same_subject_chain = bool(subject_signature and subject_signature in anchor_subject_signatures)
|
| focus_compatible = _path_type_is_focus_compatible(
|
| path_type,
|
| focused_answer_type=focused_answer_type,
|
| question_analysis=question_analysis,
|
| )
|
| utility_score = path_score + (0.20 * overlap_ratio) + (0.04 if same_subject_chain else 0.0)
|
| utility_scores[path_id] = round(float(utility_score), 6)
|
| utility_overlap_tokens[path_id] = overlap_tokens[:12]
|
| if not focus_compatible:
|
| role = "drift_noise"
|
| reason = "focus_incompatible"
|
| elif overlap_ratio >= 0.18 and path_score >= score_threshold:
|
| role = "direct_support"
|
| reason = "query_overlap_and_tunnel_score"
|
| elif same_subject_chain and path_score >= score_threshold:
|
| role = "contrast_support"
|
| reason = "same_subject_deep_chain"
|
| elif path_score >= score_threshold:
|
| role = "latent_context"
|
| reason = "tunnel_score_without_current_turn_utility"
|
| else:
|
| role = "drift_noise"
|
| reason = "below_utility_threshold"
|
| utility_roles[path_id] = role
|
| utility_reasons[path_id] = reason
|
| if role == "direct_support":
|
| direct_path_ids.append(path_id)
|
| elif role == "contrast_support":
|
| contrast_path_ids.append(path_id)
|
| elif role == "latent_context":
|
| latent_path_ids.append(path_id)
|
| else:
|
| noise_path_ids.append(path_id)
|
| injected_path_ids = _dedupe([*direct_path_ids, *contrast_path_ids], max_items=max(0, int(limit)))
|
| overflow_latent_path_ids = [
|
| path_id
|
| for path_id in [*direct_path_ids, *contrast_path_ids]
|
| if path_id not in set(injected_path_ids)
|
| ]
|
| latent_path_ids = _dedupe([*latent_path_ids, *overflow_latent_path_ids])
|
| return {
|
| "enabled": True,
|
| "candidate_path_ids": list(_dedupe(candidate_path_ids)),
|
| "direct_support_path_ids": list(direct_path_ids),
|
| "contrast_support_path_ids": list(contrast_path_ids),
|
| "latent_context_path_ids": list(latent_path_ids),
|
| "drift_noise_path_ids": list(noise_path_ids),
|
| "injected_path_ids": list(injected_path_ids),
|
| "roles": dict(utility_roles),
|
| "reasons": dict(utility_reasons),
|
| "scores": dict(utility_scores),
|
| "overlap_tokens": dict(utility_overlap_tokens),
|
| "anchor_event_ids": list(anchor_event_ids),
|
| "anchor_subject_signatures": sorted(anchor_subject_signatures),
|
| }
|
|
|
|
|
| def _calibrated_path_score(
|
| *,
|
| path: Dict[str, Any],
|
| base_score: float,
|
| temporal_scores: Dict[str, Any],
|
| question_analysis: Dict[str, Any],
|
| answer_type_scores: Dict[str, Any],
|
| ) -> float:
|
| normalized_path_type = _clean_text(path.get("type", ""))
|
| support_node_id = _path_support_node_id(path)
|
| temporal_score = float(temporal_scores.get(support_node_id, 0.0) or 0.0)
|
| normalized_answer_scores = {
|
| _clean_text(answer_type): float(value or 0.0)
|
| for answer_type, value in dict(answer_type_scores or {}).items()
|
| if _clean_text(answer_type)
|
| }
|
| dominant_answer_type = _dominant_answer_type(question_analysis, normalized_answer_scores)
|
| calibrated = float(base_score)
|
| if dominant_answer_type == "time" or bool(question_analysis.get("is_temporal", False)):
|
| if normalized_path_type == "speaker_event_time":
|
| calibrated += (0.30 * temporal_score) + (0.12 * normalized_answer_scores.get("time", 0.0))
|
| elif normalized_path_type == "speaker_event_source_turn":
|
| calibrated -= 0.08 + (0.04 * normalized_answer_scores.get("time", 0.0))
|
| elif normalized_path_type == "speaker_event_status":
|
| calibrated -= 0.12
|
| elif normalized_path_type == "speaker_event_profile":
|
| calibrated -= 0.22 + (0.06 * normalized_answer_scores.get("time", 0.0))
|
| elif dominant_answer_type == "profile":
|
| if normalized_path_type == "speaker_event_profile":
|
| calibrated += 0.12 * normalized_answer_scores.get("profile", 0.0)
|
| elif normalized_path_type != "speaker_event_source_turn":
|
| calibrated -= 0.08
|
| else:
|
| if normalized_path_type == "speaker_event_source_turn":
|
| calibrated += 0.08 * max(
|
| normalized_answer_scores.get("event_text", 0.0),
|
| normalized_answer_scores.get("multi_evidence", 0.0),
|
| )
|
| elif normalized_path_type == "speaker_event_profile":
|
| calibrated -= 0.04
|
| return calibrated
|
|
|
|
|
| def _group_metadata_value(
|
| group_hits: Sequence[MemoryHit],
|
| key: str,
|
| *,
|
| source_kinds: Sequence[str] = (),
|
| ) -> str:
|
| source_kind_set = {_clean_text(item) for item in list(source_kinds) if _clean_text(item)}
|
| candidates = [
|
| hit
|
| for hit in group_hits
|
| if not source_kind_set or _clean_text(hit.source_kind) in source_kind_set
|
| ]
|
| candidates.sort(key=lambda item: (float(item.score), int(item.turn_index)), reverse=True)
|
| for hit in candidates:
|
| metadata = dict(hit.metadata or {})
|
| value = _clean_text(metadata.get(key, ""))
|
| if value:
|
| return value
|
| return ""
|
|
|
|
|
| def _runtime_event_sequence_key(session_name: str, turn_index: int, event_id: str) -> tuple[Any, ...]:
|
| normalized_session = _clean_text(session_name)
|
| session_number_match = re.search(r"(\d+)$", normalized_session)
|
| if session_number_match:
|
| return (0, int(session_number_match.group(1)), int(turn_index), _clean_text(event_id))
|
| return (1, normalized_session, int(turn_index), _clean_text(event_id))
|
|
|
|
|
| def _runtime_source_turn_text(hit: MemoryHit | None, *, speaker: str) -> str:
|
| if hit is None:
|
| return ""
|
| metadata = dict(hit.metadata or {})
|
| source_turn_text = _clean_text(metadata.get("source_turn_text", ""))
|
| if source_turn_text:
|
| return source_turn_text
|
| raw_text = _clean_text(metadata.get("raw_text", ""))
|
| if raw_text:
|
| auxiliary_text = _clean_text(metadata.get("auxiliary_evidence_text", ""))
|
| if auxiliary_text and auxiliary_text.lower() not in raw_text.lower():
|
| return f"{raw_text}\nAuxiliary evidence: {auxiliary_text}"
|
| return raw_text
|
| origin_query = _clean_text(metadata.get("origin_query", ""))
|
| if origin_query:
|
| return origin_query
|
| text = _clean_text(hit.value)
|
| if _clean_text(hit.source_kind) == "public_dialog_turn":
|
| text = re.sub(r"^\[[^\]]+\]\s*", "", text)
|
| if _clean_text(speaker):
|
| text = re.sub(rf"^{re.escape(_clean_text(speaker))}\s*:\s*", "", text, flags=re.IGNORECASE)
|
| text = re.sub(r"^[A-Za-z][A-Za-z0-9_' -]{0,40}:\s*", "", text)
|
| return text
|
|
|
|
|
| def _runtime_event_signature(
|
| *,
|
| group_hits: Sequence[MemoryHit],
|
| representative: MemoryHit,
|
| speaker: str,
|
| semantic_slot: str,
|
| source_turn_hit: MemoryHit | None,
|
| ) -> str:
|
| existing = _group_metadata_value(group_hits, "event_signature") or _clean_text(dict(representative.metadata or {}).get("event_signature", ""))
|
| if existing:
|
| return existing
|
| event_phrase = _group_metadata_value(group_hits, "event_phrase") or _clean_text(dict(representative.metadata or {}).get("event_phrase", ""))
|
| source_turn_text = _runtime_source_turn_text(source_turn_hit, speaker=speaker)
|
| base_text = event_phrase or _clean_text(representative.value) or source_turn_text
|
| if not base_text:
|
| return ""
|
| return compute_public_event_signature(
|
| base_text,
|
| speaker=_clean_text(speaker),
|
| semantic_slot=_clean_text(semantic_slot),
|
| ) or _clean_text(base_text)
|
|
|
|
|
| def _build_runtime_graph_from_hits(query: str, hits: Sequence[MemoryHit]) -> Dict[str, Any]:
|
| nodes: List[Dict[str, Any]] = []
|
| edges: List[Dict[str, Any]] = []
|
| paths: List[Dict[str, Any]] = []
|
| node_ids = set()
|
| grouped_hits: Dict[str, List[MemoryHit]] = {}
|
| ordered_events: List[tuple[Any, ...]] = []
|
| event_typed_metadata_by_id: Dict[str, Dict[str, Any]] = {}
|
| typed_tunnel_edges: List[Dict[str, Any]] = []
|
|
|
| def add_node(node: Dict[str, Any]) -> None:
|
| node_id = _clean_text(node.get("id", ""))
|
| if not node_id or node_id in node_ids:
|
| return
|
| node_ids.add(node_id)
|
| nodes.append(node)
|
|
|
| for hit in hits:
|
| event_id = _runtime_event_key(hit)
|
| grouped_hits.setdefault(event_id, []).append(hit)
|
|
|
| for event_id, group_hits in grouped_hits.items():
|
| representative = _representative_event_hit(group_hits)
|
| if representative is None:
|
| continue
|
| metadata = dict(representative.metadata or {})
|
| event_time_hit = _support_hit_for_path("speaker_event_time", group_hits)
|
| event_profile_hit = _support_hit_for_path("speaker_event_profile", group_hits)
|
| source_turn_hit = _support_hit_for_path("speaker_event_source_turn", group_hits)
|
| event_time_display_value = (
|
| _group_metadata_value(group_hits, "time_display_value", source_kinds=("public_dialog_time",))
|
| or _group_metadata_value(group_hits, "resolved_date")
|
| or _group_metadata_value(group_hits, "time_display_value")
|
| )
|
| event_time_value = (
|
| _group_metadata_value(group_hits, "resolved_time_value", source_kinds=("public_dialog_time",))
|
| or _group_metadata_value(group_hits, "resolved_time_value")
|
| or _group_metadata_value(group_hits, "time_value")
|
| )
|
| event_time_granularity = (
|
| _group_metadata_value(group_hits, "time_granularity", source_kinds=("public_dialog_time",))
|
| or _group_metadata_value(group_hits, "time_granularity")
|
| )
|
| event_profile_type = (
|
| _group_metadata_value(group_hits, "profile_type", source_kinds=("public_dialog_profile",))
|
| or _group_metadata_value(group_hits, "semantic_slot", source_kinds=("public_dialog_profile",))
|
| or _group_metadata_value(group_hits, "profile_type")
|
| or _group_metadata_value(group_hits, "semantic_slot")
|
| )
|
| profile_hit = _support_hit_for_path("speaker_event_profile", group_hits)
|
| event_profile_value = (
|
| _group_metadata_value(group_hits, "profile_value")
|
| or _clean_text(profile_hit.value if profile_hit is not None else "")
|
| or (_clean_text(representative.value) if event_profile_type else "")
|
| )
|
| event_target_status = _group_metadata_value(group_hits, "target_status")
|
| depth_layer = (
|
| _clean_text(metadata.get("depth_layer", ""))
|
| or _group_metadata_value(group_hits, "depth_layer")
|
| or _clean_text(metadata.get("memory_chain_depth_layer", ""))
|
| or _group_metadata_value(group_hits, "memory_chain_depth_layer")
|
| )
|
| subject_signature = (
|
| _clean_text(metadata.get("subject_signature", ""))
|
| or _group_metadata_value(group_hits, "subject_signature")
|
| or _clean_text(metadata.get("memory_chain_subject_signature", ""))
|
| or _group_metadata_value(group_hits, "memory_chain_subject_signature")
|
| )
|
| session_name = (
|
| _group_metadata_value(group_hits, "session_name")
|
| or _group_metadata_value(group_hits, "session_key")
|
| or _group_metadata_value(group_hits, "scope_id")
|
| or "runtime_session"
|
| )
|
| speaker = (
|
| _clean_text(metadata.get("speaker", ""))
|
| or _group_metadata_value(group_hits, "speaker")
|
| or _clean_text(metadata.get("subject_signature", ""))
|
| or (_clean_text(representative.anchors[0]) if representative.anchors else "")
|
| or "speaker"
|
| )
|
| event_turn_index = int(getattr(representative, "turn_index", 0) or 0)
|
| semantic_slot = (
|
| _group_metadata_value(group_hits, "semantic_slot")
|
| or _clean_text(metadata.get("semantic_slot", ""))
|
| or _clean_text(metadata.get("profile_type", ""))
|
| or ("profile" if event_profile_type else _clean_text(representative.category))
|
| or "event"
|
| )
|
| teacher_fields = {
|
| "event_phrase": _clean_text(metadata.get("event_phrase", "")) or _clean_text(representative.value),
|
| "semantic_slot": semantic_slot,
|
| "target_status": event_target_status,
|
| "time_expression_span": event_time_display_value,
|
| "time_granularity": event_time_granularity,
|
| "profile_type": event_profile_type,
|
| }
|
| base_event_signature = _runtime_event_signature(
|
| group_hits=group_hits,
|
| representative=representative,
|
| speaker=speaker,
|
| semantic_slot=event_profile_type or teacher_fields["semantic_slot"],
|
| source_turn_hit=source_turn_hit,
|
| )
|
| typed_tunnel_metadata = merge_typed_metadata([metadata, *[dict(hit.metadata or {}) for hit in group_hits]])
|
| event_typed_metadata_by_id[event_id] = typed_tunnel_metadata
|
| typed_signature = typed_tunnel_signature_text(typed_tunnel_metadata)
|
| event_text = _clean_text(metadata.get("event_phrase", "")) or representative.value
|
| depth_prefix = " ".join(
|
| item
|
| for item in (
|
| f"subject {subject_signature.replace('_', ' ')}" if subject_signature else "",
|
| f"depth layer {depth_layer.replace('_', ' ')}" if depth_layer else "",
|
| )
|
| if item
|
| )
|
| runtime_event_text = f"{depth_prefix} {event_text}".strip() if depth_prefix else event_text
|
| event_signature = (
|
| compute_public_event_signature(
|
| runtime_event_text,
|
| speaker=_clean_text(speaker),
|
| semantic_slot=_clean_text(event_profile_type or teacher_fields["semantic_slot"]),
|
| )
|
| or base_event_signature
|
| )
|
| if typed_signature and typed_signature not in event_signature:
|
| event_signature = f"{event_signature} {typed_signature}".strip()
|
| speaker_node_id = f"{event_id}:speaker:{_normalize(speaker).replace(' ', '_') or 'speaker'}"
|
| add_node({"id": speaker_node_id, "type": "speaker", "text": speaker, "metadata": {"speaker": speaker}})
|
| add_node(
|
| {
|
| "id": event_id,
|
| "type": "event",
|
| "text": runtime_event_text,
|
| "speaker": speaker,
|
| "turn_index": event_turn_index,
|
| "session_name": session_name,
|
| "dia_id": _clean_text(metadata.get("dia_id", "")),
|
| "event_signature": event_signature,
|
| "slot_key": _clean_text(representative.slot_key),
|
| "state_signature": _clean_text(metadata.get("state_signature", "")),
|
| "memory_signature": _clean_text(metadata.get("memory_signature", "")),
|
| "target_status": event_target_status,
|
| "time_granularity": event_time_granularity,
|
| "time_value": event_time_value,
|
| "time_display_value": event_time_display_value,
|
| "profile_type": event_profile_type,
|
| "profile_value": event_profile_value,
|
| "depth_layer": depth_layer,
|
| "subject_signature": subject_signature,
|
| "tmcra_node_tags": list(typed_tunnel_metadata.get("tmcra_node_tags", []) or []),
|
| "tmcra_path_tags": list(typed_tunnel_metadata.get("tmcra_path_tags", []) or []),
|
| "tmcra_tunnel_roles": list(typed_tunnel_metadata.get("tmcra_tunnel_roles", []) or []),
|
| "tmcra_tunnel_group_key": _clean_text(typed_tunnel_metadata.get("tmcra_tunnel_group_key", "")),
|
| "teacher_fields": teacher_fields,
|
| "metadata": {
|
| "speaker": speaker,
|
| "session_name": session_name,
|
| "dia_id": _clean_text(metadata.get("dia_id", "")),
|
| "slot_key": _clean_text(representative.slot_key),
|
| "state_signature": _clean_text(metadata.get("state_signature", "")),
|
| "memory_signature": _clean_text(metadata.get("memory_signature", "")),
|
| "target_status": event_target_status,
|
| "time_granularity": event_time_granularity,
|
| "time_value": event_time_value,
|
| "time_display_value": event_time_display_value,
|
| "profile_type": event_profile_type,
|
| "profile_value": event_profile_value,
|
| "event_signature": event_signature,
|
| "depth_layer": depth_layer,
|
| "subject_signature": subject_signature,
|
| **typed_tunnel_metadata,
|
| },
|
| }
|
| )
|
| edges.append({"id": f"{speaker_node_id}->{event_id}:speaker_of", "source": speaker_node_id, "target": event_id, "type": "speaker_of"})
|
|
|
| time_node_ids: List[str] = []
|
| profile_node_ids: List[str] = []
|
| status_node_ids: List[str] = []
|
| source_turn_node_ids: List[str] = []
|
|
|
| if event_time_display_value or event_time_value:
|
| time_node_id = f"{event_id}:time"
|
| time_hit_metadata = dict((event_time_hit or representative).metadata or {})
|
| add_node(
|
| {
|
| "id": time_node_id,
|
| "type": "time",
|
| "text": event_time_display_value or event_time_value,
|
| "turn_index": int(getattr(event_time_hit, "turn_index", 0) or 0),
|
| "time_display_value": event_time_display_value,
|
| "time_value": event_time_value,
|
| "time_granularity": event_time_granularity,
|
| "metadata": {
|
| "time_display_value": event_time_display_value,
|
| "time_value": event_time_value,
|
| "time_granularity": event_time_granularity,
|
| "resolved_date": _clean_text(time_hit_metadata.get("resolved_date", "")),
|
| },
|
| }
|
| )
|
| edges.append({"id": f"{event_id}->{time_node_id}:time_of", "source": event_id, "target": time_node_id, "type": "time_of"})
|
| time_node_ids.append(time_node_id)
|
| if event_profile_value:
|
| profile_node_id = f"{event_id}:profile:{_normalize(event_profile_type).replace(' ', '_') or 'profile'}"
|
| add_node(
|
| {
|
| "id": profile_node_id,
|
| "type": "profile",
|
| "text": event_profile_value,
|
| "turn_index": int(getattr(event_profile_hit or representative, "turn_index", 0) or 0),
|
| "profile_type": event_profile_type,
|
| "profile_value": event_profile_value,
|
| "metadata": {
|
| "profile_type": event_profile_type,
|
| "profile_value": event_profile_value,
|
| },
|
| }
|
| )
|
| edges.append({"id": f"{event_id}->{profile_node_id}:profile_of", "source": event_id, "target": profile_node_id, "type": "profile_of"})
|
| profile_node_ids.append(profile_node_id)
|
| source_turn_text = _runtime_source_turn_text(source_turn_hit or representative, speaker=speaker)
|
| if source_turn_text:
|
| source_turn_node_id = f"{event_id}:source_turn"
|
| add_node(
|
| {
|
| "id": source_turn_node_id,
|
| "type": "source_turn",
|
| "text": source_turn_text,
|
| "turn_index": int(getattr(source_turn_hit or representative, "turn_index", 0) or 0),
|
| "metadata": {
|
| "speaker": speaker,
|
| "dia_id": _clean_text(dict((source_turn_hit or representative).metadata or {}).get("dia_id", "")),
|
| },
|
| }
|
| )
|
| edges.append({"id": f"{event_id}->{source_turn_node_id}:supported_by_turn", "source": event_id, "target": source_turn_node_id, "type": "supported_by_turn"})
|
| source_turn_node_ids.append(source_turn_node_id)
|
| if event_target_status:
|
| status_node_id = f"{event_id}:status"
|
| add_node({"id": status_node_id, "type": "status", "text": event_target_status, "metadata": {"target_status": event_target_status}})
|
| edges.append({"id": f"{event_id}->{status_node_id}:status_of", "source": event_id, "target": status_node_id, "type": "status_of"})
|
| status_node_ids.append(status_node_id)
|
| event_paths = build_default_path_templates(
|
| event_id=event_id,
|
| speaker_node_id=speaker_node_id,
|
| time_node_ids=time_node_ids,
|
| profile_node_ids=profile_node_ids,
|
| status_node_ids=status_node_ids,
|
| source_turn_node_ids=source_turn_node_ids,
|
| )
|
| for path in event_paths:
|
| path_metadata = dict(path.get("metadata", {}) or {})
|
| path_metadata["tmcra_path_tags"] = list(typed_tunnel_metadata.get("tmcra_path_tags", []) or [])
|
| path_metadata["tmcra_tunnel_group_key"] = _clean_text(typed_tunnel_metadata.get("tmcra_tunnel_group_key", ""))
|
| path["metadata"] = path_metadata
|
| path["tmcra_path_tags"] = path_metadata["tmcra_path_tags"]
|
| paths.extend(event_paths)
|
| ordered_events.append(_runtime_event_sequence_key(session_name, event_turn_index, event_id))
|
| ordered_event_ids = [event_id for _, _, _, event_id in sorted(ordered_events)]
|
| for previous_event_id, next_event_id in zip(ordered_event_ids, ordered_event_ids[1:]):
|
| typed_edge_tags = typed_edge_tags_between(
|
| event_typed_metadata_by_id.get(previous_event_id, {}),
|
| event_typed_metadata_by_id.get(next_event_id, {}),
|
| )
|
| edges.append(
|
| {
|
| "id": f"{previous_event_id}->{next_event_id}:same_session_next",
|
| "source": previous_event_id,
|
| "target": next_event_id,
|
| "type": "same_session_next",
|
| "metadata": {
|
| "tmcra_edge_tags": typed_edge_tags,
|
| "typed_tunnel_edge": bool(typed_edge_tags),
|
| },
|
| }
|
| )
|
| for index, source_event_id in enumerate(ordered_event_ids):
|
| for target_event_id in ordered_event_ids[index + 1 : index + 12]:
|
| typed_edge_tags = typed_edge_tags_between(
|
| event_typed_metadata_by_id.get(source_event_id, {}),
|
| event_typed_metadata_by_id.get(target_event_id, {}),
|
| )
|
| if not typed_edge_tags:
|
| continue
|
| typed_tunnel_edges.append(
|
| {
|
| "id": f"{source_event_id}->{target_event_id}:typed_tunnel",
|
| "source": source_event_id,
|
| "target": target_event_id,
|
| "type": "typed_tunnel_candidate",
|
| "metadata": {
|
| "tmcra_edge_tags": typed_edge_tags,
|
| "typed_tunnel_edge": True,
|
| },
|
| }
|
| )
|
| if len(typed_tunnel_edges) >= 64:
|
| break
|
| if len(typed_tunnel_edges) >= 64:
|
| break
|
| return {
|
| "conversation_id": "runtime",
|
| "query": query,
|
| "nodes": nodes,
|
| "edges": edges,
|
| "typed_tunnel_edges": typed_tunnel_edges,
|
| "paths": paths,
|
| "grouped_hits": grouped_hits,
|
| }
|
|
|
|
|
| def _public_graph_hits(graph: SessionMemoryGraphV2) -> List[MemoryHit]:
|
| public_hits: List[MemoryHit] = []
|
| for record in graph.records_by_id.values():
|
| if record.state != "active":
|
| continue
|
| if not _clean_text(record.source_kind).startswith("public_dialog"):
|
| continue
|
| metadata = dict(record.metadata or {})
|
| public_hits.append(
|
| MemoryHit(
|
| memory_id=record.memory_id,
|
| category=record.category,
|
| value=record.value,
|
| relation=record.relation,
|
| anchors=list(record.anchor_concepts),
|
| score=max(float(record.confidence), float(record.salience), 0.01),
|
| source_kind=record.source_kind,
|
| slot_key=record.slot_key,
|
| state=record.state,
|
| turn_index=int(record.turn_index),
|
| metadata=metadata,
|
| )
|
| )
|
| public_hits.sort(key=lambda item: (int(item.turn_index), float(item.score)), reverse=True)
|
| return public_hits
|
|
|
|
|
| _AUDIT_ANCHOR_QUERY_RE = re.compile(
|
| r"(?i)\b(?:remember|recall|earlier|previous|previously|old|before|mentioned|said|quote|verbatim|original|"
|
| r"that time|last time|bring back|return to|go back to|turn\s*\d+)\b|"
|
| r"(?:\u7b2c\s*\d+\s*[\u8f6e\u6b21]|\d+\s*\u8f6e|\u4e4b\u524d|\u4ee5\u524d|\u521a\u521a|"
|
| r"\u539f\u8bdd|\u90a3\u6b21|\u65e7\u8bdd\u9898|\u56de\u5230|\u63d0\u8d77|\u8bb0\u5f97)"
|
| )
|
|
|
| _AUDIT_TURN_ANCHOR_RE = re.compile(
|
| r"(?i)\bturn\s*#?\s*(\d{1,6})\b|"
|
| r"(?:\u7b2c\s*(\d{1,6})\s*[\u8f6e\u6b21]|\b(\d{1,6})\s*\u8f6e\b)"
|
| )
|
|
|
| _AUDIT_ANCHOR_STOPWORDS = set(_HYBRID_SYMBOLIC_STOPWORDS) | {
|
| "about",
|
| "again",
|
| "back",
|
| "bring",
|
| "can",
|
| "could",
|
| "did",
|
| "discuss",
|
| "discussed",
|
| "earlier",
|
| "from",
|
| "just",
|
| "keep",
|
| "mentioned",
|
| "old",
|
| "one",
|
| "previous",
|
| "previously",
|
| "really",
|
| "recall",
|
| "remember",
|
| "return",
|
| "said",
|
| "still",
|
| "that",
|
| "the",
|
| "thing",
|
| "think",
|
| "this",
|
| "turn",
|
| "what",
|
| "when",
|
| "where",
|
| "with",
|
| "you",
|
| }
|
|
|
| _AUDIT_ANCHOR_GENERIC_MATCH_TOKENS = {
|
| "body",
|
| "coherence",
|
| "continuity",
|
| "fiction",
|
| "fray",
|
| "memories",
|
| "memory",
|
| "narrative",
|
| "ourselves",
|
| "physical",
|
| "really",
|
| "stories",
|
| "story",
|
| "tell",
|
| "trust",
|
| }
|
|
|
| _AUDIT_ANCHOR_PHRASE_TOKEN_SETS = [
|
| {"spine", "book"},
|
| {"body", "spine"},
|
| {"stories", "pages"},
|
| {"story", "pages"},
|
| {"islands", "self"},
|
| {"archipelago", "self"},
|
| {"loom", "body"},
|
| {"braid", "body"},
|
| ]
|
|
|
|
|
| def _audit_anchor_query(query: str) -> bool:
|
| return bool(_AUDIT_ANCHOR_QUERY_RE.search(_clean_text(query)))
|
|
|
|
|
| def _audit_anchor_turn_numbers(query: str) -> List[int]:
|
| numbers: List[int] = []
|
| for match in _AUDIT_TURN_ANCHOR_RE.finditer(_clean_text(query)):
|
| for group in match.groups():
|
| if not group:
|
| continue
|
| try:
|
| value = int(group)
|
| except Exception:
|
| continue
|
| if value > 0 and value not in numbers:
|
| numbers.append(value)
|
| return numbers
|
|
|
|
|
| def _hit_event_id(hit: MemoryHit) -> str:
|
| metadata = dict(hit.metadata or {})
|
| event_id = _clean_text(metadata.get("event_id", ""))
|
| if event_id:
|
| return event_id
|
| dia_id = _clean_text(metadata.get("dia_id", ""))
|
| if dia_id:
|
| return f"event::{dia_id}"
|
| if int(hit.turn_index or 0) > 0:
|
| return f"event::realchat:{int(hit.turn_index)}"
|
| return ""
|
|
|
|
|
| def _audit_anchor_hit_text(hit: MemoryHit) -> str:
|
| metadata = dict(hit.metadata or {})
|
| values = [
|
| hit.value,
|
| hit.category,
|
| hit.source_kind,
|
| hit.slot_key,
|
| metadata.get("event_text", ""),
|
| metadata.get("source_span", ""),
|
| metadata.get("raw_text", ""),
|
| ]
|
| return " ".join(_clean_text(value) for value in values if _clean_text(value))
|
|
|
|
|
| def _audit_anchor_content_tokens(text: str) -> set[str]:
|
| tokens = set()
|
| for token in _tokenize(text):
|
| norm = _normalize(token)
|
| if not norm or norm in _AUDIT_ANCHOR_STOPWORDS:
|
| continue
|
| if len(norm) < 3 and not any("\u4e00" <= ch <= "\u9fff" for ch in norm):
|
| continue
|
| tokens.add(norm)
|
| return tokens
|
|
|
|
|
| def _audit_anchor_phrase_bonus(query_tokens: set[str], hit_tokens: set[str]) -> tuple[float, List[str]]:
|
| matched_phrases: List[str] = []
|
| bonus = 0.0
|
| for phrase_tokens in _AUDIT_ANCHOR_PHRASE_TOKEN_SETS:
|
| if phrase_tokens <= query_tokens and phrase_tokens <= hit_tokens:
|
| matched_phrases.append("+".join(sorted(phrase_tokens)))
|
| bonus += 5.0 if phrase_tokens == {"spine", "book"} else 3.0
|
| return bonus, matched_phrases
|
|
|
|
|
| def _copy_hit_with_audit_anchor_boost(hit: MemoryHit, *, score: float, reason: str, matched_tokens: Sequence[str]) -> MemoryHit:
|
| metadata = dict(hit.metadata or {})
|
| metadata.update(
|
| {
|
| "audit_anchor_protected": True,
|
| "audit_anchor_reason": reason,
|
| "audit_anchor_original_score": round(float(hit.score), 6),
|
| "audit_anchor_score": round(float(score), 6),
|
| "audit_anchor_matched_tokens": list(matched_tokens)[:20],
|
| }
|
| )
|
| return MemoryHit(
|
| memory_id=hit.memory_id,
|
| category=hit.category,
|
| value=hit.value,
|
| relation=hit.relation,
|
| anchors=list(hit.anchors),
|
| score=max(float(hit.score), round(1.0 + float(score), 6)),
|
| source_kind=hit.source_kind,
|
| slot_key=hit.slot_key,
|
| state=hit.state,
|
| turn_index=int(hit.turn_index),
|
| metadata=metadata,
|
| )
|
|
|
|
|
| def _audit_anchor_protected_hits(
|
| *,
|
| query: str,
|
| final_hits: Sequence[MemoryHit],
|
| candidate_hits: Sequence[MemoryHit],
|
| metadata: Mapping[str, Any],
|
| top_k: int,
|
| ) -> Dict[str, Any]:
|
| if not _audit_anchor_query(query):
|
| return {"enabled": False, "hits": list(final_hits), "promoted_hits": [], "metadata": {"audit_anchor_enabled": False}}
|
| query_tokens = _audit_anchor_content_tokens(query)
|
| if not query_tokens:
|
| return {"enabled": True, "hits": list(final_hits), "promoted_hits": [], "metadata": {"audit_anchor_enabled": True, "audit_anchor_reason": "no_content_tokens"}}
|
|
|
| explicit_turns = _audit_anchor_turn_numbers(query)
|
| explicit_turn_window = set()
|
| for number in explicit_turns:
|
| explicit_turn_window.update({number - 1, number, number + 1})
|
|
|
| symbolic_ids = list(dict.fromkeys(str(item) for item in dict(metadata or {}).get("symbolic_recall_event_ids", []) or []))
|
| learned_ids = list(dict.fromkeys(str(item) for item in dict(metadata or {}).get("learned_recall_event_ids", []) or []))
|
| selected_ids = set(str(item) for item in dict(metadata or {}).get("selected_event_ids", []) or [])
|
| symbolic_rank = {event_id: index for index, event_id in enumerate(symbolic_ids, start=1)}
|
| learned_rank = {event_id: index for index, event_id in enumerate(learned_ids, start=1)}
|
|
|
| pool: Dict[str, MemoryHit] = {}
|
| for hit in list(final_hits) + list(candidate_hits):
|
| key = hit.memory_id or f"{_hit_event_id(hit)}::{hit.slot_key}::{hit.value[:80]}"
|
| if key and key not in pool:
|
| pool[key] = hit
|
|
|
| scored: List[tuple[float, str, List[str], MemoryHit]] = []
|
| for hit in pool.values():
|
| event_id = _hit_event_id(hit)
|
| hit_tokens = _audit_anchor_content_tokens(_audit_anchor_hit_text(hit))
|
| matched = sorted(query_tokens & hit_tokens)
|
| distinctive_matched = [token for token in matched if token not in _AUDIT_ANCHOR_GENERIC_MATCH_TOKENS]
|
| phrase_bonus, matched_phrases = _audit_anchor_phrase_bonus(query_tokens, hit_tokens)
|
| if not matched and int(hit.turn_index or 0) not in explicit_turn_window:
|
| continue
|
| score = 0.0
|
| reason_parts: List[str] = []
|
| if int(hit.turn_index or 0) in explicit_turn_window:
|
| score += 8.0
|
| reason_parts.append("explicit_turn_anchor")
|
| if matched:
|
| weighted_overlap = 0.0
|
| for token in matched:
|
| weighted_overlap += 1.8 if token not in _AUDIT_ANCHOR_GENERIC_MATCH_TOKENS else 0.35
|
| score += min(8.0, weighted_overlap)
|
| score += len(matched) / max(1.0, float(min(len(query_tokens), len(hit_tokens))))
|
| reason_parts.append("lexical_anchor_overlap")
|
| if phrase_bonus > 0:
|
| score += phrase_bonus
|
| reason_parts.append("distinctive_phrase_anchor")
|
| if event_id in symbolic_rank:
|
| score += max(0.0, 2.0 - (symbolic_rank[event_id] - 1) * 0.08)
|
| reason_parts.append("symbolic_recall_anchor")
|
| if event_id in learned_rank:
|
| score += max(0.0, 1.0 - (learned_rank[event_id] - 1) * 0.02)
|
| reason_parts.append("learned_recall_anchor")
|
| if event_id in selected_ids:
|
| score -= 0.25
|
|
|
|
|
| if not explicit_turns and len(distinctive_matched) < 1 and phrase_bonus <= 0:
|
| continue
|
| if score > 0:
|
| scored.append((score, event_id, sorted(set(matched + matched_phrases)), hit))
|
|
|
| scored.sort(key=lambda item: (item[0], -abs(int(item[3].turn_index or 0))), reverse=True)
|
| max_promoted = 2 if not explicit_turns else 3
|
| promoted: List[MemoryHit] = []
|
| promoted_event_ids: set[str] = set()
|
| for score, event_id, matched, hit in scored:
|
| if event_id in promoted_event_ids:
|
| continue
|
| if any((hit.memory_id and hit.memory_id == current.memory_id) for current in final_hits[: max(1, top_k)]):
|
| continue
|
| reason = "explicit_turn_anchor" if int(hit.turn_index or 0) in explicit_turn_window else "old_topic_anchor"
|
| promoted.append(_copy_hit_with_audit_anchor_boost(hit, score=score, reason=reason, matched_tokens=matched))
|
| promoted_event_ids.add(event_id)
|
| if len(promoted) >= max_promoted:
|
| break
|
|
|
| if not promoted:
|
| return {
|
| "enabled": True,
|
| "hits": list(final_hits),
|
| "promoted_hits": [],
|
| "metadata": {
|
| "audit_anchor_enabled": True,
|
| "audit_anchor_turn_numbers": explicit_turns,
|
| "audit_anchor_query_tokens": sorted(query_tokens)[:50],
|
| "audit_anchor_promoted_event_ids": [],
|
| },
|
| }
|
|
|
| merged: List[MemoryHit] = []
|
| seen_keys: set[str] = set()
|
| for hit in promoted + list(final_hits):
|
| key = hit.memory_id or f"{_hit_event_id(hit)}::{hit.slot_key}::{hit.value[:80]}"
|
| if key in seen_keys:
|
| continue
|
| seen_keys.add(key)
|
| merged.append(hit)
|
| if len(merged) >= max(1, int(top_k)):
|
| break
|
|
|
| return {
|
| "enabled": True,
|
| "hits": merged,
|
| "promoted_hits": promoted,
|
| "metadata": {
|
| "audit_anchor_enabled": True,
|
| "audit_anchor_turn_numbers": explicit_turns,
|
| "audit_anchor_query_tokens": sorted(query_tokens)[:50],
|
| "audit_anchor_promoted_event_ids": [_hit_event_id(hit) for hit in promoted],
|
| "audit_anchor_promoted_turns": [int(hit.turn_index) for hit in promoted],
|
| "audit_anchor_promoted_hit_count": len(promoted),
|
| },
|
| }
|
|
|
|
|
| def _learnable_graph_hits(graph: SessionMemoryGraphV2) -> List[MemoryHit]:
|
| learnable_hits: List[MemoryHit] = []
|
| for record in graph.records_by_id.values():
|
| state = _normalize(record.state)
|
| metadata = dict(record.metadata or {})
|
| is_source_grounded_evidence = (
|
| state == "evidence"
|
| and (
|
| _clean_text(record.source_kind).startswith("public_dialog")
|
| or _clean_text(metadata.get("content_variant", "")) in {"source_turn", "llm_semantic_write"}
|
| or _clean_text(metadata.get("write_path", "")) == "llm_semantic_writer_gate"
|
| )
|
| )
|
| if state not in {"active", "parallel_active"} and not is_source_grounded_evidence:
|
| continue
|
| if _clean_text(record.slot_key).startswith("noise."):
|
| continue
|
| learnable_hits.append(
|
| MemoryHit(
|
| memory_id=record.memory_id,
|
| category=record.category,
|
| value=record.value,
|
| relation=record.relation,
|
| anchors=list(record.anchor_concepts),
|
| score=max(float(record.confidence), float(record.salience), 0.01),
|
| source_kind=record.source_kind,
|
| slot_key=record.slot_key,
|
| state=record.state,
|
| turn_index=int(record.turn_index),
|
| metadata=metadata,
|
| )
|
| )
|
| learnable_hits.sort(
|
| key=lambda item: (
|
| _is_public_dialog_hit(item),
|
| int(item.turn_index),
|
| float(item.score),
|
| ),
|
| reverse=True,
|
| )
|
| return learnable_hits
|
|
|
|
|
| def _parse_structured_records(
|
| payload: Dict[str, Any] | None,
|
| *,
|
| turn_index: int,
|
| profile: TMCRAProfile | None = None,
|
| ) -> List[SessionMemoryRecordV2]:
|
| profile = profile or TMCRAProfile()
|
| records: List[SessionMemoryRecordV2] = []
|
| structured_rows: List[tuple[str, Mapping[str, Any]]] = []
|
| for raw in (payload or {}).get("replacement_memory_records", []) or []:
|
| if isinstance(raw, Mapping):
|
| structured_rows.append(("formal", raw))
|
| for raw in (payload or {}).get("suspect_memory_records", []) or []:
|
| if isinstance(raw, Mapping):
|
| structured_rows.append(("suspect", raw))
|
| for index, (buffer_state, raw) in enumerate(structured_rows):
|
| if not isinstance(raw, dict):
|
| continue
|
| category = _clean_text(raw.get("category", "memory")) or "memory"
|
| value = _clean_text(raw.get("value", ""))
|
| if not value:
|
| continue
|
| anchors = _dedupe(raw.get("anchors", []) or [], max_items=8)
|
| metadata = dict(raw.get("metadata", {}) or {})
|
| slot_key = profile.stable_slot_key(
|
| category=category,
|
| value=value,
|
| anchors=anchors,
|
| slot_key=_clean_text(raw.get("slot_key", "")) or _clean_text(raw.get("slot", "")),
|
| relation=_clean_text(raw.get("relation", "")),
|
| metadata=metadata,
|
| )
|
| metadata = {
|
| **metadata,
|
| "memory_role": _clean_text(metadata.get("memory_role", "")) or "user",
|
| "authority": _clean_text(metadata.get("authority", "")) or "source",
|
| "canonical_slot_key": _clean_text(metadata.get("canonical_slot_key", "")) or slot_key,
|
| "writeback_class": _clean_text(metadata.get("writeback_class", "")),
|
| "origin_query": _clean_text(metadata.get("origin_query", "")),
|
| "origin_answer_id": _clean_text(metadata.get("origin_answer_id", "")),
|
| "support_memory_ids": _dedupe(metadata.get("support_memory_ids", []) or []),
|
| "support_fact_refs": _dedupe(metadata.get("support_fact_refs", []) or []),
|
| "support_path_refs": _dedupe(metadata.get("support_path_refs", []) or []),
|
| "promotion_state": _clean_text(metadata.get("promotion_state", "")) or "none",
|
| "memory_buffer_state": buffer_state,
|
| }
|
| memory_id = f"{slot_key}:{turn_index}:{index}"
|
| default_source_kind = "suspect_memory" if buffer_state == "suspect" else "replacement_memory"
|
| default_state = "suspect" if buffer_state == "suspect" else ("active" if bool(raw.get("active", True)) else "historical")
|
| records.append(
|
| SessionMemoryRecordV2(
|
| memory_id=memory_id,
|
| category=category,
|
| slot_key=slot_key,
|
| value=value,
|
| relation=_clean_text(raw.get("relation", "")) or f"{category}_memory",
|
| anchor_concepts=anchors,
|
| evidence_anchors=anchors,
|
| salience=float(raw.get("salience", 0.88 if category in {"goal", "constraint"} else 0.74) or 0.74),
|
| confidence=float(raw.get("confidence", 0.82) or 0.82),
|
| source_kind=_clean_text(raw.get("source_kind", "")) or default_source_kind,
|
| turn_index=int(raw.get("turn_index", turn_index) or turn_index),
|
| state=_clean_text(raw.get("state", "")) or default_state,
|
| metadata=metadata,
|
| )
|
| )
|
| return records
|
|
|
|
|
| _WRITE_MARKERS = (
|
| "goal update",
|
| "goal seed",
|
| "constraint update",
|
| "constraint overwrite",
|
| "constraint seed",
|
| "preference update",
|
| "preference overwrite",
|
| "preference seed",
|
| "terminology:",
|
| "term seed",
|
| "term overwrite",
|
| "stage update",
|
| "stage overwrite",
|
| "path fact",
|
| "fact:",
|
| "memory update",
|
| )
|
|
|
|
|
| _OVERWRITE_MARKERS = (
|
| "overwrite",
|
| "replace",
|
| "supersede",
|
| "覆盖",
|
| "替换",
|
| "改成",
|
| "更新为",
|
| )
|
|
|
|
|
| _TOPIC_BUCKET_CUE_GROUPS: tuple[tuple[tuple[str, ...], tuple[str, ...]], ...] = (
|
| (
|
| (
|
| "食物",
|
| "饮品",
|
| "甜点",
|
| "早餐",
|
| "餐饮",
|
| "口味",
|
| "酸辣",
|
| "热食",
|
| "过敏",
|
| "芒果",
|
| "配料",
|
| "推荐",
|
| "吃",
|
| "food",
|
| "breakfast",
|
| "allergy",
|
| "mango",
|
| ),
|
| ("餐饮偏好安全", "食物偏好", "过敏约束", "口味偏好"),
|
| ),
|
| (
|
| (
|
| "界面",
|
| "首页",
|
| "首屏",
|
| "配色",
|
| "营销",
|
| "横幅",
|
| "电商",
|
| "网站",
|
| "移动端",
|
| "视觉",
|
| "工具型",
|
| "ui",
|
| "homepage",
|
| "ecommerce",
|
| "mobile",
|
| "design",
|
| ),
|
| ("界面产品设计", "电商页面", "视觉布局", "移动端体验"),
|
| ),
|
| (
|
| (
|
| "api",
|
| "writer",
|
| "延迟",
|
| "算法",
|
| "耗时",
|
| "评估",
|
| "指标",
|
| "调用",
|
| "服务",
|
| "key",
|
| "runtime",
|
| "latency",
|
| "metric",
|
| ),
|
| ("api评估运行", "算法服务", "调用指标", "writer延迟"),
|
| ),
|
| )
|
|
|
|
|
| _TOPIC_BUCKET_STOPWORDS = {
|
| "不要",
|
| "不用",
|
| "复述",
|
| "原话",
|
| "只说",
|
| "关键",
|
| "最关键",
|
| "以后",
|
| "需要",
|
| "必须",
|
| "可以",
|
| "时候",
|
| "这个",
|
| "那个",
|
| "现在",
|
| "做",
|
| "说",
|
| "的",
|
| "了",
|
| "和",
|
| "与",
|
| "to",
|
| "the",
|
| "and",
|
| "for",
|
| }
|
|
|
|
|
| def _topic_bucket_keywords(text: str, *, max_items: int = 18) -> List[str]:
|
| normalized = _normalize(text)
|
| tokens: List[str] = []
|
| for cues, anchors in _TOPIC_BUCKET_CUE_GROUPS:
|
| if any(cue and _normalize(cue) in normalized for cue in cues):
|
| tokens.extend(anchors)
|
| tokens.extend(cue for cue in cues if len(cue) >= 2)
|
| tokens.extend(re.findall(r"[\u4e00-\u9fff]{2,8}", str(text or "")))
|
| tokens.extend(_tokenize(text))
|
| cleaned = []
|
| for token in tokens:
|
| value = _clean_text(token)
|
| if not value:
|
| continue
|
| normalized_value = _normalize(value)
|
| if normalized_value in _TOPIC_BUCKET_STOPWORDS:
|
| continue
|
| if len(value) < 2:
|
| continue
|
| cleaned.append(value)
|
| return _dedupe(cleaned, max_items=max_items)
|
|
|
|
|
| def _topic_bucket_id_from_keywords(keywords: Sequence[str]) -> str:
|
| basis = "|".join(_normalize(item) for item in keywords[:8] if _clean_text(item))
|
| if not basis:
|
| basis = "general"
|
| return "topic-" + str(uuid.uuid5(uuid.NAMESPACE_URL, f"tmcra-topic-bucket:{basis}"))[:12]
|
|
|
|
|
| def _topic_bucket_label_from_keywords(keywords: Sequence[str]) -> str:
|
| visible = [
|
| _clean_text(item)
|
| for item in keywords
|
| if _clean_text(item) and not _normalize(item).startswith("topic:")
|
| ]
|
| return " / ".join(visible[:3]) if visible else "动态话题"
|
|
|
|
|
| def _topic_bucket_overlap_score(left_keywords: Sequence[str], right_keywords: Sequence[str]) -> float:
|
| left = {_normalize(item) for item in left_keywords if _clean_text(item)}
|
| right = {_normalize(item) for item in right_keywords if _clean_text(item)}
|
| if not left or not right:
|
| return 0.0
|
| overlap = left & right
|
| if not overlap:
|
| return 0.0
|
| return len(overlap) / max(1, min(len(left), len(right)))
|
|
|
|
|
| def _explicit_topic_bucket_from_payload(answer_payload: Dict[str, Any] | None) -> Dict[str, Any]:
|
| metadata = dict((answer_payload or {}).get("metadata", {}) or {})
|
| explicit = metadata.get("topic_bucket")
|
| if not isinstance(explicit, Mapping):
|
| return {}
|
| keywords = _dedupe(explicit.get("topic_keywords", explicit.get("keywords", [])) or [], max_items=24)
|
| label = _clean_text(explicit.get("topic_label", explicit.get("label", "")))
|
| if not keywords and label:
|
| keywords = _topic_bucket_keywords(label, max_items=12)
|
| bucket_id = _clean_text(explicit.get("topic_bucket_id", explicit.get("id", "")))
|
| if not bucket_id:
|
| bucket_id = _topic_bucket_id_from_keywords(keywords or [label])
|
| return {
|
| "topic_bucket_id": bucket_id,
|
| "topic_label": label or _topic_bucket_label_from_keywords(keywords),
|
| "topic_keywords": keywords,
|
| "topic_confidence": float(explicit.get("confidence", explicit.get("topic_confidence", 0.92)) or 0.92),
|
| "topic_assignment_source": "explicit_payload",
|
| }
|
|
|
|
|
| def _coerce_topic_bucket(metadata: Mapping[str, Any]) -> Dict[str, Any]:
|
| bucket_id = _clean_text(metadata.get("topic_bucket_id", ""))
|
| if not bucket_id:
|
| nested = metadata.get("topic_bucket")
|
| if isinstance(nested, Mapping):
|
| bucket_id = _clean_text(nested.get("topic_bucket_id", nested.get("id", "")))
|
| if not bucket_id:
|
| return {}
|
| keywords = _dedupe(metadata.get("topic_keywords", []) or [], max_items=32)
|
| nested = metadata.get("topic_bucket")
|
| if isinstance(nested, Mapping):
|
| keywords = _dedupe([*keywords, *(nested.get("topic_keywords", nested.get("keywords", [])) or [])], max_items=32)
|
| label = _clean_text(metadata.get("topic_label", ""))
|
| if not label and isinstance(nested, Mapping):
|
| label = _clean_text(nested.get("topic_label", nested.get("label", "")))
|
| return {
|
| "topic_bucket_id": bucket_id,
|
| "topic_label": label or _topic_bucket_label_from_keywords(keywords),
|
| "topic_keywords": keywords,
|
| }
|
|
|
|
|
| def _collect_topic_buckets(graph: SessionMemoryGraphV2) -> Dict[str, Dict[str, Any]]:
|
| buckets: Dict[str, Dict[str, Any]] = {}
|
|
|
| def merge(bucket: Mapping[str, Any], *, record_id: str = "", turn_index: int = 0) -> None:
|
| bucket_id = _clean_text(bucket.get("topic_bucket_id", bucket.get("id", "")))
|
| if not bucket_id:
|
| return
|
| current = buckets.setdefault(
|
| bucket_id,
|
| {
|
| "topic_bucket_id": bucket_id,
|
| "topic_label": _clean_text(bucket.get("topic_label", bucket.get("label", ""))),
|
| "topic_keywords": [],
|
| "record_ids": [],
|
| "last_turn_index": 0,
|
| },
|
| )
|
| current["topic_label"] = current.get("topic_label") or _clean_text(bucket.get("topic_label", bucket.get("label", "")))
|
| current["topic_keywords"] = _dedupe(
|
| [*list(current.get("topic_keywords", []) or []), *(bucket.get("topic_keywords", bucket.get("keywords", [])) or [])],
|
| max_items=32,
|
| )
|
| if record_id:
|
| current["record_ids"] = _dedupe([*list(current.get("record_ids", []) or []), record_id], max_items=200)
|
| current["last_turn_index"] = max(int(current.get("last_turn_index", 0) or 0), int(turn_index or 0))
|
|
|
| for record in getattr(graph, "records_by_id", {}).values():
|
| metadata = dict(record.metadata or {})
|
| bucket = _coerce_topic_bucket(metadata)
|
| if bucket:
|
| merge(bucket, record_id=record.memory_id, turn_index=int(record.turn_index))
|
| for turn in list(getattr(graph, "turn_log", []) or []):
|
| metadata = dict(turn.get("metadata", {}) if isinstance(turn, Mapping) else getattr(turn, "metadata", {}) or {})
|
| bucket = _coerce_topic_bucket(metadata)
|
| if bucket:
|
| merge(bucket, turn_index=int(turn.get("turn_index", 0) if isinstance(turn, Mapping) else getattr(turn, "turn_index", 0) or 0))
|
| for bucket in buckets.values():
|
| if not bucket.get("topic_label"):
|
| bucket["topic_label"] = _topic_bucket_label_from_keywords(bucket.get("topic_keywords", []) or [])
|
| return buckets
|
|
|
|
|
| def _assign_topic_bucket_for_text(
|
| graph: SessionMemoryGraphV2,
|
| text: str,
|
| *,
|
| answer_payload: Dict[str, Any] | None = None,
|
| turn_index: int = 0,
|
| create: bool = True,
|
| ) -> Dict[str, Any]:
|
| explicit = _explicit_topic_bucket_from_payload(answer_payload)
|
| if explicit:
|
| explicit["turn_index"] = int(turn_index or 0)
|
| return explicit
|
| keywords = _topic_bucket_keywords(text, max_items=24)
|
| if not keywords:
|
| keywords = _dedupe([_clean_text(text)[:40]], max_items=1)
|
| buckets = _collect_topic_buckets(graph)
|
| best_bucket: Dict[str, Any] | None = None
|
| best_score = 0.0
|
| for bucket in buckets.values():
|
| score = _topic_bucket_overlap_score(keywords, bucket.get("topic_keywords", []) or [])
|
| if score > best_score:
|
| best_bucket = bucket
|
| best_score = score
|
| if best_bucket and best_score >= 0.22:
|
| merged_keywords = _dedupe([*list(best_bucket.get("topic_keywords", []) or []), *keywords], max_items=32)
|
| return {
|
| "topic_bucket_id": best_bucket["topic_bucket_id"],
|
| "topic_label": best_bucket.get("topic_label") or _topic_bucket_label_from_keywords(merged_keywords),
|
| "topic_keywords": merged_keywords,
|
| "topic_confidence": round(min(0.97, 0.68 + best_score * 0.24), 6),
|
| "topic_assignment_source": "reused_by_overlap",
|
| "topic_match_score": round(best_score, 6),
|
| "turn_index": int(turn_index or 0),
|
| }
|
| bucket_id = _topic_bucket_id_from_keywords(keywords)
|
| return {
|
| "topic_bucket_id": bucket_id,
|
| "topic_label": _topic_bucket_label_from_keywords(keywords),
|
| "topic_keywords": keywords,
|
| "topic_confidence": 0.72 if create else 0.58,
|
| "topic_assignment_source": "created_by_dialog" if create else "query_probe",
|
| "topic_match_score": round(best_score, 6),
|
| "turn_index": int(turn_index or 0),
|
| }
|
|
|
|
|
| def _apply_topic_bucket_to_records(records: List[SessionMemoryRecordV2], topic_bucket: Mapping[str, Any]) -> None:
|
| if not records or not topic_bucket:
|
| return
|
| bucket_id = _clean_text(topic_bucket.get("topic_bucket_id", ""))
|
| if not bucket_id:
|
| return
|
| label = _clean_text(topic_bucket.get("topic_label", "")) or "动态话题"
|
| keywords = _dedupe(topic_bucket.get("topic_keywords", []) or [], max_items=32)
|
| for record in records:
|
| metadata = dict(record.metadata or {})
|
| metadata.update(
|
| {
|
| "topic_bucket_id": bucket_id,
|
| "topic_label": label,
|
| "topic_keywords": keywords,
|
| "topic_confidence": float(topic_bucket.get("topic_confidence", 0.72) or 0.72),
|
| "topic_assignment_source": _clean_text(topic_bucket.get("topic_assignment_source", "")) or "created_by_dialog",
|
| }
|
| )
|
| record.metadata = metadata
|
| record.anchor_concepts = _dedupe([*list(record.anchor_concepts or []), f"topic:{label}", *keywords[:8]], max_items=24)
|
| evidence_anchors = list(metadata.get("evidence_anchors", []) or [])
|
| metadata["evidence_anchors"] = _dedupe([*evidence_anchors, f"topic:{label}", *keywords[:8]], max_items=24)
|
|
|
|
|
| def _last_topic_turn(graph: SessionMemoryGraphV2) -> Dict[str, Any]:
|
| for turn in reversed(list(getattr(graph, "turn_log", []) or [])):
|
| metadata = dict(turn.get("metadata", {}) if isinstance(turn, Mapping) else getattr(turn, "metadata", {}) or {})
|
| bucket = _coerce_topic_bucket(metadata)
|
| if bucket:
|
| bucket["turn_index"] = int(turn.get("turn_index", 0) if isinstance(turn, Mapping) else getattr(turn, "turn_index", 0) or 0)
|
| return bucket
|
| return {}
|
|
|
|
|
| def _add_topic_bridge_edges(
|
| graph: SessionMemoryGraphV2,
|
| *,
|
| previous_topic: Mapping[str, Any],
|
| current_topic: Mapping[str, Any],
|
| current_record_ids: Sequence[str],
|
| turn_index: int,
|
| evidence: str,
|
| ) -> Dict[str, Any]:
|
| previous_id = _clean_text(previous_topic.get("topic_bucket_id", ""))
|
| current_id = _clean_text(current_topic.get("topic_bucket_id", ""))
|
| if not previous_id or not current_id or previous_id == current_id or not current_record_ids:
|
| return {"topic_bridge_edge_count": 0}
|
| buckets = _collect_topic_buckets(graph)
|
| previous_record_ids = list((buckets.get(previous_id, {}) or {}).get("record_ids", []) or [])[-4:]
|
| if not previous_record_ids:
|
| return {"topic_bridge_edge_count": 0}
|
| edge_count = 0
|
| evidence_text = _clean_text(evidence)[:240]
|
| for source_id in previous_record_ids:
|
| for target_id in list(current_record_ids)[:4]:
|
| if source_id == target_id:
|
| continue
|
| edge = SessionMemoryEdgeV2(
|
| edge_id=f"{source_id}->{target_id}:topic_bridge:{previous_id}->{current_id}",
|
| source_memory_id=source_id,
|
| target_memory_id=target_id,
|
| edge_type="topic_bridge",
|
| score=0.54,
|
| model_score=0.0,
|
| evidence_turn=int(turn_index or 0),
|
| evidence=evidence_text,
|
| metadata={
|
| "from_topic_bucket_id": previous_id,
|
| "to_topic_bucket_id": current_id,
|
| "from_topic_label": _clean_text(previous_topic.get("topic_label", "")),
|
| "to_topic_label": _clean_text(current_topic.get("topic_label", "")),
|
| "bridge_reason": "adjacent_dialog_topic_transition",
|
| },
|
| )
|
| graph._upsert_memory_edge(edge)
|
| edge_count += 1
|
| return {
|
| "topic_bridge_edge_count": edge_count,
|
| "topic_bridge_from": previous_id,
|
| "topic_bridge_to": current_id,
|
| }
|
|
|
|
|
| def _add_dialogue_tunnel_edges(
|
| graph: SessionMemoryGraphV2,
|
| *,
|
| current_topic: Mapping[str, Any],
|
| current_record_ids: Sequence[str],
|
| turn_index: int,
|
| evidence: str,
|
| ) -> Dict[str, Any]:
|
| current_id = _clean_text(current_topic.get("topic_bucket_id", ""))
|
| if not current_id or not current_record_ids:
|
| return {"dialogue_tunnel_edge_count": 0}
|
| buckets = _collect_topic_buckets(graph)
|
| edge_count = 0
|
| evidence_text = _clean_text(evidence)[:240]
|
| source_ids: List[tuple[str, str, str]] = []
|
| for bucket_id, bucket in sorted(
|
| buckets.items(),
|
| key=lambda item: int(item[1].get("last_turn_index", 0) or 0),
|
| reverse=True,
|
| ):
|
| if bucket_id == current_id:
|
| continue
|
| for source_id in list(bucket.get("record_ids", []) or [])[-2:]:
|
| if source_id not in current_record_ids:
|
| source_ids.append((bucket_id, _clean_text(bucket.get("topic_label", "")), source_id))
|
| if len(source_ids) >= 6:
|
| break
|
| for source_bucket_id, source_label, source_id in source_ids[:6]:
|
| for target_id in list(current_record_ids)[:2]:
|
| if source_id == target_id:
|
| continue
|
| edge = SessionMemoryEdgeV2(
|
| edge_id=f"{source_id}->{target_id}:dialogue_tunnel:{source_bucket_id}->{current_id}",
|
| source_memory_id=source_id,
|
| target_memory_id=target_id,
|
| edge_type="dialogue_tunnel",
|
| score=0.24,
|
| model_score=0.0,
|
| evidence_turn=int(turn_index or 0),
|
| evidence=evidence_text,
|
| metadata={
|
| "from_topic_bucket_id": source_bucket_id,
|
| "to_topic_bucket_id": current_id,
|
| "from_topic_label": source_label,
|
| "to_topic_label": _clean_text(current_topic.get("topic_label", "")),
|
| "bridge_reason": "high_resistance_dialogue_level_tunnel",
|
| },
|
| )
|
| graph._upsert_memory_edge(edge)
|
| edge_count += 1
|
| return {"dialogue_tunnel_edge_count": edge_count}
|
|
|
|
|
| def _topic_adjacent_bucket_ids(graph: SessionMemoryGraphV2, bucket_id: str) -> set[str]:
|
| adjacent: set[str] = set()
|
| if not bucket_id:
|
| return adjacent
|
| for edge in getattr(graph, "memory_edges", {}).values():
|
| if _normalize(edge.edge_type) != "topic_bridge":
|
| continue
|
| metadata = dict(edge.metadata or {})
|
| left = _clean_text(metadata.get("from_topic_bucket_id", ""))
|
| right = _clean_text(metadata.get("to_topic_bucket_id", ""))
|
| if left == bucket_id and right:
|
| adjacent.add(right)
|
| if right == bucket_id and left:
|
| adjacent.add(left)
|
| return adjacent
|
|
|
|
|
| def _dialogue_tunnel_bucket_ids(graph: SessionMemoryGraphV2, bucket_id: str) -> set[str]:
|
| adjacent: set[str] = set()
|
| if not bucket_id:
|
| return adjacent
|
| for edge in getattr(graph, "memory_edges", {}).values():
|
| if _normalize(edge.edge_type) != "dialogue_tunnel":
|
| continue
|
| metadata = dict(edge.metadata or {})
|
| left = _clean_text(metadata.get("from_topic_bucket_id", ""))
|
| right = _clean_text(metadata.get("to_topic_bucket_id", ""))
|
| if left == bucket_id and right:
|
| adjacent.add(right)
|
| if right == bucket_id and left:
|
| adjacent.add(left)
|
| return adjacent
|
|
|
|
|
| def _topic_bridge_requested(query: str) -> bool:
|
| text = _normalize(query)
|
| if not text:
|
| return False
|
| bridge_markers = (
|
| "关联",
|
| "联系",
|
| "链条",
|
| "脉络",
|
| "延展",
|
| "深入",
|
| "对比",
|
| "整合",
|
| "整体",
|
| "上下文",
|
| "刚才",
|
| "之前",
|
| "上面",
|
| "跨话题",
|
| "隧穿",
|
| "related",
|
| "connect",
|
| "compare",
|
| "context",
|
| "chain",
|
| )
|
| return any(marker in text for marker in bridge_markers)
|
|
|
|
|
| def _dialogue_tunnel_requested(query: str) -> bool:
|
| text = _normalize(query)
|
| if not text:
|
| return False
|
| markers = (
|
| "跨话题",
|
| "跨对话",
|
| "不同话题",
|
| "不同对话",
|
| "历史对话",
|
| "长期脉络",
|
| "整体脉络",
|
| "所有相关",
|
| "全局",
|
| "全局记忆",
|
| "远一点",
|
| "更深",
|
| "深层关联",
|
| "对话级隧穿",
|
| "记忆隧穿",
|
| "cross topic",
|
| "cross-topic",
|
| "cross dialogue",
|
| "cross-session",
|
| "global context",
|
| "long range",
|
| )
|
| return any(marker in text for marker in markers)
|
|
|
|
|
| def _topic_bucket_record_to_hit(
|
| record: SessionMemoryRecordV2,
|
| *,
|
| query_topic: Mapping[str, Any],
|
| rank: int,
|
| rescue_kind: str = "topic_bucket",
|
| ) -> MemoryHit:
|
| metadata = dict(record.metadata or {})
|
| dialogue_tunnel = _normalize(rescue_kind) == "dialogue_tunnel"
|
| metadata.update(
|
| {
|
| "topic_bucket_rescue": not dialogue_tunnel,
|
| "dialogue_tunnel_rescue": dialogue_tunnel,
|
| "topic_bucket_rescue_rank": int(rank),
|
| "topic_bucket_query_id": _clean_text(query_topic.get("topic_bucket_id", "")),
|
| "topic_bucket_query_label": _clean_text(query_topic.get("topic_label", "")),
|
| "topic_bucket_same": not dialogue_tunnel,
|
| "topic_bucket_bridge": False,
|
| "topic_bucket_bridge_allowed": False,
|
| "topic_bucket_dialogue_tunnel_allowed": dialogue_tunnel,
|
| "topic_bucket_overlap": 0.0 if dialogue_tunnel else 1.0,
|
| }
|
| )
|
| category = _normalize(record.category)
|
| value_text = _normalize(record.value)
|
| hardish = (
|
| category == "constraint"
|
| or _normalize(metadata.get("memory_type", "")) == "hard_constraint"
|
| or _normalize(metadata.get("durability", "")) == "hard"
|
| or _normalize(metadata.get("conflict_policy", "")) == "must_preserve"
|
| or any(marker in value_text for marker in ("过敏", "必须", "避开", "禁止", "不能", "must", "avoid", "allergy"))
|
| )
|
| base_score = max(float(record.confidence), float(record.salience), 0.62)
|
| if dialogue_tunnel:
|
| if hardish:
|
| base_score += 0.42
|
| metadata.setdefault("memory_type", "hard_constraint")
|
| metadata.setdefault("durability", "hard")
|
| metadata.setdefault("conflict_policy", "must_preserve")
|
| elif category == "preference":
|
| base_score += 0.22
|
| metadata.setdefault("memory_type", "durable_preference")
|
| metadata.setdefault("durability", "long_term")
|
| else:
|
| base_score += 0.12
|
| metadata["dialogue_tunnel_resistance"] = "high"
|
| elif hardish:
|
| base_score += 1.35
|
| metadata.setdefault("memory_type", "hard_constraint")
|
| metadata.setdefault("durability", "hard")
|
| metadata.setdefault("conflict_policy", "must_preserve")
|
| elif category == "preference":
|
| base_score += 0.72
|
| metadata.setdefault("memory_type", "durable_preference")
|
| metadata.setdefault("durability", "long_term")
|
| else:
|
| base_score += 0.38
|
| return MemoryHit(
|
| memory_id=record.memory_id,
|
| category=record.category,
|
| value=record.value,
|
| relation=record.relation,
|
| anchors=list(record.anchor_concepts),
|
| score=base_score,
|
| source_kind=record.source_kind,
|
| slot_key=record.slot_key,
|
| state=record.state,
|
| turn_index=int(record.turn_index),
|
| metadata=metadata,
|
| )
|
|
|
|
|
| def _profile_query_rescue_hits(
|
| graph: SessionMemoryGraphV2,
|
| query: str,
|
| *,
|
| top_k: int,
|
| ) -> List[MemoryHit]:
|
| query_raw_tokens = set(_path_utility_tokens(query))
|
| query_tokens = _profile_query_expanded_tokens(query)
|
| intent = infer_profile_query_intent(query)
|
| if not bool(intent.get("enabled")):
|
| return []
|
| rescued: List[tuple[float, MemoryHit]] = []
|
| for record in getattr(graph, "records_by_id", {}).values():
|
| metadata = dict(record.metadata or {})
|
| if record.state != "active":
|
| continue
|
| if not is_profile_layer_record(
|
| category=record.category,
|
| source_kind=record.source_kind,
|
| semantic_slot=metadata.get("semantic_slot", ""),
|
| metadata=metadata,
|
| ):
|
| continue
|
| delta, reason = profile_query_score_delta(
|
| query=query,
|
| query_tokens=query_tokens,
|
| category=record.category,
|
| source_kind=record.source_kind,
|
| semantic_slot=metadata.get("semantic_slot", ""),
|
| value=record.value,
|
| anchors=record.anchor_concepts,
|
| metadata=metadata,
|
| )
|
| if delta <= 0:
|
| continue
|
| match_score, overlap_tokens, raw_overlap_tokens = _profile_hit_match_score(
|
| query_raw_tokens,
|
| query_tokens,
|
| MemoryHit(
|
| memory_id=record.memory_id,
|
| category=record.category,
|
| value=record.value,
|
| relation=record.relation,
|
| anchors=list(record.anchor_concepts),
|
| score=max(float(record.confidence), float(record.salience), 0.01),
|
| source_kind=record.source_kind,
|
| slot_key=record.slot_key,
|
| state=record.state,
|
| turn_index=int(record.turn_index),
|
| metadata=metadata,
|
| ),
|
| )
|
| if match_score <= 0.0:
|
| continue
|
| if match_score < 0.34 and not raw_overlap_tokens:
|
| continue
|
| metadata.update(
|
| {
|
| "profile_query_rescue": True,
|
| "profile_query_rescue_reason": reason or "profile_route",
|
| "profile_query_match_score": round(match_score, 6),
|
| "profile_query_overlap_tokens": list(overlap_tokens),
|
| "profile_query_raw_overlap_tokens": list(raw_overlap_tokens),
|
| "topic_bucket_profile_route_preserved": True,
|
| "match_reason": ",".join(_dedupe([metadata.get("match_reason", ""), reason or "profile_route"], max_items=4)),
|
| }
|
| )
|
| hit = MemoryHit(
|
| memory_id=record.memory_id,
|
| category=record.category,
|
| value=record.value,
|
| relation=record.relation,
|
| anchors=list(record.anchor_concepts),
|
| score=max(float(record.confidence), float(record.salience), 0.62) + float(delta) + float(match_score),
|
| source_kind=record.source_kind,
|
| slot_key=record.slot_key,
|
| state=record.state,
|
| turn_index=int(record.turn_index),
|
| metadata=metadata,
|
| )
|
| rescued.append((match_score, hit))
|
| rescued.sort(key=lambda item: (float(item[0]), float(item[1].score), int(item[1].turn_index)), reverse=True)
|
| return [
|
| hit
|
| for _, hit in rescued[: max(1, min(24, int(top_k or 1) * 3))]
|
| ]
|
|
|
|
|
| def _memory_hit_from_record(record: SessionMemoryRecordV2, *, score: float | None = None, metadata: Mapping[str, Any] | None = None) -> MemoryHit:
|
| record_metadata = {**dict(record.metadata or {}), **dict(metadata or {})}
|
| return MemoryHit(
|
| memory_id=record.memory_id,
|
| category=record.category,
|
| value=record.value,
|
| relation=record.relation,
|
| anchors=list(record.anchor_concepts),
|
| score=max(float(record.confidence), float(record.salience), 0.01) if score is None else float(score),
|
| source_kind=record.source_kind,
|
| slot_key=record.slot_key,
|
| state=record.state,
|
| turn_index=int(record.turn_index),
|
| metadata=record_metadata,
|
| )
|
|
|
|
|
| _FACET_NUMERIC_QUERY_TOKENS = {
|
| "amount",
|
| "count",
|
| "counts",
|
| "duration",
|
| "durations",
|
| "many",
|
| "much",
|
| "number",
|
| "quantity",
|
| "sum",
|
| "total",
|
| "totals",
|
| "weeks",
|
| "week",
|
| "hours",
|
| "hour",
|
| "dollars",
|
| "dollar",
|
| "tenants",
|
| "tickets",
|
| }
|
| _FACET_TEMPORAL_QUERY_TOKENS = {
|
| "after",
|
| "before",
|
| "date",
|
| "deadline",
|
| "end",
|
| "finish",
|
| "finished",
|
| "start",
|
| "started",
|
| "time",
|
| "when",
|
| }
|
|
|
|
|
| def _facet_query_pack_hits(
|
| graph: SessionMemoryGraphV2,
|
| query: str,
|
| final_hits: Sequence[MemoryHit],
|
| *,
|
| top_k: int,
|
| ) -> Dict[str, Any]:
|
| query_tokens = set(_path_utility_tokens(query))
|
| if not query_tokens:
|
| return {"hits": list(final_hits), "metadata": {"facet_query_pack_enabled": False, "facet_query_pack_reason": "empty_query_tokens"}}
|
| numeric_query = bool(query_tokens & _FACET_NUMERIC_QUERY_TOKENS)
|
| temporal_query = bool(query_tokens & _FACET_TEMPORAL_QUERY_TOKENS)
|
| if not numeric_query and not temporal_query and not any("facet" in _normalize(token) for token in query_tokens):
|
| return {"hits": list(final_hits), "metadata": {"facet_query_pack_enabled": False, "facet_query_pack_reason": "no_facet_intent"}}
|
|
|
| candidate_rows: List[tuple[float, SessionMemoryRecordV2, SessionMemoryRecordV2 | None, Dict[str, Any]]] = []
|
| for record in getattr(graph, "records_by_id", {}).values():
|
| metadata = dict(record.metadata or {})
|
| if _normalize(metadata.get("content_variant", "")) != "event_facet_write":
|
| continue
|
| if record.state not in {"active", "parallel_active", "evidence"}:
|
| continue
|
| facet_type = _normalize(metadata.get("facet_type", ""))
|
| parent_slot_key = _clean_text(metadata.get("facet_parent_slot_key", ""))
|
| parent = next(
|
| (
|
| candidate
|
| for candidate in getattr(graph, "records_by_id", {}).values()
|
| if _clean_text(candidate.slot_key).lower() == parent_slot_key.lower()
|
| ),
|
| None,
|
| )
|
| parent_text = " ".join(
|
| [
|
| _clean_text(parent.value if parent else ""),
|
| " ".join(parent.anchor_concepts if parent else []),
|
| _clean_text(dict(parent.metadata or {}).get("source_span", "") if parent else ""),
|
| ]
|
| )
|
| facet_text = " ".join(
|
| [
|
| record.value,
|
| " ".join(record.anchor_concepts or []),
|
| _clean_text(metadata.get("facet_type", "")),
|
| _clean_text(metadata.get("facet_role", "")),
|
| _clean_text(metadata.get("facet_source_span", "")),
|
| parent_text,
|
| ]
|
| )
|
| facet_tokens = set(_path_utility_tokens(facet_text))
|
| parent_tokens = set(_path_utility_tokens(parent_text))
|
| overlap_tokens = query_tokens & facet_tokens
|
| parent_overlap_tokens = query_tokens & parent_tokens
|
| unit_overlap = bool(query_tokens & set(_path_utility_tokens(record.value)))
|
| score = 0.0
|
| if overlap_tokens:
|
| score += min(0.72, len(overlap_tokens) / max(1.0, len(query_tokens)) * 1.2)
|
| if parent_overlap_tokens:
|
| score += min(0.72, len(parent_overlap_tokens) / max(1.0, len(query_tokens)) * 1.35)
|
| if numeric_query and facet_type == "numeric":
|
| score += 0.42
|
| if unit_overlap:
|
| score += 0.42
|
| if temporal_query and facet_type == "temporal":
|
| score += 0.32
|
| if facet_type == "entity" and parent_overlap_tokens:
|
| score += 0.26
|
| if parent is not None and _normalize(dict(parent.metadata or {}).get("content_variant", "")) == "llm_semantic_write":
|
| score += 0.08
|
| if score < 0.58:
|
| continue
|
| candidate_rows.append(
|
| (
|
| round(min(2.75, 1.18 + score), 6),
|
| record,
|
| parent,
|
| {
|
| "facet_query_pack_overlap_tokens": sorted(overlap_tokens)[:12],
|
| "facet_query_pack_parent_overlap_tokens": sorted(parent_overlap_tokens)[:12],
|
| "facet_query_pack_unit_overlap": bool(unit_overlap),
|
| "facet_query_pack_score": round(score, 6),
|
| },
|
| )
|
| )
|
|
|
| if not candidate_rows:
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {
|
| "facet_query_pack_enabled": True,
|
| "facet_query_pack_inserted_hit_count": 0,
|
| "facet_query_pack_candidate_count": 0,
|
| },
|
| }
|
|
|
| candidate_rows.sort(key=lambda item: (float(item[0]), int(item[1].turn_index)), reverse=True)
|
| selected = candidate_rows[: max(4, min(18, int(top_k or 1) * 2))]
|
| packed_hits: List[MemoryHit] = []
|
| for score, facet_record, parent, extra_metadata in selected:
|
| facet_metadata = {
|
| **extra_metadata,
|
| "facet_query_pack": True,
|
| "evidence_snippet_role": "facet_query_attribute",
|
| }
|
| packed_hits.append(_memory_hit_from_record(facet_record, score=score, metadata=facet_metadata))
|
| if parent is not None:
|
| packed_hits.append(
|
| _memory_hit_from_record(
|
| parent,
|
| score=max(1.12, score - 0.04),
|
| metadata={
|
| **extra_metadata,
|
| "facet_query_pack": True,
|
| "evidence_snippet_role": "facet_parent_event",
|
| "facet_query_pack_child_id": facet_record.memory_id,
|
| },
|
| )
|
| )
|
|
|
| merged: List[MemoryHit] = []
|
| seen_ids: set[str] = set()
|
| for hit in [*packed_hits, *list(final_hits)]:
|
| if hit.memory_id and hit.memory_id in seen_ids:
|
| continue
|
| if hit.memory_id:
|
| seen_ids.add(hit.memory_id)
|
| merged.append(hit)
|
| return {
|
| "hits": merged,
|
| "metadata": {
|
| "facet_query_pack_enabled": True,
|
| "facet_query_pack_candidate_count": len(candidate_rows),
|
| "facet_query_pack_inserted_hit_count": len(packed_hits),
|
| "facet_query_pack_numeric_query": bool(numeric_query),
|
| "facet_query_pack_temporal_query": bool(temporal_query),
|
| },
|
| }
|
|
|
|
|
| _UNIT_COVERAGE_COUNT_TOKENS = {
|
| "amount",
|
| "amounts",
|
| "count",
|
| "counts",
|
| "cost",
|
| "costs",
|
| "dollar",
|
| "dollars",
|
| "how",
|
| "many",
|
| "much",
|
| "number",
|
| "minimum",
|
| "maximum",
|
| "paid",
|
| "percent",
|
| "percentage",
|
| "price",
|
| "prices",
|
| "sale",
|
| "sales",
|
| "sell",
|
| "sold",
|
| "total",
|
| "totals",
|
| "sum",
|
| "value",
|
| "valued",
|
| "values",
|
| "worth",
|
| "items",
|
| "projects",
|
| "events",
|
| "things",
|
| }
|
| _UNIT_COVERAGE_TEMPORAL_TOKENS = {
|
| "ago",
|
| "after",
|
| "before",
|
| "between",
|
| "consecutive",
|
| "date",
|
| "days",
|
| "day",
|
| "first",
|
| "last",
|
| "months",
|
| "month",
|
| "order",
|
| "passed",
|
| "since",
|
| "weeks",
|
| "week",
|
| }
|
| _MULTI_UNIT_CHAIN_TEMPORAL_COMPARISON_TOKENS = {
|
| "after",
|
| "before",
|
| "between",
|
| "consecutive",
|
| "earlier",
|
| "first",
|
| "later",
|
| "last",
|
| "order",
|
| "since",
|
| }
|
| _UNIT_COVERAGE_QUERY_DROP_TOKENS = {
|
| "date",
|
| "fri",
|
| "friday",
|
| "mon",
|
| "monday",
|
| "question",
|
| "sat",
|
| "saturday",
|
| "sun",
|
| "sunday",
|
| "thu",
|
| "thursday",
|
| "tue",
|
| "tuesday",
|
| "wed",
|
| "wednesday",
|
| }
|
| _MULTI_UNIT_CHAIN_DISABLED_MODES = {"", "off", "disabled", "none", "false", "0"}
|
| _MULTI_UNIT_CHAIN_COUNT_TOKENS = {
|
| *_UNIT_COVERAGE_COUNT_TOKENS,
|
| "which",
|
| "each",
|
| "all",
|
| "both",
|
| }
|
| _MULTI_UNIT_CHAIN_FOCUS_DROP_TOKENS = {
|
| *_MULTI_UNIT_CHAIN_COUNT_TOKENS,
|
| *_UNIT_COVERAGE_TEMPORAL_TOKENS,
|
| "i",
|
| "me",
|
| "my",
|
| "mine",
|
| "am",
|
| "is",
|
| "are",
|
| "was",
|
| "were",
|
| "need",
|
| "needs",
|
| "needed",
|
| "currently",
|
| "did",
|
| "does",
|
| "fri",
|
| "friday",
|
| "mon",
|
| "monday",
|
| "or",
|
| "question",
|
| "sat",
|
| "saturday",
|
| "sun",
|
| "sunday",
|
| "thu",
|
| "thursday",
|
| "tue",
|
| "tuesday",
|
| "wed",
|
| "wednesday",
|
| "what",
|
| "which",
|
| "who",
|
| }
|
| _MULTI_UNIT_CHAIN_FACET_TYPES = {"action", "entity", "numeric", "role", "evidence_role", "state", "temporal"}
|
| _MULTI_UNIT_CHAIN_UNIT_KINDS = {
|
| "action_unit",
|
| "target_entity",
|
| "numeric_quantity",
|
| "leadership_role",
|
| "participation_role",
|
| "state_status",
|
| "temporal_anchor",
|
| "evidence_role",
|
| "profile_shadow_unit",
|
| }
|
| _MULTI_UNIT_CHAIN_NUMERIC_VALUE_TOKENS = {
|
| "amount",
|
| "appraisal",
|
| "appraised",
|
| "cost",
|
| "costs",
|
| "dollar",
|
| "dollars",
|
| "minimum",
|
| "paid",
|
| "price",
|
| "prices",
|
| "sale",
|
| "sell",
|
| "sold",
|
| "total",
|
| "value",
|
| "valued",
|
| "values",
|
| "worth",
|
| }
|
|
|
|
|
| def _multi_unit_chain_numeric_signal(
|
| unit_kind: str,
|
| facet_type: str,
|
| text: str,
|
| tokens: set[str],
|
| ) -> float:
|
| compact_text = _clean_text(text)
|
| date_like_only = bool(
|
| re.fullmatch(
|
| r"(?:\d{4}[/-]\d{1,2}(?:[/-]\d{1,2})?|\d{1,2}[/-]\d{1,2}(?:[/-]\d{2,4})?|\d{4}/\d{2})",
|
| compact_text,
|
| )
|
| )
|
| if date_like_only:
|
| return 0.0
|
| signal = 0.0
|
| if unit_kind == "numeric_quantity" or facet_type == "numeric":
|
| signal += 0.55
|
| if tokens & _MULTI_UNIT_CHAIN_NUMERIC_VALUE_TOKENS:
|
| signal += 0.28
|
| if re.search(r"(?:[$€£¥]\s*\d|\b\d+(?:,\d{3})*(?:\.\d+)?\s*(?:dollars?|usd|bucks?|yuan|rmb)\b)", text, flags=re.IGNORECASE):
|
| signal += 0.72
|
| elif re.search(r"\b\d+(?:,\d{3})*(?:\.\d+)?\s*(?:comments?|items?|pieces?|kits?|projects?|doctors?|weddings?|hours?|weeks?|days?|months?|years?|miles?)\b", text, flags=re.IGNORECASE):
|
| signal += 0.42
|
| return min(1.1, signal)
|
|
|
|
|
| def _multi_unit_chain_date_like_numeric(text: str) -> bool:
|
| compact_text = _clean_text(text)
|
| return bool(
|
| re.fullmatch(
|
| r"(?:\d{4}[/-]\d{1,2}(?:[/-]\d{1,2})?|\d{1,2}[/-]\d{1,2}(?:[/-]\d{2,4})?|\d{4}/\d{2})",
|
| compact_text,
|
| )
|
| )
|
|
|
|
|
| def _multi_unit_chain_temporal_anchor_signal(text: str, tokens: set[str]) -> float:
|
| signal = 0.0
|
| if tokens & _UNIT_COVERAGE_TEMPORAL_TOKENS:
|
| signal += 0.22
|
| if re.search(
|
| r"\b(?:about|around|roughly|few|several|couple|last|previous|earlier)\s+"
|
| r"(?:a\s+)?(?:day|days|week|weeks|month|months|year|years)\s+ago\b|"
|
| r"\b(?:yesterday|today|tomorrow|last\s+week|last\s+month|a\s+few\s+months\s+ago)\b",
|
| text,
|
| flags=re.IGNORECASE,
|
| ):
|
| signal += 0.72
|
| return min(1.0, signal)
|
|
|
|
|
| _PROFILE_SHADOW_EVENTLIKE_SLOT_HINTS = {
|
| "action",
|
| "appointment",
|
| "constraint",
|
| "deadline",
|
| "exchange",
|
| "obligation",
|
| "pickup",
|
| "plan",
|
| "preference",
|
| "return",
|
| "status",
|
| "task",
|
| }
|
| _PROFILE_SHADOW_EVENTLIKE_TEXT_HINTS = {
|
| "bought",
|
| "buy",
|
| "completed",
|
| "did",
|
| "exchange",
|
| "exchanged",
|
| "finish",
|
| "finished",
|
| "got",
|
| "have to",
|
| "need",
|
| "needed",
|
| "needs",
|
| "paid",
|
| "pick",
|
| "picked",
|
| "return",
|
| "returned",
|
| "should",
|
| "still",
|
| "took",
|
| "went",
|
| }
|
|
|
|
|
| def _profile_shadow_eventlike_record(record: SessionMemoryRecordV2, metadata: Mapping[str, Any]) -> bool:
|
| if _normalize(metadata.get("content_variant", "")) != "profile_shadow_from_writer":
|
| return False
|
| if record.state not in {"active", "parallel_active", "evidence"}:
|
| return False
|
| slot_text = " ".join(
|
| [
|
| _clean_text(record.category),
|
| _clean_text(record.relation),
|
| _clean_text(record.slot_key),
|
| _clean_text(metadata.get("semantic_slot", "")),
|
| _clean_text(metadata.get("profile_type", "")),
|
| ]
|
| ).lower()
|
| value_text = " ".join(
|
| [
|
| _clean_text(record.value),
|
| _clean_text(metadata.get("source_span", "")),
|
| _clean_text(metadata.get("raw_text", "")),
|
| ]
|
| ).lower()
|
| return bool(
|
| any(hint in slot_text for hint in _PROFILE_SHADOW_EVENTLIKE_SLOT_HINTS)
|
| or any(hint in value_text for hint in _PROFILE_SHADOW_EVENTLIKE_TEXT_HINTS)
|
| )
|
|
|
|
|
| def _profile_shadow_unit_text(record: SessionMemoryRecordV2, metadata: Mapping[str, Any]) -> str:
|
| return " ".join(
|
| [
|
| _clean_text(record.value),
|
| _clean_text(record.category),
|
| _clean_text(record.relation),
|
| _clean_text(record.slot_key),
|
| _clean_text(metadata.get("semantic_slot", "")),
|
| _clean_text(metadata.get("profile_type", "")),
|
| _clean_text(metadata.get("profile_domain", "")),
|
| _clean_text(metadata.get("source_span", "")),
|
| _clean_text(metadata.get("raw_text", "")),
|
| ]
|
| )
|
|
|
|
|
| def _unit_coverage_pack_hits(
|
| graph: SessionMemoryGraphV2,
|
| query: str,
|
| final_hits: Sequence[MemoryHit],
|
| *,
|
| top_k: int,
|
| ) -> Dict[str, Any]:
|
| raw_query_tokens = set(_path_utility_tokens(query))
|
| query_tokens = {token for token in raw_query_tokens if token not in _UNIT_COVERAGE_QUERY_DROP_TOKENS}
|
| if not query_tokens:
|
| return {"hits": list(final_hits), "metadata": {"unit_coverage_pack_enabled": False, "unit_coverage_reason": "empty_query"}}
|
| count_intent = bool(query_tokens & _UNIT_COVERAGE_COUNT_TOKENS)
|
| percentage_intent = bool(query_tokens & {"percent", "percentage"})
|
| temporal_intent = bool(raw_query_tokens & _MULTI_UNIT_CHAIN_TEMPORAL_COMPARISON_TOKENS)
|
| direct_unit_intent = not count_intent and not temporal_intent and len(query_tokens) >= 2
|
| if not count_intent and not temporal_intent and not direct_unit_intent:
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {"unit_coverage_pack_enabled": False, "unit_coverage_reason": "no_unit_intent"},
|
| }
|
|
|
| records = list(getattr(graph, "records_by_id", {}).values())
|
| parent_by_slot = {_clean_text(record.slot_key).lower(): record for record in records}
|
| candidates: List[tuple[float, SessionMemoryRecordV2, SessionMemoryRecordV2 | None, Dict[str, Any]]] = []
|
| for record in records:
|
| metadata = dict(record.metadata or {})
|
| if record.state not in {"active", "parallel_active", "evidence"}:
|
| continue
|
| profile_shadow_unit = _profile_shadow_eventlike_record(record, metadata)
|
| if profile_shadow_unit:
|
| unit_kind = "profile_shadow_unit"
|
| facet_type = "action"
|
| parent_slot = _clean_text(record.slot_key).lower()
|
| parent = None
|
| parent_text = ""
|
| unit_text = _profile_shadow_unit_text(record, metadata)
|
| else:
|
| if _normalize(metadata.get("content_variant", "")) != "event_facet_write":
|
| continue
|
| if _normalize(metadata.get("facet_layer_version", "")) not in {"event_unit_v1", "event_facet_v1"}:
|
| continue
|
| unit_kind = _normalize(metadata.get("unit_kind", ""))
|
| facet_type = _normalize(metadata.get("facet_type", ""))
|
| parent_slot = _clean_text(metadata.get("facet_parent_slot_key", "")).lower()
|
| parent = parent_by_slot.get(parent_slot)
|
| parent_text = " ".join(
|
| [
|
| _clean_text(parent.value if parent else ""),
|
| _clean_text(dict(parent.metadata or {}).get("source_span", "") if parent else ""),
|
| " ".join(parent.anchor_concepts if parent else []),
|
| ]
|
| )
|
| unit_text = " ".join(
|
| [
|
| _clean_text(record.value),
|
| _clean_text(metadata.get("facet_role", "")),
|
| _clean_text(metadata.get("unit_kind", "")),
|
| _clean_text(metadata.get("action", "")),
|
| _clean_text(metadata.get("target", "")),
|
| _clean_text(metadata.get("quantity", "")),
|
| _clean_text(metadata.get("unit", "")),
|
| _clean_text(metadata.get("normalized_time", "")),
|
| _clean_text(metadata.get("status", "")),
|
| _clean_text(metadata.get("facet_source_span", "")),
|
| parent_text,
|
| ]
|
| )
|
| unit_tokens = set(_path_utility_tokens(unit_text))
|
| semantic_event_unit = _normalize(record.memory_id).startswith("tmcra.event.") or _normalize(parent_slot).startswith("tmcra.event.")
|
| overlap = query_tokens & unit_tokens
|
| score = 0.0
|
| if overlap:
|
| score += min(0.92, len(overlap) / max(1.0, len(query_tokens)) * 1.55)
|
| if count_intent and facet_type in {"action", "entity", "numeric", "role", "evidence_role"}:
|
| score += 0.42
|
| if temporal_intent and facet_type in {"temporal", "action", "role", "entity"}:
|
| score += 0.38
|
| if direct_unit_intent and facet_type in {"action", "entity", "state", "numeric"}:
|
| score += 0.30
|
| if direct_unit_intent and len(overlap) >= 2:
|
| score += 0.36
|
| if unit_kind in {"action_unit", "target_entity", "leadership_role", "participation_role"}:
|
| score += 0.16
|
| if unit_kind in {"numeric_quantity", "temporal_anchor", "state_status"}:
|
| score += 0.12
|
| if profile_shadow_unit:
|
| score += 0.28
|
| if count_intent and facet_type in {"action", "entity"}:
|
| score += 0.24
|
| semantic_event_priority = bool(semantic_event_unit and (count_intent or percentage_intent))
|
| if semantic_event_priority:
|
| score += 0.46
|
| if percentage_intent and (
|
| unit_kind == "numeric_quantity"
|
| or facet_type == "numeric"
|
| or re.search(r"\b\d+(?:\.\d+)?\s*%|\b\d+(?:\.\d+)?\s*percent\b", unit_text, flags=re.IGNORECASE)
|
| ):
|
| score += 0.72
|
| if parent is not None:
|
| score += 0.08
|
| if score < 0.55:
|
| continue
|
| candidates.append(
|
| (
|
| round(min(3.2, 1.28 + score), 6),
|
| record,
|
| parent,
|
| {
|
| "unit_coverage_overlap_tokens": sorted(overlap)[:14],
|
| "unit_coverage_score": round(score, 6),
|
| "unit_kind": unit_kind,
|
| "facet_type": facet_type,
|
| "unit_coverage_count_intent": bool(count_intent),
|
| "unit_coverage_percentage_intent": bool(percentage_intent),
|
| "unit_coverage_temporal_intent": bool(temporal_intent),
|
| "unit_coverage_direct_unit_intent": bool(direct_unit_intent),
|
| "unit_coverage_profile_shadow_unit": bool(profile_shadow_unit),
|
| "unit_coverage_semantic_event_unit": bool(semantic_event_unit),
|
| "unit_coverage_semantic_event_priority": bool(semantic_event_priority),
|
| },
|
| )
|
| )
|
|
|
| if not candidates:
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {
|
| "unit_coverage_pack_enabled": True,
|
| "unit_coverage_candidate_count": 0,
|
| "unit_coverage_inserted_hit_count": 0,
|
| "unit_coverage_count_intent": bool(count_intent),
|
| "unit_coverage_percentage_intent": bool(percentage_intent),
|
| "unit_coverage_temporal_intent": bool(temporal_intent),
|
| "unit_coverage_direct_unit_intent": bool(direct_unit_intent),
|
| },
|
| }
|
| candidates.sort(
|
| key=lambda item: (
|
| 1 if bool(item[3].get("unit_coverage_semantic_event_priority", False)) else 0,
|
| 1
|
| if bool(item[3].get("unit_coverage_percentage_intent"))
|
| and (
|
| _normalize(item[3].get("unit_kind", "")) == "numeric_quantity"
|
| or _normalize(item[3].get("facet_type", "")) == "numeric"
|
| )
|
| else 0,
|
| float(item[0]),
|
| int(item[1].turn_index),
|
| ),
|
| reverse=True,
|
| )
|
| selected: List[tuple[float, SessionMemoryRecordV2, SessionMemoryRecordV2 | None, Dict[str, Any]]] = []
|
| seen_unit_values: set[str] = set()
|
| try:
|
| max_selected_units = max(1, int(os.getenv("TMCRA_UNIT_COVERAGE_PACK_MAX_UNITS", "6") or 6))
|
| except (TypeError, ValueError):
|
| max_selected_units = 6
|
| for item in candidates:
|
| _, record, parent, metadata = item
|
| key = "|".join(
|
| [
|
| _normalize(metadata.get("unit_kind", "")),
|
| _normalize(record.value),
|
| _normalize(dict(record.metadata or {}).get("facet_parent_slot_key", "")),
|
| ]
|
| )
|
| if key in seen_unit_values:
|
| continue
|
| seen_unit_values.add(key)
|
| selected.append(item)
|
| if len(selected) >= max_selected_units:
|
| break
|
| if count_intent:
|
| try:
|
| max_profile_shadow_units = max(0, int(os.getenv("TMCRA_UNIT_COVERAGE_PROFILE_SHADOW_MAX_UNITS", "2") or 2))
|
| except (TypeError, ValueError):
|
| max_profile_shadow_units = 2
|
| selected_ids = {item[1].memory_id for item in selected}
|
| added_profile_shadow_units = 0
|
| for item in candidates:
|
| _, record, _parent, metadata = item
|
| if added_profile_shadow_units >= max_profile_shadow_units:
|
| break
|
| if record.memory_id in selected_ids:
|
| continue
|
| if not bool(metadata.get("unit_coverage_profile_shadow_unit", False)):
|
| continue
|
| selected.append(item)
|
| selected_ids.add(record.memory_id)
|
| added_profile_shadow_units += 1
|
| packed_hits: List[MemoryHit] = []
|
| for score, unit_record, parent, extra in selected:
|
| packed_hits.append(
|
| _memory_hit_from_record(
|
| unit_record,
|
| score=score,
|
| metadata={
|
| **extra,
|
| "unit_coverage_pack": True,
|
| "evidence_snippet_role": "unit_coverage_evidence_unit",
|
| },
|
| )
|
| )
|
| if parent is not None:
|
| unit_metadata = dict(unit_record.metadata or {})
|
| parent_hit = _memory_hit_from_record(
|
| parent,
|
| score=max(1.12, score - 0.06),
|
| metadata={
|
| **extra,
|
| "unit_coverage_pack": True,
|
| "evidence_snippet_role": "unit_coverage_parent_event",
|
| "unit_coverage_parent_memory_id": parent.memory_id,
|
| "unit_coverage_child_id": unit_record.memory_id,
|
| "unit_coverage_child_value": unit_record.value,
|
| "unit_coverage_child_source_span": _clean_text(
|
| unit_metadata.get("facet_source_span", "") or unit_metadata.get("source_span", "")
|
| ),
|
| },
|
| )
|
| parent_hit.memory_id = f"{parent.memory_id}#unit_parent:{unit_record.memory_id}"
|
| packed_hits.append(parent_hit)
|
| merged: List[MemoryHit] = []
|
| seen_ids: set[str] = set()
|
| try:
|
| insertion_index = max(0, min(len(final_hits), int(os.getenv("TMCRA_UNIT_COVERAGE_PACK_INSERT_AFTER", "2") or 2)))
|
| except (TypeError, ValueError):
|
| insertion_index = min(len(final_hits), 2)
|
| ordered_hits = [*list(final_hits[:insertion_index]), *packed_hits, *list(final_hits[insertion_index:])]
|
| for hit in ordered_hits:
|
| if hit.memory_id and hit.memory_id in seen_ids:
|
| continue
|
| if hit.memory_id:
|
| seen_ids.add(hit.memory_id)
|
| merged.append(hit)
|
| return {
|
| "hits": merged,
|
| "metadata": {
|
| "unit_coverage_pack_enabled": True,
|
| "unit_coverage_candidate_count": len(candidates),
|
| "unit_coverage_selected_unit_count": len(selected),
|
| "unit_coverage_inserted_hit_count": len(packed_hits),
|
| "unit_coverage_count_intent": bool(count_intent),
|
| "unit_coverage_percentage_intent": bool(percentage_intent),
|
| "unit_coverage_temporal_intent": bool(temporal_intent),
|
| "unit_coverage_direct_unit_intent": bool(direct_unit_intent),
|
| },
|
| }
|
|
|
|
|
| def _multi_unit_chain_focus_tokens(query: str) -> set[str]:
|
| tokens = set(_path_utility_tokens(query))
|
| return {
|
| normalized
|
| for token in tokens
|
| for normalized in [_multi_unit_chain_normalize_token(token)]
|
| if normalized and normalized not in _MULTI_UNIT_CHAIN_FOCUS_DROP_TOKENS and len(normalized) > 2
|
| }
|
|
|
|
|
| def _multi_unit_chain_normalize_token(token: str) -> str:
|
| raw = _normalize(token)
|
| if not raw:
|
| return ""
|
| aliases = {
|
| "bought": "buy",
|
| "buying": "buy",
|
| "purchased": "buy",
|
| "purchasing": "buy",
|
| "worked": "work",
|
| "working": "work",
|
| "started": "start",
|
| "starting": "start",
|
| "finished": "finish",
|
| "finishing": "finish",
|
| "completed": "complete",
|
| "completing": "complete",
|
| "returned": "return",
|
| "returning": "return",
|
| "met": "meet",
|
| "meeting": "meet",
|
| "picked": "pick",
|
| "picking": "pick",
|
| "led": "lead",
|
| "leading": "lead",
|
| "kits": "kit",
|
| "items": "item",
|
| "projects": "project",
|
| "events": "event",
|
| "clothes": "clothing",
|
| }
|
| if raw in aliases:
|
| return aliases[raw]
|
| if len(raw) > 4 and raw.endswith("ies"):
|
| return raw[:-3] + "y"
|
| if len(raw) > 4 and raw.endswith("ing"):
|
| stem = raw[:-3]
|
| if len(stem) > 3 and stem[-1] == stem[-2]:
|
| stem = stem[:-1]
|
| return stem
|
| if len(raw) > 3 and raw.endswith("ed"):
|
| stem = raw[:-2]
|
| if len(stem) > 3 and stem[-1] == stem[-2]:
|
| stem = stem[:-1]
|
| return stem
|
| if len(raw) > 3 and raw.endswith("s") and not raw.endswith("ss"):
|
| return raw[:-1]
|
| return raw
|
|
|
|
|
| def _multi_unit_chain_normalized_tokens(text: str) -> set[str]:
|
| return {
|
| normalized
|
| for token in _path_utility_tokens(text)
|
| for normalized in [_multi_unit_chain_normalize_token(token)]
|
| if normalized
|
| }
|
|
|
|
|
| def _multi_unit_chain_hit_text(record: SessionMemoryRecordV2, parent: SessionMemoryRecordV2 | None) -> str:
|
| metadata = dict(record.metadata or {})
|
| parent_metadata = dict(parent.metadata or {}) if parent is not None else {}
|
| return " ".join(
|
| [
|
| _clean_text(record.value),
|
| _clean_text(record.category),
|
| _clean_text(record.relation),
|
| _clean_text(metadata.get("facet_type", "")),
|
| _clean_text(metadata.get("unit_kind", "")),
|
| _clean_text(metadata.get("facet_role", "")),
|
| _clean_text(metadata.get("facet_value", "")),
|
| _clean_text(metadata.get("facet_source_span", "")),
|
| _clean_text(metadata.get("action", "")),
|
| _clean_text(metadata.get("target", "")),
|
| _clean_text(metadata.get("quantity", "")),
|
| _clean_text(metadata.get("status", "")),
|
| _clean_text(parent.value if parent else ""),
|
| _clean_text(parent_metadata.get("source_span", "")),
|
| _clean_text(parent_metadata.get("raw_text", "")),
|
| ]
|
| )
|
|
|
|
|
| def _multi_unit_chain_local_hit_text(record: SessionMemoryRecordV2, parent: SessionMemoryRecordV2 | None) -> str:
|
| metadata = dict(record.metadata or {})
|
| parent_metadata = dict(parent.metadata or {}) if parent is not None else {}
|
| return " ".join(
|
| [
|
| _clean_text(record.value),
|
| _clean_text(record.category),
|
| _clean_text(record.relation),
|
| _clean_text(metadata.get("facet_type", "")),
|
| _clean_text(metadata.get("unit_kind", "")),
|
| _clean_text(metadata.get("facet_role", "")),
|
| _clean_text(metadata.get("facet_value", "")),
|
| _clean_text(metadata.get("facet_source_span", "")),
|
| _clean_text(metadata.get("action", "")),
|
| _clean_text(metadata.get("target", "")),
|
| _clean_text(metadata.get("quantity", "")),
|
| _clean_text(metadata.get("status", "")),
|
| _clean_text(parent.value if parent else ""),
|
| _clean_text(parent_metadata.get("source_span", "")),
|
| ]
|
| )
|
|
|
|
|
| def _multi_unit_chain_slot_hits(
|
| graph: SessionMemoryGraphV2,
|
| query: str,
|
| final_hits: Sequence[MemoryHit],
|
| *,
|
| top_k: int,
|
| ) -> Dict[str, Any]:
|
| mode = _normalize(os.getenv("TMCRA_MULTI_UNIT_CHAIN_SLOT_MODE", "on"))
|
| if mode in _MULTI_UNIT_CHAIN_DISABLED_MODES:
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {"multi_unit_chain_slot_enabled": False, "multi_unit_chain_slot_reason": "disabled"},
|
| }
|
| query_tokens = set(_path_utility_tokens(query))
|
| focus_tokens = _multi_unit_chain_focus_tokens(query)
|
| temporal_comparison_intent = bool(query_tokens & _MULTI_UNIT_CHAIN_TEMPORAL_COMPARISON_TOKENS) and len(focus_tokens) >= 2
|
| aggregation_or_joiner_intent = bool(
|
| re.search(r"\b(?:and|or|both|each|total|sum|minimum|maximum|amount|count|many|number)\b", str(query), flags=re.IGNORECASE)
|
| )
|
| numeric_aggregation_intent = bool(
|
| query_tokens
|
| & (
|
| _MULTI_UNIT_CHAIN_NUMERIC_VALUE_TOKENS
|
| | {"comment", "comments", "number", "percent", "percentage", "sum", "total"}
|
| )
|
| )
|
| count_or_aggregation_intent = bool(query_tokens & _MULTI_UNIT_CHAIN_COUNT_TOKENS) and (aggregation_or_joiner_intent or len(focus_tokens) >= 3)
|
| multi_intent = count_or_aggregation_intent or temporal_comparison_intent
|
| if not query_tokens or not multi_intent:
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {
|
| "multi_unit_chain_slot_enabled": False,
|
| "multi_unit_chain_slot_reason": "no_multi_intent",
|
| },
|
| }
|
| if not focus_tokens:
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {
|
| "multi_unit_chain_slot_enabled": True,
|
| "multi_unit_chain_slot_formed": False,
|
| "multi_unit_chain_slot_reason": "no_focus_tokens",
|
| },
|
| }
|
|
|
| records = list(getattr(graph, "records_by_id", {}).values())
|
| parent_by_slot = {_clean_text(record.slot_key).lower(): record for record in records}
|
| candidates: List[tuple[float, SessionMemoryRecordV2, SessionMemoryRecordV2 | None, Dict[str, Any]]] = []
|
| for record in records:
|
| metadata = dict(record.metadata or {})
|
| if record.state not in {"active", "parallel_active", "evidence"}:
|
| continue
|
| profile_shadow_unit = _profile_shadow_eventlike_record(record, metadata)
|
| if profile_shadow_unit:
|
| facet_type = "action"
|
| unit_kind = "profile_shadow_unit"
|
| parent_slot = _clean_text(record.slot_key).lower()
|
| parent = None
|
| text = _profile_shadow_unit_text(record, metadata)
|
| local_text = text
|
| else:
|
| if _normalize(metadata.get("content_variant", "")) != "event_facet_write":
|
| continue
|
| if _normalize(metadata.get("facet_layer_version", "")) != "event_unit_v1":
|
| continue
|
| facet_type = _normalize(metadata.get("facet_type", ""))
|
| unit_kind = _normalize(metadata.get("unit_kind", ""))
|
| if facet_type not in _MULTI_UNIT_CHAIN_FACET_TYPES and unit_kind not in _MULTI_UNIT_CHAIN_UNIT_KINDS:
|
| continue
|
| parent_slot = _clean_text(metadata.get("facet_parent_slot_key", "")).lower()
|
| parent = parent_by_slot.get(parent_slot)
|
| text = _multi_unit_chain_hit_text(record, parent)
|
| local_text = _multi_unit_chain_local_hit_text(record, parent)
|
| semantic_event_unit = _normalize(record.memory_id).startswith("tmcra.event.") or _normalize(parent_slot).startswith("tmcra.event.")
|
| unit_tokens = _multi_unit_chain_normalized_tokens(text)
|
| local_tokens = _multi_unit_chain_normalized_tokens(local_text)
|
| focus_overlap = focus_tokens & unit_tokens
|
| if not focus_overlap:
|
| continue
|
| score = min(1.2, len(focus_overlap) / max(1.0, len(focus_tokens)) * 1.6)
|
| local_numeric_signal = _multi_unit_chain_numeric_signal(unit_kind, facet_type, local_text, local_tokens)
|
| if _multi_unit_chain_date_like_numeric(local_text):
|
| numeric_signal = 0.0
|
| else:
|
| numeric_signal = local_numeric_signal
|
| if numeric_signal <= 0.0:
|
| numeric_signal = _multi_unit_chain_numeric_signal(unit_kind, facet_type, text, unit_tokens)
|
| temporal_anchor_signal = _multi_unit_chain_temporal_anchor_signal(text, unit_tokens)
|
| local_temporal_anchor_signal = _multi_unit_chain_temporal_anchor_signal(local_text, local_tokens)
|
| if facet_type in {"action", "entity", "role", "numeric"}:
|
| score += 0.34
|
| if unit_kind in {"action_unit", "target_entity", "leadership_role", "participation_role", "numeric_quantity"}:
|
| score += 0.24
|
| if profile_shadow_unit:
|
| score += 0.30
|
| semantic_event_priority = bool(semantic_event_unit and (count_or_aggregation_intent or numeric_aggregation_intent))
|
| if semantic_event_priority:
|
| score += 0.48
|
| if numeric_signal and numeric_aggregation_intent:
|
| score += numeric_signal
|
| if count_or_aggregation_intent and not numeric_aggregation_intent and facet_type in {"action", "entity", "role"}:
|
| score += 0.36
|
| if count_or_aggregation_intent and not numeric_aggregation_intent and unit_kind in {"action_unit", "target_entity", "leadership_role", "participation_role"}:
|
| score += 0.22
|
| if parent is not None:
|
| score += 0.12
|
| if score < 0.72:
|
| continue
|
| candidates.append(
|
| (
|
| round(min(4.0, 2.15 + score), 6),
|
| record,
|
| parent,
|
| {
|
| "multi_unit_chain_focus_overlap_tokens": sorted(focus_overlap)[:16],
|
| "multi_unit_chain_score": round(score, 6),
|
| "multi_unit_chain_numeric_signal": round(numeric_signal, 6),
|
| "multi_unit_chain_temporal_anchor_signal": round(temporal_anchor_signal, 6),
|
| "multi_unit_chain_local_temporal_anchor_signal": round(local_temporal_anchor_signal, 6),
|
| "multi_unit_chain_temporal_comparison": bool(temporal_comparison_intent),
|
| "multi_unit_chain_numeric_aggregation": bool(numeric_aggregation_intent),
|
| "unit_kind": unit_kind,
|
| "facet_type": facet_type,
|
| "multi_unit_chain_parent_slot_key": parent_slot,
|
| "multi_unit_chain_profile_shadow_unit": bool(profile_shadow_unit),
|
| "multi_unit_chain_semantic_event_unit": bool(semantic_event_unit),
|
| "multi_unit_chain_semantic_event_priority": bool(semantic_event_priority),
|
| "multi_unit_chain_count_or_aggregation": bool(count_or_aggregation_intent),
|
| },
|
| )
|
| )
|
|
|
| if not candidates:
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {
|
| "multi_unit_chain_slot_enabled": True,
|
| "multi_unit_chain_slot_formed": False,
|
| "multi_unit_chain_slot_reason": "no_matching_units",
|
| "multi_unit_chain_candidate_count": 0,
|
| "multi_unit_chain_focus_tokens": sorted(focus_tokens)[:24],
|
| },
|
| }
|
|
|
| if temporal_comparison_intent:
|
| candidates.sort(
|
| key=lambda item: (
|
| float(item[3].get("multi_unit_chain_local_temporal_anchor_signal", 0.0) or 0.0),
|
| float(item[3].get("multi_unit_chain_temporal_anchor_signal", 0.0) or 0.0),
|
| 1 if bool(item[3].get("multi_unit_chain_semantic_event_priority", False)) else 0,
|
| 0
|
| if _normalize(item[3].get("unit_kind", "")) == "numeric_quantity"
|
| or _normalize(item[3].get("facet_type", "")) == "numeric"
|
| else 1,
|
| float(item[0]),
|
| int(item[1].turn_index),
|
| ),
|
| reverse=True,
|
| )
|
| else:
|
| candidates.sort(
|
| key=lambda item: (
|
| float(item[3].get("multi_unit_chain_numeric_signal", 0.0) or 0.0)
|
| if bool(item[3].get("multi_unit_chain_numeric_aggregation", False))
|
| else (
|
| 1.2
|
| if bool(item[3].get("multi_unit_chain_semantic_event_priority", False))
|
| else
|
| 0.0
|
| if _normalize(item[3].get("unit_kind", "")) == "numeric_quantity"
|
| or _normalize(item[3].get("facet_type", "")) == "numeric"
|
| else 1.0
|
| ),
|
| float(item[0]),
|
| int(item[1].turn_index),
|
| ),
|
| reverse=True,
|
| )
|
| selected: List[tuple[float, SessionMemoryRecordV2, SessionMemoryRecordV2 | None, Dict[str, Any]]] = []
|
| seen_parent_slots: set[str] = set()
|
| seen_values: set[str] = set()
|
| max_units = max(2, min(8, int(os.getenv("TMCRA_MULTI_UNIT_CHAIN_SLOT_MAX_UNITS", "6") or 6)))
|
| for item in candidates:
|
| _, record, parent, metadata = item
|
| parent_slot = _clean_text(metadata.get("multi_unit_chain_parent_slot_key", ""))
|
| value_key = "|".join(
|
| [
|
| _normalize(record.value),
|
| _normalize(metadata.get("unit_kind", "")),
|
| parent_slot,
|
| ]
|
| )
|
| if value_key in seen_values:
|
| continue
|
| if parent_slot and parent_slot in seen_parent_slots:
|
|
|
| continue
|
| seen_values.add(value_key)
|
| if parent_slot:
|
| seen_parent_slots.add(parent_slot)
|
| selected.append(item)
|
| if len(selected) >= max_units:
|
| break
|
| if count_or_aggregation_intent:
|
| try:
|
| max_profile_shadow_units = max(0, int(os.getenv("TMCRA_MULTI_UNIT_CHAIN_PROFILE_SHADOW_MAX_UNITS", "2") or 2))
|
| except (TypeError, ValueError):
|
| max_profile_shadow_units = 2
|
| selected_ids = {item[1].memory_id for item in selected}
|
| added_profile_shadow_units = 0
|
| for item in candidates:
|
| _, record, _parent, metadata = item
|
| if added_profile_shadow_units >= max_profile_shadow_units:
|
| break
|
| if record.memory_id in selected_ids:
|
| continue
|
| if not bool(metadata.get("multi_unit_chain_profile_shadow_unit", False)):
|
| continue
|
| selected.append(item)
|
| selected_ids.add(record.memory_id)
|
| parent_slot = _clean_text(metadata.get("multi_unit_chain_parent_slot_key", ""))
|
| if parent_slot:
|
| seen_parent_slots.add(parent_slot)
|
| added_profile_shadow_units += 1
|
|
|
| min_parents = max(2, min(4, int(os.getenv("TMCRA_MULTI_UNIT_CHAIN_SLOT_MIN_PARENTS", "2") or 2)))
|
| if len(seen_parent_slots) < min_parents:
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {
|
| "multi_unit_chain_slot_enabled": True,
|
| "multi_unit_chain_slot_formed": False,
|
| "multi_unit_chain_slot_reason": "insufficient_parent_coverage",
|
| "multi_unit_chain_candidate_count": len(candidates),
|
| "multi_unit_chain_selected_unit_count": len(selected),
|
| "multi_unit_chain_parent_count": len(seen_parent_slots),
|
| "multi_unit_chain_focus_tokens": sorted(focus_tokens)[:24],
|
| },
|
| }
|
|
|
| chain_memory_ids: List[str] = []
|
| packed_hits: List[MemoryHit] = []
|
| for score, unit_record, parent, extra in selected:
|
| chain_memory_ids.append(unit_record.memory_id)
|
| packed_hits.append(
|
| _memory_hit_from_record(
|
| unit_record,
|
| score=score,
|
| metadata={
|
| **extra,
|
| "multi_unit_chain_slot": True,
|
| "multi_unit_chain_bundle": True,
|
| "evidence_snippet_role": "multi_unit_chain_evidence_unit",
|
| },
|
| )
|
| )
|
| if parent is not None:
|
| unit_metadata = dict(unit_record.metadata or {})
|
| chain_memory_ids.append(parent.memory_id)
|
| parent_hit = _memory_hit_from_record(
|
| parent,
|
| score=max(1.9, score - 0.08),
|
| metadata={
|
| **extra,
|
| "multi_unit_chain_slot": True,
|
| "multi_unit_chain_bundle": True,
|
| "evidence_snippet_role": "multi_unit_chain_parent_event",
|
| "multi_unit_chain_child_id": unit_record.memory_id,
|
| "multi_unit_chain_child_value": unit_record.value,
|
| "multi_unit_chain_child_source_span": _clean_text(
|
| unit_metadata.get("facet_source_span", "") or unit_metadata.get("source_span", "")
|
| ),
|
| },
|
| )
|
| parent_hit.memory_id = f"{parent.memory_id}#multi_parent:{unit_record.memory_id}"
|
| packed_hits.append(parent_hit)
|
|
|
| seen_ids: set[str] = set()
|
| merged: List[MemoryHit] = []
|
| insertion_index = max(0, min(len(final_hits), int(os.getenv("TMCRA_MULTI_UNIT_CHAIN_SLOT_INSERT_AFTER", "1") or 1)))
|
| ordered = [*list(final_hits[:insertion_index]), *packed_hits, *list(final_hits[insertion_index:])]
|
| for hit in ordered:
|
| if hit.memory_id and hit.memory_id in seen_ids:
|
| continue
|
| if hit.memory_id:
|
| seen_ids.add(hit.memory_id)
|
| merged.append(hit)
|
| return {
|
| "hits": merged,
|
| "metadata": {
|
| "multi_unit_chain_slot_enabled": True,
|
| "multi_unit_chain_slot_formed": True,
|
| "multi_unit_chain_slot_reason": "formed",
|
| "multi_unit_chain_candidate_count": len(candidates),
|
| "multi_unit_chain_selected_unit_count": len(selected),
|
| "multi_unit_chain_parent_count": len(seen_parent_slots),
|
| "multi_unit_chain_inserted_hit_count": len(packed_hits),
|
| "multi_unit_chain_memory_ids": _dedupe(chain_memory_ids, max_items=32),
|
| "multi_unit_chain_focus_tokens": sorted(focus_tokens)[:24],
|
| },
|
| }
|
|
|
|
|
| def _profile_support_source_hits(
|
| graph: SessionMemoryGraphV2,
|
| profile_hit: MemoryHit,
|
| *,
|
| grouped_hits: Mapping[str, Sequence[MemoryHit]],
|
| query: str,
|
| limit: int,
|
| ) -> List[MemoryHit]:
|
| metadata = dict(profile_hit.metadata or {})
|
| support_ids = _dedupe(
|
| [
|
| *list(metadata.get("profile_support_ids", []) or []),
|
| *list(metadata.get("support_memory_ids", []) or []),
|
| ],
|
| max_items=24,
|
| )
|
| if not support_ids:
|
| return []
|
| query_raw_tokens = set(_path_utility_tokens(query))
|
| query_tokens = _profile_query_expanded_tokens(query)
|
| profile_event_id = _clean_text(metadata.get("profile_first_hybrid_event_id", "")) or _runtime_event_key(profile_hit)
|
| candidates: List[tuple[float, MemoryHit]] = []
|
| for support_id in support_ids:
|
| record = getattr(graph, "records_by_id", {}).get(support_id)
|
| if record is None or record.state not in {"active", "parallel_active", "evidence"}:
|
| continue
|
| support_hit = _memory_hit_from_record(record)
|
| event_id = _runtime_event_key(support_hit)
|
| if not event_id or event_id == profile_event_id or event_id not in grouped_hits:
|
| continue
|
| match_score, overlap_tokens, raw_overlap_tokens = _profile_hit_match_score(query_raw_tokens, query_tokens, support_hit)
|
| source_score = max(0.80, float(profile_hit.score) - 0.02) + min(0.28, match_score * 0.12)
|
| support_metadata = dict(support_hit.metadata or {})
|
| profile_metadata = dict(profile_hit.metadata or {})
|
| support_metadata.update(
|
| {
|
| "profile_first_hybrid_rescue": True,
|
| "profile_first_source_support": True,
|
| "profile_first_parent_memory_id": profile_hit.memory_id,
|
| "profile_first_hybrid_event_id": event_id,
|
| "profile_first_parent_summary": _clean_text(profile_metadata.get("profile_summary", "")) or _clean_text(profile_metadata.get("profile_value", "")) or _clean_text(profile_hit.value),
|
| "profile_type": _clean_text(support_metadata.get("profile_type", "")) or _clean_text(profile_metadata.get("profile_type", "")),
|
| "profile_domain": _clean_text(support_metadata.get("profile_domain", "")) or _clean_text(profile_metadata.get("profile_domain", "")),
|
| "profile_domain_label": _clean_text(support_metadata.get("profile_domain_label", "")) or _clean_text(profile_metadata.get("profile_domain_label", "")),
|
| "profile_value": _clean_text(support_metadata.get("profile_value", "")) or _clean_text(profile_metadata.get("profile_value", "")) or _clean_text(profile_hit.value),
|
| "profile_summary": _clean_text(support_metadata.get("profile_summary", "")) or _clean_text(profile_metadata.get("profile_summary", "")) or _clean_text(profile_hit.value),
|
| "profile_query_match_score": round(match_score, 6),
|
| "profile_query_overlap_tokens": list(overlap_tokens),
|
| "profile_query_raw_overlap_tokens": list(raw_overlap_tokens),
|
| "profile_source_pack_role": "source_event_support",
|
| }
|
| )
|
| candidates.append(
|
| (
|
| source_score,
|
| MemoryHit(
|
| memory_id=support_hit.memory_id,
|
| category=support_hit.category,
|
| value=support_hit.value,
|
| relation=support_hit.relation,
|
| anchors=list(support_hit.anchors),
|
| score=round(source_score, 6),
|
| source_kind=support_hit.source_kind,
|
| slot_key=support_hit.slot_key,
|
| state=support_hit.state,
|
| turn_index=int(support_hit.turn_index),
|
| metadata=support_metadata,
|
| ),
|
| )
|
| )
|
| candidates.sort(
|
| key=lambda item: (
|
| bool((item[1].metadata or {}).get("profile_query_raw_overlap_tokens")),
|
| float(item[0]),
|
| int(item[1].turn_index),
|
| ),
|
| reverse=True,
|
| )
|
| return [hit for _, hit in candidates[: max(0, int(limit))]]
|
|
|
|
|
| def _profile_same_event_source_hit(profile_hit: MemoryHit, group_hits: Sequence[MemoryHit], *, event_id: str) -> MemoryHit | None:
|
| source_hit = _support_hit_for_path("speaker_event_source_turn", group_hits) or _representative_event_hit(group_hits)
|
| if source_hit is None or source_hit.memory_id == profile_hit.memory_id:
|
| return None
|
| metadata = dict(source_hit.metadata or {})
|
| profile_metadata = dict(profile_hit.metadata or {})
|
| metadata.update(
|
| {
|
| "profile_first_hybrid_rescue": True,
|
| "profile_first_source_support": True,
|
| "profile_first_same_event_support": True,
|
| "profile_first_parent_memory_id": profile_hit.memory_id,
|
| "profile_first_hybrid_event_id": event_id,
|
| "profile_first_parent_summary": _clean_text(profile_metadata.get("profile_summary", "")) or _clean_text(profile_metadata.get("profile_value", "")) or _clean_text(profile_hit.value),
|
| "profile_type": _clean_text(metadata.get("profile_type", "")) or _clean_text(profile_metadata.get("profile_type", "")),
|
| "profile_domain": _clean_text(metadata.get("profile_domain", "")) or _clean_text(profile_metadata.get("profile_domain", "")),
|
| "profile_domain_label": _clean_text(metadata.get("profile_domain_label", "")) or _clean_text(profile_metadata.get("profile_domain_label", "")),
|
| "profile_value": _clean_text(metadata.get("profile_value", "")) or _clean_text(profile_metadata.get("profile_value", "")) or _clean_text(profile_hit.value),
|
| "profile_summary": _clean_text(metadata.get("profile_summary", "")) or _clean_text(profile_metadata.get("profile_summary", "")) or _clean_text(profile_hit.value),
|
| "profile_source_pack_role": "same_event_source_turn",
|
| "evidence_snippet_role": "profile_source_support",
|
| }
|
| )
|
| return MemoryHit(
|
| memory_id=source_hit.memory_id,
|
| category=source_hit.category,
|
| value=source_hit.value,
|
| relation=source_hit.relation,
|
| anchors=list(source_hit.anchors),
|
| score=max(float(source_hit.score), float(profile_hit.score) - 0.01),
|
| source_kind=source_hit.source_kind,
|
| slot_key=source_hit.slot_key,
|
| state=source_hit.state,
|
| turn_index=int(source_hit.turn_index),
|
| metadata=metadata,
|
| )
|
|
|
|
|
| def _profile_first_hybrid_rescue(
|
| graph: SessionMemoryGraphV2,
|
| query: str,
|
| *,
|
| grouped_hits: Mapping[str, Sequence[MemoryHit]],
|
| top_k: int,
|
| ) -> Dict[str, Any]:
|
| intent = infer_profile_query_intent(query)
|
| if not bool(intent.get("enabled")):
|
| return {"hits": [], "event_ids": []}
|
| raw_hits = _profile_query_rescue_hits(graph, query, top_k=max(4, int(top_k or 1) * 3))
|
| limit = max(2, min(4, int(top_k or 1)))
|
| selected_hits: List[MemoryHit] = []
|
| selected_event_ids: List[str] = []
|
| seen_event_ids: set[str] = set()
|
| for rank, hit in enumerate(raw_hits, start=1):
|
| event_id = _runtime_event_key(hit)
|
| if not event_id or event_id not in grouped_hits or event_id in seen_event_ids:
|
| continue
|
| metadata = dict(hit.metadata or {})
|
| metadata.update(
|
| {
|
| "profile_first_hybrid_rescue": True,
|
| "profile_first_hybrid_rank": rank,
|
| "profile_first_hybrid_event_id": event_id,
|
| }
|
| )
|
| hit.metadata = metadata
|
| hit.score = round(max(float(hit.score), 0.88) + max(0.0, 0.08 - (0.01 * len(selected_hits))), 6)
|
| selected_hits.append(hit)
|
| selected_event_ids.append(event_id)
|
| seen_event_ids.add(event_id)
|
| same_event_source = _profile_same_event_source_hit(hit, grouped_hits.get(event_id, []), event_id=event_id)
|
| if same_event_source is not None:
|
| selected_hits.append(same_event_source)
|
| if len(selected_hits) >= limit:
|
| break
|
| support_hits = _profile_support_source_hits(
|
| graph,
|
| hit,
|
| grouped_hits=grouped_hits,
|
| query=query,
|
| limit=max(1, min(3, limit - len(selected_hits) + 1)),
|
| )
|
| for support_hit in support_hits:
|
| support_event_id = _clean_text((support_hit.metadata or {}).get("profile_first_hybrid_event_id", "")) or _runtime_event_key(support_hit)
|
| if not support_event_id or support_event_id in seen_event_ids:
|
| continue
|
| selected_hits.append(support_hit)
|
| selected_event_ids.append(support_event_id)
|
| seen_event_ids.add(support_event_id)
|
| if len(selected_hits) >= limit:
|
| break
|
| if len(selected_hits) >= limit:
|
| break
|
| return {
|
| "hits": selected_hits,
|
| "event_ids": selected_event_ids,
|
| "memory_ids": [hit.memory_id for hit in selected_hits],
|
| }
|
|
|
|
|
| def _inject_profile_first_hits(
|
| final_hits: Sequence[MemoryHit],
|
| profile_first_hits: Sequence[MemoryHit],
|
| *,
|
| selected_event_ids: Sequence[str],
|
| selected_path_ids: Sequence[str],
|
| ) -> List[MemoryHit]:
|
| merged: List[MemoryHit] = []
|
| seen_memory_ids: set[str] = set()
|
| for hit in list(profile_first_hits) + list(final_hits):
|
| if not hit or not hit.memory_id or hit.memory_id in seen_memory_ids:
|
| continue
|
| metadata = dict(hit.metadata or {})
|
| if bool(metadata.get("profile_first_hybrid_rescue")):
|
| event_id = _clean_text(metadata.get("profile_first_hybrid_event_id", "")) or _runtime_event_key(hit)
|
| metadata.update(
|
| {
|
| "event_id": event_id,
|
| "path_id": "",
|
| "evidence_snippet_role": "selected_event_representative",
|
| "hybrid_score_source": "profile_first_hybrid_rescue",
|
| "selected_event_ids": list(selected_event_ids),
|
| "selected_path_ids": list(selected_path_ids),
|
| }
|
| )
|
| hit = MemoryHit(
|
| memory_id=hit.memory_id,
|
| category=hit.category,
|
| value=hit.value,
|
| relation=hit.relation,
|
| anchors=list(hit.anchors),
|
| score=float(hit.score),
|
| source_kind=hit.source_kind,
|
| slot_key=hit.slot_key,
|
| state=hit.state,
|
| turn_index=int(hit.turn_index),
|
| metadata=metadata,
|
| )
|
| merged.append(hit)
|
| seen_memory_ids.add(hit.memory_id)
|
| return merged
|
|
|
|
|
| def _profile_focused_pack_hits(
|
| graph: SessionMemoryGraphV2,
|
| query: str,
|
| final_hits: Sequence[MemoryHit],
|
| *,
|
| top_k: int,
|
| ) -> Dict[str, Any]:
|
| intent = infer_profile_query_intent(query)
|
| if not bool(intent.get("enabled")):
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {
|
| "profile_focused_pack_enabled": False,
|
| "profile_focused_pack_reason": "profile_intent_not_requested",
|
| },
|
| }
|
| source_hits = _learnable_graph_hits(graph)
|
| if not source_hits:
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {
|
| "profile_focused_pack_enabled": False,
|
| "profile_focused_pack_reason": "no_learnable_hits",
|
| },
|
| }
|
| runtime_graph = _build_runtime_graph_from_hits(query, source_hits)
|
| grouped_hits = dict(runtime_graph.get("grouped_hits", {}) or {})
|
| profile_first_payload = _profile_first_hybrid_rescue(
|
| graph,
|
| query,
|
| grouped_hits=grouped_hits,
|
| top_k=max(2, min(max(1, int(top_k or 1)), 4)),
|
| )
|
| profile_first_hits = list(profile_first_payload.get("hits", []) or [])
|
| profile_first_event_ids = list(profile_first_payload.get("event_ids", []) or [])
|
| profile_first_memory_ids = list(profile_first_payload.get("memory_ids", []) or [])
|
| if not profile_first_hits:
|
| return {
|
| "hits": list(final_hits),
|
| "metadata": {
|
| "profile_focused_pack_enabled": True,
|
| "profile_focused_pack_reason": "no_profile_hits",
|
| "profile_focused_pack_event_ids": [],
|
| "profile_focused_pack_memory_ids": [],
|
| },
|
| }
|
| selected_event_ids = _dedupe([*profile_first_event_ids, *_event_ids_from_hits(final_hits)], max_items=max(1, int(top_k or 1) * 2))
|
| selected_path_ids = _dedupe(
|
| [
|
| _clean_text((hit.metadata or {}).get("path_id", ""))
|
| for hit in final_hits
|
| if _clean_text((hit.metadata or {}).get("path_id", ""))
|
| ],
|
| max_items=max(1, int(top_k or 1)),
|
| )
|
| merged_hits = _inject_profile_first_hits(
|
| final_hits,
|
| profile_first_hits,
|
| selected_event_ids=selected_event_ids,
|
| selected_path_ids=selected_path_ids,
|
| )
|
| merged_hits = _coverage_preserving_final_hits(merged_hits, selected_event_ids=selected_event_ids, top_k=top_k)
|
| return {
|
| "hits": merged_hits,
|
| "metadata": {
|
| "profile_focused_pack_enabled": True,
|
| "profile_focused_pack_reason": "profile_first_pack_injected",
|
| "profile_focused_pack_event_ids": list(profile_first_event_ids),
|
| "profile_focused_pack_memory_ids": list(profile_first_memory_ids),
|
| "profile_focused_pack_hit_count": len(profile_first_hits),
|
| "profile_first_hybrid_enabled": True,
|
| "profile_first_event_ids": list(profile_first_event_ids),
|
| "profile_first_memory_ids": list(profile_first_memory_ids),
|
| },
|
| }
|
|
|
|
|
| def _topic_bucket_rerank_hits(
|
| graph: SessionMemoryGraphV2,
|
| query: str,
|
| hits: Sequence[MemoryHit],
|
| *,
|
| top_k: int,
|
| ) -> Dict[str, Any]:
|
| if not hits:
|
| profile_rescue_hits = _profile_query_rescue_hits(graph, query, top_k=top_k)
|
| if profile_rescue_hits:
|
| return {
|
| "hits": profile_rescue_hits[: max(1, int(top_k or 1))],
|
| "metadata": {
|
| "topic_bucket_rerank_enabled": True,
|
| "topic_bucket_rerank_reason": "profile_query_rescue_from_empty_hits",
|
| "topic_bucket_profile_query_rescue_count": len(profile_rescue_hits),
|
| },
|
| }
|
| return {
|
| "hits": [],
|
| "metadata": {"topic_bucket_rerank_enabled": True, "topic_bucket_rerank_reason": "no_hits"},
|
| }
|
| query_topic = _assign_topic_bucket_for_text(graph, query, turn_index=0, create=False)
|
| query_bucket_id = _clean_text(query_topic.get("topic_bucket_id", ""))
|
| adjacent_ids = _topic_adjacent_bucket_ids(graph, query_bucket_id)
|
| dialogue_tunnel_ids = _dialogue_tunnel_bucket_ids(graph, query_bucket_id)
|
| query_keywords = list(query_topic.get("topic_keywords", []) or [])
|
| bridge_requested = _topic_bridge_requested(query)
|
| dialogue_requested = _dialogue_tunnel_requested(query)
|
| profile_query_requested = bool(infer_profile_query_intent(query).get("enabled"))
|
| reranked: List[MemoryHit] = []
|
| stats = {
|
| "same_bucket": 0,
|
| "bridge_bucket": 0,
|
| "blocked_bridge_bucket": 0,
|
| "dialogue_tunnel_bucket": 0,
|
| "blocked_dialogue_tunnel_bucket": 0,
|
| "profile_route_preserved": 0,
|
| "profile_query_rescue": 0,
|
| "overlap_bucket": 0,
|
| "off_topic": 0,
|
| }
|
| for hit in list(hits):
|
| metadata = dict(hit.metadata or {})
|
| hit_bucket_id = _clean_text(metadata.get("topic_bucket_id", ""))
|
| hit_keywords = _dedupe(metadata.get("topic_keywords", []) or [], max_items=32)
|
| overlap = _topic_bucket_overlap_score(query_keywords, hit_keywords)
|
| same_bucket = bool(query_bucket_id and hit_bucket_id and query_bucket_id == hit_bucket_id)
|
| bridge_bucket = bool(hit_bucket_id and hit_bucket_id in adjacent_ids)
|
| dialogue_tunnel_bucket = bool(hit_bucket_id and hit_bucket_id in dialogue_tunnel_ids)
|
| overlap_bucket = overlap >= 0.22
|
| bridge_allowed = bridge_bucket and bridge_requested
|
| dialogue_allowed = dialogue_tunnel_bucket and dialogue_requested
|
| profile_route_preserved = bool(
|
| profile_query_requested
|
| and (
|
| bool(metadata.get("profile_layer"))
|
| or "profile_route" in _normalize(metadata.get("match_reason", ""))
|
| )
|
| )
|
| current_subject_preserved = bool(metadata.get("current_subject_resolver") or metadata.get("public_subject_match"))
|
| delta = 0.0
|
| if same_bucket:
|
| delta += 1.35
|
| stats["same_bucket"] += 1
|
| elif current_subject_preserved:
|
| delta += 0.18
|
| stats["profile_route_preserved"] += 1
|
| elif bridge_allowed:
|
| delta += 0.42
|
| stats["bridge_bucket"] += 1
|
| elif dialogue_allowed:
|
| delta += 0.16
|
| stats["dialogue_tunnel_bucket"] += 1
|
| elif overlap_bucket:
|
| delta += 0.28 + overlap
|
| stats["overlap_bucket"] += 1
|
| elif bridge_bucket:
|
| delta -= 1.05
|
| stats["blocked_bridge_bucket"] += 1
|
| elif dialogue_tunnel_bucket:
|
| delta -= 1.35
|
| stats["blocked_dialogue_tunnel_bucket"] += 1
|
| elif profile_route_preserved:
|
| delta += 0.08
|
| stats["profile_route_preserved"] += 1
|
| else:
|
| delta -= 0.72
|
| stats["off_topic"] += 1
|
| memory_type = _normalize(metadata.get("memory_type", ""))
|
| durability = _normalize(metadata.get("durability", ""))
|
| conflict_policy = _normalize(metadata.get("conflict_policy", ""))
|
| if (memory_type == "hard_constraint" or durability == "hard" or conflict_policy == "must_preserve") and (
|
| same_bucket or bridge_allowed or dialogue_allowed or overlap_bucket
|
| ):
|
| delta += 1.10
|
| match_reason = _clean_text(metadata.get("match_reason", ""))
|
| if profile_route_preserved and "profile_route" not in _normalize(match_reason):
|
| match_reason = ",".join(_dedupe([match_reason, "profile_route"], max_items=4))
|
| metadata.update(
|
| {
|
| "topic_bucket_rerank": True,
|
| "topic_bucket_query_id": query_bucket_id,
|
| "topic_bucket_query_label": _clean_text(query_topic.get("topic_label", "")),
|
| "topic_bucket_overlap": round(overlap, 6),
|
| "topic_bucket_delta": round(delta, 6),
|
| "topic_bucket_same": same_bucket,
|
| "topic_bucket_bridge": bridge_bucket,
|
| "topic_bucket_bridge_allowed": bridge_allowed,
|
| "topic_bucket_dialogue_tunnel": dialogue_tunnel_bucket,
|
| "topic_bucket_dialogue_tunnel_allowed": dialogue_allowed,
|
| "topic_bucket_profile_route_preserved": profile_route_preserved,
|
| "topic_bucket_current_subject_preserved": current_subject_preserved,
|
| "match_reason": match_reason,
|
| }
|
| )
|
| hit.metadata = metadata
|
| hit.score = float(hit.score) + delta
|
| reranked.append(hit)
|
| seen_ids = {hit.memory_id for hit in reranked if hit.memory_id}
|
| rescue_records = [
|
| record
|
| for record in getattr(graph, "records_by_id", {}).values()
|
| if record.memory_id not in seen_ids
|
| and record.state == "active"
|
| and _clean_text((record.metadata or {}).get("topic_bucket_id", "")) == query_bucket_id
|
| and _normalize(record.category) != "question"
|
| ]
|
| rescue_records.sort(
|
| key=lambda record: (
|
| int(any(marker in _normalize(record.value) for marker in ("过敏", "必须", "避开", "禁止", "不能", "must", "avoid", "allergy"))),
|
| int(_normalize(record.category) in {"constraint", "preference"}),
|
| float(record.confidence),
|
| float(record.salience),
|
| -int(record.turn_index),
|
| ),
|
| reverse=True,
|
| )
|
| rescue_hits = [
|
| _topic_bucket_record_to_hit(record, query_topic=query_topic, rank=index)
|
| for index, record in enumerate(rescue_records[: max(2, min(12, int(top_k or 1) * 2))], start=1)
|
| ]
|
| reranked.extend(rescue_hits)
|
| dialogue_rescue_hits: List[MemoryHit] = []
|
| if dialogue_requested and dialogue_tunnel_ids:
|
| seen_ids.update(hit.memory_id for hit in rescue_hits if hit.memory_id)
|
| dialogue_records = [
|
| record
|
| for record in getattr(graph, "records_by_id", {}).values()
|
| if record.memory_id not in seen_ids
|
| and record.state == "active"
|
| and _clean_text((record.metadata or {}).get("topic_bucket_id", "")) in dialogue_tunnel_ids
|
| and _normalize(record.category) != "question"
|
| ]
|
| def dialogue_record_rank(record: SessionMemoryRecordV2) -> tuple[int, int, float, float, int]:
|
| return (
|
| int(any(marker in _normalize(record.value) for marker in ("过敏", "必须", "避开", "禁止", "不能", "must", "avoid", "allergy"))),
|
| int(_normalize(record.category) in {"constraint", "preference"}),
|
| float(record.confidence),
|
| float(record.salience),
|
| -int(record.turn_index),
|
| )
|
|
|
| dialogue_records.sort(key=dialogue_record_rank, reverse=True)
|
| selected_dialogue_records: List[SessionMemoryRecordV2] = []
|
| selected_dialogue_ids: set[str] = set()
|
| for bucket_id in sorted(dialogue_tunnel_ids):
|
| bucket_records = [
|
| record
|
| for record in dialogue_records
|
| if _clean_text((record.metadata or {}).get("topic_bucket_id", "")) == bucket_id
|
| ]
|
| if not bucket_records:
|
| continue
|
| best = bucket_records[0]
|
| selected_dialogue_records.append(best)
|
| selected_dialogue_ids.add(best.memory_id)
|
| target_dialogue_rescue = max(len(selected_dialogue_records), max(1, min(4, int(top_k or 1) // 2 or 1)))
|
| for record in dialogue_records:
|
| if len(selected_dialogue_records) >= target_dialogue_rescue:
|
| break
|
| if record.memory_id in selected_dialogue_ids:
|
| continue
|
| selected_dialogue_records.append(record)
|
| selected_dialogue_ids.add(record.memory_id)
|
| dialogue_rescue_hits = [
|
| _topic_bucket_record_to_hit(record, query_topic=query_topic, rank=index, rescue_kind="dialogue_tunnel")
|
| for index, record in enumerate(selected_dialogue_records, start=1)
|
| ]
|
| reranked.extend(dialogue_rescue_hits)
|
| profile_rescue_hits: List[MemoryHit] = []
|
| if profile_query_requested:
|
| seen_ids.update(hit.memory_id for hit in reranked if hit.memory_id)
|
| for index, hit in enumerate(_profile_query_rescue_hits(graph, query, top_k=top_k), start=1):
|
| if hit.memory_id in seen_ids:
|
| continue
|
| metadata = dict(hit.metadata or {})
|
| metadata.update(
|
| {
|
| "profile_query_rescue_rank": index,
|
| "topic_bucket_query_id": query_bucket_id,
|
| "topic_bucket_query_label": _clean_text(query_topic.get("topic_label", "")),
|
| "topic_bucket_profile_route_preserved": True,
|
| }
|
| )
|
| hit.metadata = metadata
|
| hit.score = float(hit.score) + 0.08
|
| profile_rescue_hits.append(hit)
|
| seen_ids.add(hit.memory_id)
|
| stats["profile_query_rescue"] = len(profile_rescue_hits)
|
| reranked.extend(profile_rescue_hits)
|
| reranked.sort(key=lambda item: (float(item.score), int(item.turn_index)), reverse=True)
|
| limit = max(1, int(top_k or 1))
|
| focused_hits = [
|
| hit
|
| for hit in reranked
|
| if bool((hit.metadata or {}).get("topic_bucket_same"))
|
| or float((hit.metadata or {}).get("topic_bucket_overlap", 0.0) or 0.0) >= 0.22
|
| or bool((hit.metadata or {}).get("topic_bucket_bridge_allowed"))
|
| or bool((hit.metadata or {}).get("topic_bucket_dialogue_tunnel_allowed"))
|
| or bool((hit.metadata or {}).get("topic_bucket_profile_route_preserved"))
|
| or bool((hit.metadata or {}).get("topic_bucket_current_subject_preserved"))
|
| or bool((hit.metadata or {}).get("current_subject_resolver"))
|
| ]
|
| model_path_fallback = False
|
| no_bucket_model_fallback = False
|
| if not focused_hits:
|
| model_supported_hits = [
|
| hit
|
| for hit in reranked
|
| if _clean_text((hit.metadata or {}).get("path_id", ""))
|
| or _clean_text((hit.metadata or {}).get("hybrid_score_source", ""))
|
| ]
|
| if model_supported_hits:
|
| focused_hits = model_supported_hits
|
| model_path_fallback = True
|
| generic_memory_fallback = False
|
| if not focused_hits and any(not _is_public_dialog_hit(hit) for hit in reranked):
|
| focused_hits = list(reranked)
|
| generic_memory_fallback = True
|
| if not focused_hits and not query_bucket_id:
|
| focused_hits = list(reranked)
|
| no_bucket_model_fallback = True
|
| time_focused_model_path = any(
|
| "speaker_event_time" in _clean_text((hit.metadata or {}).get("path_id", ""))
|
| or _clean_text((hit.metadata or {}).get("model_focused_answer_type", "")) == "time"
|
| for hit in focused_hits
|
| )
|
| if time_focused_model_path:
|
| focused_hits = [
|
| hit
|
| for hit in focused_hits
|
| if not (
|
| _clean_text(hit.source_kind) == "public_dialog_profile"
|
| and not _clean_text((hit.metadata or {}).get("path_id", ""))
|
| )
|
| ]
|
| filtered_count = max(0, len(reranked) - len(focused_hits))
|
| query_subject_signature = _public_subject_signature(_public_query_subject(query))
|
|
|
| def _hit_subject_signatures(hit: MemoryHit) -> set[str]:
|
| metadata = dict(hit.metadata or {})
|
| signatures = {
|
| _normalize(metadata.get("subject_signature", "")).replace("-", "_"),
|
| _public_subject_signature(metadata.get("subject", "")),
|
| }
|
| canonical_slot_key = _clean_text(metadata.get("canonical_slot_key", ""))
|
| if ".subject." in canonical_slot_key:
|
| signatures.add(_public_subject_signature(canonical_slot_key.split(".subject.", 1)[-1]))
|
| if ".subject." in hit.slot_key:
|
| signatures.add(_public_subject_signature(hit.slot_key.split(".subject.", 1)[-1]))
|
| signatures.discard("")
|
| return signatures
|
|
|
| protected_current_hits = [
|
| hit
|
| for hit in focused_hits
|
| if bool((hit.metadata or {}).get("current_subject_resolver"))
|
| ]
|
| protected_model_path_hits = [
|
| hit
|
| for hit in focused_hits
|
| if _clean_text((hit.metadata or {}).get("path_id", ""))
|
| and _clean_text((hit.metadata or {}).get("hybrid_score_source", ""))
|
| ]
|
| protected_generic_hits = [
|
| hit
|
| for hit in focused_hits
|
| if not _is_public_dialog_hit(hit)
|
| and not bool((hit.metadata or {}).get("profile_layer"))
|
| ]
|
| if protected_current_hits and query_subject_signature:
|
| protected_ids = {hit.memory_id for hit in protected_current_hits if hit.memory_id}
|
| compacted_focused_hits: List[MemoryHit] = []
|
| for hit in focused_hits:
|
| if hit.memory_id in protected_ids:
|
| compacted_focused_hits.append(hit)
|
| continue
|
| same_current_subject = query_subject_signature in _hit_subject_signatures(hit)
|
| inactive_state = _normalize(hit.state) in {"superseded", "evidence", "historical", "stale", "false"}
|
| if same_current_subject and inactive_state:
|
| continue
|
| compacted_focused_hits.append(hit)
|
| focused_hits = compacted_focused_hits
|
| filtered_count = max(0, len(reranked) - len(focused_hits))
|
|
|
| selected_hits: List[MemoryHit] = []
|
| selected_keys: set[str] = set()
|
| for hit in [*protected_current_hits, *protected_model_path_hits, *protected_generic_hits, *focused_hits]:
|
| key = hit.memory_id or f"{_hit_event_id(hit)}::{hit.slot_key}::{hit.value[:80]}"
|
| if key in selected_keys:
|
| continue
|
| selected_hits.append(hit)
|
| selected_keys.add(key)
|
| if len(selected_hits) >= limit:
|
| break
|
| dialogue_reserved_count = 0
|
| if dialogue_requested and dialogue_tunnel_ids:
|
| selected_ids = {hit.memory_id for hit in selected_hits if hit.memory_id}
|
| selected_bucket_ids = {
|
| _clean_text((hit.metadata or {}).get("topic_bucket_id", ""))
|
| for hit in selected_hits
|
| if _clean_text((hit.metadata or {}).get("topic_bucket_id", ""))
|
| }
|
| for bucket_id in sorted(dialogue_tunnel_ids):
|
| if bucket_id in selected_bucket_ids:
|
| continue
|
| candidate = next(
|
| (
|
| hit
|
| for hit in focused_hits
|
| if hit.memory_id not in selected_ids
|
| and _clean_text((hit.metadata or {}).get("topic_bucket_id", "")) == bucket_id
|
| and bool((hit.metadata or {}).get("topic_bucket_dialogue_tunnel_allowed"))
|
| ),
|
| None,
|
| )
|
| if candidate is None:
|
| continue
|
| if len(selected_hits) < limit:
|
| selected_hits.append(candidate)
|
| else:
|
| replace_index = len(selected_hits) - 1
|
| for index in range(len(selected_hits) - 1, -1, -1):
|
| metadata = dict(selected_hits[index].metadata or {})
|
| hardish = (
|
| _normalize(metadata.get("memory_type", "")) == "hard_constraint"
|
| or _normalize(metadata.get("durability", "")) == "hard"
|
| or _normalize(metadata.get("conflict_policy", "")) == "must_preserve"
|
| )
|
| if not hardish and not bool(metadata.get("topic_bucket_dialogue_tunnel_allowed")):
|
| replace_index = index
|
| break
|
| selected_hits[replace_index] = candidate
|
| selected_ids.add(candidate.memory_id)
|
| selected_bucket_ids.add(bucket_id)
|
| dialogue_reserved_count += 1
|
| selected_hits.sort(
|
| key=lambda item: (
|
| int(bool((item.metadata or {}).get("current_subject_resolver"))),
|
| float(item.score),
|
| int(item.turn_index),
|
| ),
|
| reverse=True,
|
| )
|
| return {
|
| "hits": selected_hits[:limit],
|
| "metadata": {
|
| "topic_bucket_rerank_enabled": True,
|
| "topic_bucket_no_fill_policy": True,
|
| "topic_bucket_query_id": query_bucket_id,
|
| "topic_bucket_query_label": _clean_text(query_topic.get("topic_label", "")),
|
| "topic_bucket_query_keywords": query_keywords,
|
| "topic_bucket_adjacent_ids": sorted(adjacent_ids),
|
| "dialogue_tunnel_adjacent_ids": sorted(dialogue_tunnel_ids),
|
| "topic_bucket_bridge_requested": bridge_requested,
|
| "dialogue_tunnel_requested": dialogue_requested,
|
| "topic_bucket_model_path_fallback": model_path_fallback,
|
| "topic_bucket_generic_memory_fallback": generic_memory_fallback,
|
| "topic_bucket_no_bucket_model_fallback": no_bucket_model_fallback,
|
| "topic_bucket_candidate_count": len(hits),
|
| "topic_bucket_rescue_count": len(rescue_hits),
|
| "dialogue_tunnel_rescue_count": len(dialogue_rescue_hits),
|
| "profile_query_rescue_count": len(profile_rescue_hits),
|
| "dialogue_tunnel_reserved_count": dialogue_reserved_count,
|
| "topic_bucket_focused_count": len(focused_hits),
|
| "topic_bucket_final_count": len(selected_hits[:limit]),
|
| "topic_bucket_filtered_count": filtered_count,
|
| "topic_bucket_stats": stats,
|
| },
|
| }
|
|
|
|
|
| def _looks_like_write_turn(user_text: str, *, answer_payload: Dict[str, Any] | None = None) -> bool:
|
| text = _normalize(user_text)
|
| if not text:
|
| return False
|
| if "?" in text:
|
| return False
|
| if any(marker in text for marker in _WRITE_MARKERS):
|
| return True
|
| metadata = dict((answer_payload or {}).get("metadata", {}) or {})
|
| return bool(metadata.get("memory_write"))
|
|
|
|
|
| def _explicit_overwrite_requested(user_text: str) -> bool:
|
| text = _normalize(user_text)
|
| return bool(text) and any(marker in text for marker in _OVERWRITE_MARKERS)
|
|
|
|
|
| def _apply_turn_write_intent(records: List[SessionMemoryRecordV2], *, user_text: str) -> List[SessionMemoryRecordV2]:
|
| if not records or not _explicit_overwrite_requested(user_text):
|
| return records
|
| for record in records:
|
| metadata = dict(record.metadata or {})
|
| metadata.setdefault("write_intent", "overwrite")
|
| metadata.setdefault("memory_gate_decision", "explicit_overwrite")
|
| metadata["allow_parallel_state"] = False
|
| record.metadata = metadata
|
| return records
|
|
|
|
|
| def _records_from_extractor(
|
| extractor: SessionMemoryExtractor,
|
| *,
|
| query: str,
|
| answer_payload: Dict[str, Any] | None,
|
| extraction_result: Dict[str, Any] | None,
|
| turn_index: int,
|
| profile: TMCRAProfile | None = None,
|
| ) -> List[SessionMemoryRecordV2]:
|
| profile = profile or TMCRAProfile()
|
| raw_records = extractor.extract(
|
| query=query,
|
| extraction_result=extraction_result,
|
| answer_bundle=answer_payload,
|
| answer_mode=str((answer_payload or {}).get("answer_mode", "transparent")),
|
| turn_index=turn_index,
|
| )
|
| results: List[SessionMemoryRecordV2] = []
|
| for index, record in enumerate(raw_records):
|
| slot_key = profile.stable_slot_key(
|
| category=record.category,
|
| value=record.value,
|
| anchors=record.anchor_concepts,
|
| slot_key=record.metadata.get("slot_key", "") if isinstance(record.metadata, dict) else "",
|
| relation=record.relation,
|
| metadata=dict(record.metadata or {}),
|
| )
|
| metadata = {
|
| **dict(record.metadata or {}),
|
| "memory_role": _clean_text(dict(record.metadata or {}).get("memory_role", "")) or "user",
|
| "authority": _clean_text(dict(record.metadata or {}).get("authority", "")) or "source",
|
| "canonical_slot_key": _clean_text(dict(record.metadata or {}).get("canonical_slot_key", "")) or slot_key,
|
| "writeback_class": _clean_text(dict(record.metadata or {}).get("writeback_class", "")),
|
| "origin_query": _clean_text(dict(record.metadata or {}).get("origin_query", "")) or _clean_text(query),
|
| "origin_answer_id": _clean_text(dict(record.metadata or {}).get("origin_answer_id", "")),
|
| "support_memory_ids": _dedupe(dict(record.metadata or {}).get("support_memory_ids", []) or []),
|
| "support_fact_refs": _dedupe(dict(record.metadata or {}).get("support_fact_refs", []) or []),
|
| "support_path_refs": _dedupe(dict(record.metadata or {}).get("support_path_refs", []) or []),
|
| "promotion_state": _clean_text(dict(record.metadata or {}).get("promotion_state", "")) or "none",
|
| }
|
| results.append(
|
| SessionMemoryRecordV2(
|
| memory_id=f"{slot_key}:{turn_index}:auto:{index}",
|
| category=record.category,
|
| slot_key=slot_key,
|
| value=_clean_text(record.value),
|
| relation=_clean_text(record.relation) or f"{record.category}_memory",
|
| anchor_concepts=_dedupe(record.anchor_concepts, max_items=8),
|
| evidence_anchors=_dedupe(record.anchor_concepts, max_items=8),
|
| salience=float(record.salience),
|
| confidence=float(record.confidence),
|
| source_kind=_clean_text(record.source_kind) or "session_memory",
|
| turn_index=int(record.turn_index),
|
| state="active",
|
| metadata=metadata,
|
| )
|
| )
|
| return results
|
|
|
|
|
| def _build_turn_records(
|
| extractor: SessionMemoryExtractor,
|
| *,
|
| user_text: str,
|
| answer_payload: Dict[str, Any] | None,
|
| extraction_result: Dict[str, Any] | None,
|
| turn_index: int,
|
| allow_auto_extract: bool,
|
| profile: TMCRAProfile | None = None,
|
| ) -> List[SessionMemoryRecordV2]:
|
| profile = profile or TMCRAProfile()
|
| structured_records = _parse_structured_records(answer_payload, turn_index=turn_index, profile=profile)
|
| if structured_records:
|
| return _apply_typed_tunnel_annotations(
|
| _apply_turn_write_intent(structured_records, user_text=user_text),
|
| source_text=user_text,
|
| )
|
| structured_records = _parse_structured_records(extraction_result, turn_index=turn_index, profile=profile)
|
| if structured_records:
|
| return _apply_typed_tunnel_annotations(
|
| _apply_turn_write_intent(structured_records, user_text=user_text),
|
| source_text=user_text,
|
| )
|
| if not allow_auto_extract and not _looks_like_write_turn(user_text, answer_payload=answer_payload):
|
| return []
|
| return _apply_typed_tunnel_annotations(
|
| _records_from_extractor(
|
| extractor,
|
| query=user_text,
|
| answer_payload=None,
|
| extraction_result=extraction_result,
|
| turn_index=turn_index,
|
| profile=profile,
|
| ),
|
| source_text=user_text,
|
| )
|
|
|
|
|
| class NullMemoryAdapter(MemoryAdapter):
|
| name = "null_memory"
|
|
|
| def reset(self) -> None:
|
| return None
|
|
|
| def ingest_turn(
|
| self,
|
| user_text: str,
|
| assistant_text: str = "",
|
| *,
|
| answer_payload: Dict[str, Any] | None = None,
|
| extraction_result: Dict[str, Any] | None = None,
|
| ) -> None:
|
| _ = user_text, assistant_text, answer_payload, extraction_result
|
|
|
| def retrieve(self, query: str, *, top_k: int = 6) -> MemoryRetrieval:
|
| _ = query, top_k
|
| return MemoryRetrieval()
|
|
|
| def stats(self) -> Dict[str, Any]:
|
| return _state_stats(storage_bytes=0, retrieval_context_tokens=0, total_state_tokens=0, records=0)
|
|
|
| def storage_bytes(self) -> int:
|
| return 0
|
|
|
| def build_prompt_context(self, query: str, *, top_k: int = 8) -> Dict[str, Any]:
|
| _ = top_k
|
| return {
|
| "mode": "null_memory",
|
| "query": query,
|
| "retrieval": MemoryRetrieval().to_dict(),
|
| "stats": self.stats(),
|
| "state": {},
|
| }
|
|
|
|
|
| class GraphSessionMemoryAdapter(MemoryAdapter):
|
| name = "graph_session_memory_v2"
|
|
|
| def __init__(
|
| self,
|
| *,
|
| auto_extract: bool = False,
|
| storage_backend: str = "sqlite",
|
| storage_path: str = "",
|
| scope_id: str = "",
|
| audit_retention: int = 256,
|
| lightweight_stats: bool = True,
|
| retrieval_mode: str = "heuristic",
|
| node_model_path: str = "",
|
| path_model_path: str = "",
|
| node_model_device: str = "",
|
| candidate_event_k: int = 24,
|
| support_path_k: int = 3,
|
| path_tunnel_rescue_k: int = 0,
|
| path_tunnel_rescue_score_floor: float = 0.0,
|
| path_tunnel_rescue_min_age: int = 0,
|
| path_tunnel_rescue_min_score_margin: float = 0.0,
|
| event_rerank_mode: str = "matrix",
|
| matrix_event_top_k: int = DEFAULT_MATRIX_EVENT_TOP_K,
|
| memory_router_mode: str = "",
|
| memory_router_threshold: float = _MEMORY_ROUTER_DEFAULT_THRESHOLD,
|
| memory_router_margin: float = _MEMORY_ROUTER_DEFAULT_MARGIN,
|
| injection_planner_mode: str = "",
|
| injection_planner_model_path: str = "",
|
| injection_planner_latest_path: str = "",
|
| injection_planner_device: str = "",
|
| injection_planner_selection_threshold: float = -1.0,
|
| injection_planner_row_threshold: float = -1.0,
|
| injection_planner_logic_threshold: float = -1.0,
|
| temporal_layer_mode: str = "",
|
| temporal_router_mode: str = "",
|
| temporal_router_dir: str = "",
|
| temporal_router_latest_path: str = "",
|
| temporal_router_device: str = "",
|
| ) -> None:
|
| prewarm_embedder_mode = (
|
| _normalize(os.getenv("TMCRA_EMBEDDER_INDEX_RECALL_MODE", ""))
|
| or _normalize(os.getenv("TMCRA_WRITE_EMBEDDER_INDEX_MODE", ""))
|
| )
|
| self._embedder_prewarm_metadata = _prewarm_embedder_dense_if_requested(mode=prewarm_embedder_mode)
|
| self.extractor = SessionMemoryExtractor()
|
| self.profile = TMCRAProfile()
|
| self.temporal_organizer = TemporalOrganizer()
|
| self.temporal_query_planner = TemporalQueryPlanner()
|
| self.timeline_evidence_builder = TimelineEvidencePackBuilder()
|
| self.auto_extract = bool(auto_extract)
|
| self.storage_backend = _normalize(storage_backend) or "sqlite"
|
| self.audit_retention = max(1, int(audit_retention))
|
| self.lightweight_stats = bool(lightweight_stats)
|
| self.scope_id = _clean_text(scope_id) or f"graph-session-{uuid.uuid4().hex}"
|
| self._store: SQLiteSessionMemoryStore | None = None
|
| self.storage_path = ""
|
| if self.storage_backend == "sqlite":
|
| resolved_storage_path = _clean_text(storage_path) or str((Path(tempfile.gettempdir()) / "tmcra_graph_session_memory.sqlite3").resolve())
|
| self._store = SQLiteSessionMemoryStore(resolved_storage_path, audit_retention=self.audit_retention)
|
| self.storage_path = str(self._store.storage_path)
|
| self.graph = self._store.load_graph(self.scope_id)
|
| elif self.storage_backend == "memory":
|
| self.storage_path = _clean_text(storage_path)
|
| self.graph = SessionMemoryGraphV2(
|
| audit_retention=self.audit_retention,
|
| persistence_backend="memory",
|
| persistence_path=self.storage_path,
|
| )
|
| else:
|
| raise ValueError(f"Unsupported storage backend: {self.storage_backend}")
|
| self._last_retrieval_context_tokens = 0
|
| self._last_writeback_summary: Dict[str, Any] = {}
|
| self.retrieval_mode = _normalize(retrieval_mode) or "heuristic"
|
| self.node_model_path = _clean_text(node_model_path)
|
| self.path_model_path = _clean_text(path_model_path)
|
| self.node_model_device = _clean_text(node_model_device)
|
| self.candidate_event_k = max(1, int(candidate_event_k))
|
| self.support_path_k = max(1, int(support_path_k))
|
| self.path_tunnel_rescue_k = max(0, int(path_tunnel_rescue_k or 0))
|
| self.path_tunnel_rescue_score_floor = max(0.0, float(path_tunnel_rescue_score_floor or 0.0))
|
| self.path_tunnel_rescue_min_age = max(0, int(path_tunnel_rescue_min_age or 0))
|
| self.path_tunnel_rescue_min_score_margin = max(0.0, float(path_tunnel_rescue_min_score_margin or 0.0))
|
| self.event_rerank_mode = _normalize(event_rerank_mode) or "matrix"
|
| self.matrix_event_top_k = max(1, int(matrix_event_top_k or DEFAULT_MATRIX_EVENT_TOP_K))
|
| self.write_embedder_index_mode = _normalize(os.getenv("TMCRA_WRITE_EMBEDDER_INDEX_MODE", ""))
|
| if not self.write_embedder_index_mode:
|
| self.write_embedder_index_mode = "off"
|
| try:
|
| self.write_embedder_index_max_terms = max(
|
| 8,
|
| int(os.getenv("TMCRA_WRITE_EMBEDDER_INDEX_MAX_TERMS", "96") or 96),
|
| )
|
| except (TypeError, ValueError):
|
| self.write_embedder_index_max_terms = 96
|
| env_embedder_recall_mode = _normalize(os.getenv("TMCRA_EMBEDDER_INDEX_RECALL_MODE", ""))
|
| self.embedder_index_recall_mode = env_embedder_recall_mode or self.write_embedder_index_mode
|
| try:
|
| self.embedder_index_recall_k = max(
|
| 0,
|
| int(os.getenv("TMCRA_EMBEDDER_INDEX_RECALL_K", "0") or 0),
|
| )
|
| except (TypeError, ValueError):
|
| self.embedder_index_recall_k = 0
|
| self.embedder_pre_recall_mode = _normalize(os.getenv("TMCRA_EMBEDDER_PRE_RECALL_MODE", ""))
|
| if not self.embedder_pre_recall_mode:
|
| self.embedder_pre_recall_mode = "off"
|
| try:
|
| self.embedder_pre_recall_k = max(
|
| 0,
|
| int(os.getenv("TMCRA_EMBEDDER_PRE_RECALL_K", "0") or 0),
|
| )
|
| except (TypeError, ValueError):
|
| self.embedder_pre_recall_k = 0
|
| self.embedder_fusion_mode = _normalize(os.getenv("TMCRA_EMBEDDER_FUSION_MODE", ""))
|
| try:
|
| self.embedder_fusion_weight = max(
|
| 0.0,
|
| float(os.getenv("TMCRA_EMBEDDER_FUSION_WEIGHT", "0.35") or 0.35),
|
| )
|
| except (TypeError, ValueError):
|
| self.embedder_fusion_weight = 0.35
|
| try:
|
| self.embedder_fusion_score_floor = max(
|
| 0.0,
|
| float(os.getenv("TMCRA_EMBEDDER_FUSION_SCORE_FLOOR", "0.62") or 0.62),
|
| )
|
| except (TypeError, ValueError):
|
| self.embedder_fusion_score_floor = 0.62
|
| try:
|
| self.embedder_fusion_top_k = max(
|
| 0,
|
| int(os.getenv("TMCRA_EMBEDDER_FUSION_TOP_K", "16") or 16),
|
| )
|
| except (TypeError, ValueError):
|
| self.embedder_fusion_top_k = 16
|
| try:
|
| self.embedder_fusion_select_k = max(
|
| 0,
|
| int(os.getenv("TMCRA_EMBEDDER_FUSION_SELECT_K", "4") or 4),
|
| )
|
| except (TypeError, ValueError):
|
| self.embedder_fusion_select_k = 4
|
| try:
|
| self.embedder_fusion_max_boost = max(
|
| 0.0,
|
| float(os.getenv("TMCRA_EMBEDDER_FUSION_MAX_BOOST", "0.42") or 0.42),
|
| )
|
| except (TypeError, ValueError):
|
| self.embedder_fusion_max_boost = 0.42
|
| env_router_mode = _clean_text(os.getenv("TMCRA_MEMORY_ROUTER_MODE", ""))
|
| self.memory_router_mode = _normalize(memory_router_mode) or _normalize(env_router_mode) or "observe"
|
| try:
|
| self.memory_router_threshold = float(
|
| os.getenv("TMCRA_MEMORY_ROUTER_THRESHOLD", "")
|
| or memory_router_threshold
|
| or _MEMORY_ROUTER_DEFAULT_THRESHOLD
|
| )
|
| except (TypeError, ValueError):
|
| self.memory_router_threshold = _MEMORY_ROUTER_DEFAULT_THRESHOLD
|
| try:
|
| self.memory_router_margin = float(
|
| os.getenv("TMCRA_MEMORY_ROUTER_MARGIN", "")
|
| or memory_router_margin
|
| or _MEMORY_ROUTER_DEFAULT_MARGIN
|
| )
|
| except (TypeError, ValueError):
|
| self.memory_router_margin = _MEMORY_ROUTER_DEFAULT_MARGIN
|
| self._loaded_node_scorer: LoadedNodeMemoryScorer | None = None
|
| self._node_scorer_error = ""
|
| env_planner_mode = _clean_text(os.getenv("TMCRA_INJECTION_PLANNER_MODE", ""))
|
| self.injection_planner_mode = _normalize(injection_planner_mode) or _normalize(env_planner_mode) or "observe"
|
| self.injection_planner_model_path = _clean_text(
|
| injection_planner_model_path or os.getenv("TMCRA_INJECTION_PLANNER_MODEL_PATH", "")
|
| )
|
| self.injection_planner_latest_path = _clean_text(
|
| injection_planner_latest_path or os.getenv("TMCRA_INJECTION_PLANNER_LATEST", "")
|
| )
|
| self.injection_planner_device = _clean_text(
|
| injection_planner_device or os.getenv("TMCRA_INJECTION_PLANNER_DEVICE", "")
|
| )
|
| self.injection_planner_selection_threshold_override = float(injection_planner_selection_threshold)
|
| self.injection_planner_row_threshold_override = float(injection_planner_row_threshold)
|
| self.injection_planner_logic_threshold_override = float(injection_planner_logic_threshold)
|
| self._loaded_injection_planner: Any | None = None
|
| self._injection_planner_config: Any | None = None
|
| self._injection_planner_payload: Dict[str, Any] = {}
|
| self._injection_planner_thresholds: Dict[str, float] = {}
|
| self._injection_planner_resolved_path = ""
|
| self._injection_planner_error = ""
|
| self._injection_planner_evidence_role_supported = False
|
| env_temporal_layer_mode = _clean_text(os.getenv("TMCRA_TEMPORAL_LAYER_MODE", ""))
|
| self.temporal_layer_mode = _normalize(temporal_layer_mode) or _normalize(env_temporal_layer_mode) or "observe"
|
| env_temporal_router_mode = _clean_text(os.getenv("TMCRA_TEMPORAL_ROUTER_MODE", ""))
|
| self.temporal_router_mode = _normalize(temporal_router_mode) or _normalize(env_temporal_router_mode) or "observe"
|
| self.temporal_router_dir = _clean_text(
|
| temporal_router_dir or os.getenv("TMCRA_TEMPORAL_ROUTER_DIR", "")
|
| )
|
| self.temporal_router_latest_path = _clean_text(
|
| temporal_router_latest_path
|
| or os.getenv("TMCRA_TEMPORAL_ROUTER_LATEST", "")
|
| or "models/temporal_router_v1_latest.txt"
|
| )
|
| self.temporal_router_device = _clean_text(
|
| temporal_router_device or os.getenv("TMCRA_TEMPORAL_ROUTER_DEVICE", "")
|
| ) or "cpu"
|
| self._loaded_temporal_router: Any | None = None
|
| self._temporal_router_resolved_dir = ""
|
| self._temporal_router_error = ""
|
|
|
| def _empty_graph(self) -> SessionMemoryGraphV2:
|
| return SessionMemoryGraphV2(
|
| audit_retention=self.audit_retention,
|
| persistence_backend=self.storage_backend,
|
| persistence_path=self.storage_path,
|
| )
|
|
|
| def _reload_graph(self) -> None:
|
| if self._store is not None:
|
| self.graph = self._store.load_graph(self.scope_id)
|
| else:
|
| self.graph.configure_persistence(
|
| backend=self.storage_backend,
|
| path=self.storage_path,
|
| audit_retention=self.audit_retention,
|
| )
|
|
|
| def _persist_graph(self) -> None:
|
| self.graph.configure_persistence(
|
| backend=self.storage_backend,
|
| path=self.storage_path,
|
| audit_retention=self.audit_retention,
|
| )
|
| if self._store is not None:
|
| self._store.save_graph(self.scope_id, self.graph)
|
|
|
| def replace_graph(self, graph: SessionMemoryGraphV2) -> None:
|
| self.graph = graph
|
| self._persist_graph()
|
|
|
| def _storage_breakdown(self) -> Dict[str, int]:
|
| core_payload = self.graph._core_payload()
|
| audit_payload_full = self.graph._audit_payload()
|
| if self.lightweight_stats:
|
| audit_token_payload = {
|
| "totals": dict(self.graph.audit_event_totals),
|
| "trimmed": dict(self.graph.audit_trimmed_counts),
|
| "retained": {
|
| "turn_log": len(self.graph.turn_log),
|
| "retrieval_log": len(self.graph.retrieval_log),
|
| "answer_support_log": len(self.graph.answer_support_log),
|
| },
|
| "audit_retention": int(self.graph.audit_retention),
|
| }
|
| else:
|
| audit_token_payload = audit_payload_full
|
| core_storage_bytes = len(json.dumps(core_payload, ensure_ascii=False).encode("utf-8"))
|
| audit_storage_bytes = len(json.dumps(audit_payload_full, ensure_ascii=False).encode("utf-8"))
|
| core_state_token_estimate = _estimate_tokens(json.dumps(core_payload, ensure_ascii=False))
|
| audit_state_token_estimate = _estimate_tokens(json.dumps(audit_token_payload, ensure_ascii=False))
|
| return {
|
| "core_storage_bytes": int(core_storage_bytes),
|
| "audit_storage_bytes": int(audit_storage_bytes),
|
| "storage_bytes": int(core_storage_bytes + audit_storage_bytes),
|
| "core_state_token_estimate": int(core_state_token_estimate),
|
| "audit_state_token_estimate": int(audit_state_token_estimate),
|
| "total_state_token_estimate": int(core_state_token_estimate + audit_state_token_estimate),
|
| }
|
|
|
| def reset(self) -> None:
|
| if self._store is not None:
|
| self._store.clear_scope(self.scope_id)
|
| self.graph = self._store.load_graph(self.scope_id)
|
| else:
|
| self.graph = self._empty_graph()
|
| self._last_retrieval_context_tokens = 0
|
| self._last_writeback_summary = {}
|
|
|
| def _temporal_layer_enabled(self) -> bool:
|
| return _normalize(self.temporal_layer_mode) not in _TEMPORAL_LAYER_DISABLED_MODES
|
|
|
| def _temporal_router_enabled(self) -> bool:
|
| return self._temporal_layer_enabled() and _normalize(self.temporal_router_mode) not in _TEMPORAL_LAYER_DISABLED_MODES
|
|
|
| def _resolve_temporal_router_dir(self) -> str:
|
| direct_dir = _clean_text(self.temporal_router_dir)
|
| if direct_dir:
|
| root = Path(direct_dir)
|
| if (root / "writer_temporal_router.pt").exists() and (root / "query_temporal_router.pt").exists():
|
| return str(root)
|
| latest_path = _clean_text(self.temporal_router_latest_path)
|
| if latest_path:
|
| pointer = Path(latest_path)
|
| if pointer.exists():
|
| lines = pointer.read_text(encoding="utf-8").splitlines()
|
| candidate = _clean_text(lines[0] if lines else "")
|
| if candidate:
|
| root = Path(candidate)
|
| if (root / "writer_temporal_router.pt").exists() and (root / "query_temporal_router.pt").exists():
|
| return str(root)
|
| return ""
|
|
|
| def _load_temporal_router(self) -> LoadedTemporalRouter | None:
|
| if not self._temporal_router_enabled():
|
| self._temporal_router_error = "disabled"
|
| return None
|
| model_dir = self._resolve_temporal_router_dir()
|
| self._temporal_router_resolved_dir = model_dir
|
| if not model_dir:
|
| self._temporal_router_error = "model_dir_missing"
|
| return None
|
| if self._loaded_temporal_router is not None:
|
| return self._loaded_temporal_router
|
| try:
|
| self._loaded_temporal_router = LoadedTemporalRouter.from_dir(
|
| model_dir,
|
| device=self.temporal_router_device or "cpu",
|
| )
|
| self._temporal_router_error = ""
|
| except Exception as exc:
|
| self._loaded_temporal_router = None
|
| self._temporal_router_error = f"{type(exc).__name__}: {exc}"
|
| return self._loaded_temporal_router
|
|
|
| def _temporal_router_status_metadata(self) -> Dict[str, Any]:
|
| router = self._load_temporal_router()
|
| return {
|
| "temporal_router_enabled": router is not None,
|
| "temporal_router_mode": _normalize(self.temporal_router_mode) or "observe",
|
| "temporal_router_model_dir": self._temporal_router_resolved_dir,
|
| "temporal_router_error": self._temporal_router_error,
|
| }
|
|
|
| def _session_timestamp_from_payloads(self, *payloads: Mapping[str, Any] | None) -> str:
|
| for payload in payloads:
|
| data = dict(payload or {})
|
| for key in ("session_timestamp", "timestamp", "created_at", "turn_timestamp"):
|
| value = _clean_text(data.get(key, ""))
|
| if value:
|
| return value
|
| metadata = data.get("metadata")
|
| if isinstance(metadata, Mapping):
|
| for key in ("session_timestamp", "timestamp", "created_at", "turn_timestamp"):
|
| value = _clean_text(metadata.get(key, ""))
|
| if value:
|
| return value
|
| return ""
|
|
|
| def _temporal_frame_for_turn(
|
| self,
|
| *,
|
| user_text: str,
|
| previous_turn: str = "",
|
| answer_payload: Dict[str, Any] | None = None,
|
| extraction_result: Dict[str, Any] | None = None,
|
| ) -> TemporalFrame | None:
|
| if not self._temporal_layer_enabled():
|
| return None
|
| session_timestamp = self._session_timestamp_from_payloads(answer_payload, extraction_result)
|
| model_frame = None
|
| for payload in (answer_payload, extraction_result):
|
| data = dict(payload or {})
|
| candidate = data.get("temporal_frame") or dict(data.get("metadata", {}) or {}).get("temporal_frame")
|
| if isinstance(candidate, Mapping):
|
| model_frame = candidate
|
| break
|
| fallback_frame = self.temporal_organizer.organize_turn(
|
| current_turn=user_text,
|
| previous_turn=previous_turn,
|
| session_timestamp=session_timestamp,
|
| speaker="user",
|
| )
|
| if model_frame is None:
|
| router = self._load_temporal_router()
|
| if router is not None and router.writer_available():
|
| predicted = router.predict_writer_frame(
|
| current_turn=user_text,
|
| previous_turn=previous_turn,
|
| session_timestamp=session_timestamp,
|
| )
|
| writer_confidence = float(predicted.get("confidence", 0.0) or 0.0) if predicted else 0.0
|
| if predicted and writer_confidence >= _float_env("TMCRA_TEMPORAL_ROUTER_WRITER_MIN_CONFIDENCE", _TEMPORAL_ROUTER_DEFAULT_WRITER_MIN_CONFIDENCE):
|
| frame_payload = fallback_frame.to_dict()
|
| frame_payload.update(
|
| {
|
| key: value
|
| for key, value in predicted.items()
|
| if key in {"temporal_intent", "anchor_type", "granularity", "state_operation"}
|
| and _clean_text(value)
|
| }
|
| )
|
| if "should_create_timeline_edge" in predicted:
|
| frame_payload["should_create_timeline_edge"] = bool(predicted.get("should_create_timeline_edge", False))
|
| if writer_confidence > 0.0:
|
| frame_payload["confidence"] = writer_confidence
|
| frame_payload["metadata"] = {
|
| **dict(frame_payload.get("metadata", {}) or {}),
|
| **dict(predicted.get("metadata", {}) or {}),
|
| **self._temporal_router_status_metadata(),
|
| }
|
| model_frame = frame_payload
|
| if model_frame is None:
|
| return fallback_frame
|
| return self.temporal_organizer.organize_turn(
|
| current_turn=user_text,
|
| previous_turn=previous_turn,
|
| session_timestamp=session_timestamp,
|
| speaker="user",
|
| model_frame=model_frame,
|
| )
|
|
|
| def _temporal_turn_metadata(self, frame: TemporalFrame | None) -> Dict[str, Any]:
|
| if frame is None:
|
| return {
|
| "temporal_layer_enabled": False,
|
| "temporal_layer_mode": _normalize(self.temporal_layer_mode) or "observe",
|
| **self._temporal_router_status_metadata(),
|
| }
|
| return {
|
| "temporal_layer_enabled": True,
|
| "temporal_layer_mode": _normalize(self.temporal_layer_mode) or "observe",
|
| "temporal_frame": frame.to_dict(),
|
| "temporal_intent": frame.temporal_intent,
|
| "temporal_subject_key": frame.subject_key,
|
| "temporal_state_operation": frame.state_operation,
|
| **self._temporal_router_status_metadata(),
|
| }
|
|
|
| def _apply_temporal_frame_to_records(self, records: Sequence[SessionMemoryRecordV2], frame: TemporalFrame | None) -> None:
|
| if frame is None:
|
| return
|
| if frame.temporal_intent == "non_temporal" and not frame.subject_key:
|
| return
|
| for record in records:
|
| metadata = dict(record.metadata or {})
|
| metadata.update(
|
| {
|
| "temporal_layer": True,
|
| "temporal_frame": frame.to_dict(),
|
| "temporal_intent": frame.temporal_intent,
|
| "temporal_subject": frame.subject,
|
| "temporal_subject_key": frame.subject_key,
|
| "temporal_state_operation": frame.state_operation,
|
| "temporal_event_time": frame.event_time,
|
| "temporal_state_valid_from": frame.state_valid_from,
|
| "temporal_state_valid_to": frame.state_valid_to,
|
| }
|
| )
|
| record.metadata = metadata
|
|
|
| def _build_timeline_state_layer(self) -> TimelineStateLayer:
|
| layer = TimelineStateLayer()
|
| for turn in sorted(list(getattr(self.graph, "turn_log", []) or []), key=lambda item: int(item.get("turn_index", 0) or 0)):
|
| metadata = dict(turn.get("metadata", {}) or {})
|
| frame_payload = metadata.get("temporal_frame")
|
| if not isinstance(frame_payload, Mapping):
|
| continue
|
| frame = TemporalFrame.from_mapping(frame_payload)
|
| if frame.temporal_intent == "non_temporal" and not frame.new_state:
|
| continue
|
| record_ids = [item for item in list(turn.get("record_ids", []) or []) if _clean_text(item)]
|
| source_event_id = _clean_text(metadata.get("temporal_source_record_id", "")) or (record_ids[0] if record_ids else "")
|
| frame.metadata = {
|
| **dict(frame.metadata or {}),
|
| "source_text": _clean_text(turn.get("text", "")) or frame.evidence_span,
|
| }
|
| layer.apply_frame(
|
| frame,
|
| source_event_id=source_event_id,
|
| source_turn_id=_clean_text(turn.get("turn_id", "")),
|
| state_type=_clean_text(metadata.get("temporal_state_type", "")) or "profile",
|
| )
|
| return layer
|
|
|
| def _temporal_runtime_pack(self, query: str) -> Dict[str, Any]:
|
| metadata: Dict[str, Any] = {
|
| "temporal_runtime_enabled": False,
|
| "temporal_layer_mode": _normalize(self.temporal_layer_mode) or "observe",
|
| **self._temporal_router_status_metadata(),
|
| }
|
| if not self._temporal_layer_enabled():
|
| metadata["temporal_runtime_reason"] = "disabled"
|
| return {"metadata": metadata}
|
| plan = self.temporal_query_planner.plan(query)
|
| router = self._load_temporal_router()
|
| if router is not None and router.query_available():
|
| predicted_plan = router.predict_query_plan(query=query)
|
| query_confidence = float(predicted_plan.get("confidence", 0.0) or 0.0) if predicted_plan else 0.0
|
| router_confidences = dict(dict(predicted_plan.get("metadata", {}) or {}).get("temporal_router_confidences", {}) or {}) if predicted_plan else {}
|
| intent_confidence = float(router_confidences.get("query_temporal_intent", 0.0) or 0.0)
|
| if (
|
| predicted_plan
|
| and query_confidence >= _float_env("TMCRA_TEMPORAL_ROUTER_QUERY_MIN_CONFIDENCE", _TEMPORAL_ROUTER_DEFAULT_QUERY_MIN_CONFIDENCE)
|
| and intent_confidence >= _float_env("TMCRA_TEMPORAL_ROUTER_QUERY_INTENT_MIN_CONFIDENCE", _TEMPORAL_ROUTER_DEFAULT_QUERY_INTENT_MIN_CONFIDENCE)
|
| ):
|
| plan_payload = plan.to_dict()
|
| plan_payload.update(
|
| {
|
| key: value
|
| for key, value in predicted_plan.items()
|
| if key in {"query_temporal_intent", "timeline_operation"} and _clean_text(value)
|
| }
|
| )
|
| for key in ("prefer_current_state", "prefer_previous_state", "requires_ordered_chain", "requires_comparison"):
|
| if key in predicted_plan:
|
| plan_payload[key] = bool(predicted_plan.get(key, False))
|
| if float(predicted_plan.get("confidence", 0.0) or 0.0) > 0.0:
|
| plan_payload["confidence"] = float(predicted_plan.get("confidence", 0.0) or 0.0)
|
| plan_payload["metadata"] = {
|
| **dict(plan_payload.get("metadata", {}) or {}),
|
| **dict(predicted_plan.get("metadata", {}) or {}),
|
| **self._temporal_router_status_metadata(),
|
| }
|
| plan = TemporalQueryPlan(**{key: value for key, value in plan_payload.items() if key in TemporalQueryPlan.__dataclass_fields__})
|
| metadata["temporal_query_plan"] = plan.to_dict()
|
| if plan.query_temporal_intent == "non_temporal" or plan.timeline_operation == "none":
|
| metadata["temporal_runtime_reason"] = "non_temporal_query"
|
| return {"plan": plan, "metadata": metadata}
|
| if plan.timeline_operation in {"query_current", "query_previous"} and not _clean_text(plan.target_subject_key):
|
| metadata["temporal_runtime_reason"] = "missing_target_subject"
|
| return {"plan": plan, "metadata": metadata}
|
| timeline_layer = self._build_timeline_state_layer()
|
| pack = self.timeline_evidence_builder.build(plan=plan, timeline_layer=timeline_layer)
|
| metadata.update(
|
| {
|
| "temporal_runtime_enabled": True,
|
| "temporal_runtime_reason": "ok",
|
| "temporal_evidence_pack": pack.to_dict(),
|
| "temporal_selected_answer_value": _clean_text(pack.selected_evidence.get("answer_value", "")),
|
| "temporal_selected_state_id": _clean_text(pack.selected_evidence.get("state_id", "")),
|
| "temporal_timeline_state_count": len(pack.timeline),
|
| }
|
| )
|
| return {"plan": plan, "pack": pack, "metadata": metadata}
|
|
|
| def _temporal_state_hit(
|
| self,
|
| *,
|
| state_payload: Mapping[str, Any],
|
| plan_payload: Mapping[str, Any],
|
| selected: bool,
|
| rank: int,
|
| ) -> MemoryHit | None:
|
| state_value = _clean_text(state_payload.get("state", ""))
|
| state_id = _clean_text(state_payload.get("state_id", ""))
|
| if not state_value or not state_id:
|
| return None
|
| source_event_id = _clean_text(state_payload.get("source_event_id", ""))
|
| source_record = self.graph.records_by_id.get(source_event_id)
|
| source_metadata = dict(source_record.metadata or {}) if source_record is not None else {}
|
| score = 0.98 if selected else max(0.75, 0.92 - (rank * 0.03))
|
| return MemoryHit(
|
| memory_id=f"temporal_state:{state_id}",
|
| category="time",
|
| value=state_value,
|
| relation="temporal_state",
|
| anchors=_dedupe([state_payload.get("time", ""), state_payload.get("source_text", ""), plan_payload.get("target_subject", "")], max_items=6),
|
| score=round(score, 6),
|
| source_kind="temporal_state_layer",
|
| slot_key=f"temporal.{_clean_text(plan_payload.get('target_subject_key', 'general'))}",
|
| state="active" if bool(state_payload.get("is_current", False)) else "history",
|
| turn_index=int(source_record.turn_index) if source_record is not None else 0,
|
| metadata={
|
| **source_metadata,
|
| "temporal_runtime_hit": True,
|
| "temporal_runtime_selected": bool(selected),
|
| "temporal_state_id": state_id,
|
| "temporal_state_value": state_value,
|
| "temporal_state_time": _clean_text(state_payload.get("time", "")),
|
| "temporal_state_valid_to": _clean_text(state_payload.get("valid_to", "")),
|
| "temporal_state_is_current": bool(state_payload.get("is_current", False)),
|
| "temporal_source_event_id": source_event_id,
|
| "temporal_source_turn_id": _clean_text(state_payload.get("source_turn_id", "")),
|
| "temporal_query_plan": dict(plan_payload),
|
| },
|
| )
|
|
|
| def _apply_temporal_evidence_pack_to_hits(
|
| self,
|
| hits: Sequence[MemoryHit],
|
| temporal_payload: Mapping[str, Any],
|
| *,
|
| top_k: int,
|
| ) -> Dict[str, Any]:
|
| metadata = dict(temporal_payload.get("metadata", {}) or {})
|
| pack = temporal_payload.get("pack")
|
| plan = temporal_payload.get("plan")
|
| if pack is None or plan is None or not bool(metadata.get("temporal_runtime_enabled", False)):
|
| return {"hits": list(hits), "metadata": metadata}
|
| pack_payload = pack.to_dict() if hasattr(pack, "to_dict") else dict(pack)
|
| plan_payload = plan.to_dict() if hasattr(plan, "to_dict") else dict(plan)
|
| selected_state_id = _clean_text(dict(pack_payload.get("selected_evidence", {}) or {}).get("state_id", ""))
|
| selected_answer_value = _clean_text(dict(pack_payload.get("selected_evidence", {}) or {}).get("answer_value", ""))
|
| timeline = list(pack_payload.get("timeline", []) or [])
|
| synthetic_hits: List[MemoryHit] = []
|
| if selected_state_id:
|
| selected_state = next((dict(item) for item in timeline if _clean_text(dict(item).get("state_id", "")) == selected_state_id), None)
|
| if selected_state is None and selected_answer_value:
|
| selected_state = {
|
| "state_id": selected_state_id,
|
| "state": selected_answer_value,
|
| "source_event_id": dict(pack_payload.get("selected_evidence", {}) or {}).get("source_event_id", ""),
|
| "source_turn_id": dict(pack_payload.get("selected_evidence", {}) or {}).get("source_turn_id", ""),
|
| "is_current": bool(plan_payload.get("prefer_current_state", False)),
|
| }
|
| if selected_state:
|
| hit = self._temporal_state_hit(state_payload=selected_state, plan_payload=plan_payload, selected=True, rank=0)
|
| if hit is not None:
|
| synthetic_hits.append(hit)
|
| if bool(plan_payload.get("requires_ordered_chain", False)) or bool(plan_payload.get("requires_comparison", False)):
|
| for index, state_payload in enumerate(timeline):
|
| if _clean_text(dict(state_payload).get("state_id", "")) == selected_state_id:
|
| continue
|
| hit = self._temporal_state_hit(
|
| state_payload=dict(state_payload),
|
| plan_payload=plan_payload,
|
| selected=False,
|
| rank=index + 1,
|
| )
|
| if hit is not None:
|
| synthetic_hits.append(hit)
|
| existing: List[MemoryHit] = []
|
| synthetic_source_ids = {
|
| _clean_text((hit.metadata or {}).get("temporal_source_event_id", ""))
|
| for hit in synthetic_hits
|
| if _clean_text((hit.metadata or {}).get("temporal_source_event_id", ""))
|
| }
|
| for hit in hits:
|
| hit_metadata = dict(hit.metadata or {})
|
| source_event_id = _clean_text(hit_metadata.get("event_id", hit.memory_id))
|
| if source_event_id in synthetic_source_ids:
|
| hit_metadata.update(
|
| {
|
| "temporal_runtime_support": True,
|
| "temporal_query_plan": dict(plan_payload),
|
| "temporal_selected_answer_value": selected_answer_value,
|
| "temporal_selected_state_id": selected_state_id,
|
| }
|
| )
|
| hit = MemoryHit(
|
| memory_id=hit.memory_id,
|
| category=hit.category,
|
| value=hit.value,
|
| relation=hit.relation,
|
| anchors=list(hit.anchors),
|
| score=hit.score,
|
| source_kind=hit.source_kind,
|
| slot_key=hit.slot_key,
|
| state=hit.state,
|
| turn_index=int(hit.turn_index),
|
| metadata=hit_metadata,
|
| )
|
| existing.append(hit)
|
| merged: List[MemoryHit] = []
|
| seen = set()
|
| for hit in [*synthetic_hits, *existing]:
|
| key = _clean_text(hit.memory_id)
|
| if key and key in seen:
|
| continue
|
| if key:
|
| seen.add(key)
|
| merged.append(hit)
|
| metadata.update(
|
| {
|
| "temporal_runtime_injected_hit_count": len(synthetic_hits),
|
| "temporal_runtime_injected_hit_ids": [hit.memory_id for hit in synthetic_hits],
|
| }
|
| )
|
| return {"hits": merged[: max(len(merged), top_k)], "metadata": metadata}
|
|
|
| def ingest_turn(
|
| self,
|
| user_text: str,
|
| assistant_text: str = "",
|
| *,
|
| answer_payload: Dict[str, Any] | None = None,
|
| extraction_result: Dict[str, Any] | None = None,
|
| ) -> None:
|
| self._reload_graph()
|
| turn_index = self.graph.next_turn()
|
| previous_topic = _last_topic_turn(self.graph)
|
| previous_turn_text = _clean_text(self.graph.turn_log[-1].get("text", "")) if self.graph.turn_log else ""
|
| temporal_frame = self._temporal_frame_for_turn(
|
| user_text=user_text,
|
| previous_turn=previous_turn_text,
|
| answer_payload=answer_payload,
|
| extraction_result=extraction_result,
|
| )
|
| topic_bucket = _assign_topic_bucket_for_text(
|
| self.graph,
|
| user_text,
|
| answer_payload=answer_payload,
|
| turn_index=turn_index,
|
| create=True,
|
| )
|
| records = _build_turn_records(
|
| self.extractor,
|
| user_text=user_text,
|
| answer_payload=answer_payload,
|
| extraction_result=extraction_result,
|
| turn_index=turn_index,
|
| allow_auto_extract=self.auto_extract,
|
| profile=self.profile,
|
| )
|
| self._apply_temporal_frame_to_records(records, temporal_frame)
|
| _apply_topic_bucket_to_records(records, topic_bucket)
|
| stored_ids = self.graph.add_records(records)
|
| write_embedder_metadata = _apply_write_embedder_index_to_graph(
|
| self.graph,
|
| stored_ids=stored_ids,
|
| turn_text=user_text,
|
| turn_index=turn_index,
|
| mode=self.write_embedder_index_mode,
|
| max_terms=self.write_embedder_index_max_terms,
|
| )
|
| topic_bridge_metadata = _add_topic_bridge_edges(
|
| self.graph,
|
| previous_topic=previous_topic,
|
| current_topic=topic_bucket,
|
| current_record_ids=stored_ids,
|
| turn_index=turn_index,
|
| evidence=user_text,
|
| )
|
| dialogue_tunnel_metadata = _add_dialogue_tunnel_edges(
|
| self.graph,
|
| current_topic=topic_bucket,
|
| current_record_ids=stored_ids,
|
| turn_index=turn_index,
|
| evidence=user_text,
|
| )
|
| turn_kind = "memory_write" if stored_ids else "noise"
|
| self.graph.record_turn(
|
| turn_kind=turn_kind,
|
| text=user_text,
|
| turn_index=turn_index,
|
| record_ids=stored_ids,
|
| speaker="user",
|
| assistant_text=assistant_text,
|
| metadata={
|
| "source": "user_turn",
|
| "auto_extract": bool(self.auto_extract),
|
| "topic_bucket_id": _clean_text(topic_bucket.get("topic_bucket_id", "")),
|
| "topic_label": _clean_text(topic_bucket.get("topic_label", "")),
|
| "topic_keywords": list(topic_bucket.get("topic_keywords", []) or []),
|
| "temporal_source_record_id": stored_ids[0] if stored_ids else "",
|
| **self._temporal_turn_metadata(temporal_frame),
|
| **write_embedder_metadata,
|
| **topic_bridge_metadata,
|
| **dialogue_tunnel_metadata,
|
| },
|
| )
|
| self._persist_graph()
|
|
|
| def ingest_answer_writeback(
|
| self,
|
| *,
|
| query_text: str,
|
| answer_text: str,
|
| answer_id: str,
|
| writeback_records: List[Dict[str, Any]],
|
| trace: Dict[str, Any] | None = None,
|
| ) -> List[str]:
|
| self._reload_graph()
|
| if not writeback_records:
|
| self._last_writeback_summary = {"stored_record_ids": [], "promotion_events": []}
|
| return []
|
| turn_index = self.graph.next_turn()
|
| records: List[SessionMemoryRecordV2] = []
|
| writeback_classes: List[str] = []
|
| for index, raw in enumerate(writeback_records):
|
| if not isinstance(raw, dict):
|
| continue
|
| category = _clean_text(raw.get("category", "fact")) or "fact"
|
| value = _clean_text(raw.get("value", ""))
|
| raw_slot_key = _clean_text(raw.get("slot_key", "")) or _clean_text(raw.get("slot", ""))
|
| anchors = _dedupe(raw.get("anchors", []) or [], max_items=8)
|
| slot_key = self.profile.stable_slot_key(
|
| category=category,
|
| value=value,
|
| anchors=anchors,
|
| slot_key=raw_slot_key,
|
| relation=_clean_text(raw.get("relation", "")),
|
| metadata=dict(raw.get("metadata", {}) or {}),
|
| )
|
| if not value or not slot_key:
|
| continue
|
| raw_metadata = dict(raw.get("metadata", {}) or {})
|
| writeback_class = _clean_text(raw_metadata.get("writeback_class", "")) or "fact"
|
| writeback_classes.append(writeback_class)
|
| metadata = {
|
| **raw_metadata,
|
| "memory_role": _clean_text(raw_metadata.get("memory_role", "")) or "assistant",
|
| "authority": _clean_text(raw_metadata.get("authority", "")) or "derived",
|
| "canonical_slot_key": _clean_text(raw_metadata.get("canonical_slot_key", "")) or slot_key.removeprefix("assistant.").split(".fact", 1)[0].split(".state_change", 1)[0].split(".high_conf_conclusion", 1)[0],
|
| "writeback_class": writeback_class,
|
| "origin_query": _clean_text(raw_metadata.get("origin_query", "")) or _clean_text(query_text),
|
| "origin_answer_id": _clean_text(raw_metadata.get("origin_answer_id", "")) or answer_id,
|
| "origin_answer_ids": _dedupe([*(raw_metadata.get("origin_answer_ids", []) or []), _clean_text(raw_metadata.get("origin_answer_id", "")) or answer_id]),
|
| "support_memory_ids": _dedupe(raw_metadata.get("support_memory_ids", []) or []),
|
| "support_fact_refs": _dedupe(raw_metadata.get("support_fact_refs", []) or []),
|
| "support_path_refs": _dedupe(raw_metadata.get("support_path_refs", []) or []),
|
| "promotion_state": _clean_text(raw_metadata.get("promotion_state", "")) or "candidate",
|
| "answer_id": answer_id,
|
| }
|
| record = SessionMemoryRecordV2(
|
| memory_id=f"{slot_key}:{turn_index}:assistant:{index}",
|
| category=category,
|
| slot_key=slot_key,
|
| value=value,
|
| relation=_clean_text(raw.get("relation", "")) or "assistant_memory",
|
| anchor_concepts=anchors,
|
| evidence_anchors=anchors,
|
| salience=float(raw.get("salience", 0.62) or 0.62),
|
| confidence=float(raw.get("confidence", 0.88) or 0.88),
|
| source_kind=_clean_text(raw.get("source_kind", "")) or "assistant_writeback",
|
| turn_index=turn_index,
|
| state=_clean_text(raw.get("state", "")) or "active",
|
| metadata=metadata,
|
| )
|
| records.append(record)
|
| stored_ids = self.graph.add_records(records)
|
| write_embedder_metadata = _apply_write_embedder_index_to_graph(
|
| self.graph,
|
| stored_ids=stored_ids,
|
| turn_text=" ".join([_clean_text(query_text), _clean_text(answer_text)]),
|
| turn_index=turn_index,
|
| mode=self.write_embedder_index_mode,
|
| max_terms=self.write_embedder_index_max_terms,
|
| )
|
| promotion_events = self._apply_writeback_promotions(stored_ids)
|
| writeback_class = writeback_classes[0] if len(set(writeback_classes)) == 1 and writeback_classes else ("mixed" if writeback_classes else "")
|
| self.graph.record_turn(
|
| turn_kind="assistant_writeback" if stored_ids else "assistant_writeback_empty",
|
| text=query_text,
|
| turn_index=turn_index,
|
| record_ids=stored_ids,
|
| speaker="assistant",
|
| assistant_text=answer_text,
|
| writeback_class=writeback_class,
|
| metadata={
|
| "source": "assistant_writeback",
|
| "answer_id": answer_id,
|
| "trace": dict(trace or {}),
|
| **write_embedder_metadata,
|
| },
|
| )
|
| self._last_writeback_summary = {
|
| "stored_record_ids": list(stored_ids),
|
| "promotion_events": list(promotion_events),
|
| **write_embedder_metadata,
|
| }
|
| self._persist_graph()
|
| return stored_ids
|
|
|
| def last_writeback_summary(self) -> Dict[str, Any]:
|
| return dict(self._last_writeback_summary)
|
|
|
| def _resolve_injection_planner_model_path(self) -> str:
|
| explicit_path = _clean_text(self.injection_planner_model_path)
|
| if explicit_path:
|
| path = Path(explicit_path).expanduser()
|
| if path.is_dir():
|
| path = path / "injection_planner.pt"
|
| return str(path)
|
| latest_path = _clean_text(self.injection_planner_latest_path)
|
| if not latest_path:
|
| return ""
|
| latest = Path(latest_path).expanduser()
|
| if latest.is_dir():
|
| return str(latest / "injection_planner.pt")
|
| if latest.suffix == ".pt":
|
| return str(latest)
|
| try:
|
| target = Path(latest.read_text(encoding="utf-8").strip()).expanduser()
|
| except Exception as exc:
|
| self._injection_planner_error = f"latest_pointer_read_failed: {exc}"
|
| return ""
|
| if target.is_dir():
|
| target = target / "injection_planner.pt"
|
| return str(target)
|
|
|
| def _load_injection_planner(self) -> Any | None:
|
| normalized_mode = _normalize(self.injection_planner_mode)
|
| if normalized_mode in _INJECTION_PLANNER_DISABLED_MODES:
|
| self._injection_planner_error = "disabled"
|
| return None
|
| if self._loaded_injection_planner is not None:
|
| return self._loaded_injection_planner
|
| torch_module = getattr(injection_planner_runtime, "torch", None)
|
| if torch_module is None:
|
| self._injection_planner_error = "torch_unavailable"
|
| return None
|
| model_path_text = self._resolve_injection_planner_model_path()
|
| if not model_path_text:
|
| if not self._injection_planner_error:
|
| self._injection_planner_error = "model_path_missing"
|
| return None
|
| model_path = Path(model_path_text)
|
| if not model_path.exists():
|
| self._injection_planner_error = f"model_path_not_found: {model_path}"
|
| return None
|
| try:
|
| device = torch_module.device(self.injection_planner_device or "cpu")
|
| payload = torch_module.load(model_path, map_location=device, weights_only=True) |
| config = injection_planner_runtime.InjectionPlannerConfig.from_dict(dict(payload.get("config", {}) or {}))
|
| model = injection_planner_runtime.InjectionPlannerModel(config).to(device)
|
| state_dict = dict(payload.get("state_dict", {}) or {})
|
| model.load_state_dict(state_dict, strict=False)
|
| model.eval()
|
| self._loaded_injection_planner = model
|
| self._injection_planner_config = config
|
| self._injection_planner_payload = dict(payload)
|
| self._injection_planner_resolved_path = str(model_path)
|
| self._injection_planner_error = ""
|
| self._injection_planner_evidence_role_supported = any(
|
| str(key).startswith("evidence_role_head.") for key in state_dict
|
| )
|
| thresholds = {"selection_threshold": 0.5, "row_threshold": 0.5, "logic_threshold": 0.5}
|
| summary_path = model_path.parent / "train_summary.json"
|
| if summary_path.exists():
|
| try:
|
| summary = json.loads(summary_path.read_text(encoding="utf-8"))
|
| calibration = dict(summary.get("calibration", {}) or {})
|
| logic_calibration = dict(summary.get("logic_calibration", {}) or {})
|
| if calibration.get("selection_threshold") is not None:
|
| thresholds["selection_threshold"] = float(calibration.get("selection_threshold"))
|
| if calibration.get("row_threshold") is not None:
|
| thresholds["row_threshold"] = float(calibration.get("row_threshold"))
|
| if logic_calibration.get("logic_threshold") is not None:
|
| thresholds["logic_threshold"] = float(logic_calibration.get("logic_threshold"))
|
| except Exception:
|
| pass
|
| if self.injection_planner_selection_threshold_override >= 0.0:
|
| thresholds["selection_threshold"] = float(self.injection_planner_selection_threshold_override)
|
| if self.injection_planner_row_threshold_override >= 0.0:
|
| thresholds["row_threshold"] = float(self.injection_planner_row_threshold_override)
|
| if self.injection_planner_logic_threshold_override >= 0.0:
|
| thresholds["logic_threshold"] = float(self.injection_planner_logic_threshold_override)
|
| self._injection_planner_thresholds = thresholds
|
| except Exception as exc:
|
| self._loaded_injection_planner = None
|
| self._injection_planner_config = None
|
| self._injection_planner_payload = {}
|
| self._injection_planner_resolved_path = str(model_path)
|
| self._injection_planner_error = str(exc)
|
| self._injection_planner_evidence_role_supported = False
|
| return self._loaded_injection_planner
|
|
|
| def _injection_planner_candidate_from_hit(
|
| self,
|
| query: str,
|
| hit: MemoryHit,
|
| *,
|
| index: int,
|
| current_turn_index: int,
|
| ) -> Dict[str, Any]:
|
| metadata = dict(hit.metadata or {})
|
| category = _normalize(hit.category)
|
| source_kind = _normalize(hit.source_kind)
|
| logic_roles = [
|
| _normalize(item)
|
| for item in (
|
| metadata.get("logic_roles", [])
|
| if isinstance(metadata.get("logic_roles", []), list)
|
| else [metadata.get("logic_roles", "")]
|
| )
|
| if _normalize(item) in set(injection_planner_runtime.LOGIC_ROLES)
|
| ]
|
| if category in set(injection_planner_runtime.LOGIC_ROLES):
|
| logic_roles.append(category)
|
| if _normalize(metadata.get("writeback_class", "")) in set(injection_planner_runtime.LOGIC_ROLES):
|
| logic_roles.append(_normalize(metadata.get("writeback_class", "")))
|
| if not logic_roles:
|
| logic_roles = ["negative"] if _normalize(hit.state) in {"stale", "superseded", "false"} else ["evidence"]
|
| evidence_snippet_role = _normalize(metadata.get("evidence_snippet_role", ""))
|
| if bool(metadata.get("profile_layer")) or category == "profile" or source_kind.endswith("profile"):
|
| layer = "profile"
|
| elif category == "time" or float(metadata.get("temporal_score", 0.0) or 0.0) > 0.0:
|
| layer = "temporal"
|
| elif "resource" in logic_roles or _clean_text(metadata.get("resource_key", "")):
|
| layer = "resource"
|
| elif (
|
| source_kind in {"path_tunnel", "path_support", "public_dialog_path"}
|
| or evidence_snippet_role in {"selected_path_support", "path_tunnel_support"}
|
| or bool(metadata.get("path_tunnel_node"))
|
| ):
|
| layer = "path_tunnel"
|
| elif bool(metadata.get("topic_bucket_rerank")) or bool(metadata.get("topic_bucket_dialogue_tunnel_allowed")):
|
| layer = "topic_tunnel"
|
| else:
|
| layer = "event"
|
| normalized_state = _normalize(hit.state)
|
| if normalized_state in {"stale", "superseded", "false"}:
|
| temporal_state = "superseded"
|
| elif bool(metadata.get("topic_bucket_current_subject_preserved")) or bool(metadata.get("current_subject_protected")):
|
| temporal_state = "current"
|
| elif current_turn_index and int(hit.turn_index or 0) and current_turn_index - int(hit.turn_index) <= 1:
|
| temporal_state = "current"
|
| elif normalized_state == "active":
|
| temporal_state = "stable"
|
| else:
|
| temporal_state = "historical"
|
| query_tokens = set(_tokenize(query))
|
| hit_tokens = set(_tokenize(" ".join([hit.value, " ".join(hit.anchors), _clean_text(metadata.get("topic_label", ""))])))
|
| overlap = len(query_tokens & hit_tokens) / max(1, len(query_tokens | hit_tokens))
|
| candidate_id = _clean_text(hit.memory_id) or f"hit:{index}"
|
| topic_label = _clean_text(metadata.get("topic_label", "")) or _clean_text(metadata.get("topic_bucket_query_label", ""))
|
| profile_key = _clean_text(metadata.get("profile_type", "")) or _clean_text(metadata.get("semantic_slot", ""))
|
| if not profile_key and layer in {"profile", "temporal", "topic_tunnel"}:
|
| profile_key = topic_label
|
| semantic_similarity = max(
|
| 0.0,
|
| min(
|
| 1.0,
|
| float(
|
| metadata.get(
|
| "semantic_similarity",
|
| metadata.get(
|
| "answer_window_semantic_similarity",
|
| metadata.get(
|
| "embedder_similarity",
|
| metadata.get("dense_similarity", metadata.get("bge_m3_similarity", hit.score)),
|
| ),
|
| ),
|
| )
|
| or 0.0
|
| ),
|
| ),
|
| )
|
| return {
|
| "id": candidate_id,
|
| "text": hit.value,
|
| "summary": _clean_text(metadata.get("source_turn_text", "")) or _clean_text(metadata.get("event_summary", "")),
|
| "topic": topic_label,
|
| "profile_key": profile_key,
|
| "resource_key": _clean_text(metadata.get("resource_key", "")) or (_clean_text(hit.slot_key) if layer == "resource" else ""),
|
| "layer": layer,
|
| "temporal_state": temporal_state,
|
| "logic_roles": _dedupe(logic_roles),
|
| "query_overlap": round(float(overlap), 6),
|
| "retrieval_score": max(0.0, min(1.0, float(hit.score or 0.0))),
|
| "graph_score": max(
|
| 0.0,
|
| min(
|
| 1.0,
|
| float(
|
| metadata.get(
|
| "event_score",
|
| metadata.get("recall_score", metadata.get("hybrid_score", hit.score)),
|
| )
|
| or 0.0
|
| ),
|
| ),
|
| ),
|
| "tunnel_score": max(
|
| 0.0,
|
| min(
|
| 1.0,
|
| max(
|
| float(metadata.get("path_tunnel_support_score", 0.0) or 0.0),
|
| float(metadata.get("event_tunnel_support_score", 0.0) or 0.0),
|
| float(metadata.get("path_chain_extension_delta_score", 0.0) or 0.0),
|
| ),
|
| ),
|
| ),
|
| "topic_similarity": max(0.0, min(1.0, float(metadata.get("topic_bucket_overlap", 0.0) or 0.0))),
|
| "semantic_similarity": semantic_similarity,
|
| "confidence": max(0.0, min(1.0, float(metadata.get("confidence", hit.score or 0.0) or 0.0))),
|
| "rank_score": round(1.0 / float(index + 1), 6),
|
| "age_turns": max(0, int(current_turn_index) - int(hit.turn_index or 0)) if current_turn_index else 0,
|
| "branch_depth": max(0, len(list(metadata.get("selected_path_ids", []) or []))),
|
| "contradicts_current": normalized_state in {"stale", "superseded", "false"} or bool(metadata.get("contradicts_current")),
|
| "is_current": temporal_state == "current",
|
| }
|
|
|
| def _apply_injection_planner_to_hits(
|
| self,
|
| query: str,
|
| hits: Sequence[MemoryHit],
|
| *,
|
| top_k: int,
|
| ) -> Dict[str, Any]:
|
| normalized_mode = _normalize(self.injection_planner_mode)
|
| base_metadata: Dict[str, Any] = {
|
| "injection_planner_enabled": False,
|
| "injection_planner_mode": normalized_mode or "observe",
|
| }
|
| if normalized_mode in _INJECTION_PLANNER_DISABLED_MODES:
|
| base_metadata["injection_planner_reason"] = "disabled"
|
| return {"hits": list(hits), "metadata": base_metadata}
|
| if not hits:
|
| base_metadata["injection_planner_reason"] = "no_hits"
|
| return {"hits": list(hits), "metadata": base_metadata}
|
| model = self._load_injection_planner()
|
| if model is None or self._injection_planner_config is None:
|
| base_metadata.update(
|
| {
|
| "injection_planner_reason": self._injection_planner_error or "load_failed",
|
| "injection_planner_model_path": self._injection_planner_resolved_path,
|
| }
|
| )
|
| return {"hits": list(hits), "metadata": base_metadata}
|
| torch_module = getattr(injection_planner_runtime, "torch", None)
|
| if torch_module is None:
|
| base_metadata["injection_planner_reason"] = "torch_unavailable"
|
| return {"hits": list(hits), "metadata": base_metadata}
|
| current_turn_index = max(
|
| [int(getattr(self.graph, "turn_index", 0) or 0), *[int(hit.turn_index or 0) for hit in hits]],
|
| default=0,
|
| )
|
| candidates = [
|
| self._injection_planner_candidate_from_hit(query, hit, index=index, current_turn_index=current_turn_index)
|
| for index, hit in enumerate(hits)
|
| ]
|
| row = {"id": "runtime_injection_plan", "query": query, "candidates": candidates, "gold": {}}
|
| try:
|
| dataset = injection_planner_runtime.InjectionPlannerDataset([row], self._injection_planner_config)
|
| batch = injection_planner_runtime.collate_injection_batch([dataset[0]])
|
| device = next(model.parameters()).device
|
| model_batch = {
|
| key: value.to(device) if hasattr(value, "to") else value
|
| for key, value in dict(batch).items()
|
| }
|
| with torch_module.no_grad():
|
| outputs = model(model_batch["features"], model_batch["valid_mask"])
|
| selection_scores = torch_module.sigmoid(outputs["selection_logits"])[0].detach().cpu().tolist()
|
| should_inject_score = float(torch_module.sigmoid(outputs["should_inject_logits"])[0].detach().cpu().item())
|
| mode_probs = torch_module.softmax(outputs["injection_mode_logits"], dim=-1)[0].detach().cpu()
|
| mode_index = int(torch_module.argmax(mode_probs).item())
|
| temporal_indices = torch_module.argmax(outputs["temporal_logits"], dim=-1)[0].detach().cpu().tolist()
|
| logic_scores = torch_module.sigmoid(outputs["logic_logits"])[0].detach().cpu().tolist()
|
| if bool(self._injection_planner_evidence_role_supported) and "evidence_role_logits" in outputs:
|
| evidence_role_indices = torch_module.argmax(outputs["evidence_role_logits"], dim=-1)[0].detach().cpu().tolist()
|
| else:
|
| evidence_role_indices = [
|
| injection_planner_runtime.EVIDENCE_ROLES.index("direct_answer")
|
| for _ in candidates
|
| ]
|
| except Exception as exc:
|
| base_metadata.update(
|
| {
|
| "injection_planner_reason": f"inference_failed: {exc}",
|
| "injection_planner_model_path": self._injection_planner_resolved_path,
|
| }
|
| )
|
| return {"hits": list(hits), "metadata": base_metadata}
|
| thresholds = {
|
| "selection_threshold": float(self._injection_planner_thresholds.get("selection_threshold", 0.5)),
|
| "row_threshold": float(self._injection_planner_thresholds.get("row_threshold", 0.5)),
|
| "logic_threshold": float(self._injection_planner_thresholds.get("logic_threshold", 0.5)),
|
| }
|
| injection_mode = injection_planner_runtime.INJECTION_MODES[mode_index]
|
| row_allows_injection = should_inject_score >= thresholds["row_threshold"] and injection_mode != "none"
|
| predictions: Dict[str, Dict[str, Any]] = {}
|
| for index, candidate in enumerate(candidates):
|
| logic_roles = [
|
| role
|
| for role, score in zip(injection_planner_runtime.LOGIC_ROLES, logic_scores[index])
|
| if float(score) >= thresholds["logic_threshold"]
|
| ]
|
| evidence_role = injection_planner_runtime.EVIDENCE_ROLES[int(evidence_role_indices[index])]
|
| role_allows_selection = evidence_role not in {"noise", "negative_evidence"}
|
| predictions[candidate["id"]] = {
|
| "selection_score": float(selection_scores[index]),
|
| "selected": bool(
|
| row_allows_injection
|
| and role_allows_selection
|
| and float(selection_scores[index]) >= thresholds["selection_threshold"]
|
| ),
|
| "temporal_state": injection_planner_runtime.TEMPORAL_STATES[int(temporal_indices[index])],
|
| "evidence_role": evidence_role,
|
| "logic_roles": logic_roles or ["evidence"],
|
| "candidate_layer": candidate.get("layer", ""),
|
| "role_allows_selection": bool(role_allows_selection),
|
| }
|
| annotated_hits: List[MemoryHit] = []
|
| for index, hit in enumerate(hits):
|
| candidate_id = _clean_text(hit.memory_id) or f"hit:{index}"
|
| prediction = predictions.get(candidate_id, {})
|
| metadata = dict(hit.metadata or {})
|
| metadata.update(
|
| {
|
| "injection_planner_enabled": True,
|
| "injection_planner_mode": normalized_mode or "observe",
|
| "injection_planner_model_path": self._injection_planner_resolved_path,
|
| "injection_planner_candidate_id": candidate_id,
|
| "injection_planner_score": round(float(prediction.get("selection_score", 0.0)), 6),
|
| "injection_planner_selected": bool(prediction.get("selected", False)),
|
| "injection_planner_temporal_state": _clean_text(prediction.get("temporal_state", "")),
|
| "injection_planner_evidence_role": _clean_text(prediction.get("evidence_role", "")),
|
| "injection_planner_role_allows_selection": bool(prediction.get("role_allows_selection", False)),
|
| "injection_planner_logic_roles": list(prediction.get("logic_roles", []) or []),
|
| "injection_planner_candidate_layer": _clean_text(prediction.get("candidate_layer", "")),
|
| "injection_planner_injection_mode": injection_mode,
|
| "injection_planner_should_inject_score": round(float(should_inject_score), 6),
|
| "injection_planner_thresholds": dict(thresholds),
|
| "injection_planner_evidence_role_supported": bool(self._injection_planner_evidence_role_supported),
|
| }
|
| )
|
| planner_score = float(prediction.get("selection_score", 0.0))
|
| next_score = max(float(hit.score), planner_score) if bool(prediction.get("selected", False)) else float(hit.score)
|
| annotated_hits.append(
|
| MemoryHit(
|
| memory_id=hit.memory_id,
|
| category=hit.category,
|
| value=hit.value,
|
| relation=hit.relation,
|
| anchors=list(hit.anchors),
|
| score=round(next_score, 6),
|
| source_kind=hit.source_kind,
|
| slot_key=hit.slot_key,
|
| state=hit.state,
|
| turn_index=int(hit.turn_index),
|
| metadata=metadata,
|
| )
|
| )
|
| selected_count = sum(1 for item in annotated_hits if bool((item.metadata or {}).get("injection_planner_selected")))
|
| if normalized_mode in _INJECTION_PLANNER_FORCE_MODES and selected_count:
|
| selected_ids = {hit.memory_id for hit in annotated_hits if bool((hit.metadata or {}).get("injection_planner_selected"))}
|
| planned_hits = [hit for hit in annotated_hits if hit.memory_id in selected_ids]
|
| elif normalized_mode in _INJECTION_PLANNER_GUIDED_MODES and selected_count:
|
| planned_hits = sorted(
|
| annotated_hits,
|
| key=lambda hit: (
|
| not bool((hit.metadata or {}).get("injection_planner_selected")),
|
| -float((hit.metadata or {}).get("injection_planner_score", 0.0) or 0.0),
|
| -float(hit.score or 0.0),
|
| ),
|
| )
|
| else:
|
| planned_hits = annotated_hits
|
| base_metadata.update(
|
| {
|
| "injection_planner_enabled": True,
|
| "injection_planner_reason": "ok",
|
| "injection_planner_model_path": self._injection_planner_resolved_path,
|
| "injection_planner_candidate_count": len(candidates),
|
| "injection_planner_selected_count": int(selected_count),
|
| "injection_planner_should_inject_score": round(float(should_inject_score), 6),
|
| "injection_planner_injection_mode": injection_mode,
|
| "injection_planner_thresholds": dict(thresholds),
|
| "injection_planner_evidence_role_supported": bool(self._injection_planner_evidence_role_supported),
|
| "injection_planner_guided": bool(normalized_mode in _INJECTION_PLANNER_GUIDED_MODES | _INJECTION_PLANNER_FORCE_MODES),
|
| "injection_planner_prediction_ids": [
|
| candidate_id
|
| for candidate_id, prediction in predictions.items()
|
| if bool(prediction.get("selected", False))
|
| ],
|
| }
|
| )
|
| return {"hits": planned_hits[: max(len(planned_hits), top_k)], "metadata": base_metadata}
|
|
|
| def _node_scorer(self) -> LoadedNodeMemoryScorer | None:
|
| if self.retrieval_mode != "hybrid_node_scored":
|
| return None
|
| if self._loaded_node_scorer is not None:
|
| return self._loaded_node_scorer
|
| if not self.node_model_path:
|
| self._node_scorer_error = "node_model_path_missing"
|
| return None
|
| try:
|
| self._loaded_node_scorer = LoadedNodeMemoryScorer(
|
| node_model_path=Path(self.node_model_path),
|
| path_model_path=Path(self.path_model_path) if self.path_model_path else None,
|
| device=self.node_model_device or None,
|
| )
|
| except Exception as exc:
|
| self._node_scorer_error = str(exc)
|
| self._loaded_node_scorer = None
|
| return self._loaded_node_scorer
|
|
|
| def _apply_writeback_promotions(self, stored_ids: Sequence[str]) -> List[Dict[str, Any]]:
|
| promotion_events: List[Dict[str, Any]] = []
|
| for memory_id in stored_ids:
|
| record = self.graph.records_by_id.get(memory_id)
|
| if record is None or not isinstance(record.metadata, dict):
|
| continue
|
| if _normalize(record.metadata.get("memory_role", "")) != "assistant" or _normalize(record.metadata.get("authority", "")) != "derived":
|
| continue
|
| canonical_slot_key = _clean_text(record.metadata.get("canonical_slot_key", ""))
|
| writeback_class = _clean_text(record.metadata.get("writeback_class", ""))
|
| if not canonical_slot_key or not writeback_class:
|
| continue
|
| support_refs = self._support_ref_union(record.metadata)
|
| confidence = float(record.confidence or 0.0)
|
| same_records = [
|
| item
|
| for item in self.graph.records_by_id.values()
|
| if isinstance(item.metadata, dict)
|
| and _normalize(item.metadata.get("memory_role", "")) == "assistant"
|
| and _normalize(item.metadata.get("canonical_slot_key", "")) == _normalize(canonical_slot_key)
|
| and _normalize(item.metadata.get("writeback_class", "")) == _normalize(writeback_class)
|
| and _normalize(item.value) == _normalize(record.value)
|
| ]
|
| qualifying = [item for item in same_records if float(item.confidence or 0.0) >= 0.9]
|
| distinct_answers = {
|
| _clean_text(answer_id)
|
| for item in qualifying
|
| for answer_id in [*list(item.metadata.get("origin_answer_ids", []) or []), _clean_text(item.metadata.get("origin_answer_id", ""))]
|
| if _clean_text(answer_id)
|
| }
|
| aggregated_support = set()
|
| for item in same_records:
|
| aggregated_support.update(self._support_ref_union(item.metadata))
|
| fast_promotion = writeback_class in {"fact", "state_change"} and confidence >= 0.97 and len(support_refs) >= 3
|
| standard_promotion = len(distinct_answers) >= 2 and len(aggregated_support) >= 2
|
| if not (fast_promotion or standard_promotion):
|
| record.metadata["promotion_state"] = "candidate"
|
| continue
|
| source_head = self._source_head_for_canonical(canonical_slot_key)
|
| blocked_conflict = source_head is not None and _normalize(source_head.value) != _normalize(record.value)
|
| promoted_slot = f"promoted.{canonical_slot_key}"
|
| promoted_metadata = {
|
| **dict(record.metadata or {}),
|
| "memory_role": "assistant",
|
| "authority": "promoted",
|
| "canonical_slot_key": canonical_slot_key,
|
| "writeback_class": writeback_class,
|
| "promotion_state": "blocked_conflict" if blocked_conflict else "promoted",
|
| "support_memory_ids": sorted({*list(record.metadata.get("support_memory_ids", []) or []), *[ref for ref in aggregated_support if ref.startswith("fact:") is False and ref.startswith("path:") is False]}),
|
| "support_fact_refs": sorted({*list(record.metadata.get("support_fact_refs", []) or []), *[ref for ref in aggregated_support if ref.startswith("fact:")]}),
|
| "support_path_refs": sorted({*list(record.metadata.get("support_path_refs", []) or []), *[ref for ref in aggregated_support if ref.startswith("path:")]}),
|
| }
|
| promoted_record = SessionMemoryRecordV2(
|
| memory_id=f"{promoted_slot}:{record.turn_index}:promoted",
|
| category=record.category,
|
| slot_key=promoted_slot,
|
| value=record.value,
|
| relation=record.relation,
|
| anchor_concepts=list(record.anchor_concepts),
|
| evidence_anchors=list(record.evidence_anchors),
|
| salience=max(float(record.salience), 0.78),
|
| confidence=max(float(record.confidence), 0.9),
|
| source_kind=f"promoted_{record.source_kind}",
|
| turn_index=int(record.turn_index),
|
| state="active",
|
| metadata=promoted_metadata,
|
| )
|
| promoted_ids = self.graph.add_records([promoted_record])
|
| record.metadata["promotion_state"] = "blocked_conflict" if blocked_conflict else "promoted"
|
| promotion_events.append(
|
| {
|
| "source_memory_id": record.memory_id,
|
| "promoted_record_ids": list(promoted_ids),
|
| "canonical_slot_key": canonical_slot_key,
|
| "writeback_class": writeback_class,
|
| "promotion_state": record.metadata["promotion_state"],
|
| "blocked_conflict": bool(blocked_conflict),
|
| }
|
| )
|
| return promotion_events
|
|
|
| def _source_head_for_canonical(self, canonical_slot_key: str) -> SessionMemoryRecordV2 | None:
|
| head_id = self.graph.slot_heads.get(canonical_slot_key)
|
| if not head_id:
|
| return None
|
| record = self.graph.records_by_id.get(head_id)
|
| if record is None or not isinstance(record.metadata, dict):
|
| return None
|
| if _normalize(record.metadata.get("memory_role", "")) != "user" or _normalize(record.metadata.get("authority", "")) != "source":
|
| return None
|
| return record
|
|
|
| def _support_ref_union(self, metadata: Dict[str, Any]) -> set[str]:
|
| return {
|
| *[_clean_text(item) for item in list(metadata.get("support_memory_ids", []) or []) if _clean_text(item)],
|
| *[_clean_text(item) for item in list(metadata.get("support_fact_refs", []) or []) if _clean_text(item)],
|
| *[_clean_text(item) for item in list(metadata.get("support_path_refs", []) or []) if _clean_text(item)],
|
| }
|
|
|
| def _hybrid_node_scored_hits(
|
| self,
|
| query: str,
|
| hits: Sequence[MemoryHit],
|
| *,
|
| top_k: int,
|
| public_hits: Sequence[MemoryHit] | None = None,
|
| ) -> Dict[str, Any]:
|
| scorer = self._node_scorer()
|
| if scorer is None:
|
| return {
|
| "hits": list(hits),
|
| "metadata": {
|
| "retrieval_mode": "heuristic",
|
| "hybrid_enabled": False,
|
| "hybrid_error": self._node_scorer_error,
|
| },
|
| }
|
| source_hits = _learnable_graph_hits(self.graph)
|
| if not source_hits:
|
| return {
|
| "hits": list(hits),
|
| "metadata": {
|
| "retrieval_mode": "heuristic",
|
| "hybrid_enabled": False,
|
| "hybrid_error": "no_learnable_hits",
|
| },
|
| }
|
| has_public_hits = any(
|
| _is_public_dialog_hit(hit)
|
| and not bool((hit.metadata or {}).get("profile_layer"))
|
| for hit in source_hits
|
| )
|
| has_generic_hits = any(not _is_public_dialog_hit(hit) for hit in source_hits)
|
| hybrid_source = "mixed_full_graph"
|
| if has_public_hits and not has_generic_hits:
|
| hybrid_source = "public_full_graph"
|
| elif has_generic_hits and not has_public_hits:
|
| hybrid_source = "generic_full_graph"
|
| runtime_graph = _build_runtime_graph_from_hits(query, source_hits)
|
| grouped_hits = dict(runtime_graph.get("grouped_hits", {}) or {})
|
| profile_first_payload = _profile_first_hybrid_rescue(
|
| self.graph,
|
| query,
|
| grouped_hits=grouped_hits,
|
| top_k=top_k,
|
| )
|
| profile_first_hits = list(profile_first_payload.get("hits", []) or [])
|
| profile_first_event_ids = list(profile_first_payload.get("event_ids", []) or [])
|
| profile_first_memory_ids = list(profile_first_payload.get("memory_ids", []) or [])
|
| candidate_event_ids = sorted(grouped_hits.keys())
|
| if not candidate_event_ids:
|
| return {
|
| "hits": list(hits),
|
| "metadata": {
|
| "retrieval_mode": "heuristic",
|
| "hybrid_enabled": False,
|
| "hybrid_error": "no_event_candidates",
|
| },
|
| }
|
| question_analysis = extract_question_features(query)
|
| hybrid_candidate_limit = min(
|
| len(candidate_event_ids),
|
| max(
|
| _HYBRID_SELECTED_EVENT_FLOOR,
|
| int(self.candidate_event_k) * 4,
|
| int(self.support_path_k) * 4,
|
| int(top_k) * 4,
|
| ),
|
| )
|
| embedder_pre_recall_mode = _normalize(getattr(self, "embedder_pre_recall_mode", ""))
|
| embedder_pre_recall_enabled = embedder_pre_recall_mode not in _EMBEDDER_INDEX_DISABLED_MODES
|
| embedder_pre_recall_index_mode = self.embedder_index_recall_mode
|
| if embedder_pre_recall_mode not in {"1", "true", "yes", "on", "auto", "seed", "candidate", "candidates"}:
|
| embedder_pre_recall_index_mode = embedder_pre_recall_mode
|
| pre_embedder_index_payload: Dict[str, Any] = {
|
| "event_ids": [],
|
| "metadata": {
|
| "embedder_pre_recall_enabled": False,
|
| "embedder_pre_recall_mode": embedder_pre_recall_mode or "off",
|
| "embedder_pre_recall_index_mode": embedder_pre_recall_index_mode or "off",
|
| "embedder_pre_recall_event_ids": [],
|
| },
|
| }
|
| pre_embedder_event_ids: List[str] = []
|
| pre_candidate_event_ids: List[str] = []
|
| if embedder_pre_recall_enabled:
|
| pre_embedder_index_payload = _embedder_index_recall_event_ids(
|
| query,
|
| grouped_hits=grouped_hits,
|
| mode=embedder_pre_recall_index_mode,
|
| limit=self.embedder_pre_recall_k or self.embedder_index_recall_k or hybrid_candidate_limit,
|
| max_terms=self.write_embedder_index_max_terms,
|
| )
|
| pre_embedder_event_ids = list(pre_embedder_index_payload.get("event_ids", []) or [])
|
| pre_candidate_event_ids = _bounded_event_id_union(
|
| pre_embedder_event_ids,
|
| max_items=hybrid_candidate_limit,
|
| )
|
| pre_score_kwargs = {
|
| "graph": runtime_graph,
|
| "question": query,
|
| "question_features": question_analysis,
|
| "rerank_top_k": self.candidate_event_k,
|
| "event_rerank_mode": self.event_rerank_mode,
|
| "matrix_event_top_k": self.matrix_event_top_k,
|
| "support_path_k": self.support_path_k,
|
| "top_k": top_k,
|
| }
|
| if pre_candidate_event_ids:
|
| pre_score_kwargs["candidate_event_ids"] = pre_candidate_event_ids
|
| scored = _call_with_supported_kwargs(
|
| scorer.score_runtime,
|
| **pre_score_kwargs,
|
| )
|
| memory_router_decision = _memory_router_decision(
|
| scored,
|
| mode=self.memory_router_mode,
|
| threshold=self.memory_router_threshold,
|
| margin=self.memory_router_margin,
|
| )
|
| profile_first_router_suppressed = False
|
| if not _memory_router_allows(memory_router_decision, "profile", "resource"):
|
| profile_first_hits = []
|
| profile_first_event_ids = []
|
| profile_first_memory_ids = []
|
| profile_first_router_suppressed = True
|
| initial_recall_event_scores = dict(scored.get("recall_event_scores", {}) or {})
|
| model_recall_event_ids = _bounded_event_id_union(
|
| list(scored.get("recall_event_ids", []) or []),
|
| [
|
| event_id
|
| for event_id, _ in sorted(
|
| initial_recall_event_scores.items(),
|
| key=lambda item: (-float(item[1]), item[0]),
|
| )
|
| ],
|
| list(scored.get("rerank_candidate_event_ids", []) or []),
|
| max_items=max(1, len(candidate_event_ids)),
|
| )
|
| learned_recall_event_ids = list(model_recall_event_ids)
|
| symbolic_recall_event_ids = _symbolic_recall_event_ids(
|
| query,
|
| runtime_graph,
|
| grouped_hits=grouped_hits,
|
| limit=hybrid_candidate_limit,
|
| )
|
| symbolic_recall_event_ids = _bounded_event_id_union(
|
| profile_first_event_ids,
|
| symbolic_recall_event_ids,
|
| max_items=hybrid_candidate_limit,
|
| )
|
| if pre_embedder_event_ids:
|
| embedder_index_payload = pre_embedder_index_payload
|
| else:
|
| embedder_index_payload = _embedder_index_recall_event_ids(
|
| query,
|
| grouped_hits=grouped_hits,
|
| mode=self.embedder_index_recall_mode,
|
| limit=self.embedder_index_recall_k or hybrid_candidate_limit,
|
| max_terms=self.write_embedder_index_max_terms,
|
| )
|
| embedder_index_event_ids = list(embedder_index_payload.get("event_ids", []) or [])
|
| embedder_index_metadata = dict(embedder_index_payload.get("metadata", {}) or {})
|
| embedder_index_metadata.update(
|
| {
|
| "embedder_pre_recall_enabled": bool(pre_candidate_event_ids),
|
| "embedder_pre_recall_mode": embedder_pre_recall_mode or "off",
|
| "embedder_pre_recall_index_mode": embedder_pre_recall_index_mode or "off",
|
| "embedder_pre_recall_event_ids": list(pre_embedder_event_ids),
|
| "embedder_pre_recall_candidate_event_ids": list(pre_candidate_event_ids),
|
| "embedder_pre_recall_candidate_count": int(len(pre_candidate_event_ids)),
|
| }
|
| )
|
| hybrid_candidate_event_ids = _bounded_event_id_union(
|
| profile_first_event_ids,
|
| embedder_index_event_ids,
|
| learned_recall_event_ids,
|
| symbolic_recall_event_ids,
|
| max_items=hybrid_candidate_limit,
|
| )
|
| learned_candidate_event_ids = _bounded_event_id_union(
|
| learned_recall_event_ids,
|
| max_items=hybrid_candidate_limit,
|
| )
|
| learned_recall_event_id_set = set(learned_candidate_event_ids)
|
| hybrid_candidate_union_added_event_ids = [
|
| event_id
|
| for event_id in hybrid_candidate_event_ids
|
| if event_id not in learned_recall_event_id_set
|
| ]
|
| hybrid_candidate_union_priority_changed = list(hybrid_candidate_event_ids) != list(learned_candidate_event_ids)
|
| hybrid_candidate_union_rescored = False
|
| if hybrid_candidate_union_added_event_ids or (embedder_index_event_ids and hybrid_candidate_union_priority_changed):
|
| scored = _call_with_supported_kwargs(
|
| scorer.score_runtime,
|
| graph=runtime_graph,
|
| question=query,
|
| question_features=question_analysis,
|
| candidate_event_ids=hybrid_candidate_event_ids,
|
| rerank_top_k=self.candidate_event_k,
|
| event_rerank_mode=self.event_rerank_mode,
|
| matrix_event_top_k=self.matrix_event_top_k,
|
| support_path_k=self.support_path_k,
|
| top_k=top_k,
|
| )
|
| hybrid_candidate_union_rescored = True
|
| memory_router_decision = _memory_router_decision(
|
| scored,
|
| mode=self.memory_router_mode,
|
| threshold=self.memory_router_threshold,
|
| margin=self.memory_router_margin,
|
| )
|
| recall_event_scores = dict(scored.get("recall_event_scores", {}) or {})
|
| rerank_candidate_event_ids = list(scored.get("rerank_candidate_event_ids", []) or [])
|
| base_event_scores = dict(scored.get("base_event_scores", {}) or {})
|
| rerank_event_scores = dict(scored.get("rerank_event_scores", {}) or {})
|
| calibrated_event_scores = dict(scored.get("calibrated_event_scores", {}) or {})
|
| matrix_event_scores = dict(scored.get("matrix_event_scores", {}) or {})
|
| event_fusion_delta_scores = dict(scored.get("event_fusion_delta_scores", {}) or {})
|
| event_tunnel_support_scores = dict(scored.get("event_tunnel_support_scores", {}) or {})
|
| event_tunnel_delta_scores = dict(scored.get("event_tunnel_delta_scores", {}) or {})
|
| tri_maze_event_reverse_scores = dict(scored.get("tri_maze_event_reverse_scores", {}) or {})
|
| tri_maze_event_boundary_scores = dict(scored.get("tri_maze_event_boundary_scores", {}) or {})
|
| tri_maze_event_reverse_relations = dict(scored.get("tri_maze_event_reverse_relations", {}) or {})
|
| matrix_rerank_event_ids = list(scored.get("matrix_rerank_event_ids", []) or [])
|
| matrix_enabled = bool(scored.get("matrix_enabled", False))
|
| rerank_path_scores = dict(scored.get("rerank_path_scores", {}) or {})
|
| matrix_path_scores = dict(scored.get("matrix_path_scores", {}) or {})
|
| tri_maze_path_reverse_scores = dict(scored.get("tri_maze_path_reverse_scores", {}) or {})
|
| tri_maze_path_boundary_scores = dict(scored.get("tri_maze_path_boundary_scores", {}) or {})
|
| tri_maze_path_reverse_relations = dict(scored.get("tri_maze_path_reverse_relations", {}) or {})
|
| matrix_path_rerank_ids = list(scored.get("matrix_path_rerank_ids", []) or [])
|
| matrix_path_enabled = bool(scored.get("matrix_path_enabled", False))
|
| fusion_enabled = bool(scored.get("fusion_enabled", False))
|
| event_fusion_enabled = bool(scored.get("event_fusion_enabled", fusion_enabled))
|
| path_fusion_enabled = bool(scored.get("path_fusion_enabled", fusion_enabled))
|
| event_calibration_enabled = bool(scored.get("event_calibration_enabled", False))
|
| path_calibration_enabled = bool(scored.get("path_calibration_enabled", False))
|
| event_tunnel_enabled = bool(scored.get("event_tunnel_enabled", False))
|
| path_tunnel_enabled = bool(scored.get("path_tunnel_enabled", False))
|
| final_event_fusion_enabled = bool(scored.get("final_event_fusion_enabled", False))
|
| final_path_fusion_enabled = bool(scored.get("final_path_fusion_enabled", False))
|
| decision_fusion_enabled = bool(scored.get("decision_fusion_enabled", False))
|
| decision_score_source = _clean_text(scored.get("decision_score_source", ""))
|
| event_scores = dict(scored.get("event_scores", {}) or {})
|
| base_path_scores = dict(scored.get("base_path_scores", {}) or {})
|
| calibrated_path_scores = dict(scored.get("calibrated_path_scores", {}) or {})
|
| path_fusion_delta_scores = dict(scored.get("path_fusion_delta_scores", {}) or {})
|
| path_tunnel_support_scores = dict(scored.get("path_tunnel_support_scores", {}) or {})
|
| path_tunnel_delta_scores = dict(scored.get("path_tunnel_delta_scores", {}) or {})
|
| path_model_scores = dict(scored.get("path_model_scores", {}) or {})
|
| path_chain_extension_delta_scores = dict(scored.get("path_chain_extension_delta_scores", {}) or {})
|
| path_chain_extended_scores = dict(scored.get("path_chain_extended_scores", {}) or {})
|
| path_chain_extension_enabled = bool(scored.get("path_chain_extension_enabled", False))
|
| answer_type_scores = dict(scored.get("answer_type_scores", {}) or {})
|
| selected_event_ids_from_model = list(scored.get("selected_event_ids", []) or [])
|
| selected_path_ids_from_model = list(scored.get("selected_path_ids", []) or [])
|
| focused_answer_type_from_model = _clean_text(scored.get("focused_answer_type", ""))
|
| path_scores = dict(scored.get("path_scores", {}) or {})
|
| temporal_scores = dict(scored.get("temporal_scores", {}) or {})
|
| runtime_paths = {_clean_text(path.get("id", "")): dict(path) for path in list(runtime_graph.get("paths", []) or [])}
|
| answer_plan_scores_raw = dict(scored.get("answer_plan_scores", {}) or {})
|
|
|
| def _answer_plan_score_map(role: str) -> Dict[str, float]:
|
| raw_scores = answer_plan_scores_raw.get(role, {})
|
| if not isinstance(raw_scores, Mapping):
|
| return {}
|
| normalized: Dict[str, float] = {}
|
| for raw_event_id, raw_score in raw_scores.items():
|
| event_id = _clean_text(raw_event_id)
|
| if not event_id:
|
| continue
|
| try:
|
| normalized[event_id] = float(raw_score or 0.0)
|
| except (TypeError, ValueError):
|
| normalized[event_id] = 0.0
|
| return normalized
|
|
|
| answer_plan_selected_scores = _answer_plan_score_map("selected")
|
| answer_plan_current_scores = _answer_plan_score_map("current")
|
| answer_plan_historical_scores = _answer_plan_score_map("historical")
|
| answer_plan_suppressed_scores = _answer_plan_score_map("suppressed")
|
| answer_plan_scores = {
|
| "selected": dict(answer_plan_selected_scores),
|
| "current": dict(answer_plan_current_scores),
|
| "historical": dict(answer_plan_historical_scores),
|
| "suppressed": dict(answer_plan_suppressed_scores),
|
| }
|
| try:
|
| answer_plan_event_selection_threshold = float(
|
| os.getenv("TMCRA_ANSWER_PLAN_EVENT_SELECTION_THRESHOLD", "0.50") or 0.50
|
| )
|
| except (TypeError, ValueError):
|
| answer_plan_event_selection_threshold = 0.50
|
| try:
|
| answer_plan_event_selection_top_k = int(os.getenv("TMCRA_ANSWER_PLAN_EVENT_SELECTION_TOP_K", "0") or 0)
|
| except (TypeError, ValueError):
|
| answer_plan_event_selection_top_k = 0
|
| if answer_plan_event_selection_top_k <= 0:
|
| answer_plan_event_selection_top_k = max(top_k, self.support_path_k * 2, _HYBRID_SELECTED_EVENT_FLOOR)
|
| answer_plan_available_event_ids = {
|
| _clean_text(event_id)
|
| for event_id in [
|
| *list(grouped_hits.keys()),
|
| *[_clean_text(path.get("event_id", "")) for path in runtime_paths.values()],
|
| ]
|
| if _clean_text(event_id)
|
| }
|
| answer_plan_event_rows: List[tuple[str, float, float, float, float, float]] = []
|
| answer_plan_event_ids = set(answer_plan_selected_scores) | set(answer_plan_current_scores) | set(answer_plan_historical_scores)
|
| for event_id in answer_plan_event_ids:
|
| if answer_plan_available_event_ids and event_id not in answer_plan_available_event_ids:
|
| continue
|
| selected_score = float(answer_plan_selected_scores.get(event_id, 0.0) or 0.0)
|
| current_score = float(answer_plan_current_scores.get(event_id, 0.0) or 0.0)
|
| historical_score = float(answer_plan_historical_scores.get(event_id, 0.0) or 0.0)
|
| suppressed_score = float(answer_plan_suppressed_scores.get(event_id, 0.0) or 0.0)
|
| support_score = max(selected_score, current_score)
|
| adjusted_score = support_score - max(0.0, suppressed_score - support_score) * 0.5
|
| if support_score < answer_plan_event_selection_threshold or suppressed_score > support_score:
|
| continue
|
| answer_plan_event_rows.append(
|
| (event_id, adjusted_score, support_score, selected_score, current_score, historical_score)
|
| )
|
| answer_plan_event_rows.sort(
|
| key=lambda row: (-float(row[1]), -float(row[2]), -float(row[3]), -float(row[4]), row[0])
|
| )
|
| answer_plan_raw_ranked_event_ids = [
|
| event_id for event_id, *_ in answer_plan_event_rows[: max(1, answer_plan_event_selection_top_k)]
|
| ]
|
| try:
|
| answer_plan_promotion_min_margin = float(
|
| os.getenv("TMCRA_ANSWER_PLAN_PROMOTION_MIN_MARGIN", "0.02") or 0.02
|
| )
|
| except (TypeError, ValueError):
|
| answer_plan_promotion_min_margin = 0.02
|
| answer_plan_promotion_score_margin = 0.0
|
| if len(answer_plan_event_rows) == 1:
|
| answer_plan_promotion_score_margin = float(answer_plan_event_rows[0][1])
|
| answer_plan_promotion_enabled = True
|
| elif len(answer_plan_event_rows) > 1:
|
| comparison_index = min(len(answer_plan_event_rows) - 1, max(1, min(answer_plan_event_selection_top_k, 5) - 1))
|
| answer_plan_promotion_score_margin = float(answer_plan_event_rows[0][1]) - float(answer_plan_event_rows[comparison_index][1])
|
| answer_plan_promotion_enabled = answer_plan_promotion_score_margin >= answer_plan_promotion_min_margin
|
| else:
|
| answer_plan_promotion_enabled = False
|
| answer_plan_ranked_event_ids = list(answer_plan_raw_ranked_event_ids if answer_plan_promotion_enabled else [])
|
| answer_plan_selected_event_ids = [
|
| event_id
|
| for event_id in answer_plan_ranked_event_ids
|
| if float(answer_plan_selected_scores.get(event_id, 0.0) or 0.0)
|
| >= answer_plan_event_selection_threshold
|
| ]
|
| answer_plan_current_event_ids = [
|
| event_id
|
| for event_id in answer_plan_ranked_event_ids
|
| if float(answer_plan_current_scores.get(event_id, 0.0) or 0.0)
|
| >= answer_plan_event_selection_threshold
|
| ]
|
| answer_plan_support_scores = {
|
| event_id: round(float(support_score), 6)
|
| for event_id, _, support_score, *_ in answer_plan_event_rows
|
| }
|
| answer_plan_adjusted_scores = {
|
| event_id: round(float(adjusted_score), 6)
|
| for event_id, adjusted_score, *_ in answer_plan_event_rows
|
| }
|
| answer_plan_rank_lookup = {
|
| event_id: rank for rank, event_id in enumerate(answer_plan_ranked_event_ids, start=1)
|
| }
|
| answer_plan_ranked_event_id_set = set(answer_plan_ranked_event_ids)
|
|
|
| def _answer_plan_hit_metadata(event_id: str) -> Dict[str, Any]:
|
| clean_event_id = _clean_text(event_id)
|
| selected_score = float(answer_plan_selected_scores.get(clean_event_id, 0.0) or 0.0)
|
| current_score = float(answer_plan_current_scores.get(clean_event_id, 0.0) or 0.0)
|
| historical_score = float(answer_plan_historical_scores.get(clean_event_id, 0.0) or 0.0)
|
| suppressed_score = float(answer_plan_suppressed_scores.get(clean_event_id, 0.0) or 0.0)
|
| support_score = max(selected_score, current_score)
|
| return {
|
| "answer_plan_score": round(float(support_score), 6),
|
| "answer_plan_selected_score": round(float(selected_score), 6),
|
| "answer_plan_current_score": round(float(current_score), 6),
|
| "answer_plan_historical_score": round(float(historical_score), 6),
|
| "answer_plan_suppressed_score": round(float(suppressed_score), 6),
|
| "answer_plan_adjusted_score": round(float(answer_plan_adjusted_scores.get(clean_event_id, 0.0)), 6),
|
| "answer_plan_selected": bool(clean_event_id in answer_plan_ranked_event_id_set),
|
| "answer_plan_rank": int(answer_plan_rank_lookup.get(clean_event_id, 0) or 0),
|
| }
|
|
|
| embedder_fusion_mode = _normalize(self.embedder_fusion_mode)
|
| embedder_fusion_enabled = (
|
| embedder_fusion_mode not in _EMBEDDER_INDEX_DISABLED_MODES
|
| and bool(embedder_index_event_ids)
|
| and bool(self.embedder_fusion_top_k > 0)
|
| and bool(self.embedder_fusion_weight > 0.0)
|
| )
|
| embedder_fusion_applied_event_scores: Dict[str, float] = {}
|
| embedder_fusion_boosts: Dict[str, float] = {}
|
| embedder_event_scores = dict(embedder_index_metadata.get("embedder_index_event_scores", {}) or {})
|
| if embedder_fusion_enabled and embedder_event_scores:
|
| ranked_embedder_events = [
|
| (event_id, float(score))
|
| for event_id, score in sorted(
|
| embedder_event_scores.items(),
|
| key=lambda item: (-float(item[1]), item[0]),
|
| )
|
| if _clean_text(event_id)
|
| ][: max(1, int(self.embedder_fusion_top_k))]
|
| for rank, (event_id, embedder_score) in enumerate(ranked_embedder_events, start=1):
|
| if embedder_score < float(self.embedder_fusion_score_floor):
|
| continue
|
| rank_bonus = max(0.0, 0.08 - (rank - 1) * 0.006)
|
| boost = min(
|
| float(self.embedder_fusion_max_boost),
|
| (float(self.embedder_fusion_weight) * embedder_score) + rank_bonus,
|
| )
|
| current_score = max(
|
| float(recall_event_scores.get(event_id, 0.0) or 0.0),
|
| float(event_scores.get(event_id, 0.0) or 0.0),
|
| float(base_event_scores.get(event_id, 0.0) or 0.0),
|
| float(rerank_event_scores.get(event_id, 0.0) or 0.0),
|
| float(calibrated_event_scores.get(event_id, 0.0) or 0.0),
|
| )
|
| fused_score = current_score + boost
|
| recall_event_scores[event_id] = max(float(recall_event_scores.get(event_id, 0.0) or 0.0), fused_score)
|
| event_scores[event_id] = max(float(event_scores.get(event_id, 0.0) or 0.0), fused_score)
|
| base_event_scores[event_id] = max(float(base_event_scores.get(event_id, 0.0) or 0.0), fused_score)
|
| rerank_event_scores[event_id] = max(float(rerank_event_scores.get(event_id, 0.0) or 0.0), fused_score)
|
| calibrated_event_scores[event_id] = max(float(calibrated_event_scores.get(event_id, 0.0) or 0.0), fused_score)
|
| embedder_fusion_applied_event_scores[event_id] = round(fused_score, 6)
|
| embedder_fusion_boosts[event_id] = round(boost, 6)
|
| if embedder_fusion_applied_event_scores:
|
| decision_score_source = f"{decision_score_source or 'learned_decision_fusion'}+embedder_fusion"
|
| recall_event_ids = [
|
| event_id
|
| for event_id, _ in sorted(recall_event_scores.items(), key=lambda item: (-float(item[1]), item[0]))
|
| ][: self.candidate_event_k]
|
| if profile_first_event_ids:
|
| profile_first_score_by_event = {
|
| _clean_text((hit.metadata or {}).get("profile_first_hybrid_event_id", "")) or _runtime_event_key(hit): float(hit.score)
|
| for hit in profile_first_hits
|
| }
|
| for rank, event_id in enumerate(profile_first_event_ids, start=1):
|
| if not event_id:
|
| continue
|
| floor = max(0.92, float(profile_first_score_by_event.get(event_id, 0.0))) + max(0.0, 0.04 - (rank * 0.005))
|
| recall_event_scores[event_id] = max(float(recall_event_scores.get(event_id, 0.0)), floor)
|
| event_scores[event_id] = max(float(event_scores.get(event_id, 0.0)), floor)
|
| base_event_scores[event_id] = max(float(base_event_scores.get(event_id, 0.0)), floor)
|
| rerank_event_scores[event_id] = max(float(rerank_event_scores.get(event_id, 0.0)), floor)
|
| calibrated_event_scores[event_id] = max(float(calibrated_event_scores.get(event_id, 0.0)), floor)
|
| for path_id, path in runtime_paths.items():
|
| event_id = _clean_text(path.get("event_id", ""))
|
| path_type = _clean_text(path.get("type", ""))
|
| if event_id not in set(profile_first_event_ids):
|
| continue
|
| if path_type not in {"speaker_event_profile", "speaker_event_source_turn", "speaker_event_status"}:
|
| continue
|
| floor = max(0.90, float(event_scores.get(event_id, 0.0)))
|
| path_scores[path_id] = max(float(path_scores.get(path_id, 0.0)), floor)
|
| base_path_scores[path_id] = max(float(base_path_scores.get(path_id, 0.0)), floor)
|
| calibrated_path_scores[path_id] = max(float(calibrated_path_scores.get(path_id, 0.0)), floor)
|
| path_model_scores[path_id] = max(float(path_model_scores.get(path_id, 0.0)), floor)
|
| selection_consistency_repaired = False
|
| selection_consistency_reason = ""
|
| model_focused_answer_type = _clean_text(focused_answer_type_from_model)
|
| embedder_fusion_selected_event_ids: List[str] = []
|
| embedder_fusion_selected_path_ids: List[str] = []
|
| if decision_fusion_enabled and (selected_path_ids_from_model or selected_event_ids_from_model):
|
| base_selected_path_limit = max(1, min(max(1, self.support_path_k), max(1, top_k)))
|
| selected_path_ids = [
|
| path_id
|
| for path_id in selected_path_ids_from_model
|
| if _clean_text(path_id) in runtime_paths
|
| ]
|
| if not selected_path_ids:
|
| selected_path_ids = [
|
| path_id
|
| for path_id, _ in sorted(
|
| ((path_id, float(score or 0.0)) for path_id, score in path_scores.items()),
|
| key=lambda item: (-float(item[1]), item[0]),
|
| )
|
| ][:base_selected_path_limit]
|
| focused_answer_type = _reconciled_focused_answer_type(question_analysis, answer_type_scores, focused_answer_type_from_model)
|
| tunnel_rescue_path_ids: List[str] = []
|
| tunnel_rescue_pre_filter_path_ids: List[str] = []
|
| path_utility_direct_support_path_ids: List[str] = []
|
| path_utility_contrast_support_path_ids: List[str] = []
|
| path_utility_latent_context_path_ids: List[str] = []
|
| path_utility_drift_noise_path_ids: List[str] = []
|
| path_utility_roles: Dict[str, str] = {}
|
| path_utility_reasons: Dict[str, str] = {}
|
| path_utility_scores: Dict[str, float] = {}
|
| path_utility_overlap_tokens: Dict[str, List[str]] = {}
|
| path_utility_anchor_event_ids: List[str] = []
|
| path_utility_anchor_subject_signatures: List[str] = []
|
| tunnel_rescue_score_threshold = float(self.path_tunnel_rescue_score_floor)
|
| tunnel_rescue_candidate_count = 0
|
| tunnel_rescue_filtered_count = 0
|
| if (
|
| self.path_tunnel_rescue_k > 0
|
| and path_tunnel_support_scores
|
| and _memory_router_allows(memory_router_decision, "path_tunnel", "topic_tunnel")
|
| ):
|
| selected_path_id_set = set(selected_path_ids)
|
| candidate_scores = [
|
| float(score or 0.0)
|
| for path_id, score in path_tunnel_support_scores.items()
|
| if _clean_text(path_id) in runtime_paths
|
| ]
|
| tunnel_rescue_candidate_count = len(candidate_scores)
|
| if candidate_scores and self.path_tunnel_rescue_min_score_margin > 0.0:
|
| sorted_scores = sorted(candidate_scores)
|
| median_score = sorted_scores[len(sorted_scores) // 2]
|
| tunnel_rescue_score_threshold = max(
|
| tunnel_rescue_score_threshold,
|
| median_score + float(self.path_tunnel_rescue_min_score_margin),
|
| )
|
| event_turn_indices = [
|
| _runtime_event_turn_index_from_id(_clean_text(path.get("event_id", "")))
|
| for path in runtime_paths.values()
|
| ]
|
| current_turn_index = max(
|
| [int(getattr(self.graph, "turn_index", 0) or 0), *[turn for turn in event_turn_indices if turn > 0]],
|
| default=0,
|
| )
|
| ranked_tunnel_path_ids = []
|
| for path_id, score in sorted(
|
| (
|
| (path_id, float(score or 0.0))
|
| for path_id, score in path_tunnel_support_scores.items()
|
| if _clean_text(path_id) in runtime_paths
|
| ),
|
| key=lambda item: (-float(item[1]), item[0]),
|
| ):
|
| if path_id in selected_path_id_set or score < tunnel_rescue_score_threshold:
|
| continue
|
| path_event_turn = _runtime_event_turn_index_from_id(
|
| _clean_text(runtime_paths.get(path_id, {}).get("event_id", ""))
|
| )
|
| path_age = max(0, current_turn_index - path_event_turn) if current_turn_index and path_event_turn else 0
|
| if self.path_tunnel_rescue_min_age > 0 and path_age < self.path_tunnel_rescue_min_age:
|
| continue
|
| ranked_tunnel_path_ids.append(path_id)
|
| tunnel_rescue_filtered_count = len(ranked_tunnel_path_ids)
|
| tunnel_rescue_pre_filter_path_ids = ranked_tunnel_path_ids[: max(self.path_tunnel_rescue_k, self.path_tunnel_rescue_k * 4)]
|
| utility_gate = _path_utility_gate(
|
| tunnel_rescue_pre_filter_path_ids,
|
| query=query,
|
| runtime_graph=runtime_graph,
|
| runtime_paths=runtime_paths,
|
| grouped_hits=grouped_hits,
|
| selected_path_ids=selected_path_ids,
|
| selected_event_ids_from_model=selected_event_ids_from_model,
|
| path_scores=path_scores,
|
| path_tunnel_support_scores=path_tunnel_support_scores,
|
| question_analysis=question_analysis,
|
| focused_answer_type=focused_answer_type,
|
| score_threshold=tunnel_rescue_score_threshold,
|
| limit=self.path_tunnel_rescue_k,
|
| )
|
| tunnel_rescue_path_ids = list(utility_gate.get("injected_path_ids", []) or [])
|
| path_utility_direct_support_path_ids = list(utility_gate.get("direct_support_path_ids", []) or [])
|
| path_utility_contrast_support_path_ids = list(utility_gate.get("contrast_support_path_ids", []) or [])
|
| path_utility_latent_context_path_ids = list(utility_gate.get("latent_context_path_ids", []) or [])
|
| path_utility_drift_noise_path_ids = list(utility_gate.get("drift_noise_path_ids", []) or [])
|
| path_utility_roles = dict(utility_gate.get("roles", {}) or {})
|
| path_utility_reasons = dict(utility_gate.get("reasons", {}) or {})
|
| path_utility_scores = dict(utility_gate.get("scores", {}) or {})
|
| path_utility_overlap_tokens = dict(utility_gate.get("overlap_tokens", {}) or {})
|
| path_utility_anchor_event_ids = list(utility_gate.get("anchor_event_ids", []) or [])
|
| path_utility_anchor_subject_signatures = list(utility_gate.get("anchor_subject_signatures", []) or [])
|
| if tunnel_rescue_path_ids:
|
| selected_path_ids = _dedupe([*selected_path_ids, *tunnel_rescue_path_ids])
|
| effective_selected_path_limit = base_selected_path_limit + len(tunnel_rescue_path_ids)
|
| selected_event_ids = _dedupe(
|
| [
|
| *[
|
| _clean_text(runtime_paths.get(path_id, {}).get("event_id", ""))
|
| for path_id in selected_path_ids
|
| if _clean_text(runtime_paths.get(path_id, {}).get("event_id", ""))
|
| ],
|
| *[
|
| _clean_text(event_id)
|
| for event_id in selected_event_ids_from_model
|
| if _clean_text(event_id)
|
| ],
|
| ]
|
| )
|
| if answer_plan_ranked_event_ids:
|
| selected_event_ids = _dedupe([*selected_event_ids, *answer_plan_ranked_event_ids])
|
| if not selected_event_ids:
|
| selected_event_ids = [
|
| event_id
|
| for event_id, _ in sorted(
|
| ((event_id, float(score or 0.0)) for event_id, score in event_scores.items()),
|
| key=lambda item: (-float(item[1]), item[0]),
|
| )
|
| ][: max(1, min(self.candidate_event_k, max(top_k, _HYBRID_SELECTED_EVENT_FLOOR)))]
|
| repaired_path_ids, selection_consistency_repaired, selection_consistency_reason = _repair_selected_paths_for_focus(
|
| selected_path_ids,
|
| runtime_paths=runtime_paths,
|
| selected_event_ids=selected_event_ids,
|
| path_scores=path_scores,
|
| event_scores=event_scores,
|
| temporal_scores=temporal_scores,
|
| question_analysis=question_analysis,
|
| answer_type_scores=answer_type_scores,
|
| focused_answer_type=focused_answer_type,
|
| limit=max(1, effective_selected_path_limit),
|
| )
|
| if selection_consistency_repaired:
|
| selected_path_ids = repaired_path_ids
|
| selected_event_ids = _dedupe(
|
| [
|
| *[
|
| _clean_text(runtime_paths.get(path_id, {}).get("event_id", ""))
|
| for path_id in selected_path_ids
|
| if _clean_text(runtime_paths.get(path_id, {}).get("event_id", ""))
|
| ],
|
| *[
|
| _clean_text(event_id)
|
| for event_id in selected_event_ids_from_model
|
| if _clean_text(event_id)
|
| ],
|
| ]
|
| )
|
| if answer_plan_ranked_event_ids:
|
| selected_event_ids = _dedupe([*selected_event_ids, *answer_plan_ranked_event_ids])
|
| if profile_first_event_ids:
|
| selected_event_ids = _dedupe([*profile_first_event_ids, *selected_event_ids])
|
| if embedder_fusion_applied_event_scores and self.embedder_fusion_select_k > 0:
|
| ranked_fusion_event_ids = [
|
| event_id
|
| for event_id in embedder_index_event_ids
|
| if event_id in embedder_fusion_applied_event_scores
|
| ][: max(1, int(self.embedder_fusion_select_k))]
|
| selected_event_ids = _dedupe([*ranked_fusion_event_ids, *selected_event_ids])
|
| selected_path_id_set = set(selected_path_ids)
|
| selected_embedder_path_ids: List[str] = []
|
| for event_id in ranked_fusion_event_ids:
|
| candidate_paths = [
|
| (path_id, path)
|
| for path_id, path in runtime_paths.items()
|
| if _clean_text(path.get("event_id", "")) == event_id
|
| ]
|
| candidate_paths.sort(
|
| key=lambda item: (
|
| int(_clean_text(item[1].get("type", "")) in {"speaker_event_source_turn", "speaker_event_profile", "speaker_event_status", "speaker_event_time"}),
|
| float(path_scores.get(item[0], 0.0) or 0.0),
|
| item[0],
|
| ),
|
| reverse=True,
|
| )
|
| for path_id, _ in candidate_paths:
|
| if path_id in selected_path_id_set:
|
| continue
|
| selected_embedder_path_ids.append(path_id)
|
| selected_path_id_set.add(path_id)
|
| embedder_fusion_selected_path_ids.append(path_id)
|
| break
|
| if selected_embedder_path_ids:
|
| selected_path_ids = _dedupe([*selected_embedder_path_ids, *selected_path_ids])
|
| embedder_fusion_selected_event_ids = list(ranked_fusion_event_ids)
|
| if answer_plan_ranked_event_ids:
|
| selected_event_ids = _dedupe([*selected_event_ids, *answer_plan_ranked_event_ids])
|
| final_hits: List[MemoryHit] = []
|
| seen_memory_ids = set()
|
| selected_event_id_set = set(selected_event_ids)
|
| for path_id in selected_path_ids:
|
| path = runtime_paths.get(path_id, {})
|
| event_id = _clean_text(path.get("event_id", ""))
|
| if event_id not in selected_event_id_set:
|
| continue
|
| path_type = _clean_text(path.get("type", ""))
|
| support_node_id = _path_support_node_id(path)
|
| support_hit = _support_hit_for_path(path_type, grouped_hits.get(event_id, []))
|
| event_hit = _representative_event_hit(
|
| [*grouped_hits.get(event_id, []), *_event_record_hits_from_graph(self.graph, event_id)],
|
| query=query,
|
| )
|
| decision_score = round(float(path_scores.get(path_id, 0.0)), 6)
|
| candidate_hits: List[tuple[MemoryHit | None, float]] = [(support_hit, decision_score)]
|
| if event_hit is not None and (support_hit is None or event_hit.memory_id != support_hit.memory_id):
|
| candidate_hits.append((event_hit, max(0.0, decision_score - 0.0001)))
|
| for raw_hit, hit_score in candidate_hits:
|
| if raw_hit is None or raw_hit.memory_id in seen_memory_ids:
|
| continue
|
| seen_memory_ids.add(raw_hit.memory_id)
|
| recall_score = float(recall_event_scores.get(event_id, event_scores.get(event_id, 0.0)))
|
| event_score = float(event_scores.get(event_id, raw_hit.score))
|
| temporal_score = float(temporal_scores.get(support_node_id, 0.0))
|
| metadata = dict(raw_hit.metadata or {})
|
| metadata.update(
|
| {
|
| "event_id": event_id,
|
| **_answer_plan_hit_metadata(event_id),
|
| "path_id": path_id,
|
| "base_event_score": round(float(base_event_scores.get(event_id, event_score)), 6),
|
| "rerank_event_score": round(float(rerank_event_scores.get(event_id, event_score)), 6),
|
| "calibrated_event_score": round(float(calibrated_event_scores.get(event_id, event_score)), 6),
|
| "matrix_event_score": round(float(matrix_event_scores.get(event_id, 0.0)), 6),
|
| "event_fusion_delta_score": round(float(event_fusion_delta_scores.get(event_id, 0.0)), 6),
|
| "event_tunnel_support_score": round(float(event_tunnel_support_scores.get(event_id, 0.0)), 6),
|
| "event_tunnel_delta_score": round(float(event_tunnel_delta_scores.get(event_id, 0.0)), 6),
|
| "tri_maze_event_reverse_score": round(float(tri_maze_event_reverse_scores.get(event_id, 0.0)), 6),
|
| "tri_maze_event_boundary_score": round(float(tri_maze_event_boundary_scores.get(event_id, 0.0)), 6),
|
| "tri_maze_event_reverse_relation": round(float(tri_maze_event_reverse_relations.get(event_id, 0.0)), 6),
|
| "matrix_enabled": matrix_enabled,
|
| "event_calibration_enabled": event_calibration_enabled,
|
| "path_calibration_enabled": path_calibration_enabled,
|
| "event_tunnel_enabled": event_tunnel_enabled,
|
| "path_tunnel_enabled": path_tunnel_enabled,
|
| "final_event_fusion_enabled": final_event_fusion_enabled,
|
| "final_path_fusion_enabled": final_path_fusion_enabled,
|
| "decision_fusion_enabled": True,
|
| "event_fusion_enabled": event_fusion_enabled,
|
| "path_fusion_enabled": path_fusion_enabled,
|
| "event_score": round(event_score, 6),
|
| "recall_score": round(recall_score, 6),
|
| "path_score": round(float(path_scores.get(path_id, hit_score)), 6),
|
| "base_path_score": round(float(base_path_scores.get(path_id, path_scores.get(path_id, hit_score))), 6),
|
| "calibrated_path_score": round(float(calibrated_path_scores.get(path_id, path_scores.get(path_id, hit_score))), 6),
|
| "path_fusion_delta_score": round(float(path_fusion_delta_scores.get(path_id, 0.0)), 6),
|
| "path_tunnel_support_score": round(float(path_tunnel_support_scores.get(path_id, 0.0)), 6),
|
| "path_tunnel_delta_score": round(float(path_tunnel_delta_scores.get(path_id, 0.0)), 6),
|
| "path_model_score": round(float(path_model_scores.get(path_id, path_scores.get(path_id, hit_score))), 6),
|
| "path_chain_extension_enabled": path_chain_extension_enabled,
|
| "path_chain_extension_delta_score": round(float(path_chain_extension_delta_scores.get(path_id, 0.0)), 6),
|
| "path_chain_extended_score": round(float(path_chain_extended_scores.get(path_id, path_scores.get(path_id, hit_score))), 6),
|
| "tri_maze_path_reverse_score": round(float(tri_maze_path_reverse_scores.get(path_id, 0.0)), 6),
|
| "tri_maze_path_boundary_score": round(float(tri_maze_path_boundary_scores.get(path_id, 0.0)), 6),
|
| "tri_maze_path_reverse_relation": round(float(tri_maze_path_reverse_relations.get(path_id, 0.0)), 6),
|
| "effective_path_score": round(float(path_scores.get(path_id, hit_score)), 6),
|
| "temporal_score": round(temporal_score, 6),
|
| "raw_public_score": round(float(raw_hit.score), 6),
|
| "hybrid_score": round(hit_score, 6),
|
| "hybrid_score_source": decision_score_source or "learned_decision_fusion",
|
| "evidence_snippet_role": "selected_path_support" if raw_hit is support_hit else "selected_path_event",
|
| "selected_event_ids": list(selected_event_ids),
|
| "selected_path_ids": list(selected_path_ids),
|
| "path_tunnel_rescue_enabled": bool(self.path_tunnel_rescue_k > 0),
|
| "path_tunnel_rescue_k": int(self.path_tunnel_rescue_k),
|
| "path_tunnel_rescue_score_floor": round(float(self.path_tunnel_rescue_score_floor), 6),
|
| "path_tunnel_rescue_min_age": int(self.path_tunnel_rescue_min_age),
|
| "path_tunnel_rescue_min_score_margin": round(float(self.path_tunnel_rescue_min_score_margin), 6),
|
| "path_tunnel_rescue_score_threshold": round(float(tunnel_rescue_score_threshold), 6),
|
| "path_tunnel_rescue_candidate_count": int(tunnel_rescue_candidate_count),
|
| "path_tunnel_rescue_filtered_count": int(tunnel_rescue_filtered_count),
|
| "path_tunnel_rescue_path_ids": list(tunnel_rescue_path_ids),
|
| "path_utility_gate_enabled": bool(self.path_tunnel_rescue_k > 0),
|
| "path_utility_pre_filter_path_ids": list(tunnel_rescue_pre_filter_path_ids),
|
| "path_utility_injected_path_ids": list(tunnel_rescue_path_ids),
|
| "path_utility_direct_support_path_ids": list(path_utility_direct_support_path_ids),
|
| "path_utility_contrast_support_path_ids": list(path_utility_contrast_support_path_ids),
|
| "path_utility_latent_context_path_ids": list(path_utility_latent_context_path_ids),
|
| "path_utility_drift_noise_path_ids": list(path_utility_drift_noise_path_ids),
|
| "path_utility_roles": dict(path_utility_roles),
|
| "path_utility_reasons": dict(path_utility_reasons),
|
| "path_utility_scores": dict(path_utility_scores),
|
| "path_utility_overlap_tokens": dict(path_utility_overlap_tokens),
|
| "path_utility_anchor_event_ids": list(path_utility_anchor_event_ids),
|
| "path_utility_anchor_subject_signatures": list(path_utility_anchor_subject_signatures),
|
| "tunnel_recall_pre_filter_count": int(len(tunnel_rescue_pre_filter_path_ids)),
|
| "tunnel_usable_post_filter_count": int(len(tunnel_rescue_path_ids)),
|
| "model_focused_answer_type": model_focused_answer_type,
|
| "selection_consistency_repaired": bool(selection_consistency_repaired),
|
| "selection_consistency_reason": selection_consistency_reason,
|
| }
|
| )
|
| final_hits.append(
|
| MemoryHit(
|
| memory_id=raw_hit.memory_id,
|
| category=raw_hit.category,
|
| value=raw_hit.value,
|
| relation=raw_hit.relation,
|
| anchors=list(raw_hit.anchors),
|
| score=round(hit_score, 6),
|
| source_kind=raw_hit.source_kind,
|
| slot_key=raw_hit.slot_key,
|
| state=raw_hit.state,
|
| turn_index=int(raw_hit.turn_index),
|
| metadata=metadata,
|
| )
|
| )
|
| for event_rank, event_id in enumerate(selected_event_ids, start=1):
|
| event_hit = _representative_event_hit(
|
| [*grouped_hits.get(event_id, []), *_event_record_hits_from_graph(self.graph, event_id)],
|
| query=query,
|
| )
|
| if event_hit is None or event_hit.memory_id in seen_memory_ids:
|
| continue
|
| seen_memory_ids.add(event_hit.memory_id)
|
| recall_score = float(recall_event_scores.get(event_id, event_hit.score))
|
| event_score = float(event_scores.get(event_id, event_hit.score))
|
| metadata = dict(event_hit.metadata or {})
|
| metadata.update(
|
| {
|
| "event_id": event_id,
|
| **_answer_plan_hit_metadata(event_id),
|
| "path_id": "",
|
| "base_event_score": round(float(base_event_scores.get(event_id, event_score)), 6),
|
| "rerank_event_score": round(float(rerank_event_scores.get(event_id, event_score)), 6),
|
| "calibrated_event_score": round(float(calibrated_event_scores.get(event_id, event_score)), 6),
|
| "matrix_event_score": round(float(matrix_event_scores.get(event_id, 0.0)), 6),
|
| "event_fusion_delta_score": round(float(event_fusion_delta_scores.get(event_id, 0.0)), 6),
|
| "event_tunnel_support_score": round(float(event_tunnel_support_scores.get(event_id, 0.0)), 6),
|
| "event_tunnel_delta_score": round(float(event_tunnel_delta_scores.get(event_id, 0.0)), 6),
|
| "tri_maze_event_reverse_score": round(float(tri_maze_event_reverse_scores.get(event_id, 0.0)), 6),
|
| "tri_maze_event_boundary_score": round(float(tri_maze_event_boundary_scores.get(event_id, 0.0)), 6),
|
| "tri_maze_event_reverse_relation": round(float(tri_maze_event_reverse_relations.get(event_id, 0.0)), 6),
|
| "matrix_enabled": matrix_enabled,
|
| "event_calibration_enabled": event_calibration_enabled,
|
| "path_calibration_enabled": path_calibration_enabled,
|
| "event_tunnel_enabled": event_tunnel_enabled,
|
| "path_tunnel_enabled": path_tunnel_enabled,
|
| "final_event_fusion_enabled": final_event_fusion_enabled,
|
| "final_path_fusion_enabled": final_path_fusion_enabled,
|
| "decision_fusion_enabled": True,
|
| "event_fusion_enabled": event_fusion_enabled,
|
| "path_fusion_enabled": path_fusion_enabled,
|
| "event_score": round(event_score, 6),
|
| "recall_score": round(recall_score, 6),
|
| "path_score": 0.0,
|
| "base_path_score": 0.0,
|
| "calibrated_path_score": 0.0,
|
| "path_fusion_delta_score": 0.0,
|
| "path_tunnel_support_score": 0.0,
|
| "path_tunnel_delta_score": 0.0,
|
| "path_model_score": 0.0,
|
| "path_chain_extension_enabled": path_chain_extension_enabled,
|
| "path_chain_extension_delta_score": 0.0,
|
| "path_chain_extended_score": 0.0,
|
| "effective_path_score": 0.0,
|
| "temporal_score": 0.0,
|
| "raw_public_score": round(float(event_hit.score), 6),
|
| "hybrid_score": round(event_score, 6),
|
| "hybrid_score_source": decision_score_source or "learned_final_event_fusion",
|
| "evidence_snippet_role": "selected_event_representative",
|
| "selected_event_rank": int(event_rank),
|
| "selected_event_ids": list(selected_event_ids),
|
| "selected_path_ids": list(selected_path_ids),
|
| "path_tunnel_rescue_enabled": bool(self.path_tunnel_rescue_k > 0),
|
| "path_tunnel_rescue_k": int(self.path_tunnel_rescue_k),
|
| "path_tunnel_rescue_score_floor": round(float(self.path_tunnel_rescue_score_floor), 6),
|
| "path_tunnel_rescue_min_age": int(self.path_tunnel_rescue_min_age),
|
| "path_tunnel_rescue_min_score_margin": round(float(self.path_tunnel_rescue_min_score_margin), 6),
|
| "path_tunnel_rescue_score_threshold": round(float(tunnel_rescue_score_threshold), 6),
|
| "path_tunnel_rescue_candidate_count": int(tunnel_rescue_candidate_count),
|
| "path_tunnel_rescue_filtered_count": int(tunnel_rescue_filtered_count),
|
| "path_tunnel_rescue_path_ids": list(tunnel_rescue_path_ids),
|
| "path_utility_gate_enabled": bool(self.path_tunnel_rescue_k > 0),
|
| "path_utility_pre_filter_path_ids": list(tunnel_rescue_pre_filter_path_ids),
|
| "path_utility_injected_path_ids": list(tunnel_rescue_path_ids),
|
| "path_utility_direct_support_path_ids": list(path_utility_direct_support_path_ids),
|
| "path_utility_contrast_support_path_ids": list(path_utility_contrast_support_path_ids),
|
| "path_utility_latent_context_path_ids": list(path_utility_latent_context_path_ids),
|
| "path_utility_drift_noise_path_ids": list(path_utility_drift_noise_path_ids),
|
| "path_utility_roles": dict(path_utility_roles),
|
| "path_utility_reasons": dict(path_utility_reasons),
|
| "path_utility_scores": dict(path_utility_scores),
|
| "path_utility_overlap_tokens": dict(path_utility_overlap_tokens),
|
| "path_utility_anchor_event_ids": list(path_utility_anchor_event_ids),
|
| "path_utility_anchor_subject_signatures": list(path_utility_anchor_subject_signatures),
|
| "tunnel_recall_pre_filter_count": int(len(tunnel_rescue_pre_filter_path_ids)),
|
| "tunnel_usable_post_filter_count": int(len(tunnel_rescue_path_ids)),
|
| "model_focused_answer_type": model_focused_answer_type,
|
| "selection_consistency_repaired": bool(selection_consistency_repaired),
|
| "selection_consistency_reason": selection_consistency_reason,
|
| }
|
| )
|
| final_hits.append(
|
| MemoryHit(
|
| memory_id=event_hit.memory_id,
|
| category=event_hit.category,
|
| value=event_hit.value,
|
| relation=event_hit.relation,
|
| anchors=list(event_hit.anchors),
|
| score=round(event_score, 6),
|
| source_kind=event_hit.source_kind,
|
| slot_key=event_hit.slot_key,
|
| state=event_hit.state,
|
| turn_index=int(event_hit.turn_index),
|
| metadata=metadata,
|
| )
|
| )
|
| if not final_hits:
|
| final_hits = list(hits)
|
| if profile_first_hits:
|
| final_hits = _inject_profile_first_hits(
|
| final_hits,
|
| profile_first_hits,
|
| selected_event_ids=selected_event_ids,
|
| selected_path_ids=selected_path_ids,
|
| )
|
| final_hits = _coverage_preserving_final_hits(final_hits, selected_event_ids=selected_event_ids, top_k=top_k)
|
| final_hit_event_ids = _event_ids_from_hits(final_hits)
|
| final_missing_selected_event_ids = [event_id for event_id in selected_event_ids if event_id not in set(final_hit_event_ids)]
|
| return {
|
| "hits": final_hits,
|
| "metadata": {
|
| "retrieval_mode": "hybrid_node_scored",
|
| "hybrid_enabled": True,
|
| "hybrid_source": hybrid_source,
|
| **memory_router_decision,
|
| "profile_first_router_suppressed": bool(profile_first_router_suppressed),
|
| "recall_event_ids": list(recall_event_ids),
|
| "learned_recall_event_ids": list(learned_recall_event_ids),
|
| "model_recall_event_ids": list(model_recall_event_ids),
|
| "symbolic_recall_event_ids": list(symbolic_recall_event_ids),
|
| "embedder_index_recall_event_ids": list(embedder_index_event_ids),
|
| **embedder_index_metadata,
|
| "embedder_fusion_mode": embedder_fusion_mode or "off",
|
| "embedder_fusion_enabled": bool(embedder_fusion_enabled),
|
| "embedder_fusion_weight": round(float(self.embedder_fusion_weight), 6),
|
| "embedder_fusion_score_floor": round(float(self.embedder_fusion_score_floor), 6),
|
| "embedder_fusion_top_k": int(self.embedder_fusion_top_k),
|
| "embedder_fusion_select_k": int(self.embedder_fusion_select_k),
|
| "embedder_fusion_max_boost": round(float(self.embedder_fusion_max_boost), 6),
|
| "embedder_fusion_event_scores": dict(embedder_fusion_applied_event_scores),
|
| "embedder_fusion_boosts": dict(embedder_fusion_boosts),
|
| "embedder_fusion_selected_event_ids": list(embedder_fusion_selected_event_ids),
|
| "embedder_fusion_selected_path_ids": list(embedder_fusion_selected_path_ids),
|
| "profile_first_hybrid_enabled": bool(profile_first_event_ids),
|
| "profile_first_event_ids": list(profile_first_event_ids),
|
| "profile_first_memory_ids": list(profile_first_memory_ids),
|
| "hybrid_candidate_event_ids": list(hybrid_candidate_event_ids),
|
| "hybrid_candidate_union_enabled": True,
|
| "hybrid_candidate_union_rescored": bool(hybrid_candidate_union_rescored),
|
| "hybrid_candidate_union_added_event_ids": list(hybrid_candidate_union_added_event_ids),
|
| "hybrid_candidate_union_priority_changed": bool(hybrid_candidate_union_priority_changed),
|
| "rerank_candidate_event_ids": list(rerank_candidate_event_ids),
|
| "base_event_scores": dict(base_event_scores),
|
| "rerank_event_scores": dict(rerank_event_scores),
|
| "calibrated_event_scores": dict(calibrated_event_scores),
|
| "matrix_event_scores": dict(matrix_event_scores),
|
| "event_fusion_delta_scores": dict(event_fusion_delta_scores),
|
| "event_tunnel_support_scores": dict(event_tunnel_support_scores),
|
| "event_tunnel_delta_scores": dict(event_tunnel_delta_scores),
|
| "tri_maze_event_reverse_scores": dict(tri_maze_event_reverse_scores),
|
| "tri_maze_event_boundary_scores": dict(tri_maze_event_boundary_scores),
|
| "tri_maze_event_reverse_relations": dict(tri_maze_event_reverse_relations),
|
| "matrix_rerank_event_ids": list(matrix_rerank_event_ids),
|
| "matrix_enabled": matrix_enabled,
|
| "rerank_path_scores": dict(rerank_path_scores),
|
| "matrix_path_scores": dict(matrix_path_scores),
|
| "tri_maze_path_reverse_scores": dict(tri_maze_path_reverse_scores),
|
| "tri_maze_path_boundary_scores": dict(tri_maze_path_boundary_scores),
|
| "tri_maze_path_reverse_relations": dict(tri_maze_path_reverse_relations),
|
| "matrix_path_rerank_ids": list(matrix_path_rerank_ids),
|
| "matrix_path_enabled": matrix_path_enabled,
|
| "fusion_enabled": fusion_enabled,
|
| "event_calibration_enabled": event_calibration_enabled,
|
| "path_calibration_enabled": path_calibration_enabled,
|
| "event_tunnel_enabled": event_tunnel_enabled,
|
| "path_tunnel_enabled": path_tunnel_enabled,
|
| "final_event_fusion_enabled": final_event_fusion_enabled,
|
| "final_path_fusion_enabled": final_path_fusion_enabled,
|
| "decision_fusion_enabled": True,
|
| "decision_score_source": decision_score_source or "learned_decision_fusion",
|
| "event_fusion_enabled": event_fusion_enabled,
|
| "path_fusion_enabled": path_fusion_enabled,
|
| "selected_event_ids": list(selected_event_ids),
|
| "path_rescue_event_ids": [],
|
| "selected_path_ids": list(selected_path_ids),
|
| "path_tunnel_rescue_enabled": bool(self.path_tunnel_rescue_k > 0),
|
| "path_tunnel_rescue_k": int(self.path_tunnel_rescue_k),
|
| "path_tunnel_rescue_score_floor": round(float(self.path_tunnel_rescue_score_floor), 6),
|
| "path_tunnel_rescue_min_age": int(self.path_tunnel_rescue_min_age),
|
| "path_tunnel_rescue_min_score_margin": round(float(self.path_tunnel_rescue_min_score_margin), 6),
|
| "path_tunnel_rescue_score_threshold": round(float(tunnel_rescue_score_threshold), 6),
|
| "path_tunnel_rescue_candidate_count": int(tunnel_rescue_candidate_count),
|
| "path_tunnel_rescue_filtered_count": int(tunnel_rescue_filtered_count),
|
| "path_tunnel_rescue_path_ids": list(tunnel_rescue_path_ids),
|
| "path_utility_gate_enabled": bool(self.path_tunnel_rescue_k > 0),
|
| "path_utility_pre_filter_path_ids": list(tunnel_rescue_pre_filter_path_ids),
|
| "path_utility_injected_path_ids": list(tunnel_rescue_path_ids),
|
| "path_utility_direct_support_path_ids": list(path_utility_direct_support_path_ids),
|
| "path_utility_contrast_support_path_ids": list(path_utility_contrast_support_path_ids),
|
| "path_utility_latent_context_path_ids": list(path_utility_latent_context_path_ids),
|
| "path_utility_drift_noise_path_ids": list(path_utility_drift_noise_path_ids),
|
| "path_utility_roles": dict(path_utility_roles),
|
| "path_utility_reasons": dict(path_utility_reasons),
|
| "path_utility_scores": dict(path_utility_scores),
|
| "path_utility_overlap_tokens": dict(path_utility_overlap_tokens),
|
| "path_utility_anchor_event_ids": list(path_utility_anchor_event_ids),
|
| "path_utility_anchor_subject_signatures": list(path_utility_anchor_subject_signatures),
|
| "tunnel_recall_pre_filter_count": int(len(tunnel_rescue_pre_filter_path_ids)),
|
| "tunnel_usable_post_filter_count": int(len(tunnel_rescue_path_ids)),
|
| "final_hit_event_ids": list(final_hit_event_ids),
|
| "final_hit_dia_ids": _dia_ids_from_hits(final_hits),
|
| "final_missing_selected_event_ids": list(final_missing_selected_event_ids),
|
| "selected_event_count": int(len(selected_event_ids)),
|
| "selected_path_count": int(len(selected_path_ids)),
|
| "temporal_scores": dict(temporal_scores),
|
| "recall_event_scores": dict(recall_event_scores),
|
| "event_scores": dict(event_scores),
|
| "base_path_scores": dict(base_path_scores),
|
| "calibrated_path_scores": dict(calibrated_path_scores),
|
| "path_fusion_delta_scores": dict(path_fusion_delta_scores),
|
| "path_tunnel_support_scores": dict(path_tunnel_support_scores),
|
| "path_tunnel_delta_scores": dict(path_tunnel_delta_scores),
|
| "path_model_scores": dict(path_model_scores),
|
| "path_chain_extension_enabled": path_chain_extension_enabled,
|
| "path_chain_extension_delta_scores": dict(path_chain_extension_delta_scores),
|
| "path_chain_extended_scores": dict(path_chain_extended_scores),
|
| "effective_path_scores": dict(path_scores),
|
| "path_scores": dict(path_scores),
|
| "answer_type_scores": dict(answer_type_scores),
|
| "answer_plan_scores": dict(answer_plan_scores),
|
| "answer_plan_support_scores": dict(answer_plan_support_scores),
|
| "answer_plan_adjusted_scores": dict(answer_plan_adjusted_scores),
|
| "answer_plan_raw_ranked_event_ids": list(answer_plan_raw_ranked_event_ids),
|
| "answer_plan_ranked_event_ids": list(answer_plan_ranked_event_ids),
|
| "answer_plan_selected_event_ids": list(answer_plan_selected_event_ids),
|
| "answer_plan_current_event_ids": list(answer_plan_current_event_ids),
|
| "answer_plan_promotion_enabled": bool(answer_plan_promotion_enabled),
|
| "answer_plan_promotion_score_margin": round(float(answer_plan_promotion_score_margin), 6),
|
| "answer_plan_promotion_min_margin": round(float(answer_plan_promotion_min_margin), 6),
|
| "answer_plan_event_selection_threshold": round(float(answer_plan_event_selection_threshold), 6),
|
| "answer_plan_event_selection_top_k": int(answer_plan_event_selection_top_k),
|
| "focused_answer_type": focused_answer_type,
|
| "model_focused_answer_type": model_focused_answer_type,
|
| "selection_consistency_repaired": bool(selection_consistency_repaired),
|
| "selection_consistency_reason": selection_consistency_reason,
|
| "preferred_path_types": [],
|
| },
|
| }
|
| dominant_answer_type = _dominant_answer_type(question_analysis, answer_type_scores)
|
| preferred_path_types = _answer_type_preferred_path_types(question_analysis, answer_type_scores)
|
| if bool(memory_router_decision.get("memory_router_guided")):
|
| temporal_focus = _memory_router_allows(memory_router_decision, "temporal")
|
| else:
|
| temporal_focus = dominant_answer_type == "time" or bool(question_analysis.get("is_temporal", False))
|
| router_profile_focus = bool(memory_router_decision.get("memory_router_guided")) and _memory_router_allows(
|
| memory_router_decision,
|
| "profile",
|
| "resource",
|
| )
|
| learned_event_available = bool(event_scores)
|
| learned_path_available = bool(path_scores)
|
| ranked_events = [
|
| event_id
|
| for event_id, _ in sorted(
|
| (
|
| (event_id, event_scores.get(event_id, recall_event_scores.get(event_id, 0.0)))
|
| for event_id in recall_event_ids
|
| ),
|
| key=lambda item: (-float(item[1]), item[0]),
|
| )
|
| ]
|
| base_selected_event_count = min(
|
| len(ranked_events),
|
| max(
|
| self.support_path_k * 2,
|
| min(self.candidate_event_k, max(top_k, _HYBRID_SELECTED_EVENT_FLOOR)),
|
| ),
|
| )
|
| effective_path_scores = dict(path_scores)
|
| if not learned_path_available:
|
| effective_path_scores = {
|
| path_id: _calibrated_path_score(
|
| path=runtime_paths.get(path_id, {}),
|
| base_score=float(score),
|
| temporal_scores=temporal_scores,
|
| question_analysis=question_analysis,
|
| answer_type_scores=answer_type_scores,
|
| )
|
| for path_id, score in path_scores.items()
|
| }
|
| ranked_path_ids_all = [
|
| path_id
|
| for path_id, _ in sorted(
|
| ((path_id, float(score)) for path_id, score in effective_path_scores.items()),
|
| key=lambda item: (-float(item[1]), item[0]),
|
| )
|
| ]
|
| path_rescue_count = min(
|
| len(ranked_path_ids_all),
|
| max(self.support_path_k * 2, min(self.candidate_event_k, max(1, top_k))),
|
| )
|
| path_rescue_event_ids = _dedupe(
|
| _clean_text(runtime_paths.get(path_id, {}).get("event_id", ""))
|
| for path_id in ranked_path_ids_all[: max(1, path_rescue_count)]
|
| )
|
| selected_event_ids = _dedupe(
|
| [
|
| *path_rescue_event_ids,
|
| *ranked_events[: max(1, base_selected_event_count)],
|
| ]
|
| )
|
| if profile_first_event_ids:
|
| selected_event_ids = _dedupe([*profile_first_event_ids, *selected_event_ids])
|
| if answer_plan_ranked_event_ids:
|
| selected_event_ids = _dedupe([*selected_event_ids, *answer_plan_ranked_event_ids])
|
| selected_event_id_set = set(selected_event_ids)
|
| ranked_path_ids = [
|
| path_id
|
| for path_id in ranked_path_ids_all
|
| if _clean_text(runtime_paths.get(path_id, {}).get("event_id", "")) in selected_event_id_set
|
| ]
|
| if temporal_focus:
|
| focused_time_path_ids = [
|
| path_id
|
| for path_id in ranked_path_ids
|
| if _clean_text(runtime_paths.get(path_id, {}).get("type", "")) == "speaker_event_time"
|
| ]
|
| if focused_time_path_ids:
|
| ranked_path_ids = focused_time_path_ids
|
| path_limit_cap = _HYBRID_SELECTED_PATH_CAP
|
| if temporal_focus:
|
| path_limit_cap = _HYBRID_TEMPORAL_PATH_CAP
|
| elif dominant_answer_type == "profile" or router_profile_focus:
|
| path_limit_cap = _HYBRID_PROFILE_PATH_CAP
|
| selected_path_count = min(
|
| len(ranked_path_ids),
|
| max(1, min(max(1, self.support_path_k), min(max(1, top_k), path_limit_cap))),
|
| )
|
| selected_path_ids = ranked_path_ids[: max(1, selected_path_count)]
|
| final_hits: List[MemoryHit] = []
|
| seen_memory_ids = set()
|
| temporal_event_hits_added = 0
|
| for path_id in selected_path_ids:
|
| path = runtime_paths.get(path_id, {})
|
| event_id = _clean_text(path.get("event_id", ""))
|
| support_node_id = _path_support_node_id(path)
|
| path_type = _clean_text(path.get("type", ""))
|
| support_hit = _support_hit_for_path(path_type, grouped_hits.get(event_id, []))
|
| event_hit = _representative_event_hit(
|
| [*grouped_hits.get(event_id, []), *_event_record_hits_from_graph(self.graph, event_id)],
|
| query=query,
|
| )
|
| hit_pairs: List[tuple[MemoryHit | None, float]] = [(support_hit, path_scores.get(path_id, 0.0))]
|
| if not temporal_focus:
|
| hit_pairs.append((event_hit, path_scores.get(path_id, 0.0)))
|
| elif support_hit is None and event_hit is not None:
|
| hit_pairs.append((event_hit, path_scores.get(path_id, 0.0)))
|
| elif path_type == "speaker_event_time" and event_hit is not None and temporal_event_hits_added < 1:
|
| hit_pairs.append((event_hit, path_scores.get(path_id, 0.0)))
|
| temporal_event_hits_added += 1
|
| for raw_hit, path_score in hit_pairs:
|
| if raw_hit is None or raw_hit.memory_id in seen_memory_ids:
|
| continue
|
| seen_memory_ids.add(raw_hit.memory_id)
|
| recall_score = float(recall_event_scores.get(event_id, event_scores.get(event_id, 0.0)))
|
| event_score = float(event_scores.get(event_id, raw_hit.score))
|
| temporal_score = float(temporal_scores.get(support_node_id, 0.0))
|
| effective_path_score = float(effective_path_scores.get(path_id, path_score))
|
| hybrid_score = round(effective_path_score, 6) if learned_path_available else round(
|
| (0.55 * event_score)
|
| + (0.20 * float(path_score))
|
| + (0.15 * recall_score)
|
| + (0.10 * temporal_score),
|
| 6,
|
| )
|
| metadata = dict(raw_hit.metadata or {})
|
| metadata.update(
|
| {
|
| "event_id": event_id,
|
| **_answer_plan_hit_metadata(event_id),
|
| "path_id": path_id,
|
| "base_event_score": round(float(base_event_scores.get(event_id, event_score)), 6),
|
| "rerank_event_score": round(float(rerank_event_scores.get(event_id, event_score)), 6),
|
| "calibrated_event_score": round(float(calibrated_event_scores.get(event_id, event_score)), 6),
|
| "matrix_event_score": round(float(matrix_event_scores.get(event_id, 0.0)), 6),
|
| "event_fusion_delta_score": round(float(event_fusion_delta_scores.get(event_id, 0.0)), 6),
|
| "event_tunnel_support_score": round(float(event_tunnel_support_scores.get(event_id, 0.0)), 6),
|
| "event_tunnel_delta_score": round(float(event_tunnel_delta_scores.get(event_id, 0.0)), 6),
|
| "matrix_enabled": matrix_enabled,
|
| "event_calibration_enabled": event_calibration_enabled,
|
| "path_calibration_enabled": path_calibration_enabled,
|
| "event_tunnel_enabled": event_tunnel_enabled,
|
| "path_tunnel_enabled": path_tunnel_enabled,
|
| "final_event_fusion_enabled": final_event_fusion_enabled,
|
| "final_path_fusion_enabled": final_path_fusion_enabled,
|
| "decision_fusion_enabled": False,
|
| "event_fusion_enabled": event_fusion_enabled,
|
| "path_fusion_enabled": path_fusion_enabled,
|
| "event_score": round(event_score, 6),
|
| "recall_score": round(recall_score, 6),
|
| "path_score": round(float(path_score), 6),
|
| "base_path_score": round(float(base_path_scores.get(path_id, path_score)), 6),
|
| "calibrated_path_score": round(float(calibrated_path_scores.get(path_id, effective_path_score)), 6),
|
| "path_fusion_delta_score": round(float(path_fusion_delta_scores.get(path_id, 0.0)), 6),
|
| "path_tunnel_support_score": round(float(path_tunnel_support_scores.get(path_id, 0.0)), 6),
|
| "path_tunnel_delta_score": round(float(path_tunnel_delta_scores.get(path_id, 0.0)), 6),
|
| "path_model_score": round(float(path_model_scores.get(path_id, path_score)), 6),
|
| "path_chain_extension_enabled": path_chain_extension_enabled,
|
| "path_chain_extension_delta_score": round(float(path_chain_extension_delta_scores.get(path_id, 0.0)), 6),
|
| "path_chain_extended_score": round(float(path_chain_extended_scores.get(path_id, effective_path_score)), 6),
|
| "effective_path_score": round(effective_path_score, 6),
|
| "temporal_score": round(temporal_score, 6),
|
| "raw_public_score": round(float(raw_hit.score), 6),
|
| "hybrid_score": hybrid_score,
|
| "hybrid_score_source": (
|
| "learned_path_fusion"
|
| if path_fusion_enabled
|
| else "learned_path_score"
|
| if learned_path_available
|
| else "heuristic_mix"
|
| ),
|
| "evidence_snippet_role": "selected_path_support" if raw_hit is support_hit else "selected_path_event",
|
| "selected_event_ids": list(selected_event_ids),
|
| "selected_path_ids": list(selected_path_ids),
|
| }
|
| )
|
| final_hits.append(
|
| MemoryHit(
|
| memory_id=raw_hit.memory_id,
|
| category=raw_hit.category,
|
| value=raw_hit.value,
|
| relation=raw_hit.relation,
|
| anchors=list(raw_hit.anchors),
|
| score=hybrid_score,
|
| source_kind=raw_hit.source_kind,
|
| slot_key=raw_hit.slot_key,
|
| state=raw_hit.state,
|
| turn_index=int(raw_hit.turn_index),
|
| metadata=metadata,
|
| )
|
| )
|
| for event_rank, event_id in enumerate(selected_event_ids, start=1):
|
| event_hit = _representative_event_hit(
|
| [*grouped_hits.get(event_id, []), *_event_record_hits_from_graph(self.graph, event_id)],
|
| query=query,
|
| )
|
| if event_hit is None or event_hit.memory_id in seen_memory_ids:
|
| continue
|
| seen_memory_ids.add(event_hit.memory_id)
|
| recall_score = float(recall_event_scores.get(event_id, event_hit.score))
|
| event_score = float(event_scores.get(event_id, event_hit.score))
|
| hybrid_score = round(event_score, 6) if learned_event_available else round(
|
| (0.7 * event_score) + (0.3 * recall_score),
|
| 6,
|
| )
|
| metadata = dict(event_hit.metadata or {})
|
| metadata.update(
|
| {
|
| "event_id": event_id,
|
| **_answer_plan_hit_metadata(event_id),
|
| "path_id": "",
|
| "base_event_score": round(float(base_event_scores.get(event_id, event_score)), 6),
|
| "rerank_event_score": round(float(rerank_event_scores.get(event_id, event_score)), 6),
|
| "calibrated_event_score": round(float(calibrated_event_scores.get(event_id, event_score)), 6),
|
| "matrix_event_score": round(float(matrix_event_scores.get(event_id, 0.0)), 6),
|
| "event_fusion_delta_score": round(float(event_fusion_delta_scores.get(event_id, 0.0)), 6),
|
| "event_tunnel_support_score": round(float(event_tunnel_support_scores.get(event_id, 0.0)), 6),
|
| "event_tunnel_delta_score": round(float(event_tunnel_delta_scores.get(event_id, 0.0)), 6),
|
| "matrix_enabled": matrix_enabled,
|
| "event_calibration_enabled": event_calibration_enabled,
|
| "path_calibration_enabled": path_calibration_enabled,
|
| "event_tunnel_enabled": event_tunnel_enabled,
|
| "path_tunnel_enabled": path_tunnel_enabled,
|
| "final_event_fusion_enabled": final_event_fusion_enabled,
|
| "final_path_fusion_enabled": final_path_fusion_enabled,
|
| "decision_fusion_enabled": False,
|
| "event_fusion_enabled": event_fusion_enabled,
|
| "path_fusion_enabled": path_fusion_enabled,
|
| "event_score": round(event_score, 6),
|
| "recall_score": round(recall_score, 6),
|
| "path_score": 0.0,
|
| "base_path_score": 0.0,
|
| "calibrated_path_score": 0.0,
|
| "path_fusion_delta_score": 0.0,
|
| "path_tunnel_support_score": 0.0,
|
| "path_tunnel_delta_score": 0.0,
|
| "path_model_score": 0.0,
|
| "path_chain_extension_enabled": path_chain_extension_enabled,
|
| "path_chain_extension_delta_score": 0.0,
|
| "path_chain_extended_score": 0.0,
|
| "effective_path_score": 0.0,
|
| "temporal_score": 0.0,
|
| "raw_public_score": round(float(event_hit.score), 6),
|
| "hybrid_score": hybrid_score,
|
| "hybrid_score_source": (
|
| "learned_event_fusion"
|
| if event_fusion_enabled
|
| else "learned_event_score"
|
| if learned_event_available
|
| else "heuristic_event_mix"
|
| ),
|
| "evidence_snippet_role": "selected_event_representative",
|
| "selected_event_rank": int(event_rank),
|
| "selected_event_ids": list(selected_event_ids),
|
| "selected_path_ids": list(selected_path_ids),
|
| }
|
| )
|
| final_hits.append(
|
| MemoryHit(
|
| memory_id=event_hit.memory_id,
|
| category=event_hit.category,
|
| value=event_hit.value,
|
| relation=event_hit.relation,
|
| anchors=list(event_hit.anchors),
|
| score=hybrid_score,
|
| source_kind=event_hit.source_kind,
|
| slot_key=event_hit.slot_key,
|
| state=event_hit.state,
|
| turn_index=int(event_hit.turn_index),
|
| metadata=metadata,
|
| )
|
| )
|
| if not final_hits:
|
| final_hits = list(hits)
|
| if profile_first_hits:
|
| final_hits = _inject_profile_first_hits(
|
| final_hits,
|
| profile_first_hits,
|
| selected_event_ids=selected_event_ids,
|
| selected_path_ids=selected_path_ids,
|
| )
|
| final_hits = _coverage_preserving_final_hits(final_hits, selected_event_ids=selected_event_ids, top_k=top_k)
|
| final_hit_event_ids = _event_ids_from_hits(final_hits)
|
| final_missing_selected_event_ids = [event_id for event_id in selected_event_ids if event_id not in set(final_hit_event_ids)]
|
| return {
|
| "hits": final_hits,
|
| "metadata": {
|
| "retrieval_mode": "hybrid_node_scored",
|
| "hybrid_enabled": True,
|
| "hybrid_source": hybrid_source,
|
| **memory_router_decision,
|
| "profile_first_router_suppressed": bool(profile_first_router_suppressed),
|
| "recall_event_ids": list(recall_event_ids),
|
| "learned_recall_event_ids": list(learned_recall_event_ids),
|
| "model_recall_event_ids": list(model_recall_event_ids),
|
| "symbolic_recall_event_ids": list(symbolic_recall_event_ids),
|
| "embedder_index_recall_event_ids": list(embedder_index_event_ids),
|
| **embedder_index_metadata,
|
| "embedder_fusion_mode": embedder_fusion_mode or "off",
|
| "embedder_fusion_enabled": bool(embedder_fusion_enabled),
|
| "embedder_fusion_weight": round(float(self.embedder_fusion_weight), 6),
|
| "embedder_fusion_score_floor": round(float(self.embedder_fusion_score_floor), 6),
|
| "embedder_fusion_top_k": int(self.embedder_fusion_top_k),
|
| "embedder_fusion_select_k": int(self.embedder_fusion_select_k),
|
| "embedder_fusion_max_boost": round(float(self.embedder_fusion_max_boost), 6),
|
| "embedder_fusion_event_scores": dict(embedder_fusion_applied_event_scores),
|
| "embedder_fusion_boosts": dict(embedder_fusion_boosts),
|
| "embedder_fusion_selected_event_ids": list(embedder_fusion_selected_event_ids),
|
| "embedder_fusion_selected_path_ids": list(embedder_fusion_selected_path_ids),
|
| "profile_first_hybrid_enabled": bool(profile_first_event_ids),
|
| "profile_first_event_ids": list(profile_first_event_ids),
|
| "profile_first_memory_ids": list(profile_first_memory_ids),
|
| "hybrid_candidate_event_ids": list(hybrid_candidate_event_ids),
|
| "hybrid_candidate_union_enabled": True,
|
| "hybrid_candidate_union_rescored": bool(hybrid_candidate_union_rescored),
|
| "hybrid_candidate_union_added_event_ids": list(hybrid_candidate_union_added_event_ids),
|
| "hybrid_candidate_union_priority_changed": bool(hybrid_candidate_union_priority_changed),
|
| "rerank_candidate_event_ids": list(rerank_candidate_event_ids),
|
| "base_event_scores": dict(base_event_scores),
|
| "rerank_event_scores": dict(rerank_event_scores),
|
| "calibrated_event_scores": dict(calibrated_event_scores),
|
| "matrix_event_scores": dict(matrix_event_scores),
|
| "event_fusion_delta_scores": dict(event_fusion_delta_scores),
|
| "event_tunnel_support_scores": dict(event_tunnel_support_scores),
|
| "event_tunnel_delta_scores": dict(event_tunnel_delta_scores),
|
| "tri_maze_event_reverse_scores": dict(tri_maze_event_reverse_scores),
|
| "tri_maze_event_boundary_scores": dict(tri_maze_event_boundary_scores),
|
| "tri_maze_event_reverse_relations": dict(tri_maze_event_reverse_relations),
|
| "matrix_rerank_event_ids": list(matrix_rerank_event_ids),
|
| "matrix_enabled": matrix_enabled,
|
| "rerank_path_scores": dict(rerank_path_scores),
|
| "matrix_path_scores": dict(matrix_path_scores),
|
| "tri_maze_path_reverse_scores": dict(tri_maze_path_reverse_scores),
|
| "tri_maze_path_boundary_scores": dict(tri_maze_path_boundary_scores),
|
| "tri_maze_path_reverse_relations": dict(tri_maze_path_reverse_relations),
|
| "matrix_path_rerank_ids": list(matrix_path_rerank_ids),
|
| "matrix_path_enabled": matrix_path_enabled,
|
| "fusion_enabled": fusion_enabled,
|
| "event_calibration_enabled": event_calibration_enabled,
|
| "path_calibration_enabled": path_calibration_enabled,
|
| "event_tunnel_enabled": event_tunnel_enabled,
|
| "path_tunnel_enabled": path_tunnel_enabled,
|
| "final_event_fusion_enabled": final_event_fusion_enabled,
|
| "final_path_fusion_enabled": final_path_fusion_enabled,
|
| "decision_fusion_enabled": False,
|
| "decision_score_source": "",
|
| "event_fusion_enabled": event_fusion_enabled,
|
| "path_fusion_enabled": path_fusion_enabled,
|
| "selected_event_ids": list(selected_event_ids),
|
| "path_rescue_event_ids": list(path_rescue_event_ids),
|
| "selected_path_ids": list(selected_path_ids),
|
| "final_hit_event_ids": list(final_hit_event_ids),
|
| "final_hit_dia_ids": _dia_ids_from_hits(final_hits),
|
| "final_missing_selected_event_ids": list(final_missing_selected_event_ids),
|
| "selected_event_count": int(len(selected_event_ids)),
|
| "selected_path_count": int(len(selected_path_ids)),
|
| "temporal_scores": dict(temporal_scores),
|
| "recall_event_scores": dict(recall_event_scores),
|
| "event_scores": dict(event_scores),
|
| "base_path_scores": dict(base_path_scores),
|
| "calibrated_path_scores": dict(calibrated_path_scores),
|
| "path_fusion_delta_scores": dict(path_fusion_delta_scores),
|
| "path_tunnel_support_scores": dict(path_tunnel_support_scores),
|
| "path_tunnel_delta_scores": dict(path_tunnel_delta_scores),
|
| "path_model_scores": dict(path_model_scores),
|
| "path_chain_extension_enabled": path_chain_extension_enabled,
|
| "path_chain_extension_delta_scores": dict(path_chain_extension_delta_scores),
|
| "path_chain_extended_scores": dict(path_chain_extended_scores),
|
| "effective_path_scores": dict(effective_path_scores),
|
| "path_scores": dict(path_scores),
|
| "answer_type_scores": dict(answer_type_scores),
|
| "answer_plan_scores": dict(answer_plan_scores),
|
| "answer_plan_support_scores": dict(answer_plan_support_scores),
|
| "answer_plan_adjusted_scores": dict(answer_plan_adjusted_scores),
|
| "answer_plan_raw_ranked_event_ids": list(answer_plan_raw_ranked_event_ids),
|
| "answer_plan_ranked_event_ids": list(answer_plan_ranked_event_ids),
|
| "answer_plan_selected_event_ids": list(answer_plan_selected_event_ids),
|
| "answer_plan_current_event_ids": list(answer_plan_current_event_ids),
|
| "answer_plan_promotion_enabled": bool(answer_plan_promotion_enabled),
|
| "answer_plan_promotion_score_margin": round(float(answer_plan_promotion_score_margin), 6),
|
| "answer_plan_promotion_min_margin": round(float(answer_plan_promotion_min_margin), 6),
|
| "answer_plan_event_selection_threshold": round(float(answer_plan_event_selection_threshold), 6),
|
| "answer_plan_event_selection_top_k": int(answer_plan_event_selection_top_k),
|
| "focused_answer_type": dominant_answer_type,
|
| "preferred_path_types": list(preferred_path_types),
|
| },
|
| }
|
|
|
| def retrieve(self, query: str, *, top_k: int = 6) -> MemoryRetrieval:
|
| self._reload_graph()
|
| start = time.perf_counter()
|
| candidate_top_k = max(top_k, self.candidate_event_k if self.retrieval_mode == "hybrid_node_scored" else min(max(top_k, 18), 48))
|
| payload = self.graph.retrieve(query, top_k=candidate_top_k)
|
| hits = [_raw_hit_to_memory_hit(item) for item in payload.get("hits", []) or []]
|
| scored_lookup = {hit.memory_id: hit for hit in hits if hit.memory_id}
|
| active_hits = _restore_hit_scores([_raw_hit_to_memory_hit(item) for item in payload.get("active_hits", []) or []], scored_lookup)
|
| history_hits = _restore_hit_scores([_raw_hit_to_memory_hit(item) for item in payload.get("history_hits", []) or []], scored_lookup)
|
| stale_hits = _restore_hit_scores([_raw_hit_to_memory_hit(item) for item in payload.get("stale_hits", []) or []], scored_lookup)
|
| overwrite_hits = _restore_hit_scores([_raw_hit_to_memory_hit(item) for item in payload.get("overwrite_hits", []) or []], scored_lookup)
|
| false_hits = _restore_hit_scores([_raw_hit_to_memory_hit(item) for item in payload.get("false_hits", []) or []], scored_lookup)
|
| public_hits = _public_graph_hits(self.graph) if self.retrieval_mode == "hybrid_node_scored" else []
|
| hybrid_payload = self._hybrid_node_scored_hits(query, hits, top_k=top_k, public_hits=public_hits)
|
| hits = list(hybrid_payload.get("hits", []) or hits)
|
| hybrid_metadata = dict(hybrid_payload.get("metadata", {}) or {})
|
| memory_router_decision = _memory_router_decision(
|
| hybrid_metadata,
|
| mode=self.memory_router_mode,
|
| threshold=self.memory_router_threshold,
|
| margin=self.memory_router_margin,
|
| )
|
| current_subject_payload = _current_subject_protected_hits(
|
| query=query,
|
| graph=self.graph,
|
| final_hits=hits,
|
| top_k=top_k,
|
| )
|
| hits = list(current_subject_payload.get("hits", hits) or hits)
|
| current_subject_metadata = dict(current_subject_payload.get("metadata", {}) or {})
|
| audit_anchor_payload = _audit_anchor_protected_hits(
|
| query=query,
|
| final_hits=hits,
|
| candidate_hits=list(public_hits) + list(active_hits) + list(history_hits),
|
| metadata={**dict(payload.get("metadata", {}) or {}), **hybrid_metadata, **current_subject_metadata},
|
| top_k=top_k,
|
| )
|
| hits = list(audit_anchor_payload.get("hits", hits) or hits)
|
| audit_anchor_metadata = dict(audit_anchor_payload.get("metadata", {}) or {})
|
| identifier_payload = _identifier_protected_hits(
|
| query=query,
|
| final_hits=hits,
|
| candidate_hits=list(public_hits) + list(active_hits) + list(history_hits),
|
| top_k=top_k,
|
| )
|
| hits = list(identifier_payload.get("hits", hits) or hits)
|
| identifier_metadata = dict(identifier_payload.get("metadata", {}) or {})
|
| if _memory_router_allows(memory_router_decision, "path_tunnel", "topic_tunnel"):
|
| depth_chain_payload = _depth_chain_protected_hits(
|
| query=query,
|
| graph=self.graph,
|
| final_hits=hits,
|
| top_k=top_k,
|
| )
|
| hits = list(depth_chain_payload.get("hits", hits) or hits)
|
| depth_chain_metadata = dict(depth_chain_payload.get("metadata", {}) or {})
|
| else:
|
| depth_chain_metadata = {
|
| "depth_chain_protected_enabled": False,
|
| "depth_chain_router_suppressed": True,
|
| }
|
| if _memory_router_allows(memory_router_decision, "profile", "resource"):
|
| profile_focused_payload = _profile_focused_pack_hits(
|
| self.graph,
|
| query,
|
| hits,
|
| top_k=top_k,
|
| )
|
| hits = list(profile_focused_payload.get("hits", hits) or hits)
|
| profile_focused_metadata = dict(profile_focused_payload.get("metadata", {}) or {})
|
| else:
|
| profile_focused_metadata = {
|
| "profile_focused_pack_enabled": False,
|
| "profile_focused_router_suppressed": True,
|
| }
|
| if _memory_router_allows(memory_router_decision, "topic_tunnel"):
|
| topic_bucket_payload = _topic_bucket_rerank_hits(self.graph, query, hits, top_k=top_k)
|
| topic_bucket_hits = topic_bucket_payload.get("hits", hits)
|
| hits = list(hits if topic_bucket_hits is None else topic_bucket_hits)
|
| topic_bucket_metadata = dict(topic_bucket_payload.get("metadata", {}) or {})
|
| else:
|
| topic_bucket_metadata = {
|
| "topic_bucket_rerank_enabled": False,
|
| "topic_bucket_router_suppressed": True,
|
| }
|
| temporal_runtime_payload = self._temporal_runtime_pack(query)
|
| temporal_evidence_payload = self._apply_temporal_evidence_pack_to_hits(
|
| hits,
|
| temporal_runtime_payload,
|
| top_k=top_k,
|
| )
|
| hits = list(temporal_evidence_payload.get("hits", hits) or hits)
|
| temporal_runtime_metadata = dict(temporal_evidence_payload.get("metadata", {}) or {})
|
| injection_planner_payload = self._apply_injection_planner_to_hits(query, hits, top_k=top_k)
|
| hits = list(injection_planner_payload.get("hits", hits) or hits)
|
| injection_planner_metadata = dict(injection_planner_payload.get("metadata", {}) or {})
|
| facet_query_pack_payload = _facet_query_pack_hits(
|
| self.graph,
|
| query,
|
| hits,
|
| top_k=top_k,
|
| )
|
| hits = list(facet_query_pack_payload.get("hits", hits) or hits)
|
| facet_query_pack_metadata = dict(facet_query_pack_payload.get("metadata", {}) or {})
|
| unit_coverage_mode = _normalize(os.getenv("TMCRA_UNIT_COVERAGE_PACK_MODE", "on"))
|
| if unit_coverage_mode in _MULTI_UNIT_CHAIN_DISABLED_MODES:
|
| unit_coverage_metadata = {
|
| "unit_coverage_pack_enabled": False,
|
| "unit_coverage_reason": "disabled",
|
| }
|
| else:
|
| unit_coverage_payload = _unit_coverage_pack_hits(
|
| self.graph,
|
| query,
|
| hits,
|
| top_k=top_k,
|
| )
|
| hits = list(unit_coverage_payload.get("hits", hits) or hits)
|
| unit_coverage_metadata = dict(unit_coverage_payload.get("metadata", {}) or {})
|
| if _normalize(os.getenv("TMCRA_MULTI_UNIT_CHAIN_SLOT_MODE", "on")) not in _MULTI_UNIT_CHAIN_DISABLED_MODES:
|
| multi_unit_chain_slot_payload = _multi_unit_chain_slot_hits(
|
| self.graph,
|
| query,
|
| hits,
|
| top_k=top_k,
|
| )
|
| hits = list(multi_unit_chain_slot_payload.get("hits", hits) or hits)
|
| multi_unit_chain_slot_metadata = dict(multi_unit_chain_slot_payload.get("metadata", {}) or {})
|
| else:
|
| multi_unit_chain_slot_metadata = {
|
| "multi_unit_chain_slot_enabled": False,
|
| "multi_unit_chain_slot_reason": "disabled",
|
| }
|
| profile_protected_reinserted_count = 0
|
| profile_protected_ids = _dedupe(
|
| [
|
| *list(profile_focused_metadata.get("profile_focused_pack_memory_ids", []) or []),
|
| *list(profile_focused_metadata.get("profile_first_memory_ids", []) or []),
|
| ],
|
| max_items=max(1, min(6, int(top_k or 1))),
|
| )
|
| if profile_protected_ids:
|
| existing_hits_by_id = {hit.memory_id: hit for hit in hits if hit.memory_id}
|
| protected_hits: List[MemoryHit] = []
|
| for memory_id in profile_protected_ids:
|
| hit = existing_hits_by_id.get(memory_id)
|
| if hit is None:
|
| record = getattr(self.graph, "records_by_id", {}).get(memory_id)
|
| if record is None:
|
| continue
|
| hit = _memory_hit_from_record(record)
|
| metadata = dict(hit.metadata or {})
|
| metadata.update(
|
| {
|
| "profile_first_hybrid_rescue": True,
|
| "profile_protected_slot": True,
|
| "evidence_snippet_role": "profile_protected_slot",
|
| }
|
| )
|
| protected_hits.append(
|
| MemoryHit(
|
| memory_id=hit.memory_id,
|
| category=hit.category,
|
| value=hit.value,
|
| relation=hit.relation,
|
| anchors=list(hit.anchors),
|
| score=max(float(hit.score), 4.8),
|
| source_kind=hit.source_kind,
|
| slot_key=hit.slot_key,
|
| state=hit.state,
|
| turn_index=int(hit.turn_index),
|
| metadata=metadata,
|
| )
|
| )
|
| if protected_hits:
|
| profile_protected_reinserted_count = len(protected_hits)
|
| seen_profile_ids = {hit.memory_id for hit in protected_hits if hit.memory_id}
|
| hits = [*protected_hits, *[hit for hit in hits if hit.memory_id not in seen_profile_ids]]
|
| embedder_fusion_output_event_ids = [
|
| _clean_text(event_id)
|
| for event_id in list(hybrid_metadata.get("embedder_fusion_selected_event_ids", []) or [])
|
| if _clean_text(event_id)
|
| ]
|
| embedder_fusion_output_reordered = False
|
| if embedder_fusion_output_event_ids:
|
| before_order = _event_ids_from_hits(hits)
|
| seen_output_memory_ids = {hit.memory_id for hit in hits}
|
| for event_id in embedder_fusion_output_event_ids:
|
| event_hit = _representative_event_hit(_event_record_hits_from_graph(self.graph, event_id), query=query)
|
| if event_hit is None or event_hit.memory_id in seen_output_memory_ids:
|
| continue
|
| metadata = dict(event_hit.metadata or {})
|
| metadata.update(
|
| {
|
| "event_id": event_id,
|
| "evidence_snippet_role": "embedder_fusion_event_representative",
|
| "embedder_fusion_event_representative": True,
|
| }
|
| )
|
| hits.append(
|
| MemoryHit(
|
| memory_id=event_hit.memory_id,
|
| category=event_hit.category,
|
| value=event_hit.value,
|
| relation=event_hit.relation,
|
| anchors=list(event_hit.anchors),
|
| score=float(event_hit.score),
|
| source_kind=event_hit.source_kind,
|
| slot_key=event_hit.slot_key,
|
| state=event_hit.state,
|
| turn_index=int(event_hit.turn_index),
|
| metadata=metadata,
|
| )
|
| )
|
| seen_output_memory_ids.add(event_hit.memory_id)
|
| hits = _coverage_preserving_final_hits(
|
| hits,
|
| selected_event_ids=_dedupe([*embedder_fusion_output_event_ids, *before_order]),
|
| top_k=top_k,
|
| )
|
| embedder_fusion_output_reordered = before_order != _event_ids_from_hits(hits)
|
| retrieval_context_tokens = int(payload.get("context_token_estimate", _estimate_tokens_from_hits(hits)))
|
| self._last_retrieval_context_tokens = retrieval_context_tokens
|
| result = MemoryRetrieval(
|
| concepts=list(payload.get("concepts", []) or []),
|
| relations=list(payload.get("relations", []) or []),
|
| hits=hits,
|
| active_hits=active_hits,
|
| history_hits=history_hits,
|
| stale_hits=stale_hits,
|
| overwrite_hits=overwrite_hits,
|
| false_hits=false_hits,
|
| retrieval_seconds=time.perf_counter() - start,
|
| context_token_estimate=retrieval_context_tokens,
|
| retrieval_context_token_estimate=retrieval_context_tokens,
|
| metadata={
|
| "query_id": payload.get("query_id", ""),
|
| **dict(payload.get("metadata", {}) or {}),
|
| **hybrid_metadata,
|
| **memory_router_decision,
|
| **current_subject_metadata,
|
| **audit_anchor_metadata,
|
| **identifier_metadata,
|
| **depth_chain_metadata,
|
| **profile_focused_metadata,
|
| **topic_bucket_metadata,
|
| **temporal_runtime_metadata,
|
| **injection_planner_metadata,
|
| **facet_query_pack_metadata,
|
| **unit_coverage_metadata,
|
| **multi_unit_chain_slot_metadata,
|
| "profile_protected_reinserted_count": profile_protected_reinserted_count,
|
| "embedder_fusion_output_event_ids": list(embedder_fusion_output_event_ids),
|
| "embedder_fusion_output_reordered": bool(embedder_fusion_output_reordered),
|
| },
|
| )
|
| self._persist_graph()
|
| return result
|
|
|
| def export_dialog_graph(self, *, mode: str = "light") -> Dict[str, Any]:
|
| self._reload_graph()
|
| return self.graph.export_graph(
|
| snapshot_points=(1000, 5000, 10000, 20000, 50000, 100000, 200000, 300000, 500000),
|
| mode=mode,
|
| )
|
|
|
| def export_dialog_graph_mermaid(self) -> str:
|
| self._reload_graph()
|
| return self.graph.export_mermaid()
|
|
|
| def register_answer_support(self, *, answer_id: str, memory_ids: List[str], query_id: str = "", answer_text: str = "") -> None:
|
| self._reload_graph()
|
| self.graph.register_answer_support(answer_id=answer_id, memory_ids=memory_ids, query_id=query_id, answer_text=answer_text)
|
| self._persist_graph()
|
|
|
| def telemetry_snapshot(self) -> Dict[str, Any]:
|
| self._reload_graph()
|
| return self.graph.summary()
|
|
|
| def stats(self) -> Dict[str, Any]:
|
| self._reload_graph()
|
| storage = self._storage_breakdown()
|
| return _state_stats(
|
| storage_bytes=storage["storage_bytes"],
|
| retrieval_context_tokens=self._last_retrieval_context_tokens,
|
| total_state_tokens=storage["total_state_token_estimate"],
|
| core_storage_bytes=storage["core_storage_bytes"],
|
| audit_storage_bytes=storage["audit_storage_bytes"],
|
| core_state_token_estimate=storage["core_state_token_estimate"],
|
| audit_state_token_estimate=storage["audit_state_token_estimate"],
|
| lightweight_stats=bool(self.lightweight_stats),
|
| **self.graph.summary(),
|
| )
|
|
|
| def storage_bytes(self) -> int:
|
| self._reload_graph()
|
| return self._storage_breakdown()["storage_bytes"]
|
|
|
| def build_prompt_context(self, query: str, *, top_k: int = 8) -> Dict[str, Any]:
|
| retrieval = self.retrieve(query, top_k=top_k)
|
| retrieval_payload = retrieval.to_dict()
|
| stats = self.stats()
|
| context_summary = _graph_prompt_state_summary(self.graph, retrieval)
|
| prompt_context_payload = {
|
| "query": query,
|
| "retrieval": retrieval_payload,
|
| "context_summary": context_summary,
|
| }
|
| prompt_context_chars = len(json.dumps(prompt_context_payload, ensure_ascii=False))
|
| prompt_context_tokens_est = _estimate_tokens(json.dumps(prompt_context_payload, ensure_ascii=False))
|
| return {
|
| "mode": "graph_session_memory_v2",
|
| "query": query,
|
| "retrieval": retrieval_payload,
|
| "stats": stats,
|
| "state": context_summary,
|
| "context_summary": context_summary,
|
| "prompt_context_chars": int(prompt_context_chars),
|
| "prompt_context_tokens_est": int(prompt_context_tokens_est),
|
| "context_truncated": bool(context_summary.get("context_truncated", False)),
|
| "truncation_reason": _clean_text(context_summary.get("truncation_reason", "")),
|
| }
|
|
|
|
|
| class SummaryWindowMemoryAdapter(MemoryAdapter):
|
| name = "summary_window_memory"
|
|
|
| def __init__(self, *, window_size: int = 24, auto_extract: bool = False) -> None:
|
| self.extractor = SessionMemoryExtractor()
|
| self.window_size = max(4, int(window_size))
|
| self.turn_index = 0
|
| self.active_slots: Dict[str, SessionMemoryRecordV2] = {}
|
| self.recent_turns: deque[Dict[str, Any]] = deque(maxlen=self.window_size)
|
| self.auto_extract = bool(auto_extract)
|
| self._last_retrieval_context_tokens = 0
|
|
|
| def reset(self) -> None:
|
| self.turn_index = 0
|
| self.active_slots = {}
|
| self.recent_turns = deque(maxlen=self.window_size)
|
| self._last_retrieval_context_tokens = 0
|
|
|
| def ingest_turn(
|
| self,
|
| user_text: str,
|
| assistant_text: str = "",
|
| *,
|
| answer_payload: Dict[str, Any] | None = None,
|
| extraction_result: Dict[str, Any] | None = None,
|
| ) -> None:
|
| self.turn_index += 1
|
| records = _build_turn_records(
|
| self.extractor,
|
| user_text=user_text,
|
| answer_payload=answer_payload,
|
| extraction_result=extraction_result,
|
| turn_index=self.turn_index,
|
| allow_auto_extract=self.auto_extract,
|
| )
|
| for record in records:
|
| previous = self.active_slots.get(record.slot_key)
|
| if previous:
|
| previous.state = "superseded"
|
| record.supersedes.append(previous.memory_id)
|
| record.state = "active"
|
| self.active_slots[record.slot_key] = record
|
| self.recent_turns.append(
|
| {
|
| "turn_index": self.turn_index,
|
| "text": _clean_text(user_text),
|
| "assistant": _clean_text(assistant_text),
|
| }
|
| )
|
|
|
| def retrieve(self, query: str, *, top_k: int = 6) -> MemoryRetrieval:
|
| start = time.perf_counter()
|
| query_tokens = set(_tokenize(query))
|
| hints = set(infer_category_hints(query))
|
| scored: List[tuple[float, SessionMemoryRecordV2]] = []
|
| for record in self.active_slots.values():
|
| token_set = record.token_set()
|
| overlap = len(query_tokens & token_set) if query_tokens and token_set else 0
|
| score = overlap / max(1, len(query_tokens | token_set)) if query_tokens and token_set else 0.0
|
| if hints and record.category in hints:
|
| score += 0.22
|
| if record.slot_key.lower() in _normalize(query):
|
| score += 0.12
|
| score += min(0.08, record.turn_index * 0.0004)
|
| score += 0.2
|
| if score > 0:
|
| scored.append((score, record))
|
| scored.sort(key=lambda item: (item[0], item[1].turn_index), reverse=True)
|
| selected_records = [record for _, record in scored[:top_k]]
|
| hits = [
|
| MemoryHit(
|
| memory_id=record.memory_id,
|
| category=record.category,
|
| value=record.value,
|
| relation=record.relation,
|
| anchors=list(record.anchor_concepts),
|
| score=float(score),
|
| source_kind=record.source_kind,
|
| slot_key=record.slot_key,
|
| state=record.state,
|
| turn_index=record.turn_index,
|
| metadata={"window_size": self.window_size},
|
| )
|
| for score, record in scored[:top_k]
|
| ]
|
| concepts = []
|
| relations = []
|
| for hit in hits:
|
| concepts.append({"concept": hit.value, "type": hit.category, "source_kind": hit.source_kind})
|
| for anchor in hit.anchors[:2]:
|
| concepts.append({"concept": anchor, "type": "context", "source_kind": hit.source_kind})
|
| relation = _relation_hit(hit, weight_bias=0.06)
|
| if relation:
|
| relations.append(relation)
|
| retrieval_context_tokens = _estimate_tokens_from_hits(hits)
|
| self._last_retrieval_context_tokens = retrieval_context_tokens
|
| return MemoryRetrieval(
|
| concepts=concepts,
|
| relations=relations,
|
| hits=hits,
|
| active_hits=list(hits),
|
| retrieval_seconds=time.perf_counter() - start,
|
| context_token_estimate=retrieval_context_tokens,
|
| retrieval_context_token_estimate=retrieval_context_tokens,
|
| metadata={
|
| "records": len(self.active_slots),
|
| "window_size": self.window_size,
|
| "recent_turns": len(self.recent_turns),
|
| },
|
| )
|
|
|
| def stats(self) -> Dict[str, Any]:
|
| payload = {
|
| "active_slots": {slot: record.to_dict() for slot, record in self.active_slots.items()},
|
| "recent_turns": list(self.recent_turns),
|
| }
|
| total_state_tokens = _estimate_tokens(json.dumps(payload, ensure_ascii=False))
|
| return _state_stats(
|
| storage_bytes=self.storage_bytes(),
|
| retrieval_context_tokens=self._last_retrieval_context_tokens,
|
| total_state_tokens=total_state_tokens,
|
| records=len(self.active_slots),
|
| active_slots=len(self.active_slots),
|
| recent_turns=len(self.recent_turns),
|
| )
|
|
|
| def storage_bytes(self) -> int:
|
| payload = {
|
| "active_slots": {slot: record.to_dict() for slot, record in self.active_slots.items()},
|
| "recent_turns": list(self.recent_turns),
|
| }
|
| return len(json.dumps(payload, ensure_ascii=False).encode("utf-8"))
|
|
|
| def build_prompt_context(self, query: str, *, top_k: int = 8) -> Dict[str, Any]:
|
| return {
|
| "mode": "summary_window_memory",
|
| "query": query,
|
| "retrieval": self.retrieve(query, top_k=top_k).to_dict(),
|
| "stats": self.stats(),
|
| "state": {
|
| "active_slots": {slot: record.to_dict() for slot, record in self.active_slots.items()},
|
| "recent_turns": list(self.recent_turns),
|
| },
|
| }
|
|
|
|
|
| @dataclass(slots=True)
|
| class _VectorRecord:
|
| memory_id: str
|
| category: str
|
| value: str
|
| relation: str
|
| anchors: List[str]
|
| tokens: List[str]
|
| turn_index: int
|
| slot_key: str = ""
|
| active: bool = True
|
| source_kind: str = "vector_memory"
|
| metadata: Dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
| class VectorRAGMemoryAdapter(MemoryAdapter):
|
| name = "vector_rag_memory"
|
|
|
| def __init__(self, *, auto_extract: bool = False) -> None:
|
| self.extractor = SessionMemoryExtractor()
|
| self.records: List[_VectorRecord] = []
|
| self.turn_index = 0
|
| self.auto_extract = bool(auto_extract)
|
| self._last_retrieval_context_tokens = 0
|
|
|
| def reset(self) -> None:
|
| self.records = []
|
| self.turn_index = 0
|
| self._last_retrieval_context_tokens = 0
|
|
|
| def ingest_turn(
|
| self,
|
| user_text: str,
|
| assistant_text: str = "",
|
| *,
|
| answer_payload: Dict[str, Any] | None = None,
|
| extraction_result: Dict[str, Any] | None = None,
|
| ) -> None:
|
| _ = assistant_text
|
| self.turn_index += 1
|
| records = _build_turn_records(
|
| self.extractor,
|
| user_text=user_text,
|
| answer_payload=answer_payload,
|
| extraction_result=extraction_result,
|
| turn_index=self.turn_index,
|
| allow_auto_extract=self.auto_extract,
|
| )
|
| for record in records:
|
| if record.slot_key:
|
| for previous in self.records:
|
| if previous.slot_key == record.slot_key and previous.active:
|
| previous.active = False
|
| self.records.append(
|
| _VectorRecord(
|
| memory_id=record.memory_id,
|
| category=record.category,
|
| value=record.value,
|
| relation=record.relation,
|
| anchors=list(record.anchor_concepts),
|
| tokens=list(record.token_set()),
|
| turn_index=record.turn_index,
|
| slot_key=record.slot_key,
|
| active=record.state == "active",
|
| source_kind=record.source_kind,
|
| metadata=dict(record.metadata),
|
| )
|
| )
|
|
|
| def retrieve(self, query: str, *, top_k: int = 6) -> MemoryRetrieval:
|
| start = time.perf_counter()
|
| query_tokens = set(_tokenize(query))
|
| hints = set(infer_category_hints(query))
|
| scored: List[tuple[float, _VectorRecord]] = []
|
| for record in self.records:
|
| token_set = set(record.tokens)
|
| overlap = len(query_tokens & token_set) if query_tokens and token_set else 0
|
| score = overlap / max(1, len(query_tokens | token_set)) if query_tokens and token_set else 0.0
|
| if hints and record.category in hints:
|
| score += 0.18
|
| if record.slot_key and record.slot_key.lower() in _normalize(query):
|
| score += 0.1
|
| score += min(0.12, record.turn_index * 0.0004)
|
| score += 0.18 if record.active else -0.25
|
| if score > 0:
|
| scored.append((score, record))
|
| if not scored:
|
| for record in self.records[-top_k:]:
|
| scored.append((0.05 + (0.15 if record.active else 0.0), record))
|
| scored.sort(key=lambda item: (item[0], item[1].active, item[1].turn_index), reverse=True)
|
| hits = [
|
| MemoryHit(
|
| memory_id=record.memory_id,
|
| category=record.category,
|
| value=record.value,
|
| relation=record.relation,
|
| anchors=list(record.anchors),
|
| score=float(score),
|
| source_kind=record.source_kind,
|
| slot_key=record.slot_key,
|
| state="active" if record.active else "superseded",
|
| turn_index=record.turn_index,
|
| metadata=dict(record.metadata),
|
| )
|
| for score, record in scored[:top_k]
|
| ]
|
| concepts = []
|
| relations = []
|
| for hit in hits:
|
| concepts.append({"concept": hit.value, "type": hit.category, "source_kind": hit.source_kind})
|
| for anchor in hit.anchors[:2]:
|
| concepts.append({"concept": anchor, "type": "context", "source_kind": hit.source_kind})
|
| relation = _relation_hit(hit)
|
| if relation:
|
| relations.append(relation)
|
| retrieval_context_tokens = _estimate_tokens_from_hits(hits)
|
| self._last_retrieval_context_tokens = retrieval_context_tokens
|
| return MemoryRetrieval(
|
| concepts=concepts,
|
| relations=relations,
|
| hits=hits,
|
| active_hits=[hit for hit in hits if hit.state == "active"],
|
| history_hits=[hit for hit in hits if hit.state != "active"],
|
| retrieval_seconds=time.perf_counter() - start,
|
| context_token_estimate=retrieval_context_tokens,
|
| retrieval_context_token_estimate=retrieval_context_tokens,
|
| metadata={
|
| "records": len(self.records),
|
| "active_records": sum(1 for item in self.records if item.active),
|
| },
|
| )
|
|
|
| def stats(self) -> Dict[str, Any]:
|
| payload = [
|
| {
|
| "memory_id": record.memory_id,
|
| "category": record.category,
|
| "value": record.value,
|
| "relation": record.relation,
|
| "anchors": record.anchors,
|
| "slot_key": record.slot_key,
|
| "active": record.active,
|
| }
|
| for record in self.records
|
| ]
|
| total_state_tokens = _estimate_tokens(json.dumps(payload, ensure_ascii=False))
|
| return _state_stats(
|
| storage_bytes=self.storage_bytes(),
|
| retrieval_context_tokens=self._last_retrieval_context_tokens,
|
| total_state_tokens=total_state_tokens,
|
| records=len(self.records),
|
| active_records=sum(1 for item in self.records if item.active),
|
| )
|
|
|
| def storage_bytes(self) -> int:
|
| payload = [
|
| {
|
| "memory_id": record.memory_id,
|
| "category": record.category,
|
| "value": record.value,
|
| "relation": record.relation,
|
| "anchors": record.anchors,
|
| "slot_key": record.slot_key,
|
| "active": record.active,
|
| }
|
| for record in self.records
|
| ]
|
| return len(json.dumps(payload, ensure_ascii=False).encode("utf-8"))
|
|
|
| def build_prompt_context(self, query: str, *, top_k: int = 8) -> Dict[str, Any]:
|
| state = [
|
| {
|
| "memory_id": record.memory_id,
|
| "category": record.category,
|
| "value": record.value,
|
| "relation": record.relation,
|
| "anchors": list(record.anchors),
|
| "slot_key": record.slot_key,
|
| "active": bool(record.active),
|
| "turn_index": int(record.turn_index),
|
| }
|
| for record in self.records
|
| ]
|
| return {
|
| "mode": "vector_rag_memory",
|
| "query": query,
|
| "retrieval": self.retrieve(query, top_k=top_k).to_dict(),
|
| "stats": self.stats(),
|
| "state": state,
|
| }
|
|
|
|
|
| class FullHistoryMemoryAdapter(MemoryAdapter):
|
| name = "full_history_memory"
|
|
|
| def __init__(self) -> None:
|
| self.turns: List[Dict[str, str]] = []
|
| self._last_retrieval_context_tokens = 0
|
|
|
| def reset(self) -> None:
|
| self.turns = []
|
| self._last_retrieval_context_tokens = 0
|
|
|
| def ingest_turn(
|
| self,
|
| user_text: str,
|
| assistant_text: str = "",
|
| *,
|
| answer_payload: Dict[str, Any] | None = None,
|
| extraction_result: Dict[str, Any] | None = None,
|
| ) -> None:
|
| _ = answer_payload, extraction_result
|
| self.turns.append({"user": _clean_text(user_text), "assistant": _clean_text(assistant_text)})
|
|
|
| def retrieve(self, query: str, *, top_k: int = 6) -> MemoryRetrieval:
|
| start = time.perf_counter()
|
| query_tokens = set(_tokenize(query))
|
| scored: List[tuple[float, Dict[str, str], int]] = []
|
| for index, turn in enumerate(self.turns):
|
| combined = f"{turn.get('user', '')} {turn.get('assistant', '')}"
|
| token_set = set(_tokenize(combined))
|
| if not token_set:
|
| continue
|
| overlap = len(query_tokens & token_set) if query_tokens else 0
|
| score = overlap / max(1, len(query_tokens | token_set)) if query_tokens else 0.0
|
| scored.append((score, turn, index))
|
| scored.sort(key=lambda item: (item[0], item[2]), reverse=True)
|
| hits = [
|
| MemoryHit(
|
| memory_id=f"turn:{index}",
|
| category="history_turn",
|
| value=turn.get("user", ""),
|
| relation="conversation_context",
|
| anchors=[turn.get("assistant", "")] if turn.get("assistant") else [],
|
| score=float(score),
|
| source_kind="full_history",
|
| slot_key=f"turn.{index}",
|
| state="active",
|
| turn_index=index + 1,
|
| )
|
| for score, turn, index in scored[:top_k]
|
| if turn.get("user", "")
|
| ]
|
| retrieval_context_tokens = _estimate_tokens(json.dumps(self.turns, ensure_ascii=False))
|
| self._last_retrieval_context_tokens = retrieval_context_tokens
|
| return MemoryRetrieval(
|
| hits=hits,
|
| active_hits=list(hits),
|
| retrieval_seconds=time.perf_counter() - start,
|
| context_token_estimate=retrieval_context_tokens,
|
| retrieval_context_token_estimate=retrieval_context_tokens,
|
| metadata={"records": len(self.turns)},
|
| )
|
|
|
| def stats(self) -> Dict[str, Any]:
|
| total_state_tokens = _estimate_tokens(json.dumps(self.turns, ensure_ascii=False))
|
| return _state_stats(
|
| storage_bytes=self.storage_bytes(),
|
| retrieval_context_tokens=self._last_retrieval_context_tokens,
|
| total_state_tokens=total_state_tokens,
|
| records=len(self.turns),
|
| )
|
|
|
| def storage_bytes(self) -> int:
|
| return len(json.dumps(self.turns, ensure_ascii=False).encode("utf-8"))
|
|
|
| def build_prompt_context(self, query: str, *, top_k: int = 8) -> Dict[str, Any]:
|
| return {
|
| "mode": "full_history_memory",
|
| "query": query,
|
| "retrieval": self.retrieve(query, top_k=top_k).to_dict(),
|
| "stats": self.stats(),
|
| "state": {
|
| "turns": list(self.turns),
|
| },
|
| }
|
|
|