Spaces:
Running on Zero
Running on Zero
Bellok commited on
Commit ·
9fcde48
1
Parent(s): e7d33ec
feat: Add synthesis functionality with session summary and artifact ingestion
Browse files- app.py +211 -201
- tests/test_synthesis_bridge.py +99 -0
- warbler_cda/synthesis_bridge.py +188 -0
app.py
CHANGED
|
@@ -9,11 +9,14 @@ Provides a web UI for the FractalStat RAG system with GPU acceleration.
|
|
| 9 |
import gradio as gr
|
| 10 |
import os
|
| 11 |
import time
|
| 12 |
-
from typing import List, Dict, Any
|
| 13 |
|
| 14 |
from warbler_cda.answer_generator import AnswerGenerator
|
| 15 |
from warbler_cda.remote_pack_loader import RemotePackLoader
|
| 16 |
-
from warbler_cda.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
|
| 18 |
# Import the HuggingFace Spaces GPU decorator
|
| 19 |
try:
|
|
@@ -328,6 +331,17 @@ def get_system_stats() -> str:
|
|
| 328 |
output += f"**Total Documents:** {api.get_context_store_size():,}\n\n"
|
| 329 |
output += f"**Document Types:** Scientific papers, novels, education, fiction, technical docs\n\n"
|
| 330 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 331 |
# Query Performance
|
| 332 |
output += "### ⚡ Query Performance\n\n"
|
| 333 |
output += f"**Total Queries:** {metrics['retrieval_metrics']['total_queries']}\n\n"
|
|
@@ -407,72 +421,175 @@ def synthesize_document(
|
|
| 407 |
document_text: str,
|
| 408 |
micro_window_size: int = 5,
|
| 409 |
macro_trigger_count: int = 3,
|
|
|
|
|
|
|
| 410 |
) -> str:
|
| 411 |
-
"""Generate a
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
units = [part.strip() for part in cleaned.splitlines() if part.strip()]
|
| 418 |
-
if len(units) < 2:
|
| 419 |
-
units = [sentence.strip() for sentence in cleaned.split(".") if sentence.strip()]
|
| 420 |
-
|
| 421 |
-
for index, unit in enumerate(units, start=1):
|
| 422 |
-
fragments.append(
|
| 423 |
-
{
|
| 424 |
-
"id": f"fragment_{index}",
|
| 425 |
-
"text": unit,
|
| 426 |
-
"heat": min(1.0, 0.4 + (len(unit) / 400.0)),
|
| 427 |
-
}
|
| 428 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 429 |
|
| 430 |
-
|
| 431 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 432 |
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
"
|
| 436 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 437 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 438 |
)
|
| 439 |
-
report = ladder.process_fragments(fragments)
|
| 440 |
-
|
| 441 |
-
lines = ["## Synthesis Output", "", f"**Input Fragments:** {len(fragments)}", ""]
|
| 442 |
-
|
| 443 |
-
if ladder.macro_distillations:
|
| 444 |
-
latest_macro = ladder.macro_distillations[-1]
|
| 445 |
-
lines.extend([
|
| 446 |
-
"### Macro Distillation",
|
| 447 |
-
"",
|
| 448 |
-
latest_macro.distilled_essence,
|
| 449 |
-
"",
|
| 450 |
-
])
|
| 451 |
-
|
| 452 |
-
if ladder.micro_summaries:
|
| 453 |
-
lines.append("### Micro Summaries")
|
| 454 |
-
lines.append("")
|
| 455 |
-
for micro in list(ladder.micro_summaries)[-3:]:
|
| 456 |
-
lines.append(f"- {micro.compressed_text}")
|
| 457 |
-
lines.append("")
|
| 458 |
-
|
| 459 |
-
metrics = ladder.get_compression_metrics()["current_state"]
|
| 460 |
-
lines.extend([
|
| 461 |
-
"### Metrics",
|
| 462 |
-
"",
|
| 463 |
-
f"- Active Micro Summaries: {metrics['micro_summaries_active']}",
|
| 464 |
-
f"- Macro Distillations: {metrics['macro_distillations_total']}",
|
| 465 |
-
f"- Compression Ratio: {metrics['compression_ratio']:.2f}",
|
| 466 |
-
"",
|
| 467 |
-
])
|
| 468 |
-
|
| 469 |
-
if report["new_macro_distillations"]:
|
| 470 |
-
lines.append("### New Distillation IDs")
|
| 471 |
-
lines.append("")
|
| 472 |
-
for item in report["new_macro_distillations"]:
|
| 473 |
-
lines.append(f"- {item['distillation_id']}")
|
| 474 |
-
|
| 475 |
-
return "\n".join(lines)
|
| 476 |
|
| 477 |
|
| 478 |
# Create Gradio interface
|
|
@@ -518,143 +635,6 @@ with gr.Blocks(title="Warbler CDA - FractalStat RAG") as demo:
|
|
| 518 |
with gr.Column():
|
| 519 |
results_output = gr.Markdown(label="Results")
|
| 520 |
|
| 521 |
-
# Add Bob quarantine info to results
|
| 522 |
-
def format_results_with_quarantine(output: str, results_count: int, quarantined_count: int) -> str:
|
| 523 |
-
"""Format results with Bob's quarantine information."""
|
| 524 |
-
if quarantined_count > 0:
|
| 525 |
-
status_line = f"**Bob the Skeptic**: {quarantined_count} conflicting results quarantined, {results_count} results retained\n\n"
|
| 526 |
-
elif quarantined_count == 0:
|
| 527 |
-
status_line = "**Bob the Skeptic**: No conflicting results detected\n\n"
|
| 528 |
-
else: # quarantined_count is None when Bob not available
|
| 529 |
-
status_line = "**Bob the Skeptic**: Conflict detection unavailable\n\n"
|
| 530 |
-
|
| 531 |
-
return output.replace("## Query Results\n\n", "## Query Results\n\n" + status_line)
|
| 532 |
-
|
| 533 |
-
def query_warbler_with_quarantine(query_text: str, max_results: int = 5, use_hybrid: bool = True, generate_answer: bool = True) -> str:
|
| 534 |
-
"""Query with Bob's quarantine reporting."""
|
| 535 |
-
if not query_text.strip():
|
| 536 |
-
return "Please enter a query."
|
| 537 |
-
|
| 538 |
-
start_time = time.time()
|
| 539 |
-
hybrid_available = embedding_provider is not None and EMBEDDING_BOOT_LIMIT != 0
|
| 540 |
-
|
| 541 |
-
# Create query - use hybrid mode when requested
|
| 542 |
-
effective_hybrid = use_hybrid and hybrid_available
|
| 543 |
-
query_mode = RetrievalMode.HYBRID_SEMANTIC_FRACTALSTAT if effective_hybrid else RetrievalMode.SEMANTIC_SIMILARITY
|
| 544 |
-
|
| 545 |
-
query = RetrievalQuery(
|
| 546 |
-
query_id=f"gradio_{int(time.time())}",
|
| 547 |
-
mode=query_mode,
|
| 548 |
-
semantic_query=query_text,
|
| 549 |
-
max_results=max_results * 2, # Get extra results for quarantine prioritizing
|
| 550 |
-
fractalstat_hybrid=effective_hybrid,
|
| 551 |
-
confidence_threshold=0.2 if not effective_hybrid else 0.3,
|
| 552 |
-
)
|
| 553 |
-
|
| 554 |
-
# DEBUG: Log query details
|
| 555 |
-
print(f"DEBUG: Executing query '{query_text}' with mode={query_mode}, max_results={max_results}, hybrid={use_hybrid}")
|
| 556 |
-
|
| 557 |
-
# Execute query
|
| 558 |
-
assembly = api.retrieve_context(query)
|
| 559 |
-
|
| 560 |
-
elapsed_ms = (time.time() - start_time) * 1000
|
| 561 |
-
|
| 562 |
-
# Find quarantined results (those with conflict flags)
|
| 563 |
-
original_results = len(assembly.results)
|
| 564 |
-
quarantined_results = [r for r in assembly.results if r.conflict_flags]
|
| 565 |
-
retained_results = [r for r in assembly.results if not r.conflict_flags]
|
| 566 |
-
|
| 567 |
-
# Limit to requested max_results from retained results
|
| 568 |
-
final_results = retained_results[:max_results]
|
| 569 |
-
|
| 570 |
-
# DEBUG: Log results summary with Bob information
|
| 571 |
-
print(f"DEBUG: Query completed in {elapsed_ms:.0f}ms, "
|
| 572 |
-
f"found {original_results} raw results, "
|
| 573 |
-
f"Bob quarantined {len(quarantined_results)}, "
|
| 574 |
-
f"retained {len(final_results)} for display")
|
| 575 |
-
if assembly.results:
|
| 576 |
-
print(f"DEBUG: Top 3 relevance scores: {[r.relevance_score for r in assembly.results[:3]]}")
|
| 577 |
-
print(f"DEBUG: Confidence threshold was: {query.confidence_threshold}")
|
| 578 |
-
|
| 579 |
-
# Hybrid Fallback: If hybrid mode and no final results from retained, try semantic on quarantine pool
|
| 580 |
-
fallback_triggered = False
|
| 581 |
-
if effective_hybrid and len(final_results) == 0 and len(quarantined_results) > 0:
|
| 582 |
-
print(f"DEBUG: Hybrid retained 0 results, checking quarantined pool for semantic fallback")
|
| 583 |
-
# Create semantic-only query for quarantined content
|
| 584 |
-
semantic_query = RetrievalQuery(
|
| 585 |
-
query_id=f"quarantine_fallback_{int(time.time())}",
|
| 586 |
-
mode=RetrievalMode.SEMANTIC_SIMILARITY,
|
| 587 |
-
semantic_query=query_text,
|
| 588 |
-
max_results=max_results,
|
| 589 |
-
confidence_threshold=0.3,
|
| 590 |
-
fractalstat_hybrid=False
|
| 591 |
-
)
|
| 592 |
-
# Note: In a full implementation, we'd re-query just the quarantined IDs
|
| 593 |
-
# For now, trigger full semantic search as fallback
|
| 594 |
-
fallback_assembly = api.retrieve_context(semantic_query)
|
| 595 |
-
fallback_triggered = True
|
| 596 |
-
final_results = fallback_assembly.results[:max_results]
|
| 597 |
-
elapsed_ms += (time.time() - start_time - elapsed_ms/1000) * 1000
|
| 598 |
-
print(f"DEBUG: Quarantine fallback triggered - retrieved {len(final_results)} from semantic search")
|
| 599 |
-
|
| 600 |
-
hybrid_fallback_used = fallback_triggered
|
| 601 |
-
|
| 602 |
-
results_data = [
|
| 603 |
-
{
|
| 604 |
-
"id": result.content_id,
|
| 605 |
-
"content": result.content,
|
| 606 |
-
"relevance_score": result.relevance_score,
|
| 607 |
-
"semantic_similarity": result.semantic_similarity,
|
| 608 |
-
"fractalstat_resonance": result.fractalstat_resonance,
|
| 609 |
-
"metadata": result.metadata,
|
| 610 |
-
}
|
| 611 |
-
for result in final_results
|
| 612 |
-
]
|
| 613 |
-
|
| 614 |
-
generated_answer = None
|
| 615 |
-
if generate_answer:
|
| 616 |
-
generated_answer = answer_generator.generate_answer(query_text, results_data)
|
| 617 |
-
|
| 618 |
-
# Format results with Bob quarantine info
|
| 619 |
-
output = f"## Query Results\n\n"
|
| 620 |
-
output += f"**Query:** {query_text}\n\n"
|
| 621 |
-
output += f"**Found:** {len(final_results)} results in {elapsed_ms:.0f}ms\n\n"
|
| 622 |
-
if len(quarantined_results) > 0:
|
| 623 |
-
output += f"**Risk Assessment:** {len(quarantined_results)} potentially conflicting results quarantined by Bob the Skeptic\n\n"
|
| 624 |
-
elif len(final_results) < original_results:
|
| 625 |
-
output += f"**Risk Assessment:** {original_results - len(final_results)} duplicates removed\n\n"
|
| 626 |
-
else:
|
| 627 |
-
output += "**Risk Assessment:** No conflicts detected by Bob the Skeptic\n\n"
|
| 628 |
-
|
| 629 |
-
output += f"**Quality Score:** {assembly.assembly_quality:.3f}\n\n"
|
| 630 |
-
|
| 631 |
-
if generated_answer:
|
| 632 |
-
output += "### Generated Answer\n\n"
|
| 633 |
-
output += f"{generated_answer.answer}\n\n"
|
| 634 |
-
if generated_answer.citations:
|
| 635 |
-
citation_line = ", ".join(
|
| 636 |
-
f"[{c['index']}] {c.get('pack') or c.get('id')}" for c in generated_answer.citations[:5]
|
| 637 |
-
)
|
| 638 |
-
output += f"**Sources:** {citation_line}\n\n"
|
| 639 |
-
output += f"**Generator:** {generated_answer.provider}\n\n"
|
| 640 |
-
|
| 641 |
-
if final_results:
|
| 642 |
-
output += "### Top Results\n\n"
|
| 643 |
-
for i, result in enumerate(final_results[:max_results], 1):
|
| 644 |
-
output += f"**{i}. Score: {result.relevance_score:.3f}**\n\n"
|
| 645 |
-
output += f"{result.content[:300]}...\n\n"
|
| 646 |
-
if effective_hybrid and result.fractalstat_resonance is not None:
|
| 647 |
-
output += f"- Semantic: {result.semantic_similarity:.3f}\n"
|
| 648 |
-
output += f"- FractalStat: {result.fractalstat_resonance:.3f}\n\n"
|
| 649 |
-
output += "---\n\n"
|
| 650 |
-
else:
|
| 651 |
-
output += "No results found.\n"
|
| 652 |
-
|
| 653 |
-
if hybrid_fallback_used:
|
| 654 |
-
output += "*\\*Note: Used semantic search because hybrid results were quarantined\\*\n"
|
| 655 |
-
|
| 656 |
-
return output
|
| 657 |
-
|
| 658 |
query_btn.click( # pylint: disable=E1101
|
| 659 |
fn=query_warbler_with_quarantine,
|
| 660 |
inputs=[query_input, max_results, use_hybrid, generate_answer_toggle],
|
|
@@ -699,15 +679,45 @@ with gr.Blocks(title="Warbler CDA - FractalStat RAG") as demo:
|
|
| 699 |
step=1,
|
| 700 |
label="Macro Trigger Count",
|
| 701 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 702 |
synthesis_btn = gr.Button("Synthesize", variant="primary")
|
| 703 |
synthesis_output = gr.Markdown(
|
| 704 |
-
"Paste a document and click **Synthesize** to generate a TLDA-style macro distillation."
|
| 705 |
)
|
| 706 |
synthesis_btn.click(
|
| 707 |
fn=synthesize_document,
|
| 708 |
-
inputs=[
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 709 |
outputs=synthesis_output,
|
| 710 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 711 |
|
| 712 |
with gr.Tab("About"):
|
| 713 |
gr.HTML("""
|
|
|
|
| 9 |
import gradio as gr
|
| 10 |
import os
|
| 11 |
import time
|
|
|
|
| 12 |
|
| 13 |
from warbler_cda.answer_generator import AnswerGenerator
|
| 14 |
from warbler_cda.remote_pack_loader import RemotePackLoader
|
| 15 |
+
from warbler_cda.synthesis_bridge import (
|
| 16 |
+
get_synthesis_session_summary,
|
| 17 |
+
ingest_synthesis_artifacts,
|
| 18 |
+
synthesize_text,
|
| 19 |
+
)
|
| 20 |
|
| 21 |
# Import the HuggingFace Spaces GPU decorator
|
| 22 |
try:
|
|
|
|
| 331 |
output += f"**Total Documents:** {api.get_context_store_size():,}\n\n"
|
| 332 |
output += f"**Document Types:** Scientific papers, novels, education, fiction, technical docs\n\n"
|
| 333 |
|
| 334 |
+
synthesis_summary = get_synthesis_session_summary(api)
|
| 335 |
+
output += "### 🧪 Synthesis Session Artifacts\n\n"
|
| 336 |
+
output += f"**Session Artifacts:** {synthesis_summary['total']}\n\n"
|
| 337 |
+
output += f"**Source Text Entries:** {synthesis_summary['source_text']}\n\n"
|
| 338 |
+
output += f"**Macro Distillations:** {synthesis_summary['macro_distillation']}\n\n"
|
| 339 |
+
if synthesis_summary["recent_ids"]:
|
| 340 |
+
output += "**Most Recent Artifact IDs:**\n"
|
| 341 |
+
for doc_id in synthesis_summary["recent_ids"]:
|
| 342 |
+
output += f"- {doc_id}\n"
|
| 343 |
+
output += "\n"
|
| 344 |
+
|
| 345 |
# Query Performance
|
| 346 |
output += "### ⚡ Query Performance\n\n"
|
| 347 |
output += f"**Total Queries:** {metrics['retrieval_metrics']['total_queries']}\n\n"
|
|
|
|
| 421 |
document_text: str,
|
| 422 |
micro_window_size: int = 5,
|
| 423 |
macro_trigger_count: int = 3,
|
| 424 |
+
add_macro_to_search: bool = False,
|
| 425 |
+
add_source_to_search: bool = False,
|
| 426 |
) -> str:
|
| 427 |
+
"""Generate a synthesis artifact and optionally ingest it into retrieval."""
|
| 428 |
+
try:
|
| 429 |
+
artifacts = synthesize_text(
|
| 430 |
+
document_text=document_text,
|
| 431 |
+
micro_window_size=micro_window_size,
|
| 432 |
+
macro_trigger_count=macro_trigger_count,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 433 |
)
|
| 434 |
+
except ValueError as exc:
|
| 435 |
+
return str(exc)
|
| 436 |
+
|
| 437 |
+
output = artifacts.report_markdown
|
| 438 |
+
|
| 439 |
+
if add_macro_to_search or add_source_to_search:
|
| 440 |
+
ingested_ids = ingest_synthesis_artifacts(
|
| 441 |
+
api=api,
|
| 442 |
+
document_text=document_text,
|
| 443 |
+
artifacts=artifacts,
|
| 444 |
+
include_source=add_source_to_search,
|
| 445 |
+
include_macro=add_macro_to_search,
|
| 446 |
+
)
|
| 447 |
+
if ingested_ids:
|
| 448 |
+
output += "\n\n### Search Index Update\n\n"
|
| 449 |
+
output += "The following artifacts were added to the current searchable session index:\n\n"
|
| 450 |
+
for doc_id in ingested_ids:
|
| 451 |
+
output += f"- {doc_id}\n"
|
| 452 |
+
output += "\nYou can now query terms from these artifacts in the Query tab during this app session."
|
| 453 |
+
else:
|
| 454 |
+
output += "\n\n### Search Index Update\n\nNo new artifacts were added. This can happen if no macro distillation was produced."
|
| 455 |
+
|
| 456 |
+
return output
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
@GPU
|
| 460 |
+
def query_warbler_with_quarantine(
|
| 461 |
+
query_text: str,
|
| 462 |
+
max_results: int = 5,
|
| 463 |
+
use_hybrid: bool = True,
|
| 464 |
+
generate_answer: bool = True,
|
| 465 |
+
) -> str:
|
| 466 |
+
"""Query with Bob's quarantine reporting."""
|
| 467 |
+
if not query_text.strip():
|
| 468 |
+
return "Please enter a query."
|
| 469 |
+
|
| 470 |
+
start_time = time.time()
|
| 471 |
+
hybrid_available = embedding_provider is not None and EMBEDDING_BOOT_LIMIT != 0
|
| 472 |
+
effective_hybrid = use_hybrid and hybrid_available
|
| 473 |
+
query_mode = (
|
| 474 |
+
RetrievalMode.HYBRID_SEMANTIC_FRACTALSTAT
|
| 475 |
+
if effective_hybrid
|
| 476 |
+
else RetrievalMode.SEMANTIC_SIMILARITY
|
| 477 |
+
)
|
| 478 |
+
|
| 479 |
+
query = RetrievalQuery(
|
| 480 |
+
query_id=f"gradio_{int(time.time())}",
|
| 481 |
+
mode=query_mode,
|
| 482 |
+
semantic_query=query_text,
|
| 483 |
+
max_results=max_results * 2,
|
| 484 |
+
fractalstat_hybrid=effective_hybrid,
|
| 485 |
+
confidence_threshold=0.2 if not effective_hybrid else 0.3,
|
| 486 |
+
)
|
| 487 |
+
|
| 488 |
+
print(
|
| 489 |
+
f"DEBUG: Executing query '{query_text}' with mode={query_mode}, "
|
| 490 |
+
f"max_results={max_results}, hybrid={use_hybrid}"
|
| 491 |
+
)
|
| 492 |
+
|
| 493 |
+
assembly = api.retrieve_context(query)
|
| 494 |
+
elapsed_ms = (time.time() - start_time) * 1000
|
| 495 |
+
|
| 496 |
+
original_results = len(assembly.results)
|
| 497 |
+
quarantined_results = [result for result in assembly.results if result.conflict_flags]
|
| 498 |
+
retained_results = [result for result in assembly.results if not result.conflict_flags]
|
| 499 |
+
final_results = retained_results[:max_results]
|
| 500 |
+
|
| 501 |
+
print(
|
| 502 |
+
f"DEBUG: Query completed in {elapsed_ms:.0f}ms, found {original_results} raw results, "
|
| 503 |
+
f"Bob quarantined {len(quarantined_results)}, retained {len(final_results)} for display"
|
| 504 |
+
)
|
| 505 |
|
| 506 |
+
fallback_triggered = False
|
| 507 |
+
if effective_hybrid and len(final_results) == 0 and len(quarantined_results) > 0:
|
| 508 |
+
print("DEBUG: Hybrid retained 0 results, checking quarantined pool for semantic fallback")
|
| 509 |
+
semantic_query = RetrievalQuery(
|
| 510 |
+
query_id=f"quarantine_fallback_{int(time.time())}",
|
| 511 |
+
mode=RetrievalMode.SEMANTIC_SIMILARITY,
|
| 512 |
+
semantic_query=query_text,
|
| 513 |
+
max_results=max_results,
|
| 514 |
+
confidence_threshold=0.3,
|
| 515 |
+
fractalstat_hybrid=False,
|
| 516 |
+
)
|
| 517 |
+
fallback_assembly = api.retrieve_context(semantic_query)
|
| 518 |
+
fallback_triggered = True
|
| 519 |
+
final_results = fallback_assembly.results[:max_results]
|
| 520 |
+
elapsed_ms += (time.time() - start_time - elapsed_ms / 1000) * 1000
|
| 521 |
+
print(
|
| 522 |
+
f"DEBUG: Quarantine fallback triggered - retrieved {len(final_results)} from semantic search"
|
| 523 |
+
)
|
| 524 |
|
| 525 |
+
results_data = [
|
| 526 |
+
{
|
| 527 |
+
"id": result.content_id,
|
| 528 |
+
"content": result.content,
|
| 529 |
+
"relevance_score": result.relevance_score,
|
| 530 |
+
"semantic_similarity": result.semantic_similarity,
|
| 531 |
+
"fractalstat_resonance": result.fractalstat_resonance,
|
| 532 |
+
"metadata": result.metadata,
|
| 533 |
}
|
| 534 |
+
for result in final_results
|
| 535 |
+
]
|
| 536 |
+
|
| 537 |
+
generated_answer = None
|
| 538 |
+
if generate_answer:
|
| 539 |
+
generated_answer = answer_generator.generate_answer(query_text, results_data)
|
| 540 |
+
|
| 541 |
+
output = "## Query Results\n\n"
|
| 542 |
+
output += f"**Query:** {query_text}\n\n"
|
| 543 |
+
output += f"**Found:** {len(final_results)} results in {elapsed_ms:.0f}ms\n\n"
|
| 544 |
+
if len(quarantined_results) > 0:
|
| 545 |
+
output += (
|
| 546 |
+
f"**Risk Assessment:** {len(quarantined_results)} potentially conflicting results "
|
| 547 |
+
"quarantined by Bob the Skeptic\n\n"
|
| 548 |
+
)
|
| 549 |
+
elif len(final_results) < original_results:
|
| 550 |
+
output += f"**Risk Assessment:** {original_results - len(final_results)} duplicates removed\n\n"
|
| 551 |
+
else:
|
| 552 |
+
output += "**Risk Assessment:** No conflicts detected by Bob the Skeptic\n\n"
|
| 553 |
+
|
| 554 |
+
output += f"**Quality Score:** {assembly.assembly_quality:.3f}\n\n"
|
| 555 |
+
|
| 556 |
+
if generated_answer:
|
| 557 |
+
output += "### Generated Answer\n\n"
|
| 558 |
+
output += f"{generated_answer.answer}\n\n"
|
| 559 |
+
if generated_answer.citations:
|
| 560 |
+
citation_line = ", ".join(
|
| 561 |
+
f"[{citation['index']}] {citation.get('pack') or citation.get('id')}"
|
| 562 |
+
for citation in generated_answer.citations[:5]
|
| 563 |
+
)
|
| 564 |
+
output += f"**Sources:** {citation_line}\n\n"
|
| 565 |
+
output += f"**Generator:** {generated_answer.provider}\n\n"
|
| 566 |
+
|
| 567 |
+
if final_results:
|
| 568 |
+
output += "### Top Results\n\n"
|
| 569 |
+
for index, result in enumerate(final_results[:max_results], 1):
|
| 570 |
+
output += f"**{index}. Score: {result.relevance_score:.3f}**\n\n"
|
| 571 |
+
output += f"{result.content[:300]}...\n\n"
|
| 572 |
+
if effective_hybrid and result.fractalstat_resonance is not None:
|
| 573 |
+
output += f"- Semantic: {result.semantic_similarity:.3f}\n"
|
| 574 |
+
output += f"- FractalStat: {result.fractalstat_resonance:.3f}\n\n"
|
| 575 |
+
output += "---\n\n"
|
| 576 |
+
else:
|
| 577 |
+
output += "No results found.\n"
|
| 578 |
+
|
| 579 |
+
if fallback_triggered:
|
| 580 |
+
output += "*\\*Note: Used semantic search because hybrid results were quarantined\\*\n"
|
| 581 |
+
|
| 582 |
+
return output
|
| 583 |
+
|
| 584 |
+
|
| 585 |
+
def query_from_synthesis_tab(query_text: str) -> str:
|
| 586 |
+
"""Run a quick semantic follow-up query from the Synthesis tab."""
|
| 587 |
+
return query_warbler_with_quarantine(
|
| 588 |
+
query_text=query_text,
|
| 589 |
+
max_results=5,
|
| 590 |
+
use_hybrid=False,
|
| 591 |
+
generate_answer=True,
|
| 592 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 593 |
|
| 594 |
|
| 595 |
# Create Gradio interface
|
|
|
|
| 635 |
with gr.Column():
|
| 636 |
results_output = gr.Markdown(label="Results")
|
| 637 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 638 |
query_btn.click( # pylint: disable=E1101
|
| 639 |
fn=query_warbler_with_quarantine,
|
| 640 |
inputs=[query_input, max_results, use_hybrid, generate_answer_toggle],
|
|
|
|
| 679 |
step=1,
|
| 680 |
label="Macro Trigger Count",
|
| 681 |
)
|
| 682 |
+
with gr.Row():
|
| 683 |
+
synthesis_add_macro = gr.Checkbox(
|
| 684 |
+
label="Add macro distillation to current search session",
|
| 685 |
+
value=False,
|
| 686 |
+
)
|
| 687 |
+
synthesis_add_source = gr.Checkbox(
|
| 688 |
+
label="Also add original source text to current search session",
|
| 689 |
+
value=False,
|
| 690 |
+
)
|
| 691 |
synthesis_btn = gr.Button("Synthesize", variant="primary")
|
| 692 |
synthesis_output = gr.Markdown(
|
| 693 |
+
"Paste a document and click **Synthesize** to generate a TLDA-style macro distillation. Optionally add it to the current search session."
|
| 694 |
)
|
| 695 |
synthesis_btn.click(
|
| 696 |
fn=synthesize_document,
|
| 697 |
+
inputs=[
|
| 698 |
+
synthesis_input,
|
| 699 |
+
synthesis_micro_window,
|
| 700 |
+
synthesis_macro_trigger,
|
| 701 |
+
synthesis_add_macro,
|
| 702 |
+
synthesis_add_source,
|
| 703 |
+
],
|
| 704 |
outputs=synthesis_output,
|
| 705 |
)
|
| 706 |
+
gr.Markdown("### Query This Synthesis")
|
| 707 |
+
synthesis_query_input = gr.Textbox(
|
| 708 |
+
label="Follow-up Query",
|
| 709 |
+
placeholder="Ask about the newly added synthesis artifacts without leaving this tab...",
|
| 710 |
+
lines=2,
|
| 711 |
+
)
|
| 712 |
+
synthesis_query_btn = gr.Button("Query Current Session", variant="secondary")
|
| 713 |
+
synthesis_query_output = gr.Markdown(
|
| 714 |
+
"If you added synthesis artifacts to search, run a follow-up query here to inspect them immediately."
|
| 715 |
+
)
|
| 716 |
+
synthesis_query_btn.click(
|
| 717 |
+
fn=query_from_synthesis_tab,
|
| 718 |
+
inputs=[synthesis_query_input],
|
| 719 |
+
outputs=synthesis_query_output,
|
| 720 |
+
)
|
| 721 |
|
| 722 |
with gr.Tab("About"):
|
| 723 |
gr.HTML("""
|
tests/test_synthesis_bridge.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from warbler_cda.retrieval_api import RetrievalAPI, RetrievalMode, RetrievalQuery
|
| 2 |
+
from warbler_cda.synthesis_bridge import (
|
| 3 |
+
build_synthesis_fragments,
|
| 4 |
+
get_synthesis_session_summary,
|
| 5 |
+
ingest_synthesis_artifacts,
|
| 6 |
+
synthesize_text,
|
| 7 |
+
)
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def test_build_synthesis_fragments_prefers_line_structure():
|
| 11 |
+
fragments = build_synthesis_fragments("line one\n\nline two\nline three")
|
| 12 |
+
|
| 13 |
+
assert len(fragments) == 3
|
| 14 |
+
assert fragments[0]["id"] == "fragment_1"
|
| 15 |
+
assert fragments[1]["text"] == "line two"
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def test_synthesize_text_generates_macro_for_longer_input():
|
| 20 |
+
text = "\n".join(
|
| 21 |
+
[
|
| 22 |
+
"Ten of Hearts is the weekly prompt.",
|
| 23 |
+
"Use the card and its symbols as inspiration.",
|
| 24 |
+
"The jam allows joining late and taking breaks.",
|
| 25 |
+
"Projects should be submitted within the relevant week.",
|
| 26 |
+
"AI use is discouraged and should be labeled if used.",
|
| 27 |
+
"Bigotry and harassment are prohibited.",
|
| 28 |
+
"New submissions and reworks are allowed.",
|
| 29 |
+
"Participants should be kind to one another.",
|
| 30 |
+
"Submissions should connect to the card theme.",
|
| 31 |
+
"The focus is on making games you want to make.",
|
| 32 |
+
]
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
artifacts = synthesize_text(text, micro_window_size=3, macro_trigger_count=2)
|
| 36 |
+
|
| 37 |
+
assert artifacts.fragments
|
| 38 |
+
assert artifacts.macro_distillation is not None
|
| 39 |
+
assert artifacts.metrics["macro_distillations_total"] >= 1
|
| 40 |
+
assert "## Synthesis Output" in artifacts.report_markdown
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def test_ingested_synthesis_source_becomes_queryable():
|
| 45 |
+
api = RetrievalAPI(config={"enable_fractalstat_hybrid": False, "enable_lexical_index": True})
|
| 46 |
+
text = "\n".join(
|
| 47 |
+
[
|
| 48 |
+
"Card game jam entries should fit on a playing card.",
|
| 49 |
+
"Participants can use Ten of Hearts as inspiration.",
|
| 50 |
+
"The jam values quantity of output and kindness.",
|
| 51 |
+
"Submitting within the week is encouraged.",
|
| 52 |
+
"Decks of cards can be used as a randomizer.",
|
| 53 |
+
]
|
| 54 |
+
)
|
| 55 |
+
artifacts = synthesize_text(text, micro_window_size=3, macro_trigger_count=2)
|
| 56 |
+
|
| 57 |
+
ingested_ids = ingest_synthesis_artifacts(
|
| 58 |
+
api=api,
|
| 59 |
+
document_text=text,
|
| 60 |
+
artifacts=artifacts,
|
| 61 |
+
include_source=True,
|
| 62 |
+
include_macro=True,
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
assert len(ingested_ids) >= 1
|
| 66 |
+
query = RetrievalQuery(
|
| 67 |
+
query_id="bridge-test",
|
| 68 |
+
mode=RetrievalMode.SEMANTIC_SIMILARITY,
|
| 69 |
+
semantic_query="playing card kindness",
|
| 70 |
+
fractalstat_hybrid=False,
|
| 71 |
+
confidence_threshold=0.0,
|
| 72 |
+
max_results=5,
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
assembly = api.retrieve_context(query)
|
| 76 |
+
|
| 77 |
+
assert assembly.results
|
| 78 |
+
assert any(result.content_id in ingested_ids for result in assembly.results)
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def test_synthesis_session_summary_counts_ingested_artifacts():
|
| 82 |
+
api = RetrievalAPI(config={"enable_fractalstat_hybrid": False, "enable_lexical_index": True})
|
| 83 |
+
text = "alpha\nbeta\ngamma\ndelta\nepsilon\nzeta"
|
| 84 |
+
artifacts = synthesize_text(text, micro_window_size=3, macro_trigger_count=2)
|
| 85 |
+
|
| 86 |
+
ingest_synthesis_artifacts(
|
| 87 |
+
api=api,
|
| 88 |
+
document_text=text,
|
| 89 |
+
artifacts=artifacts,
|
| 90 |
+
include_source=True,
|
| 91 |
+
include_macro=True,
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
summary = get_synthesis_session_summary(api)
|
| 95 |
+
|
| 96 |
+
assert summary["total"] >= 1
|
| 97 |
+
assert summary["source_text"] == 1
|
| 98 |
+
assert summary["macro_distillation"] >= 1
|
| 99 |
+
assert summary["recent_ids"]
|
warbler_cda/synthesis_bridge.py
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Utilities for TLDA-style synthesis and optional retrieval ingestion."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from dataclasses import dataclass
|
| 6 |
+
import hashlib
|
| 7 |
+
import time
|
| 8 |
+
from typing import Any, Dict, List, Optional
|
| 9 |
+
|
| 10 |
+
from warbler_cda.summarization_ladder import SummarizationLadder
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@dataclass
|
| 14 |
+
class SynthesisArtifacts:
|
| 15 |
+
"""Structured artifacts produced by a synthesis run."""
|
| 16 |
+
|
| 17 |
+
report_markdown: str
|
| 18 |
+
fragments: List[Dict[str, Any]]
|
| 19 |
+
micro_summaries: List[str]
|
| 20 |
+
macro_distillation: Optional[str]
|
| 21 |
+
metrics: Dict[str, Any]
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def build_synthesis_fragments(document_text: str) -> List[Dict[str, Any]]:
|
| 25 |
+
"""Split text into fragments suitable for the summarization ladder."""
|
| 26 |
+
cleaned = document_text.strip()
|
| 27 |
+
if not cleaned:
|
| 28 |
+
return []
|
| 29 |
+
|
| 30 |
+
units = [part.strip() for part in cleaned.splitlines() if part.strip()]
|
| 31 |
+
if len(units) < 2:
|
| 32 |
+
units = [sentence.strip() for sentence in cleaned.split(".") if sentence.strip()]
|
| 33 |
+
|
| 34 |
+
fragments: List[Dict[str, Any]] = []
|
| 35 |
+
for index, unit in enumerate(units, start=1):
|
| 36 |
+
fragments.append(
|
| 37 |
+
{
|
| 38 |
+
"id": f"fragment_{index}",
|
| 39 |
+
"text": unit,
|
| 40 |
+
"heat": min(1.0, 0.4 + (len(unit) / 400.0)),
|
| 41 |
+
}
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
return fragments
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def synthesize_text(
|
| 48 |
+
document_text: str,
|
| 49 |
+
micro_window_size: int = 5,
|
| 50 |
+
macro_trigger_count: int = 3,
|
| 51 |
+
) -> SynthesisArtifacts:
|
| 52 |
+
"""Generate hierarchical synthesis artifacts and a markdown report."""
|
| 53 |
+
fragments = build_synthesis_fragments(document_text)
|
| 54 |
+
if not fragments:
|
| 55 |
+
raise ValueError("Please provide document text to synthesize.")
|
| 56 |
+
|
| 57 |
+
ladder = SummarizationLadder(
|
| 58 |
+
config={
|
| 59 |
+
"micro_window_size": micro_window_size,
|
| 60 |
+
"macro_trigger_count": macro_trigger_count,
|
| 61 |
+
}
|
| 62 |
+
)
|
| 63 |
+
report = ladder.process_fragments(fragments)
|
| 64 |
+
|
| 65 |
+
micro_summaries = [summary.compressed_text for summary in list(ladder.micro_summaries)]
|
| 66 |
+
macro_distillation = (
|
| 67 |
+
ladder.macro_distillations[-1].distilled_essence if ladder.macro_distillations else None
|
| 68 |
+
)
|
| 69 |
+
metrics = ladder.get_compression_metrics()["current_state"]
|
| 70 |
+
|
| 71 |
+
lines = ["## Synthesis Output", "", f"**Input Fragments:** {len(fragments)}", ""]
|
| 72 |
+
|
| 73 |
+
if macro_distillation:
|
| 74 |
+
lines.extend(["### Macro Distillation", "", macro_distillation, ""])
|
| 75 |
+
|
| 76 |
+
if micro_summaries:
|
| 77 |
+
lines.append("### Micro Summaries")
|
| 78 |
+
lines.append("")
|
| 79 |
+
for micro in micro_summaries[-3:]:
|
| 80 |
+
lines.append(f"- {micro}")
|
| 81 |
+
lines.append("")
|
| 82 |
+
|
| 83 |
+
lines.extend(
|
| 84 |
+
[
|
| 85 |
+
"### Metrics",
|
| 86 |
+
"",
|
| 87 |
+
f"- Active Micro Summaries: {metrics['micro_summaries_active']}",
|
| 88 |
+
f"- Macro Distillations: {metrics['macro_distillations_total']}",
|
| 89 |
+
f"- Compression Ratio: {metrics['compression_ratio']:.2f}",
|
| 90 |
+
"",
|
| 91 |
+
]
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
if report["new_macro_distillations"]:
|
| 95 |
+
lines.append("### New Distillation IDs")
|
| 96 |
+
lines.append("")
|
| 97 |
+
for item in report["new_macro_distillations"]:
|
| 98 |
+
lines.append(f"- {item['distillation_id']}")
|
| 99 |
+
|
| 100 |
+
return SynthesisArtifacts(
|
| 101 |
+
report_markdown="\n".join(lines),
|
| 102 |
+
fragments=fragments,
|
| 103 |
+
micro_summaries=micro_summaries,
|
| 104 |
+
macro_distillation=macro_distillation,
|
| 105 |
+
metrics=metrics,
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def ingest_synthesis_artifacts(
|
| 110 |
+
api: Any,
|
| 111 |
+
document_text: str,
|
| 112 |
+
artifacts: SynthesisArtifacts,
|
| 113 |
+
include_source: bool = False,
|
| 114 |
+
include_macro: bool = True,
|
| 115 |
+
) -> List[str]:
|
| 116 |
+
"""Persist source and/or synthesis artifacts into the retrieval context store."""
|
| 117 |
+
ingested_ids: List[str] = []
|
| 118 |
+
timestamp = int(time.time() * 1000)
|
| 119 |
+
content_hash = hashlib.sha256(document_text.encode()).hexdigest()[:12]
|
| 120 |
+
base_metadata = {
|
| 121 |
+
"source": "synthesis_tab",
|
| 122 |
+
"content_hash": content_hash,
|
| 123 |
+
"artifact_type": "synthesis_session",
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
if include_source:
|
| 127 |
+
source_id = f"synthesis-source-{timestamp}-{content_hash}"
|
| 128 |
+
added = api.add_document(
|
| 129 |
+
source_id,
|
| 130 |
+
document_text,
|
| 131 |
+
{
|
| 132 |
+
**base_metadata,
|
| 133 |
+
"artifact_type": "source_text",
|
| 134 |
+
"fragment_count": len(artifacts.fragments),
|
| 135 |
+
},
|
| 136 |
+
auto_embed=False,
|
| 137 |
+
)
|
| 138 |
+
if added:
|
| 139 |
+
ingested_ids.append(source_id)
|
| 140 |
+
|
| 141 |
+
if include_macro and artifacts.macro_distillation:
|
| 142 |
+
macro_id = f"synthesis-macro-{timestamp}-{content_hash}"
|
| 143 |
+
added = api.add_document(
|
| 144 |
+
macro_id,
|
| 145 |
+
artifacts.macro_distillation,
|
| 146 |
+
{
|
| 147 |
+
**base_metadata,
|
| 148 |
+
"artifact_type": "macro_distillation",
|
| 149 |
+
"fragment_count": len(artifacts.fragments),
|
| 150 |
+
"micro_summary_count": len(artifacts.micro_summaries),
|
| 151 |
+
},
|
| 152 |
+
auto_embed=False,
|
| 153 |
+
)
|
| 154 |
+
if added:
|
| 155 |
+
ingested_ids.append(macro_id)
|
| 156 |
+
|
| 157 |
+
return ingested_ids
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def get_synthesis_session_summary(api: Any) -> Dict[str, Any]:
|
| 161 |
+
"""Summarize synthesis artifacts currently stored in the active retrieval session."""
|
| 162 |
+
context_store = getattr(api, "_context_store", {}) or {}
|
| 163 |
+
|
| 164 |
+
summary = {
|
| 165 |
+
"total": 0,
|
| 166 |
+
"source_text": 0,
|
| 167 |
+
"macro_distillation": 0,
|
| 168 |
+
"recent_ids": [],
|
| 169 |
+
}
|
| 170 |
+
|
| 171 |
+
synthesis_entries = []
|
| 172 |
+
for doc_id, doc_data in context_store.items():
|
| 173 |
+
metadata = doc_data.get("metadata", {})
|
| 174 |
+
if metadata.get("source") != "synthesis_tab":
|
| 175 |
+
continue
|
| 176 |
+
|
| 177 |
+
summary["total"] += 1
|
| 178 |
+
artifact_type = metadata.get("artifact_type")
|
| 179 |
+
if artifact_type == "source_text":
|
| 180 |
+
summary["source_text"] += 1
|
| 181 |
+
elif artifact_type == "macro_distillation":
|
| 182 |
+
summary["macro_distillation"] += 1
|
| 183 |
+
|
| 184 |
+
synthesis_entries.append((doc_data.get("added_at", 0), doc_id))
|
| 185 |
+
|
| 186 |
+
synthesis_entries.sort(reverse=True)
|
| 187 |
+
summary["recent_ids"] = [doc_id for _, doc_id in synthesis_entries[:5]]
|
| 188 |
+
return summary
|