Bellok commited on
Commit
9fcde48
·
1 Parent(s): e7d33ec

feat: Add synthesis functionality with session summary and artifact ingestion

Browse files
app.py CHANGED
@@ -9,11 +9,14 @@ Provides a web UI for the FractalStat RAG system with GPU acceleration.
9
  import gradio as gr
10
  import os
11
  import time
12
- from typing import List, Dict, Any
13
 
14
  from warbler_cda.answer_generator import AnswerGenerator
15
  from warbler_cda.remote_pack_loader import RemotePackLoader
16
- from warbler_cda.summarization_ladder import SummarizationLadder
 
 
 
 
17
 
18
  # Import the HuggingFace Spaces GPU decorator
19
  try:
@@ -328,6 +331,17 @@ def get_system_stats() -> str:
328
  output += f"**Total Documents:** {api.get_context_store_size():,}\n\n"
329
  output += f"**Document Types:** Scientific papers, novels, education, fiction, technical docs\n\n"
330
 
 
 
 
 
 
 
 
 
 
 
 
331
  # Query Performance
332
  output += "### ⚡ Query Performance\n\n"
333
  output += f"**Total Queries:** {metrics['retrieval_metrics']['total_queries']}\n\n"
@@ -407,72 +421,175 @@ def synthesize_document(
407
  document_text: str,
408
  micro_window_size: int = 5,
409
  macro_trigger_count: int = 3,
 
 
410
  ) -> str:
411
- """Generate a hierarchical synthesis artifact for a document-sized input."""
412
- cleaned = document_text.strip()
413
- if not cleaned:
414
- return "Please provide document text to synthesize."
415
-
416
- fragments: List[Dict[str, Any]] = []
417
- units = [part.strip() for part in cleaned.splitlines() if part.strip()]
418
- if len(units) < 2:
419
- units = [sentence.strip() for sentence in cleaned.split(".") if sentence.strip()]
420
-
421
- for index, unit in enumerate(units, start=1):
422
- fragments.append(
423
- {
424
- "id": f"fragment_{index}",
425
- "text": unit,
426
- "heat": min(1.0, 0.4 + (len(unit) / 400.0)),
427
- }
428
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
429
 
430
- if not fragments:
431
- return "Unable to derive synthesis fragments from the provided text."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
432
 
433
- ladder = SummarizationLadder(
434
- config={
435
- "micro_window_size": micro_window_size,
436
- "macro_trigger_count": macro_trigger_count,
 
 
 
 
437
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
438
  )
439
- report = ladder.process_fragments(fragments)
440
-
441
- lines = ["## Synthesis Output", "", f"**Input Fragments:** {len(fragments)}", ""]
442
-
443
- if ladder.macro_distillations:
444
- latest_macro = ladder.macro_distillations[-1]
445
- lines.extend([
446
- "### Macro Distillation",
447
- "",
448
- latest_macro.distilled_essence,
449
- "",
450
- ])
451
-
452
- if ladder.micro_summaries:
453
- lines.append("### Micro Summaries")
454
- lines.append("")
455
- for micro in list(ladder.micro_summaries)[-3:]:
456
- lines.append(f"- {micro.compressed_text}")
457
- lines.append("")
458
-
459
- metrics = ladder.get_compression_metrics()["current_state"]
460
- lines.extend([
461
- "### Metrics",
462
- "",
463
- f"- Active Micro Summaries: {metrics['micro_summaries_active']}",
464
- f"- Macro Distillations: {metrics['macro_distillations_total']}",
465
- f"- Compression Ratio: {metrics['compression_ratio']:.2f}",
466
- "",
467
- ])
468
-
469
- if report["new_macro_distillations"]:
470
- lines.append("### New Distillation IDs")
471
- lines.append("")
472
- for item in report["new_macro_distillations"]:
473
- lines.append(f"- {item['distillation_id']}")
474
-
475
- return "\n".join(lines)
476
 
477
 
478
  # Create Gradio interface
@@ -518,143 +635,6 @@ with gr.Blocks(title="Warbler CDA - FractalStat RAG") as demo:
518
  with gr.Column():
519
  results_output = gr.Markdown(label="Results")
520
 
521
- # Add Bob quarantine info to results
522
- def format_results_with_quarantine(output: str, results_count: int, quarantined_count: int) -> str:
523
- """Format results with Bob's quarantine information."""
524
- if quarantined_count > 0:
525
- status_line = f"**Bob the Skeptic**: {quarantined_count} conflicting results quarantined, {results_count} results retained\n\n"
526
- elif quarantined_count == 0:
527
- status_line = "**Bob the Skeptic**: No conflicting results detected\n\n"
528
- else: # quarantined_count is None when Bob not available
529
- status_line = "**Bob the Skeptic**: Conflict detection unavailable\n\n"
530
-
531
- return output.replace("## Query Results\n\n", "## Query Results\n\n" + status_line)
532
-
533
- def query_warbler_with_quarantine(query_text: str, max_results: int = 5, use_hybrid: bool = True, generate_answer: bool = True) -> str:
534
- """Query with Bob's quarantine reporting."""
535
- if not query_text.strip():
536
- return "Please enter a query."
537
-
538
- start_time = time.time()
539
- hybrid_available = embedding_provider is not None and EMBEDDING_BOOT_LIMIT != 0
540
-
541
- # Create query - use hybrid mode when requested
542
- effective_hybrid = use_hybrid and hybrid_available
543
- query_mode = RetrievalMode.HYBRID_SEMANTIC_FRACTALSTAT if effective_hybrid else RetrievalMode.SEMANTIC_SIMILARITY
544
-
545
- query = RetrievalQuery(
546
- query_id=f"gradio_{int(time.time())}",
547
- mode=query_mode,
548
- semantic_query=query_text,
549
- max_results=max_results * 2, # Get extra results for quarantine prioritizing
550
- fractalstat_hybrid=effective_hybrid,
551
- confidence_threshold=0.2 if not effective_hybrid else 0.3,
552
- )
553
-
554
- # DEBUG: Log query details
555
- print(f"DEBUG: Executing query '{query_text}' with mode={query_mode}, max_results={max_results}, hybrid={use_hybrid}")
556
-
557
- # Execute query
558
- assembly = api.retrieve_context(query)
559
-
560
- elapsed_ms = (time.time() - start_time) * 1000
561
-
562
- # Find quarantined results (those with conflict flags)
563
- original_results = len(assembly.results)
564
- quarantined_results = [r for r in assembly.results if r.conflict_flags]
565
- retained_results = [r for r in assembly.results if not r.conflict_flags]
566
-
567
- # Limit to requested max_results from retained results
568
- final_results = retained_results[:max_results]
569
-
570
- # DEBUG: Log results summary with Bob information
571
- print(f"DEBUG: Query completed in {elapsed_ms:.0f}ms, "
572
- f"found {original_results} raw results, "
573
- f"Bob quarantined {len(quarantined_results)}, "
574
- f"retained {len(final_results)} for display")
575
- if assembly.results:
576
- print(f"DEBUG: Top 3 relevance scores: {[r.relevance_score for r in assembly.results[:3]]}")
577
- print(f"DEBUG: Confidence threshold was: {query.confidence_threshold}")
578
-
579
- # Hybrid Fallback: If hybrid mode and no final results from retained, try semantic on quarantine pool
580
- fallback_triggered = False
581
- if effective_hybrid and len(final_results) == 0 and len(quarantined_results) > 0:
582
- print(f"DEBUG: Hybrid retained 0 results, checking quarantined pool for semantic fallback")
583
- # Create semantic-only query for quarantined content
584
- semantic_query = RetrievalQuery(
585
- query_id=f"quarantine_fallback_{int(time.time())}",
586
- mode=RetrievalMode.SEMANTIC_SIMILARITY,
587
- semantic_query=query_text,
588
- max_results=max_results,
589
- confidence_threshold=0.3,
590
- fractalstat_hybrid=False
591
- )
592
- # Note: In a full implementation, we'd re-query just the quarantined IDs
593
- # For now, trigger full semantic search as fallback
594
- fallback_assembly = api.retrieve_context(semantic_query)
595
- fallback_triggered = True
596
- final_results = fallback_assembly.results[:max_results]
597
- elapsed_ms += (time.time() - start_time - elapsed_ms/1000) * 1000
598
- print(f"DEBUG: Quarantine fallback triggered - retrieved {len(final_results)} from semantic search")
599
-
600
- hybrid_fallback_used = fallback_triggered
601
-
602
- results_data = [
603
- {
604
- "id": result.content_id,
605
- "content": result.content,
606
- "relevance_score": result.relevance_score,
607
- "semantic_similarity": result.semantic_similarity,
608
- "fractalstat_resonance": result.fractalstat_resonance,
609
- "metadata": result.metadata,
610
- }
611
- for result in final_results
612
- ]
613
-
614
- generated_answer = None
615
- if generate_answer:
616
- generated_answer = answer_generator.generate_answer(query_text, results_data)
617
-
618
- # Format results with Bob quarantine info
619
- output = f"## Query Results\n\n"
620
- output += f"**Query:** {query_text}\n\n"
621
- output += f"**Found:** {len(final_results)} results in {elapsed_ms:.0f}ms\n\n"
622
- if len(quarantined_results) > 0:
623
- output += f"**Risk Assessment:** {len(quarantined_results)} potentially conflicting results quarantined by Bob the Skeptic\n\n"
624
- elif len(final_results) < original_results:
625
- output += f"**Risk Assessment:** {original_results - len(final_results)} duplicates removed\n\n"
626
- else:
627
- output += "**Risk Assessment:** No conflicts detected by Bob the Skeptic\n\n"
628
-
629
- output += f"**Quality Score:** {assembly.assembly_quality:.3f}\n\n"
630
-
631
- if generated_answer:
632
- output += "### Generated Answer\n\n"
633
- output += f"{generated_answer.answer}\n\n"
634
- if generated_answer.citations:
635
- citation_line = ", ".join(
636
- f"[{c['index']}] {c.get('pack') or c.get('id')}" for c in generated_answer.citations[:5]
637
- )
638
- output += f"**Sources:** {citation_line}\n\n"
639
- output += f"**Generator:** {generated_answer.provider}\n\n"
640
-
641
- if final_results:
642
- output += "### Top Results\n\n"
643
- for i, result in enumerate(final_results[:max_results], 1):
644
- output += f"**{i}. Score: {result.relevance_score:.3f}**\n\n"
645
- output += f"{result.content[:300]}...\n\n"
646
- if effective_hybrid and result.fractalstat_resonance is not None:
647
- output += f"- Semantic: {result.semantic_similarity:.3f}\n"
648
- output += f"- FractalStat: {result.fractalstat_resonance:.3f}\n\n"
649
- output += "---\n\n"
650
- else:
651
- output += "No results found.\n"
652
-
653
- if hybrid_fallback_used:
654
- output += "*\\*Note: Used semantic search because hybrid results were quarantined\\*\n"
655
-
656
- return output
657
-
658
  query_btn.click( # pylint: disable=E1101
659
  fn=query_warbler_with_quarantine,
660
  inputs=[query_input, max_results, use_hybrid, generate_answer_toggle],
@@ -699,15 +679,45 @@ with gr.Blocks(title="Warbler CDA - FractalStat RAG") as demo:
699
  step=1,
700
  label="Macro Trigger Count",
701
  )
 
 
 
 
 
 
 
 
 
702
  synthesis_btn = gr.Button("Synthesize", variant="primary")
703
  synthesis_output = gr.Markdown(
704
- "Paste a document and click **Synthesize** to generate a TLDA-style macro distillation."
705
  )
706
  synthesis_btn.click(
707
  fn=synthesize_document,
708
- inputs=[synthesis_input, synthesis_micro_window, synthesis_macro_trigger],
 
 
 
 
 
 
709
  outputs=synthesis_output,
710
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
711
 
712
  with gr.Tab("About"):
713
  gr.HTML("""
 
9
  import gradio as gr
10
  import os
11
  import time
 
12
 
13
  from warbler_cda.answer_generator import AnswerGenerator
14
  from warbler_cda.remote_pack_loader import RemotePackLoader
15
+ from warbler_cda.synthesis_bridge import (
16
+ get_synthesis_session_summary,
17
+ ingest_synthesis_artifacts,
18
+ synthesize_text,
19
+ )
20
 
21
  # Import the HuggingFace Spaces GPU decorator
22
  try:
 
331
  output += f"**Total Documents:** {api.get_context_store_size():,}\n\n"
332
  output += f"**Document Types:** Scientific papers, novels, education, fiction, technical docs\n\n"
333
 
334
+ synthesis_summary = get_synthesis_session_summary(api)
335
+ output += "### 🧪 Synthesis Session Artifacts\n\n"
336
+ output += f"**Session Artifacts:** {synthesis_summary['total']}\n\n"
337
+ output += f"**Source Text Entries:** {synthesis_summary['source_text']}\n\n"
338
+ output += f"**Macro Distillations:** {synthesis_summary['macro_distillation']}\n\n"
339
+ if synthesis_summary["recent_ids"]:
340
+ output += "**Most Recent Artifact IDs:**\n"
341
+ for doc_id in synthesis_summary["recent_ids"]:
342
+ output += f"- {doc_id}\n"
343
+ output += "\n"
344
+
345
  # Query Performance
346
  output += "### ⚡ Query Performance\n\n"
347
  output += f"**Total Queries:** {metrics['retrieval_metrics']['total_queries']}\n\n"
 
421
  document_text: str,
422
  micro_window_size: int = 5,
423
  macro_trigger_count: int = 3,
424
+ add_macro_to_search: bool = False,
425
+ add_source_to_search: bool = False,
426
  ) -> str:
427
+ """Generate a synthesis artifact and optionally ingest it into retrieval."""
428
+ try:
429
+ artifacts = synthesize_text(
430
+ document_text=document_text,
431
+ micro_window_size=micro_window_size,
432
+ macro_trigger_count=macro_trigger_count,
 
 
 
 
 
 
 
 
 
 
 
433
  )
434
+ except ValueError as exc:
435
+ return str(exc)
436
+
437
+ output = artifacts.report_markdown
438
+
439
+ if add_macro_to_search or add_source_to_search:
440
+ ingested_ids = ingest_synthesis_artifacts(
441
+ api=api,
442
+ document_text=document_text,
443
+ artifacts=artifacts,
444
+ include_source=add_source_to_search,
445
+ include_macro=add_macro_to_search,
446
+ )
447
+ if ingested_ids:
448
+ output += "\n\n### Search Index Update\n\n"
449
+ output += "The following artifacts were added to the current searchable session index:\n\n"
450
+ for doc_id in ingested_ids:
451
+ output += f"- {doc_id}\n"
452
+ output += "\nYou can now query terms from these artifacts in the Query tab during this app session."
453
+ else:
454
+ output += "\n\n### Search Index Update\n\nNo new artifacts were added. This can happen if no macro distillation was produced."
455
+
456
+ return output
457
+
458
+
459
+ @GPU
460
+ def query_warbler_with_quarantine(
461
+ query_text: str,
462
+ max_results: int = 5,
463
+ use_hybrid: bool = True,
464
+ generate_answer: bool = True,
465
+ ) -> str:
466
+ """Query with Bob's quarantine reporting."""
467
+ if not query_text.strip():
468
+ return "Please enter a query."
469
+
470
+ start_time = time.time()
471
+ hybrid_available = embedding_provider is not None and EMBEDDING_BOOT_LIMIT != 0
472
+ effective_hybrid = use_hybrid and hybrid_available
473
+ query_mode = (
474
+ RetrievalMode.HYBRID_SEMANTIC_FRACTALSTAT
475
+ if effective_hybrid
476
+ else RetrievalMode.SEMANTIC_SIMILARITY
477
+ )
478
+
479
+ query = RetrievalQuery(
480
+ query_id=f"gradio_{int(time.time())}",
481
+ mode=query_mode,
482
+ semantic_query=query_text,
483
+ max_results=max_results * 2,
484
+ fractalstat_hybrid=effective_hybrid,
485
+ confidence_threshold=0.2 if not effective_hybrid else 0.3,
486
+ )
487
+
488
+ print(
489
+ f"DEBUG: Executing query '{query_text}' with mode={query_mode}, "
490
+ f"max_results={max_results}, hybrid={use_hybrid}"
491
+ )
492
+
493
+ assembly = api.retrieve_context(query)
494
+ elapsed_ms = (time.time() - start_time) * 1000
495
+
496
+ original_results = len(assembly.results)
497
+ quarantined_results = [result for result in assembly.results if result.conflict_flags]
498
+ retained_results = [result for result in assembly.results if not result.conflict_flags]
499
+ final_results = retained_results[:max_results]
500
+
501
+ print(
502
+ f"DEBUG: Query completed in {elapsed_ms:.0f}ms, found {original_results} raw results, "
503
+ f"Bob quarantined {len(quarantined_results)}, retained {len(final_results)} for display"
504
+ )
505
 
506
+ fallback_triggered = False
507
+ if effective_hybrid and len(final_results) == 0 and len(quarantined_results) > 0:
508
+ print("DEBUG: Hybrid retained 0 results, checking quarantined pool for semantic fallback")
509
+ semantic_query = RetrievalQuery(
510
+ query_id=f"quarantine_fallback_{int(time.time())}",
511
+ mode=RetrievalMode.SEMANTIC_SIMILARITY,
512
+ semantic_query=query_text,
513
+ max_results=max_results,
514
+ confidence_threshold=0.3,
515
+ fractalstat_hybrid=False,
516
+ )
517
+ fallback_assembly = api.retrieve_context(semantic_query)
518
+ fallback_triggered = True
519
+ final_results = fallback_assembly.results[:max_results]
520
+ elapsed_ms += (time.time() - start_time - elapsed_ms / 1000) * 1000
521
+ print(
522
+ f"DEBUG: Quarantine fallback triggered - retrieved {len(final_results)} from semantic search"
523
+ )
524
 
525
+ results_data = [
526
+ {
527
+ "id": result.content_id,
528
+ "content": result.content,
529
+ "relevance_score": result.relevance_score,
530
+ "semantic_similarity": result.semantic_similarity,
531
+ "fractalstat_resonance": result.fractalstat_resonance,
532
+ "metadata": result.metadata,
533
  }
534
+ for result in final_results
535
+ ]
536
+
537
+ generated_answer = None
538
+ if generate_answer:
539
+ generated_answer = answer_generator.generate_answer(query_text, results_data)
540
+
541
+ output = "## Query Results\n\n"
542
+ output += f"**Query:** {query_text}\n\n"
543
+ output += f"**Found:** {len(final_results)} results in {elapsed_ms:.0f}ms\n\n"
544
+ if len(quarantined_results) > 0:
545
+ output += (
546
+ f"**Risk Assessment:** {len(quarantined_results)} potentially conflicting results "
547
+ "quarantined by Bob the Skeptic\n\n"
548
+ )
549
+ elif len(final_results) < original_results:
550
+ output += f"**Risk Assessment:** {original_results - len(final_results)} duplicates removed\n\n"
551
+ else:
552
+ output += "**Risk Assessment:** No conflicts detected by Bob the Skeptic\n\n"
553
+
554
+ output += f"**Quality Score:** {assembly.assembly_quality:.3f}\n\n"
555
+
556
+ if generated_answer:
557
+ output += "### Generated Answer\n\n"
558
+ output += f"{generated_answer.answer}\n\n"
559
+ if generated_answer.citations:
560
+ citation_line = ", ".join(
561
+ f"[{citation['index']}] {citation.get('pack') or citation.get('id')}"
562
+ for citation in generated_answer.citations[:5]
563
+ )
564
+ output += f"**Sources:** {citation_line}\n\n"
565
+ output += f"**Generator:** {generated_answer.provider}\n\n"
566
+
567
+ if final_results:
568
+ output += "### Top Results\n\n"
569
+ for index, result in enumerate(final_results[:max_results], 1):
570
+ output += f"**{index}. Score: {result.relevance_score:.3f}**\n\n"
571
+ output += f"{result.content[:300]}...\n\n"
572
+ if effective_hybrid and result.fractalstat_resonance is not None:
573
+ output += f"- Semantic: {result.semantic_similarity:.3f}\n"
574
+ output += f"- FractalStat: {result.fractalstat_resonance:.3f}\n\n"
575
+ output += "---\n\n"
576
+ else:
577
+ output += "No results found.\n"
578
+
579
+ if fallback_triggered:
580
+ output += "*\\*Note: Used semantic search because hybrid results were quarantined\\*\n"
581
+
582
+ return output
583
+
584
+
585
+ def query_from_synthesis_tab(query_text: str) -> str:
586
+ """Run a quick semantic follow-up query from the Synthesis tab."""
587
+ return query_warbler_with_quarantine(
588
+ query_text=query_text,
589
+ max_results=5,
590
+ use_hybrid=False,
591
+ generate_answer=True,
592
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
593
 
594
 
595
  # Create Gradio interface
 
635
  with gr.Column():
636
  results_output = gr.Markdown(label="Results")
637
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
638
  query_btn.click( # pylint: disable=E1101
639
  fn=query_warbler_with_quarantine,
640
  inputs=[query_input, max_results, use_hybrid, generate_answer_toggle],
 
679
  step=1,
680
  label="Macro Trigger Count",
681
  )
682
+ with gr.Row():
683
+ synthesis_add_macro = gr.Checkbox(
684
+ label="Add macro distillation to current search session",
685
+ value=False,
686
+ )
687
+ synthesis_add_source = gr.Checkbox(
688
+ label="Also add original source text to current search session",
689
+ value=False,
690
+ )
691
  synthesis_btn = gr.Button("Synthesize", variant="primary")
692
  synthesis_output = gr.Markdown(
693
+ "Paste a document and click **Synthesize** to generate a TLDA-style macro distillation. Optionally add it to the current search session."
694
  )
695
  synthesis_btn.click(
696
  fn=synthesize_document,
697
+ inputs=[
698
+ synthesis_input,
699
+ synthesis_micro_window,
700
+ synthesis_macro_trigger,
701
+ synthesis_add_macro,
702
+ synthesis_add_source,
703
+ ],
704
  outputs=synthesis_output,
705
  )
706
+ gr.Markdown("### Query This Synthesis")
707
+ synthesis_query_input = gr.Textbox(
708
+ label="Follow-up Query",
709
+ placeholder="Ask about the newly added synthesis artifacts without leaving this tab...",
710
+ lines=2,
711
+ )
712
+ synthesis_query_btn = gr.Button("Query Current Session", variant="secondary")
713
+ synthesis_query_output = gr.Markdown(
714
+ "If you added synthesis artifacts to search, run a follow-up query here to inspect them immediately."
715
+ )
716
+ synthesis_query_btn.click(
717
+ fn=query_from_synthesis_tab,
718
+ inputs=[synthesis_query_input],
719
+ outputs=synthesis_query_output,
720
+ )
721
 
722
  with gr.Tab("About"):
723
  gr.HTML("""
tests/test_synthesis_bridge.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from warbler_cda.retrieval_api import RetrievalAPI, RetrievalMode, RetrievalQuery
2
+ from warbler_cda.synthesis_bridge import (
3
+ build_synthesis_fragments,
4
+ get_synthesis_session_summary,
5
+ ingest_synthesis_artifacts,
6
+ synthesize_text,
7
+ )
8
+
9
+
10
+ def test_build_synthesis_fragments_prefers_line_structure():
11
+ fragments = build_synthesis_fragments("line one\n\nline two\nline three")
12
+
13
+ assert len(fragments) == 3
14
+ assert fragments[0]["id"] == "fragment_1"
15
+ assert fragments[1]["text"] == "line two"
16
+
17
+
18
+
19
+ def test_synthesize_text_generates_macro_for_longer_input():
20
+ text = "\n".join(
21
+ [
22
+ "Ten of Hearts is the weekly prompt.",
23
+ "Use the card and its symbols as inspiration.",
24
+ "The jam allows joining late and taking breaks.",
25
+ "Projects should be submitted within the relevant week.",
26
+ "AI use is discouraged and should be labeled if used.",
27
+ "Bigotry and harassment are prohibited.",
28
+ "New submissions and reworks are allowed.",
29
+ "Participants should be kind to one another.",
30
+ "Submissions should connect to the card theme.",
31
+ "The focus is on making games you want to make.",
32
+ ]
33
+ )
34
+
35
+ artifacts = synthesize_text(text, micro_window_size=3, macro_trigger_count=2)
36
+
37
+ assert artifacts.fragments
38
+ assert artifacts.macro_distillation is not None
39
+ assert artifacts.metrics["macro_distillations_total"] >= 1
40
+ assert "## Synthesis Output" in artifacts.report_markdown
41
+
42
+
43
+
44
+ def test_ingested_synthesis_source_becomes_queryable():
45
+ api = RetrievalAPI(config={"enable_fractalstat_hybrid": False, "enable_lexical_index": True})
46
+ text = "\n".join(
47
+ [
48
+ "Card game jam entries should fit on a playing card.",
49
+ "Participants can use Ten of Hearts as inspiration.",
50
+ "The jam values quantity of output and kindness.",
51
+ "Submitting within the week is encouraged.",
52
+ "Decks of cards can be used as a randomizer.",
53
+ ]
54
+ )
55
+ artifacts = synthesize_text(text, micro_window_size=3, macro_trigger_count=2)
56
+
57
+ ingested_ids = ingest_synthesis_artifacts(
58
+ api=api,
59
+ document_text=text,
60
+ artifacts=artifacts,
61
+ include_source=True,
62
+ include_macro=True,
63
+ )
64
+
65
+ assert len(ingested_ids) >= 1
66
+ query = RetrievalQuery(
67
+ query_id="bridge-test",
68
+ mode=RetrievalMode.SEMANTIC_SIMILARITY,
69
+ semantic_query="playing card kindness",
70
+ fractalstat_hybrid=False,
71
+ confidence_threshold=0.0,
72
+ max_results=5,
73
+ )
74
+
75
+ assembly = api.retrieve_context(query)
76
+
77
+ assert assembly.results
78
+ assert any(result.content_id in ingested_ids for result in assembly.results)
79
+
80
+
81
+ def test_synthesis_session_summary_counts_ingested_artifacts():
82
+ api = RetrievalAPI(config={"enable_fractalstat_hybrid": False, "enable_lexical_index": True})
83
+ text = "alpha\nbeta\ngamma\ndelta\nepsilon\nzeta"
84
+ artifacts = synthesize_text(text, micro_window_size=3, macro_trigger_count=2)
85
+
86
+ ingest_synthesis_artifacts(
87
+ api=api,
88
+ document_text=text,
89
+ artifacts=artifacts,
90
+ include_source=True,
91
+ include_macro=True,
92
+ )
93
+
94
+ summary = get_synthesis_session_summary(api)
95
+
96
+ assert summary["total"] >= 1
97
+ assert summary["source_text"] == 1
98
+ assert summary["macro_distillation"] >= 1
99
+ assert summary["recent_ids"]
warbler_cda/synthesis_bridge.py ADDED
@@ -0,0 +1,188 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Utilities for TLDA-style synthesis and optional retrieval ingestion."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ import hashlib
7
+ import time
8
+ from typing import Any, Dict, List, Optional
9
+
10
+ from warbler_cda.summarization_ladder import SummarizationLadder
11
+
12
+
13
+ @dataclass
14
+ class SynthesisArtifacts:
15
+ """Structured artifacts produced by a synthesis run."""
16
+
17
+ report_markdown: str
18
+ fragments: List[Dict[str, Any]]
19
+ micro_summaries: List[str]
20
+ macro_distillation: Optional[str]
21
+ metrics: Dict[str, Any]
22
+
23
+
24
+ def build_synthesis_fragments(document_text: str) -> List[Dict[str, Any]]:
25
+ """Split text into fragments suitable for the summarization ladder."""
26
+ cleaned = document_text.strip()
27
+ if not cleaned:
28
+ return []
29
+
30
+ units = [part.strip() for part in cleaned.splitlines() if part.strip()]
31
+ if len(units) < 2:
32
+ units = [sentence.strip() for sentence in cleaned.split(".") if sentence.strip()]
33
+
34
+ fragments: List[Dict[str, Any]] = []
35
+ for index, unit in enumerate(units, start=1):
36
+ fragments.append(
37
+ {
38
+ "id": f"fragment_{index}",
39
+ "text": unit,
40
+ "heat": min(1.0, 0.4 + (len(unit) / 400.0)),
41
+ }
42
+ )
43
+
44
+ return fragments
45
+
46
+
47
+ def synthesize_text(
48
+ document_text: str,
49
+ micro_window_size: int = 5,
50
+ macro_trigger_count: int = 3,
51
+ ) -> SynthesisArtifacts:
52
+ """Generate hierarchical synthesis artifacts and a markdown report."""
53
+ fragments = build_synthesis_fragments(document_text)
54
+ if not fragments:
55
+ raise ValueError("Please provide document text to synthesize.")
56
+
57
+ ladder = SummarizationLadder(
58
+ config={
59
+ "micro_window_size": micro_window_size,
60
+ "macro_trigger_count": macro_trigger_count,
61
+ }
62
+ )
63
+ report = ladder.process_fragments(fragments)
64
+
65
+ micro_summaries = [summary.compressed_text for summary in list(ladder.micro_summaries)]
66
+ macro_distillation = (
67
+ ladder.macro_distillations[-1].distilled_essence if ladder.macro_distillations else None
68
+ )
69
+ metrics = ladder.get_compression_metrics()["current_state"]
70
+
71
+ lines = ["## Synthesis Output", "", f"**Input Fragments:** {len(fragments)}", ""]
72
+
73
+ if macro_distillation:
74
+ lines.extend(["### Macro Distillation", "", macro_distillation, ""])
75
+
76
+ if micro_summaries:
77
+ lines.append("### Micro Summaries")
78
+ lines.append("")
79
+ for micro in micro_summaries[-3:]:
80
+ lines.append(f"- {micro}")
81
+ lines.append("")
82
+
83
+ lines.extend(
84
+ [
85
+ "### Metrics",
86
+ "",
87
+ f"- Active Micro Summaries: {metrics['micro_summaries_active']}",
88
+ f"- Macro Distillations: {metrics['macro_distillations_total']}",
89
+ f"- Compression Ratio: {metrics['compression_ratio']:.2f}",
90
+ "",
91
+ ]
92
+ )
93
+
94
+ if report["new_macro_distillations"]:
95
+ lines.append("### New Distillation IDs")
96
+ lines.append("")
97
+ for item in report["new_macro_distillations"]:
98
+ lines.append(f"- {item['distillation_id']}")
99
+
100
+ return SynthesisArtifacts(
101
+ report_markdown="\n".join(lines),
102
+ fragments=fragments,
103
+ micro_summaries=micro_summaries,
104
+ macro_distillation=macro_distillation,
105
+ metrics=metrics,
106
+ )
107
+
108
+
109
+ def ingest_synthesis_artifacts(
110
+ api: Any,
111
+ document_text: str,
112
+ artifacts: SynthesisArtifacts,
113
+ include_source: bool = False,
114
+ include_macro: bool = True,
115
+ ) -> List[str]:
116
+ """Persist source and/or synthesis artifacts into the retrieval context store."""
117
+ ingested_ids: List[str] = []
118
+ timestamp = int(time.time() * 1000)
119
+ content_hash = hashlib.sha256(document_text.encode()).hexdigest()[:12]
120
+ base_metadata = {
121
+ "source": "synthesis_tab",
122
+ "content_hash": content_hash,
123
+ "artifact_type": "synthesis_session",
124
+ }
125
+
126
+ if include_source:
127
+ source_id = f"synthesis-source-{timestamp}-{content_hash}"
128
+ added = api.add_document(
129
+ source_id,
130
+ document_text,
131
+ {
132
+ **base_metadata,
133
+ "artifact_type": "source_text",
134
+ "fragment_count": len(artifacts.fragments),
135
+ },
136
+ auto_embed=False,
137
+ )
138
+ if added:
139
+ ingested_ids.append(source_id)
140
+
141
+ if include_macro and artifacts.macro_distillation:
142
+ macro_id = f"synthesis-macro-{timestamp}-{content_hash}"
143
+ added = api.add_document(
144
+ macro_id,
145
+ artifacts.macro_distillation,
146
+ {
147
+ **base_metadata,
148
+ "artifact_type": "macro_distillation",
149
+ "fragment_count": len(artifacts.fragments),
150
+ "micro_summary_count": len(artifacts.micro_summaries),
151
+ },
152
+ auto_embed=False,
153
+ )
154
+ if added:
155
+ ingested_ids.append(macro_id)
156
+
157
+ return ingested_ids
158
+
159
+
160
+ def get_synthesis_session_summary(api: Any) -> Dict[str, Any]:
161
+ """Summarize synthesis artifacts currently stored in the active retrieval session."""
162
+ context_store = getattr(api, "_context_store", {}) or {}
163
+
164
+ summary = {
165
+ "total": 0,
166
+ "source_text": 0,
167
+ "macro_distillation": 0,
168
+ "recent_ids": [],
169
+ }
170
+
171
+ synthesis_entries = []
172
+ for doc_id, doc_data in context_store.items():
173
+ metadata = doc_data.get("metadata", {})
174
+ if metadata.get("source") != "synthesis_tab":
175
+ continue
176
+
177
+ summary["total"] += 1
178
+ artifact_type = metadata.get("artifact_type")
179
+ if artifact_type == "source_text":
180
+ summary["source_text"] += 1
181
+ elif artifact_type == "macro_distillation":
182
+ summary["macro_distillation"] += 1
183
+
184
+ synthesis_entries.append((doc_data.get("added_at", 0), doc_id))
185
+
186
+ synthesis_entries.sort(reverse=True)
187
+ summary["recent_ids"] = [doc_id for _, doc_id in synthesis_entries[:5]]
188
+ return summary