Spaces:
Sleeping
Sleeping
File size: 1,706 Bytes
53bb779 29809c8 53bb779 29809c8 53bb779 29809c8 53bb779 29809c8 53bb779 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | """
Ablation study on chunk size effect on faithfulness and retrieval quality.
"""
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from modules.knowledge_compiler import create_knowledge_compiler
from modules.reasoning_engine import create_reasoning_engine
from evaluation.metrics import MetricsRunner
def run_chunk_ablation(agent_name: str, parsed_data: list, system_prompt: str, prompt_analysis: dict, test_queries: list):
sizes = [64, 128, 256, 512, 1024]
metrics = MetricsRunner()
for size in sizes:
print(f"\n=====================")
print(f"Testing Chunk Size: {size}")
print(f"=====================")
compiler = create_knowledge_compiler()
original_chunk_text = compiler._chunk_text
compiler._chunk_text = lambda text, chunk_size=size, overlap=size//10: original_chunk_text(text, chunk_size, overlap)
# Recompile
try:
compiler.compile(agent_name, parsed_data, system_prompt, prompt_analysis)
# Test
engine = create_reasoning_engine()
for q in test_queries:
res = engine.reason(agent_name, q)
faithfulness = metrics.extract_faithfulness(res)
print(f"Q: {q}")
if faithfulness is None:
print("Faithfulness: N/A")
else:
print(f"Faithfulness: {faithfulness:.3f}")
except Exception as e:
print(f"Failed ablation step for size {size}: {e}")
if __name__ == "__main__":
print("Chunk size ablation script ready. Needs actual parsed data to recompile.")
|