scripts/23_weight_grid.py

"""
Dense/BM25 가중치 그리드 서치
===============================
실행:
    .venv/Scripts/python.exe scripts/23_weight_grid.py
"""
import sys, re, json
from pathlib import Path
from collections import defaultdict

sys.path.insert(0, str(Path(__file__).parent.parent))

import core.search_engine as se_mod
from core.search_engine import MenuSearchEngine

EVAL_PATH = Path(__file__).parent.parent / "data" / "eval_queries_300.json"


def normalize_path(path):
    if not path: return ''
    path = re.sub(r'\s*>\s*', '>', str(path))
    path = re.sub(r'\(.*?\)', '', path)
    return path.strip().lower()


def matches(pred, ans):
    if not pred or not ans: return False
    if pred == ans: return True
    pred_end = pred.split('>')[-1].strip()
    ans_end  = ans.split('>')[-1].strip()
    if pred_end and ans_end and pred_end == ans_end:
        pp = pred.split('>')
        ap = ans.split('>')
        return sum(1 for p in pp if p in ap) >= min(2, len(ap))
    return False


def run_eval(engine, eval_set, use_hyde=False):
    acc1, acc3, acc5, mrr_list = [], [], [], []
    for item in eval_set:
        hits  = engine.search(item['query'], top_n=5, threshold=0.0, use_hyde=use_hyde)
        paths = [normalize_path(h.get('menu_path', '')) for h in hits]
        ans   = normalize_path(item['menu_path'])
        h1 = matches(paths[0], ans) if paths else False
        h3 = any(matches(p, ans) for p in paths[:3])
        h5 = any(matches(p, ans) for p in paths[:5])
        rr = next((1.0/k for k, p in enumerate(paths[:5], 1) if matches(p, ans)), 0.0)
        acc1.append(h1); acc3.append(h3); acc5.append(h5); mrr_list.append(rr)
    n = len(acc1)
    return {
        'acc1': sum(acc1)/n, 'acc3': sum(acc3)/n,
        'acc5': sum(acc5)/n, 'mrr':  sum(mrr_list)/n, 'n': n,
    }


def main():
    with open(EVAL_PATH, encoding='utf-8') as f:
        eval_set = json.load(f)

    print(f"평가셋: {len(eval_set)}개")
    print("검색엔진 초기화 중...")
    engine = MenuSearchEngine.get_instance()
    print("준비 완료\n")

    configs = [
        ('Dense=1.5  BM25=0.5  (현재)', 1.5, 0.5),
        ('Dense=1.0  BM25=1.0  (동등)', 1.0, 1.0),
        ('Dense=1.0  BM25=1.5  (BM25↑)', 1.0, 1.5),
        ('Dense=1.2  BM25=0.8  (중간)', 1.2, 0.8),
    ]

    print(f"{'설정':<28} {'Acc@1':>6} {'Acc@3':>6} {'Acc@5':>6} {'MRR':>6}")
    print("-" * 60)
    for label, w_d, w_b in configs:
        se_mod.W_DENSE = w_d
        se_mod.W_BM25  = w_b
        r = run_eval(engine, eval_set, use_hyde=False)
        print(f"  {label:<26} {r['acc1']:>5.1%} {r['acc3']:>6.1%} {r['acc5']:>6.1%} {r['mrr']:>6.3f}")

    # 원래 값 복원
    se_mod.W_DENSE = 1.5
    se_mod.W_BM25  = 0.5
    print("\n가중치 원복 완료 (Dense=1.5, BM25=0.5)")


if __name__ == "__main__":
    main()