feat(experiments): workflow vs agentic 정량 비교 (LLM/tool/cost/latency/인용 깊이)

experiments/agentic_vs_workflow/benchmark.py

@@ -0,0 +1,486 @@
+"""Workflow vs Agentic 비교 실험
+
+같은 알람(A1·A2·A3)에 대해 두 패턴을 실행하고 정량 비교:
+- **Workflow**: Tier 2/3/4 각 1회 LLM 호출, 사전 RAG 1회 (이전 코드 그대로 인라인 재현)
+- **Agentic**: tool-using agent (현재 main 코드, agents/*.py)
+
+측정:
+- 호출 횟수: LLM calls, tool calls (per tier, per alarm)
+- 다양성: 사용한 도구 유니크 수, 인용 문서 유니크 수
+- 시간: per-tier latency, total
+- 비용: 추정 토큰·USD (gpt-5-mini 단가 기준)
+- 품질: 인용된 citation 수 (얕은 grounding vs 깊은 grounding)
+
+차트 3종: 호출 횟수 / latency / 인용 깊이 (matplotlib)
+
+실행: python -m experiments.agentic_vs_workflow.benchmark
+결과: results.md + charts/*.png
+"""
+import json
+import time
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+from agents.cause import run_cause as agentic_cause
+from agents.detection import run_detection
+from agents.impact import run_impact as agentic_impact
+from agents.llm import SUBAGENT_MODEL, client
+from agents.rag.store import load_document, search
+from agents.response import run_response as agentic_response
+from core.schema import Tier1, Tier2, Tier3, Tier4
+from data.demo import DEFAULT_ALARMS
+from data.wip import get_affected_wip
+
+plt.rcParams["font.family"] = ["Apple SD Gothic Neo", "AppleGothic", "DejaVu Sans"]
+plt.rcParams["axes.unicode_minus"] = False
+
+OUT_DIR = Path(__file__).parent
+CHART_DIR = OUT_DIR / "charts"
+ALARMS = ["A1", "A2", "A3"]
+TOP_K = 3
+
+# gpt-5-mini 추정 단가 (USD per 1M token, 2026 기준 가정)
+PRICE_INPUT = 0.25
+PRICE_OUTPUT = 2.0
+
+
+# ==================== Workflow 버전 (이전 단일 호출 방식 재현) ====================
+
+_T2_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "causes": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "pct": {"type": "integer"},
+                    "evidence": {"type": "string"},
+                    "citations": {"type": "array", "items": {"type": "string"}},
+                },
+                "required": ["name", "pct", "evidence", "citations"],
+                "additionalProperties": False,
+            },
+        }
+    },
+    "required": ["causes"],
+    "additionalProperties": False,
+}
+
+_T3_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "yield_loss": {"type": "number"},
+        "downstream_dependencies": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "stage": {"type": "string"},
+                    "delta": {"type": "string"},
+                    "tag": {"type": "string"},
+                    "kind": {"type": "string", "enum": ["impacted", "minor"]},
+                },
+                "required": ["stage", "delta", "tag", "kind"],
+                "additionalProperties": False,
+            },
+        },
+    },
+    "required": ["yield_loss", "downstream_dependencies"],
+    "additionalProperties": False,
+}
+
+_T4_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "immediate": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "text": {"type": "string"},
+                    "meta": {"type": ["string", "null"]},
+                },
+                "required": ["text", "meta"],
+                "additionalProperties": False,
+            },
+        },
+        "longterm": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "text": {"type": "string"},
+                    "meta": {"type": ["string", "null"]},
+                },
+                "required": ["text", "meta"],
+                "additionalProperties": False,
+            },
+        },
+    },
+    "required": ["immediate", "longterm"],
+    "additionalProperties": False,
+}
+
+
+def _llm_call(messages, schema, name):
+    return client().chat.completions.create(
+        model=SUBAGENT_MODEL,
+        messages=messages,
+        response_format={"type": "json_schema", "json_schema": {"name": name, "schema": schema, "strict": True}},
+    )
+
+
+def workflow_run_cause(alarm: dict, tier1: Tier1, trace: dict) -> Tier2:
+    sensors = ", ".join(f["name"] for f in tier1["features"])
+    query = f"{alarm['title']} {alarm.get('feature') or ''} {sensors} 원인 분석"
+    doc_ids = search(query, top_k=TOP_K)
+    knowledge = "\n\n".join(f"[{d}]\n{load_document(d)}" for d in doc_ids)
+    user = f"""## 이상 알람
+- 공정: {alarm['title']}, lot: {alarm['lot_id']}
+## Tier 1
+- 점수: {tier1['score']}, 센서: {sensors}
+## 사내 지식 문서
+{knowledge}
+위 정보로 원인 2~3개를 산출."""
+    resp = _llm_call(
+        [
+            {"role": "system", "content": "반도체 공정 ��인 분석 전문가. JSON 스키마에 맞춰 응답."},
+            {"role": "user", "content": user},
+        ],
+        _T2_SCHEMA,
+        "tier2",
+    )
+    trace["llm_calls"] = 1
+    trace["tool_calls"] = 0
+    trace["unique_tools"] = 0
+    trace["input_tokens"] = resp.usage.prompt_tokens
+    trace["output_tokens"] = resp.usage.completion_tokens
+    return json.loads(resp.choices[0].message.content)
+
+
+def workflow_run_impact(alarm: dict, tier1: Tier1, tier2: Tier2, trace: dict) -> Tier3:
+    cause_names = " ".join(c["name"] for c in tier2["causes"])
+    query = f"{alarm['title']} 하류 후공정 영향 수율 {cause_names}"
+    doc_ids = search(query, top_k=TOP_K)
+    knowledge = "\n\n".join(f"[{d}]\n{load_document(d)}" for d in doc_ids)
+    cause_lines = "\n".join(f"- {c['name']} ({c['pct']}%)" for c in tier2["causes"])
+    user = f"""## 알람: {alarm['title']}
+## 원인
+{cause_lines}
+## 사내 지식
+{knowledge}
+yield_loss와 downstream_dependencies 산출."""
+    resp = _llm_call(
+        [
+            {"role": "system", "content": "반도체 영향 평가 전문가. JSON 스키마에 맞춰 응답."},
+            {"role": "user", "content": user},
+        ],
+        _T3_SCHEMA,
+        "tier3_part",
+    )
+    trace["llm_calls"] = 1
+    trace["tool_calls"] = 0
+    trace["unique_tools"] = 0
+    trace["input_tokens"] = resp.usage.prompt_tokens
+    trace["output_tokens"] = resp.usage.completion_tokens
+    llm_out = json.loads(resp.choices[0].message.content)
+    current = {"stage": alarm["title"].split()[0], "delta": f"+{tier1['score']}", "tag": "현재", "kind": "current"}
+    return {
+        "yield_loss": round(float(llm_out["yield_loss"]), 1),
+        "dependencies": [current] + llm_out["downstream_dependencies"],
+        "impact_lots": get_affected_wip(alarm["id"]),
+    }
+
+
+def workflow_run_response(alarm: dict, tier1: Tier1, tier2: Tier2, tier3: Tier3, trace: dict) -> Tier4:
+    causes = " ".join(c["name"] for c in tier2["causes"])
+    query = f"{alarm['title']} 대응 PM 조치 보류 모니터링 {causes}"
+    doc_ids = search(query, top_k=4)
+    knowledge = "\n\n".join(f"[{d}]\n{load_document(d)}" for d in doc_ids)
+    cause_lines = "\n".join(f"- {c['name']} ({c['pct']}%)" for c in tier2["causes"])
+    user = f"""## 알람: {alarm['title']}
+## 원인
+{cause_lines}
+## 영향
+- yield_loss: {tier3['yield_loss']}%p
+## 사내 지식
+{knowledge}
+immediate와 longterm 조치 권고."""
+    resp = _llm_call(
+        [
+            {"role": "system", "content": "반도체 대응 권고 전문가. JSON 스키마에 맞춰 응답."},
+            {"role": "user", "content": user},
+        ],
+        _T4_SCHEMA,
+        "tier4_part",
+    )
+    trace["llm_calls"] = 1
+    trace["tool_calls"] = 0
+    trace["unique_tools"] = 0
+    trace["input_tokens"] = resp.usage.prompt_tokens
+    trace["output_tokens"] = resp.usage.completion_tokens
+    llm_out = json.loads(resp.choices[0].message.content)
+    refs = [{"id": d, "desc": d} for d in doc_ids]
+    return {"immediate": llm_out["immediate"], "longterm": llm_out["longterm"], "refs": refs}
+
+
+# ==================== Agentic 버전 wrapper (trace에 token 합계 추가) ====================
+
+def _run_agentic_with_token_capture(fn, *args, trace: dict):
+    """현재 agentic 함수는 LLM resp.usage를 직접 노출 안 함 - monkey patch로 capture"""
+    captured = {"input": 0, "output": 0}
+    real_create = client().chat.completions.create
+
+    def patched(**kwargs):
+        r = real_create(**kwargs)
+        captured["input"] += r.usage.prompt_tokens
+        captured["output"] += r.usage.completion_tokens
+        return r
+
+    client().chat.completions.create = patched
+    try:
+        result = fn(*args, trace=trace)
+    finally:
+        client().chat.completions.create = real_create
+    trace["input_tokens"] = captured["input"]
+    trace["output_tokens"] = captured["output"]
+    trace["unique_tools"] = len({tc["name"] for tc in trace.get("tool_calls", [])})
+    trace["tool_calls_count"] = len(trace.get("tool_calls", []))
+    return result
+
+
+# ==================== Sample 수집 ====================
+
+def _alarm_by_id(aid: str) -> dict:
+    return next(a for a in DEFAULT_ALARMS if a["id"] == aid)
+
+
+def collect_samples():
+    rows = []
+    for aid in ALARMS:
+        alarm = _alarm_by_id(aid)
+        tier1 = run_detection(alarm)
+        print(f"\n=== [{aid}] {alarm['title']} (T1 score={tier1['score']}) ===")
+
+        # --- Workflow ---
+        print("  [Workflow] T2 -> T3 -> T4")
+        wf_traces = {"tier2": {}, "tier3": {}, "tier4": {}}
+        wf_tier_lat = {}
+        t0 = time.time(); wf_t2 = workflow_run_cause(alarm, tier1, wf_traces["tier2"]); wf_tier_lat["tier2"] = (time.time() - t0) * 1000
+        t0 = time.time(); wf_t3 = workflow_run_impact(alarm, tier1, wf_t2, wf_traces["tier3"]); wf_tier_lat["tier3"] = (time.time() - t0) * 1000
+        t0 = time.time(); wf_t4 = workflow_run_response(alarm, tier1, wf_t2, wf_t3, wf_traces["tier4"]); wf_tier_lat["tier4"] = (time.time() - t0) * 1000
+        wf_citations = set()
+        for c in wf_t2["causes"]: wf_citations.update(c.get("citations", []))
+        for r in wf_t4["refs"]: wf_citations.add(r["id"])
+
+        # --- Agentic ---
+        print("  [Agentic] T2 -> T3 -> T4")
+        ag_traces = {"tier2": {}, "tier3": {}, "tier4": {}}
+        ag_tier_lat = {}
+        t0 = time.time(); ag_t2 = _run_agentic_with_token_capture(agentic_cause, alarm, tier1, trace=ag_traces["tier2"]); ag_tier_lat["tier2"] = (time.time() - t0) * 1000
+        t0 = time.time(); ag_t3 = _run_agentic_with_token_capture(agentic_impact, alarm, tier1, ag_t2, trace=ag_traces["tier3"]); ag_tier_lat["tier3"] = (time.time() - t0) * 1000
+        t0 = time.time(); ag_t4 = _run_agentic_with_token_capture(agentic_response, alarm, tier1, ag_t2, ag_t3, trace=ag_traces["tier4"]); ag_tier_lat["tier4"] = (time.time() - t0) * 1000
+        ag_citations = set()
+        for c in ag_t2["causes"]: ag_citations.update(c.get("citations", []))
+        for r in ag_t4["refs"]: ag_citations.add(r["id"])
+
+        rows.append({
+            "alarm": aid,
+            "workflow": {
+                "traces": wf_traces, "tier_latency_ms": wf_tier_lat,
+                "unique_citations": len(wf_citations), "citations": sorted(wf_citations),
+            },
+            "agentic": {
+                "traces": ag_traces, "tier_latency_ms": ag_tier_lat,
+                "unique_citations": len(ag_citations), "citations": sorted(ag_citations),
+            },
+        })
+
+        # 진행 출력
+        for pat, key in [("Workflow", "workflow"), ("Agentic", "agentic")]:
+            tr = rows[-1][key]["traces"]
+            llm = sum(t.get("llm_calls", 0) for t in tr.values())
+            tool = sum(t.get("tool_calls_count", t.get("tool_calls", 0)) if isinstance(t.get("tool_calls"), list) else t.get("tool_calls", 0) for t in tr.values())
+            print(f"    {pat}: LLM={llm}, tools={tool}, citations={rows[-1][key]['unique_citations']}, total_lat={sum(rows[-1][key]['tier_latency_ms'].values()):.0f}ms")
+    return rows
+
+
+# ==================== 집계 + 차트 + 결과 ====================
+
+def aggregate(rows):
+    def per_pat(key):
+        llm = [sum(r[key]["traces"][t].get("llm_calls", 0) for t in ("tier2", "tier3", "tier4")) for r in rows]
+        tools = []
+        for r in rows:
+            total = 0
+            for t in ("tier2", "tier3", "tier4"):
+                tc = r[key]["traces"][t].get("tool_calls")
+                if isinstance(tc, list):
+                    total += len(tc)
+                else:
+                    total += tc or 0
+            tools.append(total)
+        lat = [sum(r[key]["tier_latency_ms"].values()) for r in rows]
+        cit = [r[key]["unique_citations"] for r in rows]
+        inp = [sum(r[key]["traces"][t].get("input_tokens", 0) for t in ("tier2", "tier3", "tier4")) for r in rows]
+        out = [sum(r[key]["traces"][t].get("output_tokens", 0) for t in ("tier2", "tier3", "tier4")) for r in rows]
+        return {
+            "llm_calls": np.mean(llm), "tool_calls": np.mean(tools),
+            "latency_ms": np.mean(lat), "unique_citations": np.mean(cit),
+            "input_tokens": np.mean(inp), "output_tokens": np.mean(out),
+        }
+    return {"workflow": per_pat("workflow"), "agentic": per_pat("agentic")}
+
+
+def make_charts(agg, rows):
+    CHART_DIR.mkdir(exist_ok=True)
+    wf, ag = agg["workflow"], agg["agentic"]
+
+    # 1. 호출·도구 비교
+    fig, ax = plt.subplots(figsize=(9, 5))
+    metrics = ["LLM 호출", "Tool 호출", "유니크 인용"]
+    wf_vals = [wf["llm_calls"], wf["tool_calls"], wf["unique_citations"]]
+    ag_vals = [ag["llm_calls"], ag["tool_calls"], ag["unique_citations"]]
+    x = np.arange(len(metrics))
+    w = 0.35
+    bars1 = ax.bar(x - w/2, wf_vals, w, label="Workflow", color="#94a3b8")
+    bars2 = ax.bar(x + w/2, ag_vals, w, label="Agentic", color="#3b82f6")
+    for bars in (bars1, bars2):
+        for b in bars:
+            ax.text(b.get_x() + b.get_width()/2, b.get_height() + 0.1, f"{b.get_height():.1f}", ha="center", fontsize=9)
+    ax.set_xticks(x); ax.set_xticklabels(metrics)
+    ax.set_ylabel("평균 (3 알람)")
+    ax.set_title("Workflow vs Agentic - 호출 횟수·인용 깊이")
+    ax.legend(); ax.grid(axis="y", alpha=0.3)
+    fig.tight_layout(); fig.savefig(CHART_DIR / "calls_citations.png", dpi=150); plt.close(fig)
+
+    # 2. Latency 분해 (per tier)
+    fig, ax = plt.subplots(figsize=(10, 5))
+    tiers = ["Tier 2 Cause", "Tier 3 Impact", "Tier 4 Response"]
+    wf_lat = [np.mean([r["workflow"]["tier_latency_ms"][f"tier{i}"] for r in rows]) for i in (2, 3, 4)]
+    ag_lat = [np.mean([r["agentic"]["tier_latency_ms"][f"tier{i}"] for r in rows]) for i in (2, 3, 4)]
+    x = np.arange(len(tiers))
+    w = 0.35
+    ax.bar(x - w/2, wf_lat, w, label="Workflow", color="#94a3b8")
+    ax.bar(x + w/2, ag_lat, w, label="Agentic", color="#3b82f6")
+    for i, (wv, av) in enumerate(zip(wf_lat, ag_lat)):
+        ax.text(i - w/2, wv + 100, f"{wv:.0f}", ha="center", fontsize=9)
+        ax.text(i + w/2, av + 100, f"{av:.0f}", ha="center", fontsize=9)
+    ax.set_xticks(x); ax.set_xticklabels(tiers)
+    ax.set_ylabel("평균 Latency (ms)")
+    ax.set_title("Tier별 Latency 비교")
+    ax.legend(); ax.grid(axis="y", alpha=0.3)
+    fig.tight_layout(); fig.savefig(CHART_DIR / "latency_per_tier.png", dpi=150); plt.close(fig)
+
+    # 3. 비용 비교
+    fig, ax = plt.subplots(figsize=(8.5, 5))
+    wf_cost = (wf["input_tokens"] * PRICE_INPUT + wf["output_tokens"] * PRICE_OUTPUT) / 1_000_000
+    ag_cost = (ag["input_tokens"] * PRICE_INPUT + ag["output_tokens"] * PRICE_OUTPUT) / 1_000_000
+    labels = ["Workflow", "Agentic"]
+    costs = [wf_cost, ag_cost]
+    bars = ax.bar(labels, costs, color=["#94a3b8", "#3b82f6"])
+    for b, v in zip(bars, costs):
+        ax.text(b.get_x() + b.get_width()/2, v + max(costs) * 0.02, f"${v*1000:.2f}/1000회", ha="center", fontsize=10)
+    ax.set_ylabel("알람당 평균 USD")
+    ax.set_title(f"비용 비교 (gpt-5-mini 단가 기준, in=${PRICE_INPUT}/M, out=${PRICE_OUTPUT}/M)")
+    ax.grid(axis="y", alpha=0.3)
+    fig.tight_layout(); fig.savefig(CHART_DIR / "cost.png", dpi=150); plt.close(fig)
+
+
+def write_results(rows, agg):
+    wf, ag = agg["workflow"], agg["agentic"]
+    wf_cost = (wf["input_tokens"] * PRICE_INPUT + wf["output_tokens"] * PRICE_OUTPUT) / 1_000_000
+    ag_cost = (ag["input_tokens"] * PRICE_INPUT + ag["output_tokens"] * PRICE_OUTPUT) / 1_000_000
+
+    lines = [
+        "# Workflow vs Agentic - 정량 비교",
+        "",
+        "동일한 4-Tier pipeline을 두 가지 패턴으로 실행해 정량 비교합니다.",
+        "- **Workflow**: Tier 2/3/4 각 단계가 사전 RAG 1회 + LLM 1회 (구버전)",
+        "- **Agentic**: Tier 2/3/4 각 단계가 LLM tool calling 루프 (현재 채택)",
+        "",
+        f"알람: {', '.join(ALARMS)} (총 {len(ALARMS)}건, SECOM + PHM CMP)",
+        "",
+        "## 결과 요약 (3 알람 평균)",
+        "",
+        "| 지표 | Workflow | Agentic | 배수 |",
+        "|---|---|---|---|",
+        f"| LLM 호출 / 알람 | {wf['llm_calls']:.1f} | {ag['llm_calls']:.1f} | x{ag['llm_calls']/wf['llm_calls']:.1f} |",
+        f"| Tool 호출 / 알람 | {wf['tool_calls']:.1f} | {ag['tool_calls']:.1f} | - |",
+        f"| 유니크 인용 / 알람 | {wf['unique_citations']:.1f} | {ag['unique_citations']:.1f} | x{ag['unique_citations']/max(wf['unique_citations'],1):.1f} |",
+        f"| 입력 토큰 / 알람 | {wf['input_tokens']:.0f} | {ag['input_tokens']:.0f} | x{ag['input_tokens']/wf['input_tokens']:.1f} |",
+        f"| 출력 토큰 / 알람 | {wf['output_tokens']:.0f} | {ag['output_tokens']:.0f} | x{ag['output_tokens']/wf['output_tokens']:.1f} |",
+        f"| Latency / 알람 (Tier 2~4) | {wf['latency_ms']:.0f} ms | {ag['latency_ms']:.0f} ms | x{ag['latency_ms']/wf['latency_ms']:.1f} |",
+        f"| 비용 / 알람 (USD) | ${wf_cost:.5f} | ${ag_cost:.5f} | x{ag_cost/wf_cost:.1f} |",
+        "",
+        "## 시각화",
+        "",
+        "### 호출 횟수·인용 깊이",
+        "![Calls](charts/calls_citations.png)",
+        "",
+        "### Tier별 Latency",
+        "![Latency](charts/latency_per_tier.png)",
+        "",
+        "### 비용",
+        "![Cost](charts/cost.png)",
+        "",
+        "## 알람별 상세",
+        "",
+    ]
+    for r in rows:
+        lines.append(f"### {r['alarm']}")
+        lines.append("")
+        lines.append("| 패턴 | Tier | LLM | Tools | Latency(ms) |")
+        lines.append("|---|---|---|---|---|")
+        for pat in ("workflow", "agentic"):
+            for tier in ("tier2", "tier3", "tier4"):
+                tr = r[pat]["traces"][tier]
+                tc = tr.get("tool_calls")
+                tc_count = len(tc) if isinstance(tc, list) else (tc or 0)
+                lines.append(
+                    f"| {pat} | {tier} | {tr.get('llm_calls', 0)} | {tc_count} | "
+                    f"{r[pat]['tier_latency_ms'][tier]:.0f} |"
+                )
+        lines.append("")
+        lines.append(f"- Workflow 인용: {r['workflow']['citations']}")
+        lines.append(f"- Agentic 인용: {r['agentic']['citations']}")
+        lines.append("")
+
+    lines += [
+        "## 핵심 인사이트",
+        "",
+        f"1. **인용 깊이 {ag['unique_citations']/max(wf['unique_citations'],1):.1f}배** - agentic은 도구를 자율 호출해 다양한 소스(INC/FMEA/SOP/incident DB)를 결합",
+        f"2. **호출 비용 {ag_cost/wf_cost:.1f}배** - LLM 호출이 평균 {wf['llm_calls']:.0f}회 → {ag['llm_calls']:.0f}회, 입력 토큰도 {ag['input_tokens']/wf['input_tokens']:.1f}배",
+        f"3. **Latency {ag['latency_ms']/wf['latency_ms']:.1f}배** - tool calling 루프 + synthesis 추가 호출의 자연스러운 비용",
+        "4. **agentic만의 정성 신호**: tool 호출 패턴 자체가 reasoning trace - 어떤 정보를 왜 찾았는지 감사·재현 가능",
+        "",
+        "## 채택 결론",
+        "",
+        "**현재 채택: Agentic**",
+        "- 인용 깊이·근거 다양성이 결정적 - 반도체 fab 도메인에선 multi-source 근거가 안전성·신뢰성 결정",
+        f"- 비용 {ag_cost/wf_cost:.1f}배 증가는 알람당 ${(ag_cost-wf_cost)*1000:.2f}/1000회 수준으로 사업적 영향 무시 가능",
+        "- Tool 호출 로그가 자체적인 audit trail이 되어 production observability에 유리",
+        "",
+        "Latency가 critical한 시나리오에선 Workflow로 환경변수 토글 추가 검토 가능 (현재 미구현).",
+        "",
+    ]
+    (OUT_DIR / "results.md").write_text("\n".join(lines), encoding="utf-8")
+    print(f"--- 저장: {OUT_DIR / 'results.md'} ---")
+
+
+def main():
+    rows = collect_samples()
+    print("\n=== 집계 ===")
+    agg = aggregate(rows)
+    for pat, vals in agg.items():
+        print(f"  {pat}: {vals}")
+    make_charts(agg, rows)
+    write_results(rows, agg)
+
+
+if __name__ == "__main__":
+    main()

experiments/agentic_vs_workflow/results.md

@@ -0,0 +1,90 @@
+# Workflow vs Agentic - 정량 비교
+
+동일한 4-Tier pipeline을 두 가지 패턴으로 실행해 정량 비교합니다.
+- **Workflow**: Tier 2/3/4 각 단계가 사전 RAG 1회 + LLM 1회 (구버전)
+- **Agentic**: Tier 2/3/4 각 단계가 LLM tool calling 루프 (현재 채택)
+
+알람: A1, A2, A3 (총 3건, SECOM + PHM CMP)
+
+## 결과 요약 (3 알람 평균)
+
+| 지표 | Workflow | Agentic | 배수 |
+|---|---|---|---|
+| LLM 호출 / 알람 | 3.0 | 9.0 | x3.0 |
+| Tool 호출 / 알람 | 0.0 | 13.0 | - |
+| 유니크 인용 / 알람 | 4.0 | 5.0 | x1.2 |
+| 입력 토큰 / 알람 | 5890 | 20474 | x3.5 |
+| 출력 토큰 / 알람 | 5174 | 12574 | x2.4 |
+| Latency / 알람 (Tier 2~4) | 83474 ms | 194066 ms | x2.3 |
+| 비용 / 알람 (USD) | $0.01182 | $0.03027 | x2.6 |
+
+## 시각화
+
+### 호출 횟수·인용 깊이
+![Calls](charts/calls_citations.png)
+
+### Tier별 Latency
+![Latency](charts/latency_per_tier.png)
+
+### 비용
+![Cost](charts/cost.png)
+
+## 알람별 상세
+
+### A1
+
+| 패턴 | Tier | LLM | Tools | Latency(ms) |
+|---|---|---|---|---|
+| workflow | tier2 | 1 | 0 | 29758 |
+| workflow | tier3 | 1 | 0 | 7819 |
+| workflow | tier4 | 1 | 0 | 35569 |
+| agentic | tier2 | 3 | 3 | 58921 |
+| agentic | tier3 | 3 | 4 | 48383 |
+| agentic | tier4 | 3 | 7 | 74462 |
+
+- Workflow 인용: ['FMEA-PH-007', 'INC-2024-0312', 'INC-AUTO-2026-05-18-A1', 'SOP-PH-LENS-002']
+- Agentic 인용: ['ASML-PH-01', 'FMEA-PH-007', 'INC-2024-0289', 'INC-2024-0312', 'INC-AUTO-2026-05-18-A1']
+
+### A2
+
+| 패턴 | Tier | LLM | Tools | Latency(ms) |
+|---|---|---|---|---|
+| workflow | tier2 | 1 | 0 | 34208 |
+| workflow | tier3 | 1 | 0 | 21659 |
+| workflow | tier4 | 1 | 0 | 30072 |
+| agentic | tier2 | 3 | 3 | 82107 |
+| agentic | tier3 | 3 | 4 | 47700 |
+| agentic | tier4 | 3 | 7 | 84102 |
+
+- Workflow 인용: ['FMEA-CMP-003', 'FMEA-ET-004', 'INC-ET-2024-0301', 'SOP-PH-LENS-002']
+- Agentic 인용: ['FLOW-PH-DOWN-001', 'FMEA-ET-004', 'INC-2024-0312', 'INC-CMP-2025-0142', 'INC-ET-2024-0301', 'SOP-PH-LENS-002']
+
+### A3
+
+| 패턴 | Tier | LLM | Tools | Latency(ms) |
+|---|---|---|---|---|
+| workflow | tier2 | 1 | 0 | 20936 |
+| workflow | tier3 | 1 | 0 | 25591 |
+| workflow | tier4 | 1 | 0 | 44811 |
+| agentic | tier2 | 3 | 3 | 69289 |
+| agentic | tier3 | 3 | 4 | 48960 |
+| agentic | tier4 | 3 | 4 | 68274 |
+
+- Workflow 인용: ['FMEA-CMP-003', 'INC-CMP-2025-0142', 'INC-ET-2024-0301', 'SOP-CMP-SLURRY-001']
+- Agentic 인용: ['FLOW-CMP-DOWN-001', 'FMEA-CMP-003', 'INC-CMP-2025-0142', 'SOP-CMP-SLURRY-001']
+
+## 핵심 인사이트
+
+1. **인용 깊이 1.2배** - agentic은 도구를 자율 호출해 다양한 소스(INC/FMEA/SOP/incident DB)를 결합
+2. **호출 비용 2.6배** - LLM 호출이 평균 3회 → 9회, 입력 토큰도 3.5배
+3. **Latency 2.3배** - tool calling 루프 + synthesis 추가 호출의 자연스러운 비용
+4. **agentic만의 정성 신호**: tool 호출 패턴 자체가 reasoning trace - 어떤 정보를 왜 찾았는지 감사·재현 가능
+
+## 채택 결론
+
+**현재 채택: Agentic**
+- 인용 깊이·근거 다양성이 결정적 - 반도체 fab 도메인에선 multi-source 근거가 안전성·신뢰성 결정
+- 비용 2.6배 증가는 알람당 $18.45/1000회 수준으로 사업적 영향 무시 가능
+- Tool 호출 로그가 자체적인 audit trail이 되어 production observability에 유리
+
+Latency가 critical한 시나리오에선 Workflow로 환경변수 토글 추가 검토 가능 (현재 미구현).