""" evaluator.py — Eval 框架执行器（v2：Match / Priority 分离评估） """ from __future__ import annotations import json, re, sys, time from pathlib import Path from typing import Any, Optional from src.eval_schema import ( EvalTrace, ERROR_TAXONOMY, detect_errors, load_golden_cases, compute_ndcg, compute_mrr, golden_case_to_relevance, ) # --------------------------------------------------------------------------- # Agent 流程条配置 # --------------------------------------------------------------------------- AGENTS = [ {"key": "profile", "name": "Profile Builder", "desc": "解析简历 → 学生画像", "icon": "👤"}, {"key": "jd", "name": "JD Intelligence", "desc": "解析 JD → 岗位画像", "icon": "📋"}, {"key": "scout", "name": "Opportunity Scout", "desc": "召回候选岗位池", "icon": "🔍"}, {"key": "ranker", "name": "Application Ranker", "desc": "计算 Match/Pass/Risk/Growth", "icon": "📊"}, {"key": "gap", "name": "Gap Diagnosis", "desc": "能力缺口 + 简历风险", "icon": "⚠️"}, {"key": "conversion", "name": "Resume Conversion", "desc": "简历改写建议", "icon": "✏️"}, {"key": "strategy", "name": "Strategy Planner", "desc": "投递策略 + 7 天计划", "icon": "🗺️"}, ] STATUS_ICONS = {"completed": "✅", "running": "⏳", "pending": "⬜"} # 技术选型说明（底部折叠） _TECH_EXPLANATION = """ **项目定位：基于多 Agent 协作的大模型应用算法系统**（不是商业 HR 工具）。 --- ### 一、多 Agent 如何分工 | Agent | 职责 | LLM 是否参与 | |---|---|---| | Profile Builder | 简历 → 结构化画像 | 可选（理解非结构化文本） | | JD Intelligence | JD → 结构化岗位画像 | 可选（归一化技能表述） | | Opportunity Scout | 关键词/TF-IDF 召回候选池 | 否（保证稳定可解释） | | Application Ranker | 多维度加权公式计算排序 | 否（**排序必须可解释**） | | Gap Diagnosis | 规则检测能力缺口 | 可选（生成可读解释） | | Resume Conversion | 规则约束的简历改写 | 可选（语言优化） | | Strategy Planner | 规则策略 + 自然语言总结 | 可选（总结生成） | **核心设计：LLM 是"可选增强层"，非依赖项。无 API Key 时所有 Agent 仍可完整运行。** --- ### 二、为什么排序/决策用可解释公式，而不是黑盒 LLM 打分？ 1. **可解释性约束**：面试官/用户需要知道"为什么推荐这个岗位"，黑盒 LLM 打分无法回答 2. **权重可调试**：`ApplyPriority = 0.40×Match + 0.30×Pass - 0.15×Risk + 0.15×Growth`，每个权重都有业务含义 3. **稳定性**：LLM API 可能抖动，排序公式稳定可复现 4. **数据效率**：公式不需要标注数据，LLM 微调才需要 --- ### 三、语义召回为什么用轻量模型（bge-small-zh）而不是满血 LLM？ 1. **延迟**：LLM 调用 ~200ms，嵌入模型 ~20ms 2. **成本**：嵌入模型本地运行，LLM API 按 token 计费 3. **稳定性**：嵌入模型输出稳定，LLM 可能返回不一致相似度 4. **可解释性**：嵌入向量可以可视化（t-SNE），LLM 相似度是黑盒 --- ### 四、Eval/Error Analysis 如何保证系统质量？ 1. **Golden Cases**：8 个核心 case（覆盖同方向/跨方向/不同城市/不同风险偏好） 2. **Error Taxonomy**：10 类错误（E1–E10），每类有自动检测逻辑 3. **指标**：Top1 Acc、Recall@5、NDCG@5、MRR@5 4. **持续迭代**：每次修改排序公式后跑 eval，确保不降低核心指标 --- ### 五、为什么项目强调"算法深度"而不是"功能完整"？ 1. **求职目标**：算法岗位（大模型应用算法/LLM 应用算法/Agent 算法） 2. **核心叙事**：排序公式设计、语义召回优化、证据链构建、Eval/Error Analysis 迭代 3. **技术深度**：不是"调包侠"，而是"算法设计者" 4. **差异化**：大多数 Demo 项目只做"功能完整"，我们做"算法深度" --- """ # --------------------------------------------------------------------------- # 主流程 # --------------------------------------------------------------------------- def run_case(case: dict, jobs_path: str | Path) -> tuple[Any, list[dict], dict]: """运行单个 golden case，返回 (trace, errors, metrics)。""" if EvalTrace is None: raise RuntimeError("eval_schema.py not loaded") trace = EvalTrace() trace.case_id = case.get("case_id", "unknown") trace.timestamp = time.strftime("%Y-%m-%dT%H:%M:%S") trace.resume_text = case.get("resume_text", "") trace.target_role = case.get("target_role", "") trace.target_city = case.get("target_city", "") trace.stage = case.get("stage", "") errors: list[dict] = [] # ---- read new golden labels ---- exp_matches = set(case.get("expected_top_matches", [])) exp_priorities = set(case.get("expected_top_priorities", [])) exp_recall = set(case.get("expected_recall_jobs", [])) exp_action = case.get("expected_action", "") # ---- Agent 1: Profile Builder ---- t0 = time.time() profile: dict = {} try: from src.resume_parser import parse_resume profile = parse_resume(trace.resume_text) except Exception: pass # 注入用户目标偏好到 profile（供 StrategyPlanner 使用） if profile is None: profile = {} profile["_city"] = trace.target_city profile["_stage"] = trace.stage profile["_target_role"] = trace.target_role trace.profile_output = profile trace.agent_timings["profile"] = round(time.time() - t0, 3) # E1/E2 if profile: actual = set(profile.get("skills", [])) missed = set(case.get("expected_skills", [])) - actual if missed: errors.append({"code": "E1_PROFILE_MISS", "agent": "ProfileBuilder", "message": f"漏抽技能：{missed}", "severity": "high"}) rl = trace.resume_text.lower() for s in actual: if s.lower() not in rl: errors.append({"code": "E2_PROFILE_HALLUCINATION", "agent": "ProfileBuilder", "message": f"可能幻觉技能：{s}", "severity": "high"}) break # ---- Agent 3/4: Scout + Ranker ---- t1 = time.time() scored: list[dict] = [] try: from src.matcher import rank_jobs scored = rank_jobs( resume_text=trace.resume_text, profile=profile, target_role=trace.target_role, target_city=trace.target_city, stage=trace.stage, top_k=8, jobs_path=Path(jobs_path), ) except Exception: pass trace.scout_candidates = scored trace.agent_timings["scout"] = round(time.time() - t1, 3) # construct both rankings by_match = sorted(scored, key=lambda x: x.get("match_score", 0), reverse=True) by_priority = sorted(scored, key=lambda x: x.get("apply_priority", 0), reverse=True) top5_m = [j.get("title", "") for j in by_match[:5]] top5_p = [j.get("title", "") for j in by_priority[:5]] top1_m = by_match[0].get("title", "") if by_match else "" top1_p = by_priority[0].get("title", "") if by_priority else "" # E4: recall miss (expected_recall_jobs not in ANY top5) if exp_recall: hit_recall = exp_recall & set(top5_m) or exp_recall & set(top5_p) if not hit_recall: errors.append({"code": "E4_RECALL_MISS", "agent": "OpportunityScout", "message": f"expected_recall_jobs {exp_recall} 未进入任一 Top5", "severity": "high"}) # E4_MATCH: expected_top_matches not in match top5 if exp_matches and not (exp_matches & set(top5_m)): errors.append({"code": "E4_MATCH_RECALL_MISS", "agent": "OpportunityScout", "message": f"expected_top_matches {exp_matches} 未进入 Match Top5: {top5_m}", "severity": "high"}) # E4_PRIORITY: expected_top_priorities not in priority top5 if exp_priorities and not (exp_priorities & set(top5_p)): errors.append({"code": "E4_PRIORITY_RECALL_MISS", "agent": "OpportunityScout", "message": f"expected_top_priorities {exp_priorities} 未进入 Priority Top5: {top5_p}", "severity": "high"}) # E5: rank misorder (top1 not in expected for each list) if exp_matches and top1_m and top1_m not in exp_matches: errors.append({"code": "E5_MATCH_RANK_MISORDER", "agent": "ApplicationRanker", "message": f"Match Top1={top1_m}, expected {exp_matches}", "severity": "high"}) if exp_priorities and top1_p and top1_p not in exp_priorities: errors.append({"code": "E5_PRIORITY_RANK_MISORDER", "agent": "ApplicationRanker", "message": f"Priority Top1={top1_p}, expected {exp_priorities}", "severity": "high"}) # E6: score calibration if by_priority: ms = by_priority[0].get("match_score", 0) ap = by_priority[0].get("apply_priority", 0) if isinstance(ms, (int,float)) and isinstance(ap, (int,float)) and ms < 30 and ap > 70: errors.append({"code": "E6_SCORE_CALIBRATION", "agent": "ApplicationRanker", "message": f"Top1 match_score={ms:.1f} 过低但 apply_priority={ap:.1f} 过高", "severity": "medium"}) trace.ranker_scores = { "top1_match_title": top1_m, "top1_priority_title": top1_p, "top5_match": top5_m, "top5_priority": top5_p, } # ---- Agent 5/6: Gap + Conversion ---- t2 = time.time() # score_job() in matcher.py already called attach_conversion_scores(), # so conv_scored already has correct pass/risk/growth scores. conv_scored = scored gaps: list[str] = [] rewrites: list[dict] = [] for j in conv_scored: g = j.get("gaps", []) if isinstance(g, list): for item in g: if isinstance(item, str): gaps.append(item) elif isinstance(item, dict): gaps.append(item.get("skill", "")) r = j.get("rewrites", []) if isinstance(r, list): for item in r: if isinstance(item, dict): rewrites.append(item) trace.gap_diagnosis_output = {"gaps": gaps[:5]} trace.resume_conversion_output = {"rewrites": rewrites[:5]} # E7: gap diagnosis unsupported rl = trace.resume_text.lower() for gs in gaps: if isinstance(gs, str): from src.eval_schema import _extract_missing_terms_from_gap missing_terms = _extract_missing_terms_from_gap(gs) unsupported_terms = [term for term in missing_terms if term.lower() in rl] if unsupported_terms: errors.append({"code": "E7_GAP_UNSUPPORTED", "agent": "GapDiagnosis", "message": f"缺口诊断称缺失技能「{unsupported_terms[0]}」，但简历中存在相关证据", "severity": "medium"}) break elif isinstance(gs, dict): skill = gs.get("skill", "") if skill and skill.lower() in rl: errors.append({"code": "E7_GAP_UNSUPPORTED", "agent": "GapDiagnosis", "message": f"缺口诊断称缺失技能「{skill}」，但简历中存在相关证据", "severity": "medium"}) break # E8: rewrite overclaim for rw in rewrites: sl = rw.get("score_lift", 0) if isinstance(sl, (int,float)) and sl > 30: errors.append({"code": "E8_REWRITE_OVERCLAIM", "agent": "ResumeConversion", "message": f"改写声称过大：{rw.get('original','')} -> +{sl}", "severity": "high"}) break trace.agent_timings["conversion"] = round(time.time() - t2, 3) # ---- Agent 7: Strategy Planner ---- t3 = time.time() strategy: dict = {} try: from src.strategy_planner import gen_strategy_package strategy = gen_strategy_package(conv_scored, profile) except Exception: strategy = {} trace.strategy_output = strategy trace.agent_timings["strategy"] = round(time.time() - t3, 3) # E9: strategy conflict top3 = strategy.get("priority_top3", []) actions = [t.get("apply_action", "") for t in top3] if "立即投递" in str(actions) and "暂缓" in str(actions): errors.append({"code": "E9_STRATEGY_CONFLICT", "agent": "StrategyPlanner", "message": "投递策略冲突：同时建议立即投递和暂缓", "severity": "medium"}) # E9_ACTION_MISMATCH: actual action vs expected actual_action = top3[0].get("apply_action", "") if top3 else "" if exp_action and actual_action and exp_action != actual_action: errors.append({"code": "E9_ACTION_MISMATCH", "agent": "StrategyPlanner", "message": f"expected_action={exp_action}, actual_action={actual_action}", "severity": "medium"}) # ---- Report ---- t4 = time.time() try: from src.report_generator import generate_report generate_report(profile, conv_scored, strategy) except Exception: pass trace.agent_timings["report"] = round(time.time() - t4, 3) # ---- Metrics ---- metrics = _compute_metrics(trace, case, by_match, by_priority, exp_matches, exp_priorities, exp_recall, exp_action, actual_action) # ---- detect_errors (supplemental) ---- if detect_errors: extra = detect_errors(trace, case) existing = {e["code"] for e in errors} for ee in extra: if ee["code"] not in existing: errors.append(ee) existing.add(ee["code"]) trace.errors = errors trace.overall_pass = len(errors) == 0 trace.pass_reason = "" if trace.overall_pass else "; ".join(e["code"] for e in errors) trace.agent_metrics = metrics return trace, errors, metrics def _compute_metrics(trace, case, by_match, by_priority, exp_matches, exp_priorities, exp_recall, exp_action, actual_action): """Compute metrics for a single case.""" m = {} # Match t1m = by_match[0].get("title", "") if by_match else "" t5m = [j.get("title", "") for j in by_match[:5]] m["match_top1_hit"] = (t1m in exp_matches) if exp_matches else None m["match_recall_at_5"] = (len(exp_matches & set(t5m))/len(exp_matches)) if exp_matches else None # Priority t1p = by_priority[0].get("title", "") if by_priority else "" t5p = [j.get("title", "") for j in by_priority[:5]] m["priority_top1_hit"] = (t1p in exp_priorities) if exp_priorities else None m["priority_recall_at_5"] = (len(exp_priorities & set(t5p))/len(exp_priorities)) if exp_priorities else None # Recall (global) all5 = set(t5m) | set(t5p) m["recall_hit"] = (len(exp_recall & all5))/len(exp_recall) if exp_recall else None # Action m["action_hit"] = (exp_action == actual_action) if exp_action and actual_action else None m["expected_action"] = exp_action m["actual_action"] = actual_action # NDCG / MRR (需要 golden_case_to_relevance / compute_ndcg / compute_mrr） match_titles = [j.get("title", "") for j in by_match] pri_titles = [j.get("title", "") for j in by_priority] if golden_case_to_relevance: match_rel = golden_case_to_relevance(match_titles, case) pri_rel = golden_case_to_relevance(pri_titles, case) if match_rel and compute_ndcg: m["match_ndcg_5"] = round(compute_ndcg(match_rel, k=5), 4) if match_rel and compute_mrr: m["mrr_5"] = round(compute_mrr(match_rel, k=5), 4) if pri_rel and compute_ndcg: m["priority_ndcg_5"] = round(compute_ndcg(pri_rel, k=5), 4) m["top1_match_title"] = t1m m["top1_priority_title"] = t1p m["top5_match"] = t5m m["top5_priority"] = t5p m["expected_top_matches"] = list(exp_matches) m["expected_top_priorities"] = list(exp_priorities) m["expected_recall_jobs"] = list(exp_recall) # errors m["error_count"] = len(trace.errors) m["error_types"] = list({e["code"] for e in trace.errors}) ae: dict[str, int] = {} for e in trace.errors: ae[e.get("agent", "?")] = ae.get(e.get("agent", "?"), 0) + 1 m["agent_error_counts"] = ae return m # --------------------------------------------------------------------------- # Golden Case 管理 # --------------------------------------------------------------------------- CORE_CASE_IDS = {f"case_{i:02d}" for i in range(1, 9)} def _case_split(case: dict) -> str: """Return the evaluation split for a golden case.""" explicit = case.get("eval_split") if explicit: return str(explicit) if "jd_text" in case: return "jd_intake" return "core" if case.get("case_id") in CORE_CASE_IDS else "stress" def _should_run_case(case: dict, eval_split: str) -> bool: """Check if a case should be run.""" if case.get("skip_eval"): return False if "jd_text" in case: return False split = _case_split(case) if eval_split == "all": return split in {"core", "stress"} return split == eval_split def run_all_cases(golden_path: str|Path, jobs_path: str|Path, eval_split: str = "core"): """Run all golden cases and return traces + summary.""" cases = load_golden_cases(golden_path) traces, all_errs, all_met = [], [], [] for c in cases: if not _should_run_case(c, eval_split): continue t, err, met = run_case(c, jobs_path) traces.append(t); all_errs.extend(err); all_met.append(met) summary = _summarize(traces, all_errs, all_met) summary["eval_split"] = eval_split return traces, summary def _summarize(traces, all_errs, all_met): """Summarize eval results.""" n = len(traces) if n == 0: return {"total_cases": 0} def _avg(seq): valid = [x for x in seq if x is not None] return round(sum(valid)/len(valid), 4) if valid else None return { "total_cases": n, "pass_cases": sum(1 for t in traces if t.overall_pass), "pass_rate": round(sum(1 for t in traces if t.overall_pass)/n, 4), "match_top1_acc": _avg([m.get("match_top1_hit") for m in all_met]), "priority_top1_acc": _avg([m.get("priority_top1_hit") for m in all_met]), "action_acc": _avg([m.get("action_hit") for m in all_met]), "match_recall_at_5": _avg([m.get("match_recall_at_5") for m in all_met]), "priority_recall_at_5": _avg([m.get("priority_recall_at_5") for m in all_met]), "error_counts": {c: sum(1 for e in all_errs if e["code"]==c) for c in sorted({e["code"] for e in all_errs})}, "agent_error_counts": dict(__import__("collections").Counter(e.get("agent", "?") for e in all_errs)), } # --------------------------------------------------------------------------- # 报告生成 # --------------------------------------------------------------------------- def generate_eval_report(traces, summary, output_path): """Generate eval report (Markdown).""" lines = ["# Eval Report (v2: Match/Priority split)", "", f"Generated: {time.strftime('%Y-%m-%d %H:%M:%S')}", ""] lines += ["## Summary", "", f"| Metric | Value |", "|---|---|", f"| Eval split | {summary.get('eval_split','core')} |", f"| Total cases | {summary.get('total_cases','-')} |", f"| Pass cases | {summary.get('pass_cases','-')} |", f"| Pass rate | {round(summary.get('pass_rate',0)*100,1)}% |", f"| Match Top1 Acc | {_pct(summary.get('match_top1_acc'))} |", f"| Priority Top1 Acc | {_pct(summary.get('priority_top1_acc'))} |", f"| Action Acc | {_pct(summary.get('action_acc'))} |", f"| Match Recall@5 | {_pct(summary.get('match_recall_at_5'))} |", f"| Priority Recall@5 | {_pct(summary.get('priority_recall_at_5'))} |", ""] ec = summary.get("error_counts", {}) if ec: lines += ["## Error Taxonomy", "", "| Code | Count | Agent | Severity |", "|---|---|---|---|"] for code, cnt in sorted(ec.items(), key=lambda x:-x[1]): tax = (ERROR_TAXONOMY or {}).get(code, {}) lines.append(f"| {code} | {cnt} | {tax.get('agent','-')} | {tax.get('severity','-')} |") lines.append("") ae = summary.get("agent_error_counts", {}) if ae: lines += ["## Agent-level Errors", "", "| Agent | Count |", "|---|---|"] for a, c in sorted(ae.items(), key=lambda x:-x[1]): lines.append(f"| {a} | {c} |") lines.append("") lines += ["## Per-Case Results", ""] for i, t in enumerate(traces, 1): lines += [f"### Case {i}: {t.case_id}", ""] m = t.agent_metrics lines += [f"- Target: {t.target_role} | City: {t.target_city} | Stage: {t.stage}", f"- Match Top1: {m.get('top1_match_title','-')} | Expected: {m.get('expected_top_matches',[])}", f"- Priority Top1: {m.get('top1_priority_title','-')} | Expected: {m.get('expected_top_priorities',[])}", f"- Action: actual={m.get('actual_action','-')}, expected={m.get('expected_action','-')}", f"- Match Top1 Hit: {m.get('match_top1_hit','-')} | Priority Top1 Hit: {m.get('priority_top1_hit','-')}", f"- Errors({len(t.errors)}): " + (", ".join(e["code"] for e in t.errors) if t.errors else "none")] if t.errors: for e in t.errors: lines.append(f" - `{e['code']}`: {e['message']}") lines.append("") content = "\n".join(lines) Path(output_path).write_text(content, encoding="utf-8") return content def generate_error_analysis(traces, summary, output_path): """Generate error analysis report.""" lines = ["# Error Analysis (v2: Match/Priority split)", "", f"Generated: {time.strftime('%Y-%m-%d %H:%M:%S')}", ""] ec = summary.get("error_counts", {}) lines += ["## Error Ranking", ""] if ec: lines += ["| Rank | Code | Count | Agent |", "|---|---|---|---|"] for rank, (code, cnt) in enumerate(sorted(ec.items(), key=lambda x:-x[1]), 1): tax = (ERROR_TAXONOMY or {}).get(code, {}) lines.append(f"| {rank} | {code} | {cnt} | {tax.get('agent','-')} |") else: lines.append("[OK] No errors.") lines.append("") # Root cause analysis total = summary.get("total_cases", 0) pass_cases = summary.get("pass_cases", 0) match_top1 = summary.get("match_top1_acc") priority_top1 = summary.get("priority_top1_acc") lines += ["## Root Cause: Matcher vs Label", ""] lines.append(f"- Pass rate: {pass_cases}/{total} ({round(summary.get('pass_rate',0)*100,1)}%)") lines.append(f"- Match Top1 Acc: {_pct(match_top1)}") lines.append(f"- Priority Top1 Acc: {_pct(priority_top1)}") lines.append("") # E4 analysis e4_match = ec.get("E4_MATCH_RECALL_MISS", 0) e4_pri = ec.get("E4_PRIORITY_RECALL_MISS", 0) e4 = ec.get("E4_RECALL_MISS", 0) lines += ["## E4 Recall Miss Analysis", ""] if e4 or e4_match or e4_pri: lines.append(f"- E4_MATCH_RECALL_MISS: {e4_match} | E4_PRIORITY_RECALL_MISS: {e4_pri} | E4_RECALL_MISS: {e4}") lines.append("") lines.append("**Possible causes:**") lines.append("1. If E4_MATCH_RECALL_MISS > 0: match_score is too low for expected roles — skill keyword coverage insufficient or expected label not aligned with jobs.json") lines.append("2. If E4_PRIORITY_RECALL_MISS > 0: apply_priority is too low for expected roles — risk_score is pulling down the priority (by design)") lines.append("3. If E4_RECALL_MISS > 0: expected_recall_jobs has jobs not in the top5 of either list — check if the job title exists in jobs.json") lines.append("") else: lines.append("[OK] No recall issues.") lines.append("") # E5 analysis e5_match = ec.get("E5_MATCH_RANK_MISORDER", 0) e5_pri = ec.get("E5_PRIORITY_RANK_MISORDER", 0) lines += ["## E5 Rank Misorder Analysis", ""] if e5_match or e5_pri: lines.append(f"- E5_MATCH_RANK_MISORDER: {e5_match} | E5_PRIORITY_RANK_MISORDER: {e5_pri}") lines.append("") lines.append("**Diagnosis per case:**") for t in traces: errs = [e["code"] for e in t.errors] if "E5_MATCH_RANK_MISORDER" in errs or "E5_PRIORITY_RANK_MISORDER" in errs: m = t.agent_metrics lines.append(f"- **{t.case_id}**: Match Top1={m.get('top1_match_title','-')}, Expected Match={m.get('expected_top_matches',[])}, Priority Top1={m.get('top1_priority_title','-')}, Expected Pri={m.get('expected_top_priorities',[])}") lines.append("") lines.append("**Verdict:**") if e5_match > 0: lines.append("- E5_MATCH errors: matcher's match_score weights may need tuning, OR expected_top_matches label is unreasonable") if e5_pri > 0: lines.append("- E5_PRIORITY errors: apply_priority incorporates risk/pass scores — this is intentional, not a bug. If expected_top_priorities should be about pure match, adjust labels.") lines.append("") else: lines.append("[OK] No rank issues.") lines.append("") # case_02 analysis lines += ["## Case_02 (Recommendation Algorithm) Analysis", ""] case02 = [t for t in traces if t.case_id == "case_02"] if case02: t = case02[0] m = t.agent_metrics errs = [e["code"] for e in t.errors] lines.append(f"- Expected: {m.get('expected_top_matches',[])}") lines.append(f"- Match Top1: {m.get('top1_match_title','-')}") lines.append(f"- Match Top5: {m.get('top5_match',[])}") lines.append(f"- Errors: {errs if errs else 'none'}") if "E5_MATCH_RANK_MISORDER" in errs or "E4_MATCH_RECALL_MISS" in errs: lines.append("- **Diagnosis**: jobs.json has 'LLM 推荐算法实习生', not '推荐算法实习生'. The expectation (pure recommendation) and actual jobs (LLM + recommendation hybrid) may differ in match_score.") lines.append("- **Action**: Either adjust expected_top_matches to 'LLM 推荐算法实习生' (if that's acceptable), or acknowledge that pure rec roles don't exist in current job pool.") else: lines.append("- [OK] No recall/rank issues for case_02 after fixing job title alignment.") lines.append("") # case_03, case_08 analysis lines += ["## Case_03/08 (NLP/CV -> LLM Transfer) Analysis", ""] for cid in ["case_03", "case_08"]: ct = [t for t in traces if t.case_id == cid] if ct: t = ct[0]; m = t.agent_metrics; errs = [e["code"] for e in t.errors] lines.append(f"**{cid}**: Errors={errs if errs else 'none'}, Match Top1={m.get('top1_match_title','-')}, Priority Top1={m.get('top1_priority_title','-')}") lines.append("") lines.append("**Diagnosis:**") lines.append("- case_03 (NLP -> LLM): Has NLP fundamentals but missing RAG/Agent skills. match_score may be reasonable but mismatched roles could rank higher.") lines.append("- case_08 (CV -> LLM): No LLM skills at all. Expected labels are empty (no good match). If system still suggests an LLM role, it means the ranking is over-optimistic for untransferable skill sets.") lines.append("- **Recommendation**: Adjust GrowthScore/RiskScore to reflect transfer difficulty more aggressively for case_08.") lines.append("") # Next steps lines += ["## Next Steps", ""] lines.append("1. If E5_MATCH errors persist: tune SKILL_KEYWORDS or match weight in matcher.py") lines.append("2. If E5_PRIORITY errors persist: review whether expected_top_priorities should be about pure match (then fix labels) or risk-adjusted priority (then matcher is correct)") lines.append("3. Expand jobs.json with more diverse roles (pure rec, pure CV) to test edge cases better") lines.append("4. Add more golden cases for transfer-learning scenarios (X background -> LLM role)") lines.append("") content = "\n".join(lines) Path(output_path).write_text(content, encoding="utf-8") return content def _pct(val): """Format a float as percentage.""" if val is None: return "N/A" return f"{round(val*100,1)}%" # --------------------------------------------------------------------------- # CLI 入口 # --------------------------------------------------------------------------- def main_cli(): """CLI entry point for eval.""" import argparse ap = argparse.ArgumentParser(description="Offer Catcher Eval v2") ap.add_argument("--golden", default="eval/golden_cases.json") ap.add_argument("--jobs", default="data/jobs.json") ap.add_argument("--report", default="reports/eval_report.md") ap.add_argument("--error-analysis", default="reports/error_analysis.md") ap.add_argument("--split", choices=["core", "stress", "all"], default="core") args = ap.parse_args() Path(args.error_analysis).parent.mkdir(parents=True, exist_ok=True) traces, summary = run_all_cases(args.golden, args.jobs, eval_split=args.split) print(f"EVAL_CASES={summary.get('total_cases',0)}") print(f"PASS_CASES={summary.get('pass_cases',0)}") print(f"PASS_RATE={summary.get('pass_rate',0)}") print(f"MATCH_TOP1_ACC={_pct(summary.get('match_top1_acc'))}") print(f"PRIORITY_TOP1_ACC={_pct(summary.get('priority_top1_acc'))}") print(f"ACTION_ACC={_pct(summary.get('action_acc'))}") print(f"MATCH_RECALL_AT_5={_pct(summary.get('match_recall_at_5'))}") print(f"PRIORITY_RECALL_AT_5={_pct(summary.get('priority_recall_at_5'))}") ec = summary.get('error_counts', {}) print(f"ERROR_COUNTS={ec}") generate_eval_report(traces, summary, args.report) print(f"[OK] Report: {args.report}") generate_error_analysis(traces, summary, args.error_analysis) print(f"[OK] Error Analysis: {args.error_analysis}") return traces, summary if __name__ == "__main__": main_cli()