Prospector v0.2.3 release (part 21)

data/runs/q2bpool32-s108--tsmc_core_agentic/summary.json

@@ -0,0 +1,192 @@
+{
+  "model": "vllm:Qwen/Qwen3.5-2B",
+  "benchmark": "data/benchmarks/tsmc/tsmc_core_agentic.jsonl",
+  "agentic": true,
+  "shard": null,
+  "ids_file": null,
+  "adapter": "models/traj_sft_2b_flash20g2/final,models/traj_grpo_2b_flash20d1/checkpoint-100,models/traj_grpo_2b_flash20d2/checkpoint-120",
+  "backend": "tsmc",
+  "kg_fingerprint": "ba22334639149e9d",
+  "kb_path": "None",
+  "source_text": "absent",
+  "chunks_dir": null,
+  "chat_template": "plain_chatml",
+  "max_model_len": 16384,
+  "timing": {
+    "driver": "batched",
+    "sequential": false,
+    "wall_ms_semantics": "since_batch_start",
+    "concurrency": null,
+    "batch_size": null,
+    "n_episodes": 195,
+    "num_samples": 1,
+    "group_size": null,
+    "total_wall_ms": 148340,
+    "cache_hits": null,
+    "api_calls": null
+  },
+  "usage": {
+    "prompt_tokens": null,
+    "completion_tokens": null,
+    "reasoning_tokens": null,
+    "cached_prompt_tokens": null,
+    "service_tier": null,
+    "served_tiers": null,
+    "tier_fallbacks": null,
+    "truncated": null,
+    "refused": null
+  },
+  "policy": {
+    "prompt_version": "agent_v1",
+    "max_turns": 20,
+    "turn_max_tokens": 1024,
+    "final_max_tokens": 900,
+    "max_edges_seen": 400,
+    "max_repeats": 3,
+    "repeat_echo": true,
+    "page_size": 40,
+    "hub_threshold": 150,
+    "hub_summary_limit": 400,
+    "connect_max_hops": 3,
+    "find_degree_tiebreak": false,
+    "tools": null,
+    "temperature": 0.7,
+    "reasoning_effort": null,
+    "seed": 108,
+    "give_anchor": false,
+    "relation_vocab": true,
+    "forced_prefill": false,
+    "user_note": null,
+    "gold_scaffold": false,
+    "citation_repair": false,
+    "offer_revision": false
+  },
+  "aggregate": {
+    "easy": {
+      "n": 70,
+      "format_rate": 1.0,
+      "halluc_strict": 0.011904761904761906,
+      "halluc_soft": 0.0,
+      "path_recall": 0.7928571428571428,
+      "path_precision": 0.6228571428571429,
+      "em_strict": 0.8285714285714286,
+      "em_claim": 0.8285714285714286,
+      "success": 0.8285714285714286,
+      "evidence_recall": 0.9071428571428571,
+      "reached_answer": 0.9142857142857143,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.15714285714285714,
+      "efficiency": 0.03883034285766838,
+      "gate_pass": 0.9714285714285714,
+      "gated_acc": 0.8285714285714286,
+      "gated_abstained": 0.02857142857142857,
+      "n_calls": 12.228571428571428,
+      "n_edges_seen": 107.25714285714285,
+      "n_repeats": 0.0,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.02857142857142857,
+      "median_n_edges_seen": 98.5,
+      "median_edges_to_answer": 21.5,
+      "median_n_calls": 13.0,
+      "reached_rate": 0.9142857142857143,
+      "censored": 0.0
+    },
+    "hard": {
+      "n": 57,
+      "format_rate": 0.9824561403508771,
+      "halluc_strict": 0.0071428571428571435,
+      "halluc_soft": 0.0,
+      "path_recall": 0.37719298245614036,
+      "path_precision": 0.37234977324263036,
+      "em_strict": 0.631578947368421,
+      "em_claim": 0.631578947368421,
+      "success": 0.6140350877192983,
+      "evidence_recall": 0.6842105263157895,
+      "reached_answer": 0.6666666666666666,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.14035087719298245,
+      "efficiency": 0.04676203562190346,
+      "gate_pass": 0.9649122807017544,
+      "gated_acc": 0.6140350877192983,
+      "gated_abstained": 0.03508771929824561,
+      "n_calls": 15.473684210526315,
+      "n_edges_seen": 117.78947368421052,
+      "n_repeats": 0.017543859649122806,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.017543859649122806,
+      "median_n_edges_seen": 112.0,
+      "median_edges_to_answer": 31.5,
+      "median_n_calls": 16.0,
+      "reached_rate": 0.6666666666666666,
+      "censored": 0.17543859649122806
+    },
+    "medium": {
+      "n": 68,
+      "format_rate": 1.0,
+      "halluc_strict": 0.004201680672268907,
+      "halluc_soft": 0.0021008403361344546,
+      "path_recall": 0.6617647058823529,
+      "path_precision": 0.6017156862745098,
+      "em_strict": 0.8382352941176471,
+      "em_claim": 0.8382352941176471,
+      "success": 0.8235294117647058,
+      "evidence_recall": 0.8480392156862745,
+      "reached_answer": 0.9117647058823529,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.10294117647058823,
+      "efficiency": 0.041853396670698415,
+      "gate_pass": 0.9852941176470589,
+      "gated_acc": 0.8235294117647058,
+      "gated_abstained": 0.014705882352941176,
+      "n_calls": 13.397058823529411,
+      "n_edges_seen": 127.0,
+      "n_repeats": 0.0,
+      "n_malformed": 0.014705882352941176,
+      "n_tool_errors": 0.029411764705882353,
+      "median_n_edges_seen": 111.0,
+      "median_edges_to_answer": 21.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.9117647058823529,
+      "censored": 0.08823529411764706
+    },
+    "ALL": {
+      "n": 195,
+      "format_rate": 0.9948717948717949,
+      "halluc_strict": 0.00783014236622484,
+      "halluc_soft": 0.000736377025036819,
+      "path_recall": 0.6256410256410256,
+      "path_precision": 0.5431353297332678,
+      "em_strict": 0.7743589743589744,
+      "em_claim": 0.7743589743589744,
+      "success": 0.764102564102564,
+      "evidence_recall": 0.8213675213675213,
+      "reached_answer": 0.841025641025641,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.13333333333333333,
+      "efficiency": 0.04220303079021937,
+      "gate_pass": 0.9743589743589743,
+      "gated_acc": 0.764102564102564,
+      "gated_abstained": 0.02564102564102564,
+      "n_calls": 13.584615384615384,
+      "n_edges_seen": 117.22051282051282,
+      "n_repeats": 0.005128205128205128,
+      "n_malformed": 0.005128205128205128,
+      "n_tool_errors": 0.02564102564102564,
+      "median_n_edges_seen": 104.0,
+      "median_edges_to_answer": 23.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.841025641025641,
+      "censored": 0.08205128205128205
+    }
+  },
+  "latency": {
+    "dir": "latency",
+    "n": 15,
+    "p25_s": 3.5,
+    "p50_s": 4.1,
+    "p75_s": 4.9,
+    "questions_per_minute": 14.55,
+    "turns_per_episode": 15.0,
+    "s_per_turn": 0.27
+  }
+}

data/runs/q2bpool32-s109--tsmc_core_agentic/summary.json

@@ -0,0 +1,192 @@
+{
+  "model": "vllm:Qwen/Qwen3.5-2B",
+  "benchmark": "data/benchmarks/tsmc/tsmc_core_agentic.jsonl",
+  "agentic": true,
+  "shard": null,
+  "ids_file": null,
+  "adapter": "models/traj_sft_2b_flash20g2/final,models/traj_grpo_2b_flash20d1/checkpoint-100,models/traj_grpo_2b_flash20d2/checkpoint-120",
+  "backend": "tsmc",
+  "kg_fingerprint": "ba22334639149e9d",
+  "kb_path": "None",
+  "source_text": "absent",
+  "chunks_dir": null,
+  "chat_template": "plain_chatml",
+  "max_model_len": 16384,
+  "timing": {
+    "driver": "batched",
+    "sequential": false,
+    "wall_ms_semantics": "since_batch_start",
+    "concurrency": null,
+    "batch_size": null,
+    "n_episodes": 195,
+    "num_samples": 1,
+    "group_size": null,
+    "total_wall_ms": 146119,
+    "cache_hits": null,
+    "api_calls": null
+  },
+  "usage": {
+    "prompt_tokens": null,
+    "completion_tokens": null,
+    "reasoning_tokens": null,
+    "cached_prompt_tokens": null,
+    "service_tier": null,
+    "served_tiers": null,
+    "tier_fallbacks": null,
+    "truncated": null,
+    "refused": null
+  },
+  "policy": {
+    "prompt_version": "agent_v1",
+    "max_turns": 20,
+    "turn_max_tokens": 1024,
+    "final_max_tokens": 900,
+    "max_edges_seen": 400,
+    "max_repeats": 3,
+    "repeat_echo": true,
+    "page_size": 40,
+    "hub_threshold": 150,
+    "hub_summary_limit": 400,
+    "connect_max_hops": 3,
+    "find_degree_tiebreak": false,
+    "tools": null,
+    "temperature": 0.7,
+    "reasoning_effort": null,
+    "seed": 109,
+    "give_anchor": false,
+    "relation_vocab": true,
+    "forced_prefill": false,
+    "user_note": null,
+    "gold_scaffold": false,
+    "citation_repair": false,
+    "offer_revision": false
+  },
+  "aggregate": {
+    "easy": {
+      "n": 70,
+      "format_rate": 1.0,
+      "halluc_strict": 0.004081632653061224,
+      "halluc_soft": 0.0,
+      "path_recall": 0.8,
+      "path_precision": 0.6188095238095238,
+      "em_strict": 0.9142857142857143,
+      "em_claim": 0.9142857142857143,
+      "success": 0.9,
+      "evidence_recall": 0.95,
+      "reached_answer": 0.9714285714285714,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.12857142857142856,
+      "efficiency": 0.03567801917123511,
+      "gate_pass": 0.9857142857142858,
+      "gated_acc": 0.9,
+      "gated_abstained": 0.014285714285714285,
+      "n_calls": 12.157142857142857,
+      "n_edges_seen": 104.68571428571428,
+      "n_repeats": 0.0,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.02857142857142857,
+      "median_n_edges_seen": 96.0,
+      "median_edges_to_answer": 28.5,
+      "median_n_calls": 13.5,
+      "reached_rate": 0.9714285714285714,
+      "censored": 0.014285714285714285
+    },
+    "hard": {
+      "n": 57,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0043859649122807015,
+      "halluc_soft": 0.0,
+      "path_recall": 0.4517543859649123,
+      "path_precision": 0.4288011695906433,
+      "em_strict": 0.6666666666666666,
+      "em_claim": 0.6666666666666666,
+      "success": 0.6491228070175439,
+      "evidence_recall": 0.7149122807017544,
+      "reached_answer": 0.7368421052631579,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.10526315789473684,
+      "efficiency": 0.04681528260663048,
+      "gate_pass": 0.9824561403508771,
+      "gated_acc": 0.6491228070175439,
+      "gated_abstained": 0.017543859649122806,
+      "n_calls": 14.982456140350877,
+      "n_edges_seen": 115.84210526315789,
+      "n_repeats": 0.0,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.0,
+      "median_n_edges_seen": 115.0,
+      "median_edges_to_answer": 37.5,
+      "median_n_calls": 16.0,
+      "reached_rate": 0.7368421052631579,
+      "censored": 0.10526315789473684
+    },
+    "medium": {
+      "n": 68,
+      "format_rate": 1.0,
+      "halluc_strict": 0.015686274509803918,
+      "halluc_soft": 0.0,
+      "path_recall": 0.6323529411764706,
+      "path_precision": 0.5623774509803922,
+      "em_strict": 0.7794117647058824,
+      "em_claim": 0.7794117647058824,
+      "success": 0.7352941176470589,
+      "evidence_recall": 0.8186274509803921,
+      "reached_answer": 0.8529411764705882,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.07352941176470588,
+      "efficiency": 0.03908246736282574,
+      "gate_pass": 0.9264705882352942,
+      "gated_acc": 0.7352941176470589,
+      "gated_abstained": 0.07352941176470588,
+      "n_calls": 13.676470588235293,
+      "n_edges_seen": 129.77941176470588,
+      "n_repeats": 0.0,
+      "n_malformed": 0.014705882352941176,
+      "n_tool_errors": 0.10294117647058823,
+      "median_n_edges_seen": 115.0,
+      "median_edges_to_answer": 25.0,
+      "median_n_calls": 15.0,
+      "reached_rate": 0.8529411764705882,
+      "censored": 0.10294117647058823
+    },
+    "ALL": {
+      "n": 195,
+      "format_rate": 1.0,
+      "halluc_strict": 0.008217338217338216,
+      "halluc_soft": 0.0,
+      "path_recall": 0.6397435897435897,
+      "path_precision": 0.5435897435897435,
+      "em_strict": 0.7948717948717948,
+      "em_claim": 0.7948717948717948,
+      "success": 0.7692307692307693,
+      "evidence_recall": 0.8354700854700854,
+      "reached_answer": 0.8615384615384616,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.10256410256410256,
+      "efficiency": 0.04012071913454639,
+      "gate_pass": 0.9641025641025641,
+      "gated_acc": 0.7692307692307693,
+      "gated_abstained": 0.035897435897435895,
+      "n_calls": 13.512820512820513,
+      "n_edges_seen": 116.6974358974359,
+      "n_repeats": 0.0,
+      "n_malformed": 0.005128205128205128,
+      "n_tool_errors": 0.046153846153846156,
+      "median_n_edges_seen": 108.0,
+      "median_edges_to_answer": 32.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8615384615384616,
+      "censored": 0.07179487179487179
+    }
+  },
+  "latency": {
+    "dir": "latency",
+    "n": 15,
+    "p25_s": 4.1,
+    "p50_s": 4.6,
+    "p75_s": 5.3,
+    "questions_per_minute": 12.94,
+    "turns_per_episode": 15.6,
+    "s_per_turn": 0.3
+  }
+}

data/runs/q2bpool32-s110--tsmc_core_agentic/summary.json

@@ -0,0 +1,192 @@
+{
+  "model": "vllm:Qwen/Qwen3.5-2B",
+  "benchmark": "data/benchmarks/tsmc/tsmc_core_agentic.jsonl",
+  "agentic": true,
+  "shard": null,
+  "ids_file": null,
+  "adapter": "models/traj_sft_2b_flash20g2/final,models/traj_grpo_2b_flash20d1/checkpoint-100,models/traj_grpo_2b_flash20d2/checkpoint-120",
+  "backend": "tsmc",
+  "kg_fingerprint": "ba22334639149e9d",
+  "kb_path": "None",
+  "source_text": "absent",
+  "chunks_dir": null,
+  "chat_template": "plain_chatml",
+  "max_model_len": 16384,
+  "timing": {
+    "driver": "batched",
+    "sequential": false,
+    "wall_ms_semantics": "since_batch_start",
+    "concurrency": null,
+    "batch_size": null,
+    "n_episodes": 195,
+    "num_samples": 1,
+    "group_size": null,
+    "total_wall_ms": 145375,
+    "cache_hits": null,
+    "api_calls": null
+  },
+  "usage": {
+    "prompt_tokens": null,
+    "completion_tokens": null,
+    "reasoning_tokens": null,
+    "cached_prompt_tokens": null,
+    "service_tier": null,
+    "served_tiers": null,
+    "tier_fallbacks": null,
+    "truncated": null,
+    "refused": null
+  },
+  "policy": {
+    "prompt_version": "agent_v1",
+    "max_turns": 20,
+    "turn_max_tokens": 1024,
+    "final_max_tokens": 900,
+    "max_edges_seen": 400,
+    "max_repeats": 3,
+    "repeat_echo": true,
+    "page_size": 40,
+    "hub_threshold": 150,
+    "hub_summary_limit": 400,
+    "connect_max_hops": 3,
+    "find_degree_tiebreak": false,
+    "tools": null,
+    "temperature": 0.7,
+    "reasoning_effort": null,
+    "seed": 110,
+    "give_anchor": false,
+    "relation_vocab": true,
+    "forced_prefill": false,
+    "user_note": null,
+    "gold_scaffold": false,
+    "citation_repair": false,
+    "offer_revision": false
+  },
+  "aggregate": {
+    "easy": {
+      "n": 70,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0,
+      "halluc_soft": 0.0,
+      "path_recall": 0.7714285714285715,
+      "path_precision": 0.6086394557823129,
+      "em_strict": 0.8571428571428571,
+      "em_claim": 0.8571428571428571,
+      "success": 0.8571428571428571,
+      "evidence_recall": 0.8785714285714286,
+      "reached_answer": 0.9,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.15714285714285714,
+      "efficiency": 0.030334458876691624,
+      "gate_pass": 0.9857142857142858,
+      "gated_acc": 0.8571428571428571,
+      "gated_abstained": 0.014285714285714285,
+      "n_calls": 12.314285714285715,
+      "n_edges_seen": 105.65714285714286,
+      "n_repeats": 0.0,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.02857142857142857,
+      "median_n_edges_seen": 101.5,
+      "median_edges_to_answer": 21.5,
+      "median_n_calls": 13.0,
+      "reached_rate": 0.9,
+      "censored": 0.02857142857142857
+    },
+    "hard": {
+      "n": 57,
+      "format_rate": 1.0,
+      "halluc_strict": 0.005847953216374268,
+      "halluc_soft": 0.002923976608187134,
+      "path_recall": 0.40789473684210525,
+      "path_precision": 0.40822890559732666,
+      "em_strict": 0.6491228070175439,
+      "em_claim": 0.6491228070175439,
+      "success": 0.631578947368421,
+      "evidence_recall": 0.7412280701754386,
+      "reached_answer": 0.7368421052631579,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.10526315789473684,
+      "efficiency": 0.045004974761995054,
+      "gate_pass": 0.9649122807017544,
+      "gated_acc": 0.631578947368421,
+      "gated_abstained": 0.03508771929824561,
+      "n_calls": 15.035087719298245,
+      "n_edges_seen": 120.84210526315789,
+      "n_repeats": 0.017543859649122806,
+      "n_malformed": 0.03508771929824561,
+      "n_tool_errors": 0.03508771929824561,
+      "median_n_edges_seen": 109.0,
+      "median_edges_to_answer": 37.5,
+      "median_n_calls": 15.0,
+      "reached_rate": 0.7368421052631579,
+      "censored": 0.15789473684210525
+    },
+    "medium": {
+      "n": 68,
+      "format_rate": 1.0,
+      "halluc_strict": 0.006127450980392156,
+      "halluc_soft": 0.0,
+      "path_recall": 0.6176470588235294,
+      "path_precision": 0.5884803921568628,
+      "em_strict": 0.75,
+      "em_claim": 0.75,
+      "success": 0.7205882352941176,
+      "evidence_recall": 0.8137254901960784,
+      "reached_answer": 0.8235294117647058,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.08823529411764706,
+      "efficiency": 0.04121165509171038,
+      "gate_pass": 0.9558823529411765,
+      "gated_acc": 0.7205882352941176,
+      "gated_abstained": 0.04411764705882353,
+      "n_calls": 13.058823529411764,
+      "n_edges_seen": 118.23529411764706,
+      "n_repeats": 0.014705882352941176,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.014705882352941176,
+      "median_n_edges_seen": 109.5,
+      "median_edges_to_answer": 28.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8235294117647058,
+      "censored": 0.08823529411764706
+    },
+    "ALL": {
+      "n": 195,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0038461538461538455,
+      "halluc_soft": 0.0008547008547008545,
+      "path_recall": 0.6115384615384616,
+      "path_precision": 0.543028083028083,
+      "em_strict": 0.7589743589743589,
+      "em_claim": 0.7589743589743589,
+      "success": 0.7435897435897436,
+      "evidence_recall": 0.8158119658119659,
+      "reached_answer": 0.8256410256410256,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.11794871794871795,
+      "efficiency": 0.03841583707199199,
+      "gate_pass": 0.9692307692307692,
+      "gated_acc": 0.7435897435897436,
+      "gated_abstained": 0.03076923076923077,
+      "n_calls": 13.36923076923077,
+      "n_edges_seen": 114.48205128205129,
+      "n_repeats": 0.010256410256410256,
+      "n_malformed": 0.010256410256410256,
+      "n_tool_errors": 0.02564102564102564,
+      "median_n_edges_seen": 105.0,
+      "median_edges_to_answer": 29.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8256410256410256,
+      "censored": 0.08717948717948718
+    }
+  },
+  "latency": {
+    "dir": "latency",
+    "n": 15,
+    "p25_s": 3.6,
+    "p50_s": 4.8,
+    "p75_s": 5.5,
+    "questions_per_minute": 12.43,
+    "turns_per_episode": 15.6,
+    "s_per_turn": 0.28
+  }
+}

data/runs/q2bpool32-s111--tsmc_core_agentic/summary.json

@@ -0,0 +1,192 @@
+{
+  "model": "vllm:Qwen/Qwen3.5-2B",
+  "benchmark": "data/benchmarks/tsmc/tsmc_core_agentic.jsonl",
+  "agentic": true,
+  "shard": null,
+  "ids_file": null,
+  "adapter": "models/traj_sft_2b_flash20g2/final,models/traj_grpo_2b_flash20d1/checkpoint-100,models/traj_grpo_2b_flash20d2/checkpoint-120",
+  "backend": "tsmc",
+  "kg_fingerprint": "ba22334639149e9d",
+  "kb_path": "None",
+  "source_text": "absent",
+  "chunks_dir": null,
+  "chat_template": "plain_chatml",
+  "max_model_len": 16384,
+  "timing": {
+    "driver": "batched",
+    "sequential": false,
+    "wall_ms_semantics": "since_batch_start",
+    "concurrency": null,
+    "batch_size": null,
+    "n_episodes": 195,
+    "num_samples": 1,
+    "group_size": null,
+    "total_wall_ms": 143260,
+    "cache_hits": null,
+    "api_calls": null
+  },
+  "usage": {
+    "prompt_tokens": null,
+    "completion_tokens": null,
+    "reasoning_tokens": null,
+    "cached_prompt_tokens": null,
+    "service_tier": null,
+    "served_tiers": null,
+    "tier_fallbacks": null,
+    "truncated": null,
+    "refused": null
+  },
+  "policy": {
+    "prompt_version": "agent_v1",
+    "max_turns": 20,
+    "turn_max_tokens": 1024,
+    "final_max_tokens": 900,
+    "max_edges_seen": 400,
+    "max_repeats": 3,
+    "repeat_echo": true,
+    "page_size": 40,
+    "hub_threshold": 150,
+    "hub_summary_limit": 400,
+    "connect_max_hops": 3,
+    "find_degree_tiebreak": false,
+    "tools": null,
+    "temperature": 0.7,
+    "reasoning_effort": null,
+    "seed": 111,
+    "give_anchor": false,
+    "relation_vocab": true,
+    "forced_prefill": false,
+    "user_note": null,
+    "gold_scaffold": false,
+    "citation_repair": false,
+    "offer_revision": false
+  },
+  "aggregate": {
+    "easy": {
+      "n": 70,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0038265306122448987,
+      "halluc_soft": 0.0017857142857142857,
+      "path_recall": 0.7428571428571429,
+      "path_precision": 0.5896428571428571,
+      "em_strict": 0.8285714285714286,
+      "em_claim": 0.8285714285714286,
+      "success": 0.8142857142857143,
+      "evidence_recall": 0.8571428571428571,
+      "reached_answer": 0.8857142857142857,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.14285714285714285,
+      "efficiency": 0.03646429100931058,
+      "gate_pass": 0.9714285714285714,
+      "gated_acc": 0.8142857142857143,
+      "gated_abstained": 0.02857142857142857,
+      "n_calls": 12.285714285714286,
+      "n_edges_seen": 104.71428571428571,
+      "n_repeats": 0.014285714285714285,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.014285714285714285,
+      "median_n_edges_seen": 97.5,
+      "median_edges_to_answer": 20.0,
+      "median_n_calls": 13.5,
+      "reached_rate": 0.8857142857142857,
+      "censored": 0.05714285714285714
+    },
+    "hard": {
+      "n": 57,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0058479532163742695,
+      "halluc_soft": 0.0,
+      "path_recall": 0.4298245614035088,
+      "path_precision": 0.4695071010860485,
+      "em_strict": 0.6491228070175439,
+      "em_claim": 0.6491228070175439,
+      "success": 0.631578947368421,
+      "evidence_recall": 0.7587719298245614,
+      "reached_answer": 0.7719298245614035,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.12280701754385964,
+      "efficiency": 0.04408509872960013,
+      "gate_pass": 0.9824561403508771,
+      "gated_acc": 0.631578947368421,
+      "gated_abstained": 0.017543859649122806,
+      "n_calls": 14.631578947368421,
+      "n_edges_seen": 121.87719298245614,
+      "n_repeats": 0.0,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.03508771929824561,
+      "median_n_edges_seen": 120.0,
+      "median_edges_to_answer": 40.0,
+      "median_n_calls": 15.0,
+      "reached_rate": 0.7719298245614035,
+      "censored": 0.05263157894736842
+    },
+    "medium": {
+      "n": 68,
+      "format_rate": 1.0,
+      "halluc_strict": 0.015196078431372547,
+      "halluc_soft": 0.004901960784313726,
+      "path_recall": 0.6225490196078431,
+      "path_precision": 0.5625,
+      "em_strict": 0.7352941176470589,
+      "em_claim": 0.7352941176470589,
+      "success": 0.7058823529411765,
+      "evidence_recall": 0.7941176470588235,
+      "reached_answer": 0.8235294117647058,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.08823529411764706,
+      "efficiency": 0.04428700255278451,
+      "gate_pass": 0.9411764705882353,
+      "gated_acc": 0.7058823529411765,
+      "gated_abstained": 0.058823529411764705,
+      "n_calls": 13.544117647058824,
+      "n_edges_seen": 124.05882352941177,
+      "n_repeats": 0.029411764705882353,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.029411764705882353,
+      "median_n_edges_seen": 114.5,
+      "median_edges_to_answer": 28.0,
+      "median_n_calls": 14.5,
+      "reached_rate": 0.8235294117647058,
+      "censored": 0.10294117647058823
+    },
+    "ALL": {
+      "n": 195,
+      "format_rate": 1.0,
+      "halluc_strict": 0.008382173382173381,
+      "halluc_soft": 0.0023504273504273508,
+      "path_recall": 0.6094017094017093,
+      "path_precision": 0.5450610500610501,
+      "em_strict": 0.7435897435897436,
+      "em_claim": 0.7435897435897436,
+      "success": 0.7230769230769231,
+      "evidence_recall": 0.8064102564102564,
+      "reached_answer": 0.8307692307692308,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.11794871794871795,
+      "efficiency": 0.041419831650401515,
+      "gate_pass": 0.9641025641025641,
+      "gated_acc": 0.7230769230769231,
+      "gated_abstained": 0.035897435897435895,
+      "n_calls": 13.41025641025641,
+      "n_edges_seen": 116.47692307692307,
+      "n_repeats": 0.015384615384615385,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.02564102564102564,
+      "median_n_edges_seen": 112.0,
+      "median_edges_to_answer": 29.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8307692307692308,
+      "censored": 0.07179487179487179
+    }
+  },
+  "latency": {
+    "dir": "latency",
+    "n": 15,
+    "p25_s": 3.4,
+    "p50_s": 4.3,
+    "p75_s": 5.0,
+    "questions_per_minute": 14.1,
+    "turns_per_episode": 15.1,
+    "s_per_turn": 0.29
+  }
+}

data/runs/q2bpool32-s112--tsmc_core_agentic/summary.json

@@ -0,0 +1,192 @@
+{
+  "model": "vllm:Qwen/Qwen3.5-2B",
+  "benchmark": "data/benchmarks/tsmc/tsmc_core_agentic.jsonl",
+  "agentic": true,
+  "shard": null,
+  "ids_file": null,
+  "adapter": "models/traj_sft_2b_flash20g2/final,models/traj_grpo_2b_flash20d1/checkpoint-100,models/traj_grpo_2b_flash20d2/checkpoint-120",
+  "backend": "tsmc",
+  "kg_fingerprint": "ba22334639149e9d",
+  "kb_path": "None",
+  "source_text": "absent",
+  "chunks_dir": null,
+  "chat_template": "plain_chatml",
+  "max_model_len": 16384,
+  "timing": {
+    "driver": "batched",
+    "sequential": false,
+    "wall_ms_semantics": "since_batch_start",
+    "concurrency": null,
+    "batch_size": null,
+    "n_episodes": 195,
+    "num_samples": 1,
+    "group_size": null,
+    "total_wall_ms": 149817,
+    "cache_hits": null,
+    "api_calls": null
+  },
+  "usage": {
+    "prompt_tokens": null,
+    "completion_tokens": null,
+    "reasoning_tokens": null,
+    "cached_prompt_tokens": null,
+    "service_tier": null,
+    "served_tiers": null,
+    "tier_fallbacks": null,
+    "truncated": null,
+    "refused": null
+  },
+  "policy": {
+    "prompt_version": "agent_v1",
+    "max_turns": 20,
+    "turn_max_tokens": 1024,
+    "final_max_tokens": 900,
+    "max_edges_seen": 400,
+    "max_repeats": 3,
+    "repeat_echo": true,
+    "page_size": 40,
+    "hub_threshold": 150,
+    "hub_summary_limit": 400,
+    "connect_max_hops": 3,
+    "find_degree_tiebreak": false,
+    "tools": null,
+    "temperature": 0.7,
+    "reasoning_effort": null,
+    "seed": 112,
+    "give_anchor": false,
+    "relation_vocab": true,
+    "forced_prefill": false,
+    "user_note": null,
+    "gold_scaffold": false,
+    "citation_repair": false,
+    "offer_revision": false
+  },
+  "aggregate": {
+    "easy": {
+      "n": 70,
+      "format_rate": 1.0,
+      "halluc_strict": 0.01,
+      "halluc_soft": 0.007142857142857143,
+      "path_recall": 0.8214285714285714,
+      "path_precision": 0.6192857142857143,
+      "em_strict": 0.8285714285714286,
+      "em_claim": 0.8285714285714286,
+      "success": 0.8142857142857143,
+      "evidence_recall": 0.9071428571428571,
+      "reached_answer": 0.9285714285714286,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.11428571428571428,
+      "efficiency": 0.037559916026703985,
+      "gate_pass": 0.9571428571428572,
+      "gated_acc": 0.8142857142857143,
+      "gated_abstained": 0.04285714285714286,
+      "n_calls": 12.457142857142857,
+      "n_edges_seen": 107.3,
+      "n_repeats": 0.014285714285714285,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.0,
+      "median_n_edges_seen": 113.0,
+      "median_edges_to_answer": 23.0,
+      "median_n_calls": 13.0,
+      "reached_rate": 0.9285714285714286,
+      "censored": 0.07142857142857142
+    },
+    "hard": {
+      "n": 57,
+      "format_rate": 1.0,
+      "halluc_strict": 0.012740183792815373,
+      "halluc_soft": 0.0,
+      "path_recall": 0.5175438596491229,
+      "path_precision": 0.49490392648287385,
+      "em_strict": 0.7192982456140351,
+      "em_claim": 0.7192982456140351,
+      "success": 0.6666666666666666,
+      "evidence_recall": 0.7631578947368421,
+      "reached_answer": 0.8070175438596491,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.05263157894736842,
+      "efficiency": 0.039974207485221,
+      "gate_pass": 0.9473684210526315,
+      "gated_acc": 0.6666666666666666,
+      "gated_abstained": 0.05263157894736842,
+      "n_calls": 14.964912280701755,
+      "n_edges_seen": 134.7719298245614,
+      "n_repeats": 0.03508771929824561,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.017543859649122806,
+      "median_n_edges_seen": 135.0,
+      "median_edges_to_answer": 38.0,
+      "median_n_calls": 16.0,
+      "reached_rate": 0.8070175438596491,
+      "censored": 0.17543859649122806
+    },
+    "medium": {
+      "n": 68,
+      "format_rate": 1.0,
+      "halluc_strict": 0.009558823529411765,
+      "halluc_soft": 0.003676470588235294,
+      "path_recall": 0.642156862745098,
+      "path_precision": 0.5727240896358543,
+      "em_strict": 0.7205882352941176,
+      "em_claim": 0.7205882352941176,
+      "success": 0.6911764705882353,
+      "evidence_recall": 0.8039215686274509,
+      "reached_answer": 0.8529411764705882,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.10294117647058823,
+      "efficiency": 0.03941118897497814,
+      "gate_pass": 0.9705882352941176,
+      "gated_acc": 0.6911764705882353,
+      "gated_abstained": 0.029411764705882353,
+      "n_calls": 13.470588235294118,
+      "n_edges_seen": 122.0,
+      "n_repeats": 0.014705882352941176,
+      "n_malformed": 0.014705882352941176,
+      "n_tool_errors": 0.0,
+      "median_n_edges_seen": 116.0,
+      "median_edges_to_answer": 29.5,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8529411764705882,
+      "censored": 0.04411764705882353
+    },
+    "ALL": {
+      "n": 195,
+      "format_rate": 1.0,
+      "halluc_strict": 0.010647130647130649,
+      "halluc_soft": 0.0038461538461538464,
+      "path_recall": 0.67008547008547,
+      "path_precision": 0.5666910866910867,
+      "em_strict": 0.7589743589743589,
+      "em_claim": 0.7589743589743589,
+      "success": 0.7282051282051282,
+      "evidence_recall": 0.829059829059829,
+      "reached_answer": 0.8666666666666667,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.09230769230769231,
+      "efficiency": 0.03891120409654046,
+      "gate_pass": 0.958974358974359,
+      "gated_acc": 0.7282051282051282,
+      "gated_abstained": 0.041025641025641026,
+      "n_calls": 13.543589743589743,
+      "n_edges_seen": 120.45641025641025,
+      "n_repeats": 0.020512820512820513,
+      "n_malformed": 0.005128205128205128,
+      "n_tool_errors": 0.005128205128205128,
+      "median_n_edges_seen": 116.0,
+      "median_edges_to_answer": 32.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8666666666666667,
+      "censored": 0.09230769230769231
+    }
+  },
+  "latency": {
+    "dir": "latency",
+    "n": 15,
+    "p25_s": 3.7,
+    "p50_s": 4.5,
+    "p75_s": 5.3,
+    "questions_per_minute": 13.2,
+    "turns_per_episode": 15.3,
+    "s_per_turn": 0.3
+  }
+}

data/runs/q2bpool32-s113--tsmc_core_agentic/summary.json

@@ -0,0 +1,192 @@
+{
+  "model": "vllm:Qwen/Qwen3.5-2B",
+  "benchmark": "data/benchmarks/tsmc/tsmc_core_agentic.jsonl",
+  "agentic": true,
+  "shard": null,
+  "ids_file": null,
+  "adapter": "models/traj_sft_2b_flash20g2/final,models/traj_grpo_2b_flash20d1/checkpoint-100,models/traj_grpo_2b_flash20d2/checkpoint-120",
+  "backend": "tsmc",
+  "kg_fingerprint": "ba22334639149e9d",
+  "kb_path": "None",
+  "source_text": "absent",
+  "chunks_dir": null,
+  "chat_template": "plain_chatml",
+  "max_model_len": 16384,
+  "timing": {
+    "driver": "batched",
+    "sequential": false,
+    "wall_ms_semantics": "since_batch_start",
+    "concurrency": null,
+    "batch_size": null,
+    "n_episodes": 195,
+    "num_samples": 1,
+    "group_size": null,
+    "total_wall_ms": 146100,
+    "cache_hits": null,
+    "api_calls": null
+  },
+  "usage": {
+    "prompt_tokens": null,
+    "completion_tokens": null,
+    "reasoning_tokens": null,
+    "cached_prompt_tokens": null,
+    "service_tier": null,
+    "served_tiers": null,
+    "tier_fallbacks": null,
+    "truncated": null,
+    "refused": null
+  },
+  "policy": {
+    "prompt_version": "agent_v1",
+    "max_turns": 20,
+    "turn_max_tokens": 1024,
+    "final_max_tokens": 900,
+    "max_edges_seen": 400,
+    "max_repeats": 3,
+    "repeat_echo": true,
+    "page_size": 40,
+    "hub_threshold": 150,
+    "hub_summary_limit": 400,
+    "connect_max_hops": 3,
+    "find_degree_tiebreak": false,
+    "tools": null,
+    "temperature": 0.7,
+    "reasoning_effort": null,
+    "seed": 113,
+    "give_anchor": false,
+    "relation_vocab": true,
+    "forced_prefill": false,
+    "user_note": null,
+    "gold_scaffold": false,
+    "citation_repair": false,
+    "offer_revision": false
+  },
+  "aggregate": {
+    "easy": {
+      "n": 70,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0,
+      "halluc_soft": 0.0,
+      "path_recall": 0.7928571428571428,
+      "path_precision": 0.6198809523809524,
+      "em_strict": 0.8571428571428571,
+      "em_claim": 0.8571428571428571,
+      "success": 0.8571428571428571,
+      "evidence_recall": 0.8928571428571429,
+      "reached_answer": 0.9571428571428572,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.15714285714285714,
+      "efficiency": 0.032269701594644234,
+      "gate_pass": 1.0,
+      "gated_acc": 0.8571428571428571,
+      "gated_abstained": 0.0,
+      "n_calls": 12.32857142857143,
+      "n_edges_seen": 112.14285714285714,
+      "n_repeats": 0.02857142857142857,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.014285714285714285,
+      "median_n_edges_seen": 107.0,
+      "median_edges_to_answer": 25.0,
+      "median_n_calls": 13.0,
+      "reached_rate": 0.9571428571428572,
+      "censored": 0.014285714285714285
+    },
+    "hard": {
+      "n": 57,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0064327485380116945,
+      "halluc_soft": 0.0064327485380116945,
+      "path_recall": 0.4166666666666667,
+      "path_precision": 0.42857142857142855,
+      "em_strict": 0.6140350877192983,
+      "em_claim": 0.6140350877192983,
+      "success": 0.5964912280701754,
+      "evidence_recall": 0.7105263157894737,
+      "reached_answer": 0.7368421052631579,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.10526315789473684,
+      "efficiency": 0.04624964714708834,
+      "gate_pass": 0.9649122807017544,
+      "gated_acc": 0.5964912280701754,
+      "gated_abstained": 0.03508771929824561,
+      "n_calls": 15.333333333333334,
+      "n_edges_seen": 120.14035087719299,
+      "n_repeats": 0.017543859649122806,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.08771929824561403,
+      "median_n_edges_seen": 117.0,
+      "median_edges_to_answer": 28.0,
+      "median_n_calls": 16.0,
+      "reached_rate": 0.7368421052631579,
+      "censored": 0.12280701754385964
+    },
+    "medium": {
+      "n": 68,
+      "format_rate": 1.0,
+      "halluc_strict": 0.012867647058823529,
+      "halluc_soft": 0.0,
+      "path_recall": 0.6372549019607844,
+      "path_precision": 0.6051470588235294,
+      "em_strict": 0.7941176470588235,
+      "em_claim": 0.7941176470588235,
+      "success": 0.75,
+      "evidence_recall": 0.8480392156862745,
+      "reached_answer": 0.8676470588235294,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.07352941176470588,
+      "efficiency": 0.047614599319693925,
+      "gate_pass": 0.9411764705882353,
+      "gated_acc": 0.75,
+      "gated_abstained": 0.058823529411764705,
+      "n_calls": 13.455882352941176,
+      "n_edges_seen": 121.20588235294117,
+      "n_repeats": 0.014705882352941176,
+      "n_malformed": 0.014705882352941176,
+      "n_tool_errors": 0.014705882352941176,
+      "median_n_edges_seen": 111.0,
+      "median_edges_to_answer": 25.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8676470588235294,
+      "censored": 0.07352941176470588
+    },
+    "ALL": {
+      "n": 195,
+      "format_rate": 1.0,
+      "halluc_strict": 0.006367521367521368,
+      "halluc_soft": 0.00188034188034188,
+      "path_recall": 0.6286324786324786,
+      "path_precision": 0.5588217338217338,
+      "em_strict": 0.764102564102564,
+      "em_claim": 0.764102564102564,
+      "success": 0.7435897435897436,
+      "evidence_recall": 0.8239316239316239,
+      "reached_answer": 0.8615384615384616,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.11282051282051282,
+      "efficiency": 0.04170718847563241,
+      "gate_pass": 0.9692307692307692,
+      "gated_acc": 0.7435897435897436,
+      "gated_abstained": 0.03076923076923077,
+      "n_calls": 13.6,
+      "n_edges_seen": 117.64102564102564,
+      "n_repeats": 0.020512820512820513,
+      "n_malformed": 0.005128205128205128,
+      "n_tool_errors": 0.035897435897435895,
+      "median_n_edges_seen": 112.0,
+      "median_edges_to_answer": 27.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8615384615384616,
+      "censored": 0.06666666666666667
+    }
+  },
+  "latency": {
+    "dir": "latency",
+    "n": 15,
+    "p25_s": 3.5,
+    "p50_s": 3.9,
+    "p75_s": 5.1,
+    "questions_per_minute": 15.41,
+    "turns_per_episode": 14.8,
+    "s_per_turn": 0.28
+  }
+}

data/runs/q2bpool32-s114--tsmc_core_agentic/summary.json

@@ -0,0 +1,192 @@
+{
+  "model": "vllm:Qwen/Qwen3.5-2B",
+  "benchmark": "data/benchmarks/tsmc/tsmc_core_agentic.jsonl",
+  "agentic": true,
+  "shard": null,
+  "ids_file": null,
+  "adapter": "models/traj_sft_2b_flash20g2/final,models/traj_grpo_2b_flash20d1/checkpoint-100,models/traj_grpo_2b_flash20d2/checkpoint-120",
+  "backend": "tsmc",
+  "kg_fingerprint": "ba22334639149e9d",
+  "kb_path": "None",
+  "source_text": "absent",
+  "chunks_dir": null,
+  "chat_template": "plain_chatml",
+  "max_model_len": 16384,
+  "timing": {
+    "driver": "batched",
+    "sequential": false,
+    "wall_ms_semantics": "since_batch_start",
+    "concurrency": null,
+    "batch_size": null,
+    "n_episodes": 195,
+    "num_samples": 1,
+    "group_size": null,
+    "total_wall_ms": 146284,
+    "cache_hits": null,
+    "api_calls": null
+  },
+  "usage": {
+    "prompt_tokens": null,
+    "completion_tokens": null,
+    "reasoning_tokens": null,
+    "cached_prompt_tokens": null,
+    "service_tier": null,
+    "served_tiers": null,
+    "tier_fallbacks": null,
+    "truncated": null,
+    "refused": null
+  },
+  "policy": {
+    "prompt_version": "agent_v1",
+    "max_turns": 20,
+    "turn_max_tokens": 1024,
+    "final_max_tokens": 900,
+    "max_edges_seen": 400,
+    "max_repeats": 3,
+    "repeat_echo": true,
+    "page_size": 40,
+    "hub_threshold": 150,
+    "hub_summary_limit": 400,
+    "connect_max_hops": 3,
+    "find_degree_tiebreak": false,
+    "tools": null,
+    "temperature": 0.7,
+    "reasoning_effort": null,
+    "seed": 114,
+    "give_anchor": false,
+    "relation_vocab": true,
+    "forced_prefill": false,
+    "user_note": null,
+    "gold_scaffold": false,
+    "citation_repair": false,
+    "offer_revision": false
+  },
+  "aggregate": {
+    "easy": {
+      "n": 70,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0,
+      "halluc_soft": 0.0,
+      "path_recall": 0.8,
+      "path_precision": 0.6223809523809524,
+      "em_strict": 0.8714285714285714,
+      "em_claim": 0.8714285714285714,
+      "success": 0.8714285714285714,
+      "evidence_recall": 0.8928571428571429,
+      "reached_answer": 0.9142857142857143,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.1,
+      "efficiency": 0.03134302176995173,
+      "gate_pass": 0.9857142857142858,
+      "gated_acc": 0.8714285714285714,
+      "gated_abstained": 0.014285714285714285,
+      "n_calls": 12.757142857142858,
+      "n_edges_seen": 111.25714285714285,
+      "n_repeats": 0.02857142857142857,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.04285714285714286,
+      "median_n_edges_seen": 112.0,
+      "median_edges_to_answer": 21.5,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.9142857142857143,
+      "censored": 0.04285714285714286
+    },
+    "hard": {
+      "n": 57,
+      "format_rate": 1.0,
+      "halluc_strict": 0.013157894736842105,
+      "halluc_soft": 0.008771929824561403,
+      "path_recall": 0.41228070175438597,
+      "path_precision": 0.42534113060428846,
+      "em_strict": 0.6140350877192983,
+      "em_claim": 0.6140350877192983,
+      "success": 0.5964912280701754,
+      "evidence_recall": 0.6973684210526315,
+      "reached_answer": 0.7017543859649122,
+      "abstained": 0.017543859649122806,
+      "answer_named_before_seen": 0.10526315789473684,
+      "efficiency": 0.044965610801762,
+      "gate_pass": 0.9473684210526315,
+      "gated_acc": 0.5964912280701754,
+      "gated_abstained": 0.05263157894736842,
+      "n_calls": 15.368421052631579,
+      "n_edges_seen": 118.21052631578948,
+      "n_repeats": 0.017543859649122806,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.05263157894736842,
+      "median_n_edges_seen": 118.0,
+      "median_edges_to_answer": 30.0,
+      "median_n_calls": 16.0,
+      "reached_rate": 0.7017543859649122,
+      "censored": 0.15789473684210525
+    },
+    "medium": {
+      "n": 68,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0024509803921568623,
+      "halluc_soft": 0.0,
+      "path_recall": 0.6715686274509803,
+      "path_precision": 0.6508578431372549,
+      "em_strict": 0.7941176470588235,
+      "em_claim": 0.7941176470588235,
+      "success": 0.7941176470588235,
+      "evidence_recall": 0.8333333333333333,
+      "reached_answer": 0.8823529411764706,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.07352941176470588,
+      "efficiency": 0.042866355812142064,
+      "gate_pass": 0.9852941176470589,
+      "gated_acc": 0.7941176470588235,
+      "gated_abstained": 0.014705882352941176,
+      "n_calls": 12.897058823529411,
+      "n_edges_seen": 120.88235294117646,
+      "n_repeats": 0.014705882352941176,
+      "n_malformed": 0.029411764705882353,
+      "n_tool_errors": 0.0,
+      "median_n_edges_seen": 125.5,
+      "median_edges_to_answer": 32.5,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8823529411764706,
+      "censored": 0.058823529411764705
+    },
+    "ALL": {
+      "n": 195,
+      "format_rate": 1.0,
+      "halluc_strict": 0.004700854700854701,
+      "halluc_soft": 0.002564102564102564,
+      "path_recall": 0.6418803418803419,
+      "path_precision": 0.5747150997150997,
+      "em_strict": 0.7692307692307693,
+      "em_claim": 0.7692307692307693,
+      "success": 0.764102564102564,
+      "evidence_recall": 0.814957264957265,
+      "reached_answer": 0.841025641025641,
+      "abstained": 0.005128205128205128,
+      "answer_named_before_seen": 0.09230769230769231,
+      "efficiency": 0.03934340274268059,
+      "gate_pass": 0.9743589743589743,
+      "gated_acc": 0.764102564102564,
+      "gated_abstained": 0.02564102564102564,
+      "n_calls": 13.569230769230769,
+      "n_edges_seen": 116.64615384615385,
+      "n_repeats": 0.020512820512820513,
+      "n_malformed": 0.010256410256410256,
+      "n_tool_errors": 0.03076923076923077,
+      "median_n_edges_seen": 116.0,
+      "median_edges_to_answer": 24.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.841025641025641,
+      "censored": 0.08205128205128205
+    }
+  },
+  "latency": {
+    "dir": "latency",
+    "n": 15,
+    "p25_s": 3.7,
+    "p50_s": 4.8,
+    "p75_s": 5.4,
+    "questions_per_minute": 12.61,
+    "turns_per_episode": 15.6,
+    "s_per_turn": 0.27
+  }
+}

data/runs/q2bpool32-s115--tsmc_core_agentic/summary.json

@@ -0,0 +1,192 @@
+{
+  "model": "vllm:Qwen/Qwen3.5-2B",
+  "benchmark": "data/benchmarks/tsmc/tsmc_core_agentic.jsonl",
+  "agentic": true,
+  "shard": null,
+  "ids_file": null,
+  "adapter": "models/traj_sft_2b_flash20g2/final,models/traj_grpo_2b_flash20d1/checkpoint-100,models/traj_grpo_2b_flash20d2/checkpoint-120",
+  "backend": "tsmc",
+  "kg_fingerprint": "ba22334639149e9d",
+  "kb_path": "None",
+  "source_text": "absent",
+  "chunks_dir": null,
+  "chat_template": "plain_chatml",
+  "max_model_len": 16384,
+  "timing": {
+    "driver": "batched",
+    "sequential": false,
+    "wall_ms_semantics": "since_batch_start",
+    "concurrency": null,
+    "batch_size": null,
+    "n_episodes": 195,
+    "num_samples": 1,
+    "group_size": null,
+    "total_wall_ms": 149482,
+    "cache_hits": null,
+    "api_calls": null
+  },
+  "usage": {
+    "prompt_tokens": null,
+    "completion_tokens": null,
+    "reasoning_tokens": null,
+    "cached_prompt_tokens": null,
+    "service_tier": null,
+    "served_tiers": null,
+    "tier_fallbacks": null,
+    "truncated": null,
+    "refused": null
+  },
+  "policy": {
+    "prompt_version": "agent_v1",
+    "max_turns": 20,
+    "turn_max_tokens": 1024,
+    "final_max_tokens": 900,
+    "max_edges_seen": 400,
+    "max_repeats": 3,
+    "repeat_echo": true,
+    "page_size": 40,
+    "hub_threshold": 150,
+    "hub_summary_limit": 400,
+    "connect_max_hops": 3,
+    "find_degree_tiebreak": false,
+    "tools": null,
+    "temperature": 0.7,
+    "reasoning_effort": null,
+    "seed": 115,
+    "give_anchor": false,
+    "relation_vocab": true,
+    "forced_prefill": false,
+    "user_note": null,
+    "gold_scaffold": false,
+    "citation_repair": false,
+    "offer_revision": false
+  },
+  "aggregate": {
+    "easy": {
+      "n": 70,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0,
+      "halluc_soft": 0.0,
+      "path_recall": 0.8214285714285714,
+      "path_precision": 0.6457086167800453,
+      "em_strict": 0.8714285714285714,
+      "em_claim": 0.8714285714285714,
+      "success": 0.8714285714285714,
+      "evidence_recall": 0.9071428571428571,
+      "reached_answer": 0.9142857142857143,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.11428571428571428,
+      "efficiency": 0.03556590366628406,
+      "gate_pass": 0.9857142857142858,
+      "gated_acc": 0.8714285714285714,
+      "gated_abstained": 0.014285714285714285,
+      "n_calls": 12.014285714285714,
+      "n_edges_seen": 106.27142857142857,
+      "n_repeats": 0.0,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.0,
+      "median_n_edges_seen": 98.5,
+      "median_edges_to_answer": 33.0,
+      "median_n_calls": 13.0,
+      "reached_rate": 0.9142857142857143,
+      "censored": 0.04285714285714286
+    },
+    "hard": {
+      "n": 57,
+      "format_rate": 1.0,
+      "halluc_strict": 0.012280701754385965,
+      "halluc_soft": 0.0035087719298245606,
+      "path_recall": 0.40789473684210525,
+      "path_precision": 0.4203425229741019,
+      "em_strict": 0.631578947368421,
+      "em_claim": 0.631578947368421,
+      "success": 0.5964912280701754,
+      "evidence_recall": 0.7105263157894737,
+      "reached_answer": 0.7894736842105263,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.10526315789473684,
+      "efficiency": 0.044506451230286445,
+      "gate_pass": 0.9473684210526315,
+      "gated_acc": 0.5964912280701754,
+      "gated_abstained": 0.05263157894736842,
+      "n_calls": 15.385964912280702,
+      "n_edges_seen": 118.54385964912281,
+      "n_repeats": 0.017543859649122806,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.0,
+      "median_n_edges_seen": 105.0,
+      "median_edges_to_answer": 31.0,
+      "median_n_calls": 16.0,
+      "reached_rate": 0.7894736842105263,
+      "censored": 0.14035087719298245
+    },
+    "medium": {
+      "n": 68,
+      "format_rate": 1.0,
+      "halluc_strict": 0.020016339869281047,
+      "halluc_soft": 0.01388888888888889,
+      "path_recall": 0.6225490196078431,
+      "path_precision": 0.5445261437908497,
+      "em_strict": 0.7352941176470589,
+      "em_claim": 0.7352941176470589,
+      "success": 0.6911764705882353,
+      "evidence_recall": 0.8137254901960784,
+      "reached_answer": 0.8382352941176471,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.07352941176470588,
+      "efficiency": 0.047119302254997435,
+      "gate_pass": 0.9117647058823529,
+      "gated_acc": 0.6911764705882353,
+      "gated_abstained": 0.08823529411764706,
+      "n_calls": 13.544117647058824,
+      "n_edges_seen": 117.29411764705883,
+      "n_repeats": 0.014705882352941176,
+      "n_malformed": 0.029411764705882353,
+      "n_tool_errors": 0.029411764705882353,
+      "median_n_edges_seen": 117.0,
+      "median_edges_to_answer": 31.0,
+      "median_n_calls": 15.0,
+      "reached_rate": 0.8382352941176471,
+      "censored": 0.14705882352941177
+    },
+    "ALL": {
+      "n": 195,
+      "format_rate": 1.0,
+      "halluc_strict": 0.010569800569800569,
+      "halluc_soft": 0.005868945868945869,
+      "path_recall": 0.6311965811965812,
+      "path_precision": 0.5445482295482296,
+      "em_strict": 0.7538461538461538,
+      "em_claim": 0.7538461538461538,
+      "success": 0.7282051282051282,
+      "evidence_recall": 0.8170940170940172,
+      "reached_answer": 0.8512820512820513,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.09743589743589744,
+      "efficiency": 0.04220817194926173,
+      "gate_pass": 0.9487179487179487,
+      "gated_acc": 0.7282051282051282,
+      "gated_abstained": 0.05128205128205128,
+      "n_calls": 13.533333333333333,
+      "n_edges_seen": 113.7025641025641,
+      "n_repeats": 0.010256410256410256,
+      "n_malformed": 0.010256410256410256,
+      "n_tool_errors": 0.010256410256410256,
+      "median_n_edges_seen": 108.0,
+      "median_edges_to_answer": 31.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8512820512820513,
+      "censored": 0.1076923076923077
+    }
+  },
+  "latency": {
+    "dir": "latency",
+    "n": 15,
+    "p25_s": 3.3,
+    "p50_s": 4.5,
+    "p75_s": 5.2,
+    "questions_per_minute": 13.46,
+    "turns_per_episode": 15.4,
+    "s_per_turn": 0.29
+  }
+}

data/runs/q2bpool32-s116--tsmc_core_agentic/summary.json

@@ -0,0 +1,192 @@
+{
+  "model": "vllm:Qwen/Qwen3.5-2B",
+  "benchmark": "data/benchmarks/tsmc/tsmc_core_agentic.jsonl",
+  "agentic": true,
+  "shard": null,
+  "ids_file": null,
+  "adapter": "models/traj_sft_2b_flash20g2/final,models/traj_grpo_2b_flash20d1/checkpoint-100,models/traj_grpo_2b_flash20d2/checkpoint-120",
+  "backend": "tsmc",
+  "kg_fingerprint": "ba22334639149e9d",
+  "kb_path": "None",
+  "source_text": "absent",
+  "chunks_dir": null,
+  "chat_template": "plain_chatml",
+  "max_model_len": 16384,
+  "timing": {
+    "driver": "batched",
+    "sequential": false,
+    "wall_ms_semantics": "since_batch_start",
+    "concurrency": null,
+    "batch_size": null,
+    "n_episodes": 195,
+    "num_samples": 1,
+    "group_size": null,
+    "total_wall_ms": 143934,
+    "cache_hits": null,
+    "api_calls": null
+  },
+  "usage": {
+    "prompt_tokens": null,
+    "completion_tokens": null,
+    "reasoning_tokens": null,
+    "cached_prompt_tokens": null,
+    "service_tier": null,
+    "served_tiers": null,
+    "tier_fallbacks": null,
+    "truncated": null,
+    "refused": null
+  },
+  "policy": {
+    "prompt_version": "agent_v1",
+    "max_turns": 20,
+    "turn_max_tokens": 1024,
+    "final_max_tokens": 900,
+    "max_edges_seen": 400,
+    "max_repeats": 3,
+    "repeat_echo": true,
+    "page_size": 40,
+    "hub_threshold": 150,
+    "hub_summary_limit": 400,
+    "connect_max_hops": 3,
+    "find_degree_tiebreak": false,
+    "tools": null,
+    "temperature": 0.7,
+    "reasoning_effort": null,
+    "seed": 116,
+    "give_anchor": false,
+    "relation_vocab": true,
+    "forced_prefill": false,
+    "user_note": null,
+    "gold_scaffold": false,
+    "citation_repair": false,
+    "offer_revision": false
+  },
+  "aggregate": {
+    "easy": {
+      "n": 70,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0023809523809523803,
+      "halluc_soft": 0.0,
+      "path_recall": 0.7785714285714286,
+      "path_precision": 0.6300566893424037,
+      "em_strict": 0.8571428571428571,
+      "em_claim": 0.8571428571428571,
+      "success": 0.8428571428571429,
+      "evidence_recall": 0.9071428571428571,
+      "reached_answer": 0.9142857142857143,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.14285714285714285,
+      "efficiency": 0.033819132916973325,
+      "gate_pass": 0.9714285714285714,
+      "gated_acc": 0.8428571428571429,
+      "gated_abstained": 0.02857142857142857,
+      "n_calls": 12.17142857142857,
+      "n_edges_seen": 112.58571428571429,
+      "n_repeats": 0.02857142857142857,
+      "n_malformed": 0.014285714285714285,
+      "n_tool_errors": 0.0,
+      "median_n_edges_seen": 100.0,
+      "median_edges_to_answer": 22.5,
+      "median_n_calls": 13.5,
+      "reached_rate": 0.9142857142857143,
+      "censored": 0.02857142857142857
+    },
+    "hard": {
+      "n": 57,
+      "format_rate": 1.0,
+      "halluc_strict": 0.007017543859649121,
+      "halluc_soft": 0.0035087719298245606,
+      "path_recall": 0.44298245614035087,
+      "path_precision": 0.44314954051796157,
+      "em_strict": 0.7543859649122807,
+      "em_claim": 0.7543859649122807,
+      "success": 0.7192982456140351,
+      "evidence_recall": 0.75,
+      "reached_answer": 0.7719298245614035,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.07017543859649122,
+      "efficiency": 0.04512994781274882,
+      "gate_pass": 0.9649122807017544,
+      "gated_acc": 0.7192982456140351,
+      "gated_abstained": 0.03508771929824561,
+      "n_calls": 15.105263157894736,
+      "n_edges_seen": 124.01754385964912,
+      "n_repeats": 0.0,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.0,
+      "median_n_edges_seen": 119.0,
+      "median_edges_to_answer": 37.0,
+      "median_n_calls": 15.0,
+      "reached_rate": 0.7719298245614035,
+      "censored": 0.12280701754385964
+    },
+    "medium": {
+      "n": 68,
+      "format_rate": 1.0,
+      "halluc_strict": 0.007352941176470588,
+      "halluc_soft": 0.007352941176470588,
+      "path_recall": 0.6568627450980392,
+      "path_precision": 0.5675420168067227,
+      "em_strict": 0.7794117647058824,
+      "em_claim": 0.7794117647058824,
+      "success": 0.75,
+      "evidence_recall": 0.8284313725490197,
+      "reached_answer": 0.8529411764705882,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.07352941176470588,
+      "efficiency": 0.04651722218574099,
+      "gate_pass": 0.9705882352941176,
+      "gated_acc": 0.75,
+      "gated_abstained": 0.029411764705882353,
+      "n_calls": 13.191176470588236,
+      "n_edges_seen": 114.82352941176471,
+      "n_repeats": 0.04411764705882353,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.014705882352941176,
+      "median_n_edges_seen": 111.5,
+      "median_edges_to_answer": 23.5,
+      "median_n_calls": 14.5,
+      "reached_rate": 0.8529411764705882,
+      "censored": 0.058823529411764705
+    },
+    "ALL": {
+      "n": 195,
+      "format_rate": 1.0,
+      "halluc_strict": 0.005470085470085469,
+      "halluc_soft": 0.0035897435897435893,
+      "path_recall": 0.6380341880341881,
+      "path_precision": 0.5536223036223036,
+      "em_strict": 0.8,
+      "em_claim": 0.8,
+      "success": 0.7743589743589744,
+      "evidence_recall": 0.8337606837606838,
+      "reached_answer": 0.8512820512820513,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.09743589743589744,
+      "efficiency": 0.04155342275971898,
+      "gate_pass": 0.9692307692307692,
+      "gated_acc": 0.7743589743589744,
+      "gated_abstained": 0.03076923076923077,
+      "n_calls": 13.384615384615385,
+      "n_edges_seen": 116.70769230769231,
+      "n_repeats": 0.02564102564102564,
+      "n_malformed": 0.005128205128205128,
+      "n_tool_errors": 0.005128205128205128,
+      "median_n_edges_seen": 113.0,
+      "median_edges_to_answer": 28.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8512820512820513,
+      "censored": 0.06666666666666667
+    }
+  },
+  "latency": {
+    "dir": "latency",
+    "n": 15,
+    "p25_s": 4.0,
+    "p50_s": 4.3,
+    "p75_s": 5.0,
+    "questions_per_minute": 13.84,
+    "turns_per_episode": 14.7,
+    "s_per_turn": 0.3
+  }
+}

data/runs/q2bpool32-s117--tsmc_core_agentic/summary.json

@@ -0,0 +1,192 @@
+{
+  "model": "vllm:Qwen/Qwen3.5-2B",
+  "benchmark": "data/benchmarks/tsmc/tsmc_core_agentic.jsonl",
+  "agentic": true,
+  "shard": null,
+  "ids_file": null,
+  "adapter": "models/traj_sft_2b_flash20g2/final,models/traj_grpo_2b_flash20d1/checkpoint-100,models/traj_grpo_2b_flash20d2/checkpoint-120",
+  "backend": "tsmc",
+  "kg_fingerprint": "ba22334639149e9d",
+  "kb_path": "None",
+  "source_text": "absent",
+  "chunks_dir": null,
+  "chat_template": "plain_chatml",
+  "max_model_len": 16384,
+  "timing": {
+    "driver": "batched",
+    "sequential": false,
+    "wall_ms_semantics": "since_batch_start",
+    "concurrency": null,
+    "batch_size": null,
+    "n_episodes": 195,
+    "num_samples": 1,
+    "group_size": null,
+    "total_wall_ms": 145789,
+    "cache_hits": null,
+    "api_calls": null
+  },
+  "usage": {
+    "prompt_tokens": null,
+    "completion_tokens": null,
+    "reasoning_tokens": null,
+    "cached_prompt_tokens": null,
+    "service_tier": null,
+    "served_tiers": null,
+    "tier_fallbacks": null,
+    "truncated": null,
+    "refused": null
+  },
+  "policy": {
+    "prompt_version": "agent_v1",
+    "max_turns": 20,
+    "turn_max_tokens": 1024,
+    "final_max_tokens": 900,
+    "max_edges_seen": 400,
+    "max_repeats": 3,
+    "repeat_echo": true,
+    "page_size": 40,
+    "hub_threshold": 150,
+    "hub_summary_limit": 400,
+    "connect_max_hops": 3,
+    "find_degree_tiebreak": false,
+    "tools": null,
+    "temperature": 0.7,
+    "reasoning_effort": null,
+    "seed": 117,
+    "give_anchor": false,
+    "relation_vocab": true,
+    "forced_prefill": false,
+    "user_note": null,
+    "gold_scaffold": false,
+    "citation_repair": false,
+    "offer_revision": false
+  },
+  "aggregate": {
+    "easy": {
+      "n": 70,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0,
+      "halluc_soft": 0.0,
+      "path_recall": 0.7714285714285715,
+      "path_precision": 0.5951360544217688,
+      "em_strict": 0.8428571428571429,
+      "em_claim": 0.8428571428571429,
+      "success": 0.8428571428571429,
+      "evidence_recall": 0.8571428571428571,
+      "reached_answer": 0.8857142857142857,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.15714285714285714,
+      "efficiency": 0.03302699817579546,
+      "gate_pass": 1.0,
+      "gated_acc": 0.8428571428571429,
+      "gated_abstained": 0.0,
+      "n_calls": 12.442857142857143,
+      "n_edges_seen": 102.2,
+      "n_repeats": 0.0,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.0,
+      "median_n_edges_seen": 89.0,
+      "median_edges_to_answer": 23.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8857142857142857,
+      "censored": 0.04285714285714286
+    },
+    "hard": {
+      "n": 57,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0,
+      "halluc_soft": 0.0,
+      "path_recall": 0.4298245614035088,
+      "path_precision": 0.4236006683375104,
+      "em_strict": 0.5964912280701754,
+      "em_claim": 0.5964912280701754,
+      "success": 0.5964912280701754,
+      "evidence_recall": 0.7192982456140351,
+      "reached_answer": 0.7017543859649122,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.07017543859649122,
+      "efficiency": 0.04568131378370273,
+      "gate_pass": 1.0,
+      "gated_acc": 0.5964912280701754,
+      "gated_abstained": 0.0,
+      "n_calls": 14.508771929824562,
+      "n_edges_seen": 126.2280701754386,
+      "n_repeats": 0.017543859649122806,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.0,
+      "median_n_edges_seen": 113.0,
+      "median_edges_to_answer": 34.0,
+      "median_n_calls": 15.0,
+      "reached_rate": 0.7017543859649122,
+      "censored": 0.07017543859649122
+    },
+    "medium": {
+      "n": 68,
+      "format_rate": 1.0,
+      "halluc_strict": 0.011029411764705883,
+      "halluc_soft": 0.003676470588235294,
+      "path_recall": 0.5882352941176471,
+      "path_precision": 0.5484593837535015,
+      "em_strict": 0.6911764705882353,
+      "em_claim": 0.6911764705882353,
+      "success": 0.6764705882352942,
+      "evidence_recall": 0.8137254901960784,
+      "reached_answer": 0.7941176470588235,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.07352941176470588,
+      "efficiency": 0.044371951454872015,
+      "gate_pass": 0.9558823529411765,
+      "gated_acc": 0.6764705882352942,
+      "gated_abstained": 0.04411764705882353,
+      "n_calls": 14.014705882352942,
+      "n_edges_seen": 120.41176470588235,
+      "n_repeats": 0.014705882352941176,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.029411764705882353,
+      "median_n_edges_seen": 109.0,
+      "median_edges_to_answer": 28.0,
+      "median_n_calls": 15.0,
+      "reached_rate": 0.7941176470588235,
+      "censored": 0.08823529411764706
+    },
+    "ALL": {
+      "n": 195,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0038461538461538464,
+      "halluc_soft": 0.001282051282051282,
+      "path_recall": 0.6076923076923076,
+      "path_precision": 0.5287179487179486,
+      "em_strict": 0.717948717948718,
+      "em_claim": 0.717948717948718,
+      "success": 0.7128205128205128,
+      "evidence_recall": 0.8017094017094017,
+      "reached_answer": 0.8,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.10256410256410256,
+      "efficiency": 0.04068214080465659,
+      "gate_pass": 0.9846153846153847,
+      "gated_acc": 0.7128205128205128,
+      "gated_abstained": 0.015384615384615385,
+      "n_calls": 13.594871794871795,
+      "n_edges_seen": 115.57435897435897,
+      "n_repeats": 0.010256410256410256,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.010256410256410256,
+      "median_n_edges_seen": 101.0,
+      "median_edges_to_answer": 28.0,
+      "median_n_calls": 15.0,
+      "reached_rate": 0.8,
+      "censored": 0.06666666666666667
+    }
+  },
+  "latency": {
+    "dir": "latency",
+    "n": 15,
+    "p25_s": 3.4,
+    "p50_s": 4.7,
+    "p75_s": 5.9,
+    "questions_per_minute": 12.9,
+    "turns_per_episode": 15.4,
+    "s_per_turn": 0.3
+  }
+}

data/runs/q2bpool32-s118--tsmc_core_agentic/summary.json

@@ -0,0 +1,192 @@
+{
+  "model": "vllm:Qwen/Qwen3.5-2B",
+  "benchmark": "data/benchmarks/tsmc/tsmc_core_agentic.jsonl",
+  "agentic": true,
+  "shard": null,
+  "ids_file": null,
+  "adapter": "models/traj_sft_2b_flash20g2/final,models/traj_grpo_2b_flash20d1/checkpoint-100,models/traj_grpo_2b_flash20d2/checkpoint-120",
+  "backend": "tsmc",
+  "kg_fingerprint": "ba22334639149e9d",
+  "kb_path": "None",
+  "source_text": "absent",
+  "chunks_dir": null,
+  "chat_template": "plain_chatml",
+  "max_model_len": 16384,
+  "timing": {
+    "driver": "batched",
+    "sequential": false,
+    "wall_ms_semantics": "since_batch_start",
+    "concurrency": null,
+    "batch_size": null,
+    "n_episodes": 195,
+    "num_samples": 1,
+    "group_size": null,
+    "total_wall_ms": 144631,
+    "cache_hits": null,
+    "api_calls": null
+  },
+  "usage": {
+    "prompt_tokens": null,
+    "completion_tokens": null,
+    "reasoning_tokens": null,
+    "cached_prompt_tokens": null,
+    "service_tier": null,
+    "served_tiers": null,
+    "tier_fallbacks": null,
+    "truncated": null,
+    "refused": null
+  },
+  "policy": {
+    "prompt_version": "agent_v1",
+    "max_turns": 20,
+    "turn_max_tokens": 1024,
+    "final_max_tokens": 900,
+    "max_edges_seen": 400,
+    "max_repeats": 3,
+    "repeat_echo": true,
+    "page_size": 40,
+    "hub_threshold": 150,
+    "hub_summary_limit": 400,
+    "connect_max_hops": 3,
+    "find_degree_tiebreak": false,
+    "tools": null,
+    "temperature": 0.7,
+    "reasoning_effort": null,
+    "seed": 118,
+    "give_anchor": false,
+    "relation_vocab": true,
+    "forced_prefill": false,
+    "user_note": null,
+    "gold_scaffold": false,
+    "citation_repair": false,
+    "offer_revision": false
+  },
+  "aggregate": {
+    "easy": {
+      "n": 70,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0,
+      "halluc_soft": 0.0,
+      "path_recall": 0.8071428571428572,
+      "path_precision": 0.5976077097505669,
+      "em_strict": 0.8428571428571429,
+      "em_claim": 0.8428571428571429,
+      "success": 0.8428571428571429,
+      "evidence_recall": 0.9,
+      "reached_answer": 0.9285714285714286,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.12857142857142856,
+      "efficiency": 0.03636460900561034,
+      "gate_pass": 0.9857142857142858,
+      "gated_acc": 0.8428571428571429,
+      "gated_abstained": 0.014285714285714285,
+      "n_calls": 12.542857142857143,
+      "n_edges_seen": 109.98571428571428,
+      "n_repeats": 0.014285714285714285,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.014285714285714285,
+      "median_n_edges_seen": 107.0,
+      "median_edges_to_answer": 21.0,
+      "median_n_calls": 13.0,
+      "reached_rate": 0.9285714285714286,
+      "censored": 0.1
+    },
+    "hard": {
+      "n": 57,
+      "format_rate": 1.0,
+      "halluc_strict": 0.008771929824561403,
+      "halluc_soft": 0.002923976608187134,
+      "path_recall": 0.3684210526315789,
+      "path_precision": 0.38350041771094406,
+      "em_strict": 0.6140350877192983,
+      "em_claim": 0.6140350877192983,
+      "success": 0.5789473684210527,
+      "evidence_recall": 0.6929824561403509,
+      "reached_answer": 0.7368421052631579,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.07017543859649122,
+      "efficiency": 0.04614022534679111,
+      "gate_pass": 0.9649122807017544,
+      "gated_acc": 0.5789473684210527,
+      "gated_abstained": 0.03508771929824561,
+      "n_calls": 15.192982456140351,
+      "n_edges_seen": 125.47368421052632,
+      "n_repeats": 0.0,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.03508771929824561,
+      "median_n_edges_seen": 119.0,
+      "median_edges_to_answer": 27.5,
+      "median_n_calls": 16.0,
+      "reached_rate": 0.7368421052631579,
+      "censored": 0.17543859649122806
+    },
+    "medium": {
+      "n": 68,
+      "format_rate": 1.0,
+      "halluc_strict": 0.011642156862745098,
+      "halluc_soft": 0.001838235294117647,
+      "path_recall": 0.6911764705882353,
+      "path_precision": 0.6207107843137255,
+      "em_strict": 0.7794117647058824,
+      "em_claim": 0.7794117647058824,
+      "success": 0.7352941176470589,
+      "evidence_recall": 0.8725490196078431,
+      "reached_answer": 0.8676470588235294,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.08823529411764706,
+      "efficiency": 0.04051561181252608,
+      "gate_pass": 0.9558823529411765,
+      "gated_acc": 0.7352941176470589,
+      "gated_abstained": 0.04411764705882353,
+      "n_calls": 12.735294117647058,
+      "n_edges_seen": 119.42647058823529,
+      "n_repeats": 0.0,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.0,
+      "median_n_edges_seen": 106.5,
+      "median_edges_to_answer": 23.5,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8676470588235294,
+      "censored": 0.029411764705882353
+    },
+    "ALL": {
+      "n": 195,
+      "format_rate": 1.0,
+      "halluc_strict": 0.006623931623931625,
+      "halluc_soft": 0.0014957264957264956,
+      "path_recall": 0.6384615384615384,
+      "path_precision": 0.5430789580789581,
+      "em_strict": 0.7538461538461538,
+      "em_claim": 0.7538461538461538,
+      "success": 0.7282051282051282,
+      "evidence_recall": 0.82991452991453,
+      "reached_answer": 0.8512820512820513,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.09743589743589744,
+      "efficiency": 0.04066962604313636,
+      "gate_pass": 0.9692307692307692,
+      "gated_acc": 0.7282051282051282,
+      "gated_abstained": 0.03076923076923077,
+      "n_calls": 13.384615384615385,
+      "n_edges_seen": 117.8051282051282,
+      "n_repeats": 0.005128205128205128,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.015384615384615385,
+      "median_n_edges_seen": 108.0,
+      "median_edges_to_answer": 24.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8512820512820513,
+      "censored": 0.09743589743589744
+    }
+  },
+  "latency": {
+    "dir": "latency",
+    "n": 15,
+    "p25_s": 3.9,
+    "p50_s": 4.7,
+    "p75_s": 5.7,
+    "questions_per_minute": 12.68,
+    "turns_per_episode": 14.9,
+    "s_per_turn": 0.3
+  }
+}

data/runs/q2bpool32-s119--tsmc_core_agentic/summary.json

@@ -0,0 +1,192 @@
+{
+  "model": "vllm:Qwen/Qwen3.5-2B",
+  "benchmark": "data/benchmarks/tsmc/tsmc_core_agentic.jsonl",
+  "agentic": true,
+  "shard": null,
+  "ids_file": null,
+  "adapter": "models/traj_sft_2b_flash20g2/final,models/traj_grpo_2b_flash20d1/checkpoint-100,models/traj_grpo_2b_flash20d2/checkpoint-120",
+  "backend": "tsmc",
+  "kg_fingerprint": "ba22334639149e9d",
+  "kb_path": "None",
+  "source_text": "absent",
+  "chunks_dir": null,
+  "chat_template": "plain_chatml",
+  "max_model_len": 16384,
+  "timing": {
+    "driver": "batched",
+    "sequential": false,
+    "wall_ms_semantics": "since_batch_start",
+    "concurrency": null,
+    "batch_size": null,
+    "n_episodes": 195,
+    "num_samples": 1,
+    "group_size": null,
+    "total_wall_ms": 145314,
+    "cache_hits": null,
+    "api_calls": null
+  },
+  "usage": {
+    "prompt_tokens": null,
+    "completion_tokens": null,
+    "reasoning_tokens": null,
+    "cached_prompt_tokens": null,
+    "service_tier": null,
+    "served_tiers": null,
+    "tier_fallbacks": null,
+    "truncated": null,
+    "refused": null
+  },
+  "policy": {
+    "prompt_version": "agent_v1",
+    "max_turns": 20,
+    "turn_max_tokens": 1024,
+    "final_max_tokens": 900,
+    "max_edges_seen": 400,
+    "max_repeats": 3,
+    "repeat_echo": true,
+    "page_size": 40,
+    "hub_threshold": 150,
+    "hub_summary_limit": 400,
+    "connect_max_hops": 3,
+    "find_degree_tiebreak": false,
+    "tools": null,
+    "temperature": 0.7,
+    "reasoning_effort": null,
+    "seed": 119,
+    "give_anchor": false,
+    "relation_vocab": true,
+    "forced_prefill": false,
+    "user_note": null,
+    "gold_scaffold": false,
+    "citation_repair": false,
+    "offer_revision": false
+  },
+  "aggregate": {
+    "easy": {
+      "n": 70,
+      "format_rate": 1.0,
+      "halluc_strict": 0.0035714285714285713,
+      "halluc_soft": 0.0035714285714285713,
+      "path_recall": 0.7857142857142857,
+      "path_precision": 0.6229931972789116,
+      "em_strict": 0.8571428571428571,
+      "em_claim": 0.8571428571428571,
+      "success": 0.8428571428571429,
+      "evidence_recall": 0.9142857142857143,
+      "reached_answer": 0.9285714285714286,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.18571428571428572,
+      "efficiency": 0.030892668502389954,
+      "gate_pass": 0.9857142857142858,
+      "gated_acc": 0.8428571428571429,
+      "gated_abstained": 0.014285714285714285,
+      "n_calls": 12.514285714285714,
+      "n_edges_seen": 103.45714285714286,
+      "n_repeats": 0.014285714285714285,
+      "n_malformed": 0.014285714285714285,
+      "n_tool_errors": 0.014285714285714285,
+      "median_n_edges_seen": 99.5,
+      "median_edges_to_answer": 21.0,
+      "median_n_calls": 13.5,
+      "reached_rate": 0.9285714285714286,
+      "censored": 0.014285714285714285
+    },
+    "hard": {
+      "n": 57,
+      "format_rate": 1.0,
+      "halluc_strict": 0.013742690058479532,
+      "halluc_soft": 0.0,
+      "path_recall": 0.35964912280701755,
+      "path_precision": 0.3783625730994152,
+      "em_strict": 0.49122807017543857,
+      "em_claim": 0.49122807017543857,
+      "success": 0.47368421052631576,
+      "evidence_recall": 0.6578947368421053,
+      "reached_answer": 0.6666666666666666,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.08771929824561403,
+      "efficiency": 0.04741264099514376,
+      "gate_pass": 0.9473684210526315,
+      "gated_acc": 0.47368421052631576,
+      "gated_abstained": 0.05263157894736842,
+      "n_calls": 15.403508771929825,
+      "n_edges_seen": 124.54385964912281,
+      "n_repeats": 0.03508771929824561,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.05263157894736842,
+      "median_n_edges_seen": 117.0,
+      "median_edges_to_answer": 32.5,
+      "median_n_calls": 15.0,
+      "reached_rate": 0.6666666666666666,
+      "censored": 0.22807017543859648
+    },
+    "medium": {
+      "n": 68,
+      "format_rate": 1.0,
+      "halluc_strict": 0.007352941176470588,
+      "halluc_soft": 0.0,
+      "path_recall": 0.6519607843137255,
+      "path_precision": 0.5833333333333333,
+      "em_strict": 0.7647058823529411,
+      "em_claim": 0.7647058823529411,
+      "success": 0.7647058823529411,
+      "evidence_recall": 0.8480392156862745,
+      "reached_answer": 0.8823529411764706,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.08823529411764706,
+      "efficiency": 0.045530267708825234,
+      "gate_pass": 0.9852941176470589,
+      "gated_acc": 0.7647058823529411,
+      "gated_abstained": 0.014705882352941176,
+      "n_calls": 13.647058823529411,
+      "n_edges_seen": 119.80882352941177,
+      "n_repeats": 0.014705882352941176,
+      "n_malformed": 0.0,
+      "n_tool_errors": 0.029411764705882353,
+      "median_n_edges_seen": 109.0,
+      "median_edges_to_answer": 36.5,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8823529411764706,
+      "censored": 0.058823529411764705
+    },
+    "ALL": {
+      "n": 195,
+      "format_rate": 1.0,
+      "halluc_strict": 0.007863247863247862,
+      "halluc_soft": 0.001282051282051282,
+      "path_recall": 0.6145299145299145,
+      "path_precision": 0.5376556776556777,
+      "em_strict": 0.717948717948718,
+      "em_claim": 0.717948717948718,
+      "success": 0.7076923076923077,
+      "evidence_recall": 0.8162393162393162,
+      "reached_answer": 0.8358974358974359,
+      "abstained": 0.0,
+      "answer_named_before_seen": 0.12307692307692308,
+      "efficiency": 0.04082597710815696,
+      "gate_pass": 0.9743589743589743,
+      "gated_acc": 0.7076923076923077,
+      "gated_abstained": 0.02564102564102564,
+      "n_calls": 13.753846153846155,
+      "n_edges_seen": 115.32307692307693,
+      "n_repeats": 0.020512820512820513,
+      "n_malformed": 0.005128205128205128,
+      "n_tool_errors": 0.03076923076923077,
+      "median_n_edges_seen": 106.0,
+      "median_edges_to_answer": 27.0,
+      "median_n_calls": 14.0,
+      "reached_rate": 0.8358974358974359,
+      "censored": 0.09230769230769231
+    }
+  },
+  "latency": {
+    "dir": "latency",
+    "n": 15,
+    "p25_s": 4.1,
+    "p50_s": 4.6,
+    "p75_s": 5.1,
+    "questions_per_minute": 12.96,
+    "turns_per_episode": 15.9,
+    "s_per_turn": 0.29
+  }
+}