feat(cp6): CLI compare — aggregate human/LLM traces by (model, difficulty)

proteus/cli.py

@@ -4,6 +4,7 @@ Subcommands:
     run             Run one session and append its trace to a JSONL file.
     list-scenarios  List the registered scenario names.
     replay          Print a saved trace's turns, outcome, and metrics.
+    compare         Aggregate traces by (model, difficulty) for baseline comparison.
 
 All commands are network-free except ``run`` with a real provider spec; use
 ``--model fake:<name>`` for an offline smoke.
@@ -137,6 +138,39 @@ def _cmd_replay(args: argparse.Namespace) -> int:
     return 0
 
 
+def _cmd_compare(args: argparse.Namespace) -> int:
+    import json
+
+    from proteus.runtime.aggregate import aggregate_traces
+
+    all_traces = []
+    for path in args.trace_files:
+        try:
+            all_traces.extend(read_traces(path))
+        except FileNotFoundError:
+            print(f"trace file not found: {path}", file=sys.stderr)
+            return 2
+    if not all_traces:
+        print("no traces to compare", file=sys.stderr)
+        return 1
+
+    groups = aggregate_traces(all_traces)
+    for (model, difficulty), info in sorted(groups.items()):
+        print(f"=== model={model} difficulty={difficulty} n={info['n']} ===")
+        for key, mean in info["metrics"].items():
+            print(f"  {key}: {mean:.2f}")
+
+    if args.out:
+        serializable = {
+            f"{model}|{difficulty}": info
+            for (model, difficulty), info in groups.items()
+        }
+        with open(args.out, "w") as fh:
+            json.dump(serializable, fh, indent=2)
+        print(f"summary written to {args.out}")
+    return 0
+
+
 def build_parser() -> argparse.ArgumentParser:
     """Build the top-level argument parser."""
     parser = argparse.ArgumentParser(
@@ -194,6 +228,15 @@ def build_parser() -> argparse.ArgumentParser:
     replay.add_argument("--fps", type=float, default=4.0, help="replay frames/sec")
     replay.set_defaults(func=_cmd_replay)
 
+    compare = sub.add_parser(
+        "compare", help="aggregate traces by (model, difficulty) for baseline comparison"
+    )
+    compare.add_argument("trace_files", nargs="+", help="one or more .jsonl trace files")
+    compare.add_argument(
+        "--out", default=None, help="optional JSON file to write the aggregate summary to"
+    )
+    compare.set_defaults(func=_cmd_compare)
+
     return parser
 
 

tests/cli/test_cli.py

@@ -149,3 +149,45 @@ def test_play_handles_stdin_eof(monkeypatch, capsys):
     ])
     assert rc == 2
     assert "stdin" in capsys.readouterr().err.lower()
+
+
+def test_compare_aggregates_traces(tmp_path, capsys):
+    out = tmp_path / "runs.jsonl"
+    # Two fake-model traces at the same difficulty (model id "demo").
+    for seed in (1, 2):
+        main([
+            "run", "--scenario", "predator_evade", "--model", "fake:demo",
+            "--seed", str(seed), "--play-turns", "4", "--no-probe", "--out", str(out),
+        ])
+    capsys.readouterr()  # drain
+    rc = main(["compare", str(out)])
+    text = capsys.readouterr().out
+    assert rc == 0
+    assert "demo" in text and "easy" in text
+    assert "n=2" in text
+    assert "motive_reading_accuracy" in text
+
+
+def test_compare_writes_summary_json(tmp_path):
+    import json
+
+    out = tmp_path / "runs.jsonl"
+    main([
+        "run", "--scenario", "predator_evade", "--model", "fake:demo",
+        "--seed", "1", "--play-turns", "4", "--no-probe", "--out", str(out),
+    ])
+    summary = tmp_path / "summary.json"
+    rc = main(["compare", str(out), "--out", str(summary)])
+    assert rc == 0
+    data = json.loads(summary.read_text())
+    assert data  # non-empty aggregate
+    # Pin the documented "model|difficulty" key format + nested shape.
+    assert "demo|easy" in data
+    assert data["demo|easy"]["n"] == 1
+    assert "motive_reading_accuracy" in data["demo|easy"]["metrics"]
+
+
+def test_compare_missing_file_errors(capsys):
+    rc = main(["compare", "/no/such/file.jsonl"])
+    assert rc == 2
+    assert "not found" in capsys.readouterr().err