comm

README.md

@@ -100,12 +100,12 @@ See [`docs/lightning_hf_runbook.md`](docs/lightning_hf_runbook.md) for the short
 Evaluate all model stages through the same environment:
 
 ```powershell
-uv run python eval.py --policy scripted_weak --label baseline --output results/baseline_eval.jsonl
-uv run python eval.py --policy oracle --label oracle --output results/oracle_eval.jsonl
-uv run python plot_results.py --inputs results/baseline_eval.jsonl results/oracle_eval.jsonl --output-dir results
+uv run python eval.py --policy scripted_weak --label baseline
+uv run python eval.py --policy oracle --label oracle
+uv run python plot_results.py --inputs results/runs --output-dir results
 ```
 
-For trained adapters on a GPU box, use `eval.py --policy hf --model <base_model> --adapter <adapter_path>`.
+For trained adapters on a GPU box, use `eval.py --policy hf --model <base_model> --adapter <adapter_path>`. If `--output` is omitted, eval files are saved under `results/runs/<model-adapter-label>/` with a `metadata.json` summary.
 
 ## OpenEnv validation
 

eval.py

@@ -1,8 +1,8 @@
 """Evaluate CORP-ENV policies through the real OpenEnv environment.
 
 Examples:
-  uv run python eval.py --policy scripted_weak --output results/baseline_eval.jsonl
-  uv run python eval.py --policy oracle --output results/oracle_eval.jsonl
+  uv run python eval.py --policy scripted_weak --label baseline
+  uv run python eval.py --policy oracle --label oracle
   uv run python eval.py --policy openai --model Qwen/Qwen2.5-7B-Instruct
   uv run python eval.py --policy hf --model outputs/sft_adapter --adapter outputs/grpo_adapter
 """
@@ -12,6 +12,7 @@ from __future__ import annotations
 import argparse
 import json
 import os
+import re
 import sys
 from pathlib import Path
 from typing import Any, Dict, List, Optional
@@ -241,6 +242,60 @@ def summarize(rows: List[Dict[str, Any]], label: str) -> None:
         )
 
 
+def slugify(value: str, fallback: str = "unknown") -> str:
+    value = (value or "").strip()
+    if not value:
+        return fallback
+    value = value.replace("\\", "/").rstrip("/")
+    value = value.split("/")[-1] if "/" in value else value
+    value = re.sub(r"[^A-Za-z0-9._-]+", "-", value)
+    value = value.strip("-._")
+    return value or fallback
+
+
+def default_output_path(args: argparse.Namespace) -> Path:
+    stage = slugify(args.label or args.policy, "eval")
+    model_slug = slugify(args.model, args.policy)
+    adapter_slug = slugify(args.adapter, "no-adapter")
+    if args.adapter:
+        run_slug = f"{model_slug}__{adapter_slug}__{stage}"
+    else:
+        run_slug = f"{model_slug}__{stage}"
+    return Path(args.results_root) / "runs" / run_slug / f"{stage}_eval.jsonl"
+
+
+def write_run_metadata(path: Path, args: argparse.Namespace, rows: List[Dict[str, Any]]) -> None:
+    by_task: Dict[str, List[Dict[str, Any]]] = {}
+    for row in rows:
+        by_task.setdefault(row["task_id"], []).append(row)
+    summary = {
+        "model_stage": args.label or args.policy,
+        "policy": args.policy,
+        "model": args.model,
+        "adapter": args.adapter,
+        "tasks": [t.strip() for t in args.tasks.split(",") if t.strip()],
+        "episodes": args.episodes,
+        "max_steps": args.max_steps,
+        "max_new_tokens": args.max_new_tokens,
+        "eval_file": str(path),
+        "metrics_by_task": {},
+    }
+    for task_id, task_rows in by_task.items():
+        summary["metrics_by_task"][task_id] = {
+            "avg_terminal_reward": round(
+                sum(r["terminal_reward"] for r in task_rows) / len(task_rows), 6
+            ),
+            "avg_verifier_pass_rate": round(
+                sum(r["verifier_pass_rate"] for r in task_rows) / len(task_rows), 6
+            ),
+            "success_rate": round(
+                sum(1 for r in task_rows if r["success"]) / len(task_rows), 6
+            ),
+        }
+    metadata_path = path.with_name("metadata.json")
+    metadata_path.write_text(json.dumps(summary, indent=2, ensure_ascii=False), encoding="utf-8")
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(description="Evaluate CORP-ENV model stages.")
     parser.add_argument("--policy", choices=["scripted_weak", "oracle", "openai", "hf"], default="scripted_weak")
@@ -251,7 +306,12 @@ def main() -> None:
     parser.add_argument("--episodes", type=int, default=1)
     parser.add_argument("--max-steps", type=int, default=30)
     parser.add_argument("--max-new-tokens", type=int, default=1536)
-    parser.add_argument("--output", default="results/eval.jsonl")
+    parser.add_argument("--results-root", default="results", help="Root for auto-organized eval output.")
+    parser.add_argument(
+        "--output",
+        default="",
+        help="Explicit JSONL path. If omitted, writes under results/runs/<model-adapter-label>/.",
+    )
     args = parser.parse_args()
 
     os.environ.setdefault("CORP_STUB_WORKERS", "1")
@@ -278,14 +338,17 @@ def main() -> None:
                 row = run_model_episode(task_id=task_id, policy=policy, max_steps=max_steps)
             row["episode_index"] = ep
             row["model_stage"] = args.label or args.policy
+            row["policy"] = args.policy
             row["model"] = args.model
             row["adapter"] = args.adapter
             rows.append(row)
 
-    out = Path(args.output)
+    out = Path(args.output) if args.output else default_output_path(args)
     write_jsonl(out, rows)
+    write_run_metadata(out, args, rows)
     summarize(rows, args.label or args.policy)
     print(f"\nWrote {out}")
+    print(f"Wrote {out.with_name('metadata.json')}")
 
 
 if __name__ == "__main__":

plot_results.py

@@ -1,4 +1,4 @@
-"""Create hackathon result plots from CORP-ENV eval JSONL files."""
+"""Create hackathon result plots from CORP-ENV eval JSONL files or run folders."""
 
 from __future__ import annotations
 
@@ -9,15 +9,39 @@ from pathlib import Path
 from typing import Any, Dict, Iterable, List
 
 
+def expand_inputs(inputs: Iterable[str]) -> List[Path]:
+    paths: List[Path] = []
+    for raw in inputs:
+        path = Path(raw)
+        if path.is_dir():
+            paths.extend(sorted(path.rglob("*_eval.jsonl")))
+            paths.extend(sorted(path.rglob("eval.jsonl")))
+        elif path.exists():
+            paths.append(path)
+        else:
+            matches = sorted(Path().glob(raw))
+            paths.extend([m for m in matches if m.is_file()])
+    # Preserve order but remove duplicates.
+    seen = set()
+    out: List[Path] = []
+    for path in paths:
+        key = str(path.resolve())
+        if key not in seen:
+            seen.add(key)
+            out.append(path)
+    return out
+
+
 def read_rows(paths: Iterable[str]) -> List[Dict[str, Any]]:
     rows: List[Dict[str, Any]] = []
-    for path_s in paths:
-        path = Path(path_s)
+    for path in expand_inputs(paths):
         with path.open("r", encoding="utf-8") as f:
             for line in f:
                 if line.strip():
                     row = json.loads(line)
                     row.setdefault("model_stage", path.stem.replace("_eval", ""))
+                    if "run_id" not in row:
+                        row["run_id"] = path.parent.name
                     steps = max(float(row.get("steps", 0) or 0), 1.0)
                     row["invalid_action_rate"] = float(row.get("invalid_action_count", 0) or 0) / steps
                     rows.append(row)
@@ -95,7 +119,7 @@ def plot_reward_curve(rows: List[Dict[str, Any]], output: Path) -> None:
 
 def main() -> None:
     parser = argparse.ArgumentParser(description="Plot CORP-ENV eval results.")
-    parser.add_argument("--inputs", nargs="+", required=True, help="Eval JSONL files.")
+    parser.add_argument("--inputs", nargs="+", required=True, help="Eval JSONL files, folders, or glob patterns.")
     parser.add_argument("--output-dir", default="results")
     args = parser.parse_args()
 

results/sft_eval.jsonl

@@ -0,0 +1,6 @@
+{"task_id": "e1_launch_readiness", "policy_kind": "model", "steps": 3, "total_reward": 1.39, "terminal_reward": 0.91, "reward_trace": [0.29000000000000004, 0.19, 0.9099999999999999], "verifier_pass_rate": 1.0, "passed_checks": ["qa_report_present", "final_rec_valid", "no_missed_milestones"], "failed_checks": [], "milestones_total": 2, "milestones_complete": 2, "milestones_missed": 0, "invalid_action_count": 0, "env_error_count": 0, "errors": [], "final_swd_version": 4, "success": true, "episode_index": 0, "model_stage": "sft", "model": "Qwen/Qwen2.5-7B-Instruct", "adapter": "Navigam/corp-gym-sft-adapter"}
+{"task_id": "m1_budget_reallocation", "policy_kind": "model", "steps": 8, "total_reward": 1.815, "terminal_reward": 0.685, "reward_trace": [0.09000000000000001, 0.29000000000000004, 0.19, 0.19, -0.01, 0.39, -0.01, 0.685], "verifier_pass_rate": 0.833333, "passed_checks": ["required_agents_consulted", "conflict_logged", "conflict_resolved", "budget_constraint_acknowledged", "reasoning_documented"], "failed_checks": ["phased_plan"], "milestones_total": 3, "milestones_complete": 2, "milestones_missed": 0, "invalid_action_count": 2, "env_error_count": 2, "errors": ["invalid_action: Expecting value: line 1 column 58 (char 57)", "invalid_action: Expecting value: line 1 column 58 (char 57)"], "final_swd_version": 7, "success": false, "episode_index": 0, "model_stage": "sft", "model": "Qwen/Qwen2.5-7B-Instruct", "adapter": "Navigam/corp-gym-sft-adapter"}
+{"task_id": "e1_launch_readiness", "policy_kind": "model", "steps": 3, "total_reward": 1.39, "terminal_reward": 0.91, "reward_trace": [0.29000000000000004, 0.19, 0.9099999999999999], "verifier_pass_rate": 1.0, "passed_checks": ["qa_report_present", "final_rec_valid", "no_missed_milestones"], "failed_checks": [], "milestones_total": 2, "milestones_complete": 2, "milestones_missed": 0, "invalid_action_count": 0, "env_error_count": 0, "errors": [], "final_swd_version": 4, "success": true, "episode_index": 1, "model_stage": "sft", "model": "Qwen/Qwen2.5-7B-Instruct", "adapter": "Navigam/corp-gym-sft-adapter"}
+{"task_id": "m1_budget_reallocation", "policy_kind": "model", "steps": 5, "total_reward": 1.766667, "terminal_reward": 0.806667, "reward_trace": [0.09000000000000001, 0.29000000000000004, 0.19, 0.39, 0.8066666666666666], "verifier_pass_rate": 0.666667, "passed_checks": ["required_agents_consulted", "conflict_logged", "conflict_resolved", "phased_plan"], "failed_checks": ["budget_constraint_acknowledged", "reasoning_documented"], "milestones_total": 3, "milestones_complete": 3, "milestones_missed": 0, "invalid_action_count": 0, "env_error_count": 0, "errors": [], "final_swd_version": 6, "success": false, "episode_index": 1, "model_stage": "sft", "model": "Qwen/Qwen2.5-7B-Instruct", "adapter": "Navigam/corp-gym-sft-adapter"}
+{"task_id": "e1_launch_readiness", "policy_kind": "model", "steps": 3, "total_reward": 1.39, "terminal_reward": 0.91, "reward_trace": [0.29000000000000004, 0.19, 0.9099999999999999], "verifier_pass_rate": 1.0, "passed_checks": ["qa_report_present", "final_rec_valid", "no_missed_milestones"], "failed_checks": [], "milestones_total": 2, "milestones_complete": 2, "milestones_missed": 0, "invalid_action_count": 0, "env_error_count": 0, "errors": [], "final_swd_version": 4, "success": true, "episode_index": 2, "model_stage": "sft", "model": "Qwen/Qwen2.5-7B-Instruct", "adapter": "Navigam/corp-gym-sft-adapter"}
+{"task_id": "m1_budget_reallocation", "policy_kind": "model", "steps": 6, "total_reward": 1.776667, "terminal_reward": 0.626667, "reward_trace": [0.09000000000000001, 0.29000000000000004, 0.19, 0.19, 0.39, 0.6266666666666666], "verifier_pass_rate": 0.666667, "passed_checks": ["required_agents_consulted", "conflict_logged", "conflict_resolved", "reasoning_documented"], "failed_checks": ["phased_plan", "budget_constraint_acknowledged"], "milestones_total": 3, "milestones_complete": 2, "milestones_missed": 0, "invalid_action_count": 0, "env_error_count": 0, "errors": [], "final_swd_version": 7, "success": false, "episode_index": 2, "model_stage": "sft", "model": "Qwen/Qwen2.5-7B-Instruct", "adapter": "Navigam/corp-gym-sft-adapter"}

training/train_sft.py

@@ -57,9 +57,10 @@ def main() -> None:
     args = parser.parse_args()
 
     try:
+        # Unsloth must load before trl/transformers/peft for full optimizations.
+        from unsloth import FastLanguageModel
         from datasets import Dataset
         from trl import SFTConfig, SFTTrainer
-        from unsloth import FastLanguageModel
     except ImportError as exc:
         raise SystemExit(
             "SFT training requires datasets, trl, and unsloth. On Lightning AI, install with:\n"
@@ -72,6 +73,11 @@ def main() -> None:
         dtype=None,
         load_in_4bit=True,
     )
+    if getattr(tokenizer, "pad_token", None) is None and getattr(
+        tokenizer, "eos_token", None
+    ) is not None:
+        tokenizer.pad_token = tokenizer.eos_token
+
     model = FastLanguageModel.get_peft_model(
         model,
         r=32,
@@ -95,7 +101,7 @@ def main() -> None:
     config = SFTConfig(
         output_dir=args.output,
         dataset_text_field="text",
-        max_seq_length=args.max_seq_length,
+        max_length=args.max_seq_length,
         per_device_train_batch_size=args.batch_size,
         gradient_accumulation_steps=args.grad_accum,
         num_train_epochs=args.epochs,