feat: add generate-baseline subcommand and -d flag to test runner

Adds `generate-baseline` for bulk expected log generation from replays
without baselines, and `-d DIR` to both headless and generate-baseline
modes for targeting subdirectories (e.g. testdata/replays/worms2d).
Extension matching is now case-insensitive (.WAgame and .wagame).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/openwa-test-runner/CLAUDE.md

@@ -4,9 +4,10 @@ Replay test runner (`openwa-test` binary). Supports headless (concurrent, log-di
 
 ## Subcommands
 
-- `openwa-test [filter] [-j N] [--no-build] [--wa-path PATH]` — Headless: discovers `testdata/replays/*.WAgame` with `*_expected.log`, runs concurrently via `/getlog`, compares output byte-for-byte.
+- `openwa-test [filter] [-j N] [-d DIR] [--no-build] [--wa-path PATH]` — Headless: discovers `*.WAgame`/`*.wagame` with `*_expected.log` in DIR (default `testdata/replays`), runs concurrently via `/getlog`, compares output byte-for-byte.
 - `openwa-test headful [filter|replay.WAgame] [--no-build] [--wa-path PATH] [--timeout SECS]` — Headful: runs with graphics+sound, checks for crashes/panics/gameplay markers. Default timeout 150s.
 - `openwa-test trace-desync <replay.WAgame> [--no-build] [--wa-path PATH]` — Compares per-frame checksums between baseline and hooked runs.
+- `openwa-test generate-baseline [filter] [-d DIR] [-j N] [--no-build] [--wa-path PATH]` — Generates `*_expected.log` baselines for replays that don't have one yet. Runs with `OPENWA_TRACE_BASELINE=1` (minimal hooks).
 
 ## Test Isolation
 
@@ -42,9 +43,18 @@ A headful test passes when: no crash, no `[PANIC]` in OpenWA.log, no `[GAMEPLAY
 - `OPENWA_LOG_PATH=<path>` — Override OpenWA.log location (used for per-instance isolation)
 - `OPENWA_ERRORLOG_PATH=<path>` — Redirect ERRORLOG.TXT to specified path (crash capture)
 
+## Worms 2D Speedrun Suite
+
+~598 replays in `testdata/replays/worms2d/` (`.wagame` extension). Must run with `-j 1` or `-j 2` — higher concurrency causes ~5-7% flakes from shared resource contention. Full suite takes ~3-4 min at `-j 1`. Use for final validation only.
+
+```bash
+openwa-test -d testdata/replays/worms2d -j 1
+```
+
 ## Key Paths
 
 - Replay files + expected logs: `testdata/replays/*.WAgame` + `*_expected.log`
+- Worms 2D replays: `testdata/replays/worms2d/*.wagame` + `*_expected.log`
 - Per-run output: `testdata/runs/<timestamp>/` (headless), `testdata/runs/headful-<timestamp>/` (headful)
 - Headless script: `run-tests.ps1`
 - Headful script: `replay-test.ps1`

crates/openwa-test-runner/src/main.rs

@@ -55,6 +55,7 @@ struct Args {
     jobs: usize,
     no_build: bool,
     wa_path: Option<PathBuf>,
+    dir: PathBuf,
 }
 
 // ─── Argument parsing ───────────────────────────────────────────────────────
@@ -65,6 +66,7 @@ fn parse_args() -> Args {
         jobs: DEFAULT_JOBS,
         no_build: false,
         wa_path: None,
+        dir: PathBuf::from(REPLAYS_DIR),
     };
 
     let argv: Vec<String> = env::args().skip(1).collect();
@@ -84,12 +86,20 @@ fn parse_args() -> Args {
                     args.wa_path = Some(PathBuf::from(&argv[i]));
                 }
             }
+            "--dir" | "-d" => {
+                i += 1;
+                if i < argv.len() {
+                    args.dir = PathBuf::from(&argv[i]);
+                }
+            }
             s if !s.starts_with('-') => {
                 args.filter = Some(s.to_string());
             }
             other => {
                 eprintln!("Unknown argument: {other}");
-                eprintln!("Usage: openwa-test [filter] [-j N] [--no-build] [--wa-path PATH]");
+                eprintln!(
+                    "Usage: openwa-test [filter] [-j N] [-d DIR] [--no-build] [--wa-path PATH]"
+                );
                 std::process::exit(1);
             }
         }
@@ -101,21 +111,25 @@ fn parse_args() -> Args {
 
 // ─── Test discovery ─────────────────────────────────────────────────────────
 
-fn discover_tests(filter: Option<&str>) -> Vec<TestCase> {
-    let replays_dir = Path::new(REPLAYS_DIR);
+fn discover_tests(replays_dir: &Path, filter: Option<&str>) -> Vec<TestCase> {
     let mut tests = Vec::new();
 
     let entries = match fs::read_dir(replays_dir) {
         Ok(e) => e,
         Err(e) => {
-            eprintln!("Cannot read {REPLAYS_DIR}: {e}");
+            eprintln!("Cannot read {}: {e}", replays_dir.display());
             return tests;
         }
     };
 
     for entry in entries.flatten() {
         let path = entry.path();
-        if path.extension().and_then(|e| e.to_str()) != Some("WAgame") {
+        let ext_match = path
+            .extension()
+            .and_then(|e| e.to_str())
+            .map(|e| e.eq_ignore_ascii_case("wagame"))
+            .unwrap_or(false);
+        if !ext_match {
             continue;
         }
 
@@ -1314,6 +1328,369 @@ fn report_startup_check_failures(results: &[TestResult], run_dir: &Path) {
     }
 }
 
+// ─── Generate-baseline subcommand ───────────────────────────────────────────
+
+struct GenerateBaselineArgs {
+    dir: PathBuf,
+    filter: Option<String>,
+    jobs: usize,
+    no_build: bool,
+    wa_path: Option<PathBuf>,
+}
+
+fn parse_generate_baseline_args(argv: &[String]) -> GenerateBaselineArgs {
+    let mut args = GenerateBaselineArgs {
+        dir: PathBuf::from(REPLAYS_DIR),
+        filter: None,
+        jobs: DEFAULT_JOBS,
+        no_build: false,
+        wa_path: None,
+    };
+    let mut i = 0;
+    while i < argv.len() {
+        match argv[i].as_str() {
+            "-j" | "--jobs" => {
+                i += 1;
+                if i < argv.len() {
+                    args.jobs = argv[i].parse().unwrap_or(DEFAULT_JOBS);
+                }
+            }
+            "--no-build" => args.no_build = true,
+            "--wa-path" => {
+                i += 1;
+                if i < argv.len() {
+                    args.wa_path = Some(PathBuf::from(&argv[i]));
+                }
+            }
+            "--dir" | "-d" => {
+                i += 1;
+                if i < argv.len() {
+                    args.dir = PathBuf::from(&argv[i]);
+                }
+            }
+            s if !s.starts_with('-') => {
+                args.filter = Some(s.to_string());
+            }
+            other => {
+                eprintln!("Unknown argument: {other}");
+                eprintln!(
+                    "Usage: openwa-test generate-baseline [filter] [-d DIR] [-j N] [--no-build] [--wa-path PATH]"
+                );
+                std::process::exit(1);
+            }
+        }
+        i += 1;
+    }
+    args
+}
+
+/// Discover .WAgame files that do NOT yet have a corresponding _expected.log.
+fn discover_baseline_targets(dir: &Path, filter: Option<&str>) -> Vec<TestCase> {
+    let entries = match fs::read_dir(dir) {
+        Ok(e) => e,
+        Err(e) => {
+            eprintln!("Cannot read {}: {e}", dir.display());
+            return Vec::new();
+        }
+    };
+
+    let mut tests = Vec::new();
+    for entry in entries.flatten() {
+        let path = entry.path();
+        let ext_match = path
+            .extension()
+            .and_then(|e| e.to_str())
+            .map(|e| e.eq_ignore_ascii_case("wagame"))
+            .unwrap_or(false);
+        if !ext_match {
+            continue;
+        }
+        let stem = match path.file_stem().and_then(|s| s.to_str()) {
+            Some(s) => s.to_string(),
+            None => continue,
+        };
+        if let Some(filter) = filter {
+            if !stem.contains(filter) {
+                continue;
+            }
+        }
+
+        let expected = dir.join(format!("{stem}_expected.log"));
+        if expected.exists() {
+            continue; // Already has a baseline — skip
+        }
+
+        let replay_abs = strip_unc(fs::canonicalize(&path).unwrap_or(path.clone()));
+        let output_abs = replay_abs.with_extension("log");
+
+        tests.push(TestCase {
+            name: stem.clone(),
+            replay_path: replay_abs,
+            expected_log: strip_unc(fs::canonicalize(dir).unwrap_or(dir.to_path_buf()))
+                .join(format!("{stem}_expected.log")),
+            output_log: output_abs,
+        });
+    }
+    tests.sort_by(|a, b| a.name.cmp(&b.name));
+    tests
+}
+
+/// Run a single baseline generation: launch with OPENWA_TRACE_BASELINE=1, then
+/// copy the output log to become the expected log.
+fn run_baseline_test(
+    test: &TestCase,
+    launcher: &Path,
+    wa_exe: &Path,
+    run_dir: &Path,
+) -> TestResult {
+    let start = Instant::now();
+
+    // Remove stale output log
+    let _ = fs::remove_file(&test.output_log);
+
+    // Per-instance log paths
+    let openwa_log = run_dir.join(format!("{}.openwa.log", test.name));
+    let errorlog_path = run_dir.join(format!("{}.errorlog.txt", test.name));
+
+    let result = Command::new(launcher)
+        .arg(wa_exe)
+        .arg("/getlog")
+        .arg(&test.replay_path)
+        .env("OPENWA_HEADLESS", "1")
+        .env("OPENWA_TRACE_BASELINE", "1")
+        .env("OPENWA_LOG_PATH", &openwa_log)
+        .env("OPENWA_ERRORLOG_PATH", &errorlog_path)
+        .stdout(std::process::Stdio::null())
+        .stderr(std::process::Stdio::null())
+        .creation_flags(CREATE_NO_WINDOW)
+        .status();
+
+    let duration = start.elapsed();
+
+    match result {
+        Err(e) => TestResult {
+            name: test.name.clone(),
+            passed: false,
+            duration,
+            diff_lines: Vec::new(),
+            error: Some(format!("Failed to launch: {e}")),
+            crashed: None,
+        },
+        Ok(status) => {
+            let exit_code = status.code().unwrap_or(0);
+
+            if duration >= Duration::from_secs(TIMEOUT_SECS) {
+                return TestResult {
+                    name: test.name.clone(),
+                    passed: false,
+                    duration,
+                    diff_lines: Vec::new(),
+                    error: Some("Timeout".to_string()),
+                    crashed: None,
+                };
+            }
+
+            // Check for crash
+            let errorlog_content = fs::read(&errorlog_path)
+                .ok()
+                .filter(|b| !b.is_empty())
+                .map(|b| String::from_utf8_lossy(&b).into_owned());
+
+            if is_crash_exit_code(exit_code) || errorlog_content.is_some() {
+                let unsigned = exit_code as u32;
+                let name = if is_crash_exit_code(exit_code) {
+                    ntstatus_name(unsigned)
+                } else {
+                    parse_errorlog_exception(errorlog_content.as_deref())
+                };
+                return TestResult {
+                    name: test.name.clone(),
+                    passed: false,
+                    duration,
+                    diff_lines: Vec::new(),
+                    error: None,
+                    crashed: Some(CrashInfo {
+                        exit_code: unsigned,
+                        name,
+                        errorlog_content,
+                    }),
+                };
+            }
+
+            // Check output log exists
+            if !test.output_log.exists() {
+                return TestResult {
+                    name: test.name.clone(),
+                    passed: false,
+                    duration,
+                    diff_lines: Vec::new(),
+                    error: Some(format!("No output log generated (exit code: {exit_code})")),
+                    crashed: None,
+                };
+            }
+
+            // Copy output log → expected log
+            match fs::copy(&test.output_log, &test.expected_log) {
+                Ok(_) => {
+                    let _ = fs::remove_file(&test.output_log);
+                    TestResult {
+                        name: test.name.clone(),
+                        passed: true,
+                        duration,
+                        diff_lines: Vec::new(),
+                        error: None,
+                        crashed: None,
+                    }
+                }
+                Err(e) => TestResult {
+                    name: test.name.clone(),
+                    passed: false,
+                    duration,
+                    diff_lines: Vec::new(),
+                    error: Some(format!("Failed to copy output to expected log: {e}")),
+                    crashed: None,
+                },
+            }
+        }
+    }
+}
+
+/// Thread pool for baseline generation (same pattern as run_tests_parallel but
+/// uses run_baseline_test).
+fn run_baseline_parallel(
+    tests: Vec<TestCase>,
+    jobs: usize,
+    launcher: &Path,
+    wa_exe: &Path,
+    run_dir: &Path,
+) -> Vec<TestResult> {
+    let (tx, rx) = mpsc::channel::<(usize, TestResult)>();
+    let (work_tx, work_rx) = mpsc::channel::<(usize, TestCase)>();
+    let work_rx = std::sync::Arc::new(std::sync::Mutex::new(work_rx));
+
+    let launcher = launcher.to_path_buf();
+    let wa_exe = wa_exe.to_path_buf();
+    let run_dir = run_dir.to_path_buf();
+
+    let mut handles = Vec::new();
+    for _ in 0..jobs.min(tests.len()) {
+        let rx = work_rx.clone();
+        let tx = tx.clone();
+        let launcher = launcher.clone();
+        let wa_exe = wa_exe.clone();
+        let run_dir = run_dir.clone();
+
+        handles.push(thread::spawn(move || loop {
+            let item = {
+                let guard = rx.lock().unwrap();
+                guard.recv()
+            };
+            match item {
+                Ok((idx, test)) => {
+                    let result = run_baseline_test(&test, &launcher, &wa_exe, &run_dir);
+                    let _ = tx.send((idx, result));
+                }
+                Err(_) => break,
+            }
+        }));
+    }
+    drop(tx);
+
+    for (idx, test) in tests.into_iter().enumerate() {
+        if idx > 0 {
+            thread::sleep(Duration::from_millis(50));
+        }
+        let _ = work_tx.send((idx, test));
+    }
+    drop(work_tx);
+
+    let mut results: Vec<Option<TestResult>> = vec![None; handles.len() + 700];
+    let mut count = 0;
+    for (idx, result) in rx {
+        if idx >= results.len() {
+            results.resize(idx + 1, None);
+        }
+        print_result(&result);
+        results[idx] = Some(result);
+        count += 1;
+    }
+
+    for h in handles {
+        let _ = h.join();
+    }
+
+    results.into_iter().take(count).flatten().collect()
+}
+
+fn run_generate_baseline(args: GenerateBaselineArgs) {
+    let tests = discover_baseline_targets(&args.dir, args.filter.as_deref());
+    if tests.is_empty() {
+        eprintln!(
+            "No replays without baselines found in {}",
+            args.dir.display()
+        );
+        if let Some(f) = &args.filter {
+            eprintln!("  (filter: \"{f}\")");
+        }
+        std::process::exit(1);
+    }
+
+    // Build
+    if !args.no_build {
+        eprint!("Building... ");
+        match build() {
+            Ok(d) => eprintln!("done ({:.1}s)", d.as_secs_f64()),
+            Err(e) => {
+                eprintln!("FAILED: {e}");
+                std::process::exit(1);
+            }
+        }
+    }
+
+    let wa_exe = find_wa_exe(args.wa_path.as_deref()).unwrap_or_else(|| {
+        eprintln!("Cannot find WA.exe. Use --wa-path or set OPENWA_WA_PATH.");
+        std::process::exit(1);
+    });
+    let launcher = find_launcher().unwrap_or_else(|| {
+        eprintln!("Cannot find openwa-launcher.exe. Run cargo build first.");
+        std::process::exit(1);
+    });
+
+    let timestamp = timestamp();
+    let run_dir = PathBuf::from(RUNS_DIR).join(format!("baseline-{timestamp}"));
+    let _ = fs::create_dir_all(&run_dir);
+    let run_dir = strip_unc(fs::canonicalize(&run_dir).unwrap_or(run_dir));
+
+    let jobs = args.jobs.max(1);
+    println!(
+        "Generating baselines for {} replay{} ({} concurrent, OPENWA_TRACE_BASELINE=1)...\n",
+        tests.len(),
+        if tests.len() == 1 { "" } else { "s" },
+        jobs
+    );
+
+    let wall_start = Instant::now();
+    let results = run_baseline_parallel(tests, jobs, &launcher, &wa_exe, &run_dir);
+    let wall_time = wall_start.elapsed();
+
+    let generated = results.iter().filter(|r| r.passed).count();
+    let failed = results.len() - generated;
+
+    print_summary(&results, wall_time);
+    write_summary(&results, wall_time, &run_dir.join("summary.txt"));
+
+    println!(
+        "\nBaseline generation: {} generated, {} failed",
+        generated, failed
+    );
+
+    cleanup_temp_files(&wa_exe);
+
+    if failed > 0 {
+        std::process::exit(1);
+    }
+}
+
 // ─── Main ───────────────────────────────────────────────────────────────────
 
 fn main() {
@@ -1330,15 +1707,20 @@ fn main() {
             run_trace_desync(sub_args);
             return;
         }
+        Some("generate-baseline") => {
+            let sub_args = parse_generate_baseline_args(&argv[1..]);
+            run_generate_baseline(sub_args);
+            return;
+        }
         _ => {}
     }
 
     let args = parse_args();
 
     // Discover tests
-    let tests = discover_tests(args.filter.as_deref());
+    let tests = discover_tests(&args.dir, args.filter.as_deref());
     if tests.is_empty() {
-        eprintln!("No replay tests found in {REPLAYS_DIR}/");
+        eprintln!("No replay tests found in {}/", args.dir.display());
         if let Some(f) = &args.filter {
             eprintln!("  (filter: \"{f}\")");
         }