fix: eliminate replay test flakiness under high concurrency

Three independent concurrency bugs caused 10-15% failure rates when
running replay tests at -j 4+ (and ~13% at -j 16 for the worms2d suite):

1. Exclusive file locking: WA.exe opens data files (.img, .wav, .bmp)
with dwShareMode=0. Force FILE_SHARE_READ on all CreateFileA calls
in headless mode so concurrent instances can read shared game assets.

2. _findfirst directory contention: WA's file-existence check for
steam.dat uses _findfirst64i32 which does FindFirstFileA directory
enumeration. Under high concurrency, NTFS directory contention from
.openwa_tmp operations causes transient failures. Hook the check
function (0x4DFA30) to always return "exists" in headless mode.

3. thm.prv race condition: MAP_VIEW_LOAD creates, reads, and deletes
a temporary thm.prv file during terrain processing. Concurrent
instances fighting over this file caused heap corruption (crash in
ntdll RtlAllocateHeap). Extend file isolation to hook FindFirstFileA
and DeleteFileA in addition to CreateFileA, so all thm.prv operations
are redirected to per-PID temp directories.

Also adds custom.dat to isolated root files, removes the now-unnecessary
50ms inter-test launch stagger, and updates worms2d suite docs to
reflect that -j 16 is now reliable (~40s for 597 tests).

Verified: 0/750 failures at -j 16 (base suite), 0/597 failures at
-j 16 (worms2d suite, 3 consecutive runs).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/openwa-core/src/address.rs

@@ -696,6 +696,8 @@ pub mod va {
         fn FRONTEND_LOCAL_MP_CTOR = 0x0049_C420;
         fn FRONTEND_TRAINING_CTOR = 0x004E_0880;
         fn FRONTEND_MISSIONS_CTOR = 0x0049_9190;
+        /// File-existence check via _findfirst (fastcall, ECX=filename)
+        fn/Fastcall FILE_EXISTS_CHECK = 0x004D_FA30;
         fn FRONTEND_POST_INIT_CTOR = 0x004C_91B0;
         fn FRONTEND_MAIN_MENU_CTOR = 0x0048_66C0;
         fn FRONTEND_SINGLE_PLAYER_CTOR = 0x004D_69F0;

crates/openwa-dll/src/replacements/file_isolation.rs

@@ -1,11 +1,11 @@
 //! Per-instance file path isolation for concurrent test execution.
 //!
-//! Hooks `kernel32!CreateFileA` to redirect temp/scratch files to a per-PID
-//! subdirectory, preventing races when multiple WA.exe instances run
-//! simultaneously. Only active when `OPENWA_HEADLESS=1` is set.
+//! Hooks `kernel32!CreateFileA`, `FindFirstFileA`, and `DeleteFileA` to redirect
+//! temp/scratch files to a per-PID subdirectory, preventing races when multiple
+//! WA.exe instances run simultaneously. Only active when `OPENWA_HEADLESS=1`.
 //!
 //! Redirected files:
-//!   Game dir:  writetest.txt, mono.tmp, custom.dat
+//!   Game dir:  writetest.txt, mono.tmp, custom.dat, thm.prv
 //!   DATA\:     land.dat, landgen.svg, current.thm, playback.thm
 //!   ERRORLOG:  ERRORLOG.TXT → OPENWA_ERRORLOG_PATH (if set)
 
@@ -30,7 +30,14 @@ type CreateFileAFn = unsafe extern "system" fn(
     hTemplateFile: HANDLE,
 ) -> HANDLE;
 
+type FindFirstFileAFn =
+    unsafe extern "system" fn(lpFileName: *const u8, lpFindFileData: *mut u8) -> HANDLE;
+
+type DeleteFileAFn = unsafe extern "system" fn(lpFileName: *const u8) -> i32;
+
 static ORIG_CREATE_FILE_A: AtomicU32 = AtomicU32::new(0);
+static ORIG_FIND_FIRST_FILE_A: AtomicU32 = AtomicU32::new(0);
+static ORIG_DELETE_FILE_A: AtomicU32 = AtomicU32::new(0);
 
 static ERRORLOG_PATH: OnceLock<Option<String>> = OnceLock::new();
 static TEMP_DIR: OnceLock<Option<String>> = OnceLock::new();
@@ -58,7 +65,9 @@ fn temp_dir() -> Option<&'static str> {
 }
 
 /// Files in the game root directory that need isolation.
-const ROOT_FILES: &[&str] = &["writetest.txt", "mono.tmp"];
+/// `thm.prv` is a temporary file created/read/deleted by MAP_VIEW_LOAD during
+/// terrain processing — concurrent instances fight over it without isolation.
+const ROOT_FILES: &[&str] = &["writetest.txt", "mono.tmp", "custom.dat", "thm.prv"];
 
 /// Files in the DATA subdirectory that need isolation.
 const DATA_FILES: &[&str] = &["land.dat", "landgen.svg", "current.thm", "playback.thm"];
@@ -80,7 +89,7 @@ fn redirect_path(path: &str) -> Option<String> {
 
     let tmp = temp_dir()?;
 
-    // Check root-level files (writetest.txt, mono.tmp, custom.dat)
+    // Check root-level files
     for &name in ROOT_FILES {
         if lower.ends_with(&format!("\\{name}")) || lower == name {
             return Some(format!("{tmp}\\{name}"));
@@ -99,6 +108,11 @@ fn redirect_path(path: &str) -> Option<String> {
     None
 }
 
+// ─── CreateFileA hook ──────────────────────────────────────────────────────
+
+/// WA.exe opens data files with `dwShareMode=0` (exclusive access), which blocks
+/// concurrent instances from reading the same files. In headless mode, force
+/// `FILE_SHARE_READ` on all file opens so multiple test instances can coexist.
 unsafe extern "system" fn hook_create_file_a(
     lp_file_name: *const u8,
     desired_access: u32,
@@ -110,6 +124,12 @@ unsafe extern "system" fn hook_create_file_a(
 ) -> HANDLE {
     let orig: CreateFileAFn = core::mem::transmute(ORIG_CREATE_FILE_A.load(Ordering::Relaxed));
 
+    // Force FILE_SHARE_READ on all opens to prevent exclusive locking between
+    // concurrent WA.exe instances. WA opens .img, .wav, .bmp etc. with share=0,
+    // which causes "File Error" failures under high concurrency.
+    const FILE_SHARE_READ: u32 = 0x00000001;
+    let share_mode = share_mode | FILE_SHARE_READ;
+
     if !lp_file_name.is_null() {
         if let Ok(path) = CStr::from_ptr(lp_file_name as *const i8).to_str() {
             if let Some(new_path) = redirect_path(path) {
@@ -138,6 +158,63 @@ unsafe extern "system" fn hook_create_file_a(
     )
 }
 
+// ─── FindFirstFileA hook ───────────────────────────────────────────────────
+
+/// Redirect FindFirstFileA for isolated files. MAP_VIEW_LOAD's file reader
+/// (FUN_004dfa70) uses FindFirstFileA to get the file size before _fopen.
+/// Without this hook, it looks in the game directory instead of the per-PID
+/// temp directory where CreateFileA wrote the file.
+unsafe extern "system" fn hook_find_first_file_a(
+    lp_file_name: *const u8,
+    lp_find_file_data: *mut u8,
+) -> HANDLE {
+    let orig: FindFirstFileAFn =
+        core::mem::transmute(ORIG_FIND_FIRST_FILE_A.load(Ordering::Relaxed));
+
+    if !lp_file_name.is_null() {
+        if let Ok(path) = CStr::from_ptr(lp_file_name as *const i8).to_str() {
+            if let Some(new_path) = redirect_path(path) {
+                let cstr: Vec<u8> = new_path.bytes().chain(std::iter::once(0)).collect();
+                return orig(cstr.as_ptr(), lp_find_file_data);
+            }
+        }
+    }
+
+    orig(lp_file_name, lp_find_file_data)
+}
+
+// ─── DeleteFileA hook ──────────────────────────────────────────────────────
+
+/// Redirect DeleteFileA for isolated files. MAP_VIEW_LOAD deletes the temporary
+/// `thm.prv` file after reading it. Without this hook, the delete targets the
+/// game directory while the file is in the per-PID temp directory.
+unsafe extern "system" fn hook_delete_file_a(lp_file_name: *const u8) -> i32 {
+    let orig: DeleteFileAFn = core::mem::transmute(ORIG_DELETE_FILE_A.load(Ordering::Relaxed));
+
+    if !lp_file_name.is_null() {
+        if let Ok(path) = CStr::from_ptr(lp_file_name as *const i8).to_str() {
+            if let Some(new_path) = redirect_path(path) {
+                let cstr: Vec<u8> = new_path.bytes().chain(std::iter::once(0)).collect();
+                return orig(cstr.as_ptr());
+            }
+        }
+    }
+
+    orig(lp_file_name)
+}
+
+// ─── File-exists check hook ────────────────────────────────────────────────
+
+/// WA's file-existence check (0x4DFA30) uses `_findfirst` which does a directory
+/// enumeration. Under high concurrency, NTFS directory contention causes transient
+/// failures. Replace with a simple "always exists" in headless mode — the checked
+/// files (steam.dat, graphics\Font.bmp) are guaranteed present in the game dir.
+unsafe extern "fastcall" fn hook_file_exists_check(_filename: *const u8) -> u32 {
+    1 // always report file as existing
+}
+
+// ─── Installation ──────────────────────────────────────────────────────────
+
 pub fn install() -> Result<(), String> {
     let is_test =
         std::env::var("OPENWA_HEADLESS").is_ok() || std::env::var("OPENWA_REPLAY_TEST").is_ok();
@@ -147,25 +224,43 @@ pub fn install() -> Result<(), String> {
     }
 
     unsafe {
-        let module = windows_sys::Win32::System::LibraryLoader::GetModuleHandleA(
+        let k32 = windows_sys::Win32::System::LibraryLoader::GetModuleHandleA(
             c"kernel32.dll".as_ptr().cast(),
         );
-        if module.is_null() {
+        if k32.is_null() {
             return Err("kernel32.dll not loaded".to_string());
         }
 
-        let proc = windows_sys::Win32::System::LibraryLoader::GetProcAddress(
-            module,
-            c"CreateFileA".as_ptr().cast(),
-        );
-        let addr = proc.ok_or("CreateFileA not found in kernel32.dll")?;
-        let target = addr as *mut c_void;
-        let trampoline = minhook::MinHook::create_hook(target, hook_create_file_a as *mut c_void)
-            .map_err(|e| format!("MinHook create_hook failed for CreateFileA: {e}"))?;
-        minhook::MinHook::queue_enable_hook(target)
-            .map_err(|e| format!("MinHook queue_enable_hook failed for CreateFileA: {e}"))?;
+        // Hook CreateFileA — path redirection + FILE_SHARE_READ forcing
+        hook_kernel32_fn(
+            k32,
+            c"CreateFileA",
+            hook_create_file_a as *mut c_void,
+            &ORIG_CREATE_FILE_A,
+        )?;
 
-        ORIG_CREATE_FILE_A.store(trampoline as u32, Ordering::Relaxed);
+        // Hook FindFirstFileA — path redirection for file size lookups
+        hook_kernel32_fn(
+            k32,
+            c"FindFirstFileA",
+            hook_find_first_file_a as *mut c_void,
+            &ORIG_FIND_FIRST_FILE_A,
+        )?;
+
+        // Hook DeleteFileA — path redirection for temp file cleanup
+        hook_kernel32_fn(
+            k32,
+            c"DeleteFileA",
+            hook_delete_file_a as *mut c_void,
+            &ORIG_DELETE_FILE_A,
+        )?;
+
+        // Hook file-exists check to avoid _findfirst contention under concurrency
+        crate::hook::install(
+            "FileExistsCheck",
+            openwa_core::address::va::FILE_EXISTS_CHECK,
+            hook_file_exists_check as *const (),
+        )?;
 
         let pid = std::process::id();
         let tmp_msg = temp_dir().map(|d| format!(" → {d}")).unwrap_or_default();
@@ -173,13 +268,35 @@ pub fn install() -> Result<(), String> {
             .map(|t| format!(", ERRORLOG.TXT → {t}"))
             .unwrap_or_default();
         let _ = log_line(&format!(
-            "[FileIsolation] Hooked CreateFileA (pid={pid}): temp files{tmp_msg}{errorlog_msg}"
+            "[FileIsolation] Hooked CreateFileA+FindFirstFileA+DeleteFileA (pid={pid}): temp files{tmp_msg}{errorlog_msg}"
         ));
     }
 
     Ok(())
 }
 
+/// Helper: hook a kernel32 function via MinHook.
+unsafe fn hook_kernel32_fn(
+    module: *mut c_void,
+    name: &core::ffi::CStr,
+    hook_fn: *mut c_void,
+    orig_store: &AtomicU32,
+) -> Result<(), String> {
+    let fn_name = name.to_str().unwrap_or("?");
+    let proc = windows_sys::Win32::System::LibraryLoader::GetProcAddress(
+        module as _,
+        name.as_ptr().cast(),
+    );
+    let addr = proc.ok_or(format!("{fn_name} not found in kernel32.dll"))?;
+    let target = addr as *mut c_void;
+    let trampoline = minhook::MinHook::create_hook(target, hook_fn)
+        .map_err(|e| format!("MinHook create_hook failed for {fn_name}: {e}"))?;
+    minhook::MinHook::queue_enable_hook(target)
+        .map_err(|e| format!("MinHook queue_enable_hook failed for {fn_name}: {e}"))?;
+    orig_store.store(trampoline as u32, Ordering::Relaxed);
+    Ok(())
+}
+
 /// Clean up the per-PID temp directory. Called during DLL detach or test cleanup.
 pub fn cleanup() {
     if let Some(tmp) = TEMP_DIR.get().and_then(|o| o.as_deref()) {

crates/openwa-test-runner/CLAUDE.md

@@ -45,10 +45,10 @@ A headful test passes when: no crash, no `[PANIC]` in OpenWA.log, no `[GAMEPLAY
 
 ## Worms 2D Speedrun Suite
 
-~598 replays in `testdata/replays/worms2d/` (`.wagame` extension). Must run with `-j 1` or `-j 2` — higher concurrency causes ~5-7% flakes from shared resource contention. Full suite takes ~3-4 min at `-j 1`. Use for final validation only.
+~598 replays in `testdata/replays/worms2d/` (`.wagame` extension). Supports high concurrency (`-j 16` or higher). Full suite takes ~40s at `-j 16`. Use for final validation after complex changes.
 
 ```bash
-openwa-test -d testdata/replays/worms2d -j 1
+openwa-test -d testdata/replays/worms2d -j 16
 ```
 
 ## Key Paths

crates/openwa-test-runner/src/main.rs

@@ -427,12 +427,7 @@ fn run_tests_parallel(
     }
     drop(tx); // Drop our sender so rx closes when all workers finish
 
-    // Small stagger between launches to reduce startup races on shared
-    // resources (thm.prv, custom.dat) that can't be safely redirected.
     for (idx, test) in tests.into_iter().enumerate() {
-        if idx > 0 {
-            thread::sleep(Duration::from_millis(50));
-        }
         let _ = work_tx.send((idx, test));
     }
     drop(work_tx); // Signal no more work
@@ -1597,9 +1592,6 @@ fn run_baseline_parallel(
     drop(tx);
 
     for (idx, test) in tests.into_iter().enumerate() {
-        if idx > 0 {
-            thread::sleep(Duration::from_millis(50));
-        }
         let _ = work_tx.send((idx, test));
     }
     drop(work_tx);