# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import subprocess
import os
import glob
import datetime
from pathlib import Path
import time
import argparse

def setup_logging_dir():
    """Create a timestamped directory for test logs and update latest symlink"""
    timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
    log_dir = Path(f"test_log_{timestamp}")
    log_dir.mkdir(exist_ok=True)

    # Check and handle the 'latest' symlink
    latest_link = Path("test_log_latest")
    if latest_link.exists():
        if not latest_link.is_symlink():
            raise RuntimeError(
                "test_log_latest exists but is not a symlink. "
                "Please remove it manually to proceed."
            )
        latest_link.unlink()  # Remove existing symlink

    # Create new symlink pointing to the new log directory
    latest_link.symlink_to(log_dir, target_is_directory=True)

    return log_dir

def get_cpp_test_executables(build_dir):
    """Find all test executables in the build directory"""
    all_tests = glob.glob(os.path.join(build_dir, "*test*"))

    # Separate multidevice and single device tests
    multidevice_tests, single_device_tests = [], []
    for test in all_tests:
        (single_device_tests, multidevice_tests)[
            "multidevice" in os.path.basename(test).lower()
        ].append(test)

    return multidevice_tests, single_device_tests

def get_python_tests(python_test_dir):
    """Find all Python test files"""
    all_tests = glob.glob(os.path.join(python_test_dir, "test_*.py"))

    # Separate multidevice and single device tests
    # This is not catching all python multidevice tests like test_communication.py
    # TODO: Change test names to separate out multidevice tests, or update to support all multidevice python tests
    multidevice_tests, single_device_tests = [], []
    for test in all_tests:
        (single_device_tests, multidevice_tests)[
            "multidevice" in os.path.basename(test).lower()
        ].append(test)

    return multidevice_tests, single_device_tests

def get_test_timeout(test_name):
    """Return timeout in seconds for a given test"""
    if test_name in ["test_nvfuser", "test_matmul", "test_direct_ops.py"]:
        return 3600  # 1 hour
    return 600  # 10 minutes

def run_multidevice_test(test_path, log_dir, num_gpus, dry_run=False):
    """Run a multidevice test using mpirun"""
    test_name = os.path.basename(test_path)
    log_base = log_dir / test_name
    timeout = get_test_timeout(test_name)

    cmd = [
        "mpirun",
        "-np",
        str(num_gpus),  # Use available GPU count
        "--output-filename",
        f"{log_base}",
        "--merge-stderr-to-stdout",
        test_path,
    ]

    if dry_run:
        print(f"Would run: {' '.join(cmd)} (timeout: {timeout / 60} minutes)")
        return True

    print(f"Running multidevice test: {test_name}")
    try:
        # Redirect output to /dev/null to suppress console output
        with open(os.devnull, "w") as devnull:
            result = subprocess.run(
                cmd, timeout=timeout, stdout=devnull, stderr=subprocess.STDOUT
            )
        return result.returncode == 0
    except subprocess.TimeoutExpired:
        with open(f"{log_base}.log", "w") as f:
            f.write(f"Test: {test_name}\n")
            f.write(f"ERROR: Test timed out after {timeout / 60} minutes\n")
        return False
    except Exception as e:
        with open(f"{log_base}.log", "w") as f:
            f.write(f"Test: {test_name}\n")
            f.write(f"ERROR: Failed to run test: {str(e)}\n")
        return False

def run_single_device_test(test_path, log_dir, gpu_id, dry_run=False):
    """Run a single device test on specified GPU"""
    test_name = os.path.basename(test_path)
    log_file = log_dir / f"{test_name}.log"

    cmd = [test_path]
    env = os.environ.copy()
    env["CUDA_VISIBLE_DEVICES"] = str(gpu_id)

    if dry_run:
        print(f"Would run: CUDA_VISIBLE_DEVICES={gpu_id} {' '.join(cmd)} > {log_file}")
        return "dry_run_process"

    print(f"Running test: {test_name} on GPU {gpu_id}")
    try:
        with open(log_file, "w") as f:
            f.write(f"Test: {test_name} on GPU {gpu_id}\n")
            process = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT, env=env)
        return process
    except Exception as e:
        with open(log_file, "w") as f:
            f.write(f"Test: {test_name} on GPU {gpu_id}\n")
            f.write(f"ERROR: Failed to start test: {str(e)}\n")
        return None

def run_python_multidevice_test(test_path, log_dir, num_gpus, dry_run=False):
    """Run a Python multidevice test using mpirun"""
    test_name = os.path.basename(test_path)
    log_base = log_dir / f"python_{test_name}"

    cmd = [
        "mpirun",
        "-np",
        str(num_gpus),  # Use available GPU count
        "--output-filename",
        f"{log_base}",
        "--merge-stderr-to-stdout",
        "pytest",
        test_path,
        "-v",
    ]

    if dry_run:
        print(f"Would run: {' '.join(cmd)}")
        return True

    print(f"Running Python multidevice test: {test_name}")
    try:
        # Redirect output to /dev/null to suppress console output
        with open(os.devnull, "w") as devnull:
            result = subprocess.run(
                cmd, timeout=1200, stdout=devnull, stderr=subprocess.STDOUT
            )
        return result.returncode == 0
    except subprocess.TimeoutExpired:
        with open(f"{log_base}.log", "w") as f:
            f.write(f"Test: {test_name}\n")
            f.write("ERROR: Test timed out after 20 minutes\n")
        return False
    except Exception as e:
        with open(f"{log_base}.log", "w") as f:
            f.write(f"Test: {test_name}\n")
            f.write(f"ERROR: Failed to run test: {str(e)}\n")
        return False

def run_python_test(test_path, log_dir, gpu_id, dry_run=False):
    """Run a Python test using pytest on specified GPU"""
    test_name = os.path.basename(test_path)
    log_file = log_dir / f"python_{test_name}.log"

    cmd = ["pytest", test_path, "-v"]
    env = os.environ.copy()
    env["CUDA_VISIBLE_DEVICES"] = str(gpu_id)

    if dry_run:
        print(f"Would run: CUDA_VISIBLE_DEVICES={gpu_id} {' '.join(cmd)} > {log_file}")
        return "dry_run_process"

    print(f"Running Python test: {test_name} on GPU {gpu_id}")
    try:
        with open(log_file, "w") as f:
            f.write(f"Test: {test_name} on GPU {gpu_id}\n")
            process = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT, env=env)
        return process
    except Exception as e:
        with open(log_file, "w") as f:
            f.write(f"Test: {test_name} on GPU {gpu_id}\n")
            f.write(f"ERROR: Failed to start test: {str(e)}\n")
        return None

def run_parallel_tests(log_dir, num_gpus, test_infos, dry_run=False):
    """Run tests in parallel across available GPUs using provided test configurations"""
    if dry_run:
        current_tests = {i: None for i in range(num_gpus)}  # Simulate GPU allocation
        test_queue = test_infos.copy()

        print(f"\nSimulating parallel execution across {num_gpus} GPUs:")

        # Initial allocation
        for gpu_id in range(num_gpus):
            if test_queue:
                test_path, run_func = test_queue.pop(0)
                current_tests[gpu_id] = (test_path, run_func)
                run_func(test_path, log_dir, gpu_id, dry_run=True)

        # Simulate rest of queue
        while test_queue:
            # Simulate round-robin GPU completion
            gpu_id = len(test_queue) % num_gpus
            current_test_path, _ = current_tests[gpu_id]
            print(f"\nGPU {gpu_id} finished {os.path.basename(current_test_path)}")
            test_path, run_func = test_queue.pop(0)
            current_tests[gpu_id] = (test_path, run_func)
            run_func(test_path, log_dir, gpu_id, dry_run=True)

        # Show final tests completing
        for gpu_id in range(num_gpus):
            if current_tests[gpu_id]:
                test_path, _ = current_tests[gpu_id]
                print(f"\nGPU {gpu_id} finished {os.path.basename(test_path)}")

        return [], []

    # Initialize test queue and tracking variables
    test_queue = test_infos.copy()
    current_processes = {i: None for i in range(num_gpus)}
    current_tests = {i: None for i in range(num_gpus)}
    start_times = {i: time.time() for i in range(num_gpus)}
    completed_tests = []
    failed_tests = []

    def start_test(test_info, gpu_id):
        """Start a test on specified GPU"""
        test_path, run_func = test_info
        current_tests[gpu_id] = test_info
        process = run_func(test_path, log_dir, gpu_id)

        if process is None:
            failed_name = os.path.basename(test_path)
            if run_func == run_python_test:
                failed_name = f"python_{failed_name}"
            failed_tests.append(failed_name)
            return False

        current_processes[gpu_id] = process
        start_times[gpu_id] = time.time()
        print(f"Started {os.path.basename(test_path)} on GPU {gpu_id}")
        return True

    def check_completion():
        """Check if any running tests have completed and return list of free GPUs"""
        free_gpus = []
        for gpu_id in range(num_gpus):
            if current_processes[gpu_id] is None:
                free_gpus.append(gpu_id)
                continue

            # Get timeout for current test
            test_path, run_func = current_tests[gpu_id]
            test_name = os.path.basename(test_path)
            timeout = get_test_timeout(test_name)

            # Check for timeout
            if time.time() - start_times[gpu_id] > timeout:
                print(
                    f"Test {test_name} on GPU {gpu_id} timed out after {timeout / 60} minutes"
                )
                current_processes[gpu_id].kill()
                log_file = log_dir / f"{test_name}.log"
                if run_func == run_python_test:
                    log_file = log_dir / f"python_{test_name}.log"
                with open(log_file, "a") as f:
                    f.write(f"\nERROR: Test timed out after {timeout / 60} minutes\n")

                failed_name = test_name
                if run_func == run_python_test:
                    failed_name = f"python_{failed_name}"
                failed_tests.append(failed_name)
                current_processes[gpu_id] = None
                current_tests[gpu_id] = None
                free_gpus.append(gpu_id)
                continue

            # Check if process completed
            if current_processes[gpu_id].poll() is not None:
                test_path, run_func = current_tests[gpu_id]
                test_name = os.path.basename(test_path)
                success = current_processes[gpu_id].returncode == 0
                print(
                    f"Completed {test_name} on GPU {gpu_id}: {'Success' if success else 'Failed'}"
                )

                # Append completion status to log file
                log_file = log_dir / f"{test_name}.log"
                if run_func == run_python_test:
                    log_file = log_dir / f"python_{test_name}.log"
                with open(log_file, "a") as f:
                    f.write(
                        f"\nTest completed with {'success' if success else 'failure'}\n"
                    )
                    f.write(f"Return code: {current_processes[gpu_id].returncode}\n")

                if success:
                    completed_tests.append(test_path)
                else:
                    failed_name = test_name
                    if run_func == run_python_test:
                        failed_name = f"python_{failed_name}"
                    failed_tests.append(failed_name)

                current_processes[gpu_id] = None
                current_tests[gpu_id] = None
                free_gpus.append(gpu_id)

        return free_gpus

    # Initial test distribution
    for gpu_id in range(num_gpus):
        if test_queue and current_processes[gpu_id] is None:
            start_test(test_queue.pop(0), gpu_id)

    # Main loop
    try:
        while test_queue or any(p is not None for p in current_processes.values()):
            # Check for completed tests
            free_gpus = check_completion()

            # Start new tests on free GPUs
            for gpu_id in free_gpus:
                if test_queue:  # If there are tests waiting to be run
                    start_test(test_queue.pop(0), gpu_id)

            # Small sleep to prevent busy waiting
            time.sleep(1)

        return completed_tests, failed_tests

    finally:
        # Ensure all processes are cleaned up
        for process in current_processes.values():
            if process is not None:
                process.kill()

def collect_test_failures(log_dir, dry_run=False):
    """Scan log files for test failures and collect context"""
    if dry_run:
        print(
            "\nWould scan log files for failures and create failure_summary.txt with:"
        )
        print("- 5 lines of context before each failure")
        print("- 20 lines of context after each failure")
        print("- Coverage for both GTest and Pytest failures")
        print("- Timeout failure information")
        return False

    failure_summary = []

    for log_file in log_dir.glob("*.log"):
        with open(log_file, "r") as f:
            lines = f.readlines()

        for i, line in enumerate(lines):
            # Look for GTest failures between square brackets
            if "[" in line and "]" in line:
                result_part = line[line.find("[") : line.find("]") + 1]
                if "FAILED" in result_part or "TIMEOUT" in result_part:
                    failure_summary.append(
                        f"\n=== GTest Failure in {log_file.name} ==="
                    )
                    # Collect 5 lines before and 20 lines after the failure
                    start = max(0, i - 5)
                    end = min(len(lines), i + 21)
                    failure_summary.extend(lines[start:end])
                continue  # Skip other checks for this line if it was a bracketed result

            # Look for pytest failures
            if "FAILED" in line and "test" in line.lower():
                failure_summary.append(f"\n=== Pytest Failure in {log_file.name} ===")
                # Collect 5 lines before and 20 lines after the failure
                start = max(0, i - 5)
                end = min(len(lines), i + 21)
                failure_summary.extend(lines[start:end])

            # Look for timeout failures
            if "ERROR: Test timed out" in line:
                failure_summary.append(f"\n=== Timeout in {log_file.name} ===")
                failure_summary.append(line)

    # Write failure summary if there were any failures
    if failure_summary:
        with open(log_dir / "failure_summary.txt", "w") as f:
            f.write("=== Test Failure Summary ===\n")
            f.write("".join(failure_summary))

        return True
    return False

def get_available_gpus():
    """Check how many NVIDIA GPUs are available"""
    try:
        # Run nvidia-smi -L | wc -l to get GPU count
        output = subprocess.check_output("nvidia-smi -L | wc -l", shell=True, text=True)
        return int(output.strip())
    except (subprocess.SubprocessError, ValueError):
        print("Warning: Could not query GPU count using nvidia-smi")
        return 0

def main():
    parser = argparse.ArgumentParser(description="Run nvFuser tests")
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Show what tests would be run without executing them",
    )
    args = parser.parse_args()

    # Check GPU availability
    gpu_count = get_available_gpus()
    if gpu_count < 1:
        print("Error: No GPUs found for testing")
        return 1

    print(f"Found {gpu_count} GPUs available for testing")

    # Hardcode paths relative to current directory
    build_dir = "bin"
    python_test_dir = "tests/python"

    log_dir = setup_logging_dir()
    multidevice_tests, single_device_tests = get_cpp_test_executables(build_dir)
    # This is not catching all python multidevice tests like test_communication.py
    # TODO: Change test names to separate out multidevice tests, or update to support all multidevice python tests
    python_multidevice_tests, python_single_tests = get_python_tests(python_test_dir)

    if not (multidevice_tests or single_device_tests) and not (
        python_multidevice_tests or python_single_tests
    ):
        print("No tests found!")
        return 0

    print(f"Found {len(multidevice_tests)} C++ multidevice tests")
    print(f"Found {len(python_multidevice_tests)} Python multidevice tests")
    print(f"Found {len(single_device_tests)} C++ single device tests")
    print(f"Found {len(python_single_tests)} Python single device tests")
    print(f"Logs will be written to: {log_dir}")

    success_count = 0
    failed_tests = []
    total_tests = (
        len(multidevice_tests)
        + len(single_device_tests)
        + len(python_multidevice_tests)
        + len(python_single_tests)
    )

    # Run all multidevice tests first
    print("\nC++ multidevice tests:")
    for test in multidevice_tests:
        if run_multidevice_test(test, log_dir, gpu_count, dry_run=args.dry_run):
            if not args.dry_run:
                success_count += 1
        else:
            if not args.dry_run:
                failed_tests.append(os.path.basename(test))

    print("\nPython multidevice tests:")
    for test in python_multidevice_tests:
        if run_python_multidevice_test(test, log_dir, gpu_count, dry_run=args.dry_run):
            if not args.dry_run:
                success_count += 1
        else:
            if not args.dry_run:
                failed_tests.append(f"python_{os.path.basename(test)}")

    # Run all single device tests (C++ and Python) in parallel
    print("\nRunning all single device tests in parallel:")

    # Create initial unordered test list
    test_infos = [(t, run_single_device_test) for t in single_device_tests] + [
        (t, run_python_test) for t in python_single_tests
    ]

    # Find and prioritize specific tests by name match in full path
    priority_names = ["test_nvfuser", "test_matmul", "test_ops.py"]
    priority_tests = []
    other_tests = []

    for test_info in test_infos:
        test_path, _ = test_info
        if any(name in test_path for name in priority_names):
            priority_tests.append(test_info)
        else:
            other_tests.append(test_info)

    test_infos = priority_tests + other_tests

    if not args.dry_run:
        completed, failed = run_parallel_tests(log_dir, gpu_count, test_infos)
        success_count += len(completed)
        failed_tests.extend(failed)
    else:
        run_parallel_tests(log_dir, gpu_count, test_infos, dry_run=True)

    if args.dry_run:
        # Show what would be written to summary.txt
        print("\nWould create summary.txt with:")
        print(f"Total tests: {total_tests}")
        print(f"Multidevice tests: {len(multidevice_tests)}")
        print(f"Multidevice python tests: {len(python_multidevice_tests)}")
        print(f"C++ single device tests: {len(single_device_tests)}")
        print(f"Python single device tests: {len(python_single_tests)}")

        # Show failure collection simulation
        collect_test_failures(log_dir, dry_run=True)

        print("\nThis was a dry run. No tests were actually executed.")
        return 0

    # Write summary
    with open(log_dir / "summary.txt", "w") as f:
        f.write(f"Total tests: {total_tests}\n")
        f.write(f"Multidevice tests: {len(multidevice_tests)}\
# ... truncated ...


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import subprocess
import os
import glob
import datetime
from pathlib import Path
import time
import argparse

def setup_logging_dir():
    """Create a timestamped directory for test logs and update latest symlink"""
    timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
    log_dir = Path(f"test_log_{timestamp}")
    log_dir.mkdir(exist_ok=True)

    # Check and handle the 'latest' symlink
    latest_link = Path("test_log_latest")
    if latest_link.exists():
        if not latest_link.is_symlink():
            raise RuntimeError(
                "test_log_latest exists but is not a symlink. "
                "Please remove it manually to proceed."
            )
        latest_link.unlink()  # Remove existing symlink

    # Create new symlink pointing to the new log directory
    latest_link.symlink_to(log_dir, target_is_directory=True)

    return log_dir

def get_cpp_test_executables(build_dir):
    """Find all test executables in the build directory"""
    all_tests = glob.glob(os.path.join(build_dir, "*test*"))

    # Separate multidevice and single device tests
    multidevice_tests, single_device_tests = [], []
    for test in all_tests:
        (single_device_tests, multidevice_tests)[
            "multidevice" in os.path.basename(test).lower()
        ].append(test)

    return multidevice_tests, single_device_tests

def get_python_tests(python_test_dir):
    """Find all Python test files"""
    all_tests = glob.glob(os.path.join(python_test_dir, "test_*.py"))


    timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
    log_dir = Path(f"test_log_{timestamp}")
    log_dir.mkdir(exist_ok=True)

    # Check and handle the 'latest' symlink
    latest_link = Path("test_log_latest")
    if latest_link.exists():
        if not latest_link.is_symlink():
            raise RuntimeError(
                "test_log_latest exists but is not a symlink. "
                "Please remove it manually to proceed."
            )
        latest_link.unlink()  # Remove existing symlink

    # Create new symlink pointing to the new log directory
    latest_link.symlink_to(log_dir, target_is_directory=True)

    return log_dir

def get_cpp_test_executables(build_dir):
    """Find all test executables in the build directory"""
    all_tests = glob.glob(os.path.join(build_dir, "*test*"))

    # Separate multidevice and single device tests
    multidevice_tests, single_device_tests = [], []
    for test in all_tests:
        (single_device_tests, multidevice_tests)[
            "multidevice" in os.path.basename(test).lower()
        ].append(test)

    return multidevice_tests, single_device_tests

def get_python_tests(python_test_dir):
    """Find all Python test files"""
    all_tests = glob.glob(os.path.join(python_test_dir, "test_*.py"))

    # Separate multidevice and single device tests
    # This is not catching all python multidevice tests like test_communication.py
    # TODO: Change test names to separate out multidevice tests, or update to support all multidevice python tests
    multidevice_tests, single_device_tests = [], []
    for test in all_tests:
        (single_device_tests, multidevice_tests)[
            "multidevice" in os.path.basename(test).lower()
        ].append(test)

    return multidevice_tests, single_device_tests

def get_test_timeout(test_name):
    """Return timeout in seconds for a given test"""


    # Create new symlink pointing to the new log directory
    latest_link.symlink_to(log_dir, target_is_directory=True)

    return log_dir

def get_cpp_test_executables(build_dir):
    """Find all test executables in the build directory"""
    all_tests = glob.glob(os.path.join(build_dir, "*test*"))

    # Separate multidevice and single device tests
    multidevice_tests, single_device_tests = [], []
    for test in all_tests:
        (single_device_tests, multidevice_tests)[
            "multidevice" in os.path.basename(test).lower()
        ].append(test)

    return multidevice_tests, single_device_tests

def get_python_tests(python_test_dir):
    """Find all Python test files"""
    all_tests = glob.glob(os.path.join(python_test_dir, "test_*.py"))

    # Separate multidevice and single device tests
    # This is not catching all python multidevice tests like test_communication.py
    # TODO: Change test names to separate out multidevice tests, or update to support all multidevice python tests
    multidevice_tests, single_device_tests = [], []
    for test in all_tests:
        (single_device_tests, multidevice_tests)[
            "multidevice" in os.path.basename(test).lower()
        ].append(test)

    return multidevice_tests, single_device_tests

def get_test_timeout(test_name):
    """Return timeout in seconds for a given test"""
    if test_name in ["test_nvfuser", "test_matmul", "test_direct_ops.py"]:
        return 3600  # 1 hour
    return 600  # 10 minutes

def run_multidevice_test(test_path, log_dir, num_gpus, dry_run=False):
    """Run a multidevice test using mpirun"""
    test_name = os.path.basename(test_path)
    log_base = log_dir / test_name
    timeout = get_test_timeout(test_name)

    cmd = [
        "mpirun",
        "-np",
        str(num_gpus),  # Use available GPU count
        "--output-filename",


    return multidevice_tests, single_device_tests

def get_python_tests(python_test_dir):
    """Find all Python test files"""
    all_tests = glob.glob(os.path.join(python_test_dir, "test_*.py"))

    # Separate multidevice and single device tests
    # This is not catching all python multidevice tests like test_communication.py
    # TODO: Change test names to separate out multidevice tests, or update to support all multidevice python tests
    multidevice_tests, single_device_tests = [], []
    for test in all_tests:
        (single_device_tests, multidevice_tests)[
            "multidevice" in os.path.basename(test).lower()
        ].append(test)

    return multidevice_tests, single_device_tests

def get_test_timeout(test_name):
    """Return timeout in seconds for a given test"""
    if test_name in ["test_nvfuser", "test_matmul", "test_direct_ops.py"]:
        return 3600  # 1 hour
    return 600  # 10 minutes

def run_multidevice_test(test_path, log_dir, num_gpus, dry_run=False):
    """Run a multidevice test using mpirun"""
    test_name = os.path.basename(test_path)
    log_base = log_dir / test_name
    timeout = get_test_timeout(test_name)

    cmd = [
        "mpirun",
        "-np",
        str(num_gpus),  # Use available GPU count
        "--output-filename",
        f"{log_base}",
        "--merge-stderr-to-stdout",
        test_path,
    ]

    if dry_run:
        print(f"Would run: {' '.join(cmd)} (timeout: {timeout / 60} minutes)")


    # Separate multidevice and single device tests
    # This is not catching all python multidevice tests like test_communication.py
    # TODO: Change test names to separate out multidevice tests, or update to support all multidevice python tests
    multidevice_tests, single_device_tests = [], []
    for test in all_tests:
        (single_device_tests, multidevice_tests)[
            "multidevice" in os.path.basename(test).lower()
        ].append(test)

    return multidevice_tests, single_device_tests

def get_test_timeout(test_name):
    """Return timeout in seconds for a given test"""
    if test_name in ["test_nvfuser", "test_matmul", "test_direct_ops.py"]:
        return 3600  # 1 hour
    return 600  # 10 minutes

def run_multidevice_test(test_path, log_dir, num_gpus, dry_run=False):
    """Run a multidevice test using mpirun"""
    test_name = os.path.basename(test_path)
    log_base = log_dir / test_name
    timeout = get_test_timeout(test_name)

    cmd = [
        "mpirun",
        "-np",
        str(num_gpus),  # Use available GPU count
        "--output-filename",
        f"{log_base}",
        "--merge-stderr-to-stdout",
        test_path,
    ]

    if dry_run:
        print(f"Would run: {' '.join(cmd)} (timeout: {timeout / 60} minutes)")
        return True

    print(f"Running multidevice test: {test_name}")
    try:
        # Redirect output to /dev/null to suppress console output
        with open(os.devnull, "w") as devnull:
            result = subprocess.run(
                cmd, timeout=timeout, stdout=devnull, stderr=subprocess.STDOUT
            )
        return result.returncode == 0
    except subprocess.TimeoutExpired:
        with open(f"{log_base}.log", "w") as f:
            f.write(f"Test: {test_name}\n")
            f.write(f"ERROR: Test timed out after {timeout / 60} minutes\n")
        return False
    except Exception as e:
        with open(f"{log_base}.log", "w") as f:
            f.write(f"Test: {test_name}\n")
            f.write(f"ERROR: Failed to run test: {str(e)}\n")
        return False

def run_single_device_test(test_path, log_dir, gpu_id, dry_run=False):
    """Run a single device test on specified GPU"""
    test_name = os.path.basename(test_path)
    log_file = log_dir / f"{test_name}.log"

    cmd = [test_path]
    env = os.environ.copy()
    env["CUDA_VISIBLE_DEVICES"] = str(gpu_id)

    if dry_run:
        print(f"Would run: CUDA_VISIBLE_DEVICES={gpu_id} {' '.join(cmd)} > {log_file}")
        return "dry_run_process"

    print(f"Running test: {test_name} on GPU {gpu_id}")
    try:
        with open(log_file, "w") as f:
            f.write(f"Test: {test_name} on GPU {gpu_id}\n")
            process = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT, env=env)


    print(f"Running multidevice test: {test_name}")
    try:
        # Redirect output to /dev/null to suppress console output
        with open(os.devnull, "w") as devnull:
            result = subprocess.run(
                cmd, timeout=timeout, stdout=devnull, stderr=subprocess.STDOUT
            )
        return result.returncode == 0
    except subprocess.TimeoutExpired:
        with open(f"{log_base}.log", "w") as f:
            f.write(f"Test: {test_name}\n")
            f.write(f"ERROR: Test timed out after {timeout / 60} minutes\n")
        return False
    except Exception as e:
        with open(f"{log_base}.log", "w") as f:
            f.write(f"Test: {test_name}\n")
            f.write(f"ERROR: Failed to run test: {str(e)}\n")
        return False

def run_single_device_test(test_path, log_dir, gpu_id, dry_run=False):
    """Run a single device test on specified GPU"""
    test_name = os.path.basename(test_path)
    log_file = log_dir / f"{test_name}.log"

    cmd = [test_path]
    env = os.environ.copy()
    env["CUDA_VISIBLE_DEVICES"] = str(gpu_id)

    if dry_run:
        print(f"Would run: CUDA_VISIBLE_DEVICES={gpu_id} {' '.join(cmd)} > {log_file}")
        return "dry_run_process"

    print(f"Running test: {test_name} on GPU {gpu_id}")
    try:
        with open(log_file, "w") as f:
            f.write(f"Test: {test_name} on GPU {gpu_id}\n")
            process = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT, env=env)
        return process
    except Exception as e:
        with open(log_file, "w") as f:
            f.write(f"Test: {test_name} on GPU {gpu_id}\n")
            f.write(f"ERROR: Failed to start test: {str(e)}\n")
        return None

def run_python_multidevice_test(test_path, log_dir, num_gpus, dry_run=False):
    """Run a Python multidevice test using mpirun"""
    test_name = os.path.basename(test_path)
    log_base = log_dir / f"python_{test_name}"

    cmd = [
        "mpirun",
        "-np",
        str(num_gpus),  # Use available GPU count
        "--output-filename",
        f"{log_base}",
        "--merge-stderr-to-stdout",
        "pytest",
        test_path,
        "-v",
    ]

    if dry_run:


    env = os.environ.copy()
    env["CUDA_VISIBLE_DEVICES"] = str(gpu_id)

    if dry_run:
        print(f"Would run: CUDA_VISIBLE_DEVICES={gpu_id} {' '.join(cmd)} > {log_file}")
        return "dry_run_process"

    print(f"Running test: {test_name} on GPU {gpu_id}")
    try:
        with open(log_file, "w") as f:
            f.write(f"Test: {test_name} on GPU {gpu_id}\n")
            process = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT, env=env)
        return process
    except Exception as e:
        with open(log_file, "w") as f:
            f.write(f"Test: {test_name} on GPU {gpu_id}\n")
            f.write(f"ERROR: Failed to start test: {str(e)}\n")
        return None

def run_python_multidevice_test(test_path, log_dir, num_gpus, dry_run=False):
    """Run a Python multidevice test using mpirun"""
    test_name = os.path.basename(test_path)
    log_base = log_dir / f"python_{test_name}"

    cmd = [
        "mpirun",
        "-np",
        str(num_gpus),  # Use available GPU count
        "--output-filename",
        f"{log_base}",
        "--merge-stderr-to-stdout",
        "pytest",
        test_path,
        "-v",
    ]

    if dry_run:
        print(f"Would run: {' '.join(cmd)}")
        return True

    print(f"Running Python multidevice test: {test_name}")
    try:
        # Redirect output to /dev/null to suppress console output
        with open(os.devnull, "w") as devnull:
            result = subprocess.run(
                cmd, timeout=1200, stdout=devnull, stderr=subprocess.STDOUT
            )
        return result.returncode == 0
    except subprocess.TimeoutExpired:
        with open(f"{log_base}.log", "w") as f:
            f.write(f"Test: {test_name}\n")
            f.write("ERROR: Test timed out after 20 minutes\n")
        return False
    except Exception as e:
        with open(f"{log_base}.log", "w") as f:
            f.write(f"Test: {test_name}\n")
            f.write(f"ERROR: Failed to run test: {str(e)}\n")
        return False

def run_python_test(test_path, log_dir, gpu_id, dry_run=False):
    """Run a Python test using pytest on specified GPU"""
    test_name = os.path.basename(test_path)
    log_file = log_dir / f"python_{test_name}.log"

    cmd = ["pytest", test_path, "-v"]
    env = os.environ.copy()
    env["CUDA_VISIBLE_DEVICES"] = str(gpu_id)

    if dry_run:
        print(f"Would run: CUDA_VISIBLE_DEVICES={gpu_id} {' '.join(cmd)} > {log_file}")
        return "dry_run_process"

    print(f"Running Python test: {test_name} on GPU {gpu_id}")
    try:
        with open(log_file, "w") as f:
            f.write(f"Test: {test_name} on GPU {gpu_id}\n")
            process = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT, env=env)


    print(f"Running Python multidevice test: {test_name}")
    try:
        # Redirect output to /dev/null to suppress console output
        with open(os.devnull, "w") as devnull:
            result = subprocess.run(
                cmd, timeout=1200, stdout=devnull, stderr=subprocess.STDOUT
            )
        return result.returncode == 0
    except subprocess.TimeoutExpired:
        with open(f"{log_base}.log", "w") as f:
            f.write(f"Test: {test_name}\n")
            f.write("ERROR: Test timed out after 20 minutes\n")
        return False
    except Exception as e:
        with open(f"{log_base}.log", "w") as f:
            f.write(f"Test: {test_name}\n")
            f.write(f"ERROR: Failed to run test: {str(e)}\n")
        return False

def run_python_test(test_path, log_dir, gpu_id, dry_run=False):
    """Run a Python test using pytest on specified GPU"""
    test_name = os.path.basename(test_path)
    log_file = log_dir / f"python_{test_name}.log"

    cmd = ["pytest", test_path, "-v"]
    env = os.environ.copy()
    env["CUDA_VISIBLE_DEVICES"] = str(gpu_id)

    if dry_run:
        print(f"Would run: CUDA_VISIBLE_DEVICES={gpu_id} {' '.join(cmd)} > {log_file}")
        return "dry_run_process"

    print(f"Running Python test: {test_name} on GPU {gpu_id}")
    try:
        with open(log_file, "w") as f:
            f.write(f"Test: {test_name} on GPU {gpu_id}\n")
            process = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT, env=env)
        return process
    except Exception as e:
        with open(log_file, "w") as f:
            f.write(f"Test: {test_name} on GPU {gpu_id}\n")
            f.write(f"ERROR: Failed to start test: {str(e)}\n")
        return None

def run_parallel_tests(log_dir, num_gpus, test_infos, dry_run=False):
    """Run tests in parallel across available GPUs using provided test configurations"""
    if dry_run:
        current_tests = {i: None for i in range(num_gpus)}  # Simulate GPU allocation
        test_queue = test_infos.copy()

        print(f"\nSimulating parallel execution across {num_gpus} GPUs:")

        # Initial allocation
        for gpu_id in range(num_gpus):
            if test_queue:
                test_path, run_func = test_queue.pop(0)
                current_tests[gpu_id] = (test_path, run_func)
                run_func(test_path, log_dir, gpu_id, dry_run=True)

        # Simulate rest of queue
        while test_queue:
            # Simulate round-robin GPU completion


    env = os.environ.copy()
    env["CUDA_VISIBLE_DEVICES"] = str(gpu_id)

    if dry_run:
        print(f"Would run: CUDA_VISIBLE_DEVICES={gpu_id} {' '.join(cmd)} > {log_file}")
        return "dry_run_process"

    print(f"Running Python test: {test_name} on GPU {gpu_id}")
    try:
        with open(log_file, "w") as f:
            f.write(f"Test: {test_name} on GPU {gpu_id}\n")
            process = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT, env=env)
        return process
    except Exception as e:
        with open(log_file, "w") as f:
            f.write(f"Test: {test_name} on GPU {gpu_id}\n")
            f.write(f"ERROR: Failed to start test: {str(e)}\n")
        return None

def run_parallel_tests(log_dir, num_gpus, test_infos, dry_run=False):
    """Run tests in parallel across available GPUs using provided test configurations"""
    if dry_run:
        current_tests = {i: None for i in range(num_gpus)}  # Simulate GPU allocation
        test_queue = test_infos.copy()

        print(f"\nSimulating parallel execution across {num_gpus} GPUs:")

        # Initial allocation
        for gpu_id in range(num_gpus):
            if test_queue:
                test_path, run_func = test_queue.pop(0)
                current_tests[gpu_id] = (test_path, run_func)
                run_func(test_path, log_dir, gpu_id, dry_run=True)

        # Simulate rest of queue
        while test_queue:
            # Simulate round-robin GPU completion
            gpu_id = len(test_queue) % num_gpus
            current_test_path, _ = current_tests[gpu_id]
            print(f"\nGPU {gpu_id} finished {os.path.basename(current_test_path)}")
            test_path, run_func = test_queue.pop(0)
            current_tests[gpu_id] = (test_path, run_func)
            run_func(test_path, log_dir, gpu_id, dry_run=True)

        # Show final tests completing
        for gpu_id in range(num_gpus):
            if current_tests[gpu_id]:
                test_path, _ = current_tests[gpu_id]
                print(f"\nGPU {gpu_id} finished {os.path.basename(test_path)}")

        return [], []

    # Initialize test queue and tracking variables
    test_queue = test_infos.copy()
    current_processes = {i: None for i in range(num_gpus)}
    current_tests = {i: None for i in range(num_gpus)}
    start_times = {i: time.time() for i in range(num_gpus)}
    completed_tests = []
    failed_tests = []

    def start_test(test_info, gpu_id):
        """Start a test on specified GPU"""
        test_path, run_func = test_info
        current_tests[gpu_id] = test_info
        process = run_func(test_path, log_dir, gpu_id)

        if process is None:
            failed_name = os.path.basename(test_path)
            if run_func == run_python_test:
                failed_name = f"python_{failed_name}"
            failed_tests.append(failed_name)
            return False

        current_processes[gpu_id] = process
        start_times[gpu_id] = time.time()
        print(f"Started {os.path.basename(test_path)} on GPU {gpu_id}")
        return True

    def check_completion():
        """Check if any running tests have completed and return list of free GPUs"""
        free_gpus = []
        for gpu_id in range(num_gpus):
            if current_processes[gpu_id] is None:
                free_gpus.append(gpu_id)
                continue

            # Get timeout for current test
            test_path, run_func = current_tests[gpu_id]
            test_name = os.path.basename(test_path)
            timeout = get_test_timeout(test_name)

            # Check for timeout
            if time.time() - start_times[gpu_id] > timeout:
                print(
                    f"Test {test_name} on GPU {gpu_id} timed out after {timeout / 60} minutes"
                )
                current_processes[gpu_id].kill()
                log_file = log_dir / f"{test_name}.log"
                if run_func == run_python_test:
                    log_file = log_dir / f"python_{test_name}.log"
                with open(log_file, "a") as f:
                    f.write(f"\nERROR: Test timed out after {timeout / 60} minutes\n")

                failed_name = test_name
                if run_func == run_python_test:
                    failed_name = f"python_{failed_name}"
                failed_tests.append(failed_name)
                current_processes[gpu_id] = None
                current_tests[gpu_id] = None
                free_gpus.append(gpu_id)
                continue

            # Check if process completed
            if current_processes[gpu_id].poll() is not None:
                test_path, run_func = current_tests[gpu_id]
                test_name = os.path.basename(test_path)
                success = current_processes[gpu_id].returncode == 0
                print(
                    f"Completed {test_name} on GPU {gpu_id}: {'Success' if success else 'Failed'}"
                )

                # Append completion status to log file
                log_file = log_dir / f"{test_name}.log"
                if run_func == run_python_test:
                    log_file = log_dir / f"python_{test_name}.log"
                with open(log_file, "a") as f:
                    f.write(
                        f"\nTest completed with {'success' if success else 'failure'}\n"
                    )
                    f.write(f"Return code: {current_processes[gpu_id].returncode}\n")

                if success:
                    completed_tests.append(test_path)
                else:
                    failed_name = test_name
                    if run_func == run_python_test:
                        failed_name = f"python_{failed_name}"
                    failed_tests.append(failed_name)

                current_processes[gpu_id] = None
                current_tests[gpu_id] = None
                free_gpus.append(gpu_id)

        return free_gpus

    # Initial test distribution
    for gpu_id in range(num_gpus):
        if test_queue and current_processes[gpu_id] is None:
            start_test(test_queue.pop(0), gpu_id)

    # Main loop
    try:
        while test_queue or any(p is not None for p in current_processes.values()):
            # Check for completed tests
            free_gpus = check_completion()

            # Start new tests on free GPUs
            for gpu_id in free_gpus:
                if test_queue:  # If there are tests waiting to be run
                    start_test(test_queue.pop(0), gpu_id)

            # Small sleep to prevent busy waiting
            time.sleep(1)

        return completed_tests, failed_tests

    finally:
        # Ensure all processes are cleaned up
        for process in current_processes.values():
            if process is not None:
                process.kill()

def collect_test_failures(log_dir, dry_run=False):
    """Scan log files for test failures and collect context"""
    if dry_run:
        print(
            "\nWould scan log files for failures and create failure_summary.txt with:"
        )
        print("- 5 lines of context before each failure")
        print("- 20 lines of context after each failure")
        print("- Coverage for both GTest and Pytest failures")
        print("- Timeout failure information")
        return False

    failure_summary = []

    for log_file in log_dir.glob("*.log"):
        with open(log_file, "r") as f:
            lines = f.readlines()


            # Check for completed tests
            free_gpus = check_completion()

            # Start new tests on free GPUs
            for gpu_id in free_gpus:
                if test_queue:  # If there are tests waiting to be run
                    start_test(test_queue.pop(0), gpu_id)

            # Small sleep to prevent busy waiting
            time.sleep(1)

        return completed_tests, failed_tests

    finally:
        # Ensure all processes are cleaned up
        for process in current_processes.values():
            if process is not None:
                process.kill()

def collect_test_failures(log_dir, dry_run=False):
    """Scan log files for test failures and collect context"""
    if dry_run:
        print(
            "\nWould scan log files for failures and create failure_summary.txt with:"
        )
        print("- 5 lines of context before each failure")
        print("- 20 lines of context after each failure")
        print("- Coverage for both GTest and Pytest failures")
        print("- Timeout failure information")
        return False

    failure_summary = []

    for log_file in log_dir.glob("*.log"):
        with open(log_file, "r") as f:
            lines = f.readlines()

        for i, line in enumerate(lines):
            # Look for GTest failures between square brackets
            if "[" in line and "]" in line:
                result_part = line[line.find("[") : line.find("]") + 1]
                if "FAILED" in result_part or "TIMEOUT" in result_part:
                    failure_summary.append(
                        f"\n=== GTest Failure in {log_file.name} ==="
                    )
                    # Collect 5 lines before and 20 lines after the failure
                    start = max(0, i - 5)
                    end = min(len(lines), i + 21)
                    failure_summary.extend(lines[start:end])
                continue  # Skip other checks for this line if it was a bracketed result

            # Look for pytest failures
            if "FAILED" in line and "test" in line.lower():
                failure_summary.append(f"\n=== Pytest Failure in {log_file.name} ===")
                # Collect 5 lines before and 20 lines after the failure
                start = max(0, i - 5)
                end = min(len(lines), i + 21)
                failure_summary.extend(lines[start:end])

            # Look for timeout failures
            if "ERROR: Test timed out" in line:
                failure_summary.append(f"\n=== Timeout in {log_file.name} ===")
                failure_summary.append(line)

    # Write failure summary if there were any failures
    if failure_summary:
        with open(log_dir / "failure_summary.txt", "w") as f:
            f.write("=== Test Failure Summary ===\n")
            f.write("".join(failure_summary))

        return True
    return False

def get_available_gpus():
    """Check how many NVIDIA GPUs are available"""
    try:
        # Run nvidia-smi -L | wc -l to get GPU count
        output = subprocess.check_output("nvidia-smi -L | wc -l", shell=True, text=True)
        return int(output.strip())
    except (subprocess.SubprocessError, ValueError):
        print("Warning: Could not query GPU count using nvidia-smi")
        return 0

def main():
    parser = argparse.ArgumentParser(description="Run nvFuser tests")
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Show what tests would be run without executing them",
    )


                # Collect 5 lines before and 20 lines after the failure
                start = max(0, i - 5)
                end = min(len(lines), i + 21)
                failure_summary.extend(lines[start:end])

            # Look for timeout failures
            if "ERROR: Test timed out" in line:
                failure_summary.append(f"\n=== Timeout in {log_file.name} ===")
                failure_summary.append(line)

    # Write failure summary if there were any failures
    if failure_summary:
        with open(log_dir / "failure_summary.txt", "w") as f:
            f.write("=== Test Failure Summary ===\n")
            f.write("".join(failure_summary))

        return True
    return False

def get_available_gpus():
    """Check how many NVIDIA GPUs are available"""
    try:
        # Run nvidia-smi -L | wc -l to get GPU count
        output = subprocess.check_output("nvidia-smi -L | wc -l", shell=True, text=True)
        return int(output.strip())
    except (subprocess.SubprocessError, ValueError):
        print("Warning: Could not query GPU count using nvidia-smi")
        return 0

def main():
    parser = argparse.ArgumentParser(description="Run nvFuser tests")
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Show what tests would be run without executing them",
    )
    args = parser.parse_args()

    # Check GPU availability
    gpu_count = get_available_gpus()
    if gpu_count < 1:
        print("Error: No GPUs found for testing")
        return 1

    print(f"Found {gpu_count} GPUs available for testing")

    # Hardcode paths relative to current directory


main

    if failure_summary:
        with open(log_dir / "failure_summary.txt", "w") as f:
            f.write("=== Test Failure Summary ===\n")
            f.write("".join(failure_summary))

        return True
    return False

def get_available_gpus():
    """Check how many NVIDIA GPUs are available"""
    try:
        # Run nvidia-smi -L | wc -l to get GPU count
        output = subprocess.check_output("nvidia-smi -L | wc -l", shell=True, text=True)
        return int(output.strip())
    except (subprocess.SubprocessError, ValueError):
        print("Warning: Could not query GPU count using nvidia-smi")
        return 0

def main():
    parser = argparse.ArgumentParser(description="Run nvFuser tests")
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Show what tests would be run without executing them",
    )
    args = parser.parse_args()

    # Check GPU availability
    gpu_count = get_available_gpus()
    if gpu_count < 1:
        print("Error: No GPUs found for testing")
        return 1

    print(f"Found {gpu_count} GPUs available for testing")

    # Hardcode paths relative to current directory
    build_dir = "bin"
    python_test_dir = "tests/python"

    log_dir = setup_logging_dir()
    multidevice_tests, single_device_tests = get_cpp_test_executables(build_dir)
    # This is not catching all python multidevice tests like test_communication.py
    # TODO: Change test names to separate out multidevice tests, or update to support all multidevice python tests
    python_multidevice_tests, python_single_tests = get_python_tests(python_test_dir)

    if not (multidevice_tests or single_device_tests) and not (
        python_multidevice_tests or python_single_tests
    ):
        print("No tests found!")
        return 0

    print(f"Found {len(multidevice_tests)} C++ multidevice tests")
    print(f"Found {len(python_multidevice_tests)} Python multidevice tests")
    print(f"Found {len(single_device_tests)} C++ single device tests")
    print(f"Found {len(python_single_tests)} Python single device tests")
    print(f"Logs will be written to: {log_dir}")

    success_count = 0
    failed_tests = []
    total_tests = (
        len(multidevice_tests)
        + len(single_device_tests)
        + len(python_multidevice_tests)
        + len(python_single_tests)
    )

    # Run all multidevice tests first
    print("\nC++ multidevice tests:")
    for test in multidevice_tests:
        if run_multidevice_test(test, log_dir, gpu_count, dry_run=args.dry_run):
            if not args.dry_run:
                success_count += 1
        else:
            if not args.dry_run:
                failed_tests.append(os.path.basename(test))

    print("\nPython multidevice tests:")
    for test in python_multidevice_tests:
        if run_python_multidevice_test(test, log_dir, gpu_count, dry_run=args.dry_run):
            if not args.dry_run:
                success_count += 1
        else:
            if not args.dry_run:
                failed_tests.append(f"python_{os.path.basename(test)}")

    # Run all single device tests (C++ and Python) in parallel
    print("\nRunning all single device tests in parallel:")

    # Create initial unordered test list
    test_infos = [(t, run_single_device_test) for t in single_device_tests] + [
        (t, run_python_test) for t in python_single_tests
    ]

    # Find and prioritize specific tests by name match in full path
    priority_names = ["test_nvfuser", "test_matmul", "test_ops.py"]
    priority_tests = []
    other_tests = []

    for test_info in test_infos:
        test_path, _ = test_info
        if any(name in test_path for name in priority_names):
            priority_tests.append(test_info)
        else:
            other_tests.append(test_info)

    test_infos = priority_tests + other_tests

    if not args.dry_run:
        completed, failed = run_parallel_tests(log_dir, gpu_count, test_infos)
        success_count += len(completed)
        failed_tests.extend(failed)
    else:
        run_parallel_tests(log_dir, gpu_count, test_infos, dry_run=True)

    if args.dry_run:
        # Show what would be written to summary.txt
        print("\nWould create summary.txt with:")
        print(f"Total tests: {total_tests}")
        print(f"Multidevice tests: {len(multidevice_tests)}")
        print(f"Multidevice python tests: {len(python_multidevice_tests)}")
        print(f"C++ single device tests: {len(single_device_tests)}")
        print(f"Python single device tests: {len(python_single_tests)}")

        # Show failure collection simulation
        collect_test_failures(log_dir, dry_run=True)

        print("\nThis was a dry run. No tests were actually executed.")
        return 0

    # Write summary
    with open(log_dir / "summary.txt", "w") as f:
        f.write(f"Total tests: {total_tests}\n")
        f.write(f"Multidevice tests: {len(multidevice_tests)}\n")
        f.write(f"Single device tests: {len(single_device_tests)}\n")
        f.write(f"Python tests: {len(python_multidevice_tests)}\n")
        f.write(f"Successful: {success_count}\n")
        f.write(f"Failed: {len(failed_tests)}\n")
        if failed_tests:
            f.write("\nFailed tests:\n")
            for test in failed_tests:
                f.write(f"- {test}\n")

    # Collect and summarize failures
    if failed_tests:
        if collect_test_failures(log_dir):
            print(
                f"Detailed failure information available in: {log_dir}/failure_summary.txt"
            )

    print(f"\nTest run complete. {success_count}/{total_tests} tests passed.")
    if failed_tests:
        print("Failed tests:")
        for test in failed_tests:
            print(f"- {test}")
    print(f"Logs available in: {log_dir}")

    return 0 if not failed_tests else 1

if __name__ == "__main__":
    main()


            test_path, run_func = test_queue.pop(0)
            current_tests[gpu_id] = (test_path, run_func)
            run_func(test_path, log_dir, gpu_id, dry_run=True)

        # Show final tests completing
        for gpu_id in range(num_gpus):
            if current_tests[gpu_id]:
                test_path, _ = current_tests[gpu_id]
                print(f"\nGPU {gpu_id} finished {os.path.basename(test_path)}")

        return [], []

    # Initialize test queue and tracking variables
    test_queue = test_infos.copy()
    current_processes = {i: None for i in range(num_gpus)}
    current_tests = {i: None for i in range(num_gpus)}
    start_times = {i: time.time() for i in range(num_gpus)}
    completed_tests = []
    failed_tests = []

    def start_test(test_info, gpu_id):
        """Start a test on specified GPU"""
        test_path, run_func = test_info
        current_tests[gpu_id] = test_info
        process = run_func(test_path, log_dir, gpu_id)

        if process is None:
            failed_name = os.path.basename(test_path)
            if run_func == run_python_test:
                failed_name = f"python_{failed_name}"
            failed_tests.append(failed_name)
            return False

        current_processes[gpu_id] = process
        start_times[gpu_id] = time.time()
        print(f"Started {os.path.basename(test_path)} on GPU {gpu_id}")
        return True

    def check_completion():
        """Check if any running tests have completed and return list of free GPUs"""
        free_gpus = []
        for gpu_id in range(num_gpus):
            if current_processes[gpu_id] is None:
                free_gpus.append(gpu_id)
                continue

            # Get timeout for current test
            test_path, run_func = current_tests[gpu_id]
            test_name = os.path.basename(test_path)
            timeout = get_test_timeout(test_name)

            # Check for timeout
            if time.time() - start_times[gpu_id] > timeout:
                print(
                    f"Test {test_name} on GPU {gpu_id} timed out after {timeout / 60} minutes"
                )
                current_processes[gpu_id].kill()


    failed_tests = []

    def start_test(test_info, gpu_id):
        """Start a test on specified GPU"""
        test_path, run_func = test_info
        current_tests[gpu_id] = test_info
        process = run_func(test_path, log_dir, gpu_id)

        if process is None:
            failed_name = os.path.basename(test_path)
            if run_func == run_python_test:
                failed_name = f"python_{failed_name}"
            failed_tests.append(failed_name)
            return False

        current_processes[gpu_id] = process
        start_times[gpu_id] = time.time()
        print(f"Started {os.path.basename(test_path)} on GPU {gpu_id}")
        return True

    def check_completion():
        """Check if any running tests have completed and return list of free GPUs"""
        free_gpus = []
        for gpu_id in range(num_gpus):
            if current_processes[gpu_id] is None:
                free_gpus.append(gpu_id)
                continue

            # Get timeout for current test
            test_path, run_func = current_tests[gpu_id]
            test_name = os.path.basename(test_path)
            timeout = get_test_timeout(test_name)

            # Check for timeout
            if time.time() - start_times[gpu_id] > timeout:
                print(
                    f"Test {test_name} on GPU {gpu_id} timed out after {timeout / 60} minutes"
                )
                current_processes[gpu_id].kill()
                log_file = log_dir / f"{test_name}.log"
                if run_func == run_python_test:
                    log_file = log_dir / f"python_{test_name}.log"
                with open(log_file, "a") as f:
                    f.write(f"\nERROR: Test timed out after {timeout / 60} minutes\n")

                failed_name = test_name
                if run_func == run_python_test:
                    failed_name = f"python_{failed_name}"
                failed_tests.append(failed_name)
                current_processes[gpu_id] = None
                current_tests[gpu_id] = None
                free_gpus.append(gpu_id)
                continue

            # Check if process completed
            if current_processes[gpu_id].poll() is not None:
                test_path, run_func = current_tests[gpu_id]
                test_name = os.path.basename(test_path)
                success = current_processes[gpu_id].returncode == 0
                print(
                    f"Completed {test_name} on GPU {gpu_id}: {'Success' if success else 'Failed'}"
                )

                # Append completion status to log file
                log_file = log_dir / f"{test_name}.log"
                if run_func == run_python_test:
                    log_file = log_dir / f"python_{test_name}.log"
                with open(log_file, "a") as f:
                    f.write(
                        f"\nTest completed with {'success' if success else 'failure'}\n"
                    )
                    f.write(f"Return code: {current_processes[gpu_id].returncode}\n")

                if success:
                    completed_tests.append(test_path)
                else:
                    failed_name = test_name
                    if run_func == run_python_test:
                        failed_name = f"python_{failed_name}"
                    failed_tests.append(failed_name)

                current_processes[gpu_id] = None
                current_tests[gpu_id] = None
                free_gpus.append(gpu_id)

        return free_gpus

    # Initial test distribution
    for gpu_id in range(num_gpus):
        if test_queue and current_processes[gpu_id] is None:
            start_test(test_queue.pop(0), gpu_id)

    # Main loop
    try:
        while test_queue or any(p is not None for p in current_processes.values()):
            # Check for completed tests
            free_gpus = check_completion()

            # Start new tests on free GPUs
            for gpu_id in free_gpus:
                if test_queue:  # If there are tests waiting to be run
                    start_test(test_queue.pop(0), gpu_id)

            # Small sleep to prevent busy waiting
            time.sleep(1)


# SPDX-FileCopyrightText: Copyright (c) 2023-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import subprocess
import sys
from pathlib import Path

UNKNOWN = "Unknown"
nvfuser_root = Path(__file__).parent.parent

# note that this root currently is still part of pytorch.
def get_sha() -> str:
    try:
        return (
            subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=nvfuser_root)
            .decode("ascii")
            .strip()
        )
    except Exception:
        import os

        # assume the $NVFUSER_VERSION is in sha form
        if nvfuser_version := os.environ.get("NVFUSER_VERSION"):
            assert (
                len(nvfuser_version) < 11
            ), "The NVFUSER_VERSION should be in sha form"
            return nvfuser_version
        return UNKNOWN

def get_version() -> str:
    sha = get_sha()
    version = (
        open((nvfuser_root / "version.txt"), "r").read().strip() + "+git" + sha[:7]
    )
    return version

def get_pytorch_cmake_prefix():
    from subprocess import Popen, PIPE

    # need to do this in a separate process so we are not going to delete nvfuser library while it's loaded by torch
    process_torch_prefix = Popen(
        [
            sys.executable,
            "-c",
            "import torch.utils; print(torch.utils.cmake_prefix_path)",
        ],
        stdout=PIPE,
    )
    stdout_msg, error_msg = process_torch_prefix.communicate()
    return stdout_msg.decode("utf-8").rstrip("\n")

def get_pytorch_use_distributed():
    from subprocess import Popen, PIPE

    # need to do this in a separate process so we are not going to delete nvfuser library while it's loaded by torch
    process_torch_prefix = Popen(
        [
            sys.executable,
            "-c",
            "import torch; print(torch._C._has_distributed())",
        ],
        stdout=PIPE,
    )
    stdout_msg, error_msg = process_torch_prefix.communicate()
    return stdout_msg.decode("utf-8").rstrip("\n")

if __name__ == "__main__":
    assert len(sys.argv) == 2
    assert sys.argv[1] == "nvfuser" or sys.argv[1] == "nvfuser_direct"
    python_module = sys.argv[1]
    version_file = nvfuser_root / python_module / "version.py"
    with open(version_file, "w") as f:
        f.write("_version_str = '{}'\n".format(get_version()))


# SPDX-FileCopyrightText: Copyright (c) 2023-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
import subprocess
import sys
from pathlib import Path

UNKNOWN = "Unknown"
nvfuser_root = Path(__file__).parent.parent

# note that this root currently is still part of pytorch.
def get_sha() -> str:
    try:
        return (
            subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=nvfuser_root)
            .decode("ascii")
            .strip()
        )
    except Exception:
        import os

        # assume the $NVFUSER_VERSION is in sha form
        if nvfuser_version := os.environ.get("NVFUSER_VERSION"):
            assert (
                len(nvfuser_version) < 11
            ), "The NVFUSER_VERSION should be in sha form"
            return nvfuser_version
        return UNKNOWN

def get_version() -> str:
    sha = get_sha()
    version = (
        open((nvfuser_root / "version.txt"), "r").read().strip() + "+git" + sha[:7]
    )
    return version

def get_pytorch_cmake_prefix():
    from subprocess import Popen, PIPE

    # need to do this in a separate process so we are not going to delete nvfuser library while it's loaded by torch
    process_torch_prefix = Popen(
        [
            sys.executable,
            "-c",
            "import torch.utils; print(torch.utils.cmake_prefix_path)",
        ],


# note that this root currently is still part of pytorch.
def get_sha() -> str:
    try:
        return (
            subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=nvfuser_root)
            .decode("ascii")
            .strip()
        )
    except Exception:
        import os

        # assume the $NVFUSER_VERSION is in sha form
        if nvfuser_version := os.environ.get("NVFUSER_VERSION"):
            assert (
                len(nvfuser_version) < 11
            ), "The NVFUSER_VERSION should be in sha form"
            return nvfuser_version
        return UNKNOWN

def get_version() -> str:
    sha = get_sha()
    version = (
        open((nvfuser_root / "version.txt"), "r").read().strip() + "+git" + sha[:7]
    )
    return version

def get_pytorch_cmake_prefix():
    from subprocess import Popen, PIPE

    # need to do this in a separate process so we are not going to delete nvfuser library while it's loaded by torch
    process_torch_prefix = Popen(
        [
            sys.executable,
            "-c",
            "import torch.utils; print(torch.utils.cmake_prefix_path)",
        ],
        stdout=PIPE,
    )
    stdout_msg, error_msg = process_torch_prefix.communicate()
    return stdout_msg.decode("utf-8").rstrip("\n")

def get_pytorch_use_distributed():
    from subprocess import Popen, PIPE


    except Exception:
        import os

        # assume the $NVFUSER_VERSION is in sha form
        if nvfuser_version := os.environ.get("NVFUSER_VERSION"):
            assert (
                len(nvfuser_version) < 11
            ), "The NVFUSER_VERSION should be in sha form"
            return nvfuser_version
        return UNKNOWN

def get_version() -> str:
    sha = get_sha()
    version = (
        open((nvfuser_root / "version.txt"), "r").read().strip() + "+git" + sha[:7]
    )
    return version

def get_pytorch_cmake_prefix():
    from subprocess import Popen, PIPE

    # need to do this in a separate process so we are not going to delete nvfuser library while it's loaded by torch
    process_torch_prefix = Popen(
        [
            sys.executable,
            "-c",
            "import torch.utils; print(torch.utils.cmake_prefix_path)",
        ],
        stdout=PIPE,
    )
    stdout_msg, error_msg = process_torch_prefix.communicate()
    return stdout_msg.decode("utf-8").rstrip("\n")

def get_pytorch_use_distributed():
    from subprocess import Popen, PIPE

    # need to do this in a separate process so we are not going to delete nvfuser library while it's loaded by torch
    process_torch_prefix = Popen(
        [
            sys.executable,
            "-c",
            "import torch; print(torch._C._has_distributed())",
        ],
        stdout=PIPE,
    )
    stdout_msg, error_msg = process_torch_prefix.communicate()
    return stdout_msg.decode("utf-8").rstrip("\n")

if __name__ == "__main__":
    assert len(sys.argv) == 2


    )
    return version

def get_pytorch_cmake_prefix():
    from subprocess import Popen, PIPE

    # need to do this in a separate process so we are not going to delete nvfuser library while it's loaded by torch
    process_torch_prefix = Popen(
        [
            sys.executable,
            "-c",
            "import torch.utils; print(torch.utils.cmake_prefix_path)",
        ],
        stdout=PIPE,
    )
    stdout_msg, error_msg = process_torch_prefix.communicate()
    return stdout_msg.decode("utf-8").rstrip("\n")

def get_pytorch_use_distributed():
    from subprocess import Popen, PIPE

    # need to do this in a separate process so we are not going to delete nvfuser library while it's loaded by torch
    process_torch_prefix = Popen(
        [
            sys.executable,
            "-c",
            "import torch; print(torch._C._has_distributed())",
        ],
        stdout=PIPE,
    )
    stdout_msg, error_msg = process_torch_prefix.communicate()
    return stdout_msg.decode("utf-8").rstrip("\n")

if __name__ == "__main__":
    assert len(sys.argv) == 2
    assert sys.argv[1] == "nvfuser" or sys.argv[1] == "nvfuser_direct"
    python_module = sys.argv[1]
    version_file = nvfuser_root / python_module / "version.py"
    with open(version_file, "w") as f:
        f.write("_version_str = '{}'\n".format(get_version()))


# SPDX-FileCopyrightText: Copyright (c) 2024-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause

"""
This script reproduces the computation used for generating the validation tolerances in csrc/validator_utils.h:
https://github.com/NVIDIA/Fuser/blob/892b7ac5646b3b873f13f2b2ea1db15fa9b9effb/csrc/validator_utils.h#L25-L46

The computation (randn + mul + sum) is repeated a large number of times.
The base tolerance is the max error seen when comparing the desired precision type with the next higher precision.
The final tolerances are computed by adding a safety factor of 3/4 to this base tolerance.

The output of this file is a .npy file corresponding to each dtype, containing a {size: safety_factor * base_tolerances} dict.

Note: The script may take a long time for a high number of num_iters, so it may be useful to only run the script for one dtype at a time. Modify `dtype_to_ref_dtypes` accordingly.

To run: python tools/generate_validation_tolerances.py
To load the files:
    ```
    import numpy as np
    tol_dict = np.load(file_name.npy, allow_pickle=True).item()
    ```
"""

import numpy as np
import torch
from datetime import datetime

sizes = [2**i for i in range(2, 22)]  # {4, 2097152}
num_iters = 10**6

def compute_max_error(size, dtype, ref_dtype):
    a, b = [torch.randn(size, dtype=dtype, device="cuda") for _ in range(2)]
    out = (a * b).sum()
    ref = (a.to(ref_dtype) * b.to(ref_dtype)).sum()
    max_error = out.sub(ref).abs().max().to(torch.double).item()
    return max_error

if __name__ == "__main__":
    assert torch.cuda.is_available(), "A CUDA device is required."

    device = torch.cuda.get_device_name(torch.cuda.current_device())
    # Comparison is made against the next higher precision,
    # since it seems sufficient and avoids incurring extra memory.
    dtype_to_ref_dtypes = {
        torch.bfloat16: torch.float32,
        torch.float16: torch.float32,
        torch.float32: torch.float64,
    }

    # The original safety factor used is not known, but reverse-engineering estimates it between 3-5.
    safety_factor = 4

    for dtype, ref_dtype in dtype_to_ref_dtypes.items():
        tolerances = {}
        for size in sizes:
            print(f"Computing validation constant for size:{size}")
            base_tolerance = torch.finfo(torch.double).min
            for i in range(num_iters):
                base_tolerance = max(
                    base_tolerance, compute_max_error(size, dtype, ref_dtype)
                )
            tolerances[size] = safety_factor * base_tolerance
        date = datetime.now().strftime("%Y%m%d%H%M%S")
        np.save(f"validation_consts_{dtype}_{device}_{date}.npy", tolerances)
        print(
            f"dtype:{dtype} device:{device} safety_factor:{safety_factor}\n{tolerances}\n"
        )


The output of this file is a .npy file corresponding to each dtype, containing a {size: safety_factor * base_tolerances} dict.

Note: The script may take a long time for a high number of num_iters, so it may be useful to only run the script for one dtype at a time. Modify `dtype_to_ref_dtypes` accordingly.

To run: python tools/generate_validation_tolerances.py
To load the files:
    ```
    import numpy as np
    tol_dict = np.load(file_name.npy, allow_pickle=True).item()
    ```
"""

import numpy as np
import torch
from datetime import datetime

sizes = [2**i for i in range(2, 22)]  # {4, 2097152}
num_iters = 10**6

def compute_max_error(size, dtype, ref_dtype):
    a, b = [torch.randn(size, dtype=dtype, device="cuda") for _ in range(2)]
    out = (a * b).sum()
    ref = (a.to(ref_dtype) * b.to(ref_dtype)).sum()
    max_error = out.sub(ref).abs().max().to(torch.double).item()
    return max_error

if __name__ == "__main__":
    assert torch.cuda.is_available(), "A CUDA device is required."

    device = torch.cuda.get_device_name(torch.cuda.current_device())
    # Comparison is made against the next higher precision,
    # since it seems sufficient and avoids incurring extra memory.
    dtype_to_ref_dtypes = {
        torch.bfloat16: torch.float32,
        torch.float16: torch.float32,
        torch.float32: torch.float64,
    }

    # The original safety factor used is not known, but reverse-engineering estimates it between 3-5.
    safety_factor = 4

    for dtype, ref_dtype in dtype_to_ref_dtypes.items():
        tolerances = {}
        for size in sizes:


tools.codediff.codediff

# SPDX-FileCopyrightText: Copyright (c) 2023-present NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
"""
Find corresponding .cu files for matching tests, even when new tests are
introduced between two commits. Diffs are displayed and the return value is the
number of mismatched corresponding tests.

Tests are skipped if they produce different numbers of .cu files, or if they
exist in only one of the given runs.

Example usage:
    python tools/diff_codegen_nvfuser_tests.py \
            codegen_comparison/{$commit1,$commit2}/binary_tests
"""

from dataclasses import asdict, dataclass, field, InitVar
import difflib
from enum import Enum
import os
import re
import subprocess
import sys

import cxxfilt
from dataclasses_json import dataclass_json

@dataclass_json
@dataclass
class GitRev:
    full_hash: str
    diff: str | None = None
    abbrev: str | None = None
    title: str | None = None
    author_name: str | None = None
    author_email: str | None = None
    author_time: str | None = None
    commit_time: str | None = None

    def __post_init__(self):
        if self.abbrev is not None:
            # Avoid running __post_init__ during deserialization
            return

        self.abbrev = (
            subprocess.run(
                ["git", "rev-parse", "--short", self.full_hash], capture_output=True
            )
            .stdout.strip()
            .decode("utf-8")
        )
        for line in (
            subprocess.run(
                ["git", "branch", "--quiet", "--color=never", self.full_hash],
                capture_output=True,
            )
            .stdout.strip()
            .decode("utf-8")
            .splitlines()
        ):
            # Possible output:
            #
            #     main
            #     * scalar_seg_edges
            #
            # In this case, we have checked out the HEAD of the
            # scalar_seg_edges branch. Here we just strip the *.
            if line[0] == "*":
                line = line[2:]
                in_branches.append(line)

        def git_show(fmt) -> str:
            return (
                subprocess.run(
                    [
                        "git",
                        "show",
                        "--no-patch",
                        f"--format={fmt}",
                        self.full_hash,
                    ],
                    capture_output=True,
                )
                .stdout.strip()
                .decode("utf-8")
            )

        self.title = git_show("%s")
        self.author_name = git_show("%an")
        self.author_email = git_show("%ae")
        self.author_time = git_show("%ad")
        self.commit_time = git_show("%cd")

@dataclass_json
@dataclass
class LaunchParams:
    blockDim: tuple[int]
    gridDim: tuple[int]
    dynamic_smem_bytes: int

@dataclass_json
@dataclass
class CompiledKernel:
    filename: str
    code: str | None = None
    ptx: str | None = None
    ptxas_info: str | None = None
    launch_params_str: str | None = None
    launch_params: LaunchParams | None = None
    gmem_bytes: int = 0
    smem_bytes: int = 0
    cmem_bank_bytes: list[int] | None = None
    registers: int | None = None
    stack_frame_bytes: int = 0
    spill_store_bytes: int = 0
    spill_load_bytes: int = 0
    mangled_name: str | None = None
    arch: str | None = None
    index_type: str | None = None

    def __post_init__(self):
        if self.launch_params is not None:
            # Avoid running __post_init__ during deserialization
            return

        self.parse_ptxas()
        self.parse_launch_params()

    def parse_ptxas(self):
        # Example input:
        #
        #   ptxas info    : 307 bytes gmem
        #   ptxas info    : Compiling entry function
        #   '_ZN76_GLOBAL__N__00000000_37___tmp_kernel_pointwise_f0_c1_r0_g0_cu_8995cef2_3255329nvfuser_pointwise_f0_c1_r0_g0ENS_6TensorIfLi2ELi2EEES1_S1_'
        #   for 'sm_86'
        #   ptxas info    : Function properties for
        #   _ZN76_GLOBAL__N__00000000_37___tmp_kernel_pointwise_f0_c1_r0_g0_cu_8995cef2_3255329nvfuser_pointwise_f0_c1_r0_g0ENS_6TensorIfLi2ELi2EEES1_S1_
        #   ptxas         .     0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads
        #   ptxas info    : Used 203 registers, 16 bytes smem, 472 bytes cmem[0], 8 bytes cmem[2]
        #
        # Here we parse this into the fields presented, and we replace the
        # mangled kernel name since it includes the kernel number and is
        # useless for the purposes of diffing since the kernel signature is
        # already included.
        if self.ptxas_info is None:
            return

        m = re.search(r"Compiling entry function '(.*)' for '(.*)'", self.ptxas_info)
        if m is not None:
            self.mangled_name, self.arch = m.groups()

        def find_unique_int(pattern) -> int | None:
            assert self.ptxas_info is not None
            m = re.search(pattern, self.ptxas_info)
            return 0 if m is None else int(m.groups()[0])

        self.stack_frame_bytes = find_unique_int(r"(\d+) bytes stack frame")
        self.spill_store_bytes = find_unique_int(r"(\d+) bytes spill stores")
        self.spill_load_bytes = find_unique_int(r"(\d+) bytes spill loads")
        self.registers = find_unique_int(r"(\d+) registers")
        self.gmem_bytes = find_unique_int(r"(\d+) bytes gmem")
        self.smem_bytes = find_unique_int(r"(\d+) bytes smem")

        self.cmem_bank_bytes = []
        cmem_banks = 0
        for m in re.finditer(r"(\d+) bytes cmem\[(\d+)\]", self.ptxas_info):
            nbytes_str, bank_str = m.groups()
            bank = int(bank_str)
            if len(self.cmem_bank_bytes) <= bank:
                self.cmem_bank_bytes += [0] * (bank + 1 - len(self.cmem_bank_bytes))
            self.cmem_bank_bytes[bank] = int(nbytes_str)
            cmem_banks += 1

    def parse_launch_params(self):
        # If NVFUSER_DUMP=launch_param is given we will get a line like this for every launch:
        #   Launch Parameters: BlockDim.x = 32, BlockDim.y = 2, BlockDim.z = 2, GridDim.x = 8, GridDim.y = 8, GridDim.z = -1, Smem Size = 49152
        # This is not done by default since we might have hundreds of thousands of these lines.
        # Still, if we recognize it, we will parse this info. If there are
        # multiple lines, we just check that they are all equal and if not then
        # we keep the first version and print a warning.
        if self.launch_params_str is None:
            return

        self.launch_params = None
        for line in self.launch_params_str.splitlines():
            m = re.search(
                r"Launch Parameters: BlockDim.x = (.*), BlockDim.y = (.*), BlockDim.z = (.*), "
                r"GridDim.x = (.*), GridDim.y = (.*), GridDim.z = (.*), Smem Size = (.*)$",
                line,
            )
            bx, by, bz, gx, gy, gz, s = m.groups()
            lp = LaunchParams((bx, by, bz), (gx, gy, gz), s)
            if self.launch_params is None:
                self.launch_params = lp
            else:
                if lp != self.launch_params:
                    # Found multiple mismatched launch params for one kernel. Only using first
                    return

@dataclass_json
@dataclass
class BenchmarkResult:
    gpu_time: float
    gpu_time_unit: str
    cpu_time: float
    cpu_time_unit: float
    iterations: int | None = None

@dataclass_json
@dataclass
class CompiledTest:
    """One grouping of kernels. A run holds multiple of these"""

    name: str
    kernels: list[CompiledKernel]
    passed: bool = True
    benchmark_result: BenchmarkResult | None = None

class CommandType(str, Enum):
    """Denotes what type of command was run"""

    UNKNOWN = "UNKNOWN"
    GOOGLETEST = "GOOGLETEST"
    GOOGLEBENCH = "GOOGLEBENCH"
    PYTEST = "PYTEST"

    def __str__(self):
        return self.name

    @classmethod
    def from_string(cls, type_str: str):
        l = type_str.lower()
        if l[:3] == "unk":
            # Specified unknown. Don't print warning
            return cls.UNKNOWN
        elif l == "gtest" or l == "googletest":
            return cls.GOOGLETEST
        elif l == "gbench" or l == "googlebench":
            return cls.GOOGLEBENCH
        elif l == "pytest":
            return cls.PYTEST
        else:
            print(
                f"WARNING: Unrecognized command type '{type_str}'. Parsing as UNKNOWN.",
                file=sys.stderr,
            )
            return cls.UNKNOWN

class LogParser:
    """General parser for STDOUT of NVFuser commands

    This parser does not group into individual tests, but rather places all
    kernels into a single CompiledTest whose name is "Ungrouped Kernels".
    """

    def __init__(self, log_file: str):
        self.compile_regex()

        self.kernel_map: dict[str, CompiledTest] = {}

        self.reset_test_state()

        self.parse(log_file)

    def compile_regex(self):
        # regex for stripping ANSI color codes
        self.ansi_re = re.compile(r"(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]")

    def reset_kernel_state(self):
        self.current_file = None
        self.ptxas_info = ""
        self.launch_params_str = ""

    def reset_test_state(self):
        """Initialize temporary variables used during parsing pass"""
        self.reset_kernel_state()
        self.current_test = None
        self.kernels = []

    def parse(self, log_file: str):
        for line in open(log_file, "r").readlines():
            line = self.ansi_re.sub("", line.rstrip())
            self.parse_line(line)
        self.finalize()

    def finalize_kernel(self):
        if self.current_file is not None:
            k = CompiledKernel(
                self.current_file,
                ptxas_info=self.ptxas_info,
                launch_params_str=self.launch_params_str,
            )
            self.kernels.append(k)
        self.reset_kernel_state()

    def finalize_test(self, passed: bool):
        assert self.current_test is not None
        self.finalize_kernel()
        new_test = CompiledTest(self.current_test, self.kernels, passed)
        self.kernel_map[self.current_test] = new_test
        self.reset_test_state()
        return new_test

    def finalize(self):
        if len(self.kernels) > 0:
            group_name = "Ungrouped Kernels"
            self.kernel_map[group_name] = CompiledTest(group_name, self.kernels)

    def parse_line(self, line):
        """Parse a line of log. Return True if consumed"""
        if line[:10] == "PRINTING: ":
            if line[-3:] == ".cu":
                self.finalize_kernel()
                # This avoids comparing the .ptx files that are created then
                # removed by the MemoryTest.LoadCache tests
                self.current_file = line[10:]
        elif line[:6] == "ptxas ":
            # NVFUSER_DUMP=ptxas_verbose corresponds to nvcc --ptxas-options=-v
            # or --resources-usage. This always prints after printing the cuda
            # filename
            if self.current_file is None:
                print("WARNING: Cannot associate ptxas info with CUDA kernel")
                return False
            self.ptxas_info += line + "\n"
        elif line[:19] == "Launch Parameters: ":
            if self.current_file is None:
                print("WARNING: Cannot associate launch params with CUDA kernel")
                return False
            self.launch_params_str += line + "\n"
        else:
            return False
        return True

class LogParserGTest(LogParser):
    """Parse output of googletest binaries like test_nvfuser"""

    def parse_line(self, line):
        if super().parse_line(line):
            return True

        if line[:13] == "[ RUN      ] ":
            self.current_test = line[13:]
        elif line[:13] == "[       OK ] ":
            self.finalize_test(True)
        elif line[:13] == "[  FAILED  ] ":
            if self.current_test is not None and self.current_file is not None:
                # Avoid the summary of failed tests, such as
                #   [  FAILED  ] 1 test, listed below:
                #   [  FAILED  ] NVFuserTest.FusionTuringMatmulSplitK_CUDA
                self.finalize_test(False)
        else:
            return False
        return True

class LogParserGBench(LogParser):
    """Parse output of google benchmark binaries like nvfuser_bench"""

    def compile_regex(self):
        super().compile_regex()

        # Example line:
        #   benchmark_name   34.0 us      1.53 ms   2007  /Launch_Parameters[block(2/2/32)/grid(32/2/2)/49664]
        # This is the only kind of line we match for benchmarks. Note that this is printed at the end of each benchmark
        self.result_re = re.compile(
            r"^(?P<testname>\S+)\s+(?P<gputime>[-+\.\d]+)\s+(?P<gputimeunit>\S+)\s+(?P<cputime>[-+\.\d]+)\s+(?P<cputimeunit>\S+)\s+(?P<iterations>\d+).*$"
        )

    def parse_line(self, line):
        if super().parse_line(line):
            return True

        m = re.match(self.result_re, line)
        if m is not None:
            d = m.groupdict()
            self.current_test = d["testname"]
            time = d["gputime"]
            time_unit = d["gputimeunit"]
            cpu = d["cputime"]
            cpu_unit = d["cputimeunit"]
            iterations = d["iterations"]
            # Skip metadata which for nvfuser_bench sometimes includes LaunchParams
            # meta = m.groups()[6]
            new_test = self.finalize_test(True)
            new_test.benchmark_result = BenchmarkResult(
                time, time_unit, cpu, cpu_unit, iterations
            )
            return True
        return False

class LogParserPyTest(LogParser):
    """Parse output of pytest tests.

    Note that the tests must be run with both the -v and -s options
    """

    def compile_regex(self):
        super().compile_regex()

        self.itemlist_re = re.compile(r"Running \d+ items in this shard: (.*)$")

        # match lines like these:
        #  [2024-10-23 02:00:20] tests/python/test_python_frontend.py::TestNvFuserFrontend::test_nanogpt_split_mha_linears
        #  tests/python/test_python_frontend.py::TestNvFuserFrontend::test_nanogpt_split_mha_linears
        #  tests/python/test_python_frontend.py::TestNvFuserFrontend::test_nanogpt_split_mha_linears PASSED

        self.wildcard_testname_re = re.compile(
            r"^((?P<timestamp>\[[\d\-: ]+\]) )?(?P<testname>\S+\.py::\S+)\s?(?P<line>.*)$"
        )

        self.extra_wildcard_testname_re = re.compile(
            r".*?(?P<testname>\S+::\S+) (?P<line>.*)$"
        )

        self.all_test_names: list[str] | None = None

    def parse_line(self, line):
        if self.all_test_names is None:
            m = re.match(self.itemlist_re, line)
            if m is not None:
                # grab the test list
                self.all_test_names = m.groups()[0].split(", ")
                return True

        m = re.match(self.wildcard_testname_re, line)
        if m is not None:
            d = m.groupdict()
            self.current_test = d["testname"]
            line = d["line"]

        if line == "PASSED":
            self.finalize_test(True)
        elif line == "FAILED" and self.current_test is not None:
            self.finalize_test(False)

        if super().parse_line(line):
            return True

        return False

@dataclass_json
@dataclass
class TestRun:
    """A single process that might contain many kernels, grouped into tests"""

    directory: str
    git: GitRev | None = None
    name: str | None = None
    command: str | None = None
    command_type: CommandType | None = None
    exit_code: int | None = None
    env: str | None = None
    gpu_names: str | None = None
    nvcc_version: str | None = None
    # map from name of test to list of kernel base filenames
    kernel_map: dict[str, CompiledTest] | None = None
    # collecting the preamble lets us skip it when diffing, and lets us compare
    # only the preamble between runs
    preamble: str | None = None
    # The following lets us skip preamble when loading kernels. Note that the
    # preamble can change length due to differing index types, so we can't rely
    # on f.seek()
    preamble_size_lines: int | None = None

    def __post_init__(self):
        if self.git is not None:
            # Avoid running __post_init__ during deserialization
            return

        if not os.path.isdir(self.directory):
            print(f"ERROR: {self.directory} does not name a directory")
            sys.exit(1)

        try:
            self.name = (
                open(os.path.join(self.directory, "run_name"), "r").read().rstrip()
            )
        except FileNotFoundError:
            self.name = os.path.basename(self.directory)

        # get description of this git rev
        gitdiff = None
        try:
            gitdiff = open(os.path.join(self.directory, "git_diff"), "r").read()
        except FileNotFoundError:
            pass
        git_hash = open(os.path.join(self.directory, "git_hash"), "r").read().rstrip()
        self.git = GitRev(git_hash, diff=gitdiff)

        self.command = open(os.path.join(self.directory, "command"), "r").read()

        try:
            self.command_type = CommandType.from_string(
                open(os.path.join(self.directory, "command_type"), "r").read().rstrip()
            )
        except FileNotFoundError:
            print(
                f"WARNING: Could not find {os.path.join(self.directory, 'command_type')}. "
                "Parsing as UNKNOWN command type means kernels will be ungrouped.",
                file=sys.stderr,
            )
            self.command_type = CommandType.UNKNOWN

        try:
            self.env = ""
            for line in open(os.path.join(self.directory, "env"), "r").readlines():
                # remove $testdir which is set by compare_codegen.sh
                # NOTE: compare_codegen.sh should have already removed these lines
                if re.search(r"^testdir=", line) is None:
                    self.env += line
        except FileNotFoundError:
            self.env = None

        try:
            self.nvcc_version = open(
                os.path.join(self.directory, "nvcc_version"), "r"
            ).read()
        except FileNotFoundError:
            self.nvcc_version = None

        try:
            self.gpu_names = list(
                open(os.path.join(self.directory, "gpu_names"), "r").readlines()
            )
        except FileNotFoundError:
            self.gpu_names = None

        self.exit_code = int(open(os.path.join(self.directory, "exitcode"), "r").read())

        self.compute_kernel_map()

        self.find_preamble()

    def compute_kernel_map(self):
        """
        Compute a map from test name to list of cuda filenames
        """
        logfile = os.path.join(self.directory, "stdout")
        if not os.path.isfile(logfile):
            raise RuntimeError(
                f"Input directory {self.directory} contains no file named 'stdout'"
            )

        if self.command_type == CommandType.GOOGLETEST:
            parser = LogParserGTest(logfile)
        elif self.command_type == CommandType.GOOGLEBENCH:
            parser = LogParserGBench(logfile)
        elif self.command_type == CommandType.PYTEST:
            parser = LogParserPyTest(logfile)
        else:
            # The base class provides a parser that groups everything into a
            # single "test" called "Ungrouped Kernels"
            parser = LogParser(logfile)

        self.kernel_map = parser.kernel_map

    def find_preamble(self):
        """Look for common preamble in collected kernels"""
        preamble_lines = []
        first = True
        files_processed = 0  # limit how many files to check
        for cufile in os.listdir(os.path.join(self.directory, "cuda")):
            cufil
# ... truncated ...


GitRev

Example usage:
    python tools/diff_codegen_nvfuser_tests.py \
            codegen_comparison/{$commit1,$commit2}/binary_tests
"""

from dataclasses import asdict, dataclass, field, InitVar
import difflib
from enum import Enum
import os
import re
import subprocess
import sys

import cxxfilt
from dataclasses_json import dataclass_json

@dataclass_json
@dataclass
class GitRev:
    full_hash: str
    diff: str | None = None
    abbrev: str | None = None
    title: str | None = None
    author_name: str | None = None
    author_email: str | None = None
    author_time: str | None = None
    commit_time: str | None = None

    def __post_init__(self):
        if self.abbrev is not None:
            # Avoid running __post_init__ during deserialization
            return

        self.abbrev = (
            subprocess.run(
                ["git", "rev-parse", "--short", self.full_hash], capture_output=True
            )
            .stdout.strip()
            .decode("utf-8")
        )
        for line in (
            subprocess.run(
                ["git", "branch", "--quiet", "--color=never", self.full_hash],
                capture_output=True,
            )
            .stdout.strip()
            .decode("utf-8")
            .splitlines()
        ):
            # Possible output:
            #
            #     main
            #     * scalar_seg_edges
            #
            # In this case, we have checked out the HEAD of the
            # scalar_seg_edges branch. Here we just strip the *.
            if line[0] == "*":
                line = line[2:]
                in_branches.append(line)

        def git_show(fmt) -> str:
            return (
                subprocess.run(
                    [
                        "git",
                        "show",
                        "--no-patch",
                        f"--format={fmt}",
                        self.full_hash,
                    ],
                    capture_output=True,
                )
                .stdout.strip()
                .decode("utf-8")
            )

        self.title = git_show("%s")
        self.author_name = git_show("%an")
        self.author_email = git_show("%ae")
        self.author_time = git_show("%ad")
        self.commit_time = git_show("%cd")

@dataclass_json
@dataclass
class LaunchParams:
    blockDim: tuple[int]
    gridDim: tuple[int]
    dynamic_smem_bytes: int

@dataclass_json
@dataclass
class CompiledKernel:
    filename: str
    code: str | None = None
    ptx: str | None = None
    ptxas_info: str | None = None
    launch_params_str: str | None = None
    launch_params: LaunchParams | None = None
    gmem_bytes: int = 0


LaunchParams

                        "show",
                        "--no-patch",
                        f"--format={fmt}",
                        self.full_hash,
                    ],
                    capture_output=True,
                )
                .stdout.strip()
                .decode("utf-8")
            )

        self.title = git_show("%s")
        self.author_name = git_show("%an")
        self.author_email = git_show("%ae")
        self.author_time = git_show("%ad")
        self.commit_time = git_show("%cd")

@dataclass_json
@dataclass
class LaunchParams:
    blockDim: tuple[int]
    gridDim: tuple[int]
    dynamic_smem_bytes: int

@dataclass_json
@dataclass
class CompiledKernel:
    filename: str
    code: str | None = None
    ptx: str | None = None
    ptxas_info: str | None = None
    launch_params_str: str | None = None
    launch_params: LaunchParams | None = None
    gmem_bytes: int = 0
    smem_bytes: int = 0
    cmem_bank_bytes: list[int] | None = None
    registers: int | None = None
    stack_frame_bytes: int = 0
    spill_store_bytes: int = 0
    spill_load_bytes: int = 0
    mangled_name: str | None = None
    arch: str | None = None