code/flash-linear-attention/scripts/check_gpu.py

import subprocess
import time

from fla.utils import device_platform


def get_xpu_memory_usage():
    """Run `xpu-smi stats -d 0` and parse its output."""
    try:
        result = subprocess.run(
            ["xpu-smi", "stats", "-d", "0"],
            capture_output=True,
            text=True,
            check=True,
        )
        return extract_gpu_memory_used(result.stdout)
    except subprocess.CalledProcessError as e:
        print(f"Failed to run xpu-smi: {e}")
        return None


def extract_gpu_memory_used(output):
    """Extract 'GPU Memory Used (MiB)' from xpu-smi output."""
    # Use regex to find the line containing "GPU Memory Used (MiB)"
    output = output.strip().replace(" ", "")

    prefix = "|GPUMemoryUsed(MiB)|"
    start_idx = output.find(prefix)
    value_start = start_idx + len(prefix)
    end_idx = output.find("|", value_start+2)
    value_str = output[value_start:end_idx-2].strip()
    if value_str:
        return int(value_str)
    else:
        print("Could not find GPU memory usage in xpu-smi output.")
        return None


def get_nvgpu_memory_usage():
    result = subprocess.run(
        ["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"],
        capture_output=True,
        text=True,
    )
    memory_used = int(result.stdout.strip())
    return memory_used


def check_gpu_memory():
    max_memory_mib = 4096  # Threshold in MiB (4 GB)
    max_wait_time = 3600    # 60 minutes in seconds
    sleep_time = 30       # Sleep for 30 seconds

    start_time = time.time()

    while True:

        # Extract GPU memory usage
        if device_platform == 'intel':
            # memory_used_mib = get_xpu_memory_usage()
            # since xpu-smi have conflicts in apt
            memory_used_mib = 0
        elif device_platform == 'nvidia':
            memory_used_mib = get_nvgpu_memory_usage()
        if memory_used_mib is None:
            exit(1)

        print(f"Current GPU memory usage: {memory_used_mib} MiB")

        if memory_used_mib > max_memory_mib:
            print(f"GPU memory usage exceeds {max_memory_mib} MiB. Sleeping for {sleep_time} seconds...")
            time.sleep(sleep_time)
        else:
            print("GPU memory usage is within limits.")
            exit(0)

        if time.time() - start_time > max_wait_time:
            print("GPU memory usage remains high for 10 minutes. Skipping this action.")
            exit(1)


if __name__ == "__main__":
    check_gpu_memory()