app/classify.py

"""Bin a pytest trace into a coarse failure category.

The goal is a single tag per failure that aggregates well in a report. We do
this with cheap regex on the raw trace string. Categories, checked in order:

  OOM             — torch OutOfMemoryError, "CUDA out of memory"
  load_error      — failure raised before forward (from_pretrained, safetensors,
                    HFValidationError, missing weight, gated/auth)
  cuda_runtime    — runtime CUDA/cuBLAS/cuDNN/nvrtc/Triton compile errors
  output_mismatch — text / tensor comparison failures (assertEqual, Expectations,
                    Tensor-likes are not close, etc.)
  import_or_config — Python-level errors before the test body runs
                    (ImportError, AttributeError on config, TypeError on
                    __init__ signatures).
  other           — fallback
"""

from __future__ import annotations

import re


_OOM_PAT = re.compile(r"OutOfMemoryError|CUDA out of memory|MallocFailure|HIP out of memory", re.I)
_LOAD_PAT = re.compile(
    r"from_pretrained|safetensors\.|HFValidationError|Repository Not Found|gated|"
    r"Cannot read|UnboundLocalError.*loading|FileNotFoundError|access requested|"
    r"401 Client Error|403 Client Error",
    re.I,
)
_CUDA_RUNTIME_PAT = re.compile(
    r"CUDA error|CUBLAS_STATUS|CUDNN_STATUS|cudnn[_ ]frontend|nvrtc|"
    r"triton\.compiler|RuntimeError: Triton|c10::Error|NCCL.*error",
    re.I,
)
_OUTPUT_MISMATCH_PAT = re.compile(
    r"Tensor-likes are not close|"
    r"assertEqual|assertSequenceEqual|self\.assertListEqual|"
    r"assertAlmostEqual|assertGreater|expected_text|"
    r"AssertionError",  # generic fallback — most assertion failures are output mismatches
    re.I | re.DOTALL,
)
_IMPORT_CFG_PAT = re.compile(
    r"^.*ImportError|ModuleNotFoundError|"
    r"AttributeError:.*(config|object has no attribute)|"
    r"TypeError:.*(__init__|got an unexpected keyword argument)|"
    r"ValueError:.*Unrecognized configuration",
    re.I | re.M,
)


def classify(trace: str) -> str:
    if not trace:
        return "other"
    for tag, pat in (
        ("OOM", _OOM_PAT),
        ("load_error", _LOAD_PAT),
        ("cuda_runtime", _CUDA_RUNTIME_PAT),
        ("import_or_config", _IMPORT_CFG_PAT),
        ("output_mismatch", _OUTPUT_MISMATCH_PAT),
    ):
        if pat.search(trace):
            return tag
    return "other"


def short_excerpt(trace: str, max_chars: int = 240) -> str:
    """Take the LAST non-empty line of the trace (the actual exception line).
    Then trim to `max_chars`.
    """
    if not trace:
        return ""
    for line in reversed(trace.splitlines()):
        line = line.strip()
        if line:
            return (line[: max_chars - 1] + "…") if len(line) > max_chars else line
    return ""


if __name__ == "__main__":
    samples = [
        "tests/...py:42: torch.OutOfMemoryError: CUDA out of memory.",
        "RuntimeError: CUDA error: CUBLAS_STATUS_EXECUTION_FAILED",
        "AssertionError: 'Paris' != 'capital of France'",
        "ImportError: cannot import name 'foo'",
        "HFValidationError: Repository Not Found for url: ...",
        "Tensor-likes are not close (...) max_abs_diff=0.5",
    ]
    for s in samples:
        print(f"{classify(s):<18}  {s}")