torch_compile/run_ast.py

import argparse
import logging
import time

import torch
from transformers import AutoFeatureExtractor, ASTForAudioClassification
from datasets import load_dataset

import torch_neuronx  # ensure Neuron backend is available

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


def main():
    parser = argparse.ArgumentParser(description="Run AST (Audio Spectrogram Transformer) on Neuron")
    parser.add_argument(
        "--model",
        type=str,
        default="MIT/ast-finetuned-audioset-10-10-0.4593",
        help="AST model name on Hugging Face Hub",
    )
    parser.add_argument("--batch-size", type=int, default=1, help="Batch size")
    args = parser.parse_args()

    torch.set_default_dtype(torch.float32)
    torch.manual_seed(42)

    # Load dataset and extract features
    dataset = load_dataset("hf-internal-testing/librispeech_asr_demo", "clean", split="validation")
    dataset = dataset.sort("id")
    sampling_rate = dataset.features["audio"].sampling_rate

    feature_extractor = AutoFeatureExtractor.from_pretrained(args.model)
    inputs = feature_extractor(
        dataset[0]["audio"]["array"], sampling_rate=sampling_rate, return_tensors="pt"
    )

    # Load AST model
    model = ASTForAudioClassification.from_pretrained(
        args.model, torch_dtype=torch.float32, attn_implementation="eager"
    )
    model.eval()

    # Pre-run once to fix shapes before compilation
    with torch.no_grad():
        logits = model(**inputs).logits

    # Compile forward pass
    model.forward = torch.compile(model.forward, backend="neuron", fullgraph=True)

    # Warmup
    warmup_start = time.time()
    with torch.no_grad():
        _ = model(**inputs)
    warmup_time = time.time() - warmup_start

    # Actual run
    run_start = time.time()
    with torch.no_grad():
        logits = model(**inputs).logits
    run_time = time.time() - run_start

    # Decode result
    predicted_class_ids = torch.argmax(logits, dim=-1).item()
    predicted_label = model.config.id2label[predicted_class_ids]

    logger.info("Warmup: %.2f s, Run: %.4f s", warmup_time, run_time)
    logger.info("Predicted label: %s", predicted_label)


if __name__ == "__main__":
    main()
    
"""
Works
"""