app/inference.py

"""
inference.py — Core logic for all 3 model variants.

Variants:
  1. Baseline   — Standard HuggingFace PyTorch model
  2. ONNX       — Exported to ONNX format (faster runtime)
  3. Quantized  — INT8 quantized ONNX (fastest + smallest)

Each variant returns: label, confidence, latency_ms, model_size_mb
"""

import math
import time
import os
import numpy as np
from pathlib import Path

MODEL_DIR = Path("models")
ONNX_PATH = MODEL_DIR / "model.onnx"
QUANTIZED_PATH = MODEL_DIR / "model_quantized.onnx"

_baseline_pipeline = None
_onnx_session = None
_quantized_session = None
_tokenizer = None


def _get_file_size_mb(path: str) -> float:
    try:
        return round(os.path.getsize(path) / (1024 * 1024), 1)
    except FileNotFoundError:
        return 0.0


def _load_baseline():
    global _baseline_pipeline
    if _baseline_pipeline is None:
        from transformers import pipeline
        print("[INFO] Loading baseline PyTorch model...")
        _baseline_pipeline = pipeline(
            "sentiment-analysis",
            model="distilbert-base-uncased-finetuned-sst-2-english",
            device=-1
        )
    return _baseline_pipeline


def _load_tokenizer():
    global _tokenizer
    if _tokenizer is None:
        from transformers import AutoTokenizer
        _tokenizer = AutoTokenizer.from_pretrained(
            "distilbert-base-uncased-finetuned-sst-2-english"
        )
    return _tokenizer


def _export_to_onnx():
    """Export model to ONNX using optimum library."""
    if not ONNX_PATH.exists():
        print("[INFO] Exporting model to ONNX (one-time setup)...")
        MODEL_DIR.mkdir(exist_ok=True)
        from optimum.onnxruntime import ORTModelForSequenceClassification
        model = ORTModelForSequenceClassification.from_pretrained(
            "distilbert-base-uncased-finetuned-sst-2-english",
            export=True
        )
        model.save_pretrained(str(MODEL_DIR))
        # Find and rename if needed
        if not ONNX_PATH.exists():
            import shutil
            for f in MODEL_DIR.glob("*.onnx"):
                shutil.copy(str(f), str(ONNX_PATH))
                break
        print(f"[INFO] ONNX model saved to {ONNX_PATH}")


def _quantize_onnx():
    """Quantize ONNX model to INT8 if not already done."""
    if not QUANTIZED_PATH.exists():
        _export_to_onnx()
        print("[INFO] Quantizing ONNX model to INT8...")
        from onnxruntime.quantization import quantize_dynamic, QuantType
        quantize_dynamic(
            str(ONNX_PATH),
            str(QUANTIZED_PATH),
            weight_type=QuantType.QInt8
        )
        print(f"[INFO] Quantized model saved to {QUANTIZED_PATH}")


def _load_onnx_session(path: str):
    import onnxruntime as ort
    opts = ort.SessionOptions()
    opts.intra_op_num_threads = 4
    return ort.InferenceSession(str(path), sess_options=opts)


def _run_onnx_inference(session, text: str) -> dict:
    tokenizer = _load_tokenizer()
    inputs = tokenizer(text, return_tensors="np", truncation=True, max_length=128)
    feed = {k: v for k, v in inputs.items() if k in [i.name for i in session.get_inputs()]}
    outputs = session.run(None, feed)
    logits = outputs[0][0]
    exp_logits = np.exp(logits - np.max(logits))
    probs = exp_logits / exp_logits.sum()
    label_idx = int(np.argmax(probs))
    labels = ["NEGATIVE", "POSITIVE"]
    return {
        "label": labels[label_idx],
        "confidence": round(float(probs[label_idx]), 4)
    }


def run_baseline(text: str) -> dict:
    pipeline = _load_baseline()
    start = time.perf_counter()
    result = pipeline(text)[0]
    latency = (time.perf_counter() - start) * 1000
    return {
        "label": result["label"],
        "confidence": round(result["score"], 4),
        "latency_ms": round(latency, 2),
        "model_size_mb": 268.0,
        "format": "PyTorch (.bin)"
    }


def run_onnx(text: str) -> dict:
    global _onnx_session
    _export_to_onnx()
    if _onnx_session is None:
        _onnx_session = _load_onnx_session(ONNX_PATH)
    start = time.perf_counter()
    prediction = _run_onnx_inference(_onnx_session, text)
    latency = (time.perf_counter() - start) * 1000
    return {
        **prediction,
        "latency_ms": round(latency, 2),
        "model_size_mb": _get_file_size_mb(ONNX_PATH) or 268.0,
        "format": "ONNX (.onnx)"
    }


def run_quantized(text: str) -> dict:
    global _quantized_session
    _quantize_onnx()
    if _quantized_session is None:
        _quantized_session = _load_onnx_session(QUANTIZED_PATH)
    start = time.perf_counter()
    prediction = _run_onnx_inference(_quantized_session, text)
    latency = (time.perf_counter() - start) * 1000
    return {
        **prediction,
        "latency_ms": round(latency, 2),
        "model_size_mb": _get_file_size_mb(QUANTIZED_PATH) or 68.0,
        "format": "Quantized ONNX INT8 (.onnx)"
    }


def run_all_models(text: str) -> dict:
    return {
        "baseline": run_baseline(text),
        "onnx": run_onnx(text),
        "quantized": run_quantized(text),
    }


def _percentile(values: list[float], percentile: float) -> float:
    if not values:
        return 0.0
    sorted_values = sorted(values)
    index = math.ceil((percentile / 100) * len(sorted_values)) - 1
    index = max(0, min(index, len(sorted_values) - 1))
    return sorted_values[index]


def run_benchmark(text: str, iterations: int = 20) -> dict:
    latencies = {
        "baseline": [],
        "onnx": [],
        "quantized": [],
    }
    latest_results = None

    for _ in range(iterations):
        latest_results = run_all_models(text)
        for model_name, result in latest_results.items():
            latencies[model_name].append(result["latency_ms"])

    stats = {}
    for model_name, model_latencies in latencies.items():
        latest = latest_results[model_name] if latest_results else {}
        stats[model_name] = {
            "avg_latency_ms": round(sum(model_latencies) / len(model_latencies), 2),
            "min_latency_ms": round(min(model_latencies), 2),
            "max_latency_ms": round(max(model_latencies), 2),
            "p95_latency_ms": round(_percentile(model_latencies, 95), 2),
            "model_size_mb": latest.get("model_size_mb", 0.0),
            "format": latest.get("format", ""),
        }

    return {
        "iterations": iterations,
        "latest_results": latest_results,
        "results": stats,
    }