File size: 3,414 Bytes
9fea2fb 3a559ac 9fea2fb 3a559ac 9fea2fb 3a559ac 9fea2fb 3a559ac 9fea2fb 3a559ac 9fea2fb 3a559ac 9fea2fb 3a559ac 9fea2fb 3a559ac 9fea2fb 3a559ac 9fea2fb 3a559ac 9fea2fb 3a559ac 9fea2fb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | #!/usr/bin/env python3
"""Evaluate model.onnx against a JSONL set with text and label fields."""
import argparse
import json
import math
from collections import defaultdict
import numpy as np
import onnxruntime as ort
def predict(session, text):
encoded = text.encode("utf-8", errors="ignore")[:256]
values = np.zeros((1, 256), dtype=np.int64)
values[0, : len(encoded)] = list(encoded)
logits = session.run(["logits"], {"input_bytes": values})[0][0]
return "EN" if int(np.argmax(logits)) == 1 else "NOT-EN", float(logits[1] - logits[0])
def main():
parser = argparse.ArgumentParser()
parser.add_argument("dataset", help="JSONL file containing text and label")
parser.add_argument("--model", default="model.onnx")
parser.add_argument("--max-errors", type=int, default=20)
args = parser.parse_args()
session = ort.InferenceSession(args.model, providers=["CPUExecutionProvider"])
rows = [json.loads(line) for line in open(args.dataset, encoding="utf-8") if line.strip()]
correct = 0
by_category = defaultdict(lambda: [0, 0])
by_language = defaultdict(lambda: [0, 0])
errors = []
predictions = []
for row in rows:
prediction, margin = predict(session, row["text"])
predictions.append(prediction)
category = row.get("category", "uncategorized")
language = row.get("language", "uncategorized")
by_category[category][1] += 1
by_language[language][1] += 1
correct += prediction == row["label"]
by_category[category][0] += prediction == row["label"]
by_language[language][0] += prediction == row["label"]
if prediction != row["label"]:
errors.append({"id": row["id"], "text": row["text"], "expected": row["label"], "predicted": prediction, "margin": round(margin, 4)})
accuracy = correct / len(rows)
margin = 1.96 * math.sqrt(accuracy * (1 - accuracy) / len(rows))
print(f"accuracy: {correct}/{len(rows)} = {accuracy:.4%} (95% Wald CI {max(0, accuracy - margin):.4%}-{min(1, accuracy + margin):.4%})")
labels = {label: [0, 0] for label in ("EN", "NOT-EN")}
for row, prediction in zip(rows, predictions):
labels[row["label"]][0] += prediction == row["label"]
labels[row["label"]][1] += 1
print("class accuracy:")
for label in labels:
hit, total = labels[label]
print(f" {label}: {hit}/{total} = {hit / total:.4%}")
macro = sum(hit / total for hit, total in by_language.values()) / len(by_language)
non_english_languages = [lang for lang in by_language if lang != "en"]
non_english_macro = sum(by_language[lang][0] / by_language[lang][1] for lang in non_english_languages) / len(non_english_languages)
print(f"language macro accuracy: {macro:.4%} across {len(by_language)} languages")
print(f"non-English language macro accuracy: {non_english_macro:.4%} across {len(non_english_languages)} languages")
print("category accuracy:")
for category in sorted(by_category):
hit, total = by_category[category]
print(f" {category}: {hit}/{total} = {hit / total:.4%}")
print(f"errors: {len(errors)}")
for error in errors[: args.max_errors]:
print(json.dumps(error, ensure_ascii=False))
if len(errors) > args.max_errors:
print(f"... {len(errors) - args.max_errors} additional errors omitted")
if __name__ == "__main__":
main()
|