verify.py

import argparse
import os
import sys
import torch

sys.path.insert(0, os.path.dirname(__file__))
from modeling import load_model, encode_text, pad_batch, TAXONOMY, JIGSAW_LABELS

SAMPLE = "You are an idiot and I will find you. My email is jane.doe@example.com."
VERIFICATION = {
    "tax": [
        0.3051,
        0.0119,
        0.1066,
        0.0103,
        0.4735,
        0.4706,
        0.4947,
        0.0363,
        0.4683,
        0.0714,
        0.4361,
        0.0968,
        0.3022,
        0.0607,
        0.0827,
        0.038,
        0.0001,
        0.5119,
        0.5415,
    ],
    "jig": [0.6895, 0.025, 0.2517, 0.0263, 0.4001, 0.244],
    "tol": 0.02,
}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--weights", default=os.environ.get("FELA_MODERATION_WEIGHTS", "."))
    args = ap.parse_args()
    model = load_model(args.weights)
    ids, _ = encode_text(SAMPLE, model.cfg.max_len)
    input_ids, mask = pad_batch([ids], model.cfg.max_len)
    with torch.no_grad():
        out = model(input_ids, mask, task="both")
    tax = torch.sigmoid(out["taxonomy"][0]).tolist()
    jig = torch.sigmoid(out["jigsaw"][0]).tolist()
    pii_shape = tuple(out["pii"].shape)
    exp_tax = (1, len(TAXONOMY))
    exp_jig = (1, len(JIGSAW_LABELS))
    if tuple(out["taxonomy"].shape) != exp_tax or tuple(out["jigsaw"].shape) != exp_jig:
        print(
            f"FAILURE: head shapes {tuple(out['taxonomy'].shape)}, {tuple(out['jigsaw'].shape)} != expected {exp_tax}, {exp_jig}"
        )
        sys.exit(1)
    if pii_shape[0] != 1 or pii_shape[2] != model.cfg.n_pii_tags:
        print(
            f"FAILURE: pii shape {pii_shape} unexpected (n_pii_tags {model.cfg.n_pii_tags})"
        )
        sys.exit(1)
    print(
        f"Shapes verified OK: taxonomy {tuple(out['taxonomy'].shape)}, jigsaw {tuple(out['jigsaw'].shape)}, pii {pii_shape}"
    )
    print("Captured taxonomy probs: " + str([round(p, 4) for p in tax]))
    print("Captured jigsaw probs: " + str([round(p, 4) for p in jig]))
    tol = VERIFICATION["tol"]
    diffs = [abs(a - b) for a, b in zip(tax, VERIFICATION["tax"])]
    diffs += [abs(a - b) for a, b in zip(jig, VERIFICATION["jig"])]
    if max(diffs) > tol:
        print(
            f"FAILURE: probs vs verification max abs diff {max(diffs):.4f} > tol {tol}"
        )
        sys.exit(1)
    print(f"Verification check OK (max abs diff {max(diffs):.4f} <= tol {tol})")


if __name__ == "__main__":
    main()