eval_lm.py

import lm_eval
import sys


sys.path.insert(
    0,
    "."
)

results = lm_eval.simple_evaluate(

    model="obsidian_multiscreen",

    model_args=(
        "checkpoint=."
        ",device=cuda"
        ",dtype=bfloat16"
        ",backend=triton"
    ),

    tasks=[
        "hellaswag",
        "arc_easy",
        "arc_challenge",
        "piqa",
        "sciq",
        "blimp",
    ],

    batch_size=8,

    device="cuda",

    limit=None,

)

print(
    lm_eval.utils.make_table(results)
)