source/src/strata/eval/__init__.py

"""STRATA evaluation harness.

Zero-shot, LM-scoring based evaluation of a trained ``StrataForCausalLM``: held-out
perplexity on packed corpora plus task adapters (minimal-pair grammaticality,
multiple-choice) for the fetched eval sets where gold labels exist. Nothing here
changes the training objective — it only reads a checkpoint and scores text.
"""

from __future__ import annotations

from strata.eval.perplexity import PerplexityResult, evaluate_perplexity
from strata.eval.scoring import encode_with_continuation, score_sequences
from strata.eval.tasks import (
    MinimalPairExample,
    MultipleChoiceExample,
    TaskResult,
    evaluate_minimal_pairs,
    evaluate_multiple_choice,
)

__all__ = [
    "PerplexityResult",
    "evaluate_perplexity",
    "score_sequences",
    "encode_with_continuation",
    "MinimalPairExample",
    "MultipleChoiceExample",
    "TaskResult",
    "evaluate_minimal_pairs",
    "evaluate_multiple_choice",
]