Spaces:
Running on Zero
Running on Zero
File size: 2,460 Bytes
fed954e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 | """qwen_test_runner — testbed for evaluating small Qwen models on JSON schema.
Public API:
from qwen_test_runner import Caption, QwenRunner, score_sample, score_run
from qwen_test_runner import SLOT_REGISTRY, SlotSpec # v0.2 registry
CLI:
qwen-bench --help
"""
from __future__ import annotations
__version__ = "0.2.0"
# Registry — single source of truth for all slot definitions
from .registry import (
SLOT_REGISTRY,
SlotSpec,
SubjectValue,
Category,
Cardinality,
Vocabulary,
Groundedness,
slots_by_category,
slot_names,
get_slot,
all_closed_vocab,
)
# Schema — generated from registry at import time
from .schema import (
Caption,
Subject, # alias for SubjectValue, kept for back-compat
CAPTION_JSON_SCHEMA,
CAPTION_GRAMMAR_GBNF,
build_gbnf_grammar,
)
# Evaluation — scoring functions
from .evaluator import (
parse_safely,
ground_check,
coverage_check,
score_sample,
score_run,
SampleResult,
RunMetrics,
GroundingReport,
CoverageReport,
)
# Eval data
from .eval_set import BUILTIN_CAPTIONS, load_eval_set
# Model runner — imported lazily so `import qwen_test_runner` doesn't drag in
# torch unless the user actually needs it. The names are still importable as
# `from qwen_test_runner import QwenRunner` thanks to __getattr__.
def __getattr__(name: str):
if name == "QwenRunner":
from .model_runner import QwenRunner
return QwenRunner
if name == "GenResult":
from .model_runner import GenResult
return GenResult
if name == "ClaudeProvider":
from .providers.claude_api import ClaudeProvider
return ClaudeProvider
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
__all__ = [
"__version__",
# registry
"SLOT_REGISTRY", "SlotSpec", "SubjectValue",
"Category", "Cardinality", "Vocabulary", "Groundedness",
"slots_by_category", "slot_names", "get_slot", "all_closed_vocab",
# schema (generated)
"Caption", "Subject",
"CAPTION_JSON_SCHEMA", "CAPTION_GRAMMAR_GBNF", "build_gbnf_grammar",
# evaluator
"parse_safely", "ground_check", "coverage_check",
"score_sample", "score_run",
"SampleResult", "RunMetrics", "GroundingReport", "CoverageReport",
# eval data
"BUILTIN_CAPTIONS", "load_eval_set",
# runners (lazy)
"QwenRunner", "GenResult", "ClaudeProvider",
]
|