Spaces:
Running
Running
File size: 2,389 Bytes
bf76155 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | # tests/test_runner.py
"""
FinanceEval – Safe Test Runner
No runtime writes. Uses only in-memory comparisons.
"""
import os, json
import pandas as pd
from core.providers import get_provider, ProviderKind
from core.preprocess import normalize_conversation, extract_model_utterances
from core.evaluators import evaluate_all_metrics
from core.fusion import weighted_total
from core.schema import METRIC_ORDER
# Load static test inputs (read-only)
TEST_INPUTS_PATH = os.path.join(os.path.dirname(__file__), "redteam_inputs.jsonl")
GOLDEN_OUTPUTS_PATH = os.path.join(os.path.dirname(__file__), "golden_outputs.json")
def load_redteam_inputs():
with open(TEST_INPUTS_PATH, "r") as f:
return [json.loads(line) for line in f]
def load_golden_outputs():
if os.path.exists(GOLDEN_OUTPUTS_PATH):
with open(GOLDEN_OUTPUTS_PATH, "r") as f:
return json.load(f)
return {}
def run_one(provider, conversation_text, alpha_map):
norm = normalize_conversation(conversation_text)
model_only = extract_model_utterances(norm)
metrics_out, usage, raw_json = evaluate_all_metrics(
provider=provider,
conversation_text=model_only,
alpha_map=alpha_map
)
# Return dict only, no file writes
return {
"metrics": {m: v["fused_0_10"] for m, v in metrics_out.items()},
"usage": usage,
"raw": raw_json
}
if __name__ == "__main__":
alpha_map = {
"trust": 0.70, "accuracy": 0.65, "explain": 0.50,
"client_first": 0.70, "risk_safety": 0.60, "clarity": 0.70
}
inputs = load_redteam_inputs()
goldens = load_golden_outputs()
# Example: run against OpenAI GPT-4o if key is available
if os.environ.get("OPENAI_API_KEY"):
provider = get_provider(ProviderKind.OPENAI, "gpt-4o")
for case in inputs:
convo = case["conversation"]
notes = case.get("notes", "")
result = run_one(provider, convo, alpha_map)
print("=== CASE ===")
print(notes)
print(pd.DataFrame([result["metrics"]]))
print("Token usage:", result["usage"])
# Golden comparison (if available)
# No saving, just console diff
case_key = notes or convo[:30]
if case_key in goldens:
print("Golden vs Result:", goldens[case_key], result["metrics"])
|