Create test_runner.py

tests/test_runner.py

@@ -0,0 +1,68 @@
+# tests/test_runner.py
+"""
+FinanceEval – Safe Test Runner
+No runtime writes. Uses only in-memory comparisons.
+"""
+
+import os, json
+import pandas as pd
+from core.providers import get_provider, ProviderKind
+from core.preprocess import normalize_conversation, extract_model_utterances
+from core.evaluators import evaluate_all_metrics
+from core.fusion import weighted_total
+from core.schema import METRIC_ORDER
+
+# Load static test inputs (read-only)
+TEST_INPUTS_PATH = os.path.join(os.path.dirname(__file__), "redteam_inputs.jsonl")
+GOLDEN_OUTPUTS_PATH = os.path.join(os.path.dirname(__file__), "golden_outputs.json")
+
+def load_redteam_inputs():
+    with open(TEST_INPUTS_PATH, "r") as f:
+        return [json.loads(line) for line in f]
+
+def load_golden_outputs():
+    if os.path.exists(GOLDEN_OUTPUTS_PATH):
+        with open(GOLDEN_OUTPUTS_PATH, "r") as f:
+            return json.load(f)
+    return {}
+
+def run_one(provider, conversation_text, alpha_map):
+    norm = normalize_conversation(conversation_text)
+    model_only = extract_model_utterances(norm)
+    metrics_out, usage, raw_json = evaluate_all_metrics(
+        provider=provider,
+        conversation_text=model_only,
+        alpha_map=alpha_map
+    )
+    # Return dict only, no file writes
+    return {
+        "metrics": {m: v["fused_0_10"] for m, v in metrics_out.items()},
+        "usage": usage,
+        "raw": raw_json
+    }
+
+if __name__ == "__main__":
+    alpha_map = {
+        "trust": 0.70, "accuracy": 0.65, "explain": 0.50,
+        "client_first": 0.70, "risk_safety": 0.60, "clarity": 0.70
+    }
+
+    inputs = load_redteam_inputs()
+    goldens = load_golden_outputs()
+
+    # Example: run against OpenAI GPT-4o if key is available
+    if os.environ.get("OPENAI_API_KEY"):
+        provider = get_provider(ProviderKind.OPENAI, "gpt-4o")
+        for case in inputs:
+            convo = case["conversation"]
+            notes = case.get("notes", "")
+            result = run_one(provider, convo, alpha_map)
+            print("=== CASE ===")
+            print(notes)
+            print(pd.DataFrame([result["metrics"]]))
+            print("Token usage:", result["usage"])
+            # Golden comparison (if available)
+            # No saving, just console diff
+            case_key = notes or convo[:30]
+            if case_key in goldens:
+                print("Golden vs Result:", goldens[case_key], result["metrics"])