tests/test_output_quality.py

import sys
import os
import unittest
import json
import sacrebleu
from pathlib import Path

# Add project root and ankahi_backend to path
sys.path.insert(0, os.getcwd())

from ankahi_backend.model_loader import model_loader
from ankahi_backend.inference import predict

class TestOutputQuality(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        model_loader.load_base_model()
        cls.eval_data_path = "data/eval/eval_pictogram_to_sentence.jsonl"
        
        if not os.path.exists(cls.eval_data_path):
            raise unittest.SkipTest(f"Eval data not found at {cls.eval_data_path}")

    def test_chrf_score(self):
        with open(self.eval_data_path, 'r') as f:
            lines = f.readlines()[:20]  # Test subset for speed
            
        hypotheses = []
        references = []
        
        for line in lines:
            data = json.loads(line)
            persona_id = data["persona_id"]
            pictograms = data["pictogram_sequence"]
            target = data["target_sentence"]
            
            # Predict
            result = predict(pictograms, persona_id, context=data.get("context", "daily"))
            response = result["primary"]
            
            hypotheses.append(response)
            references.append([target])
            print(f"[{persona_id}] Hyp: {response[:40]}... | Ref: {target[:40]}...")
            
        # Calculate chrF++
        chrf = sacrebleu.corpus_chrf(hypotheses, references)
        print(f"\nFinal average chrF++ score: {chrf.score:.2f}")
        
        # Threshold (as per spec, aim for high quality)
        self.assertGreater(chrf.score, 40.0)

if __name__ == "__main__":
    unittest.main()