lib/src/eval/evaluator.js

"use strict";
/**
 * PRIX AI Evaluation System - Evaluation Engine
 * Compares test outputs against expected results
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.Evaluator = void 0;
exports.evaluate = evaluate;
class Evaluator {
    options;
    constructor(options = {}) {
        this.options = {
            strictConfidence: false,
            strictIssueType: false,
            allowPartialMatches: true,
            ...options
        };
    }
    /**
     * Evaluate a single test output against expected results
     */
    evaluate(testCase, output) {
        const startTime = Date.now();
        const score = this.calculateScore(output, testCase.expected);
        return {
            testName: testCase.name,
            testPath: testCase.path,
            category: testCase.category,
            passed: score.overall,
            score,
            output,
            expected: testCase.expected,
            executionTime: output.executionTime,
            timestamp: new Date().toISOString()
        };
    }
    /**
     * Calculate detailed score for a test
     */
    calculateScore(output, expected) {
        // Detection check
        const detectionMatch = expected.shouldDetectIssue === output.detectedIssues;
        // Issue type check (if detection expected)
        let typeMatch = true;
        if (expected.shouldDetectIssue && expected.issueType && output.issueType) {
            typeMatch = this.matchIssueType(expected.issueType, output.issueType);
        }
        // Confidence check
        let confidenceMatch = true;
        if (expected.expectedConfidence && output.confidence) {
            confidenceMatch = this.matchConfidence(expected.expectedConfidence, output.confidence);
        }
        // Auto-PR check
        const autoPRMatch = expected.shouldAutoPR === output.autoPRTriggered;
        // Suggestion count check (if specific count expected)
        let suggestionCountMatch = true;
        if (expected.expectedSuggestions) {
            suggestionCountMatch =
                expected.expectedSuggestions.length === output.suggestions.length;
        }
        // Calculate overall score
        const allChecks = [
            detectionMatch,
            typeMatch,
            confidenceMatch,
            autoPRMatch,
            suggestionCountMatch
        ];
        // Weight detection more heavily
        const overall = detectionMatch &&
            (typeMatch || !this.options.strictIssueType) &&
            (confidenceMatch || !this.options.strictConfidence) &&
            autoPRMatch;
        return {
            detection: detectionMatch,
            classification: typeMatch,
            confidence: confidenceMatch,
            autoPR: autoPRMatch,
            overall,
            details: {
                detectionMatch,
                typeMatch,
                confidenceMatch,
                autoPRMatch,
                suggestionCountMatch
            }
        };
    }
    /**
     * Match issue types with flexibility
     */
    matchIssueType(expected, actual) {
        if (expected === actual)
            return true;
        if (!this.options.strictIssueType) {
            // Allow some flexibility in classification
            const securityTypes = ['security', 'bug'];
            const bugTypes = ['bug', 'mutation'];
            if (securityTypes.includes(expected) && securityTypes.includes(actual))
                return true;
            if (bugTypes.includes(expected) && bugTypes.includes(actual))
                return true;
        }
        return false;
    }
    /**
     * Match confidence levels
     */
    matchConfidence(expected, actual) {
        if (expected === actual)
            return true;
        if (!this.options.strictConfidence) {
            // Allow adjacent confidence levels
            const levels = ['low', 'medium', 'high'];
            const expectedIdx = levels.indexOf(expected);
            const actualIdx = levels.indexOf(actual);
            // Within 1 level is acceptable
            return Math.abs(expectedIdx - actualIdx) <= 1;
        }
        return false;
    }
    /**
     * Generate detailed evaluation report for a single test
     */
    generateReport(result) {
        const lines = [
            `\n${'='.repeat(60)}`,
            `Test: ${result.testName} (${result.category})`,
            `${'='.repeat(60)}`,
            `Status: ${result.passed ? '✅ PASS' : '❌ FAIL'}`,
            `Execution Time: ${result.executionTime}ms`,
            '',
            '--- Expected ---',
            `  Detect Issue: ${result.expected.shouldDetectIssue}`,
            `  Issue Type: ${result.expected.issueType || 'any'}`,
            `  Confidence: ${result.expected.expectedConfidence || 'any'}`,
            `  Auto-PR: ${result.expected.shouldAutoPR}`,
            '',
            '--- Actual ---',
            `  Detected: ${result.output.detectedIssues}`,
            `  Type: ${result.output.issueType || 'none'}`,
            `  Confidence: ${result.output.confidence || 'none'}`,
            `  Auto-PR: ${result.output.autoPRTriggered}`,
            `  Suggestions: ${result.output.suggestions.length}`,
            `  Comments: ${result.output.comments.length}`,
            '',
            '--- Score Breakdown ---',
            `  Detection: ${result.score.detection ? '✅' : '❌'}`,
            `  Classification: ${result.score.classification ? '✅' : '❌'}`,
            `  Confidence: ${result.score.confidence ? '✅' : '❌'}`,
            `  Auto-PR: ${result.score.autoPR ? '✅' : '❌'}`,
            '',
            '--- Errors ---',
            ...(result.output.errors.length > 0
                ? result.output.errors.map(e => `  ❌ ${e}`)
                : ['  None'])
        ];
        return lines.join('\n');
    }
    /**
     * Compare two test results for regression detection
     */
    compareResults(baseline, current) {
        const changes = [];
        if (baseline.passed && !current.passed) {
            changes.push(`Test ${current.testName} started failing`);
        }
        if (!baseline.passed && current.passed) {
            changes.push(`Test ${current.testName} now passing (improvement)`);
        }
        if (baseline.score.overall && !current.score.overall) {
            changes.push(`Overall score degraded for ${current.testName}`);
        }
        return {
            regressed: changes.some(c => c.includes('failing') || c.includes('degraded')),
            changes
        };
    }
}
exports.Evaluator = Evaluator;
/**
 * Utility function for quick evaluation
 */
function evaluate(output, expected, options) {
    const evaluator = new Evaluator(options);
    return evaluator.calculateScore(output, expected);
}
//# sourceMappingURL=evaluator.js.map