src/eval/evaluator.ts

/**
 * PRIX AI Evaluation System - Evaluation Engine
 * Compares test outputs against expected results
 */

import {
  TestOutput,
  ExpectedResult,
  TestScore,
  TestResult,
  TestCase,
  IssueType,
  ConfidenceLevel
} from './types'

export interface EvaluationOptions {
  strictConfidence?: boolean
  strictIssueType?: boolean
  allowPartialMatches?: boolean
}

export class Evaluator {
  private options: EvaluationOptions

  constructor(options: EvaluationOptions = {}) {
    this.options = {
      strictConfidence: false,
      strictIssueType: false,
      allowPartialMatches: true,
      ...options
    }
  }

  /**
   * Evaluate a single test output against expected results
   */
  evaluate(testCase: TestCase, output: TestOutput): TestResult {
    const startTime = Date.now()
    const score = this.calculateScore(output, testCase.expected)

    return {
      testName: testCase.name,
      testPath: testCase.path,
      category: testCase.category,
      passed: score.overall,
      score,
      output,
      expected: testCase.expected,
      executionTime: output.executionTime,
      timestamp: new Date().toISOString()
    }
  }

  /**
   * Calculate detailed score for a test
   */
  calculateScore(output: TestOutput, expected: ExpectedResult): TestScore {
    // Detection check
    const detectionMatch = expected.shouldDetectIssue === output.detectedIssues

    // Issue type check (if detection expected)
    let typeMatch = true
    if (expected.shouldDetectIssue && expected.issueType && output.issueType) {
      typeMatch = this.matchIssueType(expected.issueType, output.issueType)
    }

    // Confidence check
    let confidenceMatch = true
    if (expected.expectedConfidence && output.confidence) {
      confidenceMatch = this.matchConfidence(
        expected.expectedConfidence,
        output.confidence
      )
    }

    // Auto-PR check
    const autoPRMatch = expected.shouldAutoPR === output.autoPRTriggered

    // Suggestion count check (if specific count expected)
    let suggestionCountMatch = true
    if (expected.expectedSuggestions) {
      suggestionCountMatch =
        expected.expectedSuggestions.length === output.suggestions.length
    }

    // Calculate overall score
    const allChecks = [
      detectionMatch,
      typeMatch,
      confidenceMatch,
      autoPRMatch,
      suggestionCountMatch
    ]

    // Weight detection more heavily
    const overall =
      detectionMatch &&
      (typeMatch || !this.options.strictIssueType) &&
      (confidenceMatch || !this.options.strictConfidence) &&
      autoPRMatch

    return {
      detection: detectionMatch,
      classification: typeMatch,
      confidence: confidenceMatch,
      autoPR: autoPRMatch,
      overall,
      details: {
        detectionMatch,
        typeMatch,
        confidenceMatch,
        autoPRMatch,
        suggestionCountMatch
      }
    }
  }

  /**
   * Match issue types with flexibility
   */
  private matchIssueType(expected: IssueType, actual: IssueType): boolean {
    if (expected === actual) return true

    if (!this.options.strictIssueType) {
      // Allow some flexibility in classification
      const securityTypes: IssueType[] = ['security', 'bug']
      const bugTypes: IssueType[] = ['bug', 'mutation']

      if (securityTypes.includes(expected) && securityTypes.includes(actual))
        return true
      if (bugTypes.includes(expected) && bugTypes.includes(actual)) return true
    }

    return false
  }

  /**
   * Match confidence levels
   */
  private matchConfidence(
    expected: ConfidenceLevel,
    actual: ConfidenceLevel
  ): boolean {
    if (expected === actual) return true

    if (!this.options.strictConfidence) {
      // Allow adjacent confidence levels
      const levels: ConfidenceLevel[] = ['low', 'medium', 'high']
      const expectedIdx = levels.indexOf(expected)
      const actualIdx = levels.indexOf(actual)

      // Within 1 level is acceptable
      return Math.abs(expectedIdx - actualIdx) <= 1
    }

    return false
  }

  /**
   * Generate detailed evaluation report for a single test
   */
  generateReport(result: TestResult): string {
    const lines = [
      `\n${'='.repeat(60)}`,
      `Test: ${result.testName} (${result.category})`,
      `${'='.repeat(60)}`,
      `Status: ${result.passed ? '✅ PASS' : '❌ FAIL'}`,
      `Execution Time: ${result.executionTime}ms`,
      '',
      '--- Expected ---',
      `  Detect Issue: ${result.expected.shouldDetectIssue}`,
      `  Issue Type: ${result.expected.issueType || 'any'}`,
      `  Confidence: ${result.expected.expectedConfidence || 'any'}`,
      `  Auto-PR: ${result.expected.shouldAutoPR}`,
      '',
      '--- Actual ---',
      `  Detected: ${result.output.detectedIssues}`,
      `  Type: ${result.output.issueType || 'none'}`,
      `  Confidence: ${result.output.confidence || 'none'}`,
      `  Auto-PR: ${result.output.autoPRTriggered}`,
      `  Suggestions: ${result.output.suggestions.length}`,
      `  Comments: ${result.output.comments.length}`,
      '',
      '--- Score Breakdown ---',
      `  Detection: ${result.score.detection ? '✅' : '❌'}`,
      `  Classification: ${result.score.classification ? '✅' : '❌'}`,
      `  Confidence: ${result.score.confidence ? '✅' : '❌'}`,
      `  Auto-PR: ${result.score.autoPR ? '✅' : '❌'}`,
      '',
      '--- Errors ---',
      ...(result.output.errors.length > 0
        ? result.output.errors.map(e => `  ❌ ${e}`)
        : ['  None'])
    ]

    return lines.join('\n')
  }

  /**
   * Compare two test results for regression detection
   */
  compareResults(
    baseline: TestResult,
    current: TestResult
  ): {regressed: boolean; changes: string[]} {
    const changes: string[] = []

    if (baseline.passed && !current.passed) {
      changes.push(`Test ${current.testName} started failing`)
    }

    if (!baseline.passed && current.passed) {
      changes.push(`Test ${current.testName} now passing (improvement)`)
    }

    if (baseline.score.overall && !current.score.overall) {
      changes.push(`Overall score degraded for ${current.testName}`)
    }

    return {
      regressed: changes.some(
        c => c.includes('failing') || c.includes('degraded')
      ),
      changes
    }
  }
}

/**
 * Utility function for quick evaluation
 */
export function evaluate(
  output: TestOutput,
  expected: ExpectedResult,
  options?: EvaluationOptions
): TestScore {
  const evaluator = new Evaluator(options)
  return evaluator.calculateScore(output, expected)
}