lib/src/eval/cli.js

#!/usr/bin/env node
"use strict";
/**
 * PRIX AI Evaluation System - CLI
 * Command-line interface for running evaluations
 */
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
Object.defineProperty(exports, "__esModule", { value: true });
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const test_runner_1 = require("./test-runner");
const evaluator_1 = require("./evaluator");
const metrics_1 = require("./metrics");
const DEFAULT_TESTS_DIR = path.join(process.cwd(), 'tests');
const DEFAULT_OUTPUT_DIR = path.join(process.cwd(), 'eval-output');
const BASELINE_PATH = path.join(DEFAULT_OUTPUT_DIR, 'baseline_metrics.json');
const HISTORY_PATH = path.join(DEFAULT_OUTPUT_DIR, 'metrics_history.json');
function parseArgs() {
    const args = process.argv.slice(2);
    const options = {
        command: 'eval',
        testsDir: DEFAULT_TESTS_DIR,
        outputDir: DEFAULT_OUTPUT_DIR,
        verbose: false,
        format: 'text',
        failOnRegression: false
    };
    for (let i = 0; i < args.length; i++) {
        const arg = args[i];
        switch (arg) {
            case 'eval':
            case 'single':
            case 'compare':
            case 'baseline':
            case 'report':
                options.command = arg;
                break;
            case '--test':
            case '-t':
                options.testPath = args[++i];
                break;
            case '--tests-dir':
            case '-d':
                options.testsDir = args[++i];
                break;
            case '--output':
            case '-o':
                options.outputDir = args[++i];
                break;
            case '--verbose':
            case '-v':
                options.verbose = true;
                break;
            case '--format':
            case '-f':
                options.format = args[++i];
                break;
            case '--fail-on-regression':
                options.failOnRegression = true;
                break;
            case '--help':
            case '-h':
                printHelp();
                process.exit(0);
        }
    }
    return options;
}
function printHelp() {
    console.log(`
PRIX AI Evaluation System

Usage: npm run test:eval [command] [options]

Commands:
  eval       Run all test cases (default)
  single     Run a single test case (--test required)
  compare    Compare current results with baseline
  baseline   Save current results as new baseline
  report     Generate report from last run

Options:
  -t, --test <path>        Path to specific test case
  -d, --tests-dir <path>   Directory containing tests (default: ./tests)
  -o, --output <path>      Output directory (default: ./eval-output)
  -v, --verbose            Enable verbose logging
  -f, --format <format>    Output format: text, json, csv, markdown
  --fail-on-regression     Exit with error if regression detected
  -h, --help               Show this help

Examples:
  npm run test:eval                    # Run all tests
  npm run test:eval single -t tests/bugs/mutation  # Run single test
  npm run test:eval compare            # Compare with baseline
  npm run test:eval baseline           # Save new baseline
  npm run test:eval -f json            # Output as JSON
`);
}
async function runAllTests(options) {
    console.log('🚀 PRIX AI Evaluation System');
    console.log(`📁 Tests directory: ${options.testsDir}`);
    console.log(`📁 Output directory: ${options.outputDir}`);
    console.log('');
    const runner = new test_runner_1.TestRunner({
        testsDir: options.testsDir,
        outputDir: options.outputDir,
        verbose: options.verbose,
        skipAutoPR: true
    });
    const testCases = runner.loadTestCases();
    if (testCases.length === 0) {
        console.error('❌ No test cases found!');
        process.exit(1);
    }
    console.log(`📋 Found ${testCases.length} test case(s)`);
    console.log('');
    const evaluator = new evaluator_1.Evaluator();
    const results = [];
    for (const testCase of testCases) {
        const output = await runner.runTestCase(testCase);
        const result = evaluator.evaluate(testCase, output);
        results.push(result);
        // Print individual result
        if (options.format === 'text') {
            console.log(evaluator.generateReport(result));
        }
    }
    // Aggregate metrics
    const metrics = new metrics_1.MetricsAggregator({
        baselinePath: path.join(options.outputDir, 'baseline_metrics.json'),
        historyPath: path.join(options.outputDir, 'metrics_history.json')
    });
    const summary = metrics.aggregate(results);
    const regression = metrics.detectRegression(results);
    // Save results
    const resultsPath = path.join(options.outputDir, 'last_results.json');
    fs.writeFileSync(resultsPath, JSON.stringify({ results, summary }, null, 2));
    // Append to history
    metrics.appendHistory(results);
    // Generate and print report
    if (options.format === 'text') {
        console.log(metrics.generateReport(summary, regression));
    }
    else {
        console.log(metrics.exportMetrics(summary, options.format));
    }
    // Exit with error if regression detected and flag is set
    if (options.failOnRegression && regression.hasRegression) {
        console.error('\n❌ REGRESSION DETECTED - Exiting with error');
        process.exit(1);
    }
    // Exit with error if any tests failed
    if (summary.failedTests > 0) {
        process.exit(1);
    }
}
async function runSingleTest(options) {
    if (!options.testPath) {
        console.error('❌ --test path is required for single test mode');
        process.exit(1);
    }
    const fullPath = path.resolve(options.testPath);
    if (!fs.existsSync(fullPath)) {
        console.error(`❌ Test path not found: ${fullPath}`);
        process.exit(1);
    }
    console.log(`🧪 Running single test: ${fullPath}`);
    const output = await (0, test_runner_1.runTestCase)(fullPath, {
        verbose: options.verbose,
        skipAutoPR: true
    });
    const expectedPath = path.join(fullPath, 'expected.json');
    const expected = JSON.parse(fs.readFileSync(expectedPath, 'utf8'));
    const evaluator = new evaluator_1.Evaluator();
    const testCase = {
        name: path.basename(fullPath),
        path: fullPath,
        category: path.basename(path.dirname(fullPath)),
        basePath: path.join(fullPath, 'base'),
        headPath: path.join(fullPath, 'head'),
        expected
    };
    const result = evaluator.evaluate(testCase, output);
    if (options.format === 'text') {
        console.log(evaluator.generateReport(result));
    }
    else {
        console.log(JSON.stringify(result, null, 2));
    }
    if (!result.passed) {
        process.exit(1);
    }
}
async function compareWithBaseline(options) {
    const resultsPath = path.join(options.outputDir, 'last_results.json');
    if (!fs.existsSync(resultsPath)) {
        console.error('❌ No results found. Run tests first.');
        process.exit(1);
    }
    const { results } = JSON.parse(fs.readFileSync(resultsPath, 'utf8'));
    const metrics = new metrics_1.MetricsAggregator({
        baselinePath: path.join(options.outputDir, 'baseline_metrics.json'),
        historyPath: path.join(options.outputDir, 'metrics_history.json')
    });
    const regression = metrics.detectRegression(results);
    console.log('📊 Regression Analysis');
    console.log('='.repeat(50));
    console.log(`Status: ${regression.hasRegression ? '❌ REGRESSION DETECTED' : '✅ No Regression'}`);
    console.log(`Accuracy Delta: ${(regression.accuracyDelta * 100).toFixed(2)}%`);
    if (regression.newFailures.length > 0) {
        console.log('\nNew Failures:');
        for (const test of regression.newFailures) {
            console.log(`  ❌ ${test}`);
        }
    }
    if (regression.fixedTests.length > 0) {
        console.log('\nFixed Tests:');
        for (const test of regression.fixedTests) {
            console.log(`  ✅ ${test}`);
        }
    }
    if (options.failOnRegression && regression.hasRegression) {
        process.exit(1);
    }
}
async function saveBaseline(options) {
    const resultsPath = path.join(options.outputDir, 'last_results.json');
    if (!fs.existsSync(resultsPath)) {
        console.error('❌ No results found. Run tests first.');
        process.exit(1);
    }
    const { results } = JSON.parse(fs.readFileSync(resultsPath, 'utf8'));
    const metrics = new metrics_1.MetricsAggregator({
        baselinePath: path.join(options.outputDir, 'baseline_metrics.json'),
        historyPath: path.join(options.outputDir, 'metrics_history.json')
    });
    metrics.saveBaseline(results);
    console.log('✅ Baseline saved successfully');
}
async function generateReport(options) {
    const resultsPath = path.join(options.outputDir, 'last_results.json');
    if (!fs.existsSync(resultsPath)) {
        console.error('❌ No results found. Run tests first.');
        process.exit(1);
    }
    const { summary } = JSON.parse(fs.readFileSync(resultsPath, 'utf8'));
    const metrics = new metrics_1.MetricsAggregator({
        baselinePath: path.join(options.outputDir, 'baseline_metrics.json'),
        historyPath: path.join(options.outputDir, 'metrics_history.json')
    });
    const regression = metrics.detectRegression([]);
    if (options.format === 'text') {
        console.log(metrics.generateReport(summary, regression));
    }
    else {
        console.log(metrics.exportMetrics(summary, options.format));
    }
}
async function main() {
    const options = parseArgs();
    try {
        switch (options.command) {
            case 'eval':
                await runAllTests(options);
                break;
            case 'single':
                await runSingleTest(options);
                break;
            case 'compare':
                await compareWithBaseline(options);
                break;
            case 'baseline':
                await saveBaseline(options);
                break;
            case 'report':
                await generateReport(options);
                break;
        }
    }
    catch (err) {
        console.error(`❌ Error: ${err.message}`);
        if (options.verbose) {
            console.error(err.stack);
        }
        process.exit(1);
    }
}
main();
//# sourceMappingURL=cli.js.map