lib/src/eval/test-runner.js

"use strict";
/**
 * PRIX AI Evaluation System - Test Runner
 * Simulates PRs locally and runs the full review pipeline
 */
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.TestRunner = void 0;
exports.runTestCase = runTestCase;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const context_1 = require("../context");
const commenter_1 = require("../commenter");
const octokit_1 = require("../octokit");
const review_1 = require("../review");
const options_1 = require("../options");
const prompts_1 = require("../prompts");
const mock_github_1 = require("./mock-github");
class TestRunner {
    options;
    currentTest = null;
    constructor(options) {
        this.options = {
            verbose: false,
            skipAutoPR: true,
            mockAI: false,
            ...options
        };
        // Ensure output directory exists
        if (!fs.existsSync(this.options.outputDir)) {
            fs.mkdirSync(this.options.outputDir, { recursive: true });
        }
    }
    /**
     * Load all test cases from the tests directory
     */
    loadTestCases() {
        const tests = [];
        const categories = fs
            .readdirSync(this.options.testsDir)
            .filter(f => fs.statSync(path.join(this.options.testsDir, f)).isDirectory());
        for (const category of categories) {
            const categoryPath = path.join(this.options.testsDir, category);
            const testDirs = fs
                .readdirSync(categoryPath)
                .filter(f => fs.statSync(path.join(categoryPath, f)).isDirectory());
            for (const testDir of testDirs) {
                const testPath = path.join(categoryPath, testDir);
                const expectedPath = path.join(testPath, 'expected.json');
                if (!fs.existsSync(expectedPath)) {
                    console.warn(`⚠️  Skipping ${testDir}: no expected.json found`);
                    continue;
                }
                const expected = JSON.parse(fs.readFileSync(expectedPath, 'utf8'));
                tests.push({
                    name: testDir,
                    path: testPath,
                    category,
                    basePath: path.join(testPath, 'base'),
                    headPath: path.join(testPath, 'head'),
                    expected
                });
            }
        }
        return tests;
    }
    /**
     * Run a single test case
     */
    async runTestCase(testCase) {
        const startTime = Date.now();
        this.currentTest = testCase.name;
        console.log(`\n🧪 Running test: ${testCase.category}/${testCase.name}`);
        try {
            // 1. Load base and head files
            const files = this.loadDiffFiles(testCase);
            if (files.length === 0) {
                throw new Error('No files found in base/head directories');
            }
            // 2. Generate diffs
            const diffFiles = this.generateDiffs(files);
            // 3. Set up mock context
            const mockContext = {
                number: 1,
                title: `[EVAL] ${testCase.name}`,
                body: 'Test PR for evaluation',
                base: { ref: testCase.basePath, sha: 'base-sha' },
                head: { ref: testCase.headPath, sha: 'head-sha' },
                files: diffFiles
            };
            // 4. Create mock GitHub layer
            const logFile = path.join(this.options.outputDir, `${testCase.name}.log`);
            const mockLayer = new mock_github_1.MockGitHubLayer(mockContext, { logFile });
            const probotContext = mockLayer.createMockProbotContext();
            // 5. Suppress actual GitHub calls
            (0, mock_github_1.suppressGitHubCalls)();
            // 6. Run ALS context with working directory
            const mockOctokit = mockLayer.createMockOctokit();
            const store = {
                probotContext,
                octokit: mockOctokit,
                repo: { owner: 'mock-owner', repo: 'mock-repo' },
                workingDir: testCase.headPath
            };
            let output = {
                detectedIssues: false,
                autoPRTriggered: false,
                suggestions: [],
                comments: [],
                errors: [],
                executionTime: 0,
                changedLines: 0
            };
            // 7. Run inside ALS
            await context_1.als.run(store, async () => {
                // Set up context for commenter
                (0, commenter_1.setCommenterContext)(probotContext);
                (0, octokit_1.setOctokit)(probotContext.octokit);
                // Create options and prompts - Options constructor takes individual parameters
                const options = new options_1.Options(this.options.verbose || false, // debug
                false, // disableReview
                true, // disableReleaseNotes
                '0', // maxFiles
                false, // reviewSimpleChanges
                false, // reviewCommentLGTM
                null, // pathFilters
                '', // systemMessage
                'openai/gpt-oss-120b:free', // lightModel
                'openai/gpt-oss-120b:free', // heavyModel
                '0.0', // modelTemperature
                '3', // retries
                '120000', // timeoutMS
                '6', // concurrencyLimit
                '6', // githubConcurrencyLimit
                'https://openrouter.ai/api/v1', // apiBaseUrl
                'en-US', // language
                false, // createRemedyPR
                !this.options.skipAutoPR, // enableAutoPR
                false // strictMode
                );
                const prompts = new prompts_1.Prompts();
                try {
                    // Run the actual review pipeline
                    await (0, review_1.run)(probotContext, options, prompts);
                    // Collect results from mock layer
                    const comments = mockLayer.getComments();
                    const prs = mockLayer.getCreatedPRs();
                    // Calculate changed lines
                    const changedLines = diffFiles.reduce((sum, f) => {
                        return (sum +
                            f.diff
                                .split('\n')
                                .filter(l => l.startsWith('+') || l.startsWith('-')).length);
                    }, 0);
                    // Parse suggestions from comments
                    const suggestions = this.parseSuggestionsFromComments(comments);
                    output = {
                        detectedIssues: comments.length > 0 || suggestions.length > 0,
                        issueType: this.inferIssueType(comments),
                        confidence: this.inferConfidence(comments),
                        autoPRTriggered: prs.length > 0,
                        suggestions,
                        comments,
                        errors: [],
                        executionTime: Date.now() - startTime,
                        changedLines
                    };
                }
                catch (err) {
                    output.errors.push(err.message);
                    console.error(`❌ Pipeline error: ${err.message}`);
                }
            });
            return output;
        }
        catch (err) {
            return {
                detectedIssues: false,
                autoPRTriggered: false,
                suggestions: [],
                comments: [],
                errors: [err.message],
                executionTime: Date.now() - startTime,
                changedLines: 0
            };
        }
    }
    /**
     * Load files from base and head directories
     */
    loadDiffFiles(testCase) {
        const files = [];
        if (!fs.existsSync(testCase.headPath)) {
            return files;
        }
        const headFiles = this.getAllFiles(testCase.headPath);
        for (const relPath of headFiles) {
            const headFile = path.join(testCase.headPath, relPath);
            const baseFile = path.join(testCase.basePath, relPath);
            const headContent = fs.readFileSync(headFile, 'utf8');
            const baseContent = fs.existsSync(baseFile)
                ? fs.readFileSync(baseFile, 'utf8')
                : '';
            files.push({
                filename: relPath,
                base: baseContent,
                head: headContent
            });
        }
        return files;
    }
    /**
     * Get all files recursively from a directory
     */
    getAllFiles(dir, basePath = '') {
        const files = [];
        const items = fs.readdirSync(dir);
        for (const item of items) {
            const fullPath = path.join(dir, item);
            const relPath = basePath ? path.join(basePath, item) : item;
            if (fs.statSync(fullPath).isDirectory()) {
                files.push(...this.getAllFiles(fullPath, relPath));
            }
            else {
                files.push(relPath);
            }
        }
        return files;
    }
    /**
     * Generate unified diffs for files
     */
    generateDiffs(files) {
        return files.map(f => {
            const diff = this.createUnifiedDiff(f.filename, f.base, f.head);
            return {
                filename: f.filename,
                baseContent: f.base,
                headContent: f.head,
                diff
            };
        });
    }
    /**
     * Create unified diff format
     */
    createUnifiedDiff(filename, base, head) {
        const baseLines = base.split('\n');
        const headLines = head.split('\n');
        let diff = `--- a/${filename}\n+++ b/${filename}\n`;
        // Simple line-by-line diff (for testing purposes)
        const maxLen = Math.max(baseLines.length, headLines.length);
        for (let i = 0; i < maxLen; i++) {
            const baseLine = baseLines[i] || '';
            const headLine = headLines[i] || '';
            if (baseLine !== headLine) {
                if (baseLine !== undefined) {
                    diff += `-${baseLine}\n`;
                }
                if (headLine !== undefined) {
                    diff += `+${headLine}\n`;
                }
            }
            else {
                diff += ` ${baseLine}\n`;
            }
        }
        return diff;
    }
    /**
     * Parse suggestions from review comments
     */
    parseSuggestionsFromComments(comments) {
        const suggestions = [];
        for (const comment of comments) {
            // Extract suggestion blocks from comment body
            const suggestionMatch = comment.body.match(/```suggestion\n([\s\S]*?)\n```/);
            if (suggestionMatch) {
                suggestions.push({
                    filename: comment.path,
                    startLine: comment.line || 1,
                    endLine: comment.line || 1,
                    suggestion: suggestionMatch[1].trim()
                });
            }
        }
        return suggestions;
    }
    /**
     * Infer issue type from comments
     */
    inferIssueType(comments) {
        const types = ['security', 'bug', 'performance', 'style'];
        const text = comments
            .map(c => c.body)
            .join(' ')
            .toLowerCase();
        for (const type of types) {
            if (text.includes(type))
                return type;
        }
        return 'style';
    }
    /**
     * Infer confidence from comments
     */
    inferConfidence(comments) {
        const text = comments
            .map(c => c.body)
            .join(' ')
            .toLowerCase();
        if (text.includes('high confidence') || text.includes('critical'))
            return 'high';
        if (text.includes('low confidence') || text.includes('minor'))
            return 'low';
        return 'medium';
    }
}
exports.TestRunner = TestRunner;
/**
 * Utility function to run a single test by path
 */
async function runTestCase(testPath, options) {
    const expectedPath = path.join(testPath, 'expected.json');
    if (!fs.existsSync(expectedPath)) {
        throw new Error(`No expected.json found at ${testPath}`);
    }
    const expected = JSON.parse(fs.readFileSync(expectedPath, 'utf8'));
    const testCase = {
        name: path.basename(testPath),
        path: testPath,
        category: path.basename(path.dirname(testPath)),
        basePath: path.join(testPath, 'base'),
        headPath: path.join(testPath, 'head'),
        expected
    };
    const runner = new TestRunner({
        testsDir: path.dirname(path.dirname(testPath)),
        outputDir: path.join(testPath, '..', '..', 'output'),
        ...options
    });
    return runner.runTestCase(testCase);
}
//# sourceMappingURL=test-runner.js.map