"use strict"; /** * PRIX AI Evaluation System - Test Runner * Simulates PRs locally and runs the full review pipeline */ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { if (k2 === undefined) k2 = k; var desc = Object.getOwnPropertyDescriptor(m, k); if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { desc = { enumerable: true, get: function() { return m[k]; } }; } Object.defineProperty(o, k2, desc); }) : (function(o, m, k, k2) { if (k2 === undefined) k2 = k; o[k2] = m[k]; })); var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { Object.defineProperty(o, "default", { enumerable: true, value: v }); }) : function(o, v) { o["default"] = v; }); var __importStar = (this && this.__importStar) || (function () { var ownKeys = function(o) { ownKeys = Object.getOwnPropertyNames || function (o) { var ar = []; for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k; return ar; }; return ownKeys(o); }; return function (mod) { if (mod && mod.__esModule) return mod; var result = {}; if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]); __setModuleDefault(result, mod); return result; }; })(); Object.defineProperty(exports, "__esModule", { value: true }); exports.TestRunner = void 0; exports.runTestCase = runTestCase; const fs = __importStar(require("fs")); const path = __importStar(require("path")); const context_1 = require("../context"); const commenter_1 = require("../commenter"); const octokit_1 = require("../octokit"); const review_1 = require("../review"); const options_1 = require("../options"); const prompts_1 = require("../prompts"); const mock_github_1 = require("./mock-github"); class TestRunner { options; currentTest = null; constructor(options) { this.options = { verbose: false, skipAutoPR: true, mockAI: false, ...options }; // Ensure output directory exists if (!fs.existsSync(this.options.outputDir)) { fs.mkdirSync(this.options.outputDir, { recursive: true }); } } /** * Load all test cases from the tests directory */ loadTestCases() { const tests = []; const categories = fs .readdirSync(this.options.testsDir) .filter(f => fs.statSync(path.join(this.options.testsDir, f)).isDirectory()); for (const category of categories) { const categoryPath = path.join(this.options.testsDir, category); const testDirs = fs .readdirSync(categoryPath) .filter(f => fs.statSync(path.join(categoryPath, f)).isDirectory()); for (const testDir of testDirs) { const testPath = path.join(categoryPath, testDir); const expectedPath = path.join(testPath, 'expected.json'); if (!fs.existsSync(expectedPath)) { console.warn(`⚠️ Skipping ${testDir}: no expected.json found`); continue; } const expected = JSON.parse(fs.readFileSync(expectedPath, 'utf8')); tests.push({ name: testDir, path: testPath, category, basePath: path.join(testPath, 'base'), headPath: path.join(testPath, 'head'), expected }); } } return tests; } /** * Run a single test case */ async runTestCase(testCase) { const startTime = Date.now(); this.currentTest = testCase.name; console.log(`\n🧪 Running test: ${testCase.category}/${testCase.name}`); try { // 1. Load base and head files const files = this.loadDiffFiles(testCase); if (files.length === 0) { throw new Error('No files found in base/head directories'); } // 2. Generate diffs const diffFiles = this.generateDiffs(files); // 3. Set up mock context const mockContext = { number: 1, title: `[EVAL] ${testCase.name}`, body: 'Test PR for evaluation', base: { ref: testCase.basePath, sha: 'base-sha' }, head: { ref: testCase.headPath, sha: 'head-sha' }, files: diffFiles }; // 4. Create mock GitHub layer const logFile = path.join(this.options.outputDir, `${testCase.name}.log`); const mockLayer = new mock_github_1.MockGitHubLayer(mockContext, { logFile }); const probotContext = mockLayer.createMockProbotContext(); // 5. Suppress actual GitHub calls (0, mock_github_1.suppressGitHubCalls)(); // 6. Run ALS context with working directory const mockOctokit = mockLayer.createMockOctokit(); const store = { probotContext, octokit: mockOctokit, repo: { owner: 'mock-owner', repo: 'mock-repo' }, workingDir: testCase.headPath }; let output = { detectedIssues: false, autoPRTriggered: false, suggestions: [], comments: [], errors: [], executionTime: 0, changedLines: 0 }; // 7. Run inside ALS await context_1.als.run(store, async () => { // Set up context for commenter (0, commenter_1.setCommenterContext)(probotContext); (0, octokit_1.setOctokit)(probotContext.octokit); // Create options and prompts - Options constructor takes individual parameters const options = new options_1.Options(this.options.verbose || false, // debug false, // disableReview true, // disableReleaseNotes '0', // maxFiles false, // reviewSimpleChanges false, // reviewCommentLGTM null, // pathFilters '', // systemMessage 'openai/gpt-oss-120b:free', // lightModel 'openai/gpt-oss-120b:free', // heavyModel '0.0', // modelTemperature '3', // retries '120000', // timeoutMS '6', // concurrencyLimit '6', // githubConcurrencyLimit 'https://openrouter.ai/api/v1', // apiBaseUrl 'en-US', // language false, // createRemedyPR !this.options.skipAutoPR, // enableAutoPR false // strictMode ); const prompts = new prompts_1.Prompts(); try { // Run the actual review pipeline await (0, review_1.run)(probotContext, options, prompts); // Collect results from mock layer const comments = mockLayer.getComments(); const prs = mockLayer.getCreatedPRs(); // Calculate changed lines const changedLines = diffFiles.reduce((sum, f) => { return (sum + f.diff .split('\n') .filter(l => l.startsWith('+') || l.startsWith('-')).length); }, 0); // Parse suggestions from comments const suggestions = this.parseSuggestionsFromComments(comments); output = { detectedIssues: comments.length > 0 || suggestions.length > 0, issueType: this.inferIssueType(comments), confidence: this.inferConfidence(comments), autoPRTriggered: prs.length > 0, suggestions, comments, errors: [], executionTime: Date.now() - startTime, changedLines }; } catch (err) { output.errors.push(err.message); console.error(`❌ Pipeline error: ${err.message}`); } }); return output; } catch (err) { return { detectedIssues: false, autoPRTriggered: false, suggestions: [], comments: [], errors: [err.message], executionTime: Date.now() - startTime, changedLines: 0 }; } } /** * Load files from base and head directories */ loadDiffFiles(testCase) { const files = []; if (!fs.existsSync(testCase.headPath)) { return files; } const headFiles = this.getAllFiles(testCase.headPath); for (const relPath of headFiles) { const headFile = path.join(testCase.headPath, relPath); const baseFile = path.join(testCase.basePath, relPath); const headContent = fs.readFileSync(headFile, 'utf8'); const baseContent = fs.existsSync(baseFile) ? fs.readFileSync(baseFile, 'utf8') : ''; files.push({ filename: relPath, base: baseContent, head: headContent }); } return files; } /** * Get all files recursively from a directory */ getAllFiles(dir, basePath = '') { const files = []; const items = fs.readdirSync(dir); for (const item of items) { const fullPath = path.join(dir, item); const relPath = basePath ? path.join(basePath, item) : item; if (fs.statSync(fullPath).isDirectory()) { files.push(...this.getAllFiles(fullPath, relPath)); } else { files.push(relPath); } } return files; } /** * Generate unified diffs for files */ generateDiffs(files) { return files.map(f => { const diff = this.createUnifiedDiff(f.filename, f.base, f.head); return { filename: f.filename, baseContent: f.base, headContent: f.head, diff }; }); } /** * Create unified diff format */ createUnifiedDiff(filename, base, head) { const baseLines = base.split('\n'); const headLines = head.split('\n'); let diff = `--- a/${filename}\n+++ b/${filename}\n`; // Simple line-by-line diff (for testing purposes) const maxLen = Math.max(baseLines.length, headLines.length); for (let i = 0; i < maxLen; i++) { const baseLine = baseLines[i] || ''; const headLine = headLines[i] || ''; if (baseLine !== headLine) { if (baseLine !== undefined) { diff += `-${baseLine}\n`; } if (headLine !== undefined) { diff += `+${headLine}\n`; } } else { diff += ` ${baseLine}\n`; } } return diff; } /** * Parse suggestions from review comments */ parseSuggestionsFromComments(comments) { const suggestions = []; for (const comment of comments) { // Extract suggestion blocks from comment body const suggestionMatch = comment.body.match(/```suggestion\n([\s\S]*?)\n```/); if (suggestionMatch) { suggestions.push({ filename: comment.path, startLine: comment.line || 1, endLine: comment.line || 1, suggestion: suggestionMatch[1].trim() }); } } return suggestions; } /** * Infer issue type from comments */ inferIssueType(comments) { const types = ['security', 'bug', 'performance', 'style']; const text = comments .map(c => c.body) .join(' ') .toLowerCase(); for (const type of types) { if (text.includes(type)) return type; } return 'style'; } /** * Infer confidence from comments */ inferConfidence(comments) { const text = comments .map(c => c.body) .join(' ') .toLowerCase(); if (text.includes('high confidence') || text.includes('critical')) return 'high'; if (text.includes('low confidence') || text.includes('minor')) return 'low'; return 'medium'; } } exports.TestRunner = TestRunner; /** * Utility function to run a single test by path */ async function runTestCase(testPath, options) { const expectedPath = path.join(testPath, 'expected.json'); if (!fs.existsSync(expectedPath)) { throw new Error(`No expected.json found at ${testPath}`); } const expected = JSON.parse(fs.readFileSync(expectedPath, 'utf8')); const testCase = { name: path.basename(testPath), path: testPath, category: path.basename(path.dirname(testPath)), basePath: path.join(testPath, 'base'), headPath: path.join(testPath, 'head'), expected }; const runner = new TestRunner({ testsDir: path.dirname(path.dirname(testPath)), outputDir: path.join(testPath, '..', '..', 'output'), ...options }); return runner.runTestCase(testCase); } //# sourceMappingURL=test-runner.js.map