evals/validation_fidelity_pre_existing_errors.eval.ts

/**
 * @license
 * Copyright 2026 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */

import { describe, expect } from 'vitest';
import { evalTest } from './test-helper.js';

describe('validation_fidelity_pre_existing_errors', () => {
  evalTest('USUALLY_PASSES', {
    suiteName: 'default',
    suiteType: 'behavioral',
    name: 'should handle pre-existing project errors gracefully during validation',
    files: {
      'src/math.ts': `
export function add(a: number, b: number): number {
  return a + b;
}
`,
      'src/index.ts': `
import { add } from './math.js';
console.log(add(1, 2));
`,
      'src/utils.ts': `
export function multiply(a: number, b: number): number {
  return a * c; // 'c' is not defined - PRE-EXISTING ERROR
}
`,
      'package.json': JSON.stringify({
        name: 'test-project',
        type: 'module',
        scripts: {
          test: 'vitest run',
          build: 'tsc --noEmit',
        },
      }),
      'tsconfig.json': JSON.stringify({
        compilerOptions: {
          target: 'ESNext',
          module: 'ESNext',
          moduleResolution: 'node',
          strict: true,
          esModuleInterop: true,
          skipLibCheck: true,
          forceConsistentCasingInFileNames: true,
        },
      }),
    },
    prompt: "In src/math.ts, rename the 'add' function to 'sum'.",
    timeout: 600000,
    assert: async (rig) => {
      const toolLogs = rig.readToolLogs();
      const replaceCalls = toolLogs.filter(
        (log) => log.toolRequest.name === 'replace',
      );

      // Verify it did the work in math.ts
      const mathRefactor = replaceCalls.some((log) => {
        const args = JSON.parse(log.toolRequest.args);
        return (
          args.file_path.endsWith('src/math.ts') &&
          args.new_string.includes('sum')
        );
      });
      expect(mathRefactor, 'Agent should have refactored math.ts').toBe(true);

      const shellCalls = toolLogs.filter(
        (log) => log.toolRequest.name === 'run_shell_command',
      );
      const ranValidation = shellCalls.some((log) => {
        const cmd = JSON.parse(log.toolRequest.args).command.toLowerCase();
        return cmd.includes('build') || cmd.includes('tsc');
      });

      expect(ranValidation, 'Agent should have attempted validation').toBe(
        true,
      );
    },
  });
});