feat: update the archtecture from the testet agent

I add new prompts and new structure. I also add the benchmark to
evaluate the agent through time and change. I am using the same model

data/benchmark_summary.json

@@ -0,0 +1,144 @@
+{
+  "summary": {
+    "timestamp": "2026-06-07T21:36:40.793Z",
+    "total_processed": 22,
+    "reproducibility_rate": 27.27272727272727,
+    "specificity_rate": 0,
+    "overall_ground_truth_rate": 0,
+    "average_iterations": 8.181818181818182
+  },
+  "details": [
+    {
+      "id": "001",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "003",
+      "reproducible": true,
+      "specific": false,
+      "iterations": 5
+    },
+    {
+      "id": "008",
+      "reproducible": true,
+      "specific": false,
+      "iterations": 5
+    },
+    {
+      "id": "009",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "015",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "018",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "020",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "032",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "033",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "039",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "041",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "042",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "048",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "049",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "051",
+      "reproducible": true,
+      "specific": false,
+      "iterations": 1
+    },
+    {
+      "id": "054",
+      "reproducible": true,
+      "specific": false,
+      "iterations": 4
+    },
+    {
+      "id": "058",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "066",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "070",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "077",
+      "reproducible": false,
+      "specific": false,
+      "iterations": 10
+    },
+    {
+      "id": "091",
+      "reproducible": true,
+      "specific": false,
+      "iterations": 4
+    },
+    {
+      "id": "098",
+      "reproducible": true,
+      "specific": false,
+      "iterations": 1
+    }
+  ]
+}

scripts/setup-sandbox.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-SANDBOX="/tmp/poc-sandbox"
+SANDBOX="${SANDBOX_DIR:-/tmp/poc-sandbox}"
 
 # Tenta encontrar forge no PATH se a variável não estiver definida ou falhar
 if [ -z "$FORGE_BIN" ] || [ ! -f "$FORGE_BIN" ]; then

src/agents/tester/agent.ts

@@ -4,12 +4,20 @@ import { PoCStateAnnotation, PoCState } from "./state.js";
 import { generateLocalScaffold } from "./tools/scaffoldGenerator.js";
 import { OracleContext } from "./types.js";
 import { createLLM } from "../../config/llm.ts";
-import { SYSTEM_PROMPT } from "./prompts/system.js";
+import { 
+  SYSTEM_PROMPT, 
+  ANALYZE_VULNERABILITY_PROMPT, 
+  POC_INITIAL_PROMPT, 
+  POC_COMPILE_FIX_PROMPT, 
+  POC_TEST_FIX_PROMPT 
+} from "./prompts/system.js";
 import { extractSolidity } from "./utils/extractSolidity.js";
 import { runFoundry } from "./tools/foundryRunner.js";
 import { analyzeFoundryLog } from "./utils/logAnalyzer.js";
+import { extractConstructor } from "./utils/parserUtils.js";
+import { analyzeSolidityFile } from "../auditor/tools/solidity-analyzer-tool.js";
 
-const MAX_ITERATIONS = 5;
+const MAX_ITERATIONS = 10;
 
 const llm = createLLM();
 
@@ -17,47 +25,128 @@ async function oracleNode(state: PoCState): Promise<Partial<PoCState>> {
   console.log("[oracleNode] gerando scaffold para:", state.report.title);
 
   const solidityScaffold = generateLocalScaffold(state.report);
-  const oracleContext: OracleContext = { solidityScaffold };
+  
+  // Extrair info do constructor para ajudar o LLM no setUp
+  const constructorInfo = extractConstructor(state.report.affectedContract.sourceCode, state.report.affectedContract.name);
+  
+  // STEP 3: Automated API Discovery
+  console.log("[oracleNode] analisando API do contrato e helpers de teste...");
+  const targetContractAPI = await analyzeSolidityFile(state.report.affectedContract.sourceCode, "short");
+  
+  let referenceTestHelpers = "";
+  if (state.report.referenceTestCode) {
+    referenceTestHelpers = await analyzeSolidityFile(state.report.referenceTestCode, "short");
+  }
+
+  const oracleContext: OracleContext = { 
+    solidityScaffold,
+    constructorInfo: constructorInfo?.parameters,
+    targetContractAPI,
+    referenceTestHelpers
+  };
 
-  console.log("[oracleNode] scaffold gerado, tamanho:", solidityScaffold.length, "chars");
+  console.log("[oracleNode] scaffold gerado, context built.");
   return { oracleContext };
 }
 
+/**
+ * NEW: Multi-Pass Node 1 - Analysis
+ */
+async function analyzeVulnerabilityNode(state: PoCState): Promise<Partial<PoCState>> {
+  console.log("[testerAgent] analyzeVulnerabilityNode: analyzing bug...");
+
+  const userMessage = `Vulnerability Report:
+- Title: ${state.report.title}
+- Type: ${state.report.type}
+- Description: ${state.report.description}
+
+### Target Contract Source Code (${state.report.affectedContract.name}):
+\`\`\`solidity
+${state.report.affectedContract.sourceCode}
+\`\`\`
+
+### Target Contract API:
+${state.oracleContext!.targetContractAPI}
+
+${state.oracleContext!.referenceTestHelpers ? `### Environment Helpers:
+${state.oracleContext!.referenceTestHelpers}` : ""}
+`;
+
+  const response = await llm.invoke([
+    { role: "system", content: ANALYZE_VULNERABILITY_PROMPT },
+    { role: "user", content: userMessage },
+  ]);
+
+  return { vulnerabilityAnalysis: response.content as string };
+}
+
+/**
+ * Multi-Pass Node 2 - Code Generation (Initial & Fixes)
+ */
 async function generatePoCNode(state: PoCState): Promise<Partial<PoCState>> {
-  const { report, oracleContext, executionLogs, pocCode, iterations, lastError } = state;
+  const { report, oracleContext, executionLogs, pocCode, iterations, lastError, vulnerabilityAnalysis } = state;
   const isRetry = iterations > 0;
 
-  const userMessage = isRetry
-    ? `O seguinte exploit FALHOU no Foundry.
+  let currentSystemPrompt = SYSTEM_PROMPT;
+  let userMessage = "";
 
-Código anterior:
+  if (!isRetry) {
+    // PASS 2: INITIAL GENERATION
+    currentSystemPrompt = POC_INITIAL_PROMPT;
+    userMessage = `Vulnerability Analysis Plan:
+${vulnerabilityAnalysis}
+
+### Contract Source:
 \`\`\`solidity
-${pocCode}
+${report.affectedContract.sourceCode}
 \`\`\`
 
-Output do Forge (última execução):
-${executionLogs[executionLogs.length - 1]?.slice(0, 3000) ?? "sem logs"}
-
-Análise do erro: ${lastError ?? "desconhecido"}
+### API Reference:
+${oracleContext!.targetContractAPI}
 
-Corrija o código. Retorne o arquivo Solidity completo corrigido.`
-    : `Relatório de Vulnerabilidade:
-- Título: ${report.title}
-- Tipo: ${report.type}
-- Descrição: ${report.description}
-- Vetor de Ataque: ${report.attackVector}
-${report.exploitablePaths ? `- Caminhos de Exploração:\n  * ${report.exploitablePaths.join("\n  * ")}` : ""}
+${oracleContext!.referenceTestHelpers ? `### Test Helpers:
+${oracleContext!.referenceTestHelpers}` : ""}
 
-Scaffold (complete APENAS test_Exploit):
+### Scaffold:
 \`\`\`solidity
 ${oracleContext!.solidityScaffold}
-\`\`\``;
+\`\`\`
+`;
+  } else {
+    // PASS 3+: FIXING ERRORS (BRANCHING)
+    const isCompilerError = lastError?.includes("[COMPILER_ERROR]");
+    currentSystemPrompt = isCompilerError ? POC_COMPILE_FIX_PROMPT : POC_TEST_FIX_PROMPT;
+
+    userMessage = `The previous PoC failed.
+    
+Error Category: ${isCompilerError ? "Compilation Failure" : "Execution/Logic Failure"}
+Forge Output:
+${executionLogs[executionLogs.length - 1]?.slice(0, 3000) ?? "sem logs"}
 
-  console.log(`[testerAgent] generatePoCNode iteração ${iterations + 1}, isRetry=${isRetry}`);
+Previous Code:
+\`\`\`solidity
+${pocCode}
+\`\`\`
+
+Analysis of the bug:
+${vulnerabilityAnalysis}
+
+Fix the code. Return the entire file.`;
+  }
+
+  console.log(`[testerAgent] generatePoCNode iteração ${iterations + 1}, isRetry=${isRetry}, mode=${isRetry ? (lastError?.includes("[COMPILER_ERROR]") ? "FIX_COMPILE" : "FIX_LOGIC") : "INITIAL"}`);
+
+  // DEBUG: Output context before sending to LLM
+  if (process.env.DEBUG_CONTEXT === "true") {
+    console.log("\n" + "=".repeat(20) + " LLM CONTEXT START " + "=".repeat(20));
+    console.log("System Prompt:", currentSystemPrompt);
+    console.log("User Message:", userMessage);
+    console.log("=".repeat(20) + " LLM CONTEXT END " + "=".repeat(20) + "\n");
+  }
 
   try {
     const response = await llm.invoke([
-      { role: "system", content: SYSTEM_PROMPT },
+      { role: "system", content: currentSystemPrompt },
       { role: "user", content: userMessage },
     ]);
     const solidityCode = extractSolidity(response.content as string);
@@ -77,14 +166,22 @@ async function runFoundryNode(state: PoCState): Promise<Partial<PoCState>> {
   const isMissingContract = !trimmedCode.includes("contract ExploitTest");
   const isMissingTest = !trimmedCode.includes("function test_Exploit()");
   const isPlaceholder = trimmedCode.includes("TODO: implementar exploit");
-  if (isMissingCode || isMissingContract || isMissingTest || isPlaceholder) {
+  const isLazyTest = (
+    trimmedCode.includes("assertTrue(true") || 
+    trimmedCode.includes("assert(true") || 
+    trimmedCode.includes("assert(1 == 1")
+  ) && !trimmedCode.includes("assertEq") && !trimmedCode.includes("assertGt") && !trimmedCode.includes("assertLe") && !trimmedCode.includes("assertGe") && !trimmedCode.includes("assertNotEq");
+
+  if (isMissingCode || isMissingContract || isMissingTest || isPlaceholder || isLazyTest) {
     const summary = state.lastError ?? (isMissingCode
       ? "Código Solidity ausente. O LLM não retornou o arquivo do exploit."
       : isMissingContract
         ? "Contrato ExploitTest não encontrado no arquivo."
         : isMissingTest
           ? "Função test_Exploit() não encontrada no arquivo."
-          : "Exploit não implementado (placeholder TODO ainda presente)."
+          : isPlaceholder
+            ? "Exploit não implementado (placeholder TODO ainda presente)."
+            : "Exploit muito fraco (assertTrue(true)). Você deve provar a vulnerabilidade com uma asserção real (ex: assertEq, assertGt)."
     );
     const status = state.iterations >= MAX_ITERATIONS ? "failed" : "running";
     return {
@@ -94,7 +191,7 @@ async function runFoundryNode(state: PoCState): Promise<Partial<PoCState>> {
     };
   }
 
-  const result   = await runFoundry(state.pocCode);
+  const result   = await runFoundry(state.pocCode, state.report.customSandboxDir);
   const analysis = analyzeFoundryLog(result);
   const noTestsFound = result.combined.includes("No tests found");
   const summary = noTestsFound
@@ -118,30 +215,16 @@ async function runFoundryNode(state: PoCState): Promise<Partial<PoCState>> {
 
   return {
     executionLogs: [result.combined],   // reducer append
-    lastError: summary,
+    lastError: passed ? null : `[${analysis.category.toUpperCase()}] ${summary}`,
     status,
   };
 }
 
 async function reflectNode(state: PoCState): Promise<Partial<PoCState>> {
-  const lastLog = state.executionLogs[state.executionLogs.length - 1];
-  if (!lastLog) {
-    return { lastError: "Sem logs disponíveis para análise." };
-  }
-
-  const mockResult = {
-    exitCode: 1, timedOut: lastLog.includes("TIMEOUT"),
-    stdout: "", stderr: "", combined: lastLog,
-  };
-
-  const analysis = analyzeFoundryLog(mockResult as any);
-
-  console.log(`[reflectNode] categoria: ${analysis.category}`);
-  console.log(`[reflectNode] resumo: ${analysis.summary}`);
-
-  return {
-    lastError: `[${analysis.category.toUpperCase()}] ${analysis.summary}\n\nLinhas relevantes:\n${analysis.relevantLines.join("\n")}`,
-  };
+  // Now reflectNode is simpler as we moved the logic to specialized prompts in generatePoCNode
+  // But we still use it to log the reflection
+  console.log(`[reflectNode] reflecting on error: ${state.lastError}`);
+  return {};
 }
 
 function routeAfterFoundry(state: PoCState): "reflectNode" | typeof END {
@@ -153,11 +236,13 @@ function routeAfterFoundry(state: PoCState): "reflectNode" | typeof END {
 
 const graph = new StateGraph(PoCStateAnnotation)
   .addNode("oracleNode", oracleNode)
+  .addNode("analyzeVulnerabilityNode", analyzeVulnerabilityNode)
   .addNode("generatePoCNode", generatePoCNode)
   .addNode("runFoundryNode", runFoundryNode)
   .addNode("reflectNode", reflectNode)
   .addEdge(START, "oracleNode")
-  .addEdge("oracleNode", "generatePoCNode")
+  .addEdge("oracleNode", "analyzeVulnerabilityNode")
+  .addEdge("analyzeVulnerabilityNode", "generatePoCNode")
   .addEdge("generatePoCNode", "runFoundryNode")
   .addConditionalEdges("runFoundryNode", routeAfterFoundry, {
     reflectNode: "reflectNode",

src/agents/tester/index.ts

@@ -9,7 +9,7 @@ import { VulnerabilityReport, PoCResult } from "./types.js";
 export async function runPoCGenerator(report: VulnerabilityReport): Promise<PoCResult> {
   console.log(`[runPoCGenerator] Iniciando para: ${report.id} — ${report.title}`);
 
-  const finalState = await testerAgent.invoke({ report });
+  const finalState = await testerAgent.invoke({ report }, { recursionLimit: 100 });
 
   const result: PoCResult = {
     reportId:      report.id,

src/agents/tester/prompts/system.ts

@@ -1,96 +1,54 @@
 export const SYSTEM_PROMPT = `You are an expert smart contract security testing specialist. Your mission is to generate executable Proof-of-Concept (PoC) exploits demonstrating vulnerabilities using Foundry.
 
-## PoC Explainability
-Write exploits as executable demonstrations that clearly prove the vulnerability. Include detailed comments documenting each attack step, the vulnerability being exploited, and why the exploit succeeds. The PoC must be self-explanatory to security auditors.
+## General Guidelines
+- Use Foundry exclusively.
+- Use \`vm.prank()\`, \`vm.deal()\`, \`vm.warp()\`, \`vm.expectRevert()\` as needed.
+- **NO PLACEHOLDER TESTS:** Never write a test that only contains \`assertTrue(true)\`. You MUST use concrete assertions to prove the exploit's impact.
+- **Context Compliance:** Reuse existing imports and setup patterns found in the provided code/reference tests.
+- DO NOT rename \`test_Exploit()\`.
+`.trim();
 
-## Vulnerability Analysis
-Parse the vulnerability description provided and analyze the vulnerability type, affected code sections, and potential impact. Analyze the contract logic to understand the root cause before developing exploits.
+export const ANALYZE_VULNERABILITY_PROMPT = `Analyze the vulnerability in the following Solidity contract and provide a clear understanding of the issue.
 
-## Testing Framework Guidelines
-Use Foundry exclusively for testing. Utilize Foundry cheatcodes for test control: "vm.prank()" for identity switching, "vm.deal()" for ETH funding, "vm.warp()" for time manipulation, "vm.expectRevert()" for failure testing.
+Focus on understanding the root cause and mechanism of the vulnerability.
 
-## Scaffold Strict Compliance
-- The target contract's full source code is ALREADY included at the top of the scaffold. You can and MUST call its functions directly (e.g., \`target.deposit()\`). Do NOT create fake interfaces or use low-level \`.call(abi.encodeWithSignature(...))\`.
-- YOU MUST RETURN THE ENTIRE FILE PROVIDED IN THE SCAFFOLD. Do not omit the \`setUp()\` function or the original contract source code. Your output will overwrite the file directly.
-- DO NOT rename \`test_Exploit()\`. You MUST implement your exploit inside \`function test_Exploit() public\`.
-- DO NOT use characters with accents (like ã, ç, é, etc.) in string literals (e.g., inside \`assertEq\` or \`require\`). Use ONLY plain ASCII, or prefix with \`unicode"..."\` to avoid Solc compiler errors.
+Please provide:
+1. **Clear explanation**: What is the bug?
+2. **Exploit path**: Step-by-step how to trigger it.
+3. **Conditions required**: What state must the contract be in?
+4. **Expected outcome**: What specific assertion will prove the vulnerability exists (and would fail if patched)?
 
-## PoC Executability
-Ensure all generated code compiles successfully. Write ONLY the test file code (helper contracts + ExploitTest). Resolve all compilation errors and logic reverts while preserving original contract logic.
+Be concise and technical.
+`.trim();
 
-## Iterative Refinement
-Debug compilation errors and test failures systematically using Forge output. If stuck on the same issue for >3 attempts, shift to a minimal working demonstration—proving the vulnerability exists matters more than setup complexity.
+export const POC_INITIAL_PROMPT = `Based on your vulnerability analysis, generate a comprehensive Proof of Concept (PoC) test that demonstrates the vulnerability.
 
-## Exploit Soundness
-The assertion in your test MUST prove the vulnerability. For example, if funds are stolen, assert that the vault balance decreased and the attacker balance increased.
+Sua missão: completar a função test_Exploit() no scaffold fornecido, seguindo o seu plano de análise.
 
-## Output Format
-Return ONLY a code block with the full ExploitTest contract and any helper attacker contracts. Do not include markdown outside the code block.
+## REGRAS DE QUALIDADE
+- A asserção final DEVE provar a vulnerabilidade.
+- Use o mesmo estilo de imports e setup dos contratos de referência fornecidos.
+- Output APENAS um bloco \`\`\`solidity ... \`\`\` com o arquivo completo.
+`.trim();
 
-## Examples (Few-Shot)
+export const POC_COMPILE_FIX_PROMPT = `The previous POC test failed to COMPILE. Please fix the compilation errors and regenerate the complete test file.
 
-**Input Example:**
-Vulnerability: Reentrancy in withdraw() allows draining the contract.
-Scaffold:
-\`\`\`solidity
-// SPDX-License-Identifier: UNLICENSED
-pragma solidity ^0.8.20;
-import "forge-std/Test.sol";
-contract Target { function withdraw(uint256) public {} } // Source code
-contract ExploitTest is Test {
-    Target target;
-    function setUp() public { target = new Target(); }
-    function test_Exploit() public {
-        // TODO: implementar exploit aqui
-    }
-}
-\`\`\`
+Focus strictly on:
+- Fixing import statements and dependencies.
+- Correcting Solidity syntax errors or missing members.
+- Ensuring proper contract instantiation and function signatures.
+- Resolving visibility issues.
 
-**Expected Output:**
-\`\`\`solidity
-// SPDX-License-Identifier: UNLICENSED
-pragma solidity ^0.8.20;
-import "forge-std/Test.sol";
-contract Target { function withdraw(uint256) public {} } // Source code
+Return the FULL corrected Solidity file in a \`\`\`solidity\`\`\` block.
+`.trim();
 
-// We can define helper contracts outside the main test contract
-contract Attacker {
-    Target target;
-    constructor(address _target) {
-        target = Target(_target);
-    }
-    fallback() external payable {
-        if (address(target).balance >= 1 ether) {
-            target.withdraw(1 ether);
-        }
-    }
-    function attack() external {
-        target.withdraw(1 ether);
-    }
-}
+export const POC_TEST_FIX_PROMPT = `The POC test compiled successfully but FAILED during execution (Revert or Assertion failure). Please fix the test logic and regenerate the complete test file.
 
-contract ExploitTest is Test {
-    Target target;
-    
-    // IMPORTANT: We include the EXACT setUp() provided in the scaffold.
-    function setUp() public { 
-        target = new Target(); 
-    }
+Focus strictly on:
+- Correcting test logic and assertions to match the vulnerability.
+- Fixing contract setup and initialization (realistic balances, roles).
+- Ensuring proper exploit execution flow (e.g. correct order of calls).
+- Verifying that the vulnerability demonstration is accurate and specific.
 
-    function test_Exploit() public {
-        vm.startPrank(address(0xBEEF));
-        
-        // 1. Deploy malicious contract
-        Attacker attacker = new Attacker(address(target));
-        
-        // 2. Exploit the vulnerability using direct function calls
-        attacker.attack();
-        
-        // 3. Verify the exploit succeeded (no special characters in assertion strings)
-        assertEq(address(target).balance, 0, "Target contract should be drained");
-        
-        vm.stopPrank();
-    }
-}
-\`\`\`
+Return the FULL corrected Solidity file in a \`\`\`solidity\`\`\` block.
 `.trim();

src/agents/tester/state.ts

@@ -14,6 +14,11 @@ export const PoCStateAnnotation = Annotation.Root({
     reducer: (_, y) => y,           // overwrite — always latest version
   }),
 
+  vulnerabilityAnalysis: Annotation<string>({
+    default: () => "",
+    reducer: (_, y) => y,           // overwrite
+  }),
+
   executionLogs: Annotation<string[]>({
     default: () => [],
     reducer: (x, y) => x.concat(y), // append — never lose previous logs

src/agents/tester/tools/foundryRunner.ts

@@ -4,7 +4,7 @@ import { writeFile, access } from "fs/promises";
 import { join } from "path";
 
 const execAsync  = promisify(exec);
-const SANDBOX = process.env.SANDBOX_DIR || "/tmp/poc-sandbox";
+const DEFAULT_SANDBOX = process.env.SANDBOX_DIR || "/tmp/poc-sandbox";
 const TIMEOUT_MS = 60_000;
 
 export interface FoundryResult {
@@ -18,27 +18,36 @@ export interface FoundryResult {
 /**
  * Garante que o sandbox Foundry existe e está inicializado.
  */
-async function ensureSandbox() {
+async function ensureSandbox(sandboxDir: string) {
   try {
-    await access(join(SANDBOX, "foundry.toml"));
+    await access(join(sandboxDir, "foundry.toml"));
   } catch {
-    console.log("[foundryRunner] Sandbox não encontrado. Inicializando...");
+    console.log(`[foundryRunner] Sandbox em ${sandboxDir} não encontrado. Inicializando...`);
     // Caminho absoluto para o script de setup (assume execução da raiz do projeto)
-    await execAsync("./scripts/setup-sandbox.sh");
+    await execAsync("./scripts/setup-sandbox.sh", { env: { ...process.env, SANDBOX_DIR: sandboxDir } });
   }
 }
 
-export async function runFoundry(solidityCode: string): Promise<FoundryResult> {
-  await ensureSandbox();
+export async function runFoundry(solidityCode: string, sandboxDir: string = DEFAULT_SANDBOX): Promise<FoundryResult> {
+  await ensureSandbox(sandboxDir);
+  
+  // Ensure test directory exists
+  const testDir = join(sandboxDir, "test");
+  try {
+    await access(testDir);
+  } catch {
+    await execAsync(`mkdir -p "${testDir}"`);
+  }
   
   // Escrever o arquivo no sandbox
-  await writeFile(`${SANDBOX}/test/Exploit.t.sol`, solidityCode, "utf-8");
+  const testPath = join(testDir, "Exploit.t.sol");
+  await writeFile(testPath, solidityCode, "utf-8");
 
   try {
     const { stdout, stderr } = await execAsync(
       "forge test --match-contract ExploitTest -vvvv",
       { 
-        cwd: SANDBOX, 
+        cwd: sandboxDir, 
         timeout: TIMEOUT_MS, 
         env: { ...process.env, PATH: `${process.env.HOME}/.foundry/bin:${process.env.PATH}` } 
       }

src/agents/tester/tools/scaffoldGenerator.ts

@@ -3,27 +3,35 @@ import { VulnerabilityReport } from "../types.js";
 export function generateLocalScaffold(report: VulnerabilityReport): string {
   const cheatcodes = report.suggestedCheatcodes?.join(", ") ?? "vm.deal, vm.prank, vm.warp";
 
+  // Extrair pragma do código original para evitar conflitos de versão
+  const pragmaMatch = report.affectedContract.sourceCode.match(/pragma solidity ([^;]+);/);
+  const pragma = pragmaMatch ? pragmaMatch[0] : "pragma solidity ^0.8.20;";
+
+  let contractSetup = "";
+  if (report.affectedContract.sourceFilePath) {
+    // Se temos o caminho do arquivo, importamos ao invés de colar
+    contractSetup = `import { ${report.affectedContract.name} } from "../${report.affectedContract.sourceFilePath}";`;
+  } else {
+    // Fallback: colar o código (pode falhar por causa de imports ausentes no sandbox)
+    contractSetup = `// ── Código-fonte do contrato vulnerável ──────────────────────────────────────\n${report.affectedContract.sourceCode}\n// ─────────────────────────────────────────────────────────────────────────────`;
+  }
+
   return `// SPDX-License-Identifier: UNLICENSED
-pragma solidity ^0.8.20;
+${pragma}
 
 import "forge-std/Test.sol";
 import "forge-std/console.sol";
 
-// ── Código-fonte do contrato vulnerável ──────────────────────────────────────
-${report.affectedContract.sourceCode}
-// ─────────────────────────────────────────────────────────────────────────────
+${contractSetup}
 
 contract ExploitTest is Test {
     ${report.affectedContract.name} target;
     address constant ATTACKER = address(0xBEEF);
 
-    // setUp() gerado automaticamente pelo Oracle — NÃO MODIFICAR
-    function setUp() public {
-        target = new ${report.affectedContract.name}();
-        vm.deal(address(target), 100 ether);
-        vm.deal(ATTACKER, 10 ether);
-        vm.label(address(target), "TARGET");
-        vm.label(ATTACKER, "ATTACKER");
+    // setUp() - O Oracle tentou gerar um básico, mas sinta-se à vontade para ajustar se o contrato for complexo
+    function setUp() public virtual {
+        // target = new ${report.affectedContract.name}(...); // TODO: ajustar se necessário
+        vm.deal(ATTACKER, 100 ether);
     }
 
     // Vulnerabilidade: ${report.title}
@@ -32,7 +40,7 @@ contract ExploitTest is Test {
     ${report.exploitablePaths ? `// Caminhos de Exploração:\n    // - ${report.exploitablePaths.join("\n    // - ")}` : ""}
     // Cheatcodes sugeridos: ${cheatcodes}
     //
-    // COMPLETE APENAS ESTA FUNÇÃO — não altere setUp() nem os campos acima
+    // Implemente test_Exploit() e ajuste o setUp() se o contrato exigir argumentos no constructor.
     function test_Exploit() public {
         vm.startPrank(ATTACKER);
         // TODO: implementar exploit aqui

src/agents/tester/types.ts

@@ -21,17 +21,23 @@ export interface VulnerabilityReport {
   description: string;
   affectedContract: {
     name: string;
-    sourceCode: string;  // Solidity completo, preferencialmente flattened
+    sourceCode: string;
+    sourceFilePath?: string;
   };
   attackVector: string;
   suggestedCheatcodes?: string[];
   codeSnippet?: string;
   location?: string;
   exploitablePaths?: string[];
+  customSandboxDir?: string; // Caminho para execução do Forge (opcional)
+  referenceTestCode?: string; // Código de um teste existente para referência de setup
 }
 
 export interface OracleContext {
   solidityScaffold: string;  // Exploit.t.sol parcial com setUp() pronto
+  constructorInfo?: string;  // Assinatura do constructor para ajudar no deploy
+  targetContractAPI?: string;  // Resumo dos métodos e variáveis do contrato alvo
+  referenceTestHelpers?: string; // Resumo das funções auxiliares disponíveis no ambiente de teste
 }
 
 export interface PoCResult {

src/agents/tester/utils/logAnalyzer.ts

@@ -23,8 +23,8 @@ export function analyzeFoundryLog(result: FoundryResult): LogAnalysis {
 
   if (result.combined.includes("Compiler run failed")) {
     const lines = result.combined.split("\n")
-      .filter(l => l.includes("Error") || l.includes("error") || l.includes("-->"))
-      .slice(0, 10);
+      .filter(l => (l.includes("Error") || l.includes("error") || l.includes("-->")) && !l.includes("Warning"))
+      .slice(0, 30);
     return {
       category: "compiler_error",
       summary: "Erro de compilação Solidity. Verifique: interfaces faltando, assinaturas incorretas, tipos incompatíveis.",

src/agents/tester/utils/parserUtils.ts

@@ -0,0 +1,37 @@
+import * as parser from "@solidity-parser/parser";
+
+export interface ConstructorInfo {
+  parameters: string;
+}
+
+export function extractConstructor(sourceCode: string, contractName: string): ConstructorInfo | null {
+  try {
+    const ast = parser.parse(sourceCode, { range: true });
+    let constructorParams = "";
+    let found = false;
+
+    parser.visit(ast, {
+      ContractDefinition: (node) => {
+        if (node.name === contractName) {
+          for (const part of node.subNodes) {
+            if (part.type === "FunctionDefinition" && part.isConstructor) {
+              found = true;
+              if (part.range) {
+                  constructorParams = sourceCode.slice(part.range[0], part.range[1]).split("{")[0].trim();
+              }
+            }
+          }
+        }
+      }
+    });
+
+    if (found) {
+      return {
+        parameters: constructorParams
+      };
+    }
+  } catch (e) {
+    // console.warn("Failed to parse Solidity for constructor:", e);
+  }
+  return null;
+}

src/benchmark/runTesterBenchmark.ts

@@ -0,0 +1,302 @@
+import fs from "fs/promises";
+import path from "path";
+import { exec } from "child_process";
+import { promisify } from "util";
+import { runPoCGenerator } from "../agents/tester/index.js";
+import { VulnerabilityReport } from "../agents/tester/types.js";
+import "dotenv/config";
+
+const execAsync = promisify(exec);
+const DATASET_PATH = "Proof-of-Patch-only-dataset";
+const METADATA_FILE = path.join(DATASET_PATH, "dataset_metadata.json");
+const SUMMARY_FILE = "data/benchmark_summary.json";
+
+/**
+ * Extracts the likely vulnerable file path from annotation text.
+ * Looks for paths ending in .sol or github links.
+ */
+/**
+ * Extracts the likely vulnerable file path from annotation text.
+ */
+function extractVulnerableFilePath(text: string): string | null {
+  // Matches GitHub blob links: /blob/branch/path/to/File.sol
+  const githubBlobRegex = /\/blob\/[^/]+\/([^#\s]+\.sol)/g;
+  let match;
+  if ((match = githubBlobRegex.exec(text)) !== null) {
+    return match[1];
+  }
+
+  // Fallback to general .sol paths
+  const solPathRegex = /(?:^|[\s])([a-zA-Z0-9._/-]+\.sol)(?:#L\d+)?/g;
+  const paths: string[] = [];
+  while ((match = solPathRegex.exec(text)) !== null) {
+    const p = match[1];
+    if (!p.includes("test/") && !p.includes("Test.sol")) {
+      paths.push(p);
+    }
+  }
+
+  // Prioritize paths containing "src"
+  const srcPath = paths.find(p => p.includes("src/"));
+  return srcPath || (paths.length > 0 ? paths[0] : null);
+}
+
+/**
+ * Recursively finds a file by name within a directory, prioritizing src/
+ */
+async function findFileRecursively(dir: string, fileName: string): Promise<string | null> {
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+  const subdirs: string[] = [];
+  
+  // Check files in current dir first
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isFile() && entry.name === fileName) {
+      return fullPath;
+    }
+    if (entry.isDirectory() && entry.name !== "lib" && entry.name !== "node_modules") {
+      subdirs.push(fullPath);
+    }
+  }
+
+  // Prioritize "src" subdirectory if it exists
+  const srcDir = subdirs.find(d => path.basename(d) === "src");
+  if (srcDir) {
+    const found = await findFileRecursively(srcDir, fileName);
+    if (found) return found;
+  }
+
+  // Check other subdirs
+  for (const subdir of subdirs) {
+    if (path.basename(subdir) === "src") continue; // Already checked
+    const found = await findFileRecursively(subdir, fileName);
+    if (found) return found;
+  }
+
+  // Fallback to lib if nothing else found
+  const libDir = entries.find(e => e.isDirectory() && e.name === "lib");
+  if (libDir) {
+    return findFileRecursively(path.join(dir, "lib"), fileName);
+  }
+
+  return null;
+}
+
+async function main() {
+  const metadataContent = await fs.readFile(METADATA_FILE, "utf-8");
+  const metadata = JSON.parse(metadataContent);
+  const findingsIds = Object.keys(metadata);
+
+  const limit = process.argv[2] ? parseInt(process.argv[2]) : findingsIds.length;
+
+  console.log(`[Benchmark] Starting evaluation (Total available: ${findingsIds.length}, Limit: ${limit})...`);
+
+  const results: any[] = [];
+  let processedCount = 0;
+
+  for (const id of findingsIds) {
+    if (processedCount >= limit) break;
+
+    const finding = metadata[id];
+    
+    if (finding.benchmark_results?.vuln_status === "success" && !process.env.FORCE_RERUN) {
+      console.log(`[${id}] Skipping: already successful.`);
+      processedCount++;
+      continue;
+    }
+
+    console.log(`\n--- [${id}] ${finding.repo_name} ---`);
+    processedCount++;
+
+    try {
+      const annotationPath = path.join(DATASET_PATH, finding.annotations);
+      let annotationText = "";
+      try {
+        annotationText = await fs.readFile(annotationPath, "utf-8");
+      } catch (e) {
+        console.warn(`[${id}] Annotation file not found at ${annotationPath}`);
+      }
+      
+      const targetDir = path.join(process.cwd(), DATASET_PATH, finding.target_directory);
+      
+      // STEP 1: Accurate Source Code Resolution
+      const extractedPath = extractVulnerableFilePath(annotationText);
+      let mainContractPath = "";
+      let relativeContractPath = finding.main_contract;
+
+      if (extractedPath) {
+        const directPath = path.join(targetDir, extractedPath);
+        try {
+          await fs.access(directPath);
+          mainContractPath = directPath;
+          relativeContractPath = extractedPath;
+        } catch {
+          const fileName = path.basename(extractedPath);
+          console.log(`[${id}] File not found at ${extractedPath}, searching for ${fileName} recursively...`);
+          const foundPath = await findFileRecursively(targetDir, fileName);
+          if (foundPath) {
+            mainContractPath = foundPath;
+            relativeContractPath = path.relative(targetDir, foundPath);
+          }
+        }
+      }
+
+      if (!mainContractPath) {
+        mainContractPath = path.join(targetDir, finding.main_contract);
+        relativeContractPath = finding.main_contract;
+      }
+      
+      console.log(`[${id}] Using source file: ${relativeContractPath}`);
+
+      let sourceCode = "";
+      try {
+        sourceCode = await fs.readFile(mainContractPath, "utf-8");
+      } catch (e) {
+        console.warn(`[${id}] Contract not found at ${mainContractPath}, falling back to metadata.main_contract`);
+        try {
+            sourceCode = await fs.readFile(path.join(targetDir, finding.main_contract), "utf-8");
+        } catch (e2) {
+            throw new Error(`Could not find any source code for ${id}`);
+        }
+      }
+
+      const tempVulnDir = path.join(process.cwd(), "temp_vuln_run", id);
+      console.log(`[${id}] Preparing isolated sandbox at ${tempVulnDir}...`);
+      await execAsync(`mkdir -p temp_vuln_run && rm -rf ${tempVulnDir} && cp -r ${targetDir} ${tempVulnDir}`);
+      await execAsync(`rm -f ${tempVulnDir}/.git`);
+
+      // STEP 1.5: Reference Test Resolution
+      let referenceTestCode = "";
+      if (finding.test_fix_commands) {
+        const match = finding.test_fix_commands.match(/--match-path\s+([^\s]+)/);
+        if (match) {
+          const testPath = path.join(targetDir, match[1]);
+          try {
+            referenceTestCode = await fs.readFile(testPath, "utf-8");
+            console.log(`[${id}] Found reference test at ${match[1]}`);
+          } catch {
+            console.warn(`[${id}] Could not read reference test at ${testPath}`);
+          }
+        }
+      }
+
+      const report: VulnerabilityReport = {
+        id: id,
+        title: `${finding.repo_name} - ${id}`,
+        severity: (finding.impact?.toLowerCase() || "medium") as any,
+        type: finding.expected_vulnerability || "unknown",
+        description: annotationText,
+        referenceTestCode: referenceTestCode,
+        affectedContract: {
+          name: relativeContractPath.split("/").pop()!.replace(".sol", ""),
+          sourceCode: sourceCode,
+          sourceFilePath: relativeContractPath
+        },
+        attackVector: "Vulnerability analysis from dataset annotations.",
+        customSandboxDir: tempVulnDir 
+      };
+
+      console.log(`[${id}] Generating PoC and running on VULNERABLE version...`);
+      const resultVuln = await runPoCGenerator(report);
+      
+      if (process.env.DEBUG_CONTEXT === "true") {
+        console.log("\n" + "=".repeat(20) + " GENERATED POC START " + "=".repeat(20));
+        console.log(resultVuln.solidityCode);
+        console.log("=".repeat(20) + " GENERATED POC END " + "=".repeat(20) + "\n");
+      }
+      
+      let statusPatch = "not_tested";
+
+      if (resultVuln.status === "success") {
+        console.log(`[${id}] Running PoC on PATCHED version to verify specificity...`);
+        
+        const tempPatchDir = path.join(process.cwd(), "temp_patch_run", id);
+        try {
+            await execAsync(`mkdir -p temp_patch_run && rm -rf ${tempPatchDir} && cp -r ${targetDir} ${tempPatchDir}`);
+            await execAsync(`rm -f ${tempPatchDir}/.git`);
+            
+            const patchSourceDir = path.join(process.cwd(), DATASET_PATH, finding.patch);
+            await execAsync(`cp -rv ${patchSourceDir}/* ${tempPatchDir}/ || true`);
+
+            const { runFoundry } = await import("../agents/tester/tools/foundryRunner.js");
+            const patchExec = await runFoundry(resultVuln.solidityCode, tempPatchDir);
+            
+            const passedOnPatch = patchExec.exitCode === 0 && patchExec.stdout.includes("ok");
+            statusPatch = passedOnPatch ? "success" : "failed";
+            
+            if (statusPatch === "failed") {
+                await execAsync(`rm -rf ${tempPatchDir}`);
+            }
+        } catch (e: any) {
+            console.error(`[${id}] Patch run error:`, e.message);
+            statusPatch = "error";
+        }
+      }
+
+      const reproducible = resultVuln.status === "success";
+      const specific = resultVuln.status === "success" && statusPatch === "failed";
+
+      finding.benchmark_results = {
+        vuln_status: resultVuln.status,
+        patch_status: statusPatch,
+        reproducibility: reproducible,
+        specificity: specific,
+        iterations: resultVuln.iterations,
+        timestamp: new Date().toISOString(),
+      };
+
+      if (resultVuln.status === "failed") {
+          finding.benchmark_results.last_vuln_error = resultVuln.executionLogs[resultVuln.executionLogs.length - 1]?.slice(0, 500);
+      }
+      
+      results.push({
+        id,
+        reproducible,
+        specific,
+        iterations: resultVuln.iterations
+      });
+
+      await fs.writeFile(METADATA_FILE, JSON.stringify(metadata, null, 2));
+      console.log(`[${id}] Result: Reproducible=${reproducible}, Specific=${specific}`);
+
+    } catch (err: any) {
+      console.error(`[${id}] Fatal Error:`, err.message);
+      finding.benchmark_results = { 
+        status: "error", 
+        error: err.message,
+        timestamp: new Date().toISOString()
+      };
+      await fs.writeFile(METADATA_FILE, JSON.stringify(metadata, null, 2));
+    }
+  }
+  
+  const total = results.length;
+  const reproCount = results.filter(r => r.reproducible).length;
+  const specCount = results.filter(r => r.specific).length;
+  const avgIter = total > 0 ? results.reduce((acc, r) => acc + r.iterations, 0) / total : 0;
+
+  const summary = {
+    timestamp: new Date().toISOString(),
+    total_processed: total,
+    reproducibility_rate: total > 0 ? (reproCount / total) * 100 : 0,
+    specificity_rate: reproCount > 0 ? (specCount / reproCount) * 100 : 0,
+    overall_ground_truth_rate: total > 0 ? (specCount / total) * 100 : 0,
+    average_iterations: avgIter
+  };
+
+  console.log("\n" + "=".repeat(50));
+  console.log("BENCHMARK SUMMARY");
+  console.log("=".repeat(50));
+  console.log(`Total Findings:      ${total}`);
+  console.log(`Reproducibility:     ${summary.reproducibility_rate.toFixed(1)}% (${reproCount}/${total})`);
+  console.log(`Specificity:         ${summary.specificity_rate.toFixed(1)}% (${specCount}/${reproCount})`);
+  console.log(`Overall Success:     ${summary.overall_ground_truth_rate.toFixed(1)}% (Verified Ground Truth)`);
+  console.log(`Avg Iterations:      ${avgIter.toFixed(2)}`);
+  console.log("=".repeat(50));
+
+  await fs.mkdir(path.dirname(SUMMARY_FILE), { recursive: true });
+  await fs.writeFile(SUMMARY_FILE, JSON.stringify({ summary, details: results }, null, 2));
+  console.log(`Summary saved to ${SUMMARY_FILE}`);
+}
+
+main().catch(console.error);