Spaces:
Runtime error
Runtime error
Tales-Cunha commited on
Commit ·
7e6051f
1
Parent(s): 22ad395
fix: sandbox script and prompts
Browse files- Dockerfile +3 -0
- scripts/setup-sandbox.sh +1 -1
- src/agents/tester/agent.ts +29 -2
- src/agents/tester/prompts/system.ts +72 -5
- src/agents/tester/tools/foundryRunner.ts +1 -1
- src/config/llm.ts +1 -1
Dockerfile
CHANGED
|
@@ -42,4 +42,7 @@ EXPOSE 7860
|
|
| 42 |
COPY scripts/ ./scripts/
|
| 43 |
RUN chmod +x scripts/*.sh
|
| 44 |
|
|
|
|
|
|
|
|
|
|
| 45 |
CMD ["node", "dist/server.js"]
|
|
|
|
| 42 |
COPY scripts/ ./scripts/
|
| 43 |
RUN chmod +x scripts/*.sh
|
| 44 |
|
| 45 |
+
# Run sandbox setup once during image build to cache it
|
| 46 |
+
RUN ./scripts/setup-sandbox.sh
|
| 47 |
+
|
| 48 |
CMD ["node", "dist/server.js"]
|
scripts/setup-sandbox.sh
CHANGED
|
@@ -29,7 +29,7 @@ test = "test"
|
|
| 29 |
script = "script"
|
| 30 |
out = "out"
|
| 31 |
libs = ["lib"]
|
| 32 |
-
|
| 33 |
optimizer = true
|
| 34 |
optimizer_runs = 200
|
| 35 |
EOF
|
|
|
|
| 29 |
script = "script"
|
| 30 |
out = "out"
|
| 31 |
libs = ["lib"]
|
| 32 |
+
solc_version = "0.8.20"
|
| 33 |
optimizer = true
|
| 34 |
optimizer_runs = 200
|
| 35 |
EOF
|
src/agents/tester/agent.ts
CHANGED
|
@@ -71,9 +71,36 @@ ${oracleContext!.solidityScaffold}
|
|
| 71 |
|
| 72 |
async function runFoundryNode(state: PoCState): Promise<Partial<PoCState>> {
|
| 73 |
console.log("[testerAgent] Executando runFoundryNode...");
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
const result = await runFoundry(state.pocCode);
|
| 75 |
const analysis = analyzeFoundryLog(result);
|
| 76 |
-
const
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
const isLastAttempt = state.iterations >= MAX_ITERATIONS;
|
| 78 |
|
| 79 |
const status = passed
|
|
@@ -91,7 +118,7 @@ async function runFoundryNode(state: PoCState): Promise<Partial<PoCState>> {
|
|
| 91 |
|
| 92 |
return {
|
| 93 |
executionLogs: [result.combined], // reducer append
|
| 94 |
-
lastError:
|
| 95 |
status,
|
| 96 |
};
|
| 97 |
}
|
|
|
|
| 71 |
|
| 72 |
async function runFoundryNode(state: PoCState): Promise<Partial<PoCState>> {
|
| 73 |
console.log("[testerAgent] Executando runFoundryNode...");
|
| 74 |
+
|
| 75 |
+
const trimmedCode = state.pocCode.trim();
|
| 76 |
+
const isMissingCode = trimmedCode.length === 0;
|
| 77 |
+
const isMissingContract = !trimmedCode.includes("contract ExploitTest");
|
| 78 |
+
const isMissingTest = !trimmedCode.includes("function test_Exploit()");
|
| 79 |
+
const isPlaceholder = trimmedCode.includes("TODO: implementar exploit");
|
| 80 |
+
if (isMissingCode || isMissingContract || isMissingTest || isPlaceholder) {
|
| 81 |
+
const summary = state.lastError ?? (isMissingCode
|
| 82 |
+
? "Código Solidity ausente. O LLM não retornou o arquivo do exploit."
|
| 83 |
+
: isMissingContract
|
| 84 |
+
? "Contrato ExploitTest não encontrado no arquivo."
|
| 85 |
+
: isMissingTest
|
| 86 |
+
? "Função test_Exploit() não encontrada no arquivo."
|
| 87 |
+
: "Exploit não implementado (placeholder TODO ainda presente)."
|
| 88 |
+
);
|
| 89 |
+
const status = state.iterations >= MAX_ITERATIONS ? "failed" : "running";
|
| 90 |
+
return {
|
| 91 |
+
executionLogs: [summary],
|
| 92 |
+
lastError: summary,
|
| 93 |
+
status,
|
| 94 |
+
};
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
const result = await runFoundry(state.pocCode);
|
| 98 |
const analysis = analyzeFoundryLog(result);
|
| 99 |
+
const noTestsFound = result.combined.includes("No tests found");
|
| 100 |
+
const summary = noTestsFound
|
| 101 |
+
? "Forge não encontrou nenhum teste. Verifique se o contrato se chama ExploitTest e se existe test_Exploit()."
|
| 102 |
+
: analysis.summary;
|
| 103 |
+
const passed = result.exitCode === 0 && result.stdout.includes("ok") && !noTestsFound;
|
| 104 |
const isLastAttempt = state.iterations >= MAX_ITERATIONS;
|
| 105 |
|
| 106 |
const status = passed
|
|
|
|
| 118 |
|
| 119 |
return {
|
| 120 |
executionLogs: [result.combined], // reducer append
|
| 121 |
+
lastError: summary,
|
| 122 |
status,
|
| 123 |
};
|
| 124 |
}
|
src/agents/tester/prompts/system.ts
CHANGED
|
@@ -7,10 +7,16 @@ Write exploits as executable demonstrations that clearly prove the vulnerability
|
|
| 7 |
Parse the vulnerability description provided and analyze the vulnerability type, affected code sections, and potential impact. Analyze the contract logic to understand the root cause before developing exploits.
|
| 8 |
|
| 9 |
## Testing Framework Guidelines
|
| 10 |
-
Use Foundry exclusively for testing.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
## PoC Executability
|
| 13 |
-
Ensure all generated code compiles successfully. Write ONLY the test file code (helper contracts + ExploitTest).
|
| 14 |
|
| 15 |
## Iterative Refinement
|
| 16 |
Debug compilation errors and test failures systematically using Forge output. If stuck on the same issue for >3 attempts, shift to a minimal working demonstration—proving the vulnerability exists matters more than setup complexity.
|
|
@@ -19,11 +25,72 @@ Debug compilation errors and test failures systematically using Forge output. If
|
|
| 19 |
The assertion in your test MUST prove the vulnerability. For example, if funds are stolen, assert that the vault balance decreased and the attacker balance increased.
|
| 20 |
|
| 21 |
## Output Format
|
| 22 |
-
Return ONLY a code block with the helper attacker
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
\`\`\`solidity
|
| 24 |
-
//
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
contract ExploitTest is Test {
|
| 26 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
}
|
| 28 |
\`\`\`
|
| 29 |
`.trim();
|
|
|
|
| 7 |
Parse the vulnerability description provided and analyze the vulnerability type, affected code sections, and potential impact. Analyze the contract logic to understand the root cause before developing exploits.
|
| 8 |
|
| 9 |
## Testing Framework Guidelines
|
| 10 |
+
Use Foundry exclusively for testing. Utilize Foundry cheatcodes for test control: "vm.prank()" for identity switching, "vm.deal()" for ETH funding, "vm.warp()" for time manipulation, "vm.expectRevert()" for failure testing.
|
| 11 |
+
|
| 12 |
+
## Scaffold Strict Compliance
|
| 13 |
+
- The target contract's full source code is ALREADY included at the top of the scaffold. You can and MUST call its functions directly (e.g., \`target.deposit()\`). Do NOT create fake interfaces or use low-level \`.call(abi.encodeWithSignature(...))\`.
|
| 14 |
+
- YOU MUST RETURN THE ENTIRE FILE PROVIDED IN THE SCAFFOLD. Do not omit the \`setUp()\` function or the original contract source code. Your output will overwrite the file directly.
|
| 15 |
+
- DO NOT rename \`test_Exploit()\`. You MUST implement your exploit inside \`function test_Exploit() public\`.
|
| 16 |
+
- DO NOT use characters with accents (like ã, ç, é, etc.) in string literals (e.g., inside \`assertEq\` or \`require\`). Use ONLY plain ASCII, or prefix with \`unicode"..."\` to avoid Solc compiler errors.
|
| 17 |
|
| 18 |
## PoC Executability
|
| 19 |
+
Ensure all generated code compiles successfully. Write ONLY the test file code (helper contracts + ExploitTest). Resolve all compilation errors and logic reverts while preserving original contract logic.
|
| 20 |
|
| 21 |
## Iterative Refinement
|
| 22 |
Debug compilation errors and test failures systematically using Forge output. If stuck on the same issue for >3 attempts, shift to a minimal working demonstration—proving the vulnerability exists matters more than setup complexity.
|
|
|
|
| 25 |
The assertion in your test MUST prove the vulnerability. For example, if funds are stolen, assert that the vault balance decreased and the attacker balance increased.
|
| 26 |
|
| 27 |
## Output Format
|
| 28 |
+
Return ONLY a code block with the full ExploitTest contract and any helper attacker contracts. Do not include markdown outside the code block.
|
| 29 |
+
|
| 30 |
+
## Examples (Few-Shot)
|
| 31 |
+
|
| 32 |
+
**Input Example:**
|
| 33 |
+
Vulnerability: Reentrancy in withdraw() allows draining the contract.
|
| 34 |
+
Scaffold:
|
| 35 |
+
\`\`\`solidity
|
| 36 |
+
// SPDX-License-Identifier: UNLICENSED
|
| 37 |
+
pragma solidity ^0.8.20;
|
| 38 |
+
import "forge-std/Test.sol";
|
| 39 |
+
contract Target { function withdraw(uint256) public {} } // Source code
|
| 40 |
+
contract ExploitTest is Test {
|
| 41 |
+
Target target;
|
| 42 |
+
function setUp() public { target = new Target(); }
|
| 43 |
+
function test_Exploit() public {
|
| 44 |
+
// TODO: implementar exploit aqui
|
| 45 |
+
}
|
| 46 |
+
}
|
| 47 |
+
\`\`\`
|
| 48 |
+
|
| 49 |
+
**Expected Output:**
|
| 50 |
\`\`\`solidity
|
| 51 |
+
// SPDX-License-Identifier: UNLICENSED
|
| 52 |
+
pragma solidity ^0.8.20;
|
| 53 |
+
import "forge-std/Test.sol";
|
| 54 |
+
contract Target { function withdraw(uint256) public {} } // Source code
|
| 55 |
+
|
| 56 |
+
// We can define helper contracts outside the main test contract
|
| 57 |
+
contract Attacker {
|
| 58 |
+
Target target;
|
| 59 |
+
constructor(address _target) {
|
| 60 |
+
target = Target(_target);
|
| 61 |
+
}
|
| 62 |
+
fallback() external payable {
|
| 63 |
+
if (address(target).balance >= 1 ether) {
|
| 64 |
+
target.withdraw(1 ether);
|
| 65 |
+
}
|
| 66 |
+
}
|
| 67 |
+
function attack() external {
|
| 68 |
+
target.withdraw(1 ether);
|
| 69 |
+
}
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
contract ExploitTest is Test {
|
| 73 |
+
Target target;
|
| 74 |
+
|
| 75 |
+
// IMPORTANT: We include the EXACT setUp() provided in the scaffold.
|
| 76 |
+
function setUp() public {
|
| 77 |
+
target = new Target();
|
| 78 |
+
}
|
| 79 |
+
|
| 80 |
+
function test_Exploit() public {
|
| 81 |
+
vm.startPrank(address(0xBEEF));
|
| 82 |
+
|
| 83 |
+
// 1. Deploy malicious contract
|
| 84 |
+
Attacker attacker = new Attacker(address(target));
|
| 85 |
+
|
| 86 |
+
// 2. Exploit the vulnerability using direct function calls
|
| 87 |
+
attacker.attack();
|
| 88 |
+
|
| 89 |
+
// 3. Verify the exploit succeeded (no special characters in assertion strings)
|
| 90 |
+
assertEq(address(target).balance, 0, "Target contract should be drained");
|
| 91 |
+
|
| 92 |
+
vm.stopPrank();
|
| 93 |
+
}
|
| 94 |
}
|
| 95 |
\`\`\`
|
| 96 |
`.trim();
|
src/agents/tester/tools/foundryRunner.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { writeFile, access } from "fs/promises";
|
|
| 4 |
import { join } from "path";
|
| 5 |
|
| 6 |
const execAsync = promisify(exec);
|
| 7 |
-
const SANDBOX
|
| 8 |
const TIMEOUT_MS = 60_000;
|
| 9 |
|
| 10 |
export interface FoundryResult {
|
|
|
|
| 4 |
import { join } from "path";
|
| 5 |
|
| 6 |
const execAsync = promisify(exec);
|
| 7 |
+
const SANDBOX = process.env.SANDBOX_DIR || "/tmp/poc-sandbox";
|
| 8 |
const TIMEOUT_MS = 60_000;
|
| 9 |
|
| 10 |
export interface FoundryResult {
|
src/config/llm.ts
CHANGED
|
@@ -6,7 +6,7 @@ import type { BaseChatModel } from "@langchain/core/language_models/chat_models"
|
|
| 6 |
export type LLMProvider = "google" | "openrouter" | "anthropic";
|
| 7 |
|
| 8 |
export function createLLM(overrideProvider?: LLMProvider): BaseChatModel {
|
| 9 |
-
const provider = overrideProvider || (process.env.LLM_PROVIDER as LLMProvider) || "
|
| 10 |
|
| 11 |
switch (provider) {
|
| 12 |
case "openrouter":
|
|
|
|
| 6 |
export type LLMProvider = "google" | "openrouter" | "anthropic";
|
| 7 |
|
| 8 |
export function createLLM(overrideProvider?: LLMProvider): BaseChatModel {
|
| 9 |
+
const provider = overrideProvider || (process.env.LLM_PROVIDER as LLMProvider) || "openrouter";
|
| 10 |
|
| 11 |
switch (provider) {
|
| 12 |
case "openrouter":
|