Spaces:
Runtime error
Runtime error
Tales-Cunha commited on
Commit ·
d26fc11
1
Parent(s): 3f143e9
fix: revert the changes from the auditor agent and update the poc agent
Browse files- frontend/src/App.tsx +2 -1
- src/agents/auditor/agent.ts +10 -32
- src/agents/auditor/prompts.ts +2 -6
- src/agents/tester/agent.ts +10 -3
- src/agents/tester/prompts/system.ts +21 -22
- src/config/llm.ts +1 -1
- src/server.ts +3 -1
frontend/src/App.tsx
CHANGED
|
@@ -64,6 +64,7 @@ export function App() {
|
|
| 64 |
|
| 65 |
const decoder = new TextDecoder();
|
| 66 |
let buffer = "";
|
|
|
|
| 67 |
|
| 68 |
while (true) {
|
| 69 |
const { done, value } = await reader.read();
|
|
@@ -73,7 +74,6 @@ export function App() {
|
|
| 73 |
const lines = buffer.split("\n");
|
| 74 |
buffer = lines.pop() || "";
|
| 75 |
|
| 76 |
-
let currentEvent = "";
|
| 77 |
for (const line of lines) {
|
| 78 |
if (line.startsWith("event:")) {
|
| 79 |
currentEvent = line.slice(6).trim();
|
|
@@ -98,6 +98,7 @@ export function App() {
|
|
| 98 |
appendLog(`❌ ERRO: ${data}`);
|
| 99 |
break;
|
| 100 |
}
|
|
|
|
| 101 |
}
|
| 102 |
}
|
| 103 |
}
|
|
|
|
| 64 |
|
| 65 |
const decoder = new TextDecoder();
|
| 66 |
let buffer = "";
|
| 67 |
+
let currentEvent = "";
|
| 68 |
|
| 69 |
while (true) {
|
| 70 |
const { done, value } = await reader.read();
|
|
|
|
| 74 |
const lines = buffer.split("\n");
|
| 75 |
buffer = lines.pop() || "";
|
| 76 |
|
|
|
|
| 77 |
for (const line of lines) {
|
| 78 |
if (line.startsWith("event:")) {
|
| 79 |
currentEvent = line.slice(6).trim();
|
|
|
|
| 98 |
appendLog(`❌ ERRO: ${data}`);
|
| 99 |
break;
|
| 100 |
}
|
| 101 |
+
currentEvent = "";
|
| 102 |
}
|
| 103 |
}
|
| 104 |
}
|
src/agents/auditor/agent.ts
CHANGED
|
@@ -24,6 +24,8 @@ import {
|
|
| 24 |
} from "./config.ts";
|
| 25 |
import { matchLines } from "./utils.ts";
|
| 26 |
|
|
|
|
|
|
|
| 27 |
const walkDirectory = (dir: string, depth: number, solFiles: string[], docFiles: string[]) => {
|
| 28 |
if (depth > MAX_DEPTH) return;
|
| 29 |
|
|
@@ -119,29 +121,17 @@ const gatherContext: GraphNode<typeof AuditorState> = async (state) => {
|
|
| 119 |
parts.push(`### ${filePath}\n\`\`\`solidity\n${source}\n\`\`\``);
|
| 120 |
}
|
| 121 |
|
| 122 |
-
const
|
| 123 |
-
const result = await
|
| 124 |
-
|
| 125 |
-
const repoContext = typeof result.content === "string" ? result.content : JSON.stringify(result.content);
|
| 126 |
|
| 127 |
-
logger.info(`gatherContext: context built (${
|
| 128 |
-
logger.debug(`gatherContext: full context:\n${
|
| 129 |
|
| 130 |
-
return { repoContext };
|
| 131 |
};
|
| 132 |
|
| 133 |
const findVulnerabilities: GraphNode<typeof AuditorState> = async (state) => {
|
| 134 |
-
|
| 135 |
-
const llm = createLLM();
|
| 136 |
-
const model = llm.withStructuredOutput(
|
| 137 |
-
z.object({
|
| 138 |
-
findings: z.array(CandidateFindingSchema),
|
| 139 |
-
}),
|
| 140 |
-
{
|
| 141 |
-
name: "vulnerability_report",
|
| 142 |
-
method: "jsonSchema",
|
| 143 |
-
},
|
| 144 |
-
);
|
| 145 |
|
| 146 |
const previousFeedback =
|
| 147 |
state.judgeReviews.length > 0
|
|
@@ -195,7 +185,6 @@ const findVulnerabilities: GraphNode<typeof AuditorState> = async (state) => {
|
|
| 195 |
};
|
| 196 |
|
| 197 |
const judgeFindings: GraphNode<typeof AuditorState> = async (state) => {
|
| 198 |
-
console.log(`[auditorAgent] Iniciando judgeFindings para ${state.candidateFindings.length} candidatos...`);
|
| 199 |
if (state.candidateFindings.length === 0) {
|
| 200 |
logger.info("judgeFindings: no candidate findings to review, skipping LLM call");
|
| 201 |
return {
|
|
@@ -205,16 +194,7 @@ const judgeFindings: GraphNode<typeof AuditorState> = async (state) => {
|
|
| 205 |
};
|
| 206 |
}
|
| 207 |
|
| 208 |
-
const
|
| 209 |
-
const model = llm.withStructuredOutput(
|
| 210 |
-
z.object({
|
| 211 |
-
review_result: JudgeReviewSchema,
|
| 212 |
-
}),
|
| 213 |
-
{
|
| 214 |
-
name: "judge_review",
|
| 215 |
-
method: "jsonSchema",
|
| 216 |
-
},
|
| 217 |
-
);
|
| 218 |
|
| 219 |
logger.info(`judgeFindings: reviewing ${state.candidateFindings.length} candidate finding(s) in parallel`);
|
| 220 |
|
|
@@ -230,13 +210,12 @@ const judgeFindings: GraphNode<typeof AuditorState> = async (state) => {
|
|
| 230 |
const findingText = `[Finding ${i + 1}] ${finding.title}\nSeverity: ${finding.severity}\nDescription: ${finding.description}\nLocation: ${finding.path} lines ${finding.location}\nCode:\n\`\`\`solidity\n${finding.codeSnippet}\n\`\`\``;
|
| 231 |
|
| 232 |
logger.debug(`judgeFindings: reviewing finding ${i + 1}: ${finding.title}`);
|
| 233 |
-
|
| 234 |
new SystemMessage(JUDGE_FINDINGS_PROMPT),
|
| 235 |
new HumanMessage(
|
| 236 |
`Contract (${finding.path}):\n\n${source}\n\nProtocol Context:\n${state.repoContext}\n\nFinding to Review:\n\n${findingText}`,
|
| 237 |
),
|
| 238 |
]);
|
| 239 |
-
return result.review_result;
|
| 240 |
}),
|
| 241 |
);
|
| 242 |
|
|
@@ -255,7 +234,6 @@ const judgeFindings: GraphNode<typeof AuditorState> = async (state) => {
|
|
| 255 |
|
| 256 |
const falsePositiveCount = state.candidateFindings.length - findings.length;
|
| 257 |
|
| 258 |
-
console.log(`[auditorAgent] Concluído: ${findings.length} confirmados, ${falsePositiveCount} falsos positivos.`);
|
| 259 |
logger.info(`judgeFindings: ${findings.length} confirmed, ${falsePositiveCount} false positive(s)`);
|
| 260 |
logger.debug(`judgeFindings: reviews:\n${JSON.stringify(reviews, null, 2)}`);
|
| 261 |
|
|
|
|
| 24 |
} from "./config.ts";
|
| 25 |
import { matchLines } from "./utils.ts";
|
| 26 |
|
| 27 |
+
const llm = createLLM();
|
| 28 |
+
|
| 29 |
const walkDirectory = (dir: string, depth: number, solFiles: string[], docFiles: string[]) => {
|
| 30 |
if (depth > MAX_DEPTH) return;
|
| 31 |
|
|
|
|
| 121 |
parts.push(`### ${filePath}\n\`\`\`solidity\n${source}\n\`\`\``);
|
| 122 |
}
|
| 123 |
|
| 124 |
+
const model = llm.withStructuredOutput(z.object({ context: z.string() }));
|
| 125 |
+
const result = await model.invoke([new SystemMessage(GATHER_CONTEXT_PROMPT), new HumanMessage(parts.join("\n\n"))]);
|
|
|
|
|
|
|
| 126 |
|
| 127 |
+
logger.info(`gatherContext: context built (${parts.join("\n\n").length} chars)`);
|
| 128 |
+
logger.debug(`gatherContext: full context:\n${parts.join("\n\n")}`);
|
| 129 |
|
| 130 |
+
return { repoContext: result.context };
|
| 131 |
};
|
| 132 |
|
| 133 |
const findVulnerabilities: GraphNode<typeof AuditorState> = async (state) => {
|
| 134 |
+
const model = llm.withStructuredOutput(z.object({ findings: z.array(CandidateFindingSchema) }));
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
|
| 136 |
const previousFeedback =
|
| 137 |
state.judgeReviews.length > 0
|
|
|
|
| 185 |
};
|
| 186 |
|
| 187 |
const judgeFindings: GraphNode<typeof AuditorState> = async (state) => {
|
|
|
|
| 188 |
if (state.candidateFindings.length === 0) {
|
| 189 |
logger.info("judgeFindings: no candidate findings to review, skipping LLM call");
|
| 190 |
return {
|
|
|
|
| 194 |
};
|
| 195 |
}
|
| 196 |
|
| 197 |
+
const model = llm.withStructuredOutput(JudgeReviewSchema);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 198 |
|
| 199 |
logger.info(`judgeFindings: reviewing ${state.candidateFindings.length} candidate finding(s) in parallel`);
|
| 200 |
|
|
|
|
| 210 |
const findingText = `[Finding ${i + 1}] ${finding.title}\nSeverity: ${finding.severity}\nDescription: ${finding.description}\nLocation: ${finding.path} lines ${finding.location}\nCode:\n\`\`\`solidity\n${finding.codeSnippet}\n\`\`\``;
|
| 211 |
|
| 212 |
logger.debug(`judgeFindings: reviewing finding ${i + 1}: ${finding.title}`);
|
| 213 |
+
return model.invoke([
|
| 214 |
new SystemMessage(JUDGE_FINDINGS_PROMPT),
|
| 215 |
new HumanMessage(
|
| 216 |
`Contract (${finding.path}):\n\n${source}\n\nProtocol Context:\n${state.repoContext}\n\nFinding to Review:\n\n${findingText}`,
|
| 217 |
),
|
| 218 |
]);
|
|
|
|
| 219 |
}),
|
| 220 |
);
|
| 221 |
|
|
|
|
| 234 |
|
| 235 |
const falsePositiveCount = state.candidateFindings.length - findings.length;
|
| 236 |
|
|
|
|
| 237 |
logger.info(`judgeFindings: ${findings.length} confirmed, ${falsePositiveCount} false positive(s)`);
|
| 238 |
logger.debug(`judgeFindings: reviews:\n${JSON.stringify(reviews, null, 2)}`);
|
| 239 |
|
src/agents/auditor/prompts.ts
CHANGED
|
@@ -24,9 +24,7 @@ Seja preciso e exaustivo — quanto mais rico o contexto, com mais precisão as
|
|
| 24 |
|
| 25 |
export const FIND_VULNERABILITIES_PROMPT = `Você é um auditor especialista em segurança de smart contracts com foco em Solidity. Analise sistematicamente o código-fonte do contrato e o contexto do protocolo para identificar vulnerabilidades de segurança.
|
| 26 |
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
Para cada vulnerabilidade em "findings", forneça TODOS os seguintes campos:
|
| 30 |
|
| 31 |
- **title**: Nome curto e preciso (ex.: "Reentrância em withdraw", "Controle de acesso ausente em setFee").
|
| 32 |
- **description**: Explique o comportamento ESPERADO versus o comportamento OBSERVADO (vulnerável) em 2 a 4 frases.
|
|
@@ -40,9 +38,7 @@ Se feedback do juiz de uma iteração anterior for fornecido, remova os falsos p
|
|
| 40 |
|
| 41 |
export const JUDGE_FINDINGS_PROMPT = `Você é um revisor rigoroso de segurança de smart contracts. Avalie cada vulnerabilidade candidata submetida pelo auditor e determine se é um verdadeiro positivo ou um falso positivo.
|
| 42 |
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
Para o achado fornecido, preencha os seguintes campos em "review_result":
|
| 46 |
|
| 47 |
- **review**: Análise detalhada (3 a 6 frases) explicando por que a vulnerabilidade é ou não real. Referencie código específico, invariantes do protocolo, pré-condições e controles mitigadores.
|
| 48 |
- **isFalsePositive**: true se o achado NÃO for explorável na prática; false se for uma vulnerabilidade real.
|
|
|
|
| 24 |
|
| 25 |
export const FIND_VULNERABILITIES_PROMPT = `Você é um auditor especialista em segurança de smart contracts com foco em Solidity. Analise sistematicamente o código-fonte do contrato e o contexto do protocolo para identificar vulnerabilidades de segurança.
|
| 26 |
|
| 27 |
+
Para cada vulnerabilidade, forneça TODOS os seguintes campos:
|
|
|
|
|
|
|
| 28 |
|
| 29 |
- **title**: Nome curto e preciso (ex.: "Reentrância em withdraw", "Controle de acesso ausente em setFee").
|
| 30 |
- **description**: Explique o comportamento ESPERADO versus o comportamento OBSERVADO (vulnerável) em 2 a 4 frases.
|
|
|
|
| 38 |
|
| 39 |
export const JUDGE_FINDINGS_PROMPT = `Você é um revisor rigoroso de segurança de smart contracts. Avalie cada vulnerabilidade candidata submetida pelo auditor e determine se é um verdadeiro positivo ou um falso positivo.
|
| 40 |
|
| 41 |
+
Para cada achado, forneça TODOS os seguintes campos:
|
|
|
|
|
|
|
| 42 |
|
| 43 |
- **review**: Análise detalhada (3 a 6 frases) explicando por que a vulnerabilidade é ou não real. Referencie código específico, invariantes do protocolo, pré-condições e controles mitigadores.
|
| 44 |
- **isFalsePositive**: true se o achado NÃO for explorável na prática; false se for uma vulnerabilidade real.
|
src/agents/tester/agent.ts
CHANGED
|
@@ -78,6 +78,15 @@ async function runFoundryNode(state: PoCState): Promise<Partial<PoCState>> {
|
|
| 78 |
const result = await runFoundry(state.pocCode);
|
| 79 |
const analysis = analyzeFoundryLog(result);
|
| 80 |
const passed = result.exitCode === 0 && result.stdout.includes("ok");
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
|
| 82 |
console.log(`[testerAgent] Resultado Foundry: exitCode=${result.exitCode}, passed=${passed}`);
|
| 83 |
if (!passed) {
|
|
@@ -87,9 +96,7 @@ async function runFoundryNode(state: PoCState): Promise<Partial<PoCState>> {
|
|
| 87 |
return {
|
| 88 |
executionLogs: [result.combined], // reducer append
|
| 89 |
lastError: analysis.summary,
|
| 90 |
-
status
|
| 91 |
-
: result.timedOut ? "timeout"
|
| 92 |
-
: "running",
|
| 93 |
};
|
| 94 |
}
|
| 95 |
|
|
|
|
| 78 |
const result = await runFoundry(state.pocCode);
|
| 79 |
const analysis = analyzeFoundryLog(result);
|
| 80 |
const passed = result.exitCode === 0 && result.stdout.includes("ok");
|
| 81 |
+
const isLastAttempt = state.iterations >= MAX_ITERATIONS;
|
| 82 |
+
|
| 83 |
+
const status = passed
|
| 84 |
+
? "success"
|
| 85 |
+
: result.timedOut
|
| 86 |
+
? "timeout"
|
| 87 |
+
: isLastAttempt
|
| 88 |
+
? "failed"
|
| 89 |
+
: "running";
|
| 90 |
|
| 91 |
console.log(`[testerAgent] Resultado Foundry: exitCode=${result.exitCode}, passed=${passed}`);
|
| 92 |
if (!passed) {
|
|
|
|
| 96 |
return {
|
| 97 |
executionLogs: [result.combined], // reducer append
|
| 98 |
lastError: analysis.summary,
|
| 99 |
+
status,
|
|
|
|
|
|
|
| 100 |
};
|
| 101 |
}
|
| 102 |
|
src/agents/tester/prompts/system.ts
CHANGED
|
@@ -1,30 +1,29 @@
|
|
| 1 |
-
export const SYSTEM_PROMPT = `
|
| 2 |
|
| 3 |
-
##
|
| 4 |
-
|
| 5 |
-
- Inclua comentários detalhados documentando cada passo do ataque.
|
| 6 |
-
- O PoC deve ser autoexplicativo para auditores de segurança.
|
| 7 |
|
| 8 |
-
##
|
| 9 |
-
|
| 10 |
-
- NÃO modifique o contrato original ou o bloco "setUp()" fornecido no scaffold.
|
| 11 |
-
- Utilize cheatcodes de forma apropriada: vm.prank(), vm.deal(), vm.warp(), vm.expectRevert().
|
| 12 |
-
- A assertion final DEVE usar assertTrue(), assertGt() ou assertEq() para provar o sucesso do exploit.
|
| 13 |
|
| 14 |
-
##
|
| 15 |
-
|
| 16 |
-
- Mantenha o PoC minimalista e focado apenas na vulnerabilidade descrita.
|
| 17 |
-
- Se necessário, crie contratos auxiliares (ex: atacante malicioso) ANTES do contrato ExploitTest.
|
| 18 |
-
- Preserve a lógica original do contrato sem modificações.
|
| 19 |
|
| 20 |
-
##
|
| 21 |
-
|
| 22 |
-
- Para erros de importação, use apenas os arquivos já presentes no projeto.
|
| 23 |
-
- Se travar no mesmo erro por >3 iterações, tente uma abordagem mais simples que ainda prove o ponto.
|
| 24 |
|
| 25 |
-
##
|
| 26 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
\`\`\`solidity
|
| 28 |
-
//
|
|
|
|
|
|
|
|
|
|
| 29 |
\`\`\`
|
| 30 |
`.trim();
|
|
|
|
| 1 |
+
export const SYSTEM_PROMPT = `You are an expert smart contract security testing specialist. Your mission is to generate executable Proof-of-Concept (PoC) exploits demonstrating vulnerabilities using Foundry.
|
| 2 |
|
| 3 |
+
## PoC Explainability
|
| 4 |
+
Write exploits as executable demonstrations that clearly prove the vulnerability. Include detailed comments documenting each attack step, the vulnerability being exploited, and why the exploit succeeds. The PoC must be self-explanatory to security auditors.
|
|
|
|
|
|
|
| 5 |
|
| 6 |
+
## Vulnerability Analysis
|
| 7 |
+
Parse the vulnerability description provided and analyze the vulnerability type, affected code sections, and potential impact. Analyze the contract logic to understand the root cause before developing exploits.
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
+
## Testing Framework Guidelines
|
| 10 |
+
Use Foundry exclusively for testing. Implement proper "setUp()" functions with realistic contract states: i.e. initializing contracts with typical production values (reasonable token balances, realistic timestamps, standard protocol roles assigned). Utilize Foundry cheatcodes for test control: "vm.prank()" for identity switching, "vm.deal()" for ETH funding, "vm.warp()" for time manipulation, "vm.expectRevert()" for failure testing.
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
+
## PoC Executability
|
| 13 |
+
Ensure all generated code compiles successfully. Write ONLY the test file code (helper contracts + ExploitTest). Do NOT modify or re-include the original contract source code provided in the scaffold. Resolve all compilation errors and logic reverts while preserving original contract logic.
|
|
|
|
|
|
|
| 14 |
|
| 15 |
+
## Iterative Refinement
|
| 16 |
+
Debug compilation errors and test failures systematically using Forge output. If stuck on the same issue for >3 attempts, shift to a minimal working demonstration—proving the vulnerability exists matters more than setup complexity.
|
| 17 |
+
|
| 18 |
+
## Exploit Soundness
|
| 19 |
+
The assertion in your test MUST prove the vulnerability. For example, if funds are stolen, assert that the vault balance decreased and the attacker balance increased.
|
| 20 |
+
|
| 21 |
+
## Output Format
|
| 22 |
+
Return ONLY a code block with the helper attacker contract (if needed) and the ExploitTest contract:
|
| 23 |
\`\`\`solidity
|
| 24 |
+
// Attacker helpers here...
|
| 25 |
+
contract ExploitTest is Test {
|
| 26 |
+
// ...
|
| 27 |
+
}
|
| 28 |
\`\`\`
|
| 29 |
`.trim();
|
src/config/llm.ts
CHANGED
|
@@ -11,7 +11,7 @@ export function createLLM(overrideProvider?: LLMProvider): BaseChatModel {
|
|
| 11 |
switch (provider) {
|
| 12 |
case "openrouter":
|
| 13 |
return new ChatOpenRouter({
|
| 14 |
-
model: process.env.OPENROUTER_MODEL || "
|
| 15 |
temperature: 0.2,
|
| 16 |
apiKey: process.env.OPENROUTER_API_KEY,
|
| 17 |
maxTokens: 4096,
|
|
|
|
| 11 |
switch (provider) {
|
| 12 |
case "openrouter":
|
| 13 |
return new ChatOpenRouter({
|
| 14 |
+
model: process.env.OPENROUTER_MODEL || "google/gemini-3.1-flash-lite",
|
| 15 |
temperature: 0.2,
|
| 16 |
apiKey: process.env.OPENROUTER_API_KEY,
|
| 17 |
maxTokens: 4096,
|
src/server.ts
CHANGED
|
@@ -84,6 +84,8 @@ app.post("/api/run", (c) => {
|
|
| 84 |
const testerResult = await testerAgent.invoke({ report });
|
| 85 |
|
| 86 |
await send("log", `[Tester] Execução concluída com status: ${testerResult.status}`);
|
|
|
|
|
|
|
| 87 |
await send(
|
| 88 |
"tester",
|
| 89 |
JSON.stringify({
|
|
@@ -95,7 +97,7 @@ app.post("/api/run", (c) => {
|
|
| 95 |
);
|
| 96 |
} else {
|
| 97 |
await send("log", "[Tester] Nenhuma vulnerabilidade para testar.");
|
| 98 |
-
await send("tester", JSON.stringify({
|
| 99 |
}
|
| 100 |
|
| 101 |
await send("log", "Pipeline concluído.");
|
|
|
|
| 84 |
const testerResult = await testerAgent.invoke({ report });
|
| 85 |
|
| 86 |
await send("log", `[Tester] Execução concluída com status: ${testerResult.status}`);
|
| 87 |
+
|
| 88 |
+
// Garante que o objeto enviado tem exatamente o que o front espera
|
| 89 |
await send(
|
| 90 |
"tester",
|
| 91 |
JSON.stringify({
|
|
|
|
| 97 |
);
|
| 98 |
} else {
|
| 99 |
await send("log", "[Tester] Nenhuma vulnerabilidade para testar.");
|
| 100 |
+
await send("tester", JSON.stringify({ status: "skipped", iterations: 0 }));
|
| 101 |
}
|
| 102 |
|
| 103 |
await send("log", "Pipeline concluído.");
|