| import { describe, it, expect, vi } from 'vitest'; |
| import { AgentLoop } from '../core/agent-loop'; |
| import type { |
| ProviderAdapter, |
| ToolExecutor, |
| ContextManager, |
| ProgressReporter, |
| CostTracker, |
| AgentLoopConfig, |
| ParsedResponse, |
| ToolResult, |
| ToolCall, |
| ToolDef, |
| Message, |
| UsageInfo, |
| } from '../core/types'; |
|
|
| |
|
|
| function createMockProvider(responses: ParsedResponse[]): ProviderAdapter { |
| let callIndex = 0; |
| return { |
| call: vi.fn(async () => { |
| if (callIndex >= responses.length) { |
| return { content: '' }; |
| } |
| return responses[callIndex++]; |
| }), |
| getModel: () => 'test-model', |
| getProvider: () => 'test-provider', |
| supportsTools: () => true, |
| }; |
| } |
|
|
| function createMockExecutor(results?: Map<string, ToolResult>): ToolExecutor { |
| return { |
| execute: vi.fn(async (toolCall: ToolCall) => { |
| if (results && results.has(toolCall.id)) { |
| return results.get(toolCall.id)!; |
| } |
| return { |
| tool_call_id: toolCall.id, |
| content: 'OK', |
| success: true, |
| }; |
| }), |
| getDefinitions: () => [{ name: 'bash', description: 'Run shell', parameters: {} }] as ToolDef[], |
| }; |
| } |
|
|
| function createMockContext(): ContextManager { |
| const messages: Message[] = []; |
| return { |
| getMessages: () => messages, |
| getSanitizedMessages: () => messages, |
| setSystemPrompt: vi.fn((prompt: string) => { |
| if (messages.length > 0 && messages[0].role === 'system') { |
| messages[0] = { role: 'system', content: prompt }; |
| } else { |
| messages.unshift({ role: 'system', content: prompt }); |
| } |
| }), |
| addUserMessage: vi.fn((content: string) => { |
| messages.push({ role: 'user', content }); |
| }), |
| addAssistantTurn: vi.fn((response: ParsedResponse) => { |
| messages.push({ |
| role: 'assistant', |
| content: response.content || '', |
| ...(response.toolCalls?.length ? { tool_calls: response.toolCalls } : {}), |
| }); |
| }), |
| addToolResults: vi.fn((results: ToolResult[]) => { |
| for (const r of results) { |
| messages.push({ role: 'tool', content: r.content, tool_call_id: r.tool_call_id }); |
| } |
| }), |
| importMessages: vi.fn(), |
| needsCompaction: () => false, |
| compact: vi.fn(async () => undefined), |
| getTokenEstimate: () => 1000, |
| getCompactionCount: () => 0, |
| }; |
| } |
|
|
| function createMockProgress(): ProgressReporter { |
| return { onEvent: vi.fn() }; |
| } |
|
|
| function createMockCost(): CostTracker { |
| return { |
| record: vi.fn(), |
| getTurnCost: () => 0, |
| getTotalCost: () => 0, |
| getTotalUsage: () => ({ promptTokens: 0, completionTokens: 0, totalTokens: 0 }), |
| resetTurn: vi.fn(), |
| }; |
| } |
|
|
| function defaultConfig(overrides?: Partial<AgentLoopConfig>): AgentLoopConfig { |
| return { |
| maxIterations: 20, |
| maxNudges: 2, |
| maxDuplicateToolCalls: 3, |
| agentType: 'orchestrator', |
| isReadOnly: false, |
| ...overrides, |
| }; |
| } |
|
|
| |
|
|
| describe('AgentLoop', () => { |
| it('completes when model returns statusComplete signal', async () => { |
| |
| const toolCall: ToolCall = { |
| id: 'tc1', |
| type: 'function', |
| function: { name: 'bash', arguments: '{"command":"status --task \\"build site\\" --done \\"all\\" --remaining \\"none\\" --complete"}' }, |
| }; |
|
|
| const provider = createMockProvider([ |
| { content: '', toolCalls: [toolCall] }, |
| { content: 'All done!' }, |
| ]); |
|
|
| const toolResult: ToolResult = { |
| tool_call_id: 'tc1', |
| content: 'Status recorded.', |
| success: true, |
| signals: { statusComplete: true, statusResult: { task: 'build site', done: 'all', remaining: 'none', complete: true, hasExplicitFlag: true } }, |
| }; |
| const results = new Map([['tc1', toolResult]]); |
| const executor = createMockExecutor(results); |
| const context = createMockContext(); |
| const progress = createMockProgress(); |
| const cost = createMockCost(); |
| const config = defaultConfig(); |
|
|
| const loop = new AgentLoop({ |
| config, |
| provider, |
| executor, |
| context, |
| progress, |
| cost, |
| }); |
|
|
| const result = await loop.run('Build me a site'); |
| expect(result.success).toBe(true); |
| expect(result.toolCount).toBe(1); |
| expect(result.turnCount).toBeGreaterThanOrEqual(1); |
| }); |
|
|
| it('stops after maxNudges when model returns text without tool calls', async () => { |
| |
| const provider = createMockProvider([ |
| { content: 'Let me think about that...' }, |
| { content: 'Still thinking...' }, |
| { content: 'Almost there...' }, |
| { content: 'One more thought...' }, |
| { content: 'Final thought.' }, |
| ]); |
|
|
| const context = createMockContext(); |
| const progress = createMockProgress(); |
| const cost = createMockCost(); |
| const config = defaultConfig({ maxNudges: 2 }); |
|
|
| const loop = new AgentLoop({ |
| config, |
| provider, |
| executor: createMockExecutor(), |
| context, |
| progress, |
| cost, |
| }); |
|
|
| const result = await loop.run('Do something'); |
| |
| expect(result.success).toBe(false); |
| expect(result.summary).toContain('nudge'); |
| }); |
|
|
| it('nudges toward bash and retries when the model calls a non-bash tool (stream early-aborted)', async () => { |
| const statusTool: ToolCall = { |
| id: 'tc1', |
| type: 'function', |
| function: { name: 'bash', arguments: '{"command":"status --task \\"x\\" --done \\"y\\" --remaining \\"none\\" --complete"}' }, |
| }; |
| |
| const provider = createMockProvider([ |
| { invalidToolName: 'cat' }, |
| { content: '', toolCalls: [statusTool] }, |
| { content: 'done' }, |
| ]); |
| const results = new Map([['tc1', { |
| tool_call_id: 'tc1', |
| content: 'Status recorded.', |
| success: true, |
| signals: { statusComplete: true, statusResult: { task: 'x', done: 'y', remaining: 'none', complete: true, hasExplicitFlag: true } }, |
| } as ToolResult]]); |
| const context = createMockContext(); |
| const progress = createMockProgress(); |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig(), |
| provider, |
| executor: createMockExecutor(results), |
| context, |
| progress, |
| cost: createMockCost(), |
| }); |
|
|
| const result = await loop.run('do it'); |
|
|
| expect(progress.onEvent).toHaveBeenCalledWith('malformed_tool_call', expect.objectContaining({ invalidToolName: 'cat' })); |
| const nudges = (context.addUserMessage as ReturnType<typeof vi.fn>).mock.calls.map((c) => c[0] as string); |
| expect(nudges.some((m) => m.includes('cat') && m.includes('bash'))).toBe(true); |
| expect(result.success).toBe(true); |
| }); |
|
|
| it('stops (does not loop forever) when the model keeps calling a non-bash tool', async () => { |
| const provider = createMockProvider(Array.from({ length: 8 }, () => ({ invalidToolName: 'cat' } as ParsedResponse))); |
| const progress = createMockProgress(); |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig({ maxNudges: 2 }), |
| provider, |
| executor: createMockExecutor(), |
| context: createMockContext(), |
| progress, |
| cost: createMockCost(), |
| }); |
|
|
| const result = await loop.run('do it'); |
| expect(result.success).toBe(false); |
| |
| const malformed = (progress.onEvent as ReturnType<typeof vi.fn>).mock.calls.filter((c) => c[0] === 'malformed_tool_call'); |
| expect(malformed.length).toBeGreaterThanOrEqual(2); |
| }); |
|
|
| it('interview agent pauses (awaiting_user) on a prose question instead of nudging', async () => { |
| |
| |
| const provider = createMockProvider([ |
| { content: "What's your company's name?" }, |
| ]); |
| const loop = new AgentLoop({ |
| config: defaultConfig({ agentType: 'interview' }), |
| provider, |
| executor: createMockExecutor(), |
| context: createMockContext(), |
| progress: createMockProgress(), |
| cost: createMockCost(), |
| }); |
|
|
| const result = await loop.run('Understand a company'); |
| expect(result.exitReason).toBe('awaiting_user'); |
| expect(result.success).toBe(true); |
| }); |
|
|
| it('interview agent does not pause on a reasoning-only turn (no user-facing content)', async () => { |
| |
| const provider = createMockProvider([ |
| { reasoningDetails: [{ type: 'reasoning.text', text: 'thinking' }] as ParsedResponse['reasoningDetails'] }, |
| ]); |
| const loop = new AgentLoop({ |
| config: defaultConfig({ agentType: 'interview', maxNudges: 1 }), |
| provider, |
| executor: createMockExecutor(), |
| context: createMockContext(), |
| progress: createMockProgress(), |
| cost: createMockCost(), |
| }); |
|
|
| const result = await loop.run('Understand a company'); |
| expect(result.exitReason).not.toBe('awaiting_user'); |
| }); |
|
|
| it('detects duplicate tool calls and stops after maxDuplicateToolCalls', async () => { |
| const duplicateCall: ToolCall = { |
| id: 'tc-dup', |
| type: 'function', |
| function: { name: 'bash', arguments: '{"command":"ls"}' }, |
| }; |
|
|
| |
| const provider = createMockProvider([ |
| { content: '', toolCalls: [{ ...duplicateCall, id: 'tc1' }] }, |
| { content: '', toolCalls: [{ ...duplicateCall, id: 'tc2' }] }, |
| { content: '', toolCalls: [{ ...duplicateCall, id: 'tc3' }] }, |
| { content: '', toolCalls: [{ ...duplicateCall, id: 'tc4' }] }, |
| ]); |
|
|
| const context = createMockContext(); |
| const progress = createMockProgress(); |
| const cost = createMockCost(); |
| const config = defaultConfig({ maxDuplicateToolCalls: 3 }); |
|
|
| const loop = new AgentLoop({ |
| config, |
| provider, |
| executor: createMockExecutor(), |
| context, |
| progress, |
| cost, |
| }); |
|
|
| const result = await loop.run('Do the thing'); |
| expect(result.success).toBe(false); |
| expect(result.summary).toContain('loop'); |
| }); |
|
|
| it('calls completionGate before accepting completion', async () => { |
| let gateCallCount = 0; |
| const completionGate = vi.fn(async () => { |
| gateCallCount++; |
| |
| if (gateCallCount === 1) return 'Runtime error: undefined is not a function'; |
| return null; |
| }); |
|
|
| |
| const toolCall1: ToolCall = { |
| id: 'tc1', |
| type: 'function', |
| function: { name: 'bash', arguments: '{"command":"status --task \\"build\\" --done \\"all\\" --remaining \\"none\\" --complete"}' }, |
| }; |
| const toolCall2: ToolCall = { |
| id: 'tc2', |
| type: 'function', |
| function: { name: 'bash', arguments: '{"command":"status --task \\"build site\\" --done \\"everything\\" --remaining \\"none\\" --complete"}' }, |
| }; |
|
|
| const toolResult1: ToolResult = { |
| tool_call_id: 'tc1', |
| content: 'Status recorded.', |
| success: true, |
| signals: { statusComplete: true, statusResult: { task: 'build', done: 'all', remaining: 'none', complete: true, hasExplicitFlag: true } }, |
| }; |
| const toolResult2: ToolResult = { |
| tool_call_id: 'tc2', |
| content: 'Status recorded.', |
| success: true, |
| signals: { statusComplete: true, statusResult: { task: 'build site', done: 'everything', remaining: 'none', complete: true, hasExplicitFlag: true } }, |
| }; |
|
|
| |
| |
| const provider = createMockProvider([ |
| { content: '', toolCalls: [toolCall1] }, |
| { content: '', toolCalls: [toolCall2] }, |
| ]); |
|
|
| const results = new Map([ |
| ['tc1', toolResult1], |
| ['tc2', toolResult2], |
| ]); |
| const executor = createMockExecutor(results); |
| const context = createMockContext(); |
| const progress = createMockProgress(); |
| const cost = createMockCost(); |
| const config = defaultConfig({ completionGate }); |
|
|
| const loop = new AgentLoop({ |
| config, |
| provider, |
| executor, |
| context, |
| progress, |
| cost, |
| }); |
|
|
| const result = await loop.run('Build it'); |
| expect(completionGate).toHaveBeenCalledTimes(2); |
| expect(result.success).toBe(true); |
| }); |
|
|
| it('handles provider error via onPausableError with continue', async () => { |
| let callCount = 0; |
| const provider: ProviderAdapter = { |
| call: vi.fn(async () => { |
| callCount++; |
| if (callCount === 1) { |
| throw new Error('Rate limited'); |
| } |
| |
| return { |
| content: '', |
| toolCalls: [{ |
| id: 'tc1', |
| type: 'function' as const, |
| function: { name: 'bash', arguments: '{"command":"status --task \\"x\\" --done \\"x\\" --remaining \\"none\\" --complete"}' }, |
| }], |
| }; |
| }), |
| getModel: () => 'test-model', |
| getProvider: () => 'test-provider', |
| supportsTools: () => true, |
| }; |
|
|
| const onPausableError = vi.fn(async () => 'continue' as const); |
|
|
| const toolResult: ToolResult = { |
| tool_call_id: 'tc1', |
| content: 'Status recorded.', |
| success: true, |
| signals: { statusComplete: true, statusResult: { task: 'x', done: 'x', remaining: 'none', complete: true, hasExplicitFlag: true } }, |
| }; |
| const results = new Map([['tc1', toolResult]]); |
| const executor = createMockExecutor(results); |
| const context = createMockContext(); |
| const progress = createMockProgress(); |
| const cost = createMockCost(); |
| const config = defaultConfig({ onPausableError }); |
|
|
| const loop = new AgentLoop({ |
| config, |
| provider, |
| executor, |
| context, |
| progress, |
| cost, |
| }); |
|
|
| const result = await loop.run('Do it'); |
| expect(onPausableError).toHaveBeenCalledTimes(1); |
| expect(result.success).toBe(true); |
| }); |
|
|
| it('explore agent exits immediately without needing status', async () => { |
| const provider = createMockProvider([ |
| { content: 'Here is the analysis of the codebase...' }, |
| ]); |
|
|
| const context = createMockContext(); |
| const progress = createMockProgress(); |
| const cost = createMockCost(); |
| const config = defaultConfig({ agentType: 'explore' }); |
|
|
| const loop = new AgentLoop({ |
| config, |
| provider, |
| executor: createMockExecutor(), |
| context, |
| progress, |
| cost, |
| }); |
|
|
| const result = await loop.run('Analyze this'); |
| expect(result.success).toBe(true); |
| expect(result.turnCount).toBe(1); |
| |
| const events = (progress.onEvent as ReturnType<typeof vi.fn>).mock.calls.map(c => c[0]); |
| expect(events).not.toContain('nudge'); |
| }); |
|
|
| it('handles onPausableError returning stop', async () => { |
| const provider: ProviderAdapter = { |
| call: vi.fn(async () => { |
| throw new Error('Fatal error'); |
| }), |
| getModel: () => 'test-model', |
| getProvider: () => 'test-provider', |
| supportsTools: () => true, |
| }; |
|
|
| const onPausableError = vi.fn(async () => 'stop' as const); |
| const context = createMockContext(); |
| const progress = createMockProgress(); |
| const cost = createMockCost(); |
| const config = defaultConfig({ onPausableError }); |
|
|
| const loop = new AgentLoop({ |
| config, |
| provider, |
| executor: createMockExecutor(), |
| context, |
| progress, |
| cost, |
| }); |
|
|
| const result = await loop.run('Do it'); |
| expect(onPausableError).toHaveBeenCalledTimes(1); |
| expect(result.success).toBe(false); |
| expect(result.summary).toContain('error'); |
| }); |
|
|
| it('extracts tool calls from text for models without native tool support', async () => { |
| const provider: ProviderAdapter = { |
| call: vi.fn(async () => ({ |
| content: '```bash\nls -la\n```', |
| toolCalls: undefined, |
| })), |
| getModel: () => 'test-model', |
| getProvider: () => 'test-provider', |
| supportsTools: () => false, |
| }; |
|
|
| |
| const executor: ToolExecutor = { |
| execute: vi.fn(async (toolCall: ToolCall) => ({ |
| tool_call_id: toolCall.id, |
| content: 'files listed', |
| success: true, |
| signals: { statusComplete: true, statusResult: { task: 'x', done: 'x', remaining: 'none', complete: true, hasExplicitFlag: true } }, |
| })), |
| getDefinitions: () => [{ name: 'bash', description: 'Run shell', parameters: {} }], |
| }; |
|
|
| const context = createMockContext(); |
| const progress = createMockProgress(); |
| const cost = createMockCost(); |
| const config = defaultConfig(); |
|
|
| const loop = new AgentLoop({ |
| config, |
| provider, |
| executor, |
| context, |
| progress, |
| cost, |
| }); |
|
|
| const result = await loop.run('List files'); |
| expect(executor.execute).toHaveBeenCalled(); |
| expect(result.toolCount).toBe(1); |
| }); |
|
|
| it('stops when stop() is called', async () => { |
| let callCount = 0; |
| const provider: ProviderAdapter = { |
| call: vi.fn(async () => { |
| callCount++; |
| return { |
| content: '', |
| toolCalls: [{ |
| id: `tc${callCount}`, |
| type: 'function' as const, |
| function: { name: 'bash', arguments: `{"command":"echo ${callCount}"}` }, |
| }], |
| }; |
| }), |
| getModel: () => 'test-model', |
| getProvider: () => 'test-provider', |
| supportsTools: () => true, |
| }; |
|
|
| const context = createMockContext(); |
| const progress = createMockProgress(); |
| const cost = createMockCost(); |
| const config = defaultConfig(); |
|
|
| const loop = new AgentLoop({ |
| config, |
| provider, |
| executor: createMockExecutor(), |
| context, |
| progress, |
| cost, |
| }); |
|
|
| |
| (provider.call as ReturnType<typeof vi.fn>).mockImplementation(async () => { |
| if (callCount > 0) loop.stop(); |
| callCount++; |
| return { |
| content: '', |
| toolCalls: [{ |
| id: `tc${callCount}`, |
| type: 'function' as const, |
| function: { name: 'bash', arguments: `{"command":"echo ${callCount}"}` }, |
| }], |
| }; |
| }); |
|
|
| const result = await loop.run('Do something'); |
| expect(result.success).toBe(false); |
| expect(result.summary).toContain('Stopped'); |
| }); |
|
|
| it('triggers compaction using token estimate when provider reports no usage', async () => { |
| const toolCall: ToolCall = { |
| id: 'tc1', |
| type: 'function', |
| function: { name: 'bash', arguments: '{"command":"status --task \\"t\\" --done \\"d\\" --remaining \\"none\\" --complete"}' }, |
| }; |
| |
| const provider = createMockProvider([ |
| { content: '', toolCalls: [toolCall] }, |
| { content: 'Done.' }, |
| ]); |
|
|
| const context = createMockContext(); |
| |
| context.getTokenEstimate = () => 999999; |
| context.needsCompaction = (tokenCount: number) => tokenCount >= 80000; |
|
|
| const toolResult: ToolResult = { |
| tool_call_id: 'tc1', |
| content: 'Status recorded.', |
| success: true, |
| signals: { statusComplete: true, statusResult: { task: 't', done: 'd', remaining: 'none', complete: true, hasExplicitFlag: true } }, |
| }; |
| const executor = createMockExecutor(new Map([['tc1', toolResult]])); |
| const progress = createMockProgress(); |
| |
| const cost = createMockCost(); |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig(), |
| provider, |
| executor, |
| context, |
| progress, |
| cost, |
| }); |
|
|
| await loop.run('Build something'); |
| expect(context.compact).toHaveBeenCalled(); |
| }); |
|
|
| it('uses reported promptTokens for compaction check when available', async () => { |
| const toolCall: ToolCall = { |
| id: 'tc1', |
| type: 'function', |
| function: { name: 'bash', arguments: '{"command":"status --task \\"t\\" --done \\"d\\" --remaining \\"none\\" --complete"}' }, |
| }; |
| const provider = createMockProvider([ |
| { content: '', toolCalls: [toolCall], usage: { promptTokens: 90000, completionTokens: 500, totalTokens: 90500 } }, |
| { content: 'Done.' }, |
| ]); |
|
|
| const context = createMockContext(); |
| context.getTokenEstimate = () => 5000; |
| context.needsCompaction = (tokenCount: number) => tokenCount >= 80000; |
|
|
| const toolResult: ToolResult = { |
| tool_call_id: 'tc1', |
| content: 'ok', |
| success: true, |
| signals: { statusComplete: true, statusResult: { task: 't', done: 'd', remaining: 'none', complete: true, hasExplicitFlag: true } }, |
| }; |
| const executor = createMockExecutor(new Map([['tc1', toolResult]])); |
| const progress = createMockProgress(); |
| const cost = createMockCost(); |
| cost.getTotalUsage = () => ({ promptTokens: 90000, completionTokens: 500, totalTokens: 90500 }); |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig(), |
| provider, |
| executor, |
| context, |
| progress, |
| cost, |
| }); |
|
|
| await loop.run('Build something'); |
| expect(context.compact).toHaveBeenCalled(); |
| }); |
|
|
| it('does not compact when both promptTokens and estimate are below threshold', async () => { |
| const toolCall: ToolCall = { |
| id: 'tc1', |
| type: 'function', |
| function: { name: 'bash', arguments: '{"command":"status --task \\"t\\" --done \\"d\\" --remaining \\"none\\" --complete"}' }, |
| }; |
| const provider = createMockProvider([ |
| { content: '', toolCalls: [toolCall] }, |
| { content: 'Done.' }, |
| ]); |
|
|
| const context = createMockContext(); |
| context.getTokenEstimate = () => 5000; |
| context.needsCompaction = (tokenCount: number) => tokenCount >= 80000; |
|
|
| const toolResult: ToolResult = { |
| tool_call_id: 'tc1', |
| content: 'ok', |
| success: true, |
| signals: { statusComplete: true, statusResult: { task: 't', done: 'd', remaining: 'none', complete: true, hasExplicitFlag: true } }, |
| }; |
| const executor = createMockExecutor(new Map([['tc1', toolResult]])); |
| const progress = createMockProgress(); |
| const cost = createMockCost(); |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig(), |
| provider, |
| executor, |
| context, |
| progress, |
| cost, |
| }); |
|
|
| await loop.run('Build something'); |
| expect(context.compact).not.toHaveBeenCalled(); |
| }); |
| }); |
|
|
| describe('AgentLoop history integrity and retry safeguards', () => { |
| const statusToolCall = (id: string): ToolCall => ({ |
| id, |
| type: 'function', |
| function: { name: 'bash', arguments: '{"command":"status --task \\"t\\" --done \\"d\\" --remaining \\"none\\" --complete"}' }, |
| }); |
| const statusResultFor = (id: string): ToolResult => ({ |
| tool_call_id: id, |
| content: 'ok', |
| success: true, |
| signals: { statusComplete: true, statusResult: { task: 't', done: 'd', remaining: 'none', complete: true, hasExplicitFlag: true } }, |
| }); |
|
|
| it('sends sanitized messages (orphan repair) to the provider', async () => { |
| const sanitized: Message[] = [{ role: 'user', content: 'SANITIZED' }]; |
| const context = createMockContext(); |
| context.getSanitizedMessages = () => sanitized; |
| const provider = createMockProvider([{ content: 'findings' }]); |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig({ agentType: 'explore' }), |
| provider, |
| executor: createMockExecutor(), |
| context, |
| progress: createMockProgress(), |
| cost: createMockCost(), |
| }); |
| await loop.run('hi'); |
|
|
| const sentMessages = (provider.call as ReturnType<typeof vi.fn>).mock.calls[0][0].messages; |
| expect(sentMessages).toBe(sanitized); |
| }); |
|
|
| it('uses its own last response promptTokens for compaction, not the shared cost tracker', async () => { |
| |
| |
| const provider = createMockProvider([ |
| { content: '', toolCalls: [statusToolCall('tc1')], usage: { promptTokens: 90000, completionTokens: 500, totalTokens: 90500 } }, |
| { content: 'Done.' }, |
| ]); |
| const context = createMockContext(); |
| context.getTokenEstimate = () => 5000; |
| const needsCompaction = vi.fn((tokenCount: number) => tokenCount >= 80000); |
| context.needsCompaction = needsCompaction; |
|
|
| const cost = createMockCost(); |
| cost.getTotalUsage = () => ({ promptTokens: 50, completionTokens: 0, totalTokens: 50 }); |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig(), |
| provider, |
| executor: createMockExecutor(new Map([['tc1', statusResultFor('tc1')]])), |
| context, |
| progress: createMockProgress(), |
| cost, |
| }); |
| await loop.run('Build something'); |
|
|
| expect(needsCompaction).toHaveBeenCalledWith(90000); |
| expect(context.compact).toHaveBeenCalled(); |
| }); |
|
|
| it('commits executed tool results to context before terminating on loop detection', async () => { |
| const dup = (id: string): ToolCall => ({ |
| id, |
| type: 'function', |
| function: { name: 'bash', arguments: '{"command":"ls"}' }, |
| }); |
| |
| |
| const provider = createMockProvider([ |
| { content: '', toolCalls: [dup('a'), dup('b'), dup('c'), dup('d')] }, |
| ]); |
| const context = createMockContext(); |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig({ maxDuplicateToolCalls: 3 }), |
| provider, |
| executor: createMockExecutor(), |
| context, |
| progress: createMockProgress(), |
| cost: createMockCost(), |
| }); |
| const result = await loop.run('Do something'); |
|
|
| expect(result.summary).toContain('loop'); |
| |
| expect(context.addAssistantTurn).toHaveBeenCalledTimes(1); |
| expect(context.addToolResults).toHaveBeenCalled(); |
| const committed = (context.addToolResults as ReturnType<typeof vi.fn>).mock.calls[0][0] as ToolResult[]; |
| expect(committed.some(r => r.tool_call_id === 'a' && r.success)).toBe(true); |
| }); |
|
|
| it('passes a per-turn turnId to the tool executor context', async () => { |
| const call = (id: string, cmd: string): ToolCall => ({ |
| id, |
| type: 'function', |
| function: { name: 'bash', arguments: `{"command":"${cmd}"}` }, |
| }); |
| const provider = createMockProvider([ |
| { content: '', toolCalls: [call('t1', 'ls')] }, |
| { content: '', toolCalls: [call('t2', 'status --task \\"t\\" --done \\"d\\" --remaining \\"none\\" --complete')] }, |
| { content: 'Done.' }, |
| ]); |
| const turnIds: Array<number | undefined> = []; |
| const executor: ToolExecutor = { |
| execute: vi.fn(async (toolCall: ToolCall, ctx) => { |
| turnIds.push(ctx.turnId); |
| if (toolCall.id === 't2') return statusResultFor('t2'); |
| return { tool_call_id: toolCall.id, content: 'OK', success: true }; |
| }), |
| getDefinitions: () => [{ name: 'bash', description: 'Run shell', parameters: {} }], |
| }; |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig(), |
| provider, |
| executor, |
| context: createMockContext(), |
| progress: createMockProgress(), |
| cost: createMockCost(), |
| }); |
| await loop.run('Do something'); |
|
|
| expect(turnIds).toEqual([1, 2]); |
| }); |
|
|
| it('passes its abort signal to compact()', async () => { |
| const provider = createMockProvider([ |
| { content: '', toolCalls: [statusToolCall('tc1')], usage: { promptTokens: 90000, completionTokens: 1, totalTokens: 90001 } }, |
| { content: 'Done.' }, |
| ]); |
| const context = createMockContext(); |
| context.needsCompaction = (tokenCount: number) => tokenCount >= 80000; |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig(), |
| provider, |
| executor: createMockExecutor(new Map([['tc1', statusResultFor('tc1')]])), |
| context, |
| progress: createMockProgress(), |
| cost: createMockCost(), |
| }); |
| await loop.run('Build something'); |
|
|
| const compactOpts = (context.compact as ReturnType<typeof vi.fn>).mock.calls[0][1]; |
| expect(compactOpts?.signal).toBeInstanceOf(AbortSignal); |
| }); |
|
|
| it('persists reasoning-only turns and injects a targeted retry instead of the status nudge', async () => { |
| const reasoningOnly = { |
| content: '', |
| reasoningDetails: [{ type: 'thinking', text: 'I should check index.html first.' }], |
| }; |
| const provider = createMockProvider([ |
| reasoningOnly, |
| { content: '', toolCalls: [statusToolCall('tc1')] }, |
| { content: 'Done.' }, |
| ]); |
| const context = createMockContext(); |
| const progress = createMockProgress(); |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig(), |
| provider, |
| executor: createMockExecutor(new Map([['tc1', statusResultFor('tc1')]])), |
| context, |
| progress, |
| cost: createMockCost(), |
| }); |
| const result = await loop.run('Build a landing page'); |
|
|
| |
| const assistantTurns = (context.addAssistantTurn as ReturnType<typeof vi.fn>).mock.calls.map(c => c[0]); |
| expect(assistantTurns.some(r => r.reasoningDetails?.length && !r.content)).toBe(true); |
|
|
| |
| const userMessages = (context.addUserMessage as ReturnType<typeof vi.fn>).mock.calls.map(c => c[0]); |
| const retryMsg = userMessages.find((m: string) => typeof m === 'string' && m.includes('only reasoning')); |
| expect(retryMsg).toBeDefined(); |
| expect(userMessages.some((m: string) => typeof m === 'string' && m.includes('status command'))).toBe(false); |
|
|
| expect((progress.onEvent as ReturnType<typeof vi.fn>).mock.calls.some(c => c[0] === 'reasoning_only_retry')).toBe(true); |
| expect(result.success).toBe(true); |
| }); |
|
|
| it('falls back to status nudges after reasoning-only retries are exhausted', async () => { |
| const reasoningOnly = () => ({ |
| content: '', |
| reasoningDetails: [{ type: 'thinking', text: 'Thinking again from scratch.' }], |
| }); |
| const provider = createMockProvider([ |
| reasoningOnly(), reasoningOnly(), reasoningOnly(), reasoningOnly(), reasoningOnly(), |
| ]); |
| const progress = createMockProgress(); |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig({ maxNudges: 1 }), |
| provider, |
| executor: createMockExecutor(), |
| context: createMockContext(), |
| progress, |
| cost: createMockCost(), |
| }); |
| const result = await loop.run('Build a landing page'); |
|
|
| const retryEvents = (progress.onEvent as ReturnType<typeof vi.fn>).mock.calls.filter(c => c[0] === 'reasoning_only_retry'); |
| expect(retryEvents).toHaveLength(2); |
| expect(result.success).toBe(false); |
| expect(result.exitReason).toBe('nudge_exhaustion'); |
| }); |
|
|
| it('keeps reasoning details on assistant turns that have content', async () => { |
| const provider = createMockProvider([ |
| { content: 'Here is my answer.', reasoningDetails: [{ type: 'thinking', text: 'Considered options.' }] }, |
| { content: '', toolCalls: [statusToolCall('tc1')] }, |
| { content: 'Done.' }, |
| ]); |
| const context = createMockContext(); |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig(), |
| provider, |
| executor: createMockExecutor(new Map([['tc1', statusResultFor('tc1')]])), |
| context, |
| progress: createMockProgress(), |
| cost: createMockCost(), |
| }); |
| await loop.run('Question'); |
|
|
| const assistantTurns = (context.addAssistantTurn as ReturnType<typeof vi.fn>).mock.calls.map(c => c[0]); |
| const withContent = assistantTurns.find(r => r.content === 'Here is my answer.'); |
| expect(withContent?.reasoningDetails?.length).toBe(1); |
| }); |
|
|
| it('wraps synthetic harness messages in automated_reminder tags, but never the real user prompt', async () => { |
| |
| const provider = createMockProvider([ |
| { content: '', reasoningDetails: [{ type: 'thinking', text: 'hmm' }] }, |
| { content: 'text without status' }, |
| { content: 'more text' }, |
| ]); |
| const context = createMockContext(); |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig({ maxNudges: 1 }), |
| provider, |
| executor: createMockExecutor(), |
| context, |
| progress: createMockProgress(), |
| cost: createMockCost(), |
| }); |
| await loop.run('Build a landing page'); |
|
|
| const userMessages = (context.addUserMessage as ReturnType<typeof vi.fn>).mock.calls |
| .map(c => c[0]) |
| .filter((m): m is string => typeof m === 'string'); |
|
|
| |
| expect(userMessages[0]).toBe('Build a landing page'); |
| |
| const synthetic = userMessages.slice(1); |
| expect(synthetic.length).toBeGreaterThan(0); |
| for (const msg of synthetic) { |
| expect(msg.startsWith('<automated_reminder>')).toBe(true); |
| expect(msg.endsWith('</automated_reminder>')).toBe(true); |
| } |
| }); |
|
|
| it('replaces large repeated tool outputs with a dedup marker', async () => { |
| const bigContent = 'SKILL FILE CONTENT '.repeat(50); |
| const call = (id: string, cmd: string): ToolCall => ({ |
| id, |
| type: 'function', |
| function: { name: 'bash', arguments: JSON.stringify({ command: cmd }) }, |
| }); |
| const provider = createMockProvider([ |
| { content: '', toolCalls: [call('t1', 'cat /skill.md')] }, |
| { content: '', toolCalls: [call('t2', 'head -n 999 /skill.md')] }, |
| { content: '', toolCalls: [call('t3', 'status --task "t" --done "d" --remaining "none" --complete')] }, |
| { content: 'Done.' }, |
| ]); |
| const executor: ToolExecutor = { |
| execute: vi.fn(async (toolCall: ToolCall) => { |
| if (toolCall.id === 't3') return statusResultFor('t3'); |
| return { tool_call_id: toolCall.id, content: bigContent, success: true }; |
| }), |
| getDefinitions: () => [{ name: 'bash', description: 'Run shell', parameters: {} }], |
| }; |
| const context = createMockContext(); |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig(), |
| provider, |
| executor, |
| context, |
| progress: createMockProgress(), |
| cost: createMockCost(), |
| }); |
| await loop.run('Do something'); |
|
|
| const toolBatches = (context.addToolResults as ReturnType<typeof vi.fn>).mock.calls.map(c => c[0] as ToolResult[]); |
| |
| expect(toolBatches[0][0].content).toBe(bigContent); |
| |
| expect(toolBatches[1][0].content).not.toBe(bigContent); |
| expect(toolBatches[1][0].content).toContain('identical'); |
| expect(toolBatches[1][0].success).toBe(true); |
| }); |
|
|
| it('does not dedup small repeated tool outputs', async () => { |
| const call = (id: string, cmd: string): ToolCall => ({ |
| id, |
| type: 'function', |
| function: { name: 'bash', arguments: JSON.stringify({ command: cmd }) }, |
| }); |
| const provider = createMockProvider([ |
| { content: '', toolCalls: [call('t1', 'mkdir /a')] }, |
| { content: '', toolCalls: [call('t2', 'mkdir /b')] }, |
| { content: '', toolCalls: [call('t3', 'status --task "t" --done "d" --remaining "none" --complete')] }, |
| { content: 'Done.' }, |
| ]); |
| const executor: ToolExecutor = { |
| execute: vi.fn(async (toolCall: ToolCall) => { |
| if (toolCall.id === 't3') return statusResultFor('t3'); |
| return { tool_call_id: toolCall.id, content: 'Command succeeded with no output', success: true }; |
| }), |
| getDefinitions: () => [{ name: 'bash', description: 'Run shell', parameters: {} }], |
| }; |
| const context = createMockContext(); |
|
|
| const loop = new AgentLoop({ |
| config: defaultConfig(), |
| provider, |
| executor, |
| context, |
| progress: createMockProgress(), |
| cost: createMockCost(), |
| }); |
| await loop.run('Do something'); |
|
|
| const toolBatches = (context.addToolResults as ReturnType<typeof vi.fn>).mock.calls.map(c => c[0] as ToolResult[]); |
| expect(toolBatches[1][0].content).toBe('Command succeeded with no output'); |
| }); |
|
|
| it('reports the exit reason in the loop result', async () => { |
| const provider = createMockProvider([ |
| { content: 'text only' }, |
| { content: 'more text' }, |
| { content: 'final text' }, |
| ]); |
| const loop = new AgentLoop({ |
| config: defaultConfig({ maxNudges: 2 }), |
| provider, |
| executor: createMockExecutor(), |
| context: createMockContext(), |
| progress: createMockProgress(), |
| cost: createMockCost(), |
| }); |
| const result = await loop.run('Do something'); |
| expect(result.exitReason).toBe('nudge_exhaustion'); |
| }); |
| }); |
|
|