tests/test-multiturn-thinking-tools.cjs

/**
 * Multi-Turn Tool Call Test (Non-Streaming)
 *
 * Simulates Claude Code's actual multi-turn conversation pattern:
 * 1. User asks question requiring tool
 * 2. Assistant responds with thinking + tool_use
 * 3. User provides tool_result
 * 4. Assistant responds with thinking + final answer
 *
 * Key aspects tested:
 * - Thinking blocks with signatures are preserved across turns
 * - Tool use/result flow works correctly
 * - Interleaved thinking with tools
 *
 * Runs for both Claude and Gemini model families.
 */
const { makeRequest, analyzeContent, commonTools } = require('./helpers/http-client.cjs');
const { getTestModels, getModelConfig, familySupportsThinking } = require('./helpers/test-models.cjs');

const tools = [commonTools.searchFiles, commonTools.readFile];

async function runTestsForModel(family, model) {
    console.log('='.repeat(60));
    console.log(`MULTI-TURN TOOL CALL TEST [${family.toUpperCase()}]`);
    console.log(`Model: ${model}`);
    console.log('Simulates Claude Code conversation pattern');
    console.log('='.repeat(60));
    console.log('');

    let messages = [];
    let allPassed = true;
    const results = [];
    const modelConfig = getModelConfig(family);
    const expectThinking = familySupportsThinking(family);

    // ===== TURN 1: Initial request =====
    console.log('TURN 1: User asks to find and read a config file');
    console.log('-'.repeat(40));

    messages.push({
        role: 'user',
        content: 'Find the package.json file and tell me what dependencies it has. Use search_files first.'
    });

    const turn1 = await makeRequest({
        model,
        max_tokens: modelConfig.max_tokens,
        stream: false,
        tools,
        thinking: modelConfig.thinking,
        messages
    });

    if (turn1.statusCode !== 200 || turn1.error) {
        console.log(`  ERROR: ${turn1.error?.message || `Status ${turn1.statusCode}`}`);
        allPassed = false;
        results.push({ name: 'Turn 1: Initial request', passed: false });
    } else {
        const analysis = analyzeContent(turn1.content || []);
        console.log(`  Thinking: ${analysis.hasThinking ? 'YES' : 'NO'} (${analysis.thinking.length} blocks)`);
        console.log(`  Signature: ${analysis.hasSignature ? 'YES' : 'NO'}`);
        console.log(`  Tool Use: ${analysis.hasToolUse ? 'YES' : 'NO'} (${analysis.toolUse.length} calls)`);
        console.log(`  Text: ${analysis.hasText ? 'YES' : 'NO'}`);

        if (analysis.hasThinking && analysis.thinking[0].thinking) {
            console.log(`  Thinking: "${analysis.thinking[0].thinking.substring(0, 60)}..."`);
        }
        if (analysis.hasToolUse) {
            console.log(`  Tool: ${analysis.toolUse[0].name}(${JSON.stringify(analysis.toolUse[0].input)})`);
        }

        // For thinking models, expect signature + tool use
        // Note: Gemini doesn't always produce thinking blocks, but does put signatures on tool_use
        const passed = expectThinking
            ? (analysis.hasSignature && analysis.hasToolUse)  // Signature required, thinking optional
            : analysis.hasToolUse;
        results.push({ name: 'Turn 1: Thinking + Signature + Tool Use', passed });
        if (!passed) allPassed = false;

        // Prepare for turn 2
        if (analysis.hasToolUse) {
            messages.push({ role: 'assistant', content: turn1.content });
        }
    }

    // ===== TURN 2: Provide tool result =====
    if (messages.length >= 2) {
        console.log('\nTURN 2: Provide tool result, expect another tool call');
        console.log('-'.repeat(40));

        const lastAssistant = messages[messages.length - 1];
        const toolUseBlock = lastAssistant.content.find(b => b.type === 'tool_use');

        messages.push({
            role: 'user',
            content: [{
                type: 'tool_result',
                tool_use_id: toolUseBlock.id,
                content: 'Found files:\n- /project/package.json (root, 2.3KB, modified 2 days ago)\n- /project/packages/core/package.json (workspace, 1.1KB, modified 1 hour ago)\n- /project/packages/legacy/package.json (deprecated, 0.8KB, modified 1 year ago)\n- /project/node_modules/lodash/package.json (dependency, 3.2KB)\n\nIMPORTANT: Before proceeding, reason through which files are most relevant. Consider: Are node_modules relevant? Should deprecated packages be included? Which workspace packages matter for the user\'s question about dependencies?'
            }]
        });

        const turn2 = await makeRequest({
            model,
            max_tokens: modelConfig.max_tokens,
            stream: false,
            tools,
            thinking: modelConfig.thinking,
            messages
        });

        if (turn2.statusCode !== 200 || turn2.error) {
            console.log(`  ERROR: ${turn2.error?.message || `Status ${turn2.statusCode}`}`);
            allPassed = false;
            results.push({ name: 'Turn 2: After tool result', passed: false });
        } else {
            const analysis = analyzeContent(turn2.content || []);
            console.log(`  Thinking: ${analysis.hasThinking ? 'YES' : 'NO'} (${analysis.thinking.length} blocks)`);
            console.log(`  Signature: ${analysis.hasSignature ? 'YES' : 'NO'}`);
            console.log(`  Tool Use: ${analysis.hasToolUse ? 'YES' : 'NO'} (${analysis.toolUse.length} calls)`);
            console.log(`  Text: ${analysis.hasText ? 'YES' : 'NO'}`);

            if (analysis.hasThinking && analysis.thinking[0].thinking) {
                console.log(`  Thinking: "${analysis.thinking[0].thinking.substring(0, 60)}..."`);
            }
            if (analysis.hasToolUse) {
                console.log(`  Tool: ${analysis.toolUse[0].name}(${JSON.stringify(analysis.toolUse[0].input)})`);
            }

            // Either tool use (to read file) or text response is acceptable
            // Note: Claude may skip thinking on obvious next steps - this is valid behavior
            // We only require thinking on the first turn to verify signatures work
            const passed = analysis.hasToolUse || analysis.hasText;
            results.push({ name: 'Turn 2: Tool or Text response', passed });
            if (!passed) allPassed = false;

            if (analysis.hasToolUse) {
                messages.push({ role: 'assistant', content: turn2.content });
            }
        }
    }

    // ===== TURN 3: Final tool result and response =====
    if (messages.length >= 4) {
        const lastAssistant = messages[messages.length - 1];
        const toolUseBlocks = lastAssistant.content?.filter(b => b.type === 'tool_use') || [];

        if (toolUseBlocks.length > 0) {
            console.log('\nTURN 3: Provide file content, expect final response');
            console.log('-'.repeat(40));

            // Provide tool_result for ALL tool_use blocks (API requires this)
            const toolResults = toolUseBlocks.map((toolUseBlock, idx) => ({
                type: 'tool_result',
                tool_use_id: toolUseBlock.id,
                content: JSON.stringify({
                    name: idx === 0 ? 'my-project' : 'core-package',
                    dependencies: idx === 0
                        ? { express: '^4.18.2', cors: '^2.8.5' }
                        : { lodash: '^4.17.21' }
                }, null, 2)
            }));

            messages.push({
                role: 'user',
                content: toolResults
            });

            const turn3 = await makeRequest({
                model,
                max_tokens: modelConfig.max_tokens,
                stream: false,
                tools,
                thinking: modelConfig.thinking,
                messages
            });

            if (turn3.statusCode !== 200 || turn3.error) {
                console.log(`  ERROR: ${turn3.error?.message || `Status ${turn3.statusCode}`}`);
                allPassed = false;
                results.push({ name: 'Turn 3: Final response', passed: false });
            } else {
                            const analysis = analyzeContent(turn3.content || []);
                console.log(`  Thinking: ${analysis.hasThinking ? 'YES' : 'NO'} (${analysis.thinking.length} blocks)`);
                console.log(`  Signature: ${analysis.hasSignature ? 'YES' : 'NO'}`);
                console.log(`  Tool Use: ${analysis.hasToolUse ? 'YES' : 'NO'} (${analysis.toolUse.length} calls)`);
                console.log(`  Text: ${analysis.hasText ? 'YES' : 'NO'}`);

                if (analysis.hasText && analysis.text[0].text) {
                    console.log(`  Response: "${analysis.text[0].text.substring(0, 100)}..."`);
                }
                if (analysis.hasToolUse) {
                    console.log(`  Tool: ${analysis.toolUse[0].name}(${JSON.stringify(analysis.toolUse[0].input)})`);
                }

                // For final turn: expect text OR another tool call (model may need more info)
                const passed = analysis.hasText || analysis.hasToolUse;
                const responseType = analysis.hasText ? 'text' : (analysis.hasToolUse ? 'tool_use' : 'none');
                results.push({ name: `Turn 3: Response (${responseType})`, passed });
                if (!passed) allPassed = false;
            }
        }
    }

    // ===== Summary =====
    console.log('\n' + '='.repeat(60));
    console.log(`SUMMARY [${family.toUpperCase()}]`);
    console.log('='.repeat(60));

    for (const result of results) {
        const status = result.passed ? 'PASS' : 'FAIL';
        console.log(`  [${status}] ${result.name}`);
    }

    console.log('\n' + '='.repeat(60));
    console.log(`[${family.toUpperCase()}] ${allPassed ? 'ALL TESTS PASSED' : 'SOME TESTS FAILED'}`);
    console.log('='.repeat(60));

    return allPassed;
}

async function runTests() {
    const models = await getTestModels();
    let allPassed = true;

    for (const { family, model } of models) {
        console.log('\n');
        const passed = await runTestsForModel(family, model);
        if (!passed) allPassed = false;
    }

    console.log('\n' + '='.repeat(60));
    console.log('FINAL RESULT');
    console.log('='.repeat(60));
    console.log(`Overall: ${allPassed ? 'ALL MODEL FAMILIES PASSED' : 'SOME MODEL FAMILIES FAILED'}`);
    console.log('='.repeat(60));

    process.exit(allPassed ? 0 : 1);
}

runTests().catch(err => {
    console.error('Test failed with error:', err);
    process.exit(1);
});