scripts/cli-agent-sim.js

#!/usr/bin/env node
/**
 * CLI Agent Simulator — mimics Claude Code behavior against WindsurfAPI.
 *
 * Sends multi-turn conversations with tool calls through the proxy,
 * verifying streaming, tool emulation, context retention, and response quality.
 * Generates structured JSONL logs for demonstration.
 *
 * Usage:
 *   node scripts/cli-agent-sim.js [--base-url http://localhost:3003] [--api-key sk-xxx] [--model claude-4.5-sonnet]
 */

import https from 'https';
import http from 'http';
import { writeFileSync, mkdirSync, appendFileSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';

const __dirname = dirname(fileURLToPath(import.meta.url));
const LOG_DIR = join(__dirname, '..', 'logs', 'agent-sim');
mkdirSync(LOG_DIR, { recursive: true });

const args = process.argv.slice(2);
function getArg(name, fallback) {
  const idx = args.indexOf(`--${name}`);
  return idx !== -1 && args[idx + 1] ? args[idx + 1] : fallback;
}

const BASE_URL = getArg('base-url', 'http://localhost:3003');
const API_KEY = getArg('api-key', 'sk-test-local-dev');
const MODEL = getArg('model', 'gpt-4o-mini');
const LOG_FILE = join(LOG_DIR, `sim-${new Date().toISOString().replace(/[:.]/g, '-')}.jsonl`);

function logEntry(entry) {
  const line = JSON.stringify({ ts: new Date().toISOString(), ...entry });
  appendFileSync(LOG_FILE, line + '\n');
  const icon = entry.type === 'error' ? '✗' : entry.type === 'response' ? '✓' : '→';
  console.log(`  ${icon} [${entry.type}] ${entry.summary || ''}`);
}

async function chatCompletion(messages, opts = {}) {
  const body = {
    model: MODEL,
    messages,
    stream: opts.stream ?? true,
    ...(opts.tools ? { tools: opts.tools } : {}),
  };

  const url = new URL('/v1/chat/completions', BASE_URL);
  const isHttps = url.protocol === 'https:';
  const mod = isHttps ? https : http;

  return new Promise((resolve, reject) => {
    const req = mod.request(url, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${API_KEY}`,
      },
    }, (res) => {
      if (!opts.stream) {
        const chunks = [];
        res.on('data', c => chunks.push(c));
        res.on('end', () => {
          try {
            const data = JSON.parse(Buffer.concat(chunks).toString());
            resolve({ status: res.statusCode, data, headers: res.headers });
          } catch (e) { reject(e); }
        });
        return;
      }

      let text = '';
      let thinking = '';
      let toolCalls = [];
      let usage = null;
      let finishReason = null;
      let lineBuf = '';

      function processLine(line) {
        if (!line.startsWith('data: ')) return;
        const payload = line.slice(6);
        if (payload === '[DONE]') return;
        try {
          const parsed = JSON.parse(payload);
          const delta = parsed.choices?.[0]?.delta;
          if (delta?.content) text += delta.content;
          if (delta?.reasoning_content) thinking += delta.reasoning_content;
          if (delta?.tool_calls) {
            for (const tc of delta.tool_calls) {
              if (!toolCalls[tc.index]) toolCalls[tc.index] = { id: '', name: '', arguments: '' };
              if (tc.id) toolCalls[tc.index].id = tc.id;
              if (tc.function?.name) toolCalls[tc.index].name = tc.function.name;
              if (tc.function?.arguments) toolCalls[tc.index].arguments += tc.function.arguments;
            }
          }
          if (parsed.choices?.[0]?.finish_reason) finishReason = parsed.choices[0].finish_reason;
          if (parsed.usage) usage = parsed.usage;
        } catch {}
      }

      res.on('data', (chunk) => {
        lineBuf += chunk.toString();
        const lines = lineBuf.split('\n');
        lineBuf = lines.pop();
        for (const line of lines) {
          const trimmed = line.trim();
          if (trimmed) processLine(trimmed);
        }
      });

      res.on('end', () => {
        for (const line of lineBuf.split('\n')) {
          const trimmed = line.trim();
          if (trimmed) processLine(trimmed);
        }
        resolve({
          status: res.statusCode,
          text,
          thinking: thinking || null,
          toolCalls: toolCalls.filter(Boolean),
          finishReason,
          usage,
          headers: res.headers,
        });
      });

      res.on('error', (e) => {
        reject(Object.assign(e, { _source: 'res.error' }));
      });
    });
    req.on('error', (e) => {
      reject(Object.assign(e, { _source: 'req.error' }));
    });
    req.write(JSON.stringify(body));
    req.end();
  });
}

// ── Test Scenarios ──────────────────────────────────

async function testBasicChat() {
  logEntry({ type: 'test', summary: 'Basic chat — single turn' });
  const start = Date.now();
  const resp = await chatCompletion([
    { role: 'user', content: 'Reply with exactly: "Hello from WindsurfAPI". Nothing else.' },
  ], { stream: false });
  const elapsed = Date.now() - start;
  const text = resp.data?.choices?.[0]?.message?.content || resp.text || '';
  logEntry({
    type: 'response', summary: `${text.slice(0, 80)} (${elapsed}ms)`,
    model: MODEL, elapsed, textLen: text.length,
    usage: resp.data?.usage || resp.usage,
  });
  return { ...resp, text };
}

async function testMultiTurn() {
  logEntry({ type: 'test', summary: 'Multi-turn context retention' });

  const messages = [
    { role: 'user', content: 'My name is TestUser42. Remember this.' },
  ];

  const r1 = await chatCompletion(messages, { stream: false });
  const r1text = r1.data?.choices?.[0]?.message?.content || '';
  logEntry({ type: 'response', summary: `Turn 1: ${r1text.slice(0, 60)}` });

  messages.push({ role: 'assistant', content: r1text });
  messages.push({ role: 'user', content: 'What is my name? Reply with just the name.' });

  const r2 = await chatCompletion(messages, { stream: false });
  const r2text = r2.data?.choices?.[0]?.message?.content || '';
  const remembered = r2text.includes('TestUser42');
  logEntry({
    type: 'response',
    summary: `Turn 2: ${r2text.slice(0, 60)} — context ${remembered ? 'RETAINED' : 'LOST'}`,
    contextRetained: remembered,
  });
  return remembered;
}

async function testToolCalls() {
  logEntry({ type: 'test', summary: 'Tool call emulation' });

  const tools = [{
    type: 'function',
    function: {
      name: 'read_file',
      description: 'Read the contents of a file at the given path',
      parameters: {
        type: 'object',
        properties: {
          path: { type: 'string', description: 'File path to read' },
        },
        required: ['path'],
      },
    },
  }];

  const resp = await chatCompletion([
    { role: 'user', content: 'Read the file at ./package.json using the read_file tool.' },
  ], { tools, stream: false });

  const msg = resp.data?.choices?.[0]?.message;
  const tcs = msg?.tool_calls || [];
  const hasToolCall = tcs.length > 0 || resp.data?.choices?.[0]?.finish_reason === 'tool_calls';
  logEntry({
    type: 'response',
    summary: `Tool calls: ${tcs.length}, finish=${resp.data?.choices?.[0]?.finish_reason}`,
    toolCalls: tcs.map(tc => ({ name: tc.function?.name, args: tc.function?.arguments?.slice(0, 100) })),
    hasToolCall,
  });
  return hasToolCall;
}

async function testNonStream() {
  logEntry({ type: 'test', summary: 'Non-streaming response' });
  const start = Date.now();
  const resp = await chatCompletion([
    { role: 'user', content: 'Say "non-stream OK" in 3 words or less.' },
  ], { stream: false });
  const elapsed = Date.now() - start;
  const text = resp.data?.choices?.[0]?.message?.content || '';
  logEntry({
    type: 'response', summary: `${text.slice(0, 60)} (${elapsed}ms)`,
    elapsed, status: resp.status,
  });
  return resp.status === 200;
}

async function testAnthropicProtocol() {
  logEntry({ type: 'test', summary: 'Anthropic /v1/messages protocol' });

  const url = new URL('/v1/messages', BASE_URL);
  const mod = url.protocol === 'https:' ? https : http;

  const body = {
    model: MODEL,
    max_tokens: 100,
    messages: [{ role: 'user', content: 'Say "anthropic OK".' }],
  };

  return new Promise((resolve) => {
    const req = mod.request(url, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'x-api-key': API_KEY,
        'anthropic-version': '2023-06-01',
      },
    }, (res) => {
      const chunks = [];
      res.on('data', c => chunks.push(c));
      res.on('end', () => {
        try {
          const data = JSON.parse(Buffer.concat(chunks).toString());
          const text = data.content?.[0]?.text || data.error?.message || '';
          logEntry({
            type: 'response',
            summary: `Anthropic: ${text.slice(0, 60)} (status=${res.statusCode})`,
            status: res.statusCode,
          });
          resolve(res.statusCode === 200);
        } catch (e) {
          logEntry({ type: 'error', summary: `Anthropic parse error: ${e.message}` });
          resolve(false);
        }
      });
    });
    req.on('error', (e) => {
      logEntry({ type: 'error', summary: `Anthropic: ${e.message}` });
      resolve(false);
    });
    req.write(JSON.stringify(body));
    req.end();
  });
}

async function testProjectWriting() {
  logEntry({ type: 'test', summary: 'Project writing simulation — ask model to write a Python script' });

  const resp = await chatCompletion([
    { role: 'system', content: 'You are a senior Python developer. Write clean, production-ready code.' },
    { role: 'user', content: 'Write a Python function called `fibonacci(n)` that returns the nth Fibonacci number using memoization. Include type hints and a docstring. Output ONLY the code, no explanation.' },
  ], { stream: false });

  const text = resp.data?.choices?.[0]?.message?.content || '';
  const hasPython = text.includes('def fibonacci') || text.includes('fibonacci');
  logEntry({
    type: 'response',
    summary: `Code gen: ${text.length} chars, has_fibonacci=${hasPython}`,
    textLen: text.length,
    hasPython,
    codePreview: text.slice(0, 200),
  });
  return hasPython;
}

// ── Main ──────────────────────────────────

async function main() {
  console.log(`\n  WindsurfAPI CLI Agent Simulator`);
  console.log(`  Base URL:  ${BASE_URL}`);
  console.log(`  Model:     ${MODEL}`);
  console.log(`  Log file:  ${LOG_FILE}\n`);

  // Preflight: check server health
  try {
    const healthResp = await new Promise((resolve, reject) => {
      const url = new URL('/health', BASE_URL);
      const mod = url.protocol === 'https:' ? https : http;
      mod.get(url, (res) => {
        const chunks = [];
        res.on('data', c => chunks.push(c));
        res.on('end', () => resolve(JSON.parse(Buffer.concat(chunks).toString())));
      }).on('error', reject);
    });
    console.log(`  Server: v${healthResp.version} accounts=${healthResp.accounts?.active || 0}`);
    if (!healthResp.accounts?.active) {
      console.log('  ✗ No active accounts — add accounts first via /auth/login or /dashboard');
      process.exit(1);
    }
  } catch (e) {
    console.log(`  ✗ Cannot reach server: ${e.message}`);
    process.exit(1);
  }

  logEntry({ type: 'start', summary: `Simulation start — model=${MODEL} base=${BASE_URL}` });

  const results = {};
  const tests = [
    ['basic_chat', testBasicChat],
    ['multi_turn', testMultiTurn],
    ['tool_calls', testToolCalls],
    ['non_stream', testNonStream],
    ['anthropic', testAnthropicProtocol],
    ['project_writing', testProjectWriting],
  ];

  for (const [name, fn] of tests) {
    try {
      const result = await fn();
      results[name] = result ? 'PASS' : 'FAIL';
    } catch (err) {
      logEntry({ type: 'error', summary: `${name}: ${err.message}` });
      results[name] = 'ERROR';
    }
  }

  console.log('\n  ── Results ──');
  for (const [name, status] of Object.entries(results)) {
    const icon = status === 'PASS' ? '✓' : status === 'FAIL' ? '✗' : '!';
    console.log(`  ${icon} ${name}: ${status}`);
  }

  logEntry({ type: 'end', summary: 'Simulation complete', results });
  console.log(`\n  Full log: ${LOG_FILE}\n`);
}

main().catch(err => {
  console.error('Fatal:', err.message);
  process.exit(1);
});