import { callNVIDIA, loadJSON, callToolViaMcp } from './task-support.mjs'; let task = null; export function taskStatus() { if (!task) return { exists: false }; return { running: task.running, paused: !!task.paused, question: task.question || null, instruction: task.instruction, plan: task.plan || [], step: task.step, maxSteps: task.maxSteps, events: task.events.slice(-14), needsHelp: task.needsHelp || null, result: task.result || null }; } export function stopTask(reason = 'user requested stop') { if (!task) return { stopped: false }; task.running = false; task.result = { outcome: 'stopped', reason }; return { stopped: true }; } function tLog(msg) { if (!task) return; task.events.push({ t: Date.now(), msg }); if (task.events.length > 120) task.events.shift(); } const CAPTCHA_SIGNS = ['captcha', 'verify you are human', 'prove you are human', 'cf-challenge', 'g-recaptcha', 'h-captcha', 'are-you-a-robot']; function looksLikeCaptcha(snapshotText) { const hay = (snapshotText || '').toLowerCase(); return CAPTCHA_SIGNS.some(s => hay.includes(s)); } export async function startTask(instruction, maxSteps = 18, answers = null, resumeData = null) { if (task && task.running) return { error: 'a task is already running', current: task.instruction }; const fresh = !resumeData; task = fresh ? { running: true, instruction, step: 0, maxSteps, events: [], transcript: [], plan: [], answers: {}, paused: null } : Object.assign(resumeData.task, { running: true, paused: null }); if (answers && task.pendingQuestion) { task.answers[task.pendingQuestion.field || 'q' + task.step] = answers.answer || ''; task.transcript.push({ role: 'user', text: '[answered] ' + (answers.answer || '') }); } runTaskLoop().catch(e => { tLog('fatal: ' + e.message.slice(0, 100)); task.running = false; task.needsHelp = e.message.slice(0, 140); }); return { started: fresh ? 'new' : 'resumed', instruction: task.instruction }; } export async function resumeTask(answer) { if (!task) return { error: 'no task' }; if (task.paused && task.paused.kind === 'question') { task.answers[task.paused.field] = answer; task.transcript.push({ role: 'user', text: '[answer] ' + answer }); task.paused = null; task.running = true; tLog(`human answered: ${String(answer).slice(0, 60)}`); runTaskLoop().catch(e => { tLog('fatal: ' + e.message); task.running = false; }); return { resumed: true }; } if (task.paused && task.paused.kind === 'captcha') { task.paused = null; task.running = true; tLog('human handled captcha - resuming'); runTaskLoop().catch(e => { tLog('fatal: ' + e.message); task.running = false; }); return { resumed: true }; } return { error: 'nothing paused' }; } async function pauseFor(kind, question, field) { task.paused = { kind, question: question || '', field: field || '' }; task.running = false; tLog(kind === 'captcha' ? 'CAPTCHA detected - human required' : `asking human: ${String(question).slice(0, 80)}`); } async function mcp(tool, args) { const { callTool } = await import('./mcp-manager.mjs'); const r = await callTool('ChromeDevTools', tool, args, 60000); return r.text || ''; } async function visionClick(targetDescription) { const { nativeScreenshot } = await import('./agent-browser.mjs'); const { visionDescribe } = await import('./scanner.mjs'); const { callTool } = await import('./mcp-manager.mjs'); const settings = loadJSON('settings'); const b64 = await nativeScreenshot(); const q = `Find this on screen: "${targetDescription}". The image is HALF-SCALE of the real screen (real screen is 2x larger). Reply ONLY JSON: {"found":true,"x":
,"y":
} or {"found":false,"reason":"..."}`; const raw = await visionDescribe(b64, q, settings); let coords = null; try { coords = JSON.parse(raw.slice(raw.indexOf('{'), raw.lastIndexOf('}') + 1)); } catch {} if (!coords || !coords.found) return { ok: false, note: raw.slice(0, 120) }; const x = Math.round((coords.x || 0) * 2); const y = Math.round((coords.y || 0) * 2); await callTool('ComputerUse', 'Click', { loc: [x, y] }); return { ok: true, x, y, target: targetDescription }; } async function observe() { const snap = await mcp('take_snapshot', {}); if (looksLikeCaptcha(snap)) return { snap, captcha: true }; return { snap, captcha: false }; } async function runTaskLoop() { const settings = loadJSON('settings'); await mcp('navigate_page', { url: 'about:blank', type: 'url' }).catch(() => {}); if (!(task.plan || []).length) { tLog('planning...'); let memLine = ''; try { const { readFileSync } = await import('node:fs'); const memPath = new URL('./data/memory.json', import.meta.url); const mems = JSON.parse(readFileSync(memPath, 'utf8').replace(/^\uFEFF/, '')); if (Array.isArray(mems) && mems.length) memLine = 'Known user facts: ' + mems.slice(-10).map(m => m.text).join('; '); } catch {} const planRaw = await callNVIDIA([ { role: 'system', content: 'Turn vague user goals into short concrete browser checklists. Reply ONLY JSON: {"plan":["step1","step2"],"questions":[{"field":"name","question":"..."}]}. Max 8 steps. Ask a question ONLY if the goal is impossible without the answer.' }, { role: 'user', content: `GOAL: ${task.instruction}\n${memLine}` } ], settings); try { const j = JSON.parse(planRaw.slice(planRaw.indexOf('{'), planRaw.lastIndexOf('}') + 1)); task.plan = j.plan || []; for (const q of (j.questions || []).slice(0, 2)) { if (!(q.field in (task.answers || {}))) { task.pendingQuestion = q; await pauseFor('question', q.question, q.field); return; } } task.plan.forEach((p2, i) => tLog(`plan ${i + 1}: ${p2}`)); } catch { tLog('planning parse failed - freestyle'); } task.planDone = true; } let lastSnap = ''; let failStreak = 0; while (task.running && task.step < task.maxSteps) { task.step++; const { snap, captcha } = await observe(); if (captcha) { await pauseFor('captcha', 'CAPTCHA appeared - solve it in the browser, then Resume.'); return; } const decisionRaw = await callNVIDIA([ { role: 'system', content: `You are Quillan's browser-hands, driving a real Chrome via the Chrome DevTools protocol. You get an accessibility-tree snapshot where every interactive element has a uid (like "1_8"). Output EXACTLY ONE JSON directive: {"op":"click","uid":"1_8","expect":"what changes after"} {"op":"vision_click","target":"description of thing to click","expect":"..."} {"op":"fill","uid":"1_5","text":"...","expect":"field now contains..."} {"op":"navigate","url":"https://...","expect":"..."} {"op":"press","key":"Enter","expect":"..."} {"op":"done","summary":"evidence the GOAL is achieved"} {"op":"help","reason":"why stuck"} Rules: one action per turn. Use uids EXACTLY as listed in the snapshot - never invent them. If the last verify said NO CHANGE, change strategy. done requires evidence the GOAL is met, not just that an action succeeded.` }, { role: 'user', content: `GOAL: ${task.instruction}\nPLAN: ${(task.plan || []).join(' | ') || 'freestyle'}\nSTEP ${task.step}/${task.maxSteps}\nKNOWN: ${JSON.stringify(task.answers || {})}\n${task.lastVerify ? 'LAST VERIFY: ' + task.lastVerify + '\n(If NO CHANGE: critique and change strategy - RCI loop)\n' : ''}\nPAGE SNAPSHOT (a11y tree with uids):\n${snap.slice(0, 6000)}` } ], settings, 700, { model: settings.deepKernelModel, timeoutMs: 90000 }); let d = null; try { d = JSON.parse(decisionRaw.match(/\{[\s\S]*\}/)[0]); } catch { tLog('unparseable directive'); continue; } const op = (d.op || '').toLowerCase(); let acted = false; try { if (op === 'navigate') { await mcp('navigate_page', { url: d.url, type: 'url' }); acted = true; tLog('goto ' + d.url); await new Promise(r => setTimeout(r, 2000)); } else if (op === 'click') { await mcp('click', { uid: d.uid }); acted = true; tLog(`clicked ${d.uid}`); await new Promise(r => setTimeout(r, 1800)); } else if (op === 'fill') { await mcp('fill', { uid: d.uid, value: String(d.text || '') }); acted = true; tLog(`filled ${d.uid}`); } else if (op === 'press') { await mcp('press_key', { key: d.key || 'Enter' }); acted = true; tLog(`pressed ${d.key}`); await new Promise(r => setTimeout(r, 1500)); } else if (op === 'vision_click') { const r = await visionClick(d.target || d.text || ''); tLog(r.ok ? `vision-clicked (${r.x},${r.y})` : `vision miss: ${r.note}`); acted = r.ok; } else if (op === 'done') { task.running = false; task.result = { outcome: 'completed', summary: d.summary || '' }; tLog('GOAL COMPLETE: ' + (d.summary || '').slice(0, 90)); return; } else if (op === 'help') { task.running = false; task.needsHelp = d.reason || 'blocked'; tLog('NEEDS HELP: ' + task.needsHelp); return; } else { tLog('unknown op ' + d.op); } } catch (e) { tLog(`action ${op} failed: ${e.message.slice(0, 60)}`); failStreak++; } if (acted) { await new Promise(r => setTimeout(r, 900)); const fresh = await observe().catch(() => ({ snap: '' })); const changed = fresh.snap !== lastSnap; task.lastVerify = `expected "${(d.expect || '').slice(0, 60)}" | snapshot changed: ${changed}`; if (!changed) { failStreak++; if (failStreak >= 2) { task.running = false; task.needsHelp = `stuck: two actions, no page change. Intent: ${d.expect || op}`; tLog('STUCK - escalating to human'); return; } } else failStreak = 0; lastSnap = fresh.snap; tLog('verify: ' + task.lastVerify); } } if (task.running) { task.running = false; task.needsHelp = 'max steps reached'; tLog('max steps reached'); } }