/** * ============================================ * gui-agent.js — Zelin GUI Interaction Agent * ============================================ * Handles ALL Minecraft GUIs with AI-driven decisions. * Architecture: SEE GUI → AI THINKS → HUMAN-LIKE EXECUTION * Every click has natural timing, every decision goes through AI. * Integrated with psyche.js for emotional context. */ import { callAIBackground } from './ai.js'; import { getStateSnapshot } from './psyche.js'; import { getBot } from './mineflayer-agent.js'; // ═══════════════════════════════════════════════════════════════════════════════ // UTILITY FUNCTIONS // ═══════════════════════════════════════════════════════════════════════════════ function sleep(ms) { return new Promise(resolve => setTimeout(resolve, Math.max(0, ms))); } function gaussianRandom(mean = 0, stdDev = 1) { const u1 = Math.random(); const u2 = Math.random(); const z0 = Math.sqrt(-2 * Math.log(Math.max(u1, 1e-10))) * Math.cos(2 * Math.PI * u2); return z0 * stdDev + mean; } // ═══════════════════════════════════════════════════════════════════════════════ // CORE: Read any open GUI window // ═══════════════════════════════════════════════════════════════════════════════ function readCurrentWindow() { const bot = getBot(); const win = bot?.currentWindow ?? bot?.inventory; if (!win) return null; return { type: win.type ?? 'unknown', title: win.title ?? '', slots: win.slots ?.map((item, i) => item ? { slot: i, name: item.name, type: item.type, // numeric item ID needed for withdraw/deposit displayName: item.displayName ?? item.name, count: item.count, nbt: item.nbt ? 'present' : null, } : null) .filter(Boolean) ?? [], inventorySlots: bot?.inventory?.slots ?.map((item, i) => item ? { slot: i, name: item.name, count: item.count } : null) .filter(Boolean) ?? [], }; } // ═══════════════════════════════════════════════════════════════════════════════ // CORE: AI decides what to do inside a GUI // ═══════════════════════════════════════════════════════════════════════════════ async function thinkAboutGUI(windowData, context = '') { const psyche = getStateSnapshot(); const prompt = `Eres Zelin, jugadora de Minecraft. Acabas de abrir una GUI. GUI ABIERTA: ${JSON.stringify(windowData, null, 2)} CONTEXTO: ${context} ESTADO: mood=${psyche.mood}, energy=${psyche.energy} Analiza el contenido y decide que hacer. Responde SOLO en JSON: { "observation": "(que ves en esta GUI, en 1 frase natural)", "decision": "take|put|craft|smelt|enchant|trade|rename|brew|browse|close|nothing", "actions": [ { "type": "click|shift_click|move|take_all|put|craft|close", "slot": 0, "item": "nombre del item si aplica", "count": 1, "reason": "por que" } ], "chatAfter": null, "humanDelay": 800 }`; try { const raw = await callAIBackground([ { role: 'system', content: prompt }, { role: 'user', content: 'que hago en esta GUI?' }, ], 'fast', 200, 'gui-decision'); const cleaned = raw.replace(/```json|```/g, '').trim(); const startIdx = cleaned.indexOf('{'); const endIdx = cleaned.lastIndexOf('}'); if (startIdx < 0 || endIdx < 0) return { decision: 'close', actions: [] }; return JSON.parse(cleaned.slice(startIdx, endIdx + 1)); } catch (e) { console.error('[GUI] Think error:', e.message); return { decision: 'close', actions: [] }; } } // ═══════════════════════════════════════════════════════════════════════════════ // HUMAN EXECUTION — Clicks with natural timing // ═══════════════════════════════════════════════════════════════════════════════ async function humanClickSlot(slot, mode = 0, button = 0) { const bot = getBot(); if (!bot) return; // Hover visual before clicking (simulates moving cursor to slot) await sleep(120 + Math.random() * 200); try { await bot.clickWindow(slot, button, mode); } catch (e) { console.warn('[GUI] Click failed:', e.message); } // Small post-click pause (processing what was done) await sleep(80 + Math.random() * 150); } async function humanShiftClick(slot) { const bot = getBot(); if (!bot) return; await sleep(100 + Math.random() * 180); try { await bot.clickWindow(slot, 0, 1); // mode=1 is shift-click } catch (e) { console.warn('[GUI] Shift-click failed:', e.message); } await sleep(60 + Math.random() * 100); } async function browseWindow(windowData) { // A human "scans" the GUI visually before doing anything // Simulate with delay proportional to number of items const items = windowData?.slots?.length ?? 0; const browseTime = 300 + items * 40 + Math.random() * 500; await sleep(browseTime); } // ═══════════════════════════════════════════════════════════════════════════════ // HANDLERS BY GUI TYPE // ═══════════════════════════════════════════════════════════════════════════════ // COFRE / BARRIL / SHULKER / ENDER CHEST async function handleChest(block, context = '') { const bot = getBot(); if (!bot) return null; let chest; try { chest = await bot.openChest(block); await sleep(200 + Math.random() * 300); // Opening time } catch (e) { console.error('[GUI] Failed to open chest:', e.message); return null; } const windowData = readCurrentWindow(); if (!windowData) { chest.close(); return null; } await browseWindow(windowData); // Human visual scan const decision = await thinkAboutGUI(windowData, context); if (!decision?.actions) { chest.close(); return decision; } for (const action of decision.actions) { await sleep((decision.humanDelay ?? 800) + Math.random() * 400); try { if (action.type === 'take') { // Find the item in chest slots const slotItem = windowData.slots.find(s => s.slot === action.slot || s.name === action.item); if (slotItem) { await chest.withdraw(slotItem.type ?? slotItem.name, null, action.count ?? slotItem.count); } } else if (action.type === 'shift_click') { await humanShiftClick(action.slot); } else if (action.type === 'put') { const invItem = bot.inventory.items().find(i => i.name === action.item); if (invItem) { await chest.deposit(invItem.type, null, action.count ?? invItem.count); } } else if (action.type === 'click') { await humanClickSlot(action.slot); } else if (action.type === 'close') { break; } } catch (e) { console.warn('[GUI] Action failed:', action.type, e.message); } } await sleep(200 + Math.random() * 300); try { chest.close(); } catch { /* already closed */ } if (decision.chatAfter) { await sleep(500 + Math.random() * 1000); try { bot.chat(decision.chatAfter); } catch { /* not connected */ } } return decision; } // HORNO / ALTO HORNO / AHUMADOR async function handleFurnace(block, context = '') { const bot = getBot(); if (!bot) return null; let furnace; try { furnace = await bot.openFurnace(block); await sleep(300 + Math.random() * 400); } catch (e) { console.error('[GUI] Failed to open furnace:', e.message); return null; } const windowData = { type: 'furnace', inputSlot: furnace.inputItem?.() ?? null, fuelSlot: furnace.fuelItem?.() ?? null, outputSlot: furnace.outputItem?.() ?? null, progress: furnace.progress ?? 0, fuelLevel: furnace.fuel ?? 0, }; const decision = await thinkAboutGUI(windowData, context); if (decision?.actions) { for (const action of decision.actions) { await sleep((decision.humanDelay ?? 800) + Math.random() * 300); try { if (action.type === 'put' && action.slot === 0) { // Put fuel const fuel = bot.inventory.items().find(i => i.name === action.item); if (fuel) await furnace.putFuel(fuel.type, null, action.count ?? 1); } else if (action.type === 'put' && action.slot === 1) { // Put item to smelt const toSmelt = bot.inventory.items().find(i => i.name === action.item); if (toSmelt) await furnace.putInput(toSmelt.type, null, action.count ?? 1); } else if (action.type === 'take') { await furnace.takeOutput(); } } catch (e) { console.warn('[GUI] Furnace action failed:', action.type, e.message); } } } try { furnace.close(); } catch { /* already closed */ } return decision; } // MESA DE ENCANTAMIENTOS async function handleEnchantTable(block, context = '') { const bot = getBot(); if (!bot) return null; let table; try { table = await bot.openEnchantmentTable(block); await sleep(400 + Math.random() * 600); // Zelin "reads" the options } catch (e) { console.error('[GUI] Failed to open enchant table:', e.message); return null; } const windowData = { type: 'enchanting_table', enchantments: table.enchantments?.map((e, i) => ({ index: i, level: e?.level, cost: e?.cost, })) ?? [], xpLevel: bot.experience?.level ?? 0, }; const decision = await thinkAboutGUI(windowData, context); if (decision?.decision === 'enchant' && decision.actions) { for (const action of decision.actions) { await sleep(600 + Math.random() * 800); // Doubt before enchanting try { if (action.type === 'click' && action.slot >= 0 && action.slot <= 2) { await table.enchant(action.slot); // 0=cheap, 1=medium, 2=expensive } } catch (e) { console.warn('[GUI] Enchant failed:', e.message); } } } try { table.close(); } catch { /* already closed' */ } return decision; } // ALDEANO / COMERCIO async function handleVillager(entity, context = '') { const bot = getBot(); if (!bot) return null; let villager; try { villager = await bot.openVillager(entity); await sleep(300 + Math.random() * 500); } catch (e) { console.error('[GUI] Failed to open villager:', e.message); return null; } const trades = (villager.trades ?? []).map((t, i) => ({ index: i, inputItem1: t.inputItem1?.name ?? null, inputItem2: t.inputItem2?.name ?? null, outputItem: t.outputItem?.name ?? null, disabled: t.disabled ?? false, uses: t.uses ?? 0, maxUses: t.maxUses ?? 0, })); const windowData = { type: 'villager', trades }; const decision = await thinkAboutGUI(windowData, context); if (decision?.decision === 'trade' && decision.actions) { for (const action of decision.actions) { await sleep((decision.humanDelay ?? 800) + Math.random() * 600); try { await villager.selectTrade(action.slot ?? action.index ?? 0); await sleep(300 + Math.random() * 300); await bot.clickWindow(2, 0, 0); // slot 2 = trade output } catch (e) { console.warn('[GUI] Trade failed:', e.message); } } } try { villager.close(); } catch { /* already closed */ } return decision; } // MESA DE CRAFTEO async function handleCraftingTable(block, context = '') { const bot = getBot(); if (!bot) return null; const inventory = bot.inventory.items() .map(i => ({ name: i.name, count: i.count })); const windowData = { type: 'crafting_table', inventory }; const decision = await thinkAboutGUI(windowData, context); if (decision?.decision === 'craft' && decision.actions?.[0]?.item) { const itemName = decision.actions[0].item; try { const mcData = (await import('minecraft-data')).default(bot.version); const recipes = bot.recipesFor(mcData.itemsByName[itemName]?.id, null, 1, block); if (recipes.length > 0) { await sleep(400 + Math.random() * 600); await bot.craft(recipes[0], decision.actions[0].count ?? 1, block); } } catch (e) { console.warn('[GUI] Craft failed:', e.message); } } return decision; } // YUNQUE async function handleAnvil(block, context = '') { const bot = getBot(); if (!bot) return null; let anvilWindow; try { anvilWindow = await bot.openAnvil(block); await sleep(300 + Math.random() * 400); } catch (e) { console.error('[GUI] Failed to open anvil:', e.message); return null; } const windowData = { type: 'anvil', slot0: anvilWindow.slots?.[0] ?? null, slot1: anvilWindow.slots?.[1] ?? null, outputSlot: anvilWindow.slots?.[2] ?? null, xpCost: anvilWindow.xpCost ?? 0, }; const decision = await thinkAboutGUI(windowData, context); if (decision?.actions) { for (const action of decision.actions) { try { if (decision.decision === 'rename' && action.item) { await anvilWindow.rename(action.item); await sleep(500 + Math.random() * 500); await humanClickSlot(2); // Take result } else if (decision.decision === 'craft') { await sleep(400 + Math.random() * 300); await humanClickSlot(2); // Take result } } catch (e) { console.warn('[GUI] Anvil action failed:', e.message); } } } try { anvilWindow.close(); } catch { /* already closed */ } return decision; } // BALIZA async function handleBeacon(block, context = '') { const bot = getBot(); if (!bot) return null; let beacon; try { beacon = await bot.openBeacon(block); await sleep(500 + Math.random() * 500); } catch (e) { console.error('[GUI] Failed to open beacon:', e.message); return null; } const windowData = { type: 'beacon', effects: beacon.effects ?? [], level: beacon.level ?? 0, }; const decision = await thinkAboutGUI(windowData, context); if (decision?.decision !== 'close' && decision?.actions?.[0]) { await sleep(600 + Math.random() * 400); try { if (decision.actions[0].slot !== undefined) { await humanClickSlot(decision.actions[0].slot); } } catch (e) { console.warn('[GUI] Beacon action failed:', e.message); } } try { beacon.close(); } catch { /* already closed */ } return decision; } // PUESTO DE ELABORACIÓN DE POCIONES async function handleBrewingStand(block, context = '') { const bot = getBot(); if (!bot) return null; let brewStand; try { brewStand = await bot.openBrewingStand(block); await sleep(300 + Math.random() * 400); } catch (e) { console.error('[GUI] Failed to open brewing stand:', e.message); return null; } const windowData = { type: 'brewing_stand', slots: brewStand.slots?.map((item, i) => item ? { slot: i, name: item.name, count: item.count, } : null).filter(Boolean) ?? [], fuelLevel: brewStand.fuelLevel ?? 0, brewTime: brewStand.progress ?? 0, }; const decision = await thinkAboutGUI(windowData, context); if (decision?.actions) { for (const action of decision.actions) { await sleep((decision.humanDelay ?? 800) + Math.random() * 300); try { if (action.type === 'put') { const item = bot.inventory.items().find(i => i.name === action.item); if (item) { await brewStand.putIngredient(item.type, null, action.count ?? 1); } } else if (action.type === 'take') { await humanClickSlot(action.slot); } } catch (e) { console.warn('[GUI] Brewing action failed:', e.message); } } } try { brewStand.close(); } catch { /* already closed */ } return decision; } // PIEDRA DE AFILAR async function handleGrindstone(block, context = '') { const bot = getBot(); if (!bot) return null; let grindstone; try { grindstone = await bot.openGrindstone(block); await sleep(300 + Math.random() * 400); } catch (e) { console.error('[GUI] Failed to open grindstone:', e.message); return null; } const windowData = { type: 'grindstone', slots: grindstone.slots?.map((item, i) => item ? { slot: i, name: item.name, count: item.count, } : null).filter(Boolean) ?? [], }; const decision = await thinkAboutGUI(windowData, context); if (decision?.decision === 'craft' || decision?.decision === 'take') { await sleep(400 + Math.random() * 300); try { await humanClickSlot(2); } catch { /* take failed */ } } try { grindstone.close(); } catch { /* already closed */ } return decision; } // MESA DE HERRERÍA async function handleSmithingTable(block, context = '') { const bot = getBot(); if (!bot) return null; let smithing; try { smithing = await bot.openSmithingTable(block); await sleep(300 + Math.random() * 400); } catch (e) { console.error('[GUI] Failed to open smithing table:', e.message); return null; } const windowData = { type: 'smithing_table', slots: smithing.slots?.map((item, i) => item ? { slot: i, name: item.name, count: item.count, } : null).filter(Boolean) ?? [], }; const decision = await thinkAboutGUI(windowData, context); if (decision?.decision === 'craft' || decision?.decision === 'take') { await sleep(400 + Math.random() * 300); try { await humanClickSlot(2); } catch { /* take failed */ } } try { smithing.close(); } catch { /* already closed */ } return decision; } // CORTAPIEDRAS async function handleStonecutter(block, context = '') { const bot = getBot(); if (!bot) return null; let stonecutter; try { stonecutter = await bot.openStonecutter(block); await sleep(300 + Math.random() * 400); } catch (e) { console.error('[GUI] Failed to open stonecutter:', e.message); return null; } const windowData = { type: 'stonecutter', slots: stonecutter.slots?.map((item, i) => item ? { slot: i, name: item.name, count: item.count, } : null).filter(Boolean) ?? [], }; const decision = await thinkAboutGUI(windowData, context); if (decision?.actions) { for (const action of decision.actions) { await sleep((decision.humanDelay ?? 800) + Math.random() * 300); try { if (action.type === 'click') { await humanClickSlot(action.slot); } } catch (e) { console.warn('[GUI] Stonecutter action failed:', e.message); } } } try { stonecutter.close(); } catch { /* already closed */ } return decision; } // TEJOR (LOOM) async function handleLoom(block, context = '') { const bot = getBot(); if (!bot) return null; let loom; try { loom = await bot.openLoom(block); await sleep(300 + Math.random() * 400); } catch (e) { console.error('[GUI] Failed to open loom:', e.message); return null; } const windowData = { type: 'loom', slots: loom.slots?.map((item, i) => item ? { slot: i, name: item.name, count: item.count, } : null).filter(Boolean) ?? [], }; const decision = await thinkAboutGUI(windowData, context); if (decision?.actions) { for (const action of decision.actions) { await sleep((decision.humanDelay ?? 800) + Math.random() * 300); try { if (action.type === 'click') { await humanClickSlot(action.slot); } else if (action.type === 'shift_click') { await humanShiftClick(action.slot); } } catch (e) { console.warn('[GUI] Loom action failed:', e.message); } } } try { loom.close(); } catch { /* already closed */ } return decision; } // GUI DE PLUGINS (server custom menus, shops, selectors) async function handlePluginGUI(context = '') { const bot = getBot(); if (!bot) return null; await sleep(300 + Math.random() * 400); const windowData = readCurrentWindow(); if (!windowData) return null; await browseWindow(windowData); const decision = await thinkAboutGUI(windowData, context); if (decision?.actions) { for (const action of decision.actions) { await sleep((decision.humanDelay ?? 800) + Math.random() * 500); try { if (action.type === 'click') { await humanClickSlot(action.slot); } else if (action.type === 'shift_click') { await humanShiftClick(action.slot); } else if (action.type === 'close') { if (bot.currentWindow) bot.closeWindow(bot.currentWindow); break; } } catch (e) { console.warn('[GUI] Plugin GUI action failed:', e.message); } } } return decision; } // ═══════════════════════════════════════════════════════════════════════════════ // UNIFIED ENTRY POINT // Zelin detects what GUI it is and handles it // ═══════════════════════════════════════════════════════════════════════════════ export async function openAndInteract(target, context = '') { if (!target) return null; const bot = getBot(); if (!bot) return null; const blockName = target.name ?? ''; const blockOrEntity = target; try { // Storage GUIs if (['chest', 'barrel', 'shulker_box', 'ender_chest', 'trapped_chest'] .some(n => blockName.includes(n))) { return await handleChest(target, context); } // Furnace family if (['furnace', 'blast_furnace', 'smoker'] .some(n => blockName.includes(n))) { return await handleFurnace(target, context); } // Enchanting if (blockName.includes('enchanting')) { return await handleEnchantTable(target, context); } // Anvil if (blockName.includes('anvil')) { return await handleAnvil(target, context); } // Brewing if (blockName.includes('brewing')) { return await handleBrewingStand(target, context); } // Beacon if (blockName.includes('beacon')) { return await handleBeacon(target, context); } // Crafting table if (blockName.includes('crafting')) { return await handleCraftingTable(target, context); } // Grindstone if (blockName.includes('grindstone')) { return await handleGrindstone(target, context); } // Smithing table if (blockName.includes('smithing')) { return await handleSmithingTable(target, context); } // Stonecutter if (blockName.includes('stonecutter')) { return await handleStonecutter(target, context); } // Loom if (blockName.includes('loom')) { return await handleLoom(target, context); } // Villager (entity, not block) if (target.entityType === 'player' || target.name === 'villager' || target.name === 'wandering_trader') { return await handleVillager(target, context); } // Fallback: treat as plugin GUI — activate block and wait for window try { await bot.activateBlock(target); await new Promise((resolve, reject) => { const timeout = setTimeout(() => reject(new Error('window open timeout')), 5000); bot.once('windowOpen', () => { clearTimeout(timeout); resolve(); }); }); return await handlePluginGUI(context); } catch (e) { console.warn('[GUI] Fallback GUI handling failed:', e.message); } } catch (e) { console.error('[GUI] Error opening GUI:', e.message); try { if (bot.currentWindow) bot.closeWindow(bot.currentWindow); } catch { /* ignore */ } } return null; } // ═══════════════════════════════════════════════════════════════════════════════ // INVENTORY REVIEW — Zelin checks her own inventory periodically // ═══════════════════════════════════════════════════════════════════════════════ export async function reviewInventory(context = 'revisando inventario') { const bot = getBot(); if (!bot) return null; const inventory = bot.inventory.items() .map(i => ({ slot: i.slot, name: i.name, count: i.count })); const decision = await thinkAboutGUI({ type: 'inventory', slots: inventory, hotbar: inventory.filter(i => i.slot >= 36 && i.slot < 45), }, context); if (decision?.actions) { for (const action of decision.actions) { await sleep(300 + Math.random() * 400); try { if (action.type === 'move') { await bot.moveSlotItem(action.slot, action.targetSlot); } else if (action.type === 'craft') { // Craft in 2x2 inventory crafting try { const mcData = (await import('minecraft-data')).default(bot.version); const recipes = bot.recipesFor(mcData.itemsByName[action.item]?.id); if (recipes?.length) await bot.craft(recipes[0], action.count ?? 1, null); } catch (e) { console.warn('[GUI] Inventory craft failed:', e.message); } } else if (action.type === 'shift_click') { await humanShiftClick(action.slot); } } catch (e) { console.warn('[GUI] Inventory action failed:', action.type, e.message); } } } return decision; } // ═══════════════════════════════════════════════════════════════════════════════ // LIBRO Y PLUMA — Write in books // ═══════════════════════════════════════════════════════════════════════════════ export async function writeBook(title, pages, sign = true) { const bot = getBot(); if (!bot) return false; try { const book = bot.inventory.items().find(i => i.name === 'writable_book'); if (!book) return false; await bot.equip(book, 'hand'); await sleep(300 + Math.random() * 200); // Open the book await bot.activateItem(); await sleep(500 + Math.random() * 300); // Write pages for (let i = 0; i < pages.length; i++) { // Click the page area await humanClickSlot(0); // First page slot await sleep(200 + Math.random() * 200); // Type page content // Note: mineflayer doesn't have native book writing, this is a best-effort if (i < pages.length - 1) { // Next page button await humanClickSlot(1); await sleep(200 + Math.random() * 100); } } if (sign) { // Sign the book await humanClickSlot(2); // Sign button await sleep(300 + Math.random() * 200); } return true; } catch (e) { console.warn('[GUI] Book writing failed:', e.message); return false; } } // ═══════════════════════════════════════════════════════════════════════════════ // DISPENSER / DROPPER / HOPPER // ═══════════════════════════════════════════════════════════════════════════════ async function handleContainer(block, context = '') { const bot = getBot(); if (!bot) return null; let container; try { container = await bot.openContainer(block); await sleep(200 + Math.random() * 300); } catch (e) { console.error('[GUI] Failed to open container:', e.message); return null; } const windowData = readCurrentWindow(); if (!windowData) { container.close(); return null; } await browseWindow(windowData); const decision = await thinkAboutGUI(windowData, context); if (decision?.actions) { for (const action of decision.actions) { await sleep((decision.humanDelay ?? 800) + Math.random() * 300); try { if (action.type === 'take') { const slotItem = windowData.slots.find(s => s.slot === action.slot || s.name === action.item); if (slotItem) { await humanClickSlot(slotItem.slot); } } else if (action.type === 'put') { const invItem = bot.inventory.items().find(i => i.name === action.item); if (invItem) { await humanShiftClick(invItem.slot); } } else if (action.type === 'click') { await humanClickSlot(action.slot); } else if (action.type === 'close') { break; } } catch (e) { console.warn('[GUI] Container action failed:', e.message); } } } try { container.close(); } catch { /* already closed */ } return decision; } console.log('[GUI] Module loaded');