AUDIT / src /lib /agentLoop /browserTools.ts
Arypulka98's picture
feat(audit): deploy full backend cluster node (part 2)
cc11e77 verified
Raw
History Blame Contribute Delete
22.7 kB
// S538 — estratto da webTools.ts: browser/accessibility/notification tool implementations
// Tool coperti: accessibility_check, extract_table, send_notification,
// browser_navigate, browser_open, browser_act, browser_close, browser_screenshot
import { directFetch, viaProxy, TIMEOUT_PAGE, makeTimedSignal } from "./networkConstants"; // Loop-10: iOS-safe
// ─── Backend URL helper ────────────────────────────────────────────────────────
function _backendBase(): string {
return (
(typeof import.meta !== "undefined"
? (import.meta as { env?: { VITE_BACKEND_URL?: string } }).env?.VITE_BACKEND_URL
: undefined) ?? "http://localhost:8000"
).replace(/\/$/, "");
}
// ─── INTERNAL_TOKEN per auth backend ────────────────────────────────────────
const _BROWSER_INTERNAL_TOKEN = (
(typeof import.meta !== "undefined"
? (import.meta as { env?: { VITE_INTERNAL_TOKEN?: string } }).env?.VITE_INTERNAL_TOKEN
: undefined) ?? ""
);
// ─── tryExecuteBrowser — dispatcher browser/accessibility/notification tools ──
/**
* Dispatcher per i tool browser, accessibilità e notifiche.
* Ritorna string con risultato, oppure null se il tool non è gestito qui.
*/
export async function tryExecuteBrowser(
name: string,
args: Record<string, unknown>,
): Promise<string | null> {
if (name === "accessibility_check") {
const { url, html } = args as { url?: string; html?: string };
let pageHtml = html ?? "";
if (!pageHtml && url) {
pageHtml = await directFetch(url, TIMEOUT_PAGE) ?? "";
if (!pageHtml) pageHtml = await viaProxy(url, TIMEOUT_PAGE) ?? "";
if (!pageHtml) return `❌ accessibility_check: impossibile caricare ${url}`;
}
if (!pageHtml) return `❌ accessibility_check: fornisci url o html.`;
const issues: Array<{ level: "A"|"AA"; type: string; desc: string; count: number }> = [];
const count = (rx: RegExp, txt: string) => (txt.match(rx) ?? []).length;
const imgs = count(/<img[^>]*/gi, pageHtml);
const imgsWithAlt = count(/<img[^>]+alt=["'][^"']+["'][^>]*/gi, pageHtml);
const imgsNoAlt = imgs - imgsWithAlt;
if (imgsNoAlt > 0) issues.push({ level:"A", type:"1.1.1 Alt Text", desc:`${imgsNoAlt} immagini senza alt text`, count: imgsNoAlt });
const inputs = count(/<input(?![^>]+type=["']?hidden["']?)[^>]*/gi, pageHtml);
const labelsCount = count(/<label[^>]*/gi, pageHtml);
if (inputs > 0 && labelsCount < inputs) issues.push({ level:"A", type:"1.3.1 Form Labels", desc:`${inputs} input ma solo ${labelsCount} label (${inputs - labelsCount} non etichettati)`, count: inputs - labelsCount });
const langMatch = pageHtml.match(/<html[^>]+lang=["']([^"']+)["']/i);
if (!langMatch) issues.push({ level:"A", type:"3.1.1 Lang", desc:"Attributo lang mancante sul tag <html>", count: 1 });
const titleMatch = pageHtml.match(/<title[^>]*>([^<]+)<\/title>/i);
if (!titleMatch) issues.push({ level:"A", type:"2.4.2 Title", desc:"Tag <title> mancante o vuoto", count: 1 });
const h1Count = count(/<h1[^>]*/gi, pageHtml);
if (h1Count === 0) issues.push({ level:"A", type:"1.3.1 Headings", desc:"Nessun <h1> trovato nella pagina", count: 1 });
if (h1Count > 1) issues.push({ level:"A", type:"1.3.1 Headings", desc:`${h1Count} elementi <h1> (dovrebbe essercene uno solo)`, count: h1Count });
const tabindex = count(/tabindex=["']-[1-9]/gi, pageHtml);
if (tabindex > 0) issues.push({ level:"AA", type:"2.1.1 Keyboard", desc:`${tabindex} elementi con tabindex negativo (non navigabili da tastiera)`, count: tabindex });
const linksNoText = count(/<a[^>]*>\s*(<img[^>]+>)?\s*<\/a>/gi, pageHtml);
if (linksNoText > 0) issues.push({ level:"AA", type:"2.4.4 Link Purpose", desc:`${linksNoText} link senza testo visibile`, count: linksNoText });
const skipsLinks = pageHtml.includes("skip-to-content") || pageHtml.includes("skip-navigation") || pageHtml.includes("skipnav");
if (!skipsLinks) issues.push({ level:"AA", type:"2.4.1 Skip Links", desc:"Nessun link 'skip to content' trovato", count: 1 });
if (!issues.length) {
return `♿ **Accessibilità: OTTIMA** per ${url ?? "HTML fornito"}\n\nNessun problema WCAG A/AA rilevato con analisi statica.`;
}
const levelA = issues.filter(i => i.level === "A");
const levelAA = issues.filter(i => i.level === "AA");
let out = `♿ **Report Accessibilità** — ${url ?? "HTML fornito"}\n`;
out += `🔴 ${levelA.length} problemi critici (WCAG A) | 🟡 ${levelAA.length} problemi medi (WCAG AA)\n\n`;
if (levelA.length) { out += `**Critici (WCAG A):**\n${levelA.map(i => `• ❌ **${i.type}**: ${i.desc}`).join("\n")}\n\n`; }
if (levelAA.length) { out += `**Medi (WCAG AA):**\n${levelAA.map(i => `• ⚠️ **${i.type}**: ${i.desc}`).join("\n")}`; }
return out.trim();
}
if (name === "extract_table") {
const { url, table_index = 0 } = args as { url: string; table_index?: number };
let html = await directFetch(url, TIMEOUT_PAGE);
if (!html) html = await viaProxy(url, TIMEOUT_PAGE);
if (!html) return `❌ Pagina non accessibile: ${url}`;
const tables = [...html.matchAll(/<table[^>]*>([\s\S]*?)<\/table>/gi)];
if (!tables.length) return `❌ Nessuna tabella HTML trovata in ${url}`;
const idx = Math.min(Number(table_index) || 0, tables.length - 1);
const tableHtml = tables[idx]?.[0] ?? "";
const rows: string[][] = [];
for (const rowMatch of tableHtml.matchAll(/<tr[^>]*>([\s\S]*?)<\/tr>/gi)) {
const row: string[] = [];
for (const cellMatch of rowMatch[1].matchAll(/<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi)) {
row.push(cellMatch[1].replace(/<[^>]+>/g, "").replace(/\s+/g, " ").replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").trim());
}
if (row.length > 0) rows.push(row);
}
if (!rows.length) return `❌ Tabella ${idx} vuota o non parsabile.`;
const maxCols = Math.max(...rows.map(r => r.length));
const header = rows[0].map(c => c.slice(0, 30)).join(" | ");
const sep = Array(maxCols).fill("---").join(" | ");
const body = rows.slice(1, 51).map(r => r.map(c => c.slice(0, 30)).join(" | ")).join("\n");
const note = rows.length > 51 ? `\n\n_…e altre ${rows.length - 51} righe_` : "";
return `📊 **Tabella ${idx + 1}/${tables.length}** da _${url}_\n(${rows.length} righe, ${maxCols} colonne)\n\n| ${header} |\n| ${sep} |\n| ${body.split("\n").join(" |\n| ")} |${note}`;
}
if (name === "send_notification") {
const { title, body = "" } = args as { title: string; body?: string };
if (typeof Notification === "undefined") return "❌ Notifiche non supportate in questo browser.";
if (Notification.permission === "denied") return "❌ Notifiche bloccate dall'utente. Riabilita dalle impostazioni del browser.";
const sendNow = () => {
try { new Notification(title, { body, icon: "/icon-192.png" }); } catch { /**/ }
return `✅ Notifica inviata: "${title}"${body ? ` — ${body}` : ""}`;
};
if (Notification.permission === "granted") return sendNow();
const perm = await Notification.requestPermission();
return perm === "granted" ? sendNow() : "❌ Permesso notifiche negato.";
}
if (name === "browser_navigate") {
const {
url,
actions = [],
width = 1280,
mobile = false,
wait_ms = 2000,
caption = "",
} = args as {
url: string;
actions?: Array<{ type: string; selector?: string; value?: string; key?: string; ms?: number }>;
width?: number;
mobile?: boolean;
wait_ms?: number;
caption?: string;
};
const backendBase = _backendBase();
// S274: fallback rapido se backend non configurato o punta a localhost (offline su iPhone)
// GAP-FIX-3: aggiunto viaProxy fallback — CORS block su iPhone senza backend
if (!backendBase || backendBase.includes("localhost")) {
let pageResult = await directFetch(url, TIMEOUT_PAGE) ?? "";
if (!pageResult || pageResult.length < 50) pageResult = await viaProxy(url, TIMEOUT_PAGE) ?? "";
if (pageResult && !pageResult.startsWith("❌")) return pageResult;
return `❌ browser_navigate: impossibile accedere a ${url} (CORS block su mobile, backend non configurato). Usa read_page o web_search.`;
}
try {
const ctrl = new AbortController();
const tid = setTimeout(() => ctrl.abort(), 45000);
const res = await fetch(`${backendBase}/api/browser/navigate`, {
method: "POST",
headers: { "Content-Type": "application/json", ...(_BROWSER_INTERNAL_TOKEN ? { "X-Internal-Token": _BROWSER_INTERNAL_TOKEN } : {}) },
body: JSON.stringify({ url, actions, width, mobile, wait_ms }),
signal: ctrl.signal,
});
clearTimeout(tid);
if (!res.ok) return `❌ browser_navigate: backend ${res.status} — assicurati che HF Space sia online`;
const data = await res.json() as {
ok: boolean;
screenshot_b64?: string;
title?: string;
url?: string;
text_content?: string;
error?: string;
};
if (!data.ok) return `❌ browser_navigate: ${data.error ?? "errore sconosciuto"}`;
const device = mobile ? "📱 Mobile" : "🖥️ Desktop";
const pageTitle = data.title ?? url;
const finalUrl = data.url ?? url;
const cap = caption || pageTitle;
const actLog = (actions as Array<{type:string;selector?:string}>).length > 0
? `\n_Azioni: ${(actions as Array<{type:string;selector?:string}>).map(a => a.type + (a.selector ? ` → ${a.selector}` : "")).join(", ")}_`
: "";
let out = `🌐 **Browser Automation — ${device}**\n\n`;
out += `**Titolo:** ${pageTitle}\n**URL:** ${finalUrl}${actLog}\n\n`;
if (data.screenshot_b64) {
out += `![${cap}](data:image/png;base64,${data.screenshot_b64})\n\n`;
}
if (data.text_content?.trim()) {
const txt = data.text_content.slice(0, 800);
out += `**Contenuto:**\n${txt}${data.text_content.length > 800 ? "…" : ""}`;
}
return out;
} catch (e) {
return `❌ browser_navigate: ${e instanceof Error ? e.message : String(e)}`;
}
}
// ─── browser_open — sessione persistente ──────────────────────────────────
if (name === "browser_open") {
const {
url,
actions = [],
mobile = false,
width = 1280,
wait_ms = 1500,
} = args as {
url: string;
actions?: Array<{ type: string; selector?: string; value?: string; key?: string; ms?: number }>;
mobile?: boolean;
width?: number;
wait_ms?: number;
};
const backendBase = _backendBase();
// ── max_sessions_guard: verifica sessioni aperte prima di aprire ────────
// Limite HF free tier: SESSION_LIMIT = 2 (Chromium ~300 MB/sessione).
// Se sessioni >= 2, il backend chiuderà automaticamente la più inattiva (OOM guard).
// Mostriamo il warning all'LLM: può scegliere di chiamare browser_close prima
// per evitare di perdere sessioni importanti in corso.
let _sessionGuard = "";
try {
const _sRes = await fetch(`${backendBase}/api/browser/sessions`, {
signal: makeTimedSignal(4000),
});
if (_sRes.ok) {
const _sData = await _sRes.json() as Record<string, { url: string; idle_s: number; age_s: number }>;
const _open = Object.entries(_sData);
const _limit = 2; // SESSION_LIMIT backend (browser.py)
if (_open.length >= _limit) {
const _oldest = _open.sort((a, b) => b[1].idle_s - a[1].idle_s)[0];
const _idleMin = Math.round(_oldest[1].idle_s / 60);
_sessionGuard =
`⚠️ **OOM Guard — ${_open.length}/${_limit} sessioni attive.**\n` +
`La sessione più inattiva (\`${_oldest[0]}\` · ${_oldest[1].url.slice(0, 55)} · idle ${_idleMin}min) ` +
`sarà chiusa automaticamente.\n` +
`👉 Usa \`browser_close\` per chiuderla tu stesso prima di aprire una nuova sessione.\n\n`;
} else if (_open.length === _limit - 1) {
const _s = _open[0];
_sessionGuard =
`ℹ️ Sessioni aperte: ${_open.length}/${_limit} (\`${_s[0]}\` · ${_s[1].url.slice(0, 55)}). Apertura OK.\n\n`;
}
// 0 sessioni: nessun messaggio
}
} catch { /* silenzioso — backend non raggiungibile o timeout */ }
try {
const ctrl = new AbortController();
const tid = setTimeout(() => ctrl.abort(), 30000);
const res = await fetch(`${backendBase}/api/browser/open`, {
method: "POST",
headers: { "Content-Type": "application/json", ...(_BROWSER_INTERNAL_TOKEN ? { "X-Internal-Token": _BROWSER_INTERNAL_TOKEN } : {}) },
body: JSON.stringify({ url, actions, mobile, width, wait_ms }),
signal: ctrl.signal,
});
clearTimeout(tid);
if (!res.ok) return `❌ browser_open: backend ${res.status}`;
const data = await res.json() as {
ok: boolean; session_id?: string; screenshot_b64?: string;
title?: string; url?: string;
text_content?: string; // W-NAV: trafilatura mainbody (fino a 6000 chars)
dom?: { title?: string; desc?: string; text?: string; links?: Array<{text:string;href:string}>; inputs?: Array<{tag:string;type?:string;label?:string;selector?:string}> };
ax_tree?: Record<string, unknown>; // GAP-AX: Playwright Accessibility Tree (MCP-style)
error?: string;
};
if (!data.ok) return `❌ browser_open: ${data.error ?? "errore sconosciuto"}`;
const sid = data.session_id ?? "";
const title = data.title ?? url;
// Prependi il guard (vuoto se tutto OK)
let out = _sessionGuard + `🌐 **Browser aperto** — \`${sid}\`\n\n**Titolo:** ${title}\n**URL:** ${data.url ?? url}\n\n`;
if (data.screenshot_b64) {
out += `![${title}](data:image/png;base64,${data.screenshot_b64})\n\n`;
}
if (data.dom) {
const { links = [], inputs = [], text = "" } = data.dom;
if (inputs.length > 0) {
out += `**Form/Input trovati (${inputs.length}):**\n`;
inputs.slice(0, 8).forEach(inp => {
out += `- \`${inp.selector ?? inp.tag}\` — ${inp.label ?? inp.type ?? inp.tag}\n`;
});
out += "\n";
}
if (links.length > 0) {
out += `**Link principali (${links.length}):**\n`;
links.slice(0, 6).forEach(l => out += `- [${l.text}](${l.href})\n`);
out += "\n";
}
// W-NAV: usa text_content (trafilatura, fino a 6000 chars) se disponibile,
// altrimenti dom.text (fino a MAX_TEXT chars, già troncato lato backend)
const bodyText = data.text_content?.trim() || text;
if (bodyText) {
const MAX_D = 800;
out += `**Contenuto:**\n${bodyText.slice(0, MAX_D)}${bodyText.length > MAX_D ? "…" : ""}`;
if (data.text_content && data.text_content.length > MAX_D) {
out += `\n_(${data.text_content.length} chars totali — usa read_page per il testo completo)_`;
}
}
}
// GAP-AX: Accessibility Tree — compact rendering per LLM (MCP-style)
if (data.ax_tree) {
const axLines: string[] = [];
const renderAx = (node: Record<string, unknown>, depth: number): void => {
if (depth > 4) return;
const role = String(node.role ?? "");
const name = node.name ? ` "${node.name}"` : "";
const val = node.value ? ` ="${String(node.value).slice(0, 40)}"` : "";
const chk = node.checked === true ? " [✓]" : node.checked === false ? " [ ]" : "";
if (role && role !== "none" && role !== "presentation") {
axLines.push(`${" ".repeat(depth)}${role}${name}${val}${chk}`);
}
const ch = node.children as Record<string, unknown>[] | undefined;
if (ch) ch.slice(0, 12).forEach(c => renderAx(c, depth + 1));
};
renderAx(data.ax_tree, 0);
if (axLines.length > 0) {
const axPreview = axLines.slice(0, 40).join("\n");
const extra = axLines.length > 40 ? `\n…(${axLines.length - 40} nodi in più)` : "";
out += `\n\n**Accessibility Tree (ARIA):**\n\`\`\`\n${axPreview}${extra}\n\`\`\``;
}
}
out += `\n\n_Sessione attiva: \`${sid}\` — usa browser_act per interagire, browser_close per chiudere._`;
return out;
} catch (e) {
return `❌ browser_open: ${e instanceof Error ? e.message : String(e)}`;
}
}
// ─── browser_act — azione su sessione persistente ─────────────────────────
if (name === "browser_act") {
const {
session_id,
actions,
wait_ms = 1000,
take_screenshot = true,
} = args as {
session_id: string;
actions: Array<{ type: string; selector?: string; value?: string; key?: string; ms?: number }>;
wait_ms?: number;
take_screenshot?: boolean;
};
const backendBase = _backendBase();
try {
const ctrl = new AbortController();
const tid = setTimeout(() => ctrl.abort(), 25000);
const res = await fetch(`${backendBase}/api/browser/act`, {
method: "POST",
headers: { "Content-Type": "application/json", ...(_BROWSER_INTERNAL_TOKEN ? { "X-Internal-Token": _BROWSER_INTERNAL_TOKEN } : {}) },
body: JSON.stringify({ session_id, actions, wait_ms, take_screenshot }),
signal: ctrl.signal,
});
clearTimeout(tid);
if (!res.ok) return `❌ browser_act: backend ${res.status}`;
const data = await res.json() as {
ok: boolean; session_id?: string; screenshot_b64?: string;
title?: string; url?: string;
dom?: { text?: string; links?: Array<{text:string;href:string}>; inputs?: Array<{tag:string;label?:string;selector?:string}> };
error?: string;
};
if (!data.ok) return `❌ browser_act: ${data.error ?? "errore sconosciuto"}`;
const actLog = actions.map(a => `${a.type}${a.selector ? `(${a.selector})` : ""}${a.value ? `="${a.value}"` : ""}`).join(" → ");
let out = `🖱️ **Azioni eseguite:** ${actLog}\n**URL:** ${data.url ?? ""}\n\n`;
if (data.screenshot_b64) {
out += `![${data.title ?? ""}](data:image/png;base64,${data.screenshot_b64})\n\n`;
}
if (data.dom?.text?.trim()) {
out += `**Contenuto aggiornato:**\n${data.dom.text.slice(0, 500)}${(data.dom.text.length ?? 0) > 500 ? "…" : ""}`;
}
return out;
} catch (e) {
return `❌ browser_act: ${e instanceof Error ? e.message : String(e)}`;
}
}
// ─── browser_close — chiude sessione persistente ──────────────────────────
if (name === "browser_close") {
const { session_id } = args as { session_id: string };
const backendBase = _backendBase();
try {
const ctrl = new AbortController();
const tid = setTimeout(() => ctrl.abort(), 10000);
const res = await fetch(`${backendBase}/api/browser/close`, {
method: "POST",
headers: { "Content-Type": "application/json", ...(_BROWSER_INTERNAL_TOKEN ? { "X-Internal-Token": _BROWSER_INTERNAL_TOKEN } : {}) },
body: JSON.stringify({ session_id }),
signal: ctrl.signal,
});
clearTimeout(tid);
const data = await res.json() as { ok: boolean };
return data.ok
? `✅ Sessione browser \`${session_id}\` chiusa.`
: `⚠️ Sessione \`${session_id}\` non trovata (già chiusa o scaduta).`;
} catch (e) {
return `❌ browser_close: ${e instanceof Error ? e.message : String(e)}`;
}
}
// ─── browser_screenshot — snapshot visivo sessione attiva ─────────────────
if (name === "browser_screenshot") {
const { session_id, full_page = false } = args as { session_id: string; full_page?: boolean };
const backendBase = _backendBase();
try {
const ctrl = new AbortController();
const tid = setTimeout(() => ctrl.abort(), 15000);
const qp = full_page ? "?full_page=true" : "";
const res = await fetch(
`${backendBase}/api/browser/screenshot/${encodeURIComponent(session_id)}${qp}`,
{ signal: ctrl.signal },
);
clearTimeout(tid);
if (!res.ok) return `❌ browser_screenshot: backend ${res.status}`;
const data = await res.json() as {
ok: boolean; session_id?: string; screenshot_b64?: string;
title?: string; url?: string; error?: string;
};
if (!data.ok) return `❌ browser_screenshot: ${data.error ?? "errore sconosciuto"}`;
let out = `📸 **Screenshot sessione \`${session_id}\`**`;
if (data.title) out += `\nPagina: **${data.title}**`;
if (data.url) out += ` — ${data.url}`;
if (data.screenshot_b64) {
out += `\n![screenshot](data:image/png;base64,${data.screenshot_b64})`;
}
return out;
} catch (e) {
return `❌ browser_screenshot: ${e instanceof Error ? e.message : String(e)}`;
}
}
// ─── browser_session_open — alias per browser_open con sessione ──────────────
if (name === "browser_session_open") {
const { url, session_id: _sid, headless: _hl, viewport: _vp } = args as {
url: string; session_id?: string; headless?: boolean; viewport?: { width: number; height: number };
};
const width = _vp?.width ?? 1280;
return tryExecuteBrowser("browser_open", { url, actions: [], mobile: false, width });
}
// ─── browser_session_act — alias per browser_act con singola azione ──────────
if (name === "browser_session_act") {
const { session_id, action, selector, value, timeout: _to } = args as {
session_id: string; action: string; selector?: string; value?: string; timeout?: number;
};
const mappedAction: Record<string, unknown> = { type: action };
if (selector) mappedAction.selector = selector;
if (value) mappedAction.value = value;
if (_to) mappedAction.ms = _to;
return tryExecuteBrowser("browser_act", { session_id, actions: [mappedAction], take_screenshot: true });
}
return null;
}