File size: 3,371 Bytes
1b0b730
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
import { chromium } from 'playwright';
import fs from 'node:fs/promises';

const root = new URL('../..', import.meta.url).pathname;
const url = process.argv[2] ?? 'https://mike0021-minicpm5-2b-webgpu-pi-http.static.hf.space/';
const gist = 'https://gist.githubusercontent.com/mbostock/4063269/raw/5e72bffb17d12e15fb9da3a8b6794ac19db802a2/README.md';
const result = { date: new Date().toISOString(), url, prompt: `Fetch ${gist} and tell me what kind of chart it describes.`, errors: [], requests: [] };
const context = await chromium.launchPersistentContext(root + '/validation/http-browser-profile', {
  executablePath: '/usr/bin/chromium', headless: true, viewport: { width: 1440, height: 1000 },
  args: ['--enable-unsafe-webgpu', '--use-angle=vulkan', '--enable-features=Vulkan', '--disable-vulkan-surface'],
});
context.setDefaultTimeout(120_000);
const page = await context.newPage();
page.on('pageerror', error => result.errors.push(error.message));
context.on('request', request => {
  if (request.url().startsWith('https://gist.githubusercontent.com/')) result.requests.push({ url: request.url(), method: request.method() });
});
let timer;
try {
  await page.goto(url);
  await page.waitForFunction(() => window.browserPi?.terminal);
  await page.evaluate(() => window.browserPi.ready);
  await page.waitForFunction(() => !window.browserPi.busy);
  await page.waitForTimeout(300);
  const loadStart = Date.now();
  await page.evaluate(() => window.browserPi.load());
  result.modelLoadMs = Date.now() - loadStart;
  await page.waitForFunction(() => !window.browserPi.busy);
  await page.locator('#new-chat').click();
  await page.waitForFunction(() => !window.browserPi.busy);
  await page.waitForTimeout(300);
  const start = Date.now();
  const pending = page.evaluate(prompt => window.browserPi.prompt(prompt), result.prompt);
  try {
    await Promise.race([pending, new Promise((_, reject) => {
      timer = setTimeout(() => reject(Error('Gist trial exceeded 120 seconds')), 120_000);
    })]);
  } catch (error) {
    await page.evaluate(() => window.browserPi.rpc('stop'));
    await pending.catch(() => {});
    throw error;
  } finally { clearTimeout(timer); }
  result.elapsedMs = Date.now() - start;
  result.state = await page.evaluate(() => window.browserPi.rpc('snapshot'));
  result.calls = result.state.messages.filter(m => m.role === 'assistant').flatMap(m => m.content.filter(c => c.type === 'toolCall'));
  result.toolResults = result.state.messages.filter(m => m.role === 'toolResult');
  result.final = result.state.messages.filter(m => m.role === 'assistant').at(-1)?.content.filter(c => c.type === 'text').map(c => c.text).join('\n') ?? '';
  result.passed = result.requests.some(r => r.url === gist) &&
    result.toolResults.some(t => !t.isError && t.toolName === 'bash' && /bubble charts/i.test(JSON.stringify(t.content))) &&
    /bubble chart/i.test(result.final) && !result.errors.length;
  console.log(JSON.stringify({ passed: result.passed, elapsedMs: result.elapsedMs, final: result.final, calls: result.calls }));
} catch (error) {
  result.failure = error.stack;
  result.passed = false;
  console.error(error.stack);
} finally {
  clearTimeout(timer);
  await fs.writeFile(root + '/validation/http-any-https.json', JSON.stringify(result, null, 2) + '\n');
  await context.close();
  if (!result.passed) process.exitCode = 1;
}