import axios from 'axios'; import { fetchWebContent } from '../engines/web/index.js'; type TestCase = { name: string; run: () => Promise; }; const originalAxiosGet = axios.get.bind(axios); const originalAxiosHead = axios.head.bind(axios); function installAxiosMock(): void { (axios as any).head = async (url: string) => { if (url.endsWith('/too-large.md')) { return { headers: { 'content-length': String(5 * 1024 * 1024) }, request: { res: { responseUrl: url } } }; } if (url.endsWith('/long.md')) { return { headers: { 'content-length': String(1024) }, request: { res: { responseUrl: url } } }; } return { headers: {}, request: { res: { responseUrl: url } } }; }; (axios as any).get = async (url: string) => { if (url.endsWith('/skill.md')) { return { headers: { 'content-type': 'text/plain; charset=utf-8' }, data: '# Skill Title\n\nThis is a markdown test document.', request: { res: { responseUrl: url } } }; } if (url.endsWith('/page')) { return { headers: { 'content-type': 'text/html; charset=utf-8' }, data: ` Skill Page

Skill Page

${'Skill body content '.repeat(12)}

`, request: { res: { responseUrl: `${url}?from=test` } } }; } if (url.endsWith('/long.md')) { return { headers: { 'content-type': 'text/markdown; charset=utf-8' }, data: `# Long\n\n${'x'.repeat(6000)}`, request: { res: { responseUrl: url } } }; } if (url.endsWith('/too-large.md')) { throw new Error('GET should not be called when HEAD indicates oversized response'); } if (url.endsWith('/spa')) { return { headers: { 'content-type': 'text/html; charset=utf-8' }, data: ` SPA Site

`, request: { res: { responseUrl: url } } }; } throw new Error(`Unexpected mocked URL: ${url}`); }; } function restoreAxiosMock(): void { (axios as any).get = originalAxiosGet; (axios as any).head = originalAxiosHead; } function assert(condition: unknown, message: string): void { if (!condition) { throw new Error(message); } } async function runCase(testCase: TestCase): Promise { try { await testCase.run(); console.log(`✅ ${testCase.name}`); return true; } catch (error) { console.error(`❌ ${testCase.name}:`, error); return false; } } async function main(): Promise { installAxiosMock(); const testCases: TestCase[] = [ { name: 'should parse markdown content by .md URL', run: async () => { const result = await fetchWebContent('https://example.com/skill.md', 5000); assert(result.title === '', 'markdown title should be empty'); assert(result.content.includes('Skill Title'), 'markdown content should keep source text'); assert(result.truncated === false, 'markdown should not be truncated'); } }, { name: 'should extract text and title from html page', run: async () => { const result = await fetchWebContent('https://example.com/page', 5000); assert(result.title === 'Skill Page', 'html title should be extracted'); assert(result.finalUrl.endsWith('/page?from=test'), 'finalUrl should follow redirect target'); assert(result.content.includes('Skill body content'), 'html content should be extracted'); } }, { name: 'should truncate long content when maxChars is small', run: async () => { const result = await fetchWebContent('https://example.com/long.md', 1200); assert(result.truncated === true, 'long content should be truncated'); assert(result.content.includes('[...truncated '), 'truncation marker should exist'); } }, { name: 'should fallback to metadata for js-rendered html pages', run: async () => { const result = await fetchWebContent('https://example.com/spa', 5000); assert(result.title === 'SPA Site', 'title should be extracted from html'); assert(result.content.includes('Rendered by JS runtime'), 'meta description fallback should be used'); } }, { name: 'should reject non-http protocol', run: async () => { let failed = false; try { await fetchWebContent('file:///tmp/skill.md', 5000); } catch { failed = true; } assert(failed, 'file protocol should be rejected'); } }, { name: 'should reject private/local network targets', run: async () => { let failed = false; try { await fetchWebContent('http://127.0.0.1/private', 5000); } catch { failed = true; } assert(failed, 'private network target should be rejected'); } }, { name: 'should reject oversized response by content-length', run: async () => { let failed = false; try { await fetchWebContent('https://example.com/too-large.md', 5000); } catch { failed = true; } assert(failed, 'oversized response should be rejected'); } } ]; let passed = 0; for (const testCase of testCases) { if (await runCase(testCase)) { passed += 1; } } restoreAxiosMock(); const total = testCases.length; console.log(`\nResult: ${passed}/${total} passed`); if (passed !== total) { process.exit(1); } } main().catch((error) => { restoreAxiosMock(); console.error('❌ test-web-content failed:', error); process.exit(1); });