File size: 2,488 Bytes
c92aa92 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 | import { fetchWebContent } from '../engines/web/index.js';
type CliArgs = {
url: string;
maxChars: number;
previewChars: number;
};
function parseArgs(argv: string[]): CliArgs {
const parsed: CliArgs = {
url: 'https://awiki.ai',
maxChars: 30000,
previewChars: 600
};
for (const arg of argv) {
if (arg.startsWith('--url=')) {
parsed.url = arg.slice('--url='.length);
} else if (arg.startsWith('--maxChars=')) {
const value = Number(arg.slice('--maxChars='.length));
if (Number.isFinite(value) && value > 0) {
parsed.maxChars = value;
}
} else if (arg.startsWith('--previewChars=')) {
const value = Number(arg.slice('--previewChars='.length));
if (Number.isFinite(value) && value > 0) {
parsed.previewChars = value;
}
}
}
return parsed;
}
async function main(): Promise<void> {
const args = parseArgs(process.argv.slice(2));
console.log('Live fetch test config:', args);
const start = Date.now();
try {
const result = await fetchWebContent(args.url, args.maxChars);
const durationMs = Date.now() - start;
console.log('\nFetch metadata:');
console.log(`- url: ${result.url}`);
console.log(`- finalUrl: ${result.finalUrl}`);
console.log(`- contentType: ${result.contentType}`);
console.log(`- title: ${result.title || '(empty)'}`);
console.log(`- truncated: ${result.truncated}`);
console.log(`- contentLength: ${result.content.length}`);
console.log(`- durationMs: ${durationMs}`);
if (!result.content.trim()) {
throw new Error('Fetched content is empty');
}
const preview = result.content.slice(0, args.previewChars);
console.log('\nContent preview:\n');
console.log(preview);
console.log('\nLive fetch test passed.');
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
console.error('\nLive fetch test failed:', message);
if (/EAI_AGAIN|getaddrinfo|TLS|socket/i.test(message)) {
console.error('Network/DNS issue detected. If needed, enable proxy: USE_PROXY=true PROXY_URL=http://127.0.0.1:7890');
}
process.exit(1);
}
}
main().catch((error) => {
console.error('Unexpected error:', error);
process.exit(1);
});
|