Spaces:
Paused
Paused
File size: 3,209 Bytes
bcf46c3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | #!/usr/bin/env node
import { Command } from 'commander';
import dotenv from 'dotenv';
import path from 'path';
import { OpticparseClient, ScrapeRequest } from './index';
// Load default environment variables from the current working directory's .env file
dotenv.config();
const program = new Command();
program
.name('opticparse')
.description('Command-line utility to scrape dynamic web pages using AI Vision & Gemini.')
.version('1.0.0');
program
.command('scrape')
.description('Execute a visual scrape operation against a webpage.')
.requiredOption('-u, --url <string>', 'The URL of the webpage to scrape.')
.requiredOption('-q, --query <string>', 'The structured query explaining what JSON data to extract.')
.option('-k, --key <string>', 'API key credential. Falls back to OPTICPARSE_API_KEY env variable.')
.option('-a, --api-url <string>', 'Custom target API server host URL.')
.option('--width <number>', 'Browser viewport width.', '1280')
.option('--height <number>', 'Browser viewport height.', '800')
.option('--wait-until <string>', 'Playwright wait criteria: networkidle, load, or domcontentloaded.', 'networkidle')
.option('-t, --timeout <number>', 'Scraper connection & execution timeout in milliseconds.', '90000')
.action(async (options) => {
try {
const apiKey = options.key || process.env.OPTICPARSE_API_KEY;
const apiUrl = options.apiUrl || process.env.OPTICPARSE_API_URL;
if (!apiKey) {
console.error('\x1b[31mError: API Key is missing.\x1b[0m');
console.error('Please specify the key using the -k/--key flag, or set the OPTICPARSE_API_KEY environment variable.');
process.exit(1);
}
const client = new OpticparseClient({
apiKey,
apiUrl,
});
const scrapeRequest: ScrapeRequest = {
targetUrl: options.url,
extractionQuery: options.query,
viewportWidth: parseInt(options.width, 10),
viewportHeight: parseInt(options.height, 10),
waitUntil: options.waitUntil as any,
timeout: parseInt(options.timeout, 10),
};
console.log(`\x1b[36mInitiating visual scraping of:\x1b[0m ${options.url}`);
console.log(`\x1b[36mQuery:\x1b[0m "${options.query}"`);
const startTime = Date.now();
const result = await client.scrape(scrapeRequest);
const duration = ((Date.now() - startTime) / 1000).toFixed(2);
console.log(`\x1b[32mSuccessfully scraped in ${duration}s!\x1b[0m`);
console.log('\n\x1b[35m=== Extracted JSON Result ===\x1b[0m');
console.log(JSON.stringify(result, null, 2));
console.log('\x1b[35m=============================\x1b[0m');
} catch (error: any) {
console.error('\n\x1b[31m=== Scraper Execution Failed ===\x1b[0m');
console.error(`\x1b[31mMessage:\x1b[0m ${error.message}`);
if (error.statusCode) {
console.error(`\x1b[31mHTTP Status Code:\x1b[0m ${error.statusCode}`);
}
if (error.details) {
console.error(`\x1b[31mServer Details:\x1b[0m ${error.details}`);
}
console.error('\x1b[31m================================\x1b[0m');
process.exit(1);
}
});
program.parse(process.argv);
|