process-all-events-webfetch.js

const fs = require('fs');
const path = require('path');
const https = require('https');

const dataDir = path.join(__dirname, 'data');
if (!fs.existsSync(dataDir)) {
    fs.mkdirSync(dataDir, { recursive: true });
}

const startId = 6;
const endId = 415;
let processed = 0;
let saved = 0;

function fetchEventPage(id) {
    return new Promise((resolve, reject) => {
        const options = {
            hostname: 'neliota.astro.noa.gr',
            path: `/DataAccess/EventDetails/${id}?AspxAutoDetectCookieSupport=1`,
            method: 'GET',
            headers: {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
            }
        };
        
        let html = '';
        const req = https.request(options, (res) => {
            res.on('data', (chunk) => {
                html += chunk;
            });
            
            res.on('end', () => {
                if (res.statusCode === 200 && html.includes('Detected NEO Lunar Impact Event')) {
                    resolve(html);
                } else {
                    resolve(null);
                }
            });
        });
        
        req.on('error', (err) => {
            console.error(`Error fetching event ${id}:`, err.message);
            resolve(null);
        });
        
        req.end();
    });
}

function parseEventData(html, eventId) {
    const data = {
        id: eventId
    };
    
    const idMatch = html.match(/ID:\s*([^\s<]+)/);
    if (idMatch) data.id = idMatch[1];
    
    const airmassMatch = html.match(/Airmass:\s*<\/td>\s*<td[^>]*>\s*([\d.]+)/);
    if (airmassMatch) data.airmass = parseFloat(airmassMatch[1]);
    
    const altitudeMatch = html.match(/Altitude \(deg\):\s*<\/td>\s*<td[^>]*>\s*([\d.-]+)/);
    if (altitudeMatch) data.altitude_deg = parseFloat(altitudeMatch[1]);
    
    const azimuthMatch = html.match(/Azimuth \(deg\):\s*<\/td>\s*<td[^>]*>\s*([\d.-]+)/);
    if (azimuthMatch) data.azimuth_deg = parseFloat(azimuthMatch[1]);
    
    const dateMatch = html.match(/UT Date \(DD\/MM\/YYYY\):\s*<\/td>\s*<td[^>]*>\s*([^\s<]+)/);
    if (dateMatch) data.ut_date = dateMatch[1];
    
    const timeMatch = html.match(/UT Time:\s*<\/td>\s*<td[^>]*>\s*([^\s<]+)/);
    if (timeMatch) data.ut_time = timeMatch[1];
    
    const magRMatch = html.match(/R \(mag\):\s*<\/td>\s*<td[^>]*>\s*([^\s<]+(?:\s*±\s*[^\s<]+)?)/);
    if (magRMatch) data.mag_r = magRMatch[1];
    
    const magIMatch = html.match(/I \(mag\):\s*<\/td>\s*<td[^>]*>\s*([^\s<]+(?:\s*±\s*[^\s<]+)?)/);
    if (magIMatch) data.mag_i = magIMatch[1];
    
    const longMatch = html.match(/Lunar Long \(deg\):\s*<\/td>\s*<td[^>]*>\s*([\d.-]+)/);
    if (longMatch) data.lunar_long_deg = parseFloat(longMatch[1]);
    
    const latMatch = html.match(/Lunar Lat \(deg\):\s*<\/td>\s*<td[^>]*>\s*([\d.-]+)/);
    if (latMatch) data.lunar_lat_deg = parseFloat(latMatch[1]);
    
    const durationMatch = html.match(/Duration \(sec\):\s*<\/td>\s*<td[^>]*>\s*([\d.]+)/);
    if (durationMatch) data.duration_sec = parseFloat(durationMatch[1]);
    
    const camerasMatch = html.match(/Number of Cameras:\s*<\/td>\s*<td[^>]*>\s*(\d+)/);
    if (camerasMatch) data.number_of_cameras = parseInt(camerasMatch[1]);
    
    const sizeMatch = html.match(/Media Size:\s*(\d+)\s*MB/);
    if (sizeMatch) data.media_size_mb = parseInt(sizeMatch[1]);
    
    return data;
}

function createEventFolder(eventId, data, html) {
    const eventDir = path.join(dataDir, `event-${eventId}`);
    const imagesDir = path.join(eventDir, 'images');
    
    if (!fs.existsSync(eventDir)) {
        fs.mkdirSync(eventDir, { recursive: true });
    }
    if (!fs.existsSync(imagesDir)) {
        fs.mkdirSync(imagesDir, { recursive: true });
    }
    
    const htmlPath = path.join(eventDir, `event-${eventId}.html`);
    fs.writeFileSync(htmlPath, html, 'utf8');
    
    const csvHeader = 'id,airmass,altitude_deg,azimuth_deg,ut_date,ut_time,mag_r,mag_i,lunar_long_deg,lunar_lat_deg,duration_sec,number_of_cameras,media_size_mb';
    const csvRow = `${data.id || eventId},${data.airmass || ''},${data.altitude_deg || ''},${data.azimuth_deg || ''},${data.ut_date || ''},${data.ut_time || ''},"${data.mag_r || ''}","${data.mag_i || ''}",${data.lunar_long_deg || ''},${data.lunar_lat_deg || ''},${data.duration_sec || ''},${data.number_of_cameras || ''},${data.media_size_mb || ''}`;
    const csvContent = csvHeader + '\n' + csvRow;
    
    const csvPath = path.join(eventDir, `event-${eventId}.csv`);
    fs.writeFileSync(csvPath, csvContent, 'utf8');
    
    return eventDir;
}

async function processAllEvents() {
    console.log('Starting to process all NELIOTA events...');
    console.log(`Event ID range: ${startId} to ${endId}`);
    console.log('This may take a while...\n');
    
    for (let id = startId; id <= endId; id++) {
        processed++;
        process.stdout.write(`Processing event ${id}... `);
        
        const html = await fetchEventPage(id);
        
        if (html) {
            const eventData = parseEventData(html, id);
            const eventDir = createEventFolder(id, eventData, html);
            saved++;
            console.log(`✓ Saved to ${eventDir}`);
        } else {
            console.log('✗ Not found');
        }
        
        await new Promise(resolve => setTimeout(resolve, 300));
    }
    
    console.log(`\nProcessing complete!`);
    console.log(`Total events processed: ${processed}`);
    console.log(`Events successfully saved: ${saved}`);
}

processAllEvents();