Hugging Face's logo

Spaces:
aigodking
/
crawler-trial
Running

App Files Community
crawler-trial / index.html
aigodking's picture
aigodking
Add 2 files
c0a77b6 verified
Raw
History Blame Contribute Delete
57.4 kB
<!DOCTYPE html>
<html lang="en">
<head>
 <meta charset="UTF-8">
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
 <title>Herholdts Frontend Web Crawler</title>
 <script src="https://cdn.tailwindcss.com"></script>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js"></script>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/FileSaver.js/2.0.5/FileSaver.min.js"></script>
 <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css">
 <style>
 .progress-bar { transition: width 0.3s ease; }
 .product-card { transition: all 0.2s ease; }
 .product-card:hover { transform: translateY(-5px); box-shadow: 0 10px 20px rgba(0,0,0,0.1); }
 .blink { animation: blink 1s infinite; }
 @keyframes blink { 0% { opacity: 1; } 50% { opacity: 0.5; } 100% { opacity: 1; } }
 /* Ensure sticky header */
 .sticky-header { position: sticky; top: 0; z-index: 10; }
 /* Style for log messages */
 .log-entry { margin-bottom: 4px; line-height: 1.4; }
 .log-time { color: #a0aec0; margin-right: 8px; } /* Gray */
 .log-info { color: #9f7aea; } /* Purple */
 .log-success { color: #48bb78; } /* Green */
 .log-warning { color: #ecc94b; } /* Yellow */
 .log-error { color: #f56565; } /* Red */
 .log-debug { color: #63b3ed; } /* Blue */
 </style>
</head>
<body class="bg-gray-100">
 <div class="container mx-auto px-4 py-8">
 <!-- Important Disclaimer -->
 <div class="bg-red-100 border border-red-400 text-red-700 px-4 py-3 rounded relative mb-6" role="alert">
 <strong class="font-bold">Important Limitations:</strong>
 <span class="block sm:inline"> This is a **frontend-only** crawler. It relies on the public `allorigins.win` proxy due to browser security restrictions (CORS). This has drawbacks:
 <ul class="list-disc list-inside mt-1">
 <li>**Reliability:** The proxy can be slow, rate-limited, or blocked.</li>
 <li>**JavaScript Rendering:** It cannot run JavaScript on the target site, potentially missing dynamic content.</li>
 <li>**Anti-Blocking:** True anti-blocking (like IP rotation) is not possible from the frontend.</li>
 <li>**Use a proper backend crawler (Python/Node.js with libraries like Scrapy, Playwright, or crawl4ai) for serious scraping.**</li>
 </ul>
 </span>
 </div>
<div class="bg-white rounded-xl shadow-lg overflow-hidden">
 <!-- Header -->
 <div class="bg-indigo-600 px-6 py-4 text-white sticky-header">
 <div class="flex justify-between items-center">
 <h1 class="text-2xl font-bold">
 <i class="fas fa-spider mr-2"></i> Herholdts Frontend Crawler
 </h1>
 <div class="flex space-x-2">
 <span id="status" class="bg-indigo-800 px-3 py-1 rounded-full text-sm">Ready</span>
 <span id="userAgentBadge" class="bg-teal-700 px-3 py-1 rounded-full text-sm text-xs" title="Current User Agent type sent to proxy">UA: Chrome</span>
 </div>
 </div>
 <p class="text-indigo-200 mt-1 text-sm">Basic crawler using CORS proxy for herholdts.co.za</p>
 </div>
<!-- Main Content -->
 <div class="p-6">
 <!-- Controls Section -->
 <div class="bg-gray-50 p-4 rounded-lg mb-6 border border-gray-200">
 <div class="grid grid-cols-1 md:grid-cols-3 gap-4">
 <div>
 <label for="startUrl" class="block text-sm font-medium text-gray-700 mb-1">Start URL</label>
 <div class="flex">
 <input type="text" id="startUrl" value="https://herholdts.co.za/"
 class="flex-1 rounded-l-md border-gray-300 shadow-sm focus:border-indigo-500 focus:ring-indigo-500 px-3 py-2 border text-sm" placeholder="https://herholdts.co.za/">
 <button onclick="testConnection()" class="bg-blue-500 text-white px-3 rounded-r-md hover:bg-blue-600 text-sm" title="Test connection via proxy">
 <i class="fas fa-plug"></i>
 </button>
 </div>
 </div>
 <div>
 <label for="maxPages" class="block text-sm font-medium text-gray-700 mb-1">Max Pages</label>
 <input type="number" id="maxPages" value="50" min="1" max="500"
 class="w-full rounded-md border-gray-300 shadow-sm focus:border-indigo-500 focus:ring-indigo-500 px-3 py-2 border text-sm">
 </div>
 <div>
 <label for="crawlSpeed" class="block text-sm font-medium text-gray-700 mb-1">Crawl Delay</label>
 <select id="crawlSpeed" class="w-full rounded-md border-gray-300 shadow-sm focus:border-indigo-500 focus:ring-indigo-500 px-3 py-2 border text-sm">
 <option value="slow">Slow (5-10s)</option>
 <option value="medium" selected>Medium (2-5s)</option>
 <option value="fast">Fast (1-2s)</option>
 <option value="random">Random (1-10s)</option>
 </select>
 </div>
 </div>
<div class="mt-4 grid grid-cols-1 md:grid-cols-2 gap-4">
 <div>
 <label class="block text-sm font-medium text-gray-700 mb-1">Options</label>
 <div class="flex flex-wrap gap-x-4 gap-y-2">
 <label class="inline-flex items-center">
 <input type="checkbox" id="rotateUA" checked class="rounded border-gray-300 text-indigo-600 shadow-sm focus:ring-indigo-200">
 <span class="ml-2 text-sm">Rotate User Agents (sent to proxy)</span>
 </label>
 <label class="inline-flex items-center">
 <input type="checkbox" id="respectRobots" checked class="rounded border-gray-300 text-indigo-600 shadow-sm focus:ring-indigo-200">
 <span class="ml-2 text-sm">Attempt Respect robots.txt</span>
 </label>
 </div>
 <p class="text-xs text-gray-500 mt-1">Note: Proxies cannot be used effectively from the frontend.</p>
 </div>
 <div>
 <label class="block text-sm font-medium text-gray-700 mb-1">Actions</label>
 <div class="flex flex-wrap gap-2">
 <button onclick="startCrawling()" id="startBtn" class="bg-green-600 hover:bg-green-700 text-white px-4 py-2 rounded-md flex items-center text-sm">
 <i class="fas fa-play mr-2"></i> Start
 </button>
 <button onclick="pauseCrawling()" id="pauseBtn" disabled class="bg-yellow-500 hover:bg-yellow-600 text-white px-4 py-2 rounded-md flex items-center text-sm">
 <i class="fas fa-pause mr-2"></i> Pause
 </button>
 <button onclick="stopCrawling()" id="stopBtn" disabled class="bg-red-600 hover:bg-red-700 text-white px-4 py-2 rounded-md flex items-center text-sm">
 <i class="fas fa-stop mr-2"></i> Stop
 </button>
 <button onclick="exportData('csv')" class="bg-blue-600 hover:bg-blue-700 text-white px-4 py-2 rounded-md flex items-center text-sm">
 <i class="fas fa-file-csv mr-2"></i> CSV
 </button>
 <button onclick="exportData('excel')" class="bg-green-700 hover:bg-green-800 text-white px-4 py-2 rounded-md flex items-center text-sm">
 <i class="fas fa-file-excel mr-2"></i> Excel
 </button>
 <button onclick="exportData('json')" class="bg-purple-600 hover:bg-purple-700 text-white px-4 py-2 rounded-md flex items-center text-sm">
 <i class="fas fa-file-code mr-2"></i> JSON
 </button>
 </div>
 </div>
 </div>
 </div>
<!-- Stats Section -->
 <div class="grid grid-cols-2 md:grid-cols-5 gap-4 mb-6">
 <div class="bg-white p-3 rounded-lg shadow border border-gray-200">
 <div class="text-gray-500 text-xs uppercase tracking-wider">Crawled</div>
 <div id="pagesCrawled" class="text-xl font-bold text-indigo-600">0</div>
 </div>
 <div class="bg-white p-3 rounded-lg shadow border border-gray-200">
 <div class="text-gray-500 text-xs uppercase tracking-wider">Products</div>
 <div id="productsFound" class="text-xl font-bold text-green-600">0</div>
 </div>
 <div class="bg-white p-3 rounded-lg shadow border border-gray-200">
 <div class="text-gray-500 text-xs uppercase tracking-wider">Images</div>
 <div id="imagesFound" class="text-xl font-bold text-blue-600">0</div>
 </div>
 <div class="bg-white p-3 rounded-lg shadow border border-gray-200">
 <div class="text-gray-500 text-xs uppercase tracking-wider">Success Rate</div>
 <div id="successRate" class="text-xl font-bold">0%</div>
 </div>
 <div class="bg-white p-3 rounded-lg shadow border border-gray-200 col-span-2 md:col-span-1">
 <div class="text-gray-500 text-xs uppercase tracking-wider">Current UA</div>
 <div id="currentUA" class="text-xs font-mono truncate pt-1" title="N/A">N/A</div>
 </div>
 </div>
<!-- Progress Section -->
 <div class="mb-6">
 <div class="flex justify-between mb-1">
 <span class="text-sm font-medium text-gray-700">Crawling Progress</span>
 <span id="progressText" class="text-sm font-medium text-gray-700">0/0 pages</span>
 </div>
 <div class="w-full bg-gray-200 rounded-full h-2.5">
 <div id="progressBar" class="bg-indigo-600 h-2.5 rounded-full progress-bar" style="width: 0%"></div>
 </div>
 </div>
<!-- Logs Section -->
 <div class="mb-6">
 <div class="flex justify-between items-center mb-2">
 <h3 class="font-medium text-gray-700">
 <i class="fas fa-terminal mr-2"></i> Crawler Logs
 </h3>
 <div>
 <button onclick="clearLogs()" class="text-sm text-gray-500 hover:text-gray-700 mr-3" title="Clear logs">
 <i class="fas fa-trash-alt mr-1"></i> Clear
 </button>
 <button onclick="toggleDebugMode()" id="debugBtn" class="text-sm bg-gray-200 hover:bg-gray-300 text-gray-700 px-2 py-1 rounded" title="Toggle detailed debug messages">
 <i class="fas fa-bug mr-1"></i> Debug: OFF
 </button>
 </div>
 </div>
 <div id="logs" class="bg-gray-800 text-sm p-3 rounded-lg h-48 overflow-y-auto font-mono">
 <div class="log-entry log-info"><span class="log-time"></span>System initialized. Ready to start crawling.</div>
 </div>
 </div>
<!-- Results Section -->
 <div>
 <div class="flex flex-col md:flex-row justify-between md:items-center mb-3 gap-2">
 <h3 class="font-medium text-gray-700">
 <i class="fas fa-box-open mr-2"></i> Extracted Products (<span id="productCount">0</span>)
 </h3>
 <div class="flex flex-col sm:flex-row gap-2">
 <input type="text" id="productSearch" placeholder="Search products..."
 class="text-sm rounded-md border-gray-300 shadow-sm focus:border-indigo-500 focus:ring-indigo-500 px-3 py-1 border w-full sm:w-auto">
 <select id="categoryFilter" class="text-sm rounded-md border-gray-300 shadow-sm focus:border-indigo-500 focus:ring-indigo-500 px-2 py-1 border w-full sm:w-auto">
 <option value="">All Categories</option>
 </select>
 </div>
 </div>
 <div id="productGrid" class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4">
 <!-- Products will be displayed here -->
 <div id="noProductsPlaceholder" class="col-span-full text-center py-10 text-gray-400">
 <i class="fas fa-box-open text-4xl mb-2"></i>
 <p>No products extracted yet</p>
 </div>
 </div>
 </div>
 </div>
 </div>
<!-- Footer -->
 <div class="text-center text-gray-500 text-xs mt-6">
 Crawler UI Template | Use responsibly and respect website terms of service.
 </div>
 </div>
<script>
 // --- Configuration ---
 const PROXY_URL = 'https://api.allorigins.win/get?url='; // CORS Proxy
 const TARGET_HOSTNAME = 'herholdts.co.za'; // Only follow links within this host
 // --- Global State ---
 let isCrawling = false;
 let isPaused = false;
 let isDebug = false;
 let crawlController = null; // AbortController instance
 let crawledPages = 0;
 let maxPagesToCrawl = 50;
 let successfulRequests = 0;
 let failedRequests = 0;
 let products = [];
 let categories = new Set();
 let images = new Set(); // Use Set for unique image URLs
 let queue = [];
 let visited = new Set();
 let robotsAllowed = true; // Assume allowed until checked
 let delaySettings = { min: 2000, max: 5000 }; // Default medium speed
 // --- User Agents ---
 const userAgents = [ /* ... (keep the same list as before) ... */
 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
 "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0",
 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:89.0) Gecko/20100101 Firefox/89.0",
 "Mozilla/5.0 (X11; Linux i686; rv:89.0) Gecko/20100101 Firefox/89.0",
 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15",
 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Edg/91.0.864.59",
 "Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
 "Mozilla/5.0 (Linux; Android 10; SM-G981B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.120 Mobile Safari/537.36"
 ];
 // --- DOM Elements ---
 const getEl = (id) => document.getElementById(id);
 const logsElement = getEl('logs');
 const progressBar = getEl('progressBar');
 const progressText = getEl('progressText');
 const pagesCrawledElement = getEl('pagesCrawled');
 const productsFoundElement = getEl('productsFound');
 const imagesFoundElement = getEl('imagesFound');
 const productCountElement = getEl('productCount');
 const productGrid = getEl('productGrid');
 const categoryFilter = getEl('categoryFilter');
 const productSearch = getEl('productSearch');
 const statusElement = getEl('status');
 const startBtn = getEl('startBtn');
 const pauseBtn = getEl('pauseBtn');
 const stopBtn = getEl('stopBtn');
 const userAgentBadge = getEl('userAgentBadge');
 const currentUAElement = getEl('currentUA');
 const successRateElement = getEl('successRate');
 const debugBtn = getEl('debugBtn');
 const rotateUACheckbox = getEl('rotateUA');
 const respectRobotsCheckbox = getEl('respectRobots');
 const startUrlInput = getEl('startUrl');
 const maxPagesInput = getEl('maxPages');
 const crawlSpeedSelect = getEl('crawlSpeed');
 const noProductsPlaceholder = getEl('noProductsPlaceholder');
 // --- Initialization ---
 document.addEventListener('DOMContentLoaded', () => {
 updateCrawlSpeed(); // Set initial delay
 addLog('System initialized. Ready to start crawling.', 'info');
 productSearch.addEventListener('input', filterProducts);
 categoryFilter.addEventListener('change', filterProducts);
 crawlSpeedSelect.addEventListener('change', updateCrawlSpeed);
 });
 function updateCrawlSpeed() {
 const speed = crawlSpeedSelect.value;
 if (speed === 'slow') delaySettings = { min: 5000, max: 10000 };
 else if (speed === 'medium') delaySettings = { min: 2000, max: 5000 };
 else if (speed === 'fast') delaySettings = { min: 1000, max: 2000 };
 else delaySettings = { min: 1000, max: 10000 }; // random
 addLog(`Crawl delay set to: ${speed} (${delaySettings.min/1000}-${delaySettings.max/1000}s)`, 'debug');
 }
 // --- Utility Functions ---
 function getRandomUserAgent() {
 return userAgents[Math.floor(Math.random() * userAgents.length)];
 }
 function getCurrentUserAgent() {
 return rotateUACheckbox.checked ? getRandomUserAgent() : userAgents[0];
 }
 function setCurrentUAUI(ua) {
 const uaShort = ua.substring(0, ua.indexOf('(') > 0 ? ua.indexOf('(') : 30) + '...';
 currentUAElement.textContent = uaShort;
 currentUAElement.title = ua;
 if (ua.includes('Chrome/')) userAgentBadge.textContent = 'UA: Chrome';
 else if (ua.includes('Firefox/')) userAgentBadge.textContent = 'UA: Firefox';
 else if (ua.includes('Safari/')) userAgentBadge.textContent = 'UA: Safari';
 else if (ua.includes('Edg/')) userAgentBadge.textContent = 'UA: Edge';
 else if (ua.includes('Mobile')) userAgentBadge.textContent = 'UA: Mobile';
 else userAgentBadge.textContent = 'UA: Custom';
 }
 function getDelay() {
 return Math.floor(Math.random() * (delaySettings.max - delaySettings.min + 1)) + delaySettings.min;
 }
 function sleep(ms) {
 return new Promise(resolve => setTimeout(resolve, ms));
 }
 function normalizeUrl(url) {
 try {
 const urlObj = new URL(url);
 // Remove hash and trailing slash for consistency
 return urlObj.origin + urlObj.pathname.replace(/\/$/, '') + urlObj.search;
 } catch (e) {
 return null; // Invalid URL
 }
 }
 // --- Logging ---
 function addLog(message, type = 'info') {
 if (type === 'debug' && !isDebug) return;
 const time = new Date().toLocaleTimeString();
 const typeClass = `log-${type}`;
 const logEntry = document.createElement('div');
 logEntry.className = `log-entry ${typeClass}`;
 logEntry.innerHTML = `<span class="log-time">${time}</span>${message}`;
 logsElement.appendChild(logEntry);
 logsElement.scrollTop = logsElement.scrollHeight; // Auto-scroll
 }
 function clearLogs() {
 logsElement.innerHTML = '';
 addLog('Logs cleared', 'info');
 }
 function toggleDebugMode() {
 isDebug = !isDebug;
 debugBtn.innerHTML = `<i class="fas fa-bug mr-1"></i> Debug: ${isDebug ? 'ON' : 'OFF'}`;
 debugBtn.classList.toggle('bg-yellow-500', isDebug);
 debugBtn.classList.toggle('text-white', isDebug);
 debugBtn.classList.toggle('bg-gray-200', !isDebug);
 debugBtn.classList.toggle('text-gray-700', !isDebug);
 debugBtn.classList.toggle('hover:bg-yellow-600', isDebug);
 debugBtn.classList.toggle('hover:bg-gray-300', !isDebug);
 addLog(`Debug mode ${isDebug ? 'enabled' : 'disabled'}`, 'info');
 }
 // --- Fetching & Anti-Blocking (Limited Frontend Version) ---
 async function fetchViaProxy(url, signal) {
 const ua = getCurrentUserAgent();
 setCurrentUAUI(ua); // Update UI
 addLog(`Fetching: ${url}`, 'debug');
 try {
 const response = await fetch(`${PROXY_URL}${encodeURIComponent(url)}`, {
 signal: signal, // Pass AbortSignal
 headers: { // These headers are sent to the proxy
 'X-Requested-With': 'XMLHttpRequest', // Might help some proxies
 // We can't directly control headers sent *from* the proxy to the target
 }
 });
 if (!response.ok) {
 throw new Error(`Proxy error: ${response.status} ${response.statusText}`);
 }
 const data = await response.json();
 // Check the structure returned by allorigins.win
 if (!data || typeof data.contents === 'undefined') {
 throw new Error('Invalid response structure from proxy.');
 }
 if (data.contents === null && data.status && data.status.http_code && data.status.http_code >= 400) {
 throw new Error(`Target server error: ${data.status.http_code} (via proxy)`);
 }
 if (data.contents === null) {
 // Sometimes proxy returns null content without error status, treat as failure
 throw new Error('Proxy returned null content.');
 }
 return data.contents; // Return the HTML content
 } catch (error) {
 if (error.name === 'AbortError') {
 addLog(`Request aborted: ${url}`, 'warning');
 } else {
 addLog(`Fetch error for ${url}: ${error.message}`, 'error');
 }
 throw error; // Re-throw for crawl loop handling
 }
 }
 // --- Robots.txt Handling (Basic) ---
 async function checkRobotsTxt(startUrl) {
 if (!respectRobotsCheckbox.checked) {
 addLog('Skipping robots.txt check.', 'info');
 robotsAllowed = true;
 return true;
 }
 try {
 const urlObj = new URL(startUrl);
 const robotsUrl = `${urlObj.origin}/robots.txt`;
 addLog(`Checking robots.txt at: ${robotsUrl}`, 'info');
 // Use a separate AbortController for this check if needed
 const robotsContent = await fetchViaProxy(robotsUrl);
 if (robotsContent) {
 // VERY Basic Parsing: Check if disallowed for all agents ('*')
 // A proper parser would be much more complex.
 const rules = robotsContent.split('\n');
 let userAgentSection = false;
 for (const line of rules) {
 const trimmedLine = line.trim().toLowerCase();
 if (trimmedLine.startsWith('user-agent: *')) {
 userAgentSection = true;
 } else if (userAgentSection && trimmedLine.startsWith('user-agent:')) {
 userAgentSection = false; // New agent section started
 } else if (userAgentSection && trimmedLine.startsWith('disallow: /')) {
 addLog('robots.txt disallows crawling for all agents (`Disallow: /`). Aborting.', 'error');
 robotsAllowed = false;
 return false;
 } else if (userAgentSection && trimmedLine.startsWith('disallow:')) {
 // More specific disallow, we'll ignore for this basic check
 // A real crawler needs to match paths.
 } else if (trimmedLine === '') {
 userAgentSection = false; // Blank line resets section
 }
 }
 addLog('robots.txt check passed (basic check).', 'success');
 robotsAllowed = true;
 return true;
 } else {
 addLog('robots.txt not found or empty, assuming allowed.', 'warning');
 robotsAllowed = true;
 return true; // Assume allowed if robots.txt is missing/empty
 }
 } catch (error) {
 addLog(`Error checking robots.txt (${error.message}). Assuming allowed.`, 'warning');
 robotsAllowed = true;
 return true; // Assume allowed if there's an error fetching/parsing
 }
 }
 // --- Connection Test ---
 async function testConnection() {
 const url = startUrlInput.value;
 if (!url) {
 addLog('Please enter a Start URL.', 'error');
 return;
 }
 addLog(`Testing connection to ${url} via proxy...`, 'info');
 statusElement.textContent = 'Testing';
 statusElement.className = 'bg-yellow-500 px-3 py-1 rounded-full text-sm';
 try {
 const content = await fetchViaProxy(url);
 if (content !== null) { // Check if content was successfully retrieved
 addLog('Connection successful! Proxy retrieved content.', 'success');
 // Basic check for blocking patterns
 if (content.toLowerCase().includes('access denied') || content.toLowerCase().includes('blocked') || content.toLowerCase().includes('captcha')) {
 addLog('Warning: Possible blocking detected in response content.', 'warning');
 statusElement.textContent = 'Limited?';
 statusElement.className = 'bg-orange-500 px-3 py-1 rounded-full text-sm';
 } else {
 statusElement.textContent = 'Ready';
 statusElement.className = 'bg-green-500 px-3 py-1 rounded-full text-sm';
 }
 } else {
 // Fetch succeeded but content was null (handled as error in fetchViaProxy)
 // This case might not be reached if fetchViaProxy throws
 addLog('Connection via proxy failed (null content).', 'error');
 statusElement.textContent = 'Failed';
 statusElement.className = 'bg-red-500 px-3 py-1 rounded-full text-sm';
 }
 } catch (error) {
 addLog(`Connection test failed: ${error.message}`, 'error');
 statusElement.textContent = 'Error';
 statusElement.className = 'bg-red-500 px-3 py-1 rounded-full text-sm';
 }
 }
 // --- Crawling Logic ---
 async function startCrawling() {
 if (isCrawling) return;
 const startUrl = startUrlInput.value;
 maxPagesToCrawl = parseInt(maxPagesInput.value) || 50;
 if (!startUrl) {
 addLog('Please enter a valid Start URL', 'error');
 return;
 }
 // Reset state
 isCrawling = true;
 isPaused = false;
 crawledPages = 0;
 successfulRequests = 0;
 failedRequests = 0;
 products = [];
 categories = new Set();
 images = new Set();
 queue = [];
 visited = new Set();
 productGrid.innerHTML = ''; // Clear previous results
 noProductsPlaceholder.style.display = 'block';
 updateCategoryFilter(); // Clear categories
 addLog(`--- Starting Crawl ---`, 'info');
 addLog(`Target: ${startUrl}`, 'info');
 addLog(`Max Pages: ${maxPagesToCrawl}`, 'info');
 updateUIState('start');
 // Check robots.txt before starting queue
 const allowedByRobots = await checkRobotsTxt(startUrl);
 if (!allowedByRobots) {
 updateUIState('stop');
 return; // Stop if robots.txt disallows
 }
 // Initialize queue with normalized start URL
 const normalizedStartUrl = normalizeUrl(startUrl);
 if(normalizedStartUrl) {
 queue.push(normalizedStartUrl);
 visited.add(normalizedStartUrl); // Add start URL to visited immediately
 } else {
 addLog('Invalid Start URL format.', 'error');
 updateUIState('stop');
 return;
 }
 // Start the crawl loop
 crawlController = new AbortController(); // Create AbortController for this crawl session
 crawlLoop(crawlController.signal);
 }
 function pauseCrawling() {
 if (!isCrawling) return;
 isPaused = !isPaused; // Toggle pause state
 if (isPaused) {
 addLog('Crawl paused.', 'warning');
 updateUIState('pause');
 if (crawlController) {
 // We don't abort on pause, just stop processing queue
 }
 } else {
 addLog('Resuming crawl...', 'info');
 updateUIState('resume');
 // Restart the loop if it wasn't already running
 crawlLoop(crawlController.signal);
 }
 }
 function stopCrawling() {
 if (!isCrawling) return;
 addLog('--- Stopping Crawl ---', 'warning');
 isCrawling = false;
 isPaused = false; // Ensure not paused
 if (crawlController) {
 crawlController.abort(); // Signal abortion to pending fetch requests
 crawlController = null;
 }
 updateUIState('stop');
 addLog(`Crawl stopped by user. Success rate: ${calculateSuccessRate()}%`, 'info');
 }
 async function crawlLoop(signal) {
 if (!isCrawling || isPaused) {
 addLog(`Loop check: Crawling=${isCrawling}, Paused=${isPaused}. Exiting loop.`, 'debug');
 return; // Stop if paused or stopped
 }
 if (queue.length === 0) {
 addLog('Queue is empty. Crawl finished.', 'success');
 updateUIState('finish');
 addLog(`Crawl completed. Success rate: ${calculateSuccessRate()}%`, 'info');
 return;
 }
 if (crawledPages >= maxPagesToCrawl) {
 addLog(`Reached maximum page limit (${maxPagesToCrawl}). Stopping crawl.`, 'success');
 updateUIState('finish');
 addLog(`Crawl limit reached. Success rate: ${calculateSuccessRate()}%`, 'info');
 return;
 }
 const currentUrl = queue.shift();
 crawledPages++;
 updateStats();
 addLog(`Crawling page ${crawledPages}/${maxPagesToCrawl}: ${currentUrl}`, 'info');
 try {
 if (signal.aborted) { // Check if stopped before fetching
 addLog('Crawl stopped before fetching.', 'warning');
 return;
 }
 const delay = getDelay();
 addLog(`Waiting ${delay/1000}s before request...`, 'debug');
 await sleep(delay);
 const htmlContent = await fetchViaProxy(currentUrl, signal);
 if (htmlContent !== null) { // Ensure content was received
 successfulRequests++;
 processPageContent(htmlContent, currentUrl);
 } else {
 // If fetchViaProxy returns null but doesn't throw (e.g., proxy issue handled), count as failed
 failedRequests++;
 }
 } catch (error) {
 if (error.name !== 'AbortError') { // Don't count aborts as fails
 failedRequests++;
 }
 // Error already logged in fetchViaProxy or here if it's not AbortError
 addLog(`Failed to process ${currentUrl}: ${error.message}`, 'error');
 } finally {
 updateStats(); // Update stats after each attempt
 // Schedule the next iteration using setTimeout to avoid deep recursion
 // and allow UI updates
 if(isCrawling && !isPaused) {
 setTimeout(() => crawlLoop(signal), 50); // Small delay before next check
 }
 }
 }
 // --- Page Processing & Data Extraction ---
 function processPageContent(htmlContent, pageUrl) {
 addLog(`Processing content from: ${pageUrl}`, 'debug');
 if (htmlContent.toLowerCase().includes('access denied') || htmlContent.toLowerCase().includes('blocked') || htmlContent.toLowerCase().includes('captcha')) {
 addLog(`Warning: Possible blocking detected on ${pageUrl}`, 'warning');
 // Could implement strategies here if needed, e.g., increase delay
 }
 const parser = new DOMParser();
 const doc = parser.parseFromString(htmlContent, 'text/html');
 // 1. Extract and queue new valid links
 const links = doc.querySelectorAll('a[href]');
 let newLinksFound = 0;
 links.forEach(link => {
 const href = link.getAttribute('href');
 if (href && !href.startsWith('#') && !href.startsWith('javascript:') && !href.startsWith('mailto:')) {
 try {
 const absoluteUrl = new URL(href, pageUrl);
 // Basic filter: stay on the same hostname
 if (absoluteUrl.hostname.endsWith(TARGET_HOSTNAME)) {
 const normalized = normalizeUrl(absoluteUrl.href);
 if (normalized && !visited.has(normalized)) {
 visited.add(normalized);
 queue.push(normalized);
 newLinksFound++;
 }
 }
 } catch (e) {
 addLog(`Invalid link URL found: ${href}`, 'debug');
 }
 }
 });
 if(newLinksFound > 0) addLog(`Added ${newLinksFound} new unique links to the queue.`, 'debug');
 // 2. Extract products (NEEDS CUSTOMIZATION)
 extractProducts(doc, pageUrl);
 }
 function extractProducts(doc, pageUrl) {
 addLog(`Attempting product extraction on: ${pageUrl}`, 'debug');
 // --- !!! CRITICAL CUSTOMIZATION POINT !!! ---
 // You MUST inspect herholdts.co.za's product listing and product pages
 // using your browser's Developer Tools (F12) to find the correct CSS selectors.
 // The selectors below are GENERIC GUESSES and WILL LIKELY FAIL.
 const productContainerSelectors = [
 '.products .product-item', // Example: Grid/list items
 '.product-details', // Example: Single product page container
 'div[data-product-id]', // Example: Attribute-based selector
 '.item-product'
 // Add more potential container selectors based on inspection
 ];
 let productsExtractedOnPage = 0;
 for (const containerSelector of productContainerSelectors) {
 const productElements = doc.querySelectorAll(containerSelector);
 if (productElements.length > 0) {
 addLog(`Found ${productElements.length} potential product element(s) using selector: "${containerSelector}"`, 'debug');
 productElements.forEach((el, index) => {
 try {
 // --- !!! MORE CUSTOMIZATION NEEDED HERE !!! ---
 const name = getTextContent(el, ['.product-title', 'h1', 'h2.name', '.product-name']) || 'Unknown Product';
 const price = getTextContent(el, ['.price .amount', '.product-price', '.special-price']) || 'N/A';
 const sku = getAttributeValue(el, ['[data-sku]', '.sku'], 'data-sku') || getTextContent(el, ['.sku-number']) || '';
 const description = getTextContent(el, ['.product-description', '.short-description', '.tab-content#description']) || '';
 const category = extractCategoryFromUrlOrBreadcrumbs(pageUrl, doc); // Helper needed
 const imageUrl = extractImageUrl(el, pageUrl); // Helper needed
 if (imageUrl) {
 images.add(imageUrl); // Add to set of unique images
 }
 // Basic check to avoid duplicates based on name/sku if possible
 const isDuplicate = products.some(p => p.name === name && (sku && p.sku === sku));
 if (!isDuplicate) {
 const product = {
 id: products.length + 1,
 name: name.trim(),
 price: price.trim(),
 sku: sku.trim(),
 description: description.trim().substring(0, 200) + (description.length > 200 ? '...' : ''), // Limit description length
 category: category,
 imageUrl: imageUrl,
 pageUrl: pageUrl,
 timestamp: new Date().toISOString()
 };
 products.push(product);
 categories.add(category);
 updateProductDisplay(product); // Add to UI immediately
 productsExtractedOnPage++;
 } else {
 addLog(`Skipping likely duplicate product: ${name}`, 'debug');
 }
 } catch (e) {
 addLog(`Error processing potential product element #${index+1} on ${pageUrl}: ${e.message}`, 'error');
 addLog(`Problematic element selector: ${containerSelector}`, 'debug');
 }
 });
 // If we found products with this selector, assume it's the right one for this page and stop checking others
 break;
 }
 }
 if (productsExtractedOnPage > 0) {
 addLog(`Successfully extracted ${productsExtractedOnPage} new products from ${pageUrl}`, 'success');
 updateStats();
 updateCategoryFilter();
 } else {
 addLog(`No products extracted from ${pageUrl} with current selectors.`, 'debug');
 }
 }
 // Helper: Get text content using multiple selectors
 function getTextContent(parentElement, selectors) {
 for (const selector of selectors) {
 const element = parentElement.querySelector(selector);
 if (element) {
 // Attempt to get clean text, removing potential hidden elements or excessive whitespace
 return element.textContent?.replace(/\s+/g, ' ').trim() || null;
 }
 }
 return null;
 }
 // Helper: Get attribute value using multiple selectors
 function getAttributeValue(parentElement, selectors, attributeName) {
 for (const selector of selectors) {
 const element = parentElement.querySelector(selector);
 if (element && element.hasAttribute(attributeName)) {
 return element.getAttribute(attributeName);
 }
 }
 return null;
 }
 // Helper: Extract image URL (NEEDS CUSTOMIZATION)
 function extractImageUrl(parentElement, baseUrl) {
 // --- !!! CUSTOMIZATION POINT !!! ---
 // Inspect how images are embedded (img src, data-src, background-image, picture element)
 const selectors = [
 'img.product-image', // Primary image selector guess
 '.product-gallery img',
 'img[data-main-image]',
 'img' // Fallback to any image within the container
 ];
 let imgUrl = null;
 for(const selector of selectors) {
 const imgElement = parentElement.querySelector(selector);
 if (imgElement) {
 imgUrl = imgElement.getAttribute('data-src') || imgElement.getAttribute('src');
 if (imgUrl) break;
 }
 }
 // Add checks for <picture> elements or background images if necessary
 if (imgUrl) {
 try {
 // Ensure URL is absolute
 return new URL(imgUrl, baseUrl).href;
 } catch (e) {
 addLog(`Invalid image URL found: ${imgUrl}`, 'debug');
 return null;
 }
 }
 return null; // No image found
 }
 // Helper: Extract category (NEEDS CUSTOMIZATION)
 function extractCategoryFromUrlOrBreadcrumbs(pageUrl, doc) {
 // --- !!! CUSTOMIZATION POINT !!! ---
 // Strategy 1: Use breadcrumbs if available
 const breadcrumbSelectors = ['.breadcrumbs .item', '.breadcrumb li a']; // Guesses
 for (const selector of breadcrumbSelectors) {
 const breadcrumbLinks = doc.querySelectorAll(selector);
 if (breadcrumbLinks.length > 1) { // Need at least Home > Category
 // Get text from all but the last link (often the product itself) or the first (Home)
 const categories = Array.from(breadcrumbLinks)
 .slice(1, breadcrumbLinks.length > 2 ? -1 : undefined) // Skip Home, optionally skip last item
 .map(el => el.textContent?.trim())
 .filter(Boolean); // Remove empty items
 if (categories.length > 0) return categories.join(' > ');
 }
 }
 // Strategy 2: Parse from URL path (less reliable)
 try {
 const pathParts = new URL(pageUrl).pathname.split('/').filter(part => part && part !== 'product' && !/\d/.test(part) && part !== 'default.asp'); // Basic filtering
 if (pathParts.length > 0) {
 return pathParts.map(p => p.replace(/-/g, ' ').replace(/_/g, ' ')).join(' > '); // Simple formatting
 }
 } catch {}
 return 'Uncategorized'; // Fallback
 }
 // --- UI Update Functions ---
 function updateUIState(state) {
 switch (state) {
 case 'start':
 case 'resume':
 statusElement.textContent = 'Crawling';
 statusElement.className = 'bg-indigo-800 px-3 py-1 rounded-full text-sm blink';
 startBtn.disabled = true;
 pauseBtn.disabled = false;
 stopBtn.disabled = false;
 pauseBtn.innerHTML = '<i class="fas fa-pause mr-2"></i> Pause';
 break;
 case 'pause':
 statusElement.textContent = 'Paused';
 statusElement.className = 'bg-yellow-500 px-3 py-1 rounded-full text-sm';
 startBtn.disabled = true;
 pauseBtn.disabled = false;
 stopBtn.disabled = false;
 pauseBtn.innerHTML = '<i class="fas fa-play mr-2"></i> Resume';
 break;
 case 'stop':
 case 'finish':
 case 'error':
 statusElement.textContent = state === 'finish' ? 'Completed' : 'Stopped';
 statusElement.className = `px-3 py-1 rounded-full text-sm ${state === 'finish' ? 'bg-green-500' : 'bg-red-500'}`;
 startBtn.disabled = false;
 pauseBtn.disabled = true;
 stopBtn.disabled = true;
 pauseBtn.innerHTML = '<i class="fas fa-pause mr-2"></i> Pause';
 if (statusElement.classList.contains('blink')) {
 statusElement.classList.remove('blink');
 }
 break;
 }
 }
 function calculateSuccessRate() {
 const totalAttempts = successfulRequests + failedRequests;
 return totalAttempts > 0 ? Math.round((successfulRequests / totalAttempts) * 100) : 0;
 }
 function updateStats() {
 const rate = calculateSuccessRate();
 // Use maxPagesToCrawl for progress calculation
 const progressPercent = maxPagesToCrawl > 0 ? (crawledPages / maxPagesToCrawl) * 100 : 0;
 progressText.textContent = `${crawledPages}/${maxPagesToCrawl} pages`;
 progressBar.style.width = `${Math.min(progressPercent, 100)}%`; // Cap at 100%
 pagesCrawledElement.textContent = crawledPages;
 productsFoundElement.textContent = products.length;
 imagesFoundElement.textContent = images.size; // Use set size
 productCountElement.textContent = products.length;
 successRateElement.textContent = `${rate}%`;
 // Update success rate color
 successRateElement.className = 'text-xl font-bold'; // Base class
 if (rate >= 95) successRateElement.classList.add('text-green-600');
 else if (rate >= 80) successRateElement.classList.add('text-yellow-600');
 else successRateElement.classList.add('text-red-600');
 }
 function updateProductDisplay(product) {
 if(noProductsPlaceholder.style.display !== 'none') {
 noProductsPlaceholder.style.display = 'none'; // Hide placeholder
 }
 const productCard = document.createElement('div');
 productCard.className = 'product-card bg-white rounded-lg shadow overflow-hidden border border-gray-200 flex flex-col'; // Added flex
 productCard.setAttribute('data-category', product.category); // For filtering
 productCard.setAttribute('data-name', product.name.toLowerCase()); // For searching
 productCard.innerHTML = `
 <div class="h-48 bg-gray-100 flex items-center justify-center overflow-hidden p-2">
 ${product.imageUrl ?
 `<img src="${product.imageUrl}" alt="${product.name}" class="max-h-full w-auto object-contain lazyload" loading="lazy">` :
 `<i class="fas fa-image text-gray-300 text-4xl"></i>`}
 </div>
 <div class="p-3 flex flex-col flex-grow">
 <h3 class="font-semibold text-sm mb-1 line-clamp-2 leading-tight" title="${product.name}">${product.name}</h3>
 <div class="text-indigo-600 font-bold text-base my-1">${product.price}</div>
 <div class="text-xs text-gray-500 mb-1">${product.category}</div>
 ${product.sku ? `<div class="text-xs text-gray-400 mb-2">SKU: ${product.sku}</div>` : ''}
 <p class="text-xs text-gray-600 line-clamp-2 flex-grow">${product.description}</p>
 <div class="mt-2 text-xs text-gray-400 truncate pt-1 border-t border-gray-100" title="${product.pageUrl}">
 <a href="${product.pageUrl}" target="_blank" rel="noopener noreferrer" class="hover:text-indigo-500">${product.pageUrl.replace(/^https?:\/\//, '')}</a>
 </div>
 </div>
 `;
 productGrid.prepend(productCard); // Add new products to the top
 }
 function updateCategoryFilter() {
 const currentSelection = categoryFilter.value;
 // Clear existing options except the first placeholder
 while (categoryFilter.options.length > 1) {
 categoryFilter.remove(1);
 }
 Array.from(categories).sort().forEach(category => {
 if(category){ // Avoid adding empty categories
 const option = document.createElement('option');
 option.value = category;
 option.textContent = category;
 categoryFilter.appendChild(option);
 }
 });
 // Try to restore previous selection
 categoryFilter.value = currentSelection;
 }
 function filterProducts() {
 const searchTerm = productSearch.value.toLowerCase().trim();
 const selectedCategory = categoryFilter.value;
 let visibleCount = 0;
 document.querySelectorAll('.product-card').forEach(card => {
 const name = card.getAttribute('data-name') || '';
 const category = card.getAttribute('data-category') || '';
 const matchesSearch = !searchTerm || name.includes(searchTerm);
 const matchesCategory = !selectedCategory || category === selectedCategory;
 const isVisible = matchesSearch && matchesCategory;
 card.style.display = isVisible ? 'flex' : 'none'; // Use flex since card uses flex display
 if(isVisible) visibleCount++;
 });
 // Show placeholder if no products match filter
 noProductsPlaceholder.style.display = (productGrid.childElementCount === 1 && visibleCount === 0) ? 'block' : 'none';
 if (productGrid.childElementCount > 1) {
 noProductsPlaceholder.style.display = visibleCount === 0 ? 'block' : 'none';
 }
 }
 // --- Data Export ---
 function exportData(format) {
 if (products.length === 0) {
 addLog('No product data to export.', 'warning');
 return;
 }
 addLog(`Preparing ${format.toUpperCase()} export...`, 'info');
 try {
 const timestamp = new Date().toISOString().replace(/:/g, '-').slice(0, 19);
 let data, filename, mimeType;
 // Use only relevant fields for export
 const exportProducts = products.map(p => ({
 Name: p.name,
 Price: p.price,
 SKU: p.sku,
 Category: p.category,
 Description: p.description,
 ImageURL: p.imageUrl,
 PageURL: p.pageUrl,
 Timestamp: p.timestamp
 }));
 switch (format) {
 case 'csv':
 data = convertToCSV(exportProducts);
 filename = `herholdts_products_${timestamp}.csv`;
 mimeType = 'text/csv;charset=utf-8;';
 const blobCsv = new Blob([data], { type: mimeType });
 saveAs(blobCsv, filename);
 break;
 case 'excel':
 const wb = XLSX.utils.book_new();
 const ws = XLSX.utils.json_to_sheet(exportProducts);
 // Optional: Adjust column widths (basic example)
 const colWidths = Object.keys(exportProducts[0]).map(key => ({ wch: Math.max(key.length, 15) })); // Basic width estimate
 ws['!cols'] = colWidths;
 XLSX.utils.book_append_sheet(wb, ws, 'Products');
 XLSX.writeFile(wb, `herholdts_products_${timestamp}.xlsx`);
 break;
 case 'json':
 data = JSON.stringify(exportProducts, null, 2);
 filename = `herholdts_products_${timestamp}.json`;
 mimeType = 'application/json;charset=utf-8;';
 const blobJson = new Blob([data], { type: mimeType });
 saveAs(blobJson, filename);
 break;
 default:
 throw new Error(`Unsupported format: ${format}`);
 }
 addLog(`${format.toUpperCase()} file exported successfully. (${products.length} items)`, 'success');
 } catch (e) {
 addLog(`Error exporting data as ${format}: ${e.message}`, 'error');
 console.error("Export error:", e);
 }
 }
 function convertToCSV(items) {
 if (!items || items.length === 0) return '';
 const header = Object.keys(items[0]);
 const csvHeader = header.join(',');
 const csvRows = items.map(item =>
 header.map(fieldName => {
 let fieldValue = item[fieldName];
 if (fieldValue === null || typeof fieldValue === 'undefined') {
 return '';
 }
 let stringValue = String(fieldValue);
 // Escape double quotes and handle commas/newlines within fields
 if (stringValue.includes('"') || stringValue.includes(',') || stringValue.includes('\n')) {
 return `"${stringValue.replace(/"/g, '""')}"`;
 }
 return stringValue;
 }).join(',')
 );
 return [csvHeader, ...csvRows].join('\n');
 }
 </script>
<p style="border-radius: 8px; text-align: center; font-size: 12px; color: #fff; margin-top: 16px;position: fixed; left: 8px; bottom: 8px; z-index: 10; background: rgba(0, 0, 0, 0.8); padding: 4px 8px;">Made with <img src="https://enzostvs-deepsite.hf.space/logo.svg" alt="DeepSite Logo" style="width: 16px; height: 16px; vertical-align: middle;display:inline-block;margin-right:3px;filter:brightness(0) invert(1);"><a href="https://enzostvs-deepsite.hf.space" style="color: #fff;text-decoration: underline;" target="_blank" >DeepSite</a> - <a href="https://enzostvs-deepsite.hf.space?remix=aigodking/crawler-trial" style="color: #fff;text-decoration: underline;" target="_blank" >🧬 Remix</a></p></body>
</html>