Upload 3 files

app.py

@@ -0,0 +1,398 @@
+import asyncio
+import base64
+import json
+import logging
+import os
+import time
+import uuid
+from datetime import datetime
+from io import BytesIO
+from typing import Dict, List, Optional, Any
+from contextlib import asynccontextmanager
+
+import gradio as gr
+import uvicorn
+from fastapi import FastAPI, HTTPException, BackgroundTasks
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from playwright.async_api import async_playwright, Browser, BrowserContext, Page
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import TimeoutException, WebDriverException
+from bs4 import BeautifulSoup
+from PIL import Image
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Global browser instances
+browser_instances: Dict[str, Dict] = {}
+playwright_instance = None
+browser_pool = None
+
+# Pydantic models
+class BrowserLaunchRequest(BaseModel):
+    headless: bool = True
+    width: int = 1920
+    height: int = 1080
+    user_agent: Optional[str] = None
+
+class NavigateRequest(BaseModel):
+    session_id: str
+    url: str
+    wait_until: str = "networkidle"
+
+class ScreenshotRequest(BaseModel):
+    session_id: str
+    full_page: bool = False
+    selector: Optional[str] = None
+
+class ElementActionRequest(BaseModel):
+    session_id: str
+    selector: str
+    action: str  # click, type, scroll, hover
+    value: Optional[str] = None
+
+class ScrapeRequest(BaseModel):
+    session_id: str
+    selectors: Optional[List[str]] = None
+    extract_all: bool = False
+
+class AIExtractionRequest(BaseModel):
+    session_id: str
+    prompt: str
+    target_elements: Optional[List[str]] = None
+
+# Global lifespan manager
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global playwright_instance, browser_pool
+    
+    # Startup
+    playwright_instance = await async_playwright().start()
+    browser_pool = await playwright_instance.chromium.launch(
+        headless=True,
+        args=['--no-sandbox', '--disable-dev-shm-usage', '--disable-gpu']
+    )
+    logger.info("Browser pool initialized")
+    
+    yield
+    
+    # Shutdown
+    if browser_pool:
+        await browser_pool.close()
+    if playwright_instance:
+        await playwright_instance.stop()
+    logger.info("Browser instances cleaned up")
+
+# Initialize FastAPI app
+app = FastAPI(
+    title="Web Scraping API Service",
+    description="Headless browser automation with Playwright and Selenium",
+    version="1.0.0",
+    lifespan=lifespan
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Utility functions
+def get_chrome_options():
+    """Get Chrome options for Selenium"""
+    options = Options()
+    options.add_argument('--headless')
+    options.add_argument('--no-sandbox')
+    options.add_argument('--disable-dev-shm-usage')
+    options.add_argument('--disable-gpu')
+    options.add_argument('--window-size=1920,1080')
+    return options
+
+def cleanup_old_sessions():
+    """Clean up sessions older than 1 hour"""
+    current_time = time.time()
+    expired_sessions = []
+    
+    for session_id, session_data in browser_instances.items():
+        if current_time - session_data.get('created_at', 0) > 3600:  # 1 hour
+            expired_sessions.append(session_id)
+    
+    for session_id in expired_sessions:
+        asyncio.create_task(close_browser_session(session_id))
+
+async def close_browser_session(session_id: str):
+    """Close a specific browser session"""
+    if session_id in browser_instances:
+        session = browser_instances[session_id]
+        
+        # Close Playwright session
+        if 'playwright_page' in session:
+            try:
+                await session['playwright_page'].close()
+                await session['playwright_context'].close()
+            except Exception as e:
+                logger.error(f"Error closing Playwright session {session_id}: {e}")
+        
+        # Close Selenium session
+        if 'selenium_driver' in session:
+            try:
+                session['selenium_driver'].quit()
+            except Exception as e:
+                logger.error(f"Error closing Selenium session {session_id}: {e}")
+        
+        del browser_instances[session_id]
+        logger.info(f"Closed browser session: {session_id}")
+
+# API Endpoints
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint"""
+    return {"status": "healthy", "timestamp": datetime.now().isoformat()}
+
+@app.post("/api/browser/launch")
+async def launch_browser(request: BrowserLaunchRequest):
+    """Launch a new browser instance"""
+    session_id = str(uuid.uuid4())
+    
+    try:
+        # Launch Playwright browser
+        context = await browser_pool.new_context(
+            viewport={'width': request.width, 'height': request.height},
+            user_agent=request.user_agent
+        )
+        page = await context.new_page()
+        
+        # Launch Selenium browser
+        chrome_options = get_chrome_options()
+        if request.user_agent:
+            chrome_options.add_argument(f'--user-agent={request.user_agent}')
+        
+        selenium_driver = webdriver.Chrome(options=chrome_options)
+        selenium_driver.set_window_size(request.width, request.height)
+        
+        # Store session
+        browser_instances[session_id] = {
+            'playwright_context': context,
+            'playwright_page': page,
+            'selenium_driver': selenium_driver,
+            'created_at': time.time(),
+            'config': request.dict()
+        }
+        
+        logger.info(f"Launched browser session: {session_id}")
+        return {"session_id": session_id, "status": "launched"}
+        
+    except Exception as e:
+        logger.error(f"Error launching browser: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/api/browser/navigate")
+async def navigate_to_url(request: NavigateRequest):
+    """Navigate to a URL"""
+    if request.session_id not in browser_instances:
+        raise HTTPException(status_code=404, detail="Session not found")
+    
+    session = browser_instances[request.session_id]
+    
+    try:
+        # Navigate with Playwright
+        await session['playwright_page'].goto(request.url, wait_until=request.wait_until)
+        
+        # Navigate with Selenium
+        session['selenium_driver'].get(request.url)
+        
+        return {"status": "navigated", "url": request.url}
+        
+    except Exception as e:
+        logger.error(f"Error navigating to {request.url}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/api/browser/screenshot")
+async def take_screenshot(request: ScreenshotRequest):
+    """Take a screenshot"""
+    if request.session_id not in browser_instances:
+        raise HTTPException(status_code=404, detail="Session not found")
+    
+    session = browser_instances[request.session_id]
+    
+    try:
+        if request.selector:
+            # Screenshot specific element with Playwright
+            element = await session['playwright_page'].locator(request.selector).first
+            screenshot_bytes = await element.screenshot()
+        else:
+            # Full page screenshot with Playwright
+            screenshot_bytes = await session['playwright_page'].screenshot(
+                full_page=request.full_page
+            )
+        
+        # Convert to base64
+        screenshot_b64 = base64.b64encode(screenshot_bytes).decode()
+        
+        return {
+            "screenshot": screenshot_b64,
+            "format": "png",
+            "timestamp": datetime.now().isoformat()
+        }
+        
+    except Exception as e:
+        logger.error(f"Error taking screenshot: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/api/elements/action")
+async def perform_element_action(request: ElementActionRequest):
+    """Perform action on an element"""
+    if request.session_id not in browser_instances:
+        raise HTTPException(status_code=404, detail="Session not found")
+    
+    session = browser_instances[request.session_id]
+    
+    try:
+        page = session['playwright_page']
+        element = page.locator(request.selector).first
+        
+        if request.action == "click":
+            await element.click()
+        elif request.action == "type":
+            await element.fill(request.value or "")
+        elif request.action == "scroll":
+            await element.scroll_into_view_if_needed()
+        elif request.action == "hover":
+            await element.hover()
+        else:
+            raise HTTPException(status_code=400, detail="Invalid action")
+        
+        return {"status": "completed", "action": request.action}
+        
+    except Exception as e:
+        logger.error(f"Error performing action {request.action}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/elements/inspect/{session_id}")
+async def inspect_page_elements(session_id: str):
+    """Get all interactive elements on the page"""
+    if session_id not in browser_instances:
+        raise HTTPException(status_code=404, detail="Session not found")
+    
+    session = browser_instances[session_id]
+    
+    try:
+        page = session['playwright_page']
+        
+        # Get page content
+        content = await page.content()
+        soup = BeautifulSoup(content, 'html.parser')
+        
+        # Find interactive elements
+        interactive_selectors = [
+            'a', 'button', 'input', 'select', 'textarea', 
+            '[onclick]', '[href]', '[role="button"]'
+        ]
+        
+        elements = []
+        for selector in interactive_selectors:
+            found_elements = soup.select(selector)
+            for i, elem in enumerate(found_elements):
+                element_info = {
+                    'tag': elem.name,
+                    'selector': f"{selector}:nth-of-type({i+1})",
+                    'text': elem.get_text(strip=True)[:100],
+                    'attributes': dict(elem.attrs),
+                    'type': elem.get('type', 'N/A')
+                }
+                elements.append(element_info)
+        
+        return {"elements": elements, "total_count": len(elements)}
+        
+    except Exception as e:
+        logger.error(f"Error inspecting elements: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/api/scrape/content")
+async def scrape_content(request: ScrapeRequest):
+    """Scrape content from the page"""
+    if request.session_id not in browser_instances:
+        raise HTTPException(status_code=404, detail="Session not found")
+    
+    session = browser_instances[request.session_id]
+    
+    try:
+        page = session['playwright_page']
+        content = await page.content()
+        soup = BeautifulSoup(content, 'html.parser')
+        
+        scraped_data = {}
+        
+        if request.extract_all:
+            # Extract common elements
+            scraped_data = {
+                'title': soup.title.string if soup.title else None,
+                'headings': [h.get_text(strip=True) for h in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])],
+                'paragraphs': [p.get_text(strip=True) for p in soup.find_all('p')],
+                'links': [{'text': a.get_text(strip=True), 'href': a.get('href')} for a in soup.find_all('a', href=True)],
+                'images': [{'src': img.get('src'), 'alt': img.get('alt')} for img in soup.find_all('img')],
+                'forms': [{'action': form.get('action'), 'method': form.get('method')} for form in soup.find_all('form')]
+            }
+        elif request.selectors:
+            # Extract specific selectors
+            for selector in request.selectors:
+                elements = soup.select(selector)
+                scraped_data[selector] = [elem.get_text(strip=True) for elem in elements]
+        
+        return {"data": scraped_data, "timestamp": datetime.now().isoformat()}
+        
+    except Exception as e:
+        logger.error(f"Error scraping content: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.delete("/api/browser/close/{session_id}")
+async def close_browser(session_id: str):
+    """Close a browser session"""
+    if session_id not in browser_instances:
+        raise HTTPException(status_code=404, detail="Session not found")
+    
+    await close_browser_session(session_id)
+    return {"status": "closed", "session_id": session_id}
+
+@app.get("/api/sessions")
+async def list_sessions():
+    """List all active browser sessions"""
+    sessions = []
+    for session_id, session_data in browser_instances.items():
+        sessions.append({
+            'session_id': session_id,
+            'created_at': datetime.fromtimestamp(session_data['created_at']).isoformat(),
+            'config': session_data['config']
+        })
+    
+    return {"sessions": sessions, "total_count": len(sessions)}
+
+# Background task to cleanup old sessions
+@app.on_event("startup")
+async def startup_event():
+    async def cleanup_task():
+        while True:
+            cleanup_old_sessions()
+            await asyncio.sleep(300)  # Clean up every 5 minutes
+    
+    asyncio.create_task(cleanup_task())
+
+if __name__ == "__main__":
+    uvicorn.run(
+        "app:app",
+        host="0.0.0.0",
+        port=7860,
+        reload=False,
+        workers=1
+    )

browser_automation_ui.html

@@ -0,0 +1,270 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>🌐 Web Scraping Server UI</title>
+    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=Source+Code+Pro:wght@400;500&display=swap" rel="stylesheet">
+    <link href="https://cdnjs.cloudflare.com/ajax/libs/lucide/0.263.1/lucide.min.css" rel="stylesheet">
+    <style>
+        * {
+            margin: 0;
+            padding: 0;
+            box-sizing: border-box;
+        }
+
+        body {
+            font-family: 'Inter', sans-serif;
+            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+            min-height: 100vh;
+            color: #2d3748;
+            overflow-x: hidden;
+        }
+
+        .container {
+            max-width: 1400px;
+            margin: 0 auto;
+            padding: 20px;
+        }
+
+        .header {
+            text-align: center;
+            margin-bottom: 30px;
+            color: white;
+        }
+
+        .header h1 {
+            font-size: 2.5rem;
+            font-weight: 700;
+            margin-bottom: 10px;
+            text-shadow: 0 2px 4px rgba(0,0,0,0.1);
+        }
+
+        .header p {
+            font-size: 1.1rem;
+            opacity: 0.9;
+        }
+
+        .status-bar {
+            background: rgba(255, 255, 255, 0.95);
+            backdrop-filter: blur(10px);
+            border-radius: 12px;
+            padding: 16px 24px;
+            margin-bottom: 24px;
+            display: flex;
+            justify-content: space-between;
+            align-items: center;
+            box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1);
+            border: 1px solid rgba(255, 255, 255, 0.2);
+        }
+
+        .status-item {
+            display: flex;
+            align-items: center;
+            gap: 8px;
+        }
+
+        .status-indicator {
+            width: 12px;
+            height: 12px;
+            border-radius: 50%;
+            animation: pulse 2s infinite;
+        }
+
+        .status-online {
+            background: #10b981;
+        }
+
+        .status-offline {
+            background: #ef4444;
+        }
+
+        @keyframes pulse {
+            0%, 100% { opacity: 1; }
+            50% { opacity: 0.5; }
+        }
+
+        .main-content {
+            background: rgba(255, 255, 255, 0.95);
+            backdrop-filter: blur(10px);
+            border-radius: 16px;
+            box-shadow: 0 20px 40px rgba(0, 0, 0, 0.1);
+            border: 1px solid rgba(255, 255, 255, 0.2);
+            overflow: hidden;
+        }
+
+        .tab-nav {
+            display: flex;
+            background: rgba(248, 250, 252, 0.8);
+            border-bottom: 1px solid #e2e8f0;
+        }
+
+        .tab-button {
+            flex: 1;
+            padding: 16px 24px;
+            background: none;
+            border: none;
+            cursor: pointer;
+            font-family: 'Inter', sans-serif;
+            font-size: 1rem;
+            font-weight: 500;
+            color: #64748b;
+            transition: all 0.3s ease;
+            position: relative;
+        }
+
+        .tab-button.active {
+            color: #4f46e5;
+            background: rgba(79, 70, 229, 0.05);
+        }
+
+        .tab-button.active::after {
+            content: '';
+            position: absolute;
+            bottom: 0;
+            left: 0;
+            right: 0;
+            height: 3px;
+            background: #4f46e5;
+            border-radius: 3px 3px 0 0;
+        }
+
+        .tab-button:hover {
+            background: rgba(79, 70, 229, 0.05);
+            color: #4f46e5;
+        }
+
+        .tab-content {
+            padding: 32px;
+            min-height: 600px;
+        }
+
+        .tab-pane {
+            display: none;
+        }
+
+        .tab-pane.active {
+            display: block;
+            animation: fadeIn 0.3s ease;
+        }
+
+        @keyframes fadeIn {
+            from { opacity: 0; transform: translateY(10px); }
+            to { opacity: 1; transform: translateY(0); }
+        }
+
+        .api-section {
+            margin-bottom: 40px;
+            background: #f8fafc;
+            border-radius: 12px;
+            border: 1px solid #e2e8f0;
+            overflow: hidden;
+        }
+
+        .api-header {
+            background: linear-gradient(135deg, #4f46e5, #7c3aed);
+            color: white;
+            padding: 20px 24px;
+            display: flex;
+            align-items: center;
+            gap: 12px;
+        }
+
+        .api-header i {
+            font-size: 1.2rem;
+        }
+
+        .api-header h3 {
+            font-size: 1.25rem;
+            font-weight: 600;
+        }
+
+        .api-body {
+            padding: 24px;
+        }
+
+        .form-group {
+            margin-bottom: 20px;
+        }
+
+        .form-label {
+            display: block;
+            margin-bottom: 8px;
+            font-weight: 500;
+            color: #374151;
+        }
+
+        .form-input, .form-textarea, .form-select {
+            width: 100%;
+            padding: 12px 16px;
+            border: 2px solid #e5e7eb;
+            border-radius: 8px;
+            font-family: 'Inter', sans-serif;
+            font-size: 0.95rem;
+            transition: all 0.3s ease;
+        }
+
+        .form-input:focus, .form-textarea:focus, .form-select:focus {
+            outline: none;
+            border-color: #4f46e5;
+            box-shadow: 0 0 0 3px rgba(79, 70, 229, 0.1);
+        }
+
+        .form-textarea {
+            min-height: 120px;
+            resize: vertical;
+            font-family: 'Source Code Pro', monospace;
+        }
+
+        .btn {
+            padding: 12px 24px;
+            border: none;
+            border-radius: 8px;
+            cursor: pointer;
+            font-family: 'Inter', sans-serif;
+            font-size: 0.95rem;
+            font-weight: 500;
+            transition: all 0.3s ease;
+            display: inline-flex;
+            align-items: center;
+            gap: 8px;
+        }
+
+        .btn-primary {
+            background: linear-gradient(135deg, #4f46e5, #7c3aed);
+            color: white;
+            box-shadow: 0 4px 12px rgba(79, 70, 229, 0.3);
+        }
+
+        .btn-primary:hover {
+            transform: translateY(-2px);
+            box-shadow: 0 6px 20px rgba(79, 70, 229, 0.4);
+        }
+
+        .btn-secondary {
+            background: #f1f5f9;
+            color: #475569;
+            border: 1px solid #e2e8f0;
+        }
+
+        .btn-secondary:hover {
+            background: #e2e8f0;
+        }
+
+        .response-area {
+            margin-top: 24px;
+            background: #1e293b;
+            border-radius: 8px;
+            padding: 20px;
+            font-family: 'Source Code Pro', monospace;
+            color: #e2e8f0;
+            white-space: pre-wrap;
+            max-height: 400px;
+            overflow-y: auto;
+            border: 1px solid #334155;
+        }
+
+        .element-inspector {
+            background: #f8fafc;
+            border-radius: 12px;
+            border

requirements.txt

@@ -0,0 +1,39 @@
+# Web framework
+fastapi==0.104.1
+uvicorn[standard]==0.24.0
+gradio==4.8.0
+
+# Browser automation
+playwright==1.40.0
+selenium==4.15.2
+webdriver-manager==4.0.1
+
+# Web scraping and parsing
+beautifulsoup4==4.12.2
+lxml==4.9.3
+requests==2.31.0
+aiohttp==3.9.1
+
+# Image processing
+Pillow==10.1.0
+
+# Data processing
+pandas==2.1.4
+numpy==1.25.2
+
+# Async support
+asyncio-throttle==1.0.2
+aiofiles==23.2.1
+
+# Utilities
+python-multipart==0.0.6
+python-dotenv==1.0.0
+pydantic==2.5.0
+
+# AI/ML for content understanding (optional)
+openai==1.3.7
+anthropic==0.7.8
+
+# Development
+pytest==7.4.3
+pytest-asyncio==0.21.1