Create app.py

app.py

@@ -0,0 +1,912 @@
+#!/usr/bin/env python3
+"""
+Hugging Face Data Processor - Single Unified Server
+
+A complete, self-contained FastAPI application that:
+1. Automatically processes all courses from samelias1/Helium and samelias1/Data
+2. Merges frame data with cursor information
+3. Searches for exact transcription matches in samfred2/ATO
+4. Generates combined JSON output ready for upload to samfred2/ALL
+5. Provides REST API for monitoring and management
+6. Includes built-in web dashboard for real-time progress tracking
+
+Run with: python server.py
+Then open: http://localhost:8000
+"""
+
+import json
+import asyncio
+import os
+import sys
+from pathlib import Path
+from typing import Optional, List, Dict, Any
+from datetime import datetime
+from enum import Enum
+from collections import defaultdict
+import traceback
+
+from fastapi import FastAPI, HTTPException, BackgroundTasks
+from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from huggingface_hub import hf_hub_download, HfApi
+import uvicorn
+
+# ============================================================================
+# Configuration
+# ============================================================================
+
+DATASET_HELIUM = "samelias1/Helium"
+DATASET_DATA = "samelias1/Data"
+DATASET_ATO = "samfred2/ATO"
+DATASET_OUTPUT = "samfred2/ALL"
+
+OUTPUT_DIR = Path("./output")
+OUTPUT_DIR.mkdir(exist_ok=True)
+
+# ============================================================================
+# Models & Enums
+# ============================================================================
+
+class JobStatus(str, Enum):
+    PENDING = "pending"
+    FETCHING_FILES = "fetching_files"
+    PROCESSING = "processing"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+
+class ProcessingJob(BaseModel):
+    job_id: str
+    status: JobStatus
+    total_files: int = 0
+    processed_files: int = 0
+    matched_transcriptions: int = 0
+    error_message: Optional[str] = None
+    created_at: str
+    started_at: Optional[str] = None
+    completed_at: Optional[str] = None
+    output_file: Optional[str] = None
+    progress_percent: float = 0.0
+
+# ============================================================================
+# Global State
+# ============================================================================
+
+jobs_db: Dict[str, ProcessingJob] = {}
+jobs_lock = asyncio.Lock()
+
+# ============================================================================
+# FastAPI App Setup
+# ============================================================================
+
+app = FastAPI(
+    title="Hugging Face Data Processor",
+    description="Process and merge Hugging Face datasets automatically",
+    version="1.0.0"
+)
+
+# Add CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# ============================================================================
+# Helper Functions
+# ============================================================================
+
+def get_hf_api() -> HfApi:
+    """Initialize Hugging Face API client."""
+    return HfApi()
+
+def list_dataset_files(dataset_id: str) -> List[str]:
+    """Fetch all file names from a Hugging Face dataset."""
+    try:
+        print(f"[HF] Listing files from {dataset_id}...")
+        api = get_hf_api()
+        files = api.list_repo_files(repo_id=dataset_id, repo_type="dataset")
+        file_list = list(files)
+        print(f"[HF] Found {len(file_list)} files in {dataset_id}")
+        return file_list
+    except Exception as e:
+        print(f"[ERROR] Failed to list files from {dataset_id}: {e}")
+        return []
+
+def download_file(repo_id: str, file_name: str) -> Optional[str]:
+    """Download a file from Hugging Face dataset to cache."""
+    try:
+        print(f"[DOWNLOAD] {file_name} from {repo_id}...")
+        path = hf_hub_download(
+            repo_id=repo_id,
+            filename=file_name,
+            repo_type="dataset"
+        )
+        print(f"[DOWNLOAD] ✓ {file_name}")
+        return path
+    except Exception as e:
+        print(f"[ERROR] Failed to download {file_name}: {e}")
+        return None
+
+def load_json_file(file_path: str) -> Optional[Dict | List]:
+    """Load and parse a JSON file."""
+    try:
+        with open(file_path, "r") as f:
+            return json.load(f)
+    except Exception as e:
+        print(f"[ERROR] Failed to load JSON from {file_path}: {e}")
+        return None
+
+def merge_course_data(helium_path: str, data_path: str) -> List[Dict]:
+    """Merge frame data from Helium with cursor data from Data dataset."""
+    try:
+        helium_data = load_json_file(helium_path)
+        data_data = load_json_file(data_path)
+        
+        if not helium_data or not data_data:
+            return []
+        
+        # Create lookup dictionary from Data dataset
+        cursor_lookup = {}
+        for item in data_data:
+            key = (item.get("course"), item.get("image_path"))
+            cursor_lookup[key] = {k: v for k, v in item.items() if k not in ["course", "image_path"]}
+        
+        # Merge with Helium data
+        merged_data = []
+        for index, item in enumerate(helium_data):
+            key = (item.get("course"), item.get("image_path"))
+            
+            merged_item = item.copy()
+            if key in cursor_lookup:
+                merged_item.update(cursor_lookup[key])
+            
+            # Clean up unwanted fields
+            merged_item.pop("server_url", None)
+            merged_item.pop("timestamp", None)
+            
+            # Renumber image_path sequentially
+            merged_item["image_path"] = index + 1
+            
+            merged_data.append(merged_item)
+        
+        return merged_data
+    except Exception as e:
+        print(f"[ERROR] Failed to merge course data: {e}")
+        return []
+
+def find_exact_transcription(course_name: str, ato_files: List[str]) -> Optional[str]:
+    """Search for exact transcription file match in ATO dataset."""
+    expected_file = course_name.replace("_frames.json", ".json")
+    
+    if expected_file in ato_files:
+        print(f"[MATCH] Found transcription: {expected_file}")
+        return expected_file
+    
+    return None
+
+async def process_single_course(
+    course_file: str,
+    job: ProcessingJob,
+    ato_files: List[str]
+) -> Optional[Dict]:
+    """Process a single course: merge data and fetch transcription if available."""
+    try:
+        print(f"\n[PROCESS] Course: {course_file}")
+        
+        # Download from Helium and Data
+        helium_path = download_file(DATASET_HELIUM, course_file)
+        data_path = download_file(DATASET_DATA, course_file)
+        
+        if not helium_path or not data_path:
+            print(f"[SKIP] Missing data files for {course_file}")
+            return None
+        
+        # Merge frame data
+        merged_frames = merge_course_data(helium_path, data_path)
+        if not merged_frames:
+            print(f"[SKIP] Failed to merge data for {course_file}")
+            return None
+        
+        # Try to find and download transcription
+        transcription_data = None
+        expected_ato_file = find_exact_transcription(course_file, ato_files)
+        
+        if expected_ato_file:
+            ato_path = download_file(DATASET_ATO, expected_ato_file)
+            if ato_path:
+                transcription_data = load_json_file(ato_path)
+                if transcription_data:
+                    job.matched_transcriptions += 1
+        
+        # Prepare output: frames + transcription (or "none")
+        course_name = course_file.replace("_frames.json", "")
+        output = {
+            "course": course_name,
+            "frames": merged_frames,
+            "transcription": transcription_data if transcription_data else "none"
+        }
+        
+        return output
+    
+    except Exception as e:
+        print(f"[ERROR] Failed to process {course_file}: {e}")
+        traceback.print_exc()
+        return None
+
+async def process_all_courses_background(job_id: str):
+    """Main background processing function."""
+    job = jobs_db.get(job_id)
+    if not job:
+        return
+    
+    try:
+        job.status = JobStatus.FETCHING_FILES
+        job.started_at = datetime.utcnow().isoformat()
+        
+        print(f"\n{'='*70}")
+        print(f"[JOB] Starting job: {job_id}")
+        print(f"{'='*70}\n")
+        
+        # Fetch file lists from all datasets
+        print("[INIT] Fetching file lists from datasets...")
+        helium_files = list_dataset_files(DATASET_HELIUM)
+        ato_files = list_dataset_files(DATASET_ATO)
+        
+        # Filter to only _frames.json files from Helium
+        course_files = [f for f in helium_files if f.endswith("_frames.json")]
+        job.total_files = len(course_files)
+        
+        print(f"[INIT] Found {len(course_files)} courses to process")
+        print(f"[INIT] Found {len(ato_files)} files in ATO dataset\n")
+        
+        # Process each course
+        job.status = JobStatus.PROCESSING
+        all_courses = []
+        
+        for idx, course_file in enumerate(course_files):
+            try:
+                course_data = await process_single_course(course_file, job, ato_files)
+                if course_data:
+                    all_courses.append(course_data)
+                
+                job.processed_files = idx + 1
+                job.progress_percent = (job.processed_files / job.total_files) * 100
+                
+                print(f"[PROGRESS] {job.processed_files}/{job.total_files} ({job.progress_percent:.1f}%)")
+                
+                # Small delay to avoid rate limiting
+                await asyncio.sleep(0.05)
+            
+            except Exception as e:
+                print(f"[ERROR] Failed to process {course_file}: {e}")
+                continue
+        
+        # Save combined output
+        timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
+        output_file = OUTPUT_DIR / f"combined_output_{timestamp}.json"
+        
+        print(f"\n[SAVE] Saving output to {output_file}...")
+        with open(output_file, "w") as f:
+            json.dump(all_courses, f, indent=2)
+        
+        job.output_file = str(output_file)
+        job.status = JobStatus.COMPLETED
+        job.completed_at = datetime.utcnow().isoformat()
+        
+        print(f"\n{'='*70}")
+        print(f"[SUCCESS] Job completed!")
+        print(f"{'='*70}")
+        print(f"Total courses processed: {len(all_courses)}")
+        print(f"Transcriptions matched: {job.matched_transcriptions}")
+        print(f"Output file: {output_file}")
+        print(f"File size: {output_file.stat().st_size / (1024*1024):.2f} MB")
+        print(f"{'='*70}\n")
+        
+    except Exception as e:
+        job.status = JobStatus.FAILED
+        job.error_message = str(e)
+        job.completed_at = datetime.utcnow().isoformat()
+        print(f"\n[FAILED] Job failed: {e}")
+        traceback.print_exc()
+
+# ============================================================================
+# API Endpoints
+# ============================================================================
+
+@app.get("/")
+async def root():
+    """Health check endpoint."""
+    return {
+        "status": "running",
+        "service": "Hugging Face Data Processor",
+        "version": "1.0.0",
+        "dashboard": "http://localhost:8000/dashboard"
+    }
+
+@app.post("/api/jobs/create")
+async def create_job(background_tasks: BackgroundTasks):
+    """Create and start a new processing job."""
+    job_id = f"job_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}"
+    
+    job = ProcessingJob(
+        job_id=job_id,
+        status=JobStatus.PENDING,
+        created_at=datetime.utcnow().isoformat()
+    )
+    
+    async with jobs_lock:
+        jobs_db[job_id] = job
+    
+    # Start processing in background
+    background_tasks.add_task(process_all_courses_background, job_id)
+    
+    return {
+        "job_id": job_id,
+        "status": "started",
+        "message": "Processing job created and started"
+    }
+
+@app.get("/api/jobs/{job_id}")
+async def get_job_status(job_id: str):
+    """Get the status of a processing job."""
+    job = jobs_db.get(job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Job not found")
+    
+    return job
+
+@app.get("/api/jobs")
+async def list_jobs():
+    """List all processing jobs."""
+    return {
+        "total_jobs": len(jobs_db),
+        "jobs": list(jobs_db.values())
+    }
+
+@app.post("/api/jobs/{job_id}/cancel")
+async def cancel_job(job_id: str):
+    """Cancel a processing job."""
+    job = jobs_db.get(job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Job not found")
+    
+    if job.status in [JobStatus.COMPLETED, JobStatus.FAILED, JobStatus.CANCELLED]:
+        raise HTTPException(status_code=400, detail="Cannot cancel completed or failed job")
+    
+    job.status = JobStatus.CANCELLED
+    job.error_message = "Job cancelled by user"
+    job.completed_at = datetime.utcnow().isoformat()
+    
+    return {"status": "cancelled", "job_id": job_id}
+
+@app.get("/api/jobs/{job_id}/output")
+async def get_job_output(job_id: str):
+    """Download the combined output JSON for a completed job."""
+    job = jobs_db.get(job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Job not found")
+    
+    if job.status != JobStatus.COMPLETED:
+        raise HTTPException(status_code=400, detail="Job not completed yet")
+    
+    if not job.output_file:
+        raise HTTPException(status_code=404, detail="Output file not found")
+    
+    try:
+        return FileResponse(
+            path=job.output_file,
+            filename=Path(job.output_file).name,
+            media_type="application/json"
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error reading output: {str(e)}")
+
+@app.get("/api/stats")
+async def get_stats():
+    """Get overall statistics about all jobs."""
+    total_jobs = len(jobs_db)
+    completed = sum(1 for j in jobs_db.values() if j.status == JobStatus.COMPLETED)
+    failed = sum(1 for j in jobs_db.values() if j.status == JobStatus.FAILED)
+    processing = sum(1 for j in jobs_db.values() if j.status in [JobStatus.PROCESSING, JobStatus.FETCHING_FILES])
+    
+    total_files = sum(j.total_files for j in jobs_db.values())
+    total_processed = sum(j.processed_files for j in jobs_db.values())
+    total_matched = sum(j.matched_transcriptions for j in jobs_db.values())
+    
+    return {
+        "total_jobs": total_jobs,
+        "completed_jobs": completed,
+        "failed_jobs": failed,
+        "processing_jobs": processing,
+        "total_files_processed": total_processed,
+        "total_files": total_files,
+        "total_transcriptions_matched": total_matched
+    }
+
+# ============================================================================
+# Web Dashboard
+# ============================================================================
+
+DASHBOARD_HTML = """
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Hugging Face Data Processor</title>
+    <style>
+        * {
+            margin: 0;
+            padding: 0;
+            box-sizing: border-box;
+        }
+        
+        body {
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
+            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+            min-height: 100vh;
+            padding: 20px;
+        }
+        
+        .container {
+            max-width: 1200px;
+            margin: 0 auto;
+        }
+        
+        header {
+            background: rgba(255, 255, 255, 0.95);
+            padding: 30px;
+            border-radius: 12px;
+            margin-bottom: 30px;
+            box-shadow: 0 10px 40px rgba(0, 0, 0, 0.1);
+        }
+        
+        h1 {
+            color: #333;
+            margin-bottom: 10px;
+            font-size: 2.5em;
+        }
+        
+        .subtitle {
+            color: #666;
+            font-size: 1.1em;
+        }
+        
+        .controls {
+            display: flex;
+            gap: 15px;
+            margin-top: 20px;
+            flex-wrap: wrap;
+        }
+        
+        button {
+            background: #667eea;
+            color: white;
+            border: none;
+            padding: 12px 24px;
+            border-radius: 6px;
+            cursor: pointer;
+            font-size: 1em;
+            font-weight: 600;
+            transition: all 0.3s ease;
+        }
+        
+        button:hover {
+            background: #764ba2;
+            transform: translateY(-2px);
+            box-shadow: 0 5px 15px rgba(0, 0, 0, 0.2);
+        }
+        
+        button:disabled {
+            background: #ccc;
+            cursor: not-allowed;
+            transform: none;
+        }
+        
+        .grid {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+            gap: 20px;
+            margin-bottom: 30px;
+        }
+        
+        .card {
+            background: rgba(255, 255, 255, 0.95);
+            padding: 25px;
+            border-radius: 12px;
+            box-shadow: 0 10px 40px rgba(0, 0, 0, 0.1);
+        }
+        
+        .card h2 {
+            color: #333;
+            margin-bottom: 15px;
+            font-size: 1.3em;
+        }
+        
+        .stat {
+            display: flex;
+            justify-content: space-between;
+            padding: 10px 0;
+            border-bottom: 1px solid #eee;
+        }
+        
+        .stat:last-child {
+            border-bottom: none;
+        }
+        
+        .stat-label {
+            color: #666;
+            font-weight: 500;
+        }
+        
+        .stat-value {
+            color: #333;
+            font-weight: 700;
+            font-size: 1.1em;
+        }
+        
+        .job-list {
+            background: rgba(255, 255, 255, 0.95);
+            padding: 25px;
+            border-radius: 12px;
+            box-shadow: 0 10px 40px rgba(0, 0, 0, 0.1);
+        }
+        
+        .job-item {
+            padding: 20px;
+            border: 1px solid #eee;
+            border-radius: 8px;
+            margin-bottom: 15px;
+            background: #f9f9f9;
+        }
+        
+        .job-header {
+            display: flex;
+            justify-content: space-between;
+            align-items: center;
+            margin-bottom: 15px;
+        }
+        
+        .job-id {
+            font-family: monospace;
+            color: #667eea;
+            font-weight: 600;
+        }
+        
+        .job-status {
+            padding: 6px 12px;
+            border-radius: 20px;
+            font-size: 0.9em;
+            font-weight: 600;
+        }
+        
+        .status-pending {
+            background: #fff3cd;
+            color: #856404;
+        }
+        
+        .status-processing {
+            background: #cfe2ff;
+            color: #084298;
+        }
+        
+        .status-completed {
+            background: #d1e7dd;
+            color: #0f5132;
+        }
+        
+        .status-failed {
+            background: #f8d7da;
+            color: #842029;
+        }
+        
+        .status-cancelled {
+            background: #e2e3e5;
+            color: #383d41;
+        }
+        
+        .progress-bar {
+            width: 100%;
+            height: 8px;
+            background: #e0e0e0;
+            border-radius: 4px;
+            overflow: hidden;
+            margin: 10px 0;
+        }
+        
+        .progress-fill {
+            height: 100%;
+            background: linear-gradient(90deg, #667eea, #764ba2);
+            transition: width 0.3s ease;
+        }
+        
+        .job-details {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
+            gap: 15px;
+            margin-top: 15px;
+        }
+        
+        .detail {
+            font-size: 0.9em;
+        }
+        
+        .detail-label {
+            color: #666;
+            font-weight: 500;
+        }
+        
+        .detail-value {
+            color: #333;
+            font-weight: 600;
+            margin-top: 5px;
+        }
+        
+        .message {
+            padding: 15px;
+            border-radius: 6px;
+            margin-bottom: 15px;
+            display: none;
+        }
+        
+        .message.show {
+            display: block;
+        }
+        
+        .message.success {
+            background: #d1e7dd;
+            color: #0f5132;
+            border: 1px solid #badbcc;
+        }
+        
+        .message.error {
+            background: #f8d7da;
+            color: #842029;
+            border: 1px solid #f5c2c7;
+        }
+        
+        .loading {
+            display: inline-block;
+            width: 20px;
+            height: 20px;
+            border: 3px solid #f3f3f3;
+            border-top: 3px solid #667eea;
+            border-radius: 50%;
+            animation: spin 1s linear infinite;
+        }
+        
+        @keyframes spin {
+            0% { transform: rotate(0deg); }
+            100% { transform: rotate(360deg); }
+        }
+        
+        .empty {
+            text-align: center;
+            padding: 40px;
+            color: #999;
+        }
+        
+        .footer {
+            text-align: center;
+            color: rgba(255, 255, 255, 0.8);
+            margin-top: 30px;
+            font-size: 0.9em;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <header>
+            <h1>🚀 Hugging Face Data Processor</h1>
+            <p class="subtitle">Automated dataset processing and merging</p>
+            <div class="controls">
+                <button onclick="createJob()" id="createBtn">Create New Job</button>
+                <button onclick="refreshAll()">Refresh</button>
+                <button onclick="clearJobs()">Clear All</button>
+            </div>
+            <div id="message" class="message"></div>
+        </header>
+        
+        <div class="grid">
+            <div class="card">
+                <h2>📊 Statistics</h2>
+                <div class="stat">
+                    <span class="stat-label">Total Jobs</span>
+                    <span class="stat-value" id="totalJobs">0</span>
+                </div>
+                <div class="stat">
+                    <span class="stat-label">Completed</span>
+                    <span class="stat-value" id="completedJobs">0</span>
+                </div>
+                <div class="stat">
+                    <span class="stat-label">Processing</span>
+                    <span class="stat-value" id="processingJobs">0</span>
+                </div>
+                <div class="stat">
+                    <span class="stat-label">Failed</span>
+                    <span class="stat-value" id="failedJobs">0</span>
+                </div>
+            </div>
+            
+            <div class="card">
+                <h2>📈 Progress</h2>
+                <div class="stat">
+                    <span class="stat-label">Files Processed</span>
+                    <span class="stat-value" id="filesProcessed">0</span>
+                </div>
+                <div class="stat">
+                    <span class="stat-label">Total Files</span>
+                    <span class="stat-value" id="totalFiles">0</span>
+                </div>
+                <div class="stat">
+                    <span class="stat-label">Transcriptions Matched</span>
+                    <span class="stat-value" id="transcriptionsMatched">0</span>
+                </div>
+            </div>
+        </div>
+        
+        <div class="job-list">
+            <h2>📋 Processing Jobs</h2>
+            <div id="jobsList" class="empty">No jobs yet. Create one to get started!</div>
+        </div>
+        
+        <div class="footer">
+            <p>Hugging Face Data Processor v1.0 | API: http://localhost:8000/docs</p>
+        </div>
+    </div>
+    
+    <script>
+        const API_BASE = '/api';
+        let autoRefresh = setInterval(refreshAll, 2000);
+        
+        function showMessage(text, type = 'success') {
+            const msg = document.getElementById('message');
+            msg.textContent = text;
+            msg.className = `message show ${type}`;
+            setTimeout(() => msg.classList.remove('show'), 5000);
+        }
+        
+        async function createJob() {
+            try {
+                document.getElementById('createBtn').disabled = true;
+                const res = await fetch(`${API_BASE}/jobs/create`, { method: 'POST' });
+                const data = await res.json();
+                showMessage(`Job created: ${data.job_id}`, 'success');
+                refreshAll();
+            } catch (e) {
+                showMessage(`Error: ${e.message}`, 'error');
+            } finally {
+                document.getElementById('createBtn').disabled = false;
+            }
+        }
+        
+        async function cancelJob(jobId) {
+            if (!confirm(`Cancel job ${jobId}?`)) return;
+            try {
+                await fetch(`${API_BASE}/jobs/${jobId}/cancel`, { method: 'POST' });
+                showMessage('Job cancelled', 'success');
+                refreshAll();
+            } catch (e) {
+                showMessage(`Error: ${e.message}`, 'error');
+            }
+        }
+        
+        async function downloadOutput(jobId) {
+            try {
+                window.location.href = `${API_BASE}/jobs/${jobId}/output`;
+            } catch (e) {
+                showMessage(`Error: ${e.message}`, 'error');
+            }
+        }
+        
+        async function refreshAll() {
+            try {
+                const statsRes = await fetch(`${API_BASE}/stats`);
+                const stats = await statsRes.json();
+                
+                const jobsRes = await fetch(`${API_BASE}/jobs`);
+                const jobsData = await jobsRes.json();
+                
+                // Update stats
+                document.getElementById('totalJobs').textContent = stats.total_jobs;
+                document.getElementById('completedJobs').textContent = stats.completed_jobs;
+                document.getElementById('processingJobs').textContent = stats.processing_jobs;
+                document.getElementById('failedJobs').textContent = stats.failed_jobs;
+                document.getElementById('filesProcessed').textContent = stats.total_files_processed;
+                document.getElementById('totalFiles').textContent = stats.total_files;
+                document.getElementById('transcriptionsMatched').textContent = stats.total_transcriptions_matched;
+                
+                // Update jobs list
+                const jobsList = document.getElementById('jobsList');
+                if (jobsData.jobs.length === 0) {
+                    jobsList.innerHTML = '<div class="empty">No jobs yet. Create one to get started!</div>';
+                } else {
+                    jobsList.innerHTML = jobsData.jobs.map(job => `
+                        <div class="job-item">
+                            <div class="job-header">
+                                <span class="job-id">${job.job_id}</span>
+                                <span class="job-status status-${job.status}">${job.status.toUpperCase()}</span>
+                            </div>
+                            ${job.status === 'processing' || job.status === 'fetching_files' ? `
+                                <div class="progress-bar">
+                                    <div class="progress-fill" style="width: ${job.progress_percent}%"></div>
+                                </div>
+                            ` : ''}
+                            <div class="job-details">
+                                <div class="detail">
+                                    <div class="detail-label">Files</div>
+                                    <div class="detail-value">${job.processed_files}/${job.total_files}</div>
+                                </div>
+                                <div class="detail">
+                                    <div class="detail-label">Transcriptions</div>
+                                    <div class="detail-value">${job.matched_transcriptions}</div>
+                                </div>
+                                <div class="detail">
+                                    <div class="detail-label">Created</div>
+                                    <div class="detail-value">${new Date(job.created_at).toLocaleString()}</div>
+                                </div>
+                            </div>
+                            ${job.error_message ? `<div style="color: #d32f2f; margin-top: 10px;">Error: ${job.error_message}</div>` : ''}
+                            <div style="margin-top: 15px; display: flex; gap: 10px;">
+                                ${job.status === 'processing' || job.status === 'pending' || job.status === 'fetching_files' ? `
+                                    <button onclick="cancelJob('${job.job_id}')" style="background: #d32f2f;">Cancel</button>
+                                ` : ''}
+                                ${job.status === 'completed' && job.output_file ? `
+                                    <button onclick="downloadOutput('${job.job_id}')" style="background: #4caf50;">Download Output</button>
+                                ` : ''}
+                            </div>
+                        </div>
+                    `).join('');
+                }
+            } catch (e) {
+                console.error('Refresh error:', e);
+            }
+        }
+        
+        function clearJobs() {
+            if (confirm('Clear all jobs from memory? (This does not delete output files)')) {
+                location.reload();
+            }
+        }
+        
+        // Initial load
+        refreshAll();
+    </script>
+</body>
+</html>
+"""
+
+@app.get("/dashboard")
+async def dashboard():
+    """Serve the web dashboard."""
+    return HTMLResponse(content=DASHBOARD_HTML)
+
+# ============================================================================
+# Main Entry Point
+# ============================================================================
+
+def main():
+    """Start the server."""
+    print("\n" + "="*70)
+    print("🚀 Hugging Face Data Processor - Starting Server")
+    print("="*70)
+    print(f"API Base URL: http://localhost:8000")
+    print(f"Dashboard:   http://localhost:8000/dashboard")
+    print(f"Swagger UI:  http://localhost:8000/docs")
+    print(f"Output Dir:  {OUTPUT_DIR.absolute()}")
+    print("="*70 + "\n")
+    
+    uvicorn.run(
+        app,
+        host="0.0.0.0",
+        port=8000,
+        log_level="info"
+    )
+
+if __name__ == "__main__":
+    main()