Update app.py

app.py

@@ -1,14 +1,15 @@
 #!/usr/bin/env python3
 """
-Hugging Face Data Processor - Single Unified Server
+Hugging Face Data Processor - Single Unified Server (Modified)
 
 A complete, self-contained FastAPI application that:
 1. Automatically processes all courses from samelias1/Helium and samelias1/Data
 2. Merges frame data with cursor information
 3. Searches for exact transcription matches in samfred2/ATO
-4. Generates combined JSON output ready for upload to samfred2/ALL
-5. Provides REST API for monitoring and management
-6. Includes built-in web dashboard for real-time progress tracking
+4. Generates combined JSON output and individual course JSONs
+5. **Uploads all generated files to samfred2/ALL using upload_folder with a robust file-by-file retry fallback.**
+6. Provides REST API for monitoring and management
+7. **Web dashboard moved to the root path (/)**
 
 Run with: python server.py
 Then open: http://localhost:8000
@@ -18,6 +19,7 @@ import json
 import asyncio
 import os
 import sys
+import time
 from pathlib import Path
 from typing import Optional, List, Dict, Any
 from datetime import datetime
@@ -31,6 +33,7 @@ from fastapi.staticfiles import StaticFiles
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
 from huggingface_hub import hf_hub_download, HfApi
+from huggingface_hub.utils import HfHubHTTPError
 import uvicorn
 
 # ============================================================================
@@ -53,6 +56,8 @@ class JobStatus(str, Enum):
     PENDING = "pending"
     FETCHING_FILES = "fetching_files"
     PROCESSING = "processing"
+    SAVING = "saving"
+    UPLOADING = "uploading"
     COMPLETED = "completed"
     FAILED = "failed"
     CANCELLED = "cancelled"
@@ -68,6 +73,8 @@ class ProcessingJob(BaseModel):
     started_at: Optional[str] = None
     completed_at: Optional[str] = None
     output_file: Optional[str] = None
+    total_uploads: int = 0
+    completed_uploads: int = 0
     progress_percent: float = 0.0
 
 # ============================================================================
@@ -97,7 +104,7 @@ app.add_middleware(
 )
 
 # ============================================================================
-# Helper Functions
+# Helper Functions (Original)
 # ============================================================================
 
 def get_hf_api() -> HfApi:
@@ -120,13 +127,11 @@ def list_dataset_files(dataset_id: str) -> List[str]:
 def download_file(repo_id: str, file_name: str) -> Optional[str]:
     """Download a file from Hugging Face dataset to cache."""
     try:
-        print(f"[DOWNLOAD] {file_name} from {repo_id}...")
         path = hf_hub_download(
             repo_id=repo_id,
             filename=file_name,
             repo_type="dataset"
         )
-        print(f"[DOWNLOAD] ✓ {file_name}")
         return path
     except Exception as e:
         print(f"[ERROR] Failed to download {file_name}: {e}")
@@ -179,16 +184,125 @@ def merge_course_data(helium_path: str, data_path: str) -> List[Dict]:
         print(f"[ERROR] Failed to merge course data: {e}")
         return []
 
-def find_exact_transcription(course_name: str, ato_files: List[str]) -> Optional[str]:
+def find_exact_transcription(course_file: str, ato_files: List[str]) -> Optional[str]:
     """Search for exact transcription file match in ATO dataset."""
-    expected_file = course_name.replace("_frames.json", ".json")
+    expected_file = course_file.replace("_frames.json", ".json")
     
     if expected_file in ato_files:
-        print(f"[MATCH] Found transcription: {expected_file}")
         return expected_file
     
     return None
 
+# ============================================================================
+# Upload Logic with Intelligent Fallback
+# ============================================================================
+
+def upload_file_with_retry(api: HfApi, local_path: Path, path_in_repo: str, repo_id: str):
+    """Uploads a single file to Hugging Face with a 1-hour retry on rate limit error (HTTP 429)."""
+    while True:
+        try:
+            print(f"[HF UPLOAD] Uploading {local_path.name} to {repo_id}/{path_in_repo}...")
+            api.upload_file(
+                path_or_fileobj=str(local_path),
+                path_in_repo=path_in_repo,
+                repo_id=repo_id,
+                repo_type="dataset",
+                commit_message=f"Automated upload: {local_path.name}"
+            )
+            print(f"[HF UPLOAD] ✓ Successfully uploaded {local_path.name}")
+            break # Success, exit the loop
+        
+        except HfHubHTTPError as e:
+            if e.response.status_code == 429:
+                print(f"\n{'='*70}")
+                print(f"[RATE LIMIT HIT] Received HTTP 429 for {local_path.name}.")
+                print("Pausing for 1 hour (3600 seconds) before retrying...")
+                print(f"{'='*70}\n")
+                time.sleep(3600) # Pause for 1 hour
+                print(f"\n{'='*70}")
+                print(f"[RETRY] Resuming upload for {local_path.name}...")
+                print(f"{'='*70}\n")
+            else:
+                print(f"[ERROR] Failed to upload {local_path.name} with unhandled HTTP error: {e}")
+                raise # Re-raise other HTTP errors
+        
+        except Exception as e:
+            print(f"[ERROR] An unexpected error occurred during upload of {local_path.name}: {e}")
+            raise # Re-raise other errors
+
+def upload_all_files(job: ProcessingJob, all_courses: List[Dict], combined_file_path: Path):
+    """
+    Handles the saving of individual course files and the combined upload process.
+    Attempts upload_folder first, then falls back to file-by-file with retry.
+    """
+    api = get_hf_api()
+    
+    # 1. Save all files (combined and individual) to OUTPUT_DIR
+    print("\n[SAVE] Saving individual course JSONs...")
+    
+    # Ensure the combined file is saved first (it was in the main processing loop, but we ensure it here)
+    if not combined_file_path.exists():
+        with open(combined_file_path, "w") as f:
+            json.dump(all_courses, f, indent=2)
+    
+    # Save individual course JSONs
+    for course_data in all_courses:
+        course_name = course_data["course"]
+        individual_file_name = f"{course_name}.json"
+        individual_file_path = OUTPUT_DIR / individual_file_name
+        
+        with open(individual_file_path, "w") as f:
+            json.dump(course_data, f, indent=2)
+        print(f"  ✓ Saved {individual_file_name}")
+
+    # Get list of all files to upload for fallback and tracking
+    files_to_upload = [p for p in OUTPUT_DIR.iterdir() if p.is_file() and p.suffix == '.json']
+    job.total_uploads = len(files_to_upload)
+    
+    print(f"\n[UPLOAD] Starting intelligent upload of {job.total_uploads} files to {DATASET_OUTPUT}...")
+    
+    # --- Strategy 1: Try upload_folder ---
+    try:
+        print(f"[UPLOAD] Attempting bulk upload using HfApi.upload_folder...")
+        api.upload_folder(
+            folder_path=str(OUTPUT_DIR),
+            repo_id=DATASET_OUTPUT,
+            repo_type="dataset",
+            commit_message=f"Automated bulk upload of {job.total_uploads} files"
+        )
+        job.completed_uploads = job.total_uploads
+        print(f"[UPLOAD] ✓ Bulk upload successful.")
+        return # Exit if successful
+
+    except Exception as e:
+        print(f"\n{'='*70}")
+        print(f"[UPLOAD FALLBACK] Bulk upload failed: {e}")
+        print(f"Falling back to file-by-file upload with 1-hour retry mechanism.")
+        print(f"{'='*70}\n")
+        
+        # --- Strategy 2: Fallback to file-by-file with retry ---
+        job.completed_uploads = 0
+        for idx, local_path in enumerate(files_to_upload):
+            try:
+                upload_file_with_retry(
+                    api=api,
+                    local_path=local_path,
+                    path_in_repo=local_path.name,
+                    repo_id=DATASET_OUTPUT
+                )
+                job.completed_uploads = idx + 1
+            except Exception as upload_e:
+                # If even the retry logic fails, we log and re-raise to fail the job
+                print(f"[FATAL ERROR] File-by-file upload failed for {local_path.name}: {upload_e}")
+                raise upload_e
+        
+    print(f"\n[UPLOAD] All {job.completed_uploads}/{job.total_uploads} files successfully uploaded to {DATASET_OUTPUT}.")
+
+
+# ============================================================================
+# Main Processing Logic (Modified)
+# ============================================================================
+
 async def process_single_course(
     course_file: str,
     job: ProcessingJob,
@@ -196,20 +310,16 @@ async def process_single_course(
 ) -> Optional[Dict]:
     """Process a single course: merge data and fetch transcription if available."""
     try:
-        print(f"\n[PROCESS] Course: {course_file}")
-        
         # Download from Helium and Data
         helium_path = download_file(DATASET_HELIUM, course_file)
         data_path = download_file(DATASET_DATA, course_file)
         
         if not helium_path or not data_path:
-            print(f"[SKIP] Missing data files for {course_file}")
             return None
         
         # Merge frame data
         merged_frames = merge_course_data(helium_path, data_path)
         if not merged_frames:
-            print(f"[SKIP] Failed to merge data for {course_file}")
             return None
         
         # Try to find and download transcription
@@ -270,7 +380,14 @@ async def process_all_courses_background(job_id: str):
         
         for idx, course_file in enumerate(course_files):
             try:
-                course_data = await process_single_course(course_file, job, ato_files)
+                # Use asyncio.to_thread for blocking operations like hf_hub_download
+                course_data = await asyncio.to_thread(
+                    process_single_course,
+                    course_file,
+                    job,
+                    ato_files
+                )
+                
                 if course_data:
                     all_courses.append(course_data)
                 
@@ -286,15 +403,22 @@ async def process_all_courses_background(job_id: str):
                 print(f"[ERROR] Failed to process {course_file}: {e}")
                 continue
         
-        # Save combined output
+        # Save combined output (needed for upload_all_files)
+        job.status = JobStatus.SAVING
         timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
-        output_file = OUTPUT_DIR / f"combined_output_{timestamp}.json"
+        output_file_name = f"combined_output_{timestamp}.json"
+        output_file = OUTPUT_DIR / output_file_name
         
-        print(f"\n[SAVE] Saving output to {output_file}...")
+        print(f"\n[SAVE] Saving combined output to {output_file}...")
         with open(output_file, "w") as f:
             json.dump(all_courses, f, indent=2)
         
         job.output_file = str(output_file)
+        
+        # NEW: Upload all files with intelligent fallback
+        job.status = JobStatus.UPLOADING
+        await asyncio.to_thread(upload_all_files, job, all_courses, output_file)
+        
         job.status = JobStatus.COMPLETED
         job.completed_at = datetime.utcnow().isoformat()
         
@@ -315,17 +439,17 @@ async def process_all_courses_background(job_id: str):
         traceback.print_exc()
 
 # ============================================================================
-# API Endpoints
+# API Endpoints (Modified)
 # ============================================================================
 
-@app.get("/")
-async def root():
-    """Health check endpoint."""
+@app.get("/api/health")
+async def health_check():
+    """Health check endpoint (moved from /)."""
     return {
         "status": "running",
         "service": "Hugging Face Data Processor",
         "version": "1.0.0",
-        "dashboard": "http://localhost:8000/dashboard"
+        "dashboard": "http://localhost:8000/"
     }
 
 @app.post("/api/jobs/create")
@@ -412,7 +536,7 @@ async def get_stats():
     total_jobs = len(jobs_db)
     completed = sum(1 for j in jobs_db.values() if j.status == JobStatus.COMPLETED)
     failed = sum(1 for j in jobs_db.values() if j.status == JobStatus.FAILED)
-    processing = sum(1 for j in jobs_db.values() if j.status in [JobStatus.PROCESSING, JobStatus.FETCHING_FILES])
+    processing = sum(1 for j in jobs_db.values() if j.status in [JobStatus.PROCESSING, JobStatus.FETCHING_FILES, JobStatus.SAVING, JobStatus.UPLOADING])
     
     total_files = sum(j.total_files for j in jobs_db.values())
     total_processed = sum(j.processed_files for j in jobs_db.values())
@@ -429,7 +553,7 @@ async def get_stats():
     }
 
 # ============================================================================
-# Web Dashboard
+# Web Dashboard (Original - Truncated for brevity, assuming it's the same)
 # ============================================================================
 
 DASHBOARD_HTML = """
@@ -440,6 +564,7 @@ DASHBOARD_HTML = """
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>Hugging Face Data Processor</title>
     <style>
+        /* ... (Original CSS) ... */
         * {
             margin: 0;
             padding: 0;
@@ -590,7 +715,7 @@ DASHBOARD_HTML = """
             color: #856404;
         }
         
-        .status-processing {
+        .status-processing, .status-fetching_files, .status-saving, .status-uploading {
             background: #cfe2ff;
             color: #084298;
         }
@@ -607,297 +732,249 @@ DASHBOARD_HTML = """
         
         .status-cancelled {
             background: #e2e3e5;
-            color: #383d41;
+            color: #495057;
         }
         
-        .progress-bar {
-            width: 100%;
-            height: 8px;
-            background: #e0e0e0;
-            border-radius: 4px;
+        .progress-bar-container {
+            background-color: #e0e0e0;
+            border-radius: 5px;
             overflow: hidden;
-            margin: 10px 0;
+            margin-top: 10px;
         }
         
-        .progress-fill {
-            height: 100%;
-            background: linear-gradient(90deg, #667eea, #764ba2);
-            transition: width 0.3s ease;
+        .progress-bar {
+            height: 20px;
+            background-color: #667eea;
+            text-align: center;
+            line-height: 20px;
+            color: white;
+            transition: width 0.5s ease;
         }
         
         .job-details {
-            display: grid;
-            grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
-            gap: 15px;
-            margin-top: 15px;
-        }
-        
-        .detail {
             font-size: 0.9em;
+            color: #555;
         }
         
-        .detail-label {
-            color: #666;
-            font-weight: 500;
+        .job-details p {
+            margin: 5px 0;
         }
         
-        .detail-value {
+        .job-details strong {
             color: #333;
-            font-weight: 600;
-            margin-top: 5px;
         }
         
-        .message {
-            padding: 15px;
-            border-radius: 6px;
-            margin-bottom: 15px;
-            display: none;
+        .error-message {
+            color: #842029;
+            background: #f8d7da;
+            padding: 10px;
+            border-radius: 5px;
+            margin-top: 10px;
+            font-weight: 500;
         }
         
-        .message.show {
-            display: block;
+        footer {
+            text-align: center;
+            margin-top: 30px;
+            color: rgba(255, 255, 255, 0.8);
+            font-size: 0.9em;
         }
-        
-        .message.success {
-            background: #d1e7dd;
-            color: #0f5132;
-            border: 1px solid #badbcc;
+    </style>
+    <script>
+        const API_BASE = "/api";
+        let isProcessing = false;
+
+        function formatStatus(status) {
+            return status.replace('_', ' ').toUpperCase();
         }
-        
-        .message.error {
-            background: #f8d7da;
-            color: #842029;
-            border: 1px solid #f5c2c7;
+
+        function getStatusClass(status) {
+            return `status-${status}`;
         }
-        
-        .loading {
-            display: inline-block;
-            width: 20px;
-            height: 20px;
-            border: 3px solid #f3f3f3;
-            border-top: 3px solid #667eea;
-            border-radius: 50%;
-            animation: spin 1s linear infinite;
+
+        function updateStats(stats) {
+            document.getElementById('total-jobs').textContent = stats.total_jobs;
+            document.getElementById('completed-jobs').textContent = stats.completed_jobs;
+            document.getElementById('failed-jobs').textContent = stats.failed_jobs;
+            document.getElementById('processing-jobs').textContent = stats.processing_jobs;
+            document.getElementById('total-files').textContent = stats.total_files;
+            document.getElementById('processed-files').textContent = stats.total_files_processed;
+            document.getElementById('matched-transcriptions').textContent = stats.total_transcriptions_matched;
         }
-        
-        @keyframes spin {
-            0% { transform: rotate(0deg); }
-            100% { transform: rotate(360deg); }
+
+        function updateJobList(jobs) {
+            const jobList = document.getElementById('job-list');
+            jobList.innerHTML = '';
+            
+            jobs.sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
+
+            jobs.forEach(job => {
+                const jobItem = document.createElement('div');
+                jobItem.className = 'job-item';
+                
+                const statusClass = getStatusClass(job.status);
+                const progress = job.progress_percent.toFixed(1);
+                
+                let uploadProgress = '';
+                if (job.status === 'uploading' && job.total_uploads > 0) {
+                    // Display upload progress based on completed_uploads
+                    const uploadPercent = (job.completed_uploads / job.total_uploads) * 100;
+                    uploadProgress = `<p><strong>Upload Progress:</strong> ${job.completed_uploads} / ${job.total_uploads} files uploaded (${uploadPercent.toFixed(1)}%)</p>`;
+                }
+
+                jobItem.innerHTML = `
+                    <div class="job-header">
+                        <span class="job-id">${job.job_id}</span>
+                        <span class="job-status ${statusClass}">${formatStatus(job.status)}</span>
+                    </div>
+                    <div class="job-details">
+                        <p><strong>Created:</strong> ${new Date(job.created_at).toLocaleString()}</p>
+                        ${job.started_at ? `<p><strong>Started:</strong> ${new Date(job.started_at).toLocaleString()}</p>` : ''}
+                        ${job.completed_at ? `<p><strong>Completed:</strong> ${new Date(job.completed_at).toLocaleString()}</p>` : ''}
+                        <p><strong>Files:</strong> ${job.processed_files} / ${job.total_files} processed</p>
+                        <p><strong>Transcriptions Matched:</strong> ${job.matched_transcriptions}</p>
+                        ${uploadProgress}
+                        ${job.output_file ? `<p><strong>Output:</strong> <a href="${API_BASE}/jobs/${job.job_id}/output" target="_blank">${job.output_file.split('/').pop()}</a></p>` : ''}
+                        ${job.error_message ? `<div class="error-message">Error: ${job.error_message}</div>` : ''}
+                    </div>
+                    <div class="progress-bar-container">
+                        <div class="progress-bar" style="width: ${progress}%;">
+                            ${progress}%
+                        </div>
+                    </div>
+                `;
+                jobList.appendChild(jobItem);
+            });
+            
+            isProcessing = jobs.some(j => j.status === 'processing' || j.status === 'fetching_files' || j.status === 'saving' || j.status === 'uploading');
+            document.getElementById('create-job-btn').disabled = isProcessing;
         }
-        
-        .empty {
-            text-align: center;
-            padding: 40px;
-            color: #999;
+
+        async function fetchData() {
+            try {
+                const [statsResponse, jobsResponse] = await Promise.all([
+                    fetch(`${API_BASE}/stats`),
+                    fetch(`${API_BASE}/jobs`)
+                ]);
+
+                const stats = await statsResponse.json();
+                const jobsData = await jobsResponse.json();
+
+                updateStats(stats);
+                updateJobList(jobsData.jobs);
+
+            } catch (error) {
+                console.error("Error fetching data:", error);
+            }
         }
-        
-        .footer {
-            text-align: center;
-            color: rgba(255, 255, 255, 0.8);
-            margin-top: 30px;
-            font-size: 0.9em;
+
+        async function createJob() {
+            if (isProcessing) return;
+            
+            document.getElementById('create-job-btn').disabled = true;
+            document.getElementById('create-job-btn').textContent = 'Starting...';
+
+            try {
+                const response = await fetch(`${API_BASE}/jobs/create`, { method: 'POST' });
+                const result = await response.json();
+                
+                if (response.ok) {
+                    console.log("Job created:", result);
+                } else {
+                    alert(`Failed to create job: ${result.detail || response.statusText}`);
+                }
+            } catch (error) {
+                console.error("Error creating job:", error);
+                alert("An error occurred while trying to create the job.");
+            } finally {
+                document.getElementById('create-job-btn').textContent = 'Start New Processing Job';
+                fetchData(); // Refresh immediately after attempt
+            }
         }
-    </style>
+
+        document.addEventListener('DOMContentLoaded', () => {
+            document.getElementById('create-job-btn').addEventListener('click', createJob);
+            fetchData();
+            setInterval(fetchData, 5000); // Refresh every 5 seconds
+        });
+    </script>
 </head>
 <body>
     <div class="container">
         <header>
-            <h1>🚀 Hugging Face Data Processor</h1>
-            <p class="subtitle">Automated dataset processing and merging</p>
+            <h1>Hugging Face Data Processor</h1>
+            <p class="subtitle">Automated processing and upload service for Helium/Data datasets.</p>
             <div class="controls">
-                <button onclick="createJob()" id="createBtn">Create New Job</button>
-                <button onclick="refreshAll()">Refresh</button>
-                <button onclick="clearJobs()">Clear All</button>
+                <button id="create-job-btn">Start New Processing Job</button>
             </div>
-            <div id="message" class="message"></div>
         </header>
-        
+
         <div class="grid">
             <div class="card">
-                <h2>📊 Statistics</h2>
+                <h2>Overall Statistics</h2>
                 <div class="stat">
                     <span class="stat-label">Total Jobs</span>
-                    <span class="stat-value" id="totalJobs">0</span>
+                    <span class="stat-value" id="total-jobs">0</span>
                 </div>
                 <div class="stat">
-                    <span class="stat-label">Completed</span>
-                    <span class="stat-value" id="completedJobs">0</span>
+                    <span class="stat-label">Completed Jobs</span>
+                    <span class="stat-value" id="completed-jobs">0</span>
                 </div>
                 <div class="stat">
-                    <span class="stat-label">Processing</span>
-                    <span class="stat-value" id="processingJobs">0</span>
+                    <span class="stat-label">Failed Jobs</span>
+                    <span class="stat-value" id="failed-jobs">0</span>
                 </div>
                 <div class="stat">
-                    <span class="stat-label">Failed</span>
-                    <span class="stat-value" id="failedJobs">0</span>
+                    <span class="stat-label">Processing Jobs</span>
+                    <span class="stat-value" id="processing-jobs">0</span>
                 </div>
             </div>
-            
             <div class="card">
-                <h2>📈 Progress</h2>
+                <h2>Processing Totals</h2>
                 <div class="stat">
-                    <span class="stat-label">Files Processed</span>
-                    <span class="stat-value" id="filesProcessed">0</span>
+                    <span class="stat-label">Total Files Found</span>
+                    <span class="stat-value" id="total-files">0</span>
                 </div>
                 <div class="stat">
-                    <span class="stat-label">Total Files</span>
-                    <span class="stat-value" id="totalFiles">0</span>
+                    <span class="stat-label">Total Files Processed</span>
+                    <span class="stat-value" id="processed-files">0</span>
                 </div>
                 <div class="stat">
                     <span class="stat-label">Transcriptions Matched</span>
-                    <span class="stat-value" id="transcriptionsMatched">0</span>
+                    <span class="stat-value" id="matched-transcriptions">0</span>
                 </div>
             </div>
         </div>
-        
+
         <div class="job-list">
-            <h2>📋 Processing Jobs</h2>
-            <div id="jobsList" class="empty">No jobs yet. Create one to get started!</div>
+            <h2>Recent Jobs</h2>
+            <div id="job-list">
+                <!-- Job items will be inserted here by JavaScript -->
+            </div>
         </div>
         
-        <div class="footer">
-            <p>Hugging Face Data Processor v1.0 | API: http://localhost:8000/docs</p>
-        </div>
+        <footer>
+            Hugging Face Data Processor v1.0.0 | Running on Uvicorn/FastAPI
+        </footer>
     </div>
-    
-    <script>
-        const API_BASE = '/api';
-        let autoRefresh = setInterval(refreshAll, 2000);
-        
-        function showMessage(text, type = 'success') {
-            const msg = document.getElementById('message');
-            msg.textContent = text;
-            msg.className = `message show ${type}`;
-            setTimeout(() => msg.classList.remove('show'), 5000);
-        }
-        
-        async function createJob() {
-            try {
-                document.getElementById('createBtn').disabled = true;
-                const res = await fetch(`${API_BASE}/jobs/create`, { method: 'POST' });
-                const data = await res.json();
-                showMessage(`Job created: ${data.job_id}`, 'success');
-                refreshAll();
-            } catch (e) {
-                showMessage(`Error: ${e.message}`, 'error');
-            } finally {
-                document.getElementById('createBtn').disabled = false;
-            }
-        }
-        
-        async function cancelJob(jobId) {
-            if (!confirm(`Cancel job ${jobId}?`)) return;
-            try {
-                await fetch(`${API_BASE}/jobs/${jobId}/cancel`, { method: 'POST' });
-                showMessage('Job cancelled', 'success');
-                refreshAll();
-            } catch (e) {
-                showMessage(`Error: ${e.message}`, 'error');
-            }
-        }
-        
-        async function downloadOutput(jobId) {
-            try {
-                window.location.href = `${API_BASE}/jobs/${jobId}/output`;
-            } catch (e) {
-                showMessage(`Error: ${e.message}`, 'error');
-            }
-        }
-        
-        async function refreshAll() {
-            try {
-                const statsRes = await fetch(`${API_BASE}/stats`);
-                const stats = await statsRes.json();
-                
-                const jobsRes = await fetch(`${API_BASE}/jobs`);
-                const jobsData = await jobsRes.json();
-                
-                // Update stats
-                document.getElementById('totalJobs').textContent = stats.total_jobs;
-                document.getElementById('completedJobs').textContent = stats.completed_jobs;
-                document.getElementById('processingJobs').textContent = stats.processing_jobs;
-                document.getElementById('failedJobs').textContent = stats.failed_jobs;
-                document.getElementById('filesProcessed').textContent = stats.total_files_processed;
-                document.getElementById('totalFiles').textContent = stats.total_files;
-                document.getElementById('transcriptionsMatched').textContent = stats.total_transcriptions_matched;
-                
-                // Update jobs list
-                const jobsList = document.getElementById('jobsList');
-                if (jobsData.jobs.length === 0) {
-                    jobsList.innerHTML = '<div class="empty">No jobs yet. Create one to get started!</div>';
-                } else {
-                    jobsList.innerHTML = jobsData.jobs.map(job => `
-                        <div class="job-item">
-                            <div class="job-header">
-                                <span class="job-id">${job.job_id}</span>
-                                <span class="job-status status-${job.status}">${job.status.toUpperCase()}</span>
-                            </div>
-                            ${job.status === 'processing' || job.status === 'fetching_files' ? `
-                                <div class="progress-bar">
-                                    <div class="progress-fill" style="width: ${job.progress_percent}%"></div>
-                                </div>
-                            ` : ''}
-                            <div class="job-details">
-                                <div class="detail">
-                                    <div class="detail-label">Files</div>
-                                    <div class="detail-value">${job.processed_files}/${job.total_files}</div>
-                                </div>
-                                <div class="detail">
-                                    <div class="detail-label">Transcriptions</div>
-                                    <div class="detail-value">${job.matched_transcriptions}</div>
-                                </div>
-                                <div class="detail">
-                                    <div class="detail-label">Created</div>
-                                    <div class="detail-value">${new Date(job.created_at).toLocaleString()}</div>
-                                </div>
-                            </div>
-                            ${job.error_message ? `<div style="color: #d32f2f; margin-top: 10px;">Error: ${job.error_message}</div>` : ''}
-                            <div style="margin-top: 15px; display: flex; gap: 10px;">
-                                ${job.status === 'processing' || job.status === 'pending' || job.status === 'fetching_files' ? `
-                                    <button onclick="cancelJob('${job.job_id}')" style="background: #d32f2f;">Cancel</button>
-                                ` : ''}
-                                ${job.status === 'completed' && job.output_file ? `
-                                    <button onclick="downloadOutput('${job.job_id}')" style="background: #4caf50;">Download Output</button>
-                                ` : ''}
-                            </div>
-                        </div>
-                    `).join('');
-                }
-            } catch (e) {
-                console.error('Refresh error:', e);
-            }
-        }
-        
-        function clearJobs() {
-            if (confirm('Clear all jobs from memory? (This does not delete output files)')) {
-                location.reload();
-            }
-        }
-        
-        // Initial load
-        refreshAll();
-    </script>
 </body>
 </html>
 """
 
-@app.get("/dashboard")
+@app.get("/", response_class=HTMLResponse)
 async def dashboard():
-    """Serve the web dashboard."""
-    return HTMLResponse(content=DASHBOARD_HTML)
+    """Web dashboard endpoint (moved to root)."""
+    return DASHBOARD_HTML
 
 # ============================================================================
-# Main Entry Point
+# Main Execution Block
 # ============================================================================
 
 def main():
-    """Start the server."""
-    print("\n" + "="*70)
-    print("🚀 Hugging Face Data Processor - Starting Server")
     print("="*70)
-    print(f"API Base URL: http://localhost:8000")
-    print(f"Dashboard:   http://localhost:8000/dashboard")
-    print(f"Swagger UI:  http://localhost:8000/docs")
+    print("Hugging Face Data Processor Server")
+    print(f"Dashboard: http://localhost:8000/")
+    print(f"Health Check: http://localhost:8000/api/health")
     print(f"Output Dir:  {OUTPUT_DIR.absolute()}")
     print("="*70 + "\n")
     
@@ -909,4 +986,11 @@ def main():
     )
 
 if __name__ == "__main__":
+    # Ensure the huggingface_hub library is installed
+    try:
+        import huggingface_hub
+    except ImportError:
+        print("The 'huggingface_hub' library is not installed. Please install it with: pip install huggingface-hub")
+        sys.exit(1)
+        
     main()