Update app.py

app.py

@@ -1,704 +1,181 @@
-import os
-import json
-import requests
-import subprocess
-import shutil
-import time
-import sys
-import threading
-from typing import Dict, List, Optional, Any
-from huggingface_hub import HfApi, hf_hub_url
-from fastapi import FastAPI, HTTPException
-from fastapi.responses import JSONResponse
-import uvicorn
-
-# Fix Unicode encoding for Windows
-if sys.platform == 'win32':
-    import io
-    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
-
-# Initialize FastAPI app
-app = FastAPI(title="Audio Transcriber", description="Audio transcription and upload service")
-
-# ==== CONFIGURATION ====
-HF_TOKEN = os.environ.get("HF_TOKEN", "")
-SOURCE_REPO_ID = "Samfredoly/BG_Vid"  # Fetch audio files from here
-TARGET_REPO_ID = "samfred2/A_Text"  # Upload transcriptions here
-REFERENCE_REPO_ID = "Fred808/BG3"  # Reference repo to match audio filenames
-
-# Path Configuration
-DOWNLOAD_FOLDER = "downloads_audio"
-TRANSCRIPTIONS_FOLDER = "transcriptions"
-LOCAL_STATE_FOLDER = ".state_audio"
-
-os.makedirs(DOWNLOAD_FOLDER, exist_ok=True)
-os.makedirs(TRANSCRIPTIONS_FOLDER, exist_ok=True)
-os.makedirs(LOCAL_STATE_FOLDER, exist_ok=True)
-
-# State Files
-FAILED_FILES_LOG = "failed_audio_files.log"
-HF_STATE_FILE = "processing_audio_state.json"
-
-# Processing Parameters
-PROCESSING_DELAY = 2
-MAX_RETRIES = 3
-MIN_FREE_SPACE_GB = 1
-WHISPER_MODEL = "small"  # Whisper model size
-
-# Initialize HF API
-hf_api = HfApi(token=HF_TOKEN)
-
-# Global State
-processing_status = {
-    "is_running": False,
-    "current_file": None,
-    "total_files": 0,
-    "processed_files": 0,
-    "failed_files": 0,
-    "transcribed_files": 0,
-    "last_update": None,
-    "logs": []
-}
-
-def log_message(message: str, level: str = "INFO"):
-    """Log messages with timestamp"""
-    timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
-    log_entry = f"[{timestamp}] {level}: {message}"
-    print(log_entry)
-    processing_status["logs"].append(log_entry)
-    processing_status["last_update"] = timestamp
-    if len(processing_status["logs"]) > 100:
-        processing_status["logs"] = processing_status["logs"][-100:]
-
-def log_failed_file(filename: str, error: str):
-    """Log failed files to persistent file"""
-    with open(FAILED_FILES_LOG, "a") as f:
-        f.write(f"{time.strftime('%Y-%m-%d %H:%M:%S')} - {filename}: {error}\n")
-
-def get_disk_usage(path: str) -> Dict[str, float]:
-    """Get disk usage statistics in GB"""
-    statvfs = os.statvfs(path)
-    total = statvfs.f_frsize * statvfs.f_blocks / (1024**3)
-    free = statvfs.f_frsize * statvfs.f_bavail / (1024**3)
-    used = total - free
-    return {"total": total, "free": free, "used": used}
-
-def check_disk_space(path: str = ".") -> bool:
-    """Check if there's enough disk space"""
-    disk_info = get_disk_usage(path)
-    if disk_info["free"] < MIN_FREE_SPACE_GB:
-        log_message(f'⚠️ Low disk space: {disk_info["free"]:.2f}GB free, {disk_info["used"]:.2f}GB used')
-        return False
-    return True
-
-def cleanup_temp_files():
-    """Clean up temporary files to free space"""
-    log_message("🧹 Cleaning up temporary files...", "INFO")
-    
-    current_file = processing_status.get("current_file")
-    for file in os.listdir(DOWNLOAD_FOLDER):
-        if file != current_file and file.endswith((".wav", ".mp3")):
-            try:
-                os.remove(os.path.join(DOWNLOAD_FOLDER, file))
-                log_message(f"🗑️ Removed old download: {file}", "INFO")
-            except:
-                pass
-
-def load_json_state(file_path: str, default_value: Dict[str, Any]) -> Dict[str, Any]:
-    """Load state from JSON file with migration logic for new structure."""
-    if os.path.exists(file_path):
-        try:
-            with open(file_path, "r") as f:
-                data = json.load(f)
-                
-                if "file_states" not in data or not isinstance(data["file_states"], dict):
-                    log_message("ℹ️ Initializing 'file_states' dictionary.", "INFO")
-                    data["file_states"] = {}
-                    
-                if "next_download_index" not in data:
-                    data["next_download_index"] = 0
-                    
-                return data
-        except json.JSONDecodeError:
-            log_message(f"⚠️ Corrupted state file: {file_path}", "WARNING")
-    return default_value
-
-def save_json_state(file_path: str, data: Dict[str, Any]):
-    """Save state to JSON file"""
-    with open(file_path, "w") as f:
-        json.dump(data, f, indent=2)
-
-def download_hf_state(repo_id: str, filename: str) -> Dict[str, Any]:
-    """Downloads the state file from Hugging Face or returns a default state."""
-    local_path = os.path.join(LOCAL_STATE_FOLDER, filename)
-    default_state = {"next_download_index": 0, "file_states": {}}
-    
-    try:
-        files = hf_api.list_repo_files(repo_id=repo_id, repo_type="dataset")
-        if filename not in files:
-            log_message(f"ℹ️ State file {filename} not found in {repo_id}. Starting from default state.", "INFO")
-            return default_state
-
-        from huggingface_hub import hf_hub_download
-        hf_hub_download(
-            repo_id=repo_id,
-            filename=filename,
-            repo_type="dataset",
-            local_dir=LOCAL_STATE_FOLDER,
-            local_dir_use_symlinks=False
-        )
-        
-        log_message(f"✅ Successfully downloaded state file from {repo_id}.", "INFO")
-        return load_json_state(local_path, default_state)
-        
-    except Exception as e:
-        log_message(f"⚠️ Failed to download state file from Hugging Face: {str(e)}. Starting from default state.", "WARNING")
-        return default_state
-
-def upload_hf_state(repo_id: str, filename: str, state: Dict[str, Any]) -> bool:
-    """Uploads the state file to Hugging Face."""
-    local_path = os.path.join(LOCAL_STATE_FOLDER, filename)
-    
-    try:
-        save_json_state(local_path, state)
-        
-        hf_api.upload_file(
-            path_or_fileobj=local_path,
-            path_in_repo=filename,
-            repo_id=repo_id,
-            repo_type="dataset",
-            commit_message=f"Update audio processing state: next_index={state['next_download_index']}"
-        )
-        log_message(f"✅ Successfully uploaded updated state file to {repo_id}", "INFO")
-        return True
-    except Exception as e:
-        log_message(f"❌ Failed to upload state file to Hugging Face: {str(e)}", "ERROR")
-        return False
-
-def lock_file_for_processing(wav_filename: str, state: Dict[str, Any]) -> bool:
-    """Marks a file as 'processing' in the state file and uploads the lock."""
-    log_message(f"🔒 Attempting to lock file: {wav_filename} (Marking as 'processing')", "INFO")
-    
-    state["file_states"][wav_filename] = "processing"
-    
-    if upload_hf_state(TARGET_REPO_ID, HF_STATE_FILE, state):
-        log_message(f"✅ Successfully locked file: {wav_filename}", "INFO")
-        return True
-    else:
-        log_message(f"❌ Failed to upload lock for file: {wav_filename}. Aborting processing.", "ERROR")
-        if wav_filename in state["file_states"]:
-            del state["file_states"][wav_filename]
-        return False
-
-def unlock_file_as_processed(wav_filename: str, state: Dict[str, Any], next_index: int) -> bool:
-    """Marks a file as 'processed', updates the index, and uploads the state."""
-    log_message(f"🔓 Attempting to unlock file: {wav_filename} (Marking as 'processed')", "INFO")
-    
-    state["file_states"][wav_filename] = "processed"
-    state["next_download_index"] = next_index
-    
-    if upload_hf_state(TARGET_REPO_ID, HF_STATE_FILE, state):
-        log_message(f"✅ Successfully unlocked and marked as processed: {wav_filename}", "INFO")
-        return True
-    else:
-        log_message(f"❌ Failed to upload final state for file: {wav_filename}.", "ERROR")
-        return False
-
-def download_with_retry(url: str, dest_path: str, max_retries: int = 3) -> bool:
-    """Download file with retry logic and disk space checking"""
-    if not check_disk_space():
-        cleanup_temp_files()
-        if not check_disk_space():
-            log_message("❌ Insufficient disk space even after cleanup", "ERROR")
-            return False
-    
-    try:
-        os.makedirs(os.path.dirname(dest_path), exist_ok=True)
-    except Exception as e:
-        log_message(f"❌ Failed to create directory for download path {os.path.dirname(dest_path)}: {str(e)}", "ERROR")
-        return False
-        
-    headers = {"Authorization": f"Bearer {HF_TOKEN}"}
-    for attempt in range(max_retries):
-        try:
-            with requests.get(url, headers=headers, stream=True) as r:
-                r.raise_for_status()
-                
-                with open(dest_path, "wb") as f:
-                    for chunk in r.iter_content(chunk_size=8192):
-                        if chunk:
-                            f.write(chunk)
-                
-                log_message(f"✅ Download successful: {dest_path}", "INFO")
-                return True
-            
-        except requests.exceptions.RequestException as e:
-            log_message(f"❌ Download attempt {attempt + 1} failed for {url}: {str(e)}", "WARNING")
-            time.sleep(PROCESSING_DELAY)
-        except Exception as e:
-            log_message(f"❌ An unexpected error occurred during download: {str(e)}", "ERROR")
-            return False
-            
-    log_message(f"❌ Failed to download {url} after {max_retries} attempts.", "ERROR")
-    return False
-
-def fetch_reference_files(repo_id: str) -> Dict[str, str]:
-    """Fetch all files from Fred808/BG3 repo to match with audio filenames."""
-    log_message(f"📋 Fetching file list from {repo_id}...", "INFO")
-    
-    try:
-        files_list = hf_api.list_repo_files(repo_id=repo_id, repo_type="dataset")
-        
-        # Include all file types (zip, rar, wav, mp3, etc.)
-        all_files = [f for f in files_list]
-        
-        # Create a mapping of base filename (without extension) to full path
-        filename_map = {}
-        for file_path in all_files:
-            base_name = os.path.splitext(os.path.basename(file_path))[0]
-            filename_map[base_name] = file_path
-        
-        log_message(f"✅ Found {len(filename_map)} files in reference repo", "INFO")
-        return filename_map
-        
-    except Exception as e:
-        log_message(f"❌ Failed to fetch reference files: {str(e)}", "ERROR")
-        return {}
-
-def find_matching_filename(transcribed_filename: str, reference_map: Dict[str, str]) -> Optional[str]:
-    """Find matching filename in reference map from Fred808/BG3."""
-    base_name = os.path.splitext(transcribed_filename)[0]
-    
-    # Exact match first
-    if base_name in reference_map:
-        full_path = reference_map[base_name]
-        print(f"\n✅ MATCH FOUND:")
-        print(f"   Audio: {transcribed_filename}")
-        print(f"   File:  {full_path}")
-        log_message(f"✅ Found exact match: {transcribed_filename} -> {full_path}", "INFO")
-        return full_path
-    
-    # Partial/fuzzy match (check if reference contains transcribed as substring)
-    matches = []
-    for ref_base, ref_full_path in reference_map.items():
-        if base_name.lower() in ref_base.lower() or ref_base.lower() in base_name.lower():
-            matches.append((ref_base, ref_full_path))
-    
-    # Return first partial match if found
-    if matches:
-        ref_base, ref_full_path = matches[0]
-        print(f"\n✅ PARTIAL MATCH FOUND:")
-        print(f"   Audio: {transcribed_filename}")
-        print(f"   File:  {ref_full_path}")
-        log_message(f"✅ Found partial match: {transcribed_filename} -> {ref_full_path}", "INFO")
-        return ref_full_path
-    
-    print(f"\n❌ NO MATCH FOUND:")
-    print(f"   Audio: {transcribed_filename}")
-    log_message(f"⚠️ No matching filename found for: {transcribed_filename}", "WARNING")
-    return None
-
-def transcribe_audio(wav_path: str) -> Optional[Dict[str, Any]]:
-    """Transcribe audio file using Whisper from Transformers."""
-    log_message(f"🎤 Transcribing audio file: {wav_path}", "INFO")
-    
-    try:
-        from transformers import pipeline
-        import librosa
-        
-        # Load audio with librosa
-        log_message(f"Loading audio file: {wav_path}", "INFO")
-        audio, sr = librosa.load(wav_path, sr=16000)
-        
-        # Initialize Whisper pipeline
-        log_message(f"Loading Whisper {WHISPER_MODEL} model from Transformers...", "INFO")
-        pipe = pipeline(
-            "automatic-speech-recognition",
-            model=f"openai/whisper-{WHISPER_MODEL}",
-            device=0 if __import__('torch').cuda.is_available() else -1  # GPU if available, else CPU
-        )
-        
-        # Transcribe
-        log_message("Transcribing audio...", "INFO")
-        result = pipe(audio)
-        
-        # Format result to match openai-whisper format
-        formatted_result = {
-            "text": result["text"],
-            "segments": [{"text": result["text"]}]
-        }
-        
-        log_message(f"✅ Successfully transcribed: {wav_path}", "INFO")
-        return formatted_result
-        
-    except ImportError as e:
-        missing_lib = str(e)
-        log_message(f"❌ Missing library. Install with: pip install transformers librosa torch torchaudio", "ERROR")
-        log_message(f"   Error: {missing_lib}", "ERROR")
-        return None
-    except Exception as e:
-        log_message(f"❌ Failed to transcribe {wav_path}: {str(e)}", "ERROR")
-        return None
-
-def process_audio_file(wav_path: str, reference_map: Dict[str, str], matched_filename: str) -> bool:
-    """
-    Main processing logic for a single audio file:
-    1. Transcribe using Whisper
-    2. Save transcription as JSON
-    3. Upload to HF dataset
-    4. Clean up local files
-    """
-    wav_filename = os.path.basename(wav_path)
-    
-    # 1. Transcribe audio
-    transcription = transcribe_audio(wav_path)
-    if transcription is None:
-        log_failed_file(wav_filename, "Transcription failed")
-        return False
-    
-    # 2. Save transcription as JSON
-    json_filename = os.path.splitext(matched_filename)[0] + "_transcription.json"
-    json_output_path = os.path.join(TRANSCRIPTIONS_FOLDER, json_filename)
-    
-    try:
-        os.makedirs(os.path.dirname(json_output_path), exist_ok=True)
-        
-        with open(json_output_path, "w", encoding="utf-8") as f:
-            json.dump(transcription, f, indent=2, ensure_ascii=False)
-        
-        log_message(f"✅ Saved transcription: {json_output_path}", "INFO")
-        
-    except Exception as e:
-        log_message(f"❌ Failed to save transcription JSON: {str(e)}", "ERROR")
-        log_failed_file(wav_filename, f"Failed to save JSON: {str(e)}")
-        return False
-    
-    # 3. Upload to HF dataset
-    try:
-        path_in_repo = f"transcriptions/{json_filename}"
-        commit_message = f"Add transcription for: {matched_filename}"
-        
-        hf_api.upload_file(
-            path_or_fileobj=json_output_path,
-            path_in_repo=path_in_repo,
-            repo_id=TARGET_REPO_ID,
-            repo_type="dataset",
-            commit_message=commit_message
-        )
-        log_message(f"✅ Successfully uploaded transcription: {json_filename}", "INFO")
-        processing_status["transcribed_files"] += 1
-        
-    except Exception as e:
-        log_message(f"❌ Failed to upload transcription to HF: {str(e)}", "ERROR")
-        log_failed_file(wav_filename, f"Failed to upload: {str(e)}")
-        return False
-    
-    # 4. Clean up local files
-    try:
-        os.remove(json_output_path)
-        log_message(f"🗑️ Cleaned up local transcription file: {json_output_path}", "INFO")
-    except:
-        pass
-    
-    return True
-
-def get_next_file_to_process(repo_id: str, state: Dict[str, Any]) -> Optional[Dict[str, Any]]:
-    """
-    Finds the next audio file to process from the source repo in reverse order (oldest to newest).
-    Returns: { 'filename': str, 'url': str, 'index': int } or None
-    """
-    log_message(f"🔍 Searching for next audio file to process in {repo_id}", "INFO")
-    
-    try:
-        files_list = hf_api.list_repo_files(repo_id=repo_id, repo_type="dataset")
-        
-        # Filter for audio files and sort in reverse order (descending)
-        audio_files = sorted([f for f in files_list if f.endswith(('.wav', '.mp3'))], reverse=True)
-        
-        if not audio_files:
-            log_message("ℹ️ No audio files found in the source repository.", "INFO")
-            return None
-            
-        processing_status["total_files"] = len(audio_files)
-        
-        start_index = state.get("next_download_index", 0)
-        
-        for index in range(start_index, len(audio_files)):
-            filename = audio_files[index]
-            file_state = state["file_states"].get(filename)
-            
-            if file_state is None or file_state == "failed":
-                url = hf_hub_url(repo_id=repo_id, filename=filename, repo_type="dataset", subfolder=None)
-                
-                log_message(f"✅ Found next audio file: {filename} at index {index}", "INFO")
-                return {
-                    'filename': filename,
-                    'url': url,
-                    'index': index
-                }
-                
-            elif file_state == "processing":
-                log_message(f"⚠️ File {filename} is currently marked as 'processing'. Skipping for now.", "WARNING")
-                
-            elif file_state == "processed":
-                log_message(f"ℹ️ File {filename} already processed. Skipping.", "INFO")
-                
-        log_message("ℹ️ All files up to the current index have been processed or skipped.", "INFO")
-        
-        if start_index >= len(audio_files):
-            log_message("ℹ️ Reached end of file list. Resetting index to 0 for next loop.", "INFO")
-            state["next_download_index"] = 0
-            upload_hf_state(TARGET_REPO_ID, HF_STATE_FILE, state)
-            
-        return None
-        
-    except Exception as e:
-        log_message(f"❌ Failed to list files from Hugging Face: {str(e)}", "ERROR")
-        return None
-
-def main_processing_loop():
-    """The main loop that orchestrates the download, transcription, and upload cycle."""
-    
-    if processing_status["is_running"]:
-        log_message("⚠️ Processing loop is already running.", "WARNING")
-        return
-        
-    processing_status["is_running"] = True
-    
-    try:
-        log_message("🚀 Starting audio transcription processing loop...", "INFO")
-        
-        # Fetch reference files from BG_Vid repo once at the start
-        reference_map = fetch_reference_files(REFERENCE_REPO_ID)
-        
-        if not reference_map:
-            log_message("❌ No reference files found. Cannot proceed.", "ERROR")
-            return
-        
-        while processing_status["is_running"]:
-            
-            current_state = download_hf_state(TARGET_REPO_ID, HF_STATE_FILE)
-            next_file_info = get_next_file_to_process(SOURCE_REPO_ID, current_state)
-            
-            if next_file_info is None:
-                log_message("💤 No new audio files to process. Sleeping for a while...", "INFO")
-                time.sleep(PROCESSING_DELAY * 5)
-                continue
-                
-            target_file = next_file_info['filename']
-            audio_url = next_file_info['url']
-            target_index = next_file_info['index']
-            
-            processing_status["current_file"] = target_file
-            success = False
-            matched_filename = None
-            
-            try:
-                if not lock_file_for_processing(target_file, current_state):
-                    log_message(f"❌ Failed to lock file {target_file}. Skipping.", "ERROR")
-                    time.sleep(PROCESSING_DELAY)
-                    continue
-                    
-                local_wav_path = os.path.join(DOWNLOAD_FOLDER, os.path.basename(target_file))
-                log_message(f"⬇️ Downloading audio file: {target_file}", "INFO")
-                
-                if download_with_retry(audio_url, local_wav_path):
-                    
-                    # Extract base filename for matching
-                    base_filename = os.path.basename(target_file)
-                    matched_filename = find_matching_filename(base_filename, reference_map)
-                    
-                    if matched_filename:
-                        if process_audio_file(local_wav_path, reference_map, matched_filename):
-                            success = True
-                            log_message(f"✅ Finished processing: {target_file}", "INFO")
-                        else:
-                            log_message(f"❌ Processing failed for: {target_file}", "ERROR")
-                    else:
-                        log_message(f"❌ No matching filename found for: {base_filename}", "ERROR")
-                        log_failed_file(target_file, "No matching reference filename")
-                else:
-                    log_message(f"❌ Download failed for: {target_file}", "ERROR")
-                    
-            except Exception as e:
-                log_message(f"🔥 An unhandled error occurred while processing {target_file}: {str(e)}", "ERROR")
-                log_failed_file(target_file, str(e))
-                
-            finally:
-                next_index_to_save = target_index + 1
-                current_state = download_hf_state(TARGET_REPO_ID, HF_STATE_FILE)
-                
-                if success:
-                    unlock_file_as_processed(target_file, current_state, next_index_to_save)
-                    processing_status["processed_files"] += 1
-                else:
-                    log_message(f"⚠️ Processing failed for {target_file}. Marking as 'failed' and advancing index.", "WARNING")
-                    current_state["file_states"][target_file] = "failed"
-                    current_state["next_download_index"] = next_index_to_save
-                    upload_hf_state(TARGET_REPO_ID, HF_STATE_FILE, current_state)
-                    processing_status["failed_files"] += 1
-                    
-                if os.path.exists(local_wav_path):
-                    os.remove(local_wav_path)
-                    log_message(f"🗑️ Cleaned up local file: {local_wav_path}", "INFO")
-
-            time.sleep(PROCESSING_DELAY)
-
-        log_message("🎉 Processing complete!", "INFO")
-        log_message(f"📊 Final stats: {processing_status['transcribed_files']} audio files transcribed, {processing_status['processed_files']} files processed", "INFO")
-        
-    except KeyboardInterrupt:
-        log_message("⏹️ Processing interrupted by user", "WARNING")
-    except Exception as e:
-        log_message(f"❌ Fatal error: {str(e)}", "ERROR")
-    finally:
-        processing_status["is_running"] = False
-        cleanup_temp_files()
-
-if __name__ == "__main__":
-    main_processing_loop()
-
-# ===== FASTAPI ENDPOINTS =====
-
-@app.get("/")
-async def root():
-    """Root endpoint with service info"""
-    return {
-        "service": "Audio Transcriber",
-        "status": "running",
-        "version": "1.0.0",
-        "endpoints": {
-            "status": "/status",
-            "start": "/start",
-            "stop": "/stop",
-            "process": "/process/{filename}",
-            "logs": "/logs"
-        }
-    }
-
-@app.get("/status")
-async def get_status():
-    """Get current processing status"""
-    return {
-        "is_running": processing_status["is_running"],
-        "current_file": processing_status["current_file"],
-        "total_files": processing_status["total_files"],
-        "processed_files": processing_status["processed_files"],
-        "transcribed_files": processing_status["transcribed_files"],
-        "failed_files": processing_status["failed_files"],
-        "last_update": processing_status["last_update"],
-        "recent_logs": processing_status["logs"][-10:]
-    }
-
-@app.post("/start")
-async def start_processing():
-    """Start the main processing loop"""
-    if processing_status["is_running"]:
-        raise HTTPException(status_code=400, detail="Processing already running")
-    
-    # Start processing in a separate thread
-    thread = threading.Thread(target=main_processing_loop, daemon=True)
-    thread.start()
-    
-    return {
-        "message": "Processing started",
-        "status": "started"
-    }
-
-@app.post("/stop")
-async def stop_processing():
-    """Stop the main processing loop"""
-    if not processing_status["is_running"]:
-        raise HTTPException(status_code=400, detail="Processing not running")
-    
-    processing_status["is_running"] = False
-    
-    return {
-        "message": "Processing stopped",
-        "status": "stopped"
-    }
-
-@app.get("/logs")
-async def get_logs(limit: int = 50):
-    """Get recent logs"""
-    logs = processing_status["logs"][-limit:]
-    return {
-        "total_logs": len(processing_status["logs"]),
-        "recent_logs": logs
-    }
-
-@app.post("/process/{filename}")
-async def process_single_file(filename: str):
-    """Process a single audio file manually"""
-    try:
-        log_message(f"🎯 Manual processing requested for: {filename}", "INFO")
-        
-        # Download and process the file
-        reference_map = fetch_reference_files(REFERENCE_REPO_ID)
-        if not reference_map:
-            raise HTTPException(status_code=500, detail="Could not fetch reference files")
-        
-        # Get file URL
-        audio_url = hf_hub_url(repo_id=SOURCE_REPO_ID, filename=filename, repo_type="dataset", subfolder=None)
-        local_wav_path = os.path.join(DOWNLOAD_FOLDER, os.path.basename(filename))
-        
-        # Download
-        if not download_with_retry(audio_url, local_wav_path):
-            raise HTTPException(status_code=500, detail="Failed to download file")
-        
-        # Find match
-        base_filename = os.path.basename(filename)
-        matched_filename = find_matching_filename(base_filename, reference_map)
-        
-        if not matched_filename:
-            os.remove(local_wav_path)
-            raise HTTPException(status_code=404, detail="No matching filename found")
-        
-        # Process
-        if process_audio_file(local_wav_path, reference_map, matched_filename):
-            processing_status["transcribed_files"] += 1
-            
-            if os.path.exists(local_wav_path):
-                os.remove(local_wav_path)
-            
-            return {
-                "status": "success",
-                "file": filename,
-                "matched": matched_filename,
-                "message": "Audio transcribed and uploaded successfully"
-            }
-        else:
-            if os.path.exists(local_wav_path):
-                os.remove(local_wav_path)
-            raise HTTPException(status_code=500, detail="Processing failed")
-            
-    except Exception as e:
-        log_message(f"❌ Manual processing error: {str(e)}", "ERROR")
-        raise HTTPException(status_code=500, detail=str(e))
-
-@app.on_event("startup")
-async def startup_event():
-    """Auto-start processing when server starts"""
-    log_message("🚀 Server startup: Checking dependencies...", "INFO")
-    
-    try:
-        import transformers
-        log_message("✅ Transformers found", "INFO")
-    except ImportError:
-        log_message("⚠️ WARNING: Transformers not installed!", "WARNING")
-        log_message("   Install with: pip install transformers librosa torch torchaudio", "WARNING")
-    
-    log_message("🚀 Server startup: Auto-starting processing loop", "INFO")
-    
-    # Start processing in a separate thread
-    thread = threading.Thread(target=main_processing_loop, daemon=True)
-    thread.start()
-
-def run_api(host: str = "0.0.0.0", port: int = 8000):
-    """Run the FastAPI server"""
-    log_message(f"🚀 Starting FastAPI server on {host}:{port}", "INFO")
-    uvicorn.run(app, host=host, port=port)
-
-if __name__ == "__main__":
-    # Run API server (processing will auto-start via startup event)
-    run_api()
+import os
+import shutil
+import zipfile
+import asyncio
+from contextlib import asynccontextmanager
+from typing import List
+
+from fastapi import FastAPI, UploadFile, File, HTTPException
+from fastapi.responses import FileResponse
+from huggingface_hub import HfApi
+
+# --- Configuration ---
+UPLOAD_DIR = "uploaded_files"
+HF_DATASET_REPO = "samfred2/A_Text"
+HF_TOKEN = os.getenv("HF_TOKEN")
+
+# --- Utility Functions ---
+
+def get_uploaded_files() -> List[str]:
+    """Returns a list of all files in the upload directory."""
+    if not os.path.exists(UPLOAD_DIR):
+        return []
+    return [os.path.join(UPLOAD_DIR, f) for f in os.listdir(UPLOAD_DIR) if os.path.isfile(os.path.join(UPLOAD_DIR, f))]
+
+def zip_uploaded_files(zip_filename: str = "uploaded_files.zip") -> str:
+    """Zips all files in the upload directory into a single zip file."""
+    if not os.path.exists(UPLOAD_DIR) or not os.listdir(UPLOAD_DIR):
+        print("No files to zip.")
+        return None
+
+    zip_path = os.path.join(os.getcwd(), zip_filename)
+    with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
+        for root, _, files in os.walk(UPLOAD_DIR):
+            for file in files:
+                file_path = os.path.join(root, file)
+                # Add file to zip, preserving directory structure relative to UPLOAD_DIR
+                zipf.write(file_path, os.path.relpath(file_path, UPLOAD_DIR))
+    
+    print(f"Successfully created zip file at: {zip_path}")
+    return zip_path
+
+def upload_to_huggingface(zip_path: str):
+    """Uploads the zip file to the specified Hugging Face dataset."""
+    if not HF_TOKEN:
+        print("HF_TOKEN not found in environment variables. Skipping upload.")
+        return
+
+    if not zip_path or not os.path.exists(zip_path):
+        print("Zip file not found. Skipping upload.")
+        return
+
+    try:
+        api = HfApi()
+        
+        # Upload the zip file to the root of the dataset repository
+        api.upload_file(
+            path_or_fileobj=zip_path,
+            path_in_repo=os.path.basename(zip_path),
+            repo_id=HF_DATASET_REPO,
+            repo_type="dataset",
+            token=HF_TOKEN
+        )
+        print(f"Successfully uploaded {os.path.basename(zip_path)} to {HF_DATASET_REPO}")
+    except Exception as e:
+        print(f"Hugging Face upload failed: {e}")
+
+def cleanup_upload_dir():
+    """Removes the upload directory and its contents."""
+    if os.path.exists(UPLOAD_DIR):
+        shutil.rmtree(UPLOAD_DIR)
+        print(f"Cleaned up {UPLOAD_DIR} directory.")
+
+# --- Application Lifespan ---
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup: Ensure upload directory exists
+    os.makedirs(UPLOAD_DIR, exist_ok=True)
+    print(f"Application starting. Upload directory: {UPLOAD_DIR}")
+    yield
+    # Shutdown: Zip and upload files
+    print("Application shutting down. Initiating final upload...")
+    zip_path = zip_uploaded_files()
+    if zip_path:
+        upload_to_huggingface(zip_path)
+        # Clean up the created zip file after upload
+        os.remove(zip_path)
+    cleanup_upload_dir()
+    print("Shutdown complete.")
+
+# --- FastAPI App Initialization ---
+
+app = FastAPI(
+    title="File Uploader and Downloader Service",
+    description="A simple service for file management and Hugging Face dataset synchronization.",
+    version="1.0.0",
+    lifespan=lifespan
+)
+
+# --- Endpoints ---
+
+@app.post("/upload/")
+async def upload_file(file: UploadFile = File(...)):
+    """Upload a file to the server."""
+    try:
+        file_path = os.path.join(UPLOAD_DIR, file.filename)
+        # Check if file already exists to prevent overwriting without warning
+        if os.path.exists(file_path):
+            raise HTTPException(status_code=409, detail=f"File '{file.filename}' already exists.")
+            
+        # Write the file content to disk
+        with open(file_path, "wb") as buffer:
+            shutil.copyfileobj(file.file, buffer)
+        
+        return {"filename": file.filename, "message": "File successfully uploaded"}
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"An error occurred during upload: {e}")
+
+@app.get("/download/{filename}")
+async def download_file(filename: str):
+    """Download a file from the server."""
+    file_path = os.path.join(UPLOAD_DIR, filename)
+    
+    if not os.path.exists(file_path):
+        raise HTTPException(status_code=404, detail="File not found")
+    
+    return FileResponse(
+        path=file_path, 
+        filename=filename, 
+        media_type='application/octet-stream'
+    )
+
+@app.post("/sync_dataset/")
+async def sync_dataset():
+    """Manually trigger zipping of all uploaded files and uploading to the Hugging Face dataset."""
+    print("Manual dataset sync triggered.")
+    zip_path = zip_uploaded_files()
+    if not zip_path:
+        return {"message": "No files to sync. Upload directory is empty."}
+    
+    upload_to_huggingface(zip_path)
+    
+    # Clean up the created zip file after upload
+    os.remove(zip_path)
+    
+    return {"message": "Files zipped and upload to Hugging Face dataset initiated."}
+
+@app.get("/files/")
+async def list_files():
+    """List all files currently available for download."""
+    if not os.path.exists(UPLOAD_DIR):
+        return {"files": []}
+    return {"files": os.listdir(UPLOAD_DIR)}
+
+# --- Main execution block for testing/running ---
+if __name__ == "__main__":
+    import uvicorn
+    # Set the token for local testing
+    os.environ["HF_TOKEN"] = HF_TOKEN or "dummy_token_for_local_test"
+    
+    # Ensure UPLOAD_DIR exists before starting
+    os.makedirs(UPLOAD_DIR, exist_ok=True)
+    
+    # Use a short timeout for local testing to simulate a quick run
+    config = uvicorn.Config(app, host="0.0.0.0", port=8000, log_level="info")
+    server = uvicorn.Server(config)
+    
+    # This block is for local testing and won't be used in the final sandbox execution
+    # but is good practice for a runnable script.
+    try:
+        print("Starting server for local test...")
+        # server.run() # Normally we would run this, but in the sandbox we use exec
+        pass
+    except KeyboardInterrupt:
+        print("Server stopped by user.")
+    finally:
+        # Simulate cleanup that happens in the lifespan context manager
+        # when running with uvicorn in a real environment.
+        pass