Update app.py

app.py

@@ -7,12 +7,10 @@ import time
 import sys
 import threading
 from typing import Dict, List, Optional, Any
+from huggingface_hub import HfApi, hf_hub_url
 from fastapi import FastAPI, HTTPException
 from fastapi.responses import JSONResponse
 import uvicorn
-import torch
-import librosa
-from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
 
 # Fix Unicode encoding for Windows
 if sys.platform == 'win32':
@@ -23,13 +21,9 @@ if sys.platform == 'win32':
 app = FastAPI(title="Audio Transcriber", description="Audio transcription and upload service")
 
 # ==== CONFIGURATION ====
-# The new backend URL for state management and transcription upload
-# It is now read from an environment variable, falling back to the default if not set.
-BACKEND_URL = os.environ.get("BACKEND_URL", "https://samfredoly-acp.hf.space")
-# The original Hugging Face repo IDs are still needed for fetching the audio files
-# and the reference file list, as the backend only handles transcription storage.
+HF_TOKEN = os.environ.get("HF_TOKEN", "")
 SOURCE_REPO_ID = "Samfredoly/BG_Vid"  # Fetch audio files from here
-TARGET_REPO_ID = "samfred2/A_Text"  # Target repo ID is now a constant for the backend
+TARGET_REPO_ID = "samfred2/A_Text"  # Upload transcriptions here
 REFERENCE_REPO_ID = "Fred808/BG3"  # Reference repo to match audio filenames
 
 # Path Configuration
@@ -41,55 +35,9 @@ os.makedirs(DOWNLOAD_FOLDER, exist_ok=True)
 os.makedirs(TRANSCRIPTIONS_FOLDER, exist_ok=True)
 os.makedirs(LOCAL_STATE_FOLDER, exist_ok=True)
 
-# Whisper Model Setup (using transformers)
-DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
-TORCH_DTYPE = torch.float16 if torch.cuda.is_available() else torch.float32
-WHISPER_MODEL_ID = f"openai/whisper-small"
-
-# Global model cache
-_whisper_model = None
-_whisper_processor = None
-_whisper_pipeline = None
-
-def get_whisper_pipeline():
-    """Get or initialize the Whisper pipeline."""
-    global _whisper_model, _whisper_processor, _whisper_pipeline
-    
-    if _whisper_pipeline is not None:
-        return _whisper_pipeline
-    
-    try:
-        log_message(f"Loading Whisper model {WHISPER_MODEL_ID}...", "INFO")
-        
-        model = AutoModelForSpeechSeq2Seq.from_pretrained(
-            WHISPER_MODEL_ID,
-            torch_dtype=TORCH_DTYPE,
-            low_cpu_mem_usage=True,
-            use_safetensors=True
-        )
-        model = model.to(DEVICE)
-        
-        processor = AutoProcessor.from_pretrained(WHISPER_MODEL_ID)
-        
-        _whisper_pipeline = pipeline(
-            "automatic-speech-recognition",
-            model=model,
-            tokenizer=processor.tokenizer,
-            feature_extractor=processor.feature_extractor,
-            torch_dtype=TORCH_DTYPE,
-            device=DEVICE
-        )
-        
-        log_message(f"✅ Whisper model loaded successfully on {DEVICE.upper()}", "INFO")
-        return _whisper_pipeline
-        
-    except Exception as e:
-        log_message(f"❌ Failed to load Whisper model: {str(e)}", "ERROR")
-        raise
-
 # State Files
 FAILED_FILES_LOG = "failed_audio_files.log"
-HF_STATE_FILE = "processing_audio_state.json" # This is the filename the backend uses
+HF_STATE_FILE = "processing_audio_state.json"
 
 # Processing Parameters
 PROCESSING_DELAY = 2
@@ -97,9 +45,7 @@ MAX_RETRIES = 3
 MIN_FREE_SPACE_GB = 1
 WHISPER_MODEL = "small"  # Whisper model size
 
-# NOTE: The Hugging Face API is still required for listing files in SOURCE_REPO_ID and REFERENCE_REPO_ID
-from huggingface_hub import HfApi, hf_hub_url
-HF_TOKEN = os.environ.get("HF_TOKEN", "")
+# Initialize HF API
 hf_api = HfApi(token=HF_TOKEN)
 
 # Global State
@@ -158,133 +104,106 @@ def cleanup_temp_files():
             except:
                 pass
 
-# Helper function to save state locally
+def load_json_state(file_path: str, default_value: Dict[str, Any]) -> Dict[str, Any]:
+    """Load state from JSON file with migration logic for new structure."""
+    if os.path.exists(file_path):
+        try:
+            with open(file_path, "r") as f:
+                data = json.load(f)
+                
+                if "file_states" not in data or not isinstance(data["file_states"], dict):
+                    log_message("ℹ️ Initializing 'file_states' dictionary.", "INFO")
+                    data["file_states"] = {}
+                    
+                if "next_download_index" not in data:
+                    data["next_download_index"] = 0
+                    
+                return data
+        except json.JSONDecodeError:
+            log_message(f"⚠️ Corrupted state file: {file_path}", "WARNING")
+    return default_value
+
 def save_json_state(file_path: str, data: Dict[str, Any]):
     """Save state to JSON file"""
     with open(file_path, "w") as f:
         json.dump(data, f, indent=2)
 
-# --- NEW API FUNCTIONS FOR STATE MANAGEMENT AND UPLOAD ---
-
-def download_state_from_api() -> Dict[str, Any]:
-    """Downloads the state file from the backend API."""
-    url = f"{BACKEND_URL}/state/"
+def download_hf_state(repo_id: str, filename: str) -> Dict[str, Any]:
+    """Downloads the state file from Hugging Face or returns a default state."""
+    local_path = os.path.join(LOCAL_STATE_FOLDER, filename)
     default_state = {"next_download_index": 0, "file_states": {}}
     
     try:
-        response = requests.get(url, timeout=10)
-        response.raise_for_status()
-        
-        # The API returns {"state": {...}}
-        state_data = response.json().get("state", default_state)
+        files = hf_api.list_repo_files(repo_id=repo_id, repo_type="dataset")
+        if filename not in files:
+            log_message(f"ℹ️ State file {filename} not found in {repo_id}. Starting from default state.", "INFO")
+            return default_state
+
+        from huggingface_hub import hf_hub_download
+        hf_hub_download(
+            repo_id=repo_id,
+            filename=filename,
+            repo_type="dataset",
+            local_dir=LOCAL_STATE_FOLDER,
+            local_dir_use_symlinks=False
+        )
         
-        # Ensure the structure is correct (migration logic from original load_json_state)
-        if "file_states" not in state_data or not isinstance(state_data["file_states"], dict):
-            state_data["file_states"] = {}
-        if "next_download_index" not in state_data:
-            state_data["next_download_index"] = 0
-            
-        log_message(f"✅ Successfully downloaded state from API.", "INFO")
-        return state_data
+        log_message(f"✅ Successfully downloaded state file from {repo_id}.", "INFO")
+        return load_json_state(local_path, default_state)
         
-    except requests.exceptions.RequestException as e:
-        log_message(f"⚠️ Failed to download state from API ({url}): {str(e)}. Starting from default state.", "WARNING")
+    except Exception as e:
+        log_message(f"⚠️ Failed to download state file from Hugging Face: {str(e)}. Starting from default state.", "WARNING")
         return default_state
 
-def upload_state_to_api(state: Dict[str, Any]) -> bool:
-    """
-    Saves the state locally and uploads it to the backend API's /upload/ endpoint.
-    This simulates the original HF state upload for locking/unlocking.
-    """
-    local_path = os.path.join(LOCAL_STATE_FOLDER, HF_STATE_FILE)
-    url = f"{BACKEND_URL}/upload/"
+def upload_hf_state(repo_id: str, filename: str, state: Dict[str, Any]) -> bool:
+    """Uploads the state file to Hugging Face."""
+    local_path = os.path.join(LOCAL_STATE_FOLDER, filename)
     
     try:
-        # 1. Save the current state locally
         save_json_state(local_path, state)
         
-        # 2. Upload the state file to the backend
-        with open(local_path, "rb") as f:
-            files = {'file': (HF_STATE_FILE, f, 'application/json')}
-            
-            response = requests.post(url, files=files, timeout=30)
-            response.raise_for_status()
-            
-            log_message(f"✅ Successfully uploaded state file to API: {HF_STATE_FILE}", "INFO")
-            return True
-            
-    except requests.exceptions.HTTPError as e:
-        if hasattr(e, 'response') and e.response.status_code == 409:
-            log_message(f"⚠️ State file already exists on server (409 Conflict) - Treating as success.", "INFO")
-            return True
-        log_message(f"❌ Failed to upload state file to API ({url}): {str(e)}", "ERROR")
-        return False
-    except requests.exceptions.RequestException as e:
-        log_message(f"❌ Failed to upload state file to API ({url}): {str(e)}", "ERROR")
-        return False
-    except Exception as e:
-        log_message(f"❌ An unexpected error occurred during API state upload: {str(e)}", "ERROR")
-        return False
-
-def upload_transcription_to_api(json_output_path: str, matched_filename: str) -> bool:
-    """Uploads the transcription JSON file to the backend API's /upload/ endpoint."""
-    url = f"{BACKEND_URL}/upload/"
-    
-    try:
-        with open(json_output_path, "rb") as f:
-            files = {'file': (os.path.basename(json_output_path), f, 'application/json')}
-            
-            response = requests.post(url, files=files, timeout=30)
-            response.raise_for_status()
-            
-            log_message(f"✅ Successfully uploaded transcription to API: {os.path.basename(json_output_path)}", "INFO")
-            return True
-            
-    except requests.exceptions.HTTPError as e:
-        if hasattr(e, 'response') and e.response.status_code == 409:
-            log_message(f"⚠️ File already exists on server (409 Conflict) - Treating as success.", "INFO")
-            return True
-        log_message(f"❌ Failed to upload transcription to API ({url}): {str(e)}", "ERROR")
-        return False
-    except requests.exceptions.RequestException as e:
-        log_message(f"❌ Failed to upload transcription to API ({url}): {str(e)}", "ERROR")
-        return False
+        hf_api.upload_file(
+            path_or_fileobj=local_path,
+            path_in_repo=filename,
+            repo_id=repo_id,
+            repo_type="dataset",
+            commit_message=f"Update audio processing state: next_index={state['next_download_index']}"
+        )
+        log_message(f"✅ Successfully uploaded updated state file to {repo_id}", "INFO")
+        return True
     except Exception as e:
-        log_message(f"❌ An unexpected error occurred during API upload: {str(e)}", "ERROR")
+        log_message(f"❌ Failed to upload state file to Hugging Face: {str(e)}", "ERROR")
         return False
 
 def lock_file_for_processing(wav_filename: str, state: Dict[str, Any]) -> bool:
-    """Marks a file as 'processing' in the state file and uploads the lock via API."""
+    """Marks a file as 'processing' in the state file and uploads the lock."""
     log_message(f"🔒 Attempting to lock file: {wav_filename} (Marking as 'processing')", "INFO")
     
     state["file_states"][wav_filename] = "processing"
     
-    if upload_state_to_api(state):
-        log_message(f"✅ Successfully locked file: {wav_filename} via API state upload", "INFO")
+    if upload_hf_state(TARGET_REPO_ID, HF_STATE_FILE, state):
+        log_message(f"✅ Successfully locked file: {wav_filename}", "INFO")
         return True
     else:
         log_message(f"❌ Failed to upload lock for file: {wav_filename}. Aborting processing.", "ERROR")
-        # Revert local state change if upload fails
         if wav_filename in state["file_states"]:
             del state["file_states"][wav_filename]
         return False
 
 def unlock_file_as_processed(wav_filename: str, state: Dict[str, Any], next_index: int) -> bool:
-    """Marks a file as 'processed', updates the index, and uploads the state via API."""
+    """Marks a file as 'processed', updates the index, and uploads the state."""
     log_message(f"🔓 Attempting to unlock file: {wav_filename} (Marking as 'processed')", "INFO")
     
     state["file_states"][wav_filename] = "processed"
     state["next_download_index"] = next_index
     
-    if upload_state_to_api(state):
-        log_message(f"✅ Successfully unlocked and marked as processed: {wav_filename} via API state upload", "INFO")
+    if upload_hf_state(TARGET_REPO_ID, HF_STATE_FILE, state):
+        log_message(f"✅ Successfully unlocked and marked as processed: {wav_filename}", "INFO")
         return True
     else:
         log_message(f"❌ Failed to upload final state for file: {wav_filename}.", "ERROR")
         return False
 
-# --- END NEW API FUNCTIONS ---
-
 def download_with_retry(url: str, dest_path: str, max_retries: int = 3) -> bool:
     """Download file with retry logic and disk space checking"""
     if not check_disk_space():
@@ -299,8 +218,6 @@ def download_with_retry(url: str, dest_path: str, max_retries: int = 3) -> bool:
         log_message(f"❌ Failed to create directory for download path {os.path.dirname(dest_path)}: {str(e)}", "ERROR")
         return False
         
-    # The original code used HF_TOKEN for authorization headers, which is only needed 
-    # if the source repo is private. We keep it for compatibility.
     headers = {"Authorization": f"Bearer {HF_TOKEN}"}
     for attempt in range(max_retries):
         try:
@@ -312,275 +229,255 @@ def download_with_retry(url: str, dest_path: str, max_retries: int = 3) -> bool:
                         if chunk:
                             f.write(chunk)
                 
-                log_message(f"✅ Download successful: {os.path.basename(dest_path)}", "INFO")
+                log_message(f"✅ Download successful: {dest_path}", "INFO")
                 return True
+            
         except requests.exceptions.RequestException as e:
-            log_message(f"⚠️ Download attempt {attempt + 1}/{max_retries} failed for {url}: {str(e)}", "WARNING")
-            if attempt < max_retries - 1:
-                time.sleep(2 ** attempt) # Exponential backoff
-            else:
-                log_message(f"❌ Download failed after {max_retries} attempts for {url}", "ERROR")
-                return False
+            log_message(f"❌ Download attempt {attempt + 1} failed for {url}: {str(e)}", "WARNING")
+            time.sleep(PROCESSING_DELAY)
         except Exception as e:
             log_message(f"❌ An unexpected error occurred during download: {str(e)}", "ERROR")
             return False
+            
+    log_message(f"❌ Failed to download {url} after {max_retries} attempts.", "ERROR")
     return False
 
-def get_reference_map(reference_repo_id: str) -> Dict[str, str]:
-    """
-    Downloads the reference file list from the Hugging Face repo and creates a map
-    from audio filename (without extension) to the reference filename.
-    """
-    log_message(f"Fetching reference file list from {reference_repo_id}...", "INFO")
-    
-    # This is a placeholder for the actual logic to get the file list.
-    # Assuming the reference repo contains a list of files that match the audio files.
-    # In a real scenario, this would involve listing files in the repo.
-    # For now, we'll assume a simple list of files can be retrieved.
+def fetch_reference_files(repo_id: str) -> Dict[str, str]:
+    """Fetch all files from Fred808/BG3 repo to match with audio filenames."""
+    log_message(f"📋 Fetching file list from {repo_id}...", "INFO")
     
     try:
-        # Use HfApi to list files in the reference repo
-        repo_files = hf_api.list_repo_files(repo_id=reference_repo_id, repo_type="dataset")
-        
-        reference_map = {}
-        for file in repo_files:
-            # Assuming the reference files are named like 'audio_file_name.txt'
-            # and we want to map the audio file name (e.g., 'audio_file_name.wav') to it.
-            base_name, ext = os.path.splitext(file)
-            if ext.lower() in ['.txt', '.json']: # Only consider text/json files as reference
-                # The key is the audio file name without extension
-                reference_map[base_name] = file
-                
-        log_message(f"✅ Successfully created reference map with {len(reference_map)} entries.", "INFO")
-        return reference_map
+        files_list = hf_api.list_repo_files(repo_id=repo_id, repo_type="dataset")
+        
+        # Include all file types (zip, rar, wav, mp3, etc.)
+        all_files = [f for f in files_list]
+        
+        # Create a mapping of base filename (without extension) to full path
+        filename_map = {}
+        for file_path in all_files:
+            base_name = os.path.splitext(os.path.basename(file_path))[0]
+            filename_map[base_name] = file_path
+        
+        log_message(f"✅ Found {len(filename_map)} files in reference repo", "INFO")
+        return filename_map
         
     except Exception as e:
-        log_message(f"❌ Failed to fetch reference map from Hugging Face: {str(e)}", "ERROR")
+        log_message(f"❌ Failed to fetch reference files: {str(e)}", "ERROR")
         return {}
 
-def find_matching_filename(audio_filename: str, reference_map: Dict[str, str]) -> Optional[str]:
-    """Finds the matching reference filename for a given audio filename."""
-    base_name, _ = os.path.splitext(audio_filename)
-    return reference_map.get(base_name)
-
-def get_next_file_to_process(source_repo_id: str, state: Dict[str, Any]) -> Optional[Dict[str, Any]]:
-    """
-    Determines the next file to process based on the current state and the file list
-    from the source Hugging Face repository.
-    """
-    log_message(f"Determining next file to process from {source_repo_id}...", "INFO")
+def find_matching_filename(transcribed_filename: str, reference_map: Dict[str, str]) -> Optional[str]:
+    """Find matching filename in reference map from Fred808/BG3."""
+    base_name = os.path.splitext(transcribed_filename)[0]
+    
+    # Exact match first
+    if base_name in reference_map:
+        full_path = reference_map[base_name]
+        print(f"\n✅ EXACT MATCH FOUND:")
+        print(f"   Audio: {transcribed_filename}")
+        print(f"   File:  {full_path}")
+        log_message(f"✅ Found exact match: {transcribed_filename} -> {full_path}", "INFO")
+        return full_path
+    
+    # Partial/fuzzy match (check if reference contains transcribed as substring)
+    matches = []
+    for ref_base, ref_full_path in reference_map.items():
+        if base_name.lower() in ref_base.lower() or ref_base.lower() in base_name.lower():
+            matches.append((ref_base, ref_full_path))
+    
+    # Return first partial match if found
+    if matches:
+        ref_base, ref_full_path = matches[0]
+        print(f"\n✅ PARTIAL MATCH FOUND:")
+        print(f"   Audio: {transcribed_filename}")
+        print(f"   File:  {ref_full_path}")
+        log_message(f"✅ Found partial match: {transcribed_filename} -> {ref_full_path}", "INFO")
+        return ref_full_path
+    
+    print(f"\n⚠️ NO EXACT/PARTIAL MATCH FOUND (will still process):")
+    print(f"   Audio: {transcribed_filename}")
+    log_message(f"⚠️ No matching filename found for: {transcribed_filename}. Will use original filename.", "WARNING")
+    return None
+
+def transcribe_audio(wav_path: str) -> Optional[Dict[str, Any]]:
+    """Transcribe audio file using Whisper from Transformers."""
+    log_message(f"🎤 Transcribing audio file: {wav_path}", "INFO")
     
     try:
-        # 1. Get the list of all files in the source repo
-        repo_files = hf_api.list_repo_files(repo_id=source_repo_id, repo_type="dataset")
+        from transformers import pipeline
+        import librosa
         
-        # Filter for audio files (e.g., .wav, .mp3)
-        audio_files = sorted([f for f in repo_files if f.lower().endswith(('.wav', '.mp3'))])
+        # Load audio with librosa
+        log_message(f"Loading audio file: {wav_path}", "INFO")
+        audio, sr = librosa.load(wav_path, sr=16000)
         
-        processing_status["total_files"] = len(audio_files)
+        # Initialize Whisper pipeline
+        log_message(f"Loading Whisper {WHISPER_MODEL} model from Transformers...", "INFO")
+        pipe = pipeline(
+            "automatic-speech-recognition",
+            model=f"openai/whisper-{WHISPER_MODEL}",
+            device=0 if __import__('torch').cuda.is_available() else -1  # GPU if available, else CPU
+        )
         
-        if not audio_files:
-            log_message("No audio files found in the source repository.", "INFO")
-            return None
-            
-        # 2. Get the next index from the state
-        next_index = state.get("next_download_index", 0)
-        file_states = state.get("file_states", {})
-        
-        # 3. Skip forward past all processed and processing files starting from next_index
-        # This ensures we don't repeatedly find files that have already been handled
-        current_index = next_index
-        while current_index < len(audio_files):
-            filename = audio_files[current_index]
-            status = file_states.get(filename, "unprocessed")
-            
-            # If this file is processed or currently processing, skip it
-            if status in ["processed", "processing"]:
-                current_index += 1
-                continue
-            
-            # If this file failed, we can retry it, so return it
-            if status == "failed":
-                file_url = hf_hub_url(repo_id=source_repo_id, filename=filename, repo_type="dataset")
-                log_message(f"Found failed file for retry at index {current_index}: {filename}", "INFO")
-                return {
-                    "filename": filename,
-                    "url": file_url,
-                    "index": current_index
-                }
-            
-            # If this file is unprocessed, we found our next file
-            file_url = hf_hub_url(repo_id=source_repo_id, filename=filename, repo_type="dataset")
-            log_message(f"Found next file at index {current_index}: {filename}", "INFO")
-            return {
-                "filename": filename,
-                "url": file_url,
-                "index": current_index
-            }
+        # Transcribe
+        log_message("Transcribing audio...", "INFO")
+        result = pipe(audio)
         
-        log_message("All files have been processed or are locked. Checking for any failed files from the start.", "INFO")
+        # Format result to match openai-whisper format
+        formatted_result = {
+            "text": result["text"],
+            "segments": [{"text": result["text"]}]
+        }
         
-        # 4. If we've processed all files from next_index to end, check from beginning for failed files
-        for i in range(0, next_index):
-            filename = audio_files[i]
-            status = file_states.get(filename, "unprocessed")
-            
-            if status == "failed":
-                file_url = hf_hub_url(repo_id=source_repo_id, filename=filename, repo_type="dataset")
-                log_message(f"Found failed file for retry at index {i}: {filename}", "INFO")
-                return {
-                    "filename": filename,
-                    "url": file_url,
-                    "index": i
-                }
+        log_message(f"✅ Successfully transcribed: {wav_path}", "INFO")
+        return formatted_result
         
-        log_message("All files have been processed. Waiting for new files...", "INFO")
+    except ImportError as e:
+        missing_lib = str(e)
+        log_message(f"❌ Missing library. Install with: pip install transformers librosa torch torchaudio", "ERROR")
+        log_message(f"   Error: {missing_lib}", "ERROR")
         return None
-        
     except Exception as e:
-        log_message(f"❌ Failed to get next file to process: {str(e)}", "ERROR")
+        log_message(f"❌ Failed to transcribe {wav_path}: {str(e)}", "ERROR")
         return None
 
-def run_whisper_transcription(audio_path: str, output_dir: str, model: str) -> Optional[str]:
+def process_audio_file(wav_path: str, reference_map: Dict[str, str], matched_filename: str) -> bool:
     """
-    Runs Whisper transcription using the transformers library.
-    Returns the path to the generated JSON file on success.
-    No ffmpeg dependency required.
+    Main processing logic for a single audio file:
+    1. Transcribe using Whisper
+    2. Save transcription as JSON
+    3. Upload to HF dataset
+    4. Clean up local files
     """
-    log_message(f"🎙️ Starting transcription for {os.path.basename(audio_path)} with model {model}...", "INFO")
+    wav_filename = os.path.basename(wav_path)
+    
+    # 1. Transcribe audio
+    transcription = transcribe_audio(wav_path)
+    if transcription is None:
+        log_failed_file(wav_filename, "Transcription failed")
+        return False
+    
+    # 2. Save transcription as JSON
+    json_filename = os.path.splitext(matched_filename)[0] + "_transcription.json"
+    json_output_path = os.path.join(TRANSCRIPTIONS_FOLDER, json_filename)
     
     try:
-        # Get the Whisper pipeline
-        pipe = get_whisper_pipeline()
-        
-        # Load audio using librosa
-        log_message(f"Loading audio file: {audio_path}", "INFO")
-        audio_data, sample_rate = librosa.load(audio_path, sr=16000)
-        
-        # Run transcription
-        log_message(f"Running transcription...", "INFO")
-        result = pipe(
-            audio_data,
-            chunk_length_s=30,
-            batch_size=8,
-            return_timestamps=True
-        )
-        
-        # Extract text and chunks
-        transcription_text = result.get("text", "")
-        chunks = result.get("chunks", [])
-        
-        log_message(f"✅ Transcription successful: {len(transcription_text)} characters", "INFO")
-        
-        # Prepare output JSON structure
-        output_json = {
-            "text": transcription_text,
-            "chunks": chunks,
-            "language": result.get("language", "en")
-        }
-        
-        # Save to JSON file
-        base_name, _ = os.path.splitext(os.path.basename(audio_path))
-        json_output_path = os.path.join(output_dir, f"{base_name}.json")
+        os.makedirs(os.path.dirname(json_output_path), exist_ok=True)
         
         with open(json_output_path, "w", encoding="utf-8") as f:
-            json.dump(output_json, f, indent=2, ensure_ascii=False)
+            json.dump(transcription, f, indent=2, ensure_ascii=False)
         
-        log_message(f"✅ Saved transcription to: {json_output_path}", "INFO")
-        return json_output_path
+        log_message(f"✅ Saved transcription: {json_output_path}", "INFO")
         
     except Exception as e:
-        log_message(f"❌ An error occurred during transcription: {str(e)}", "ERROR")
-        import traceback
-        log_message(f"Traceback: {traceback.format_exc()}", "ERROR")
-        return None
-
-def process_audio_file(audio_path: str, reference_map: Dict[str, str], output_filename: str) -> bool:
-    """
-    Transcribes the audio file, renames the output JSON to match the reference,
-    and uploads the result to the API.
-    """
-    
-    # 1. Run transcription
-    json_output_path = run_whisper_transcription(audio_path, TRANSCRIPTIONS_FOLDER, WHISPER_MODEL)
-    
-    if not json_output_path:
+        log_message(f"❌ Failed to save transcription JSON: {str(e)}", "ERROR")
+        log_failed_file(wav_filename, f"Failed to save JSON: {str(e)}")
         return False
-        
-    # 2. Rename the JSON file to the matched filename
-    # The output_filename already includes the correct extension (e.g., .txt or .json)
-    # We assume the reference map provides the full target filename.
-    
-    # The whisper output is a JSON file named after the audio file.
-    # We need to rename it to the target filename (which should be a JSON file for the backend).
     
-    # The output_filename is the matched filename from the reference map (e.g., 'audio_file_name.txt')
-    # The backend expects a JSON file. Let's assume the matched filename should be used as the base
-    # but with a .json extension for the upload.
-    
-    # Let's stick to the original logic: the backend expects a JSON file with the name
-    # of the audio file (or the matched reference file) with a .json extension.
-    
-    # Since the whisper output is already a JSON file, we just need to rename it
-    # to the desired final name.
-    
-    # The output_filename passed here is the base name of the audio file or the matched reference file.
-    # If it's a reference file name (e.g., 'file.txt'), we should probably use 'file.json'.
+    # 3. Upload to HF dataset
+    try:
+        path_in_repo = f"transcriptions/{json_filename}"
+        commit_message = f"Add transcription for: {matched_filename}"
+        
+        hf_api.upload_file(
+            path_or_fileobj=json_output_path,
+            path_in_repo=path_in_repo,
+            repo_id=TARGET_REPO_ID,
+            repo_type="dataset",
+            commit_message=commit_message
+        )
+        log_message(f"✅ Successfully uploaded transcription: {json_filename}", "INFO")
+        processing_status["transcribed_files"] += 1
+        
+    except Exception as e:
+        log_message(f"❌ Failed to upload transcription to HF: {str(e)}", "ERROR")
+        log_failed_file(wav_filename, f"Failed to upload: {str(e)}")
+        return False
     
-    # For simplicity and to match the backend's expectation (which handles JSON),
-    # we will rename the whisper output JSON to the base name of the audio file
-    # and ensure it has a .json extension.
+    # 4. Clean up local files
+    try:
+        os.remove(json_output_path)
+        log_message(f"🗑️ Cleaned up local transcription file: {json_output_path}", "INFO")
+    except:
+        pass
     
-    base_name, _ = os.path.splitext(output_filename)
-    final_json_filename = f"{base_name}.json"
-    final_json_path = os.path.join(TRANSCRIPTIONS_FOLDER, final_json_filename)
+    return True
+
+def get_next_file_to_process(repo_id: str, state: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """
+    Finds the next audio file to process from the source repo in reverse order (oldest to newest).
+    Returns: { 'filename': str, 'url': str, 'index': int } or None
+    """
+    log_message(f"🔍 Searching for next audio file to process in {repo_id}", "INFO")
     
     try:
-        if json_output_path != final_json_path:
-            shutil.move(json_output_path, final_json_path)
-            log_message(f"✅ Renamed transcription to: {final_json_filename}", "INFO")
-    except Exception as e:
-        log_message(f"❌ Failed to rename transcription file: {str(e)}", "ERROR")
-        return False
+        files_list = hf_api.list_repo_files(repo_id=repo_id, repo_type="dataset")
         
-    # 3. Upload transcription to API
-    if upload_transcription_to_api(final_json_path, final_json_filename):
-        processing_status["transcribed_files"] += 1
-        # Clean up the local transcription file after successful upload
-        try:
-            os.remove(final_json_path)
-            log_message(f"🗑️ Cleaned up local transcription file: {final_json_path}", "INFO")
-        except Exception as e:
-            log_message(f"❌ Failed to clean up transcription file: {str(e)}", "ERROR")
-        return True
-    else:
-        log_message(f"❌ Failed to upload transcription to API: {final_json_filename}", "ERROR")
-        return False
+        # Filter for audio files and sort in reverse order (descending)
+        audio_files = sorted([f for f in files_list if f.endswith(('.wav', '.mp3'))], reverse=True)
+        
+        if not audio_files:
+            log_message("ℹ️ No audio files found in the source repository.", "INFO")
+            return None
+            
+        processing_status["total_files"] = len(audio_files)
+        
+        start_index = state.get("next_download_index", 0)
+        
+        for index in range(start_index, len(audio_files)):
+            filename = audio_files[index]
+            file_state = state["file_states"].get(filename)
+            
+            if file_state is None or file_state == "failed":
+                url = hf_hub_url(repo_id=repo_id, filename=filename, repo_type="dataset", subfolder=None)
+                
+                log_message(f"✅ Found next audio file: {filename} at index {index}", "INFO")
+                return {
+                    'filename': filename,
+                    'url': url,
+                    'index': index
+                }
+                
+            elif file_state == "processing":
+                log_message(f"⚠️ File {filename} is currently marked as 'processing'. Skipping for now.", "WARNING")
+                
+            elif file_state == "processed":
+                log_message(f"ℹ️ File {filename} already processed. Skipping.", "INFO")
+                
+        log_message("ℹ️ All files up to the current index have been processed or skipped.", "INFO")
+        
+        if start_index >= len(audio_files):
+            log_message("ℹ️ Reached end of file list. Resetting index to 0 for next loop.", "INFO")
+            state["next_download_index"] = 0
+            upload_hf_state(TARGET_REPO_ID, HF_STATE_FILE, state)
+            
+        return None
+        
+    except Exception as e:
+        log_message(f"❌ Failed to list files from Hugging Face: {str(e)}", "ERROR")
+        return None
 
 def main_processing_loop():
-    """The main loop that continuously checks for and processes new audio files."""
-    global processing_status
+    """The main loop that orchestrates the download, transcription, and upload cycle."""
     
     if processing_status["is_running"]:
-        log_message("Processing loop is already running.", "WARNING")
+        log_message("⚠️ Processing loop is already running.", "WARNING")
         return
         
     processing_status["is_running"] = True
-    log_message("🚀 Audio transcription processing loop started.", "INFO")
     
-    # 1. Get the reference map once
-    reference_map = get_reference_map(REFERENCE_REPO_ID)
-    if not reference_map:
-        log_message("❌ Could not get reference map. Stopping loop.", "CRITICAL")
-        processing_status["is_running"] = False
-        return
-        
     try:
+        log_message("🚀 Starting audio transcription processing loop...", "INFO")
+        
+        # Fetch reference files from BG_Vid repo once at the start
+        reference_map = fetch_reference_files(REFERENCE_REPO_ID)
+        
+        if not reference_map:
+            log_message("❌ No reference files found. Cannot proceed.", "ERROR")
+            return
+        
         while processing_status["is_running"]:
-            time.sleep(PROCESSING_DELAY)
             
-            # 1. Download state from the new API
-            current_state = download_state_from_api()
+            current_state = download_hf_state(TARGET_REPO_ID, HF_STATE_FILE)
             next_file_info = get_next_file_to_process(SOURCE_REPO_ID, current_state)
             
             if next_file_info is None:
@@ -597,10 +494,6 @@ def main_processing_loop():
             matched_filename = None
             
             try:
-                # 2. Lock file by updating state on the API
-                # IMPORTANT: Update next_download_index when locking to prevent other workers from picking same file
-                current_state["next_download_index"] = target_index + 1
-                
                 if not lock_file_for_processing(target_file, current_state):
                     log_message(f"❌ Failed to lock file {target_file}. Skipping.", "ERROR")
                     time.sleep(PROCESSING_DELAY)
@@ -618,7 +511,6 @@ def main_processing_loop():
                     # Use matched filename if found, otherwise use original filename
                     output_filename = matched_filename if matched_filename else base_filename
                     
-                    # 3. Process and Upload transcription to API
                     if process_audio_file(local_wav_path, reference_map, output_filename):
                         success = True
                         log_message(f"✅ Finished processing: {target_file}", "INFO")
@@ -632,87 +524,180 @@ def main_processing_loop():
                 log_failed_file(target_file, str(e))
                 
             finally:
-                # 4. Unlock/Mark as processed by updating state on the API
-                # IMPORTANT: Don't re-download the state here because it will lose the incremented index!
-                # Instead, use the current_state which already has the correct next_download_index
+                next_index_to_save = target_index + 1
+                current_state = download_hf_state(TARGET_REPO_ID, HF_STATE_FILE)
                 
                 if success:
-                    # Mark as processed and update index, then upload state
-                    unlock_file_as_processed(target_file, current_state, current_state["next_download_index"])
+                    unlock_file_as_processed(target_file, current_state, next_index_to_save)
                     processing_status["processed_files"] += 1
                 else:
-                    # Mark as failed but keep the incremented index so next worker can proceed
-                    log_message(f"⚠️ File {target_file} failed. Marking as 'failed' and updating state.", "WARNING")
+                    log_message(f"⚠️ Processing failed for {target_file}. Marking as 'failed' and advancing index.", "WARNING")
                     current_state["file_states"][target_file] = "failed"
-                    # Don't overwrite next_download_index here - keep it at the incremented value
-                    upload_state_to_api(current_state)
-                
-                # Clean up the downloaded audio file regardless of success
-                try:
-                    if os.path.exists(local_wav_path):
-                        os.remove(local_wav_path)
-                        log_message(f"🗑️ Cleaned up local audio file: {local_wav_path}", "INFO")
-                except Exception as e:
-                    log_message(f"❌ Failed to clean up audio file: {str(e)}", "ERROR")
-                
-                processing_status["current_file"] = None
-                time.sleep(PROCESSING_DELAY)
-                
-    except Exception as e:
-        log_message(f"🔥 Critical error in main processing loop: {str(e)}", "CRITICAL")
+                    current_state["next_download_index"] = next_index_to_save
+                    upload_hf_state(TARGET_REPO_ID, HF_STATE_FILE, current_state)
+                    processing_status["failed_files"] += 1
+                    
+                if os.path.exists(local_wav_path):
+                    os.remove(local_wav_path)
+                    log_message(f"🗑️ Cleaned up local file: {local_wav_path}", "INFO")
+
+            time.sleep(PROCESSING_DELAY)
+
+        log_message("🎉 Processing complete!", "INFO")
+        log_message(f"📊 Final stats: {processing_status['transcribed_files']} audio files transcribed, {processing_status['processed_files']} files processed", "INFO")
         
+    except KeyboardInterrupt:
+        log_message("⏹️ Processing interrupted by user", "WARNING")
+    except Exception as e:
+        log_message(f"❌ Fatal error: {str(e)}", "ERROR")
     finally:
         processing_status["is_running"] = False
-        log_message("🛑 Audio transcription processing loop stopped.", "INFO")
-
-# --- FastAPI Endpoints (Unchanged) ---
+        cleanup_temp_files()
 
-# Add to configuration section
-AUTO_START_PROCESSING = os.environ.get("AUTO_START_PROCESSING", "true").lower() == "true"
+if __name__ == "__main__":
+    main_processing_loop()
 
-@app.on_event("startup")
-async def startup_event():
-    """Conditionally start processing based on environment variable."""
-    if AUTO_START_PROCESSING:
-        log_message("🚀 AUTO_START_PROCESSING enabled - Starting processing loop...", "INFO")
-        thread = threading.Thread(target=main_processing_loop, daemon=True)
-        thread.start()
-        log_message("✅ Background processing thread started", "INFO")
-    else:
-        log_message("⏸️ AUTO_START_PROCESSING disabled - Use /start endpoint to begin", "INFO")
+# ===== FASTAPI ENDPOINTS =====
 
 @app.get("/")
 async def root():
-    """Root endpoint to check service status."""
-    return {"message": "Audio Transcriber Service is running", "status": processing_status}
+    """Root endpoint with service info"""
+    return {
+        "service": "Audio Transcriber",
+        "status": "running",
+        "version": "1.0.0",
+        "endpoints": {
+            "status": "/status",
+            "start": "/start",
+            "stop": "/stop",
+            "process": "/process/{filename}",
+            "logs": "/logs"
+        }
+    }
 
 @app.get("/status")
 async def get_status():
-    """Get the current processing status."""
-    return processing_status
+    """Get current processing status"""
+    return {
+        "is_running": processing_status["is_running"],
+        "current_file": processing_status["current_file"],
+        "total_files": processing_status["total_files"],
+        "processed_files": processing_status["processed_files"],
+        "transcribed_files": processing_status["transcribed_files"],
+        "failed_files": processing_status["failed_files"],
+        "last_update": processing_status["last_update"],
+        "recent_logs": processing_status["logs"][-10:]
+    }
 
 @app.post("/start")
 async def start_processing():
-    """Start the background processing loop."""
+    """Start the main processing loop"""
     if processing_status["is_running"]:
-        return JSONResponse(status_code=200, content={"message": "Processing already running."})
+        raise HTTPException(status_code=400, detail="Processing already running")
     
-    thread = threading.Thread(target=main_processing_loop)
+    # Start processing in a separate thread
+    thread = threading.Thread(target=main_processing_loop, daemon=True)
     thread.start()
-    return JSONResponse(status_code=200, content={"message": "Processing started in background."})
+    
+    return {
+        "message": "Processing started",
+        "status": "started"
+    }
 
 @app.post("/stop")
 async def stop_processing():
-    """Stop the background processing loop."""
+    """Stop the main processing loop"""
     if not processing_status["is_running"]:
-        return JSONResponse(status_code=200, content={"message": "Processing is not running."})
+        raise HTTPException(status_code=400, detail="Processing not running")
     
     processing_status["is_running"] = False
-    return JSONResponse(status_code=200, content={"message": "Processing stop requested. Will stop after current file."})
+    
+    return {
+        "message": "Processing stopped",
+        "status": "stopped"
+    }
+
+@app.get("/logs")
+async def get_logs(limit: int = 50):
+    """Get recent logs"""
+    logs = processing_status["logs"][-limit:]
+    return {
+        "total_logs": len(processing_status["logs"]),
+        "recent_logs": logs
+    }
+
+@app.post("/process/{filename}")
+async def process_single_file(filename: str):
+    """Process a single audio file manually"""
+    try:
+        log_message(f"🎯 Manual processing requested for: {filename}", "INFO")
+        
+        # Download and process the file
+        reference_map = fetch_reference_files(REFERENCE_REPO_ID)
+        if not reference_map:
+            raise HTTPException(status_code=500, detail="Could not fetch reference files")
+        
+        # Get file URL
+        audio_url = hf_hub_url(repo_id=SOURCE_REPO_ID, filename=filename, repo_type="dataset", subfolder=None)
+        local_wav_path = os.path.join(DOWNLOAD_FOLDER, os.path.basename(filename))
+        
+        # Download
+        if not download_with_retry(audio_url, local_wav_path):
+            raise HTTPException(status_code=500, detail="Failed to download file")
+        
+        # Find match
+        base_filename = os.path.basename(filename)
+        matched_filename = find_matching_filename(base_filename, reference_map)
+        
+        if not matched_filename:
+            os.remove(local_wav_path)
+            raise HTTPException(status_code=404, detail="No matching filename found")
+        
+        # Process
+        if process_audio_file(local_wav_path, reference_map, matched_filename):
+            processing_status["transcribed_files"] += 1
+            
+            if os.path.exists(local_wav_path):
+                os.remove(local_wav_path)
+            
+            return {
+                "status": "success",
+                "file": filename,
+                "matched": matched_filename,
+                "message": "Audio transcribed and uploaded successfully"
+            }
+        else:
+            if os.path.exists(local_wav_path):
+                os.remove(local_wav_path)
+            raise HTTPException(status_code=500, detail="Processing failed")
+            
+    except Exception as e:
+        log_message(f"❌ Manual processing error: {str(e)}", "ERROR")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.on_event("startup")
+async def startup_event():
+    """Auto-start processing when server starts"""
+    log_message("🚀 Server startup: Checking dependencies...", "INFO")
+    
+    try:
+        import transformers
+        log_message("✅ Transformers found", "INFO")
+    except ImportError:
+        log_message("⚠️ WARNING: Transformers not installed!", "WARNING")
+        log_message("   Install with: pip install transformers librosa torch torchaudio", "WARNING")
+    
+    log_message("🚀 Server startup: Auto-starting processing loop", "INFO")
+    
+    # Start processing in a separate thread
+    thread = threading.Thread(target=main_processing_loop, daemon=True)
+    thread.start()
 
-# --- Main Execution ---
+def run_api(host: str = "0.0.0.0", port: int = 8000):
+    """Run the FastAPI server"""
+    log_message(f"🚀 Starting FastAPI server on {host}:{port}", "INFO")
+    uvicorn.run(app, host=host, port=port)
 
 if __name__ == "__main__":
-    # This block is for local testing and won't be used in the final sandbox execution
-    # but is good practice for a runnable script.
-    uvicorn.run(app, host="0.0.0.0", port=8000)
+    # Run API server (processing will auto-start via startup event)
+    run_api()