import os import logging import shutil import re import zipfile import uvicorn import asyncio import threading import time from fastapi import FastAPI, Request, BackgroundTasks, HTTPException from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from pydantic import BaseModel from typing import Dict, Set, Tuple, List, Optional from io import BytesIO from pathlib import Path # Configure data directories for Hugging Face Spaces # HF Spaces can have persistent storage in the /data directory DATA_DIR = '/data' # Import the existing telegram bot functionality # Define these directories here to ensure they're created before importing app.py PENDING_DIR = os.path.join(DATA_DIR, 'pending') PENDING_VOICE_DIR = os.path.join(PENDING_DIR, 'voice') PENDING_TEXT_DIR = os.path.join(PENDING_DIR, 'text') PROCESSED_DIR = os.path.join(DATA_DIR, 'processed') PROCESSED_VOICE_DIR = os.path.join(PROCESSED_DIR, 'voice') PROCESSED_TEXT_DIR = os.path.join(PROCESSED_DIR, 'text') # Create directories if they don't exist os.makedirs(PENDING_VOICE_DIR, exist_ok=True) os.makedirs(PENDING_TEXT_DIR, exist_ok=True) os.makedirs(PROCESSED_VOICE_DIR, exist_ok=True) os.makedirs(PROCESSED_TEXT_DIR, exist_ok=True) # Import from app.py now that we've defined the directories try: from app import extract_sequence_number, get_files_by_sequence_number except Exception as e: # Define fallback functions if import fails def extract_sequence_number(filename: str) -> int: """Extract sequence number from filename, e.g., 'Sound 100.wav' -> 100.""" match = re.search(r'(\d+)', filename) if match: return int(match.group(1)) return None def get_files_by_sequence_number(): """Create dictionaries mapping sequence numbers to filenames.""" voice_files = {} text_files = {} # Map voice files to sequence numbers for filename in os.listdir(PENDING_VOICE_DIR): if filename.endswith('.wav'): seq_num = extract_sequence_number(filename) if seq_num is not None: voice_files[seq_num] = filename # Map text files to sequence numbers for filename in os.listdir(PENDING_TEXT_DIR): if filename.endswith('.txt'): seq_num = extract_sequence_number(filename) if seq_num is not None: text_files[seq_num] = filename return voice_files, text_files logging.error(f"Error importing from app.py: {str(e)}") # Enable logging logging.basicConfig( format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO ) logger = logging.getLogger(__name__) app = FastAPI(title="Algerian Darija Transcription Service") # Create templates instance templates = Jinja2Templates(directory="templates") # Telegram bot process bot_process = None bot_running = False bot_start_time = None # Function to start the bot in a separate process instead of a thread def start_bot_process(): from multiprocessing import Process global bot_process, bot_running # Create a new process for the bot bot_process = Process(target=run_bot_process) bot_process.daemon = True # Process will terminate when main process ends bot_process.start() bot_running = True return bot_process.pid def run_bot_process(): """Function that runs in a separate process to start the bot""" import sys import os import asyncio try: # Import and run the main function from app.py from app import main # Create a new event loop for this process loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) # Run the bot loop.run_until_complete(main()) loop.run_forever() except Exception as e: sys.stderr.write(f"Error in bot process: {str(e)}\n") sys.exit(1) @app.get("/", response_class=HTMLResponse) async def get_root(request: Request): return templates.TemplateResponse("index.html", {"request": request, "bot_running": bot_running}) @app.post("/start-bot") async def start_bot_endpoint(background_tasks: BackgroundTasks): global bot_running, bot_start_time if bot_running: return {"status": "error", "message": "Bot is already running"} # Check if TELEGRAM_TOKEN is set if not os.environ.get("TELEGRAM_TOKEN"): return {"status": "error", "message": "TELEGRAM_TOKEN environment variable is not set. Please configure it in Hugging Face Space settings."} try: # Start bot in a separate process pid = start_bot_process() bot_start_time = time.time() return {"status": "success", "message": f"Bot started successfully with PID {pid}"} except Exception as e: logger.error(f"Error starting bot: {str(e)}") return {"status": "error", "message": f"Failed to start bot: {str(e)}"} @app.post("/stop-bot") async def stop_bot_endpoint(): global bot_process, bot_running, bot_start_time if not bot_running or bot_process is None: return {"status": "error", "message": "Bot is not running"} try: # Terminate the process bot_process.terminate() bot_process.join(timeout=5) # Wait for process to terminate # If process didn't terminate, force kill it if bot_process.is_alive(): bot_process.kill() bot_process.join() bot_running = False bot_start_time = None return {"status": "success", "message": "Bot stopped successfully"} except Exception as e: logger.error(f"Error stopping bot: {str(e)}") return {"status": "error", "message": f"Failed to stop bot: {str(e)}"} @app.get("/status") async def get_status(): # Check if directories exist and create if needed os.makedirs(PENDING_VOICE_DIR, exist_ok=True) os.makedirs(PENDING_TEXT_DIR, exist_ok=True) os.makedirs(PROCESSED_VOICE_DIR, exist_ok=True) os.makedirs(PROCESSED_TEXT_DIR, exist_ok=True) # Count pending files pending_voice_count = len( [f for f in os.listdir(PENDING_VOICE_DIR) if f.endswith('.wav')]) pending_text_count = len( [f for f in os.listdir(PENDING_TEXT_DIR) if f.endswith('.txt')]) # Get sequence numbers voice_files, text_files = get_files_by_sequence_number() matching_pairs = len( set(voice_files.keys()).intersection(set(text_files.keys()))) # Count processed files - Only count text files now as voice files are no longer saved to processed processed_voice_count = 0 # Set to 0 since we no longer save voice files processed_text_count = 0 # Count processed files by user user_stats = {} # Process text files only (voice files are no longer saved to processed directory) for root, dirs, files in os.walk(PROCESSED_TEXT_DIR): txt_files = [f for f in files if f.endswith('.txt')] processed_text_count += len(txt_files) # Get user ID from directory path if os.path.basename(root).startswith("user_"): user_id = os.path.basename(root) if user_id not in user_stats: user_stats[user_id] = {"voice": 0, "text": 0} user_stats[user_id]["text"] += len(txt_files) # Calculate uptime if bot is running uptime = None if bot_running and bot_start_time: uptime = int(time.time() - bot_start_time) return { "bot_running": bot_running, "uptime_seconds": uptime, "pending_voice_count": pending_voice_count, "pending_text_count": pending_text_count, "matching_pairs": matching_pairs, "processed_voice_count": processed_voice_count, "processed_text_count": processed_text_count, "user_stats": user_stats, "is_huggingface": True } @app.get("/list-users") async def list_users(): users = set() # Get users from text directories only (since voice files are no longer saved) for item in os.listdir(PROCESSED_TEXT_DIR): if item.startswith("user_"): users.add(item) return {"users": sorted(list(users))} @app.get("/download/{user_id}") async def download_processed_data(user_id: str): if not user_id.startswith("user_"): raise HTTPException(status_code=400, detail="Invalid user ID format") # Check if user exists - only check text directory now text_dir = os.path.join(PROCESSED_TEXT_DIR, user_id) if not os.path.exists(text_dir): raise HTTPException( status_code=404, detail=f"No data found for user {user_id}") # Create a zip file in memory zip_buffer = BytesIO() with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file: # Add text files only if os.path.exists(text_dir): for file_name in os.listdir(text_dir): file_path = os.path.join(text_dir, file_name) if os.path.isfile(file_path) and file_name.endswith('.txt'): # Read the file and add it to the zip with open(file_path, 'rb') as f: zip_file.writestr(f"text/{file_name}", f.read()) # Reset buffer position zip_buffer.seek(0) # Return the zip file as a response return StreamingResponse( zip_buffer, media_type="application/zip", headers={ "Content-Disposition": f"attachment; filename={user_id}_processed_data.zip" } ) @app.get("/download-all") async def download_all_processed_data(): # Create a zip file in memory zip_buffer = BytesIO() with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file: # Add all processed text files only for root, dirs, files in os.walk(PROCESSED_TEXT_DIR): for file_name in files: if file_name.endswith('.txt'): file_path = os.path.join(root, file_name) # Get the relative path from the PROCESSED_TEXT_DIR rel_path = os.path.relpath(file_path, PROCESSED_TEXT_DIR) # Read the file and add it to the zip with open(file_path, 'rb') as f: zip_file.writestr(f"text/{rel_path}", f.read()) # Reset buffer position zip_buffer.seek(0) # Return the zip file as a response return StreamingResponse( zip_buffer, media_type="application/zip", headers={ "Content-Disposition": "attachment; filename=all_processed_data.zip" } ) @app.on_event("startup") async def setup_app(): # Ensure templates directory exists os.makedirs("templates", exist_ok=True) # Only create index.html if it doesn't exist if not os.path.exists("templates/index.html"): # Create a simple default template if needed with open("templates/index.html", "w") as f: f.write(""" Algerian Darija Transcription Service

Algerian Darija Transcription Service

Bot Control

Start or stop the Telegram bot service.

Status

Note: Only corrected text files are saved to the processed directory. Voice files are removed after processing to save storage space.
Loading status...

Users Data

Loading users...

Data Download

Download all processed corrected text files or data for specific users.

""") # Entrypoint for running the server if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=7860)