Spaces:
Running
Running
GitHub Action commited on
Commit ·
6374131
1
Parent(s): d24235e
deploy from github actions
Browse files- app/models/process.py +1 -0
- app/routers/lyrics.py +2 -2
- app/routers/songs.py +4 -0
- app/routers/workers.py +9 -0
- app/utils/hf_wake.py +149 -0
- main.py +5 -0
- migrate_worker_url.py +19 -0
app/models/process.py
CHANGED
|
@@ -58,4 +58,5 @@ class Worker(Base):
|
|
| 58 |
token = Column(String, unique=True, index=True, nullable=True)
|
| 59 |
command = Column(String, default="none") # start | stop | none
|
| 60 |
is_active = Column(Boolean, default=False)
|
|
|
|
| 61 |
|
|
|
|
| 58 |
token = Column(String, unique=True, index=True, nullable=True)
|
| 59 |
command = Column(String, default="none") # start | stop | none
|
| 60 |
is_active = Column(Boolean, default=False)
|
| 61 |
+
url = Column(String, nullable=True) # URL de despertar / HuggingFace Space URL
|
| 62 |
|
app/routers/lyrics.py
CHANGED
|
@@ -475,8 +475,8 @@ def update_repertoire(id: int, rep: RepertorioCreate, db: Session = Depends(get_
|
|
| 475 |
for idx, lyric_id in enumerate(rep.lyrics_ids):
|
| 476 |
db_rep_lyric = RepertorioLyric(repertorio_id=id, lyric_id=lyric_id, order=idx)
|
| 477 |
db.add(db_rep_lyric)
|
| 478 |
-
|
| 479 |
-
|
| 480 |
db.refresh(db_rep)
|
| 481 |
if db_rep.group_id is not None:
|
| 482 |
try:
|
|
|
|
| 475 |
for idx, lyric_id in enumerate(rep.lyrics_ids):
|
| 476 |
db_rep_lyric = RepertorioLyric(repertorio_id=id, lyric_id=lyric_id, order=idx)
|
| 477 |
db.add(db_rep_lyric)
|
| 478 |
+
|
| 479 |
+
db.commit()
|
| 480 |
db.refresh(db_rep)
|
| 481 |
if db_rep.group_id is not None:
|
| 482 |
try:
|
app/routers/songs.py
CHANGED
|
@@ -43,6 +43,7 @@ load_dotenv()
|
|
| 43 |
# Importar tasks desde el directorio padre
|
| 44 |
sys.path.append(os.getcwd())
|
| 45 |
from tasks import procesar_cancion_completa, celery_app
|
|
|
|
| 46 |
|
| 47 |
# ==========================================
|
| 48 |
# CONFIGURACIÓN Y LOGGING
|
|
@@ -240,6 +241,7 @@ async def queue_cloud_song(
|
|
| 240 |
job = procesar_cancion_completa.apply_async(**kwargs_async)
|
| 241 |
nuevo_proceso.celery_id = job.id
|
| 242 |
db.commit()
|
|
|
|
| 243 |
return {
|
| 244 |
"message": "Encolado desde la nube",
|
| 245 |
"task_id": task_id,
|
|
@@ -420,6 +422,7 @@ async def subir_cancion(
|
|
| 420 |
kwargs_async["queue"] = queue_name
|
| 421 |
job = procesar_cancion_completa.apply_async(**kwargs_async)
|
| 422 |
logger.info(f"[{task_id}] Tarea encolada: {job.id}")
|
|
|
|
| 423 |
|
| 424 |
return {
|
| 425 |
"celery_id": job.id,
|
|
@@ -504,6 +507,7 @@ def subir_youtube(
|
|
| 504 |
kwargs_async["queue"] = queue_name
|
| 505 |
job = procesar_cancion_completa.apply_async(**kwargs_async)
|
| 506 |
logger.info(f"[{task_id}] Tarea YouTube encolada: {job.id}")
|
|
|
|
| 507 |
return {
|
| 508 |
"celery_id": job.id,
|
| 509 |
"task_id": task_id,
|
|
|
|
| 43 |
# Importar tasks desde el directorio padre
|
| 44 |
sys.path.append(os.getcwd())
|
| 45 |
from tasks import procesar_cancion_completa, celery_app
|
| 46 |
+
from app.utils.hf_wake import wake_hf_workers
|
| 47 |
|
| 48 |
# ==========================================
|
| 49 |
# CONFIGURACIÓN Y LOGGING
|
|
|
|
| 241 |
job = procesar_cancion_completa.apply_async(**kwargs_async)
|
| 242 |
nuevo_proceso.celery_id = job.id
|
| 243 |
db.commit()
|
| 244 |
+
wake_hf_workers()
|
| 245 |
return {
|
| 246 |
"message": "Encolado desde la nube",
|
| 247 |
"task_id": task_id,
|
|
|
|
| 422 |
kwargs_async["queue"] = queue_name
|
| 423 |
job = procesar_cancion_completa.apply_async(**kwargs_async)
|
| 424 |
logger.info(f"[{task_id}] Tarea encolada: {job.id}")
|
| 425 |
+
wake_hf_workers()
|
| 426 |
|
| 427 |
return {
|
| 428 |
"celery_id": job.id,
|
|
|
|
| 507 |
kwargs_async["queue"] = queue_name
|
| 508 |
job = procesar_cancion_completa.apply_async(**kwargs_async)
|
| 509 |
logger.info(f"[{task_id}] Tarea YouTube encolada: {job.id}")
|
| 510 |
+
wake_hf_workers()
|
| 511 |
return {
|
| 512 |
"celery_id": job.id,
|
| 513 |
"task_id": task_id,
|
app/routers/workers.py
CHANGED
|
@@ -157,6 +157,15 @@ def delete_worker(worker_id: int, db: Session = Depends(get_db), current_admin:
|
|
| 157 |
ws_manager.broadcast_sync({"type": "refresh", "tab": "workers"})
|
| 158 |
return {"message": "Worker eliminado exitosamente"}
|
| 159 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
# ==========================================
|
| 161 |
# BYOK / LOCAL WORKERS ENDPOINTS
|
| 162 |
# ==========================================
|
|
|
|
| 157 |
ws_manager.broadcast_sync({"type": "refresh", "tab": "workers"})
|
| 158 |
return {"message": "Worker eliminado exitosamente"}
|
| 159 |
|
| 160 |
+
@router.post("/wake")
|
| 161 |
+
def wake_workers():
|
| 162 |
+
"""
|
| 163 |
+
Endpoint para despertar manualmente los workers de Hugging Face y la red.
|
| 164 |
+
"""
|
| 165 |
+
from app.utils.hf_wake import wake_hf_workers
|
| 166 |
+
wake_hf_workers()
|
| 167 |
+
return {"message": "Peticiones de despertar enviadas a los workers de Hugging Face."}
|
| 168 |
+
|
| 169 |
# ==========================================
|
| 170 |
# BYOK / LOCAL WORKERS ENDPOINTS
|
| 171 |
# ==========================================
|
app/utils/hf_wake.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import logging
|
| 3 |
+
import threading
|
| 4 |
+
import time
|
| 5 |
+
import requests
|
| 6 |
+
from datetime import datetime, timedelta
|
| 7 |
+
|
| 8 |
+
logger = logging.getLogger(__name__)
|
| 9 |
+
|
| 10 |
+
def get_hf_spaces_config():
|
| 11 |
+
"""
|
| 12 |
+
Obtiene la lista de Hugging Face Spaces o URLs de workers configuradas desde las variables de entorno.
|
| 13 |
+
Soporta repositorios (ej: 'Isaac105/melodix-worker') o URLs completas.
|
| 14 |
+
"""
|
| 15 |
+
raw = os.getenv("HF_SPACES") or os.getenv("HF_WORKER_SPACES") or os.getenv("HF_WORKERS") or ""
|
| 16 |
+
spaces = [s.strip() for s in raw.split(",") if s.strip()]
|
| 17 |
+
return spaces
|
| 18 |
+
|
| 19 |
+
def wake_hf_workers():
|
| 20 |
+
"""
|
| 21 |
+
Intenta despertar los Hugging Face Spaces o workers remotos configurados.
|
| 22 |
+
Utiliza la API de Hugging Face (HfApi.restart_space / get_space_runtime)
|
| 23 |
+
y pings HTTP directos a las URLs de los Spaces.
|
| 24 |
+
"""
|
| 25 |
+
def _do_wake():
|
| 26 |
+
hf_token = os.getenv("HF_TOKEN")
|
| 27 |
+
spaces_config = get_hf_spaces_config()
|
| 28 |
+
|
| 29 |
+
# Buscar también en la base de datos workers con URL configurada
|
| 30 |
+
try:
|
| 31 |
+
from app.database import SessionLocal
|
| 32 |
+
from app.models.process import Worker
|
| 33 |
+
db = SessionLocal()
|
| 34 |
+
try:
|
| 35 |
+
db_workers = db.query(Worker).all()
|
| 36 |
+
for w in db_workers:
|
| 37 |
+
if w.url and w.url not in spaces_config:
|
| 38 |
+
spaces_config.append(w.url)
|
| 39 |
+
finally:
|
| 40 |
+
db.close()
|
| 41 |
+
except Exception as e:
|
| 42 |
+
logger.warning(f"[HF Wake] Error consultando URLs de workers en DB: {e}")
|
| 43 |
+
|
| 44 |
+
if not spaces_config:
|
| 45 |
+
logger.info("[HF Wake] No hay Hugging Face Spaces o URLs configuradas en HF_SPACES ni en DB.")
|
| 46 |
+
return
|
| 47 |
+
|
| 48 |
+
logger.info(f"[HF Wake] Iniciando intento de despertar para {len(spaces_config)} space(s) / worker(s)...")
|
| 49 |
+
|
| 50 |
+
# Inicializar HfApi si existe token
|
| 51 |
+
hf_api = None
|
| 52 |
+
if hf_token:
|
| 53 |
+
try:
|
| 54 |
+
from huggingface_hub import HfApi
|
| 55 |
+
hf_api = HfApi(token=hf_token)
|
| 56 |
+
except Exception as e:
|
| 57 |
+
logger.warning(f"[HF Wake] No se pudo inicializar HfApi: {e}")
|
| 58 |
+
|
| 59 |
+
for target in spaces_config:
|
| 60 |
+
try:
|
| 61 |
+
# Si el target es formato repo_id ("Usuario/nombre-space")
|
| 62 |
+
if "/" in target and not target.startswith("http"):
|
| 63 |
+
space_repo = target
|
| 64 |
+
if hf_api:
|
| 65 |
+
try:
|
| 66 |
+
runtime = hf_api.get_space_runtime(repo_id=space_repo)
|
| 67 |
+
stage = getattr(runtime, 'stage', 'UNKNOWN')
|
| 68 |
+
logger.info(f"[HF Wake] Space '{space_repo}' estado runtime: {stage}")
|
| 69 |
+
if stage in ["SLEEPING", "PAUSED", "STOPPED", "OFF", "BUILD_ERROR", "RUNTIME_ERROR"]:
|
| 70 |
+
logger.info(f"[HF Wake] Reiniciando Space dormido '{space_repo}' vía HF API...")
|
| 71 |
+
hf_api.restart_space(repo_id=space_repo)
|
| 72 |
+
except Exception as ex_api:
|
| 73 |
+
logger.warning(f"[HF Wake] Error en restart_space para '{space_repo}': {ex_api}")
|
| 74 |
+
|
| 75 |
+
parts = space_repo.split('/')
|
| 76 |
+
target_url = f"https://huggingface.co/spaces/{parts[0]}/{parts[1]}"
|
| 77 |
+
else:
|
| 78 |
+
target_url = target
|
| 79 |
+
|
| 80 |
+
# Hacer ping HTTP para activar el proxy de Hugging Face
|
| 81 |
+
if target_url.startswith("http"):
|
| 82 |
+
logger.info(f"[HF Wake] Enviando ping HTTP para despertar a: {target_url}")
|
| 83 |
+
try:
|
| 84 |
+
resp = requests.get(target_url, timeout=12, headers={"User-Agent": "Melodix-Wake-Bot/1.0"})
|
| 85 |
+
logger.info(f"[HF Wake] Ping HTTP a {target_url} enviado. Status code: {resp.status_code}")
|
| 86 |
+
except Exception as http_err:
|
| 87 |
+
logger.warning(f"[HF Wake] Respuesta del ping HTTP a {target_url}: {http_err}")
|
| 88 |
+
except Exception as err:
|
| 89 |
+
logger.error(f"[HF Wake] Error despertando worker '{target}': {err}")
|
| 90 |
+
|
| 91 |
+
# Ejecutar en hilo daemon independiente para no bloquear el hilo de FastAPI
|
| 92 |
+
threading.Thread(target=_do_wake, daemon=True).start()
|
| 93 |
+
|
| 94 |
+
def check_and_wake_workers_if_needed():
|
| 95 |
+
"""
|
| 96 |
+
Verifica si hay tareas pendientes en la DB y si NO hay workers activos.
|
| 97 |
+
Si hay tareas sin atender, invoca wake_hf_workers().
|
| 98 |
+
"""
|
| 99 |
+
try:
|
| 100 |
+
from app.database import SessionLocal
|
| 101 |
+
from app.models.process import Task, Worker
|
| 102 |
+
db = SessionLocal()
|
| 103 |
+
try:
|
| 104 |
+
# 1. Tareas en espera
|
| 105 |
+
pending_count = db.query(Task).filter(Task.status.in_(["PENDING", "PROCESSING"])).count()
|
| 106 |
+
if pending_count == 0:
|
| 107 |
+
return
|
| 108 |
+
|
| 109 |
+
# 2. Workers reportando en los últimos 2 minutos
|
| 110 |
+
two_mins_ago = datetime.utcnow() - timedelta(minutes=2)
|
| 111 |
+
active_workers_count = db.query(Worker).filter(
|
| 112 |
+
Worker.status.in_(["online", "busy"]),
|
| 113 |
+
Worker.last_seen >= two_mins_ago
|
| 114 |
+
).count()
|
| 115 |
+
|
| 116 |
+
logger.info(f"[HF Monitor] Tareas pendientes: {pending_count} | Workers activos: {active_workers_count}")
|
| 117 |
+
|
| 118 |
+
if active_workers_count == 0:
|
| 119 |
+
logger.warning(f"[HF Monitor] ⚠️ {pending_count} tarea(s) en espera y 0 workers activos. Despertando Hugging Face Workers...")
|
| 120 |
+
wake_hf_workers()
|
| 121 |
+
finally:
|
| 122 |
+
db.close()
|
| 123 |
+
except Exception as e:
|
| 124 |
+
logger.error(f"[HF Monitor] Error verificando estado de tareas/workers: {e}")
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
_monitor_thread_started = False
|
| 128 |
+
|
| 129 |
+
def start_worker_monitor_loop(interval_seconds: int = 45):
|
| 130 |
+
"""
|
| 131 |
+
Inicia un hilo en segundo plano que monitorea periódicamente la cola de tareas
|
| 132 |
+
y despierta los workers de Hugging Face si están dormidos.
|
| 133 |
+
"""
|
| 134 |
+
global _monitor_thread_started
|
| 135 |
+
if _monitor_thread_started:
|
| 136 |
+
return
|
| 137 |
+
_monitor_thread_started = True
|
| 138 |
+
|
| 139 |
+
def _loop():
|
| 140 |
+
logger.info(f"[HF Monitor] Bucle de monitoreo de workers iniciado (intervalo: {interval_seconds}s).")
|
| 141 |
+
while True:
|
| 142 |
+
try:
|
| 143 |
+
check_and_wake_workers_if_needed()
|
| 144 |
+
except Exception as e:
|
| 145 |
+
logger.error(f"[HF Monitor] Error en ciclo de monitoreo: {e}")
|
| 146 |
+
time.sleep(interval_seconds)
|
| 147 |
+
|
| 148 |
+
thread = threading.Thread(target=_loop, daemon=True)
|
| 149 |
+
thread.start()
|
main.py
CHANGED
|
@@ -145,6 +145,11 @@ async def startup_event():
|
|
| 145 |
logger.info("=" * 50)
|
| 146 |
logger.info("MELODIX API INICIANDO")
|
| 147 |
logger.info("=" * 50)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
|
| 149 |
@app.on_event("shutdown")
|
| 150 |
async def shutdown_event():
|
|
|
|
| 145 |
logger.info("=" * 50)
|
| 146 |
logger.info("MELODIX API INICIANDO")
|
| 147 |
logger.info("=" * 50)
|
| 148 |
+
try:
|
| 149 |
+
from app.utils.hf_wake import start_worker_monitor_loop
|
| 150 |
+
start_worker_monitor_loop(interval_seconds=45)
|
| 151 |
+
except Exception as e:
|
| 152 |
+
logger.error(f"Error iniciando monitor de workers HF: {e}")
|
| 153 |
|
| 154 |
@app.on_event("shutdown")
|
| 155 |
async def shutdown_event():
|
migrate_worker_url.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import sys
|
| 2 |
+
import os
|
| 3 |
+
sys.path.insert(0, os.path.abspath('.'))
|
| 4 |
+
|
| 5 |
+
from sqlalchemy import text
|
| 6 |
+
from app.database import engine
|
| 7 |
+
|
| 8 |
+
def run_migration():
|
| 9 |
+
print("Ejecutando migracion para agregar columna 'url' a la tabla 'workers'...")
|
| 10 |
+
with engine.connect() as conn:
|
| 11 |
+
try:
|
| 12 |
+
conn.execute(text("ALTER TABLE workers ADD COLUMN IF NOT EXISTS url TEXT DEFAULT NULL;"))
|
| 13 |
+
conn.commit()
|
| 14 |
+
print("Columna 'url' agregada a la tabla 'workers' exitosamente.")
|
| 15 |
+
except Exception as e:
|
| 16 |
+
print(f"Error o ya existe: {e}")
|
| 17 |
+
|
| 18 |
+
if __name__ == "__main__":
|
| 19 |
+
run_migration()
|