Update app/main.py
Browse files- app/main.py +194 -239
app/main.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
import os
|
| 2 |
import uuid
|
| 3 |
import hashlib
|
|
@@ -8,7 +9,7 @@ import re
|
|
| 8 |
import math
|
| 9 |
from datetime import datetime, timedelta
|
| 10 |
from pathlib import Path
|
| 11 |
-
from typing import Dict, Any, List, Union
|
| 12 |
from enum import Enum
|
| 13 |
|
| 14 |
from fastapi import FastAPI, File, UploadFile, BackgroundTasks, HTTPException, Path as FastApiPath, Depends, Query, status
|
|
@@ -16,50 +17,53 @@ from fastapi.responses import JSONResponse, FileResponse
|
|
| 16 |
from pydantic import BaseModel, Field, validator
|
| 17 |
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
| 18 |
|
| 19 |
-
# ---
|
| 20 |
|
| 21 |
-
# API
|
| 22 |
API_KEY = os.getenv("API_KEY")
|
| 23 |
|
| 24 |
-
#
|
| 25 |
-
|
| 26 |
-
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
-
#
|
| 29 |
BASE_DATA_DIR = Path("data")
|
| 30 |
UPLOADS_DIR = BASE_DATA_DIR / "uploads"
|
| 31 |
OUTPUTS_DIR = BASE_DATA_DIR / "outputs"
|
| 32 |
LOGS_DIR = BASE_DATA_DIR / "logs"
|
| 33 |
|
| 34 |
-
#
|
| 35 |
FILES_DB: Dict[str, Dict[str, Any]] = {}
|
| 36 |
TASKS_DB: Dict[str, Dict[str, Any]] = {}
|
| 37 |
|
| 38 |
-
#
|
| 39 |
MAX_AGE_HOURS = 24
|
| 40 |
MAX_TOTAL_SIZE_GB = 50
|
| 41 |
CLEANUP_INTERVAL_MINUTES = 30
|
| 42 |
|
| 43 |
-
#
|
| 44 |
scheduler = AsyncIOScheduler()
|
| 45 |
|
| 46 |
-
# ---
|
| 47 |
|
| 48 |
-
async def verify_token(token: str = Query(..., description="
|
| 49 |
-
"""
|
| 50 |
if not API_KEY:
|
| 51 |
-
print("
|
| 52 |
raise HTTPException(
|
| 53 |
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
| 54 |
-
detail="API
|
| 55 |
)
|
| 56 |
if token != API_KEY:
|
| 57 |
raise HTTPException(
|
| 58 |
status_code=status.HTTP_401_UNAUTHORIZED,
|
| 59 |
-
detail="
|
| 60 |
)
|
| 61 |
|
| 62 |
-
# ---
|
| 63 |
|
| 64 |
class Encoder(str, Enum):
|
| 65 |
av1 = "svt-av1"
|
|
@@ -78,104 +82,84 @@ class X265Preset(str, Enum):
|
|
| 78 |
veryslow = "veryslow"
|
| 79 |
placebo = "placebo"
|
| 80 |
|
| 81 |
-
|
| 82 |
-
"""Indicates the origin of a file."""
|
| 83 |
-
UPLOAD = "upload"
|
| 84 |
-
VFR_TO_CFR = "vfr_to_cfr"
|
| 85 |
-
ENCODE_OUTPUT = "encode_output"
|
| 86 |
-
|
| 87 |
-
# --- Data Models for API (Pydantic) ---
|
| 88 |
|
| 89 |
-
class
|
| 90 |
-
|
| 91 |
file_hash: str
|
| 92 |
file_path: str
|
| 93 |
original_filename: str
|
| 94 |
-
origin: FileOrigin
|
| 95 |
-
task_id: Optional[str] = None
|
| 96 |
-
size: int
|
| 97 |
-
created_at: datetime
|
| 98 |
-
|
| 99 |
-
class UploadResponse(BaseModel):
|
| 100 |
-
message: str
|
| 101 |
-
file_info: FileInfo
|
| 102 |
is_new: bool
|
| 103 |
|
| 104 |
class BaseTaskRequest(BaseModel):
|
| 105 |
-
input_hash: str = Field(..., description="SHA256
|
| 106 |
-
encoder: Encoder = Field(Encoder.av1, description="
|
| 107 |
-
extra_options: str = Field("", description="
|
| 108 |
|
| 109 |
class AutoEncodeRequest(BaseTaskRequest):
|
| 110 |
-
preset: Union[int, X265Preset] = Field(3, description="
|
| 111 |
-
min_vmaf: int = Field(96, description="
|
| 112 |
-
vfr_to_cfr_crf: int = Field(VFR_TO_CFR_CRF_DEFAULT, description="CRF for VFR to CFR conversion.")
|
| 113 |
-
vfr_to_cfr_preset: int = Field(VFR_TO_CFR_PRESET_DEFAULT, description="Preset for VFR to CFR conversion.")
|
| 114 |
|
| 115 |
@validator('preset')
|
| 116 |
def preset_must_match_encoder(cls, v, values):
|
| 117 |
encoder = values.get('encoder')
|
| 118 |
if encoder == Encoder.av1 and not isinstance(v, int):
|
| 119 |
-
raise ValueError('
|
| 120 |
if encoder == Encoder.x265 and not isinstance(v, X265Preset):
|
| 121 |
-
raise ValueError("
|
| 122 |
if isinstance(v, int) and (v < 0 or v > 12):
|
| 123 |
-
raise ValueError('
|
| 124 |
return v
|
| 125 |
|
| 126 |
class CrfSearchRequest(AutoEncodeRequest):
|
| 127 |
pass
|
| 128 |
|
| 129 |
class EncodeRequest(BaseTaskRequest):
|
| 130 |
-
preset: Union[int, X265Preset] = Field(3, description="
|
| 131 |
-
crf: float = Field(20, description="CRF
|
| 132 |
|
| 133 |
@validator('preset')
|
| 134 |
def preset_must_match_encoder(cls, v, values):
|
| 135 |
encoder = values.get('encoder')
|
| 136 |
if encoder == Encoder.av1 and not isinstance(v, int):
|
| 137 |
-
raise ValueError('
|
| 138 |
if encoder == Encoder.x265 and not isinstance(v, X265Preset):
|
| 139 |
-
raise ValueError("
|
| 140 |
if isinstance(v, int) and (v < 0 or v > 12):
|
| 141 |
-
raise ValueError('
|
| 142 |
return v
|
| 143 |
|
| 144 |
class TaskStatusResponse(BaseModel):
|
| 145 |
task_id: str
|
| 146 |
task_type: str
|
| 147 |
status: str
|
|
|
|
| 148 |
command: str
|
| 149 |
created_at: datetime
|
| 150 |
-
started_at:
|
| 151 |
-
finished_at:
|
| 152 |
-
|
| 153 |
log_path: str
|
| 154 |
-
last_log_line:
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
min_vmaf: Optional[int] = None
|
| 163 |
-
# Found values
|
| 164 |
-
found_crf: Optional[float] = None
|
| 165 |
-
found_vmaf: Optional[float] = None
|
| 166 |
|
| 167 |
class TaskCreateResponse(BaseModel):
|
| 168 |
message: str
|
| 169 |
task_id: str
|
| 170 |
status_url: str
|
| 171 |
log_url: str
|
| 172 |
-
download_url: str
|
| 173 |
manage_url: str
|
| 174 |
|
| 175 |
-
# ---
|
| 176 |
async def cleanup_files():
|
| 177 |
-
"""
|
| 178 |
-
print(f"[{datetime.utcnow()}]
|
| 179 |
|
| 180 |
now = datetime.utcnow()
|
| 181 |
max_age_limit = now - timedelta(hours=MAX_AGE_HOURS)
|
|
@@ -188,8 +172,10 @@ async def cleanup_files():
|
|
| 188 |
active_files = set()
|
| 189 |
for task_id, task in TASKS_DB.items():
|
| 190 |
if task['status'] in ['pending', 'running']:
|
| 191 |
-
|
| 192 |
-
active_files.add(Path(
|
|
|
|
|
|
|
| 193 |
if task.get('log_path'):
|
| 194 |
active_files.add(Path(task['log_path']))
|
| 195 |
|
|
@@ -225,82 +211,60 @@ async def cleanup_files():
|
|
| 225 |
deleted_count = 0
|
| 226 |
for file_path in files_to_delete:
|
| 227 |
try:
|
| 228 |
-
file_hash_to_del = file_path.stem.split('_')[0]
|
| 229 |
file_path.unlink()
|
| 230 |
deleted_count += 1
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
except Exception as e:
|
| 238 |
-
print(f"
|
| 239 |
|
| 240 |
if deleted_count > 0:
|
| 241 |
-
print(f"
|
| 242 |
|
| 243 |
-
# --- FastAPI
|
| 244 |
app = FastAPI(
|
| 245 |
title="ab-av1 API Server",
|
| 246 |
-
description="REST API
|
| 247 |
-
version="
|
| 248 |
)
|
| 249 |
|
| 250 |
@app.on_event("startup")
|
| 251 |
async def on_startup():
|
| 252 |
-
"""
|
| 253 |
if not API_KEY:
|
| 254 |
-
print("
|
| 255 |
else:
|
| 256 |
-
print("API
|
| 257 |
|
| 258 |
for dir_path in [UPLOADS_DIR, OUTPUTS_DIR, LOGS_DIR]:
|
| 259 |
dir_path.mkdir(parents=True, exist_ok=True)
|
| 260 |
|
| 261 |
scheduler.add_job(cleanup_files, 'interval', minutes=CLEANUP_INTERVAL_MINUTES)
|
| 262 |
scheduler.start()
|
| 263 |
-
print("
|
| 264 |
|
| 265 |
@app.on_event("shutdown")
|
| 266 |
async def on_shutdown():
|
| 267 |
-
"""
|
| 268 |
scheduler.shutdown()
|
| 269 |
-
print("
|
| 270 |
for task_id, task in TASKS_DB.items():
|
| 271 |
if task.get('process') and task['status'] == 'running':
|
| 272 |
-
print(f"
|
| 273 |
task['process'].terminate()
|
| 274 |
try:
|
| 275 |
task['process'].wait(timeout=5)
|
| 276 |
except subprocess.TimeoutExpired:
|
| 277 |
task['process'].kill()
|
| 278 |
|
| 279 |
-
# ---
|
| 280 |
-
|
| 281 |
-
def _update_task_timing(task: Dict[str, Any]):
|
| 282 |
-
"""Calculates and updates task duration."""
|
| 283 |
-
if task.get("started_at") and task.get("finished_at"):
|
| 284 |
-
duration = task["finished_at"] - task["started_at"]
|
| 285 |
-
task["duration_seconds"] = round(duration.total_seconds(), 2)
|
| 286 |
-
|
| 287 |
-
def _add_file_to_db(file_path: Path, origin: FileOrigin, original_filename: str, task_id: Optional[str] = None) -> FileInfo:
|
| 288 |
-
"""Adds a file to the global file DB and returns its info."""
|
| 289 |
-
file_hash = hashlib.sha256(file_path.read_bytes()).hexdigest()
|
| 290 |
-
file_info = FileInfo(
|
| 291 |
-
file_hash=file_hash,
|
| 292 |
-
file_path=str(file_path),
|
| 293 |
-
original_filename=original_filename,
|
| 294 |
-
origin=origin,
|
| 295 |
-
task_id=task_id,
|
| 296 |
-
size=file_path.stat().st_size,
|
| 297 |
-
created_at=datetime.utcnow()
|
| 298 |
-
)
|
| 299 |
-
FILES_DB[file_hash] = file_info.model_dump()
|
| 300 |
-
return file_info
|
| 301 |
|
| 302 |
def run_simple_task(task_id: str, command: list[str], log_path: Path):
|
| 303 |
-
"""
|
| 304 |
task = TASKS_DB[task_id]
|
| 305 |
task["started_at"] = datetime.utcnow()
|
| 306 |
task["status"] = "running"
|
|
@@ -321,30 +285,30 @@ def run_simple_task(task_id: str, command: list[str], log_path: Path):
|
|
| 321 |
process.wait()
|
| 322 |
|
| 323 |
if task.get('was_cancelled'):
|
| 324 |
-
|
| 325 |
elif process.returncode == 0:
|
| 326 |
task["status"] = "completed"
|
| 327 |
else:
|
| 328 |
task["status"] = "failed"
|
| 329 |
-
task["last_log_line"] = f"
|
| 330 |
|
| 331 |
except Exception as e:
|
| 332 |
task["status"] = "failed"
|
| 333 |
-
error_message = f"
|
| 334 |
task["last_log_line"] = error_message
|
| 335 |
with open(log_path, "a", encoding='utf-8') as log_file:
|
| 336 |
log_file.write(f"\n--- PYTHON EXCEPTION ---\n{error_message}")
|
| 337 |
finally:
|
| 338 |
task["finished_at"] = datetime.utcnow()
|
| 339 |
-
_update_task_timing(task)
|
| 340 |
task.pop('process', None)
|
| 341 |
|
| 342 |
def run_auto_encode_workflow(task_id: str, request: AutoEncodeRequest):
|
| 343 |
-
"""
|
|
|
|
|
|
|
| 344 |
task = TASKS_DB[task_id]
|
| 345 |
log_path = Path(task["log_path"])
|
| 346 |
-
|
| 347 |
-
original_input_path = Path(input_file_info.file_path)
|
| 348 |
temp_cfr_path = None
|
| 349 |
|
| 350 |
def _log_message(message: str, level: str = "INFO"):
|
|
@@ -355,8 +319,8 @@ def run_auto_encode_workflow(task_id: str, request: AutoEncodeRequest):
|
|
| 355 |
task["last_log_line"] = message
|
| 356 |
|
| 357 |
def _run_sub_task(command: list[str], sub_task_name: str) -> str:
|
| 358 |
-
_log_message(f"
|
| 359 |
-
_log_message(f"
|
| 360 |
|
| 361 |
process = subprocess.Popen(
|
| 362 |
command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|
@@ -377,7 +341,7 @@ def run_auto_encode_workflow(task_id: str, request: AutoEncodeRequest):
|
|
| 377 |
task.pop('process', None)
|
| 378 |
|
| 379 |
if process.returncode != 0:
|
| 380 |
-
raise RuntimeError(f"
|
| 381 |
|
| 382 |
return "".join(full_output)
|
| 383 |
|
|
@@ -388,9 +352,9 @@ def run_auto_encode_workflow(task_id: str, request: AutoEncodeRequest):
|
|
| 388 |
f.write(f"Starting auto-encode workflow for task {task_id}\n")
|
| 389 |
|
| 390 |
if not shutil.which("mediainfo"):
|
| 391 |
-
raise RuntimeError("
|
| 392 |
|
| 393 |
-
_log_message("
|
| 394 |
vfr_check_cmd = ["mediainfo", "--Inform=Video;%FrameRate_Mode%", str(original_input_path)]
|
| 395 |
vfr_check = subprocess.run(vfr_check_cmd, capture_output=True, text=True, check=True)
|
| 396 |
frame_rate_mode = vfr_check.stdout.strip()
|
|
@@ -398,33 +362,30 @@ def run_auto_encode_workflow(task_id: str, request: AutoEncodeRequest):
|
|
| 398 |
crf_search_input = original_input_path
|
| 399 |
|
| 400 |
if "VFR" in frame_rate_mode.upper():
|
| 401 |
-
_log_message(f"VFR
|
| 402 |
|
| 403 |
max_fps_res_cmd = ["mediainfo", "--Inform=Video;%FrameRate_Maximum%", str(original_input_path)]
|
| 404 |
max_fps_res = subprocess.run(max_fps_res_cmd, capture_output=True, text=True, check=True)
|
| 405 |
target_fps = math.ceil(float(max_fps_res.stdout.strip()))
|
| 406 |
-
_log_message(f"
|
| 407 |
|
| 408 |
temp_cfr_path = OUTPUTS_DIR / f"{task_id}_temp_cfr.mp4"
|
| 409 |
|
|
|
|
| 410 |
cfr_cmd = [
|
| 411 |
"ffmpeg", "-i", str(original_input_path), "-vf", f"fps={target_fps}",
|
| 412 |
-
"-c:v", "libsvtav1",
|
| 413 |
-
"-
|
|
|
|
| 414 |
"-c:a", "copy", "-c:s", "copy", str(temp_cfr_path)
|
| 415 |
]
|
| 416 |
-
_run_sub_task(cfr_cmd, "VFR
|
| 417 |
-
|
| 418 |
-
# Add intermediate file to DB and task
|
| 419 |
-
file_info = _add_file_to_db(temp_cfr_path, FileOrigin.VFR_TO_CFR, f"{original_input_path.stem}_cfr.mp4", task_id)
|
| 420 |
-
task['associated_files'].append(file_info.model_dump())
|
| 421 |
-
|
| 422 |
crf_search_input = temp_cfr_path
|
| 423 |
-
_log_message("
|
| 424 |
else:
|
| 425 |
-
_log_message(f"CFR
|
| 426 |
|
| 427 |
-
_log_message("
|
| 428 |
|
| 429 |
preset_value = request.preset.value if isinstance(request.preset, Enum) else str(request.preset)
|
| 430 |
|
|
@@ -436,59 +397,54 @@ def run_auto_encode_workflow(task_id: str, request: AutoEncodeRequest):
|
|
| 436 |
if request.extra_options:
|
| 437 |
search_cmd.extend(request.extra_options.split())
|
| 438 |
|
| 439 |
-
search_output = _run_sub_task(search_cmd, "
|
| 440 |
|
| 441 |
-
|
| 442 |
-
match = re.search(r"
|
| 443 |
-
if not match: # Fallback for older formats
|
| 444 |
-
match = re.search(r"encode .*--crf ([\d.]+) .*--vmaf ([\d.]+)", search_output)
|
| 445 |
-
|
| 446 |
if match:
|
| 447 |
-
|
| 448 |
-
task["found_vmaf"] = float(match.group(3) if len(match.groups()) > 2 else re.search(r'VMAF score: ([\d.]+)', search_output).group(1))
|
| 449 |
-
_log_message(f"Found optimal CRF: {task['found_crf']} for VMAF {task['found_vmaf']}")
|
| 450 |
else:
|
| 451 |
-
|
|
|
|
|
|
|
| 452 |
|
| 453 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 454 |
final_output_path = Path(task["output_path"])
|
| 455 |
encode_cmd = [
|
| 456 |
"ab-av1", "encode", "-i", str(crf_search_input), "-o", str(final_output_path),
|
| 457 |
-
"--crf",
|
| 458 |
]
|
| 459 |
if request.extra_options:
|
| 460 |
encode_cmd.extend(request.extra_options.split())
|
| 461 |
|
| 462 |
-
_run_sub_task(encode_cmd, "
|
| 463 |
-
|
| 464 |
-
# Add final output file to DB and task
|
| 465 |
-
file_info = _add_file_to_db(final_output_path, FileOrigin.ENCODE_OUTPUT, f"{original_input_path.stem}_encoded.mp4", task_id)
|
| 466 |
-
task['associated_files'].append(file_info.model_dump())
|
| 467 |
|
| 468 |
task["status"] = "completed"
|
| 469 |
-
_log_message("
|
| 470 |
|
| 471 |
except Exception as e:
|
| 472 |
task["status"] = "failed"
|
| 473 |
-
error_message = f"
|
| 474 |
_log_message(error_message, level="ERROR")
|
| 475 |
finally:
|
| 476 |
if temp_cfr_path and temp_cfr_path.exists():
|
| 477 |
-
_log_message("
|
| 478 |
temp_cfr_path.unlink()
|
| 479 |
task["finished_at"] = datetime.utcnow()
|
| 480 |
-
_update_task_timing(task)
|
| 481 |
task.pop('process', None)
|
| 482 |
|
| 483 |
def create_task(task_type: str, request: BaseTaskRequest, background_tasks: BackgroundTasks):
|
| 484 |
-
"""
|
| 485 |
if request.input_hash not in FILES_DB:
|
| 486 |
-
raise HTTPException(status_code=404, detail=f"
|
| 487 |
|
| 488 |
-
|
| 489 |
-
input_path = Path(input_file_info.file_path)
|
| 490 |
if not input_path.exists():
|
| 491 |
-
raise HTTPException(status_code=404, detail=f"
|
| 492 |
|
| 493 |
task_id = str(uuid.uuid4())
|
| 494 |
output_path = OUTPUTS_DIR / f"{task_id}.mp4"
|
|
@@ -518,21 +474,20 @@ def create_task(task_type: str, request: BaseTaskRequest, background_tasks: Back
|
|
| 518 |
"task_id": task_id,
|
| 519 |
"task_type": task_type,
|
| 520 |
"status": "pending",
|
|
|
|
| 521 |
"command": command_str,
|
| 522 |
"created_at": datetime.utcnow(),
|
| 523 |
"started_at": None,
|
| 524 |
"finished_at": None,
|
| 525 |
"output_path": str(output_path) if task_type != "crf-search" else None,
|
| 526 |
"log_path": str(log_path),
|
| 527 |
-
"last_log_line": "
|
| 528 |
-
|
| 529 |
"encoder": request.encoder,
|
| 530 |
"preset": request.preset,
|
| 531 |
"extra_options": request.extra_options,
|
| 532 |
"crf": getattr(request, 'crf', None),
|
| 533 |
"min_vmaf": getattr(request, 'min_vmaf', None),
|
| 534 |
-
"found_crf": None,
|
| 535 |
-
"found_vmaf": None,
|
| 536 |
}
|
| 537 |
|
| 538 |
if task_type == "auto-encode":
|
|
@@ -541,29 +496,28 @@ def create_task(task_type: str, request: BaseTaskRequest, background_tasks: Back
|
|
| 541 |
background_tasks.add_task(run_simple_task, task_id, command, log_path)
|
| 542 |
|
| 543 |
return TaskCreateResponse(
|
| 544 |
-
message=f"
|
| 545 |
-
task_id=task_id,
|
| 546 |
-
|
| 547 |
-
log_url=f"/tasks/{task_id}/log",
|
| 548 |
-
download_url=f"/download/{task_id}", # Main download endpoint
|
| 549 |
-
manage_url="/manage"
|
| 550 |
)
|
| 551 |
|
| 552 |
-
# --- API
|
| 553 |
|
| 554 |
-
@app.get("/", summary="
|
| 555 |
def read_root():
|
| 556 |
return {"status": "ok"}
|
| 557 |
|
| 558 |
-
@app.post("/upload", response_model=UploadResponse, summary="
|
| 559 |
async def upload_file(file: UploadFile = File(...)):
|
| 560 |
contents = await file.read()
|
| 561 |
file_hash = hashlib.sha256(contents).hexdigest()
|
| 562 |
|
| 563 |
-
if file_hash in FILES_DB and Path(FILES_DB[file_hash]["
|
| 564 |
return UploadResponse(
|
| 565 |
-
message="
|
| 566 |
-
|
|
|
|
|
|
|
| 567 |
is_new=False
|
| 568 |
)
|
| 569 |
|
|
@@ -573,61 +527,74 @@ async def upload_file(file: UploadFile = File(...)):
|
|
| 573 |
with open(saved_path, "wb") as f:
|
| 574 |
f.write(contents)
|
| 575 |
|
| 576 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 577 |
|
| 578 |
return UploadResponse(
|
| 579 |
-
message="
|
| 580 |
-
|
|
|
|
|
|
|
| 581 |
is_new=True
|
| 582 |
)
|
| 583 |
|
| 584 |
-
@app.post("/tasks/auto-encode", response_model=TaskCreateResponse, status_code=202, summary="
|
| 585 |
async def task_auto_encode(request: AutoEncodeRequest, background_tasks: BackgroundTasks):
|
| 586 |
return create_task("auto-encode", request, background_tasks)
|
| 587 |
|
| 588 |
-
@app.post("/tasks/crf-search", response_model=TaskCreateResponse, status_code=202, summary="
|
| 589 |
async def task_crf_search(request: CrfSearchRequest, background_tasks: BackgroundTasks):
|
| 590 |
return create_task("crf-search", request, background_tasks)
|
| 591 |
|
| 592 |
-
@app.post("/tasks/encode", response_model=TaskCreateResponse, status_code=202, summary="
|
| 593 |
async def task_encode(request: EncodeRequest, background_tasks: BackgroundTasks):
|
| 594 |
return create_task("encode", request, background_tasks)
|
| 595 |
|
| 596 |
-
@app.get("/tasks/{task_id}/status", response_model=TaskStatusResponse, summary="
|
| 597 |
-
def get_task_status(task_id: str = FastApiPath(..., description="
|
| 598 |
if task_id not in TASKS_DB:
|
| 599 |
-
raise HTTPException(status_code=404, detail="
|
| 600 |
task_info = TASKS_DB[task_id].copy()
|
| 601 |
task_info.pop('process', None)
|
| 602 |
return task_info
|
| 603 |
|
| 604 |
-
@app.get("/tasks/{task_id}/log", summary="
|
| 605 |
-
def get_task_log(task_id: str = FastApiPath(..., description="
|
| 606 |
if task_id not in TASKS_DB:
|
| 607 |
-
raise HTTPException(status_code=404, detail="
|
| 608 |
log_path = Path(TASKS_DB[task_id]["log_path"])
|
| 609 |
if not log_path.exists():
|
| 610 |
-
return JSONResponse(status_code=404, content={"detail": "
|
| 611 |
return FileResponse(log_path, media_type="text/plain", filename=log_path.name)
|
| 612 |
|
| 613 |
-
@app.get("/download/{
|
| 614 |
-
def download_result(
|
| 615 |
-
if
|
| 616 |
-
raise HTTPException(status_code=404, detail="
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
if not
|
| 622 |
-
raise HTTPException(status_code=404, detail="
|
| 623 |
|
| 624 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 625 |
|
| 626 |
-
# ---
|
| 627 |
|
| 628 |
-
@app.get("/manage", summary="
|
| 629 |
async def get_management_page():
|
| 630 |
-
"""
|
|
|
|
|
|
|
| 631 |
tasks_list = []
|
| 632 |
for task_id, task_data in TASKS_DB.items():
|
| 633 |
task_info = task_data.copy()
|
|
@@ -635,68 +602,56 @@ async def get_management_page():
|
|
| 635 |
tasks_list.append(task_info)
|
| 636 |
|
| 637 |
sorted_tasks = sorted(tasks_list, key=lambda x: x['created_at'], reverse=True)
|
| 638 |
-
sorted_files = sorted(list(FILES_DB.values()), key=lambda x: x['created_at'], reverse=True)
|
| 639 |
|
| 640 |
-
return {"tasks": sorted_tasks, "files":
|
| 641 |
|
| 642 |
|
| 643 |
-
@app.post("/manage/task/{task_id}/cancel", summary="
|
| 644 |
-
def cancel_task(task_id: str = FastApiPath(..., description="ID
|
| 645 |
if task_id not in TASKS_DB:
|
| 646 |
-
raise HTTPException(status_code=404, detail="
|
| 647 |
task = TASKS_DB[task_id]
|
| 648 |
if task['status'] != 'running' or 'process' not in task:
|
| 649 |
-
raise HTTPException(status_code=400, detail="
|
| 650 |
|
| 651 |
-
print(f"
|
| 652 |
task['was_cancelled'] = True
|
| 653 |
os.killpg(os.getpgid(task['process'].pid), signal.SIGTERM)
|
| 654 |
|
| 655 |
-
return {"message": f"
|
| 656 |
|
| 657 |
-
@app.delete("/manage/task/{task_id}", summary="
|
| 658 |
-
def delete_task(task_id: str = FastApiPath(..., description="ID
|
| 659 |
if task_id not in TASKS_DB:
|
| 660 |
-
raise HTTPException(status_code=404, detail="
|
| 661 |
|
| 662 |
task = TASKS_DB[task_id]
|
| 663 |
if task['status'] == 'running':
|
| 664 |
-
raise HTTPException(status_code=400, detail="
|
| 665 |
-
|
| 666 |
-
# Delete associated files (outputs and intermediates)
|
| 667 |
-
for file_info_dict in task.get('associated_files', []):
|
| 668 |
-
file_info = FileInfo(**file_info_dict)
|
| 669 |
-
if file_info.origin != FileOrigin.UPLOAD:
|
| 670 |
-
Path(file_info.file_path).unlink(missing_ok=True)
|
| 671 |
-
FILES_DB.pop(file_info.file_hash, None)
|
| 672 |
|
| 673 |
-
|
|
|
|
| 674 |
if task.get('log_path') and Path(task['log_path']).exists():
|
| 675 |
Path(task['log_path']).unlink(missing_ok=True)
|
| 676 |
|
| 677 |
del TASKS_DB[task_id]
|
| 678 |
|
| 679 |
-
return {"message": f"
|
| 680 |
|
| 681 |
-
@app.delete("/manage/file/{file_hash}", summary="
|
| 682 |
-
def delete_file(file_hash: str = FastApiPath(..., description="
|
| 683 |
if file_hash not in FILES_DB:
|
| 684 |
-
raise HTTPException(status_code=404, detail="
|
| 685 |
|
| 686 |
for task in TASKS_DB.values():
|
| 687 |
-
|
| 688 |
-
|
| 689 |
-
raise HTTPException(status_code=400, detail=f"Cannot delete file, it is being used by active task {task['task_id']}.")
|
| 690 |
|
| 691 |
-
|
| 692 |
-
file_path = Path(file_info['file_path'])
|
| 693 |
if file_path.exists():
|
| 694 |
file_path.unlink()
|
| 695 |
|
| 696 |
del FILES_DB[file_hash]
|
| 697 |
|
| 698 |
-
|
| 699 |
-
for task in TASKS_DB.values():
|
| 700 |
-
task['associated_files'] = [f for f in task.get('associated_files', []) if f['file_hash'] != file_hash]
|
| 701 |
|
| 702 |
-
return {"message": f"File {file_hash} has been deleted."}
|
|
|
|
| 1 |
+
|
| 2 |
import os
|
| 3 |
import uuid
|
| 4 |
import hashlib
|
|
|
|
| 9 |
import math
|
| 10 |
from datetime import datetime, timedelta
|
| 11 |
from pathlib import Path
|
| 12 |
+
from typing import Dict, Any, List, Union
|
| 13 |
from enum import Enum
|
| 14 |
|
| 15 |
from fastapi import FastAPI, File, UploadFile, BackgroundTasks, HTTPException, Path as FastApiPath, Depends, Query, status
|
|
|
|
| 17 |
from pydantic import BaseModel, Field, validator
|
| 18 |
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
| 19 |
|
| 20 |
+
# --- Конфигурация и глобальные переменные ---
|
| 21 |
|
| 22 |
+
# Ключ API для доступа
|
| 23 |
API_KEY = os.getenv("API_KEY")
|
| 24 |
|
| 25 |
+
# --- ИЗМЕНЕНИЕ: Параметры для VFR -> CFR вынесены сюда ---
|
| 26 |
+
# Настройки для конвертации видео с переменной частотой кадров (VFR)
|
| 27 |
+
# в видео с постоянной частотой кадров (CFR).
|
| 28 |
+
# Этот шаг выполняется для стабилизации видео перед основным кодированием.
|
| 29 |
+
VFR_TO_CFR_CRF = 17 # Качество (чем ниже, тем лучше). 17 - очень высокое.
|
| 30 |
+
VFR_TO_CFR_PRESET = 4 # Скорость (чем ниже, тем качественнее, но медленнее).
|
| 31 |
|
| 32 |
+
# Директории для хранения данных
|
| 33 |
BASE_DATA_DIR = Path("data")
|
| 34 |
UPLOADS_DIR = BASE_DATA_DIR / "uploads"
|
| 35 |
OUTPUTS_DIR = BASE_DATA_DIR / "outputs"
|
| 36 |
LOGS_DIR = BASE_DATA_DIR / "logs"
|
| 37 |
|
| 38 |
+
# "Базы данных" в памяти
|
| 39 |
FILES_DB: Dict[str, Dict[str, Any]] = {}
|
| 40 |
TASKS_DB: Dict[str, Dict[str, Any]] = {}
|
| 41 |
|
| 42 |
+
# Настройки для очистки
|
| 43 |
MAX_AGE_HOURS = 24
|
| 44 |
MAX_TOTAL_SIZE_GB = 50
|
| 45 |
CLEANUP_INTERVAL_MINUTES = 30
|
| 46 |
|
| 47 |
+
# Планировщик для задачи очистки
|
| 48 |
scheduler = AsyncIOScheduler()
|
| 49 |
|
| 50 |
+
# --- Зависимость для проверки токена ---
|
| 51 |
|
| 52 |
+
async def verify_token(token: str = Query(..., description="Токен для доступа к API.")):
|
| 53 |
+
"""Проверяет, соответствует ли предоставленный токен ключу API сервера."""
|
| 54 |
if not API_KEY:
|
| 55 |
+
print("КРИТИЧЕСКАЯ ОШИБКА: Переменная окружения API_KEY не установлена!")
|
| 56 |
raise HTTPException(
|
| 57 |
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
| 58 |
+
detail="Ключ API не сконфигурирован на сервере."
|
| 59 |
)
|
| 60 |
if token != API_KEY:
|
| 61 |
raise HTTPException(
|
| 62 |
status_code=status.HTTP_401_UNAUTHORIZED,
|
| 63 |
+
detail="Неверный или отсутствующий токен API."
|
| 64 |
)
|
| 65 |
|
| 66 |
+
# --- Перечисления и константы ---
|
| 67 |
|
| 68 |
class Encoder(str, Enum):
|
| 69 |
av1 = "svt-av1"
|
|
|
|
| 82 |
veryslow = "veryslow"
|
| 83 |
placebo = "placebo"
|
| 84 |
|
| 85 |
+
# --- Модели данных для API (Pydantic) ---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
|
| 87 |
+
class UploadResponse(BaseModel):
|
| 88 |
+
message: str
|
| 89 |
file_hash: str
|
| 90 |
file_path: str
|
| 91 |
original_filename: str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
is_new: bool
|
| 93 |
|
| 94 |
class BaseTaskRequest(BaseModel):
|
| 95 |
+
input_hash: str = Field(..., description="SHA256 хеш загруженного файла.")
|
| 96 |
+
encoder: Encoder = Field(Encoder.av1, description="Выбор кодека.")
|
| 97 |
+
extra_options: str = Field("", description="Дополнительные опции для ab-av1.")
|
| 98 |
|
| 99 |
class AutoEncodeRequest(BaseTaskRequest):
|
| 100 |
+
preset: Union[int, X265Preset] = Field(3, description="Пресет для кодировщика. Для AV1: 0-12 (int). Для x265: 'slow', 'medium' и т.д. (str).")
|
| 101 |
+
min_vmaf: int = Field(96, description="Целевой минимальный VMAF.", ge=0, le=100)
|
|
|
|
|
|
|
| 102 |
|
| 103 |
@validator('preset')
|
| 104 |
def preset_must_match_encoder(cls, v, values):
|
| 105 |
encoder = values.get('encoder')
|
| 106 |
if encoder == Encoder.av1 and not isinstance(v, int):
|
| 107 |
+
raise ValueError('Для AV1 пресет должен быть числом (0-12).')
|
| 108 |
if encoder == Encoder.x265 and not isinstance(v, X265Preset):
|
| 109 |
+
raise ValueError("Для x265 пресет должен быть строкой (например, 'slow').")
|
| 110 |
if isinstance(v, int) and (v < 0 or v > 12):
|
| 111 |
+
raise ValueError('Пресет для AV1 должен быть в диапазоне от 0 до 12.')
|
| 112 |
return v
|
| 113 |
|
| 114 |
class CrfSearchRequest(AutoEncodeRequest):
|
| 115 |
pass
|
| 116 |
|
| 117 |
class EncodeRequest(BaseTaskRequest):
|
| 118 |
+
preset: Union[int, X265Preset] = Field(3, description="Пресет для кодировщика. Для AV1: 0-12 (int). Для x265: 'slow', 'medium' и т.д. (str).")
|
| 119 |
+
crf: float = Field(20, description="Значение CRF (Constant Rate Factor). Допускаются дробные значения.")
|
| 120 |
|
| 121 |
@validator('preset')
|
| 122 |
def preset_must_match_encoder(cls, v, values):
|
| 123 |
encoder = values.get('encoder')
|
| 124 |
if encoder == Encoder.av1 and not isinstance(v, int):
|
| 125 |
+
raise ValueError('Для AV1 пресет должен быть числом (0-12).')
|
| 126 |
if encoder == Encoder.x265 and not isinstance(v, X265Preset):
|
| 127 |
+
raise ValueError("Для x265 пресет должен быть строкой (например, 'slow').")
|
| 128 |
if isinstance(v, int) and (v < 0 or v > 12):
|
| 129 |
+
raise ValueError('Пресет для AV1 должен быть в диапазоне от 0 до 12.')
|
| 130 |
return v
|
| 131 |
|
| 132 |
class TaskStatusResponse(BaseModel):
|
| 133 |
task_id: str
|
| 134 |
task_type: str
|
| 135 |
status: str
|
| 136 |
+
input_hash: str
|
| 137 |
command: str
|
| 138 |
created_at: datetime
|
| 139 |
+
started_at: datetime | None
|
| 140 |
+
finished_at: datetime | None
|
| 141 |
+
output_path: str | None
|
| 142 |
log_path: str
|
| 143 |
+
last_log_line: str | None
|
| 144 |
+
|
| 145 |
+
encoder: Encoder | None = None
|
| 146 |
+
preset: Union[int, str] | None = None
|
| 147 |
+
crf: float | None = None
|
| 148 |
+
min_vmaf: int | None = None
|
| 149 |
+
extra_options: str | None = None
|
| 150 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
|
| 152 |
class TaskCreateResponse(BaseModel):
|
| 153 |
message: str
|
| 154 |
task_id: str
|
| 155 |
status_url: str
|
| 156 |
log_url: str
|
|
|
|
| 157 |
manage_url: str
|
| 158 |
|
| 159 |
+
# --- Логика очистки (Janitor) ---
|
| 160 |
async def cleanup_files():
|
| 161 |
+
"""Периодическая задача для удаления старых файлов и контроля размера хранилища."""
|
| 162 |
+
print(f"[{datetime.utcnow()}] Запуск задачи очистки...")
|
| 163 |
|
| 164 |
now = datetime.utcnow()
|
| 165 |
max_age_limit = now - timedelta(hours=MAX_AGE_HOURS)
|
|
|
|
| 172 |
active_files = set()
|
| 173 |
for task_id, task in TASKS_DB.items():
|
| 174 |
if task['status'] in ['pending', 'running']:
|
| 175 |
+
if task['input_hash'] in FILES_DB:
|
| 176 |
+
active_files.add(Path(FILES_DB[task['input_hash']]['path']))
|
| 177 |
+
if task.get('output_path'):
|
| 178 |
+
active_files.add(Path(task['output_path']))
|
| 179 |
if task.get('log_path'):
|
| 180 |
active_files.add(Path(task['log_path']))
|
| 181 |
|
|
|
|
| 211 |
deleted_count = 0
|
| 212 |
for file_path in files_to_delete:
|
| 213 |
try:
|
|
|
|
| 214 |
file_path.unlink()
|
| 215 |
deleted_count += 1
|
| 216 |
+
if str(file_path).startswith(str(UPLOADS_DIR)):
|
| 217 |
+
file_hash_to_del = file_path.stem
|
| 218 |
+
FILES_DB.pop(file_hash_to_del, None)
|
| 219 |
+
elif str(file_path).startswith(str(OUTPUTS_DIR)) or str(file_path).startswith(str(LOGS_DIR)):
|
| 220 |
+
task_id_to_del = file_path.stem
|
| 221 |
+
TASKS_DB.pop(task_id_to_del, None)
|
| 222 |
except Exception as e:
|
| 223 |
+
print(f"Ошибка при удалении файла {file_path}: {e}")
|
| 224 |
|
| 225 |
if deleted_count > 0:
|
| 226 |
+
print(f"Очистка завершена. Удалено {deleted_count} файлов.")
|
| 227 |
|
| 228 |
+
# --- Инициализация FastAPI приложения ---
|
| 229 |
app = FastAPI(
|
| 230 |
title="ab-av1 API Server",
|
| 231 |
+
description="REST API для асинхронного кодирования видео. **Интерактивная документация: `/docs`**.",
|
| 232 |
+
version="2.8.0", # Версия обновлена
|
| 233 |
)
|
| 234 |
|
| 235 |
@app.on_event("startup")
|
| 236 |
async def on_startup():
|
| 237 |
+
"""Создаем директории, проверяем API_KEY и запускаем планировщик."""
|
| 238 |
if not API_KEY:
|
| 239 |
+
print("КРИТИЧЕСКАЯ ОШИБКА: Переменная окружения API_KEY не установлена. Сервер не сможет авторизовать запросы.")
|
| 240 |
else:
|
| 241 |
+
print("Ключ API успешно загружен.")
|
| 242 |
|
| 243 |
for dir_path in [UPLOADS_DIR, OUTPUTS_DIR, LOGS_DIR]:
|
| 244 |
dir_path.mkdir(parents=True, exist_ok=True)
|
| 245 |
|
| 246 |
scheduler.add_job(cleanup_files, 'interval', minutes=CLEANUP_INTERVAL_MINUTES)
|
| 247 |
scheduler.start()
|
| 248 |
+
print("Планировщик очистки файлов запущен.")
|
| 249 |
|
| 250 |
@app.on_event("shutdown")
|
| 251 |
async def on_shutdown():
|
| 252 |
+
"""Останавливаем планировщик и активные процессы при выключении."""
|
| 253 |
scheduler.shutdown()
|
| 254 |
+
print("Планировщик остановлен.")
|
| 255 |
for task_id, task in TASKS_DB.items():
|
| 256 |
if task.get('process') and task['status'] == 'running':
|
| 257 |
+
print(f"Остановка процесса для задачи {task_id}...")
|
| 258 |
task['process'].terminate()
|
| 259 |
try:
|
| 260 |
task['process'].wait(timeout=5)
|
| 261 |
except subprocess.TimeoutExpired:
|
| 262 |
task['process'].kill()
|
| 263 |
|
| 264 |
+
# --- Вспомогательные функции ---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 265 |
|
| 266 |
def run_simple_task(task_id: str, command: list[str], log_path: Path):
|
| 267 |
+
"""Выполняет простую одношаговую команду в фоне (для encode и crf-search)."""
|
| 268 |
task = TASKS_DB[task_id]
|
| 269 |
task["started_at"] = datetime.utcnow()
|
| 270 |
task["status"] = "running"
|
|
|
|
| 285 |
process.wait()
|
| 286 |
|
| 287 |
if task.get('was_cancelled'):
|
| 288 |
+
task["status"] = "cancelled"
|
| 289 |
elif process.returncode == 0:
|
| 290 |
task["status"] = "completed"
|
| 291 |
else:
|
| 292 |
task["status"] = "failed"
|
| 293 |
+
task["last_log_line"] = f"Процесс завершился с кодом {process.returncode}"
|
| 294 |
|
| 295 |
except Exception as e:
|
| 296 |
task["status"] = "failed"
|
| 297 |
+
error_message = f"Произошло исключение Python: {e}"
|
| 298 |
task["last_log_line"] = error_message
|
| 299 |
with open(log_path, "a", encoding='utf-8') as log_file:
|
| 300 |
log_file.write(f"\n--- PYTHON EXCEPTION ---\n{error_message}")
|
| 301 |
finally:
|
| 302 |
task["finished_at"] = datetime.utcnow()
|
|
|
|
| 303 |
task.pop('process', None)
|
| 304 |
|
| 305 |
def run_auto_encode_workflow(task_id: str, request: AutoEncodeRequest):
|
| 306 |
+
"""
|
| 307 |
+
Выполняет сложный воркфлоу для auto-encode с проверкой VFR.
|
| 308 |
+
"""
|
| 309 |
task = TASKS_DB[task_id]
|
| 310 |
log_path = Path(task["log_path"])
|
| 311 |
+
original_input_path = Path(FILES_DB[request.input_hash]["path"])
|
|
|
|
| 312 |
temp_cfr_path = None
|
| 313 |
|
| 314 |
def _log_message(message: str, level: str = "INFO"):
|
|
|
|
| 319 |
task["last_log_line"] = message
|
| 320 |
|
| 321 |
def _run_sub_task(command: list[str], sub_task_name: str) -> str:
|
| 322 |
+
_log_message(f"Запуск подзадачи: {sub_task_name}")
|
| 323 |
+
_log_message(f"Команда: {' '.join(command)}")
|
| 324 |
|
| 325 |
process = subprocess.Popen(
|
| 326 |
command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|
|
|
| 341 |
task.pop('process', None)
|
| 342 |
|
| 343 |
if process.returncode != 0:
|
| 344 |
+
raise RuntimeError(f"Подзадача '{sub_task_name}' завершилась с ошибкой (код: {process.returncode}). Смотрите лог для деталей.")
|
| 345 |
|
| 346 |
return "".join(full_output)
|
| 347 |
|
|
|
|
| 352 |
f.write(f"Starting auto-encode workflow for task {task_id}\n")
|
| 353 |
|
| 354 |
if not shutil.which("mediainfo"):
|
| 355 |
+
raise RuntimeError("Утилита 'mediainfo' не найдена в системном PATH. Пожалуйста, установите ее.")
|
| 356 |
|
| 357 |
+
_log_message("Шаг 1/4: Проверка на Variable Frame Rate (VFR)")
|
| 358 |
vfr_check_cmd = ["mediainfo", "--Inform=Video;%FrameRate_Mode%", str(original_input_path)]
|
| 359 |
vfr_check = subprocess.run(vfr_check_cmd, capture_output=True, text=True, check=True)
|
| 360 |
frame_rate_mode = vfr_check.stdout.strip()
|
|
|
|
| 362 |
crf_search_input = original_input_path
|
| 363 |
|
| 364 |
if "VFR" in frame_rate_mode.upper():
|
| 365 |
+
_log_message(f"Обнаружен VFR ({frame_rate_mode}). Начало конвертации в CFR.")
|
| 366 |
|
| 367 |
max_fps_res_cmd = ["mediainfo", "--Inform=Video;%FrameRate_Maximum%", str(original_input_path)]
|
| 368 |
max_fps_res = subprocess.run(max_fps_res_cmd, capture_output=True, text=True, check=True)
|
| 369 |
target_fps = math.ceil(float(max_fps_res.stdout.strip()))
|
| 370 |
+
_log_message(f"Целевой FPS для CFR: {target_fps}")
|
| 371 |
|
| 372 |
temp_cfr_path = OUTPUTS_DIR / f"{task_id}_temp_cfr.mp4"
|
| 373 |
|
| 374 |
+
# --- ИЗМЕНЕНИЕ: Используются переменные ��з конфигурации ---
|
| 375 |
cfr_cmd = [
|
| 376 |
"ffmpeg", "-i", str(original_input_path), "-vf", f"fps={target_fps}",
|
| 377 |
+
"-c:v", "libsvtav1",
|
| 378 |
+
"-crf", str(VFR_TO_CFR_CRF),
|
| 379 |
+
"-preset", str(VFR_TO_CFR_PRESET),
|
| 380 |
"-c:a", "copy", "-c:s", "copy", str(temp_cfr_path)
|
| 381 |
]
|
| 382 |
+
_run_sub_task(cfr_cmd, "VFR в CFR конвертация")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 383 |
crf_search_input = temp_cfr_path
|
| 384 |
+
_log_message("Конвертация в CFR успешно завершена.")
|
| 385 |
else:
|
| 386 |
+
_log_message(f"Обнаружен CFR ({frame_rate_mode}). Конвертация не требуется.")
|
| 387 |
|
| 388 |
+
_log_message("Шаг 2/4: Поиск оптимального CRF (crf-search)")
|
| 389 |
|
| 390 |
preset_value = request.preset.value if isinstance(request.preset, Enum) else str(request.preset)
|
| 391 |
|
|
|
|
| 397 |
if request.extra_options:
|
| 398 |
search_cmd.extend(request.extra_options.split())
|
| 399 |
|
| 400 |
+
search_output = _run_sub_task(search_cmd, "Поиск CRF")
|
| 401 |
|
| 402 |
+
found_crf = None
|
| 403 |
+
match = re.search(r"encode .*--crf ([\d.]+)", search_output)
|
|
|
|
|
|
|
|
|
|
| 404 |
if match:
|
| 405 |
+
found_crf = match.group(1)
|
|
|
|
|
|
|
| 406 |
else:
|
| 407 |
+
match = re.search(r"^crf ([\d.]+) VMAF", search_output, re.MULTILINE)
|
| 408 |
+
if match:
|
| 409 |
+
found_crf = match.group(1)
|
| 410 |
|
| 411 |
+
if not found_crf:
|
| 412 |
+
raise RuntimeError("Не удалось найти рекомендованный CRF в выводе crf-search.")
|
| 413 |
+
_log_message(f"Найден оптимальный CRF: {found_crf}")
|
| 414 |
+
|
| 415 |
+
_log_message("Шаг 3/4: Финальное кодирование")
|
| 416 |
final_output_path = Path(task["output_path"])
|
| 417 |
encode_cmd = [
|
| 418 |
"ab-av1", "encode", "-i", str(crf_search_input), "-o", str(final_output_path),
|
| 419 |
+
"--crf", found_crf, "-e", request.encoder.value, "--preset", preset_value
|
| 420 |
]
|
| 421 |
if request.extra_options:
|
| 422 |
encode_cmd.extend(request.extra_options.split())
|
| 423 |
|
| 424 |
+
_run_sub_task(encode_cmd, "Финальное кодирование")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 425 |
|
| 426 |
task["status"] = "completed"
|
| 427 |
+
_log_message("Шаг 4/4: Задача успешно завершена!")
|
| 428 |
|
| 429 |
except Exception as e:
|
| 430 |
task["status"] = "failed"
|
| 431 |
+
error_message = f"Воркфлоу завершился с ошибкой: {e}"
|
| 432 |
_log_message(error_message, level="ERROR")
|
| 433 |
finally:
|
| 434 |
if temp_cfr_path and temp_cfr_path.exists():
|
| 435 |
+
_log_message("Очистка временных файлов...")
|
| 436 |
temp_cfr_path.unlink()
|
| 437 |
task["finished_at"] = datetime.utcnow()
|
|
|
|
| 438 |
task.pop('process', None)
|
| 439 |
|
| 440 |
def create_task(task_type: str, request: BaseTaskRequest, background_tasks: BackgroundTasks):
|
| 441 |
+
"""Общая функция для создания и запуска любого типа задачи."""
|
| 442 |
if request.input_hash not in FILES_DB:
|
| 443 |
+
raise HTTPException(status_code=404, detail=f"Файл с хешем {request.input_hash} не найден.")
|
| 444 |
|
| 445 |
+
input_path = Path(FILES_DB[request.input_hash]["path"])
|
|
|
|
| 446 |
if not input_path.exists():
|
| 447 |
+
raise HTTPException(status_code=404, detail=f"Исходный файл для хеша {request.input_hash} был удален.")
|
| 448 |
|
| 449 |
task_id = str(uuid.uuid4())
|
| 450 |
output_path = OUTPUTS_DIR / f"{task_id}.mp4"
|
|
|
|
| 474 |
"task_id": task_id,
|
| 475 |
"task_type": task_type,
|
| 476 |
"status": "pending",
|
| 477 |
+
"input_hash": request.input_hash,
|
| 478 |
"command": command_str,
|
| 479 |
"created_at": datetime.utcnow(),
|
| 480 |
"started_at": None,
|
| 481 |
"finished_at": None,
|
| 482 |
"output_path": str(output_path) if task_type != "crf-search" else None,
|
| 483 |
"log_path": str(log_path),
|
| 484 |
+
"last_log_line": "Задача поставлена в очередь.",
|
| 485 |
+
|
| 486 |
"encoder": request.encoder,
|
| 487 |
"preset": request.preset,
|
| 488 |
"extra_options": request.extra_options,
|
| 489 |
"crf": getattr(request, 'crf', None),
|
| 490 |
"min_vmaf": getattr(request, 'min_vmaf', None),
|
|
|
|
|
|
|
| 491 |
}
|
| 492 |
|
| 493 |
if task_type == "auto-encode":
|
|
|
|
| 496 |
background_tasks.add_task(run_simple_task, task_id, command, log_path)
|
| 497 |
|
| 498 |
return TaskCreateResponse(
|
| 499 |
+
message=f"Задача '{task_type}' успешно создана.",
|
| 500 |
+
task_id=task_id, status_url=f"/tasks/{task_id}/status",
|
| 501 |
+
log_url=f"/tasks/{task_id}/log", manage_url="/manage"
|
|
|
|
|
|
|
|
|
|
| 502 |
)
|
| 503 |
|
| 504 |
+
# --- Эндпоинты API ---
|
| 505 |
|
| 506 |
+
@app.get("/", summary="Статус API")
|
| 507 |
def read_root():
|
| 508 |
return {"status": "ok"}
|
| 509 |
|
| 510 |
+
@app.post("/upload", response_model=UploadResponse, summary="Загрузка видеофайла", dependencies=[Depends(verify_token)])
|
| 511 |
async def upload_file(file: UploadFile = File(...)):
|
| 512 |
contents = await file.read()
|
| 513 |
file_hash = hashlib.sha256(contents).hexdigest()
|
| 514 |
|
| 515 |
+
if file_hash in FILES_DB and Path(FILES_DB[file_hash]["path"]).exists():
|
| 516 |
return UploadResponse(
|
| 517 |
+
message="Файл с таким хешем уже существует.",
|
| 518 |
+
file_hash=file_hash,
|
| 519 |
+
file_path=FILES_DB[file_hash]["path"],
|
| 520 |
+
original_filename=FILES_DB[file_hash]["original_filename"],
|
| 521 |
is_new=False
|
| 522 |
)
|
| 523 |
|
|
|
|
| 527 |
with open(saved_path, "wb") as f:
|
| 528 |
f.write(contents)
|
| 529 |
|
| 530 |
+
FILES_DB[file_hash] = {
|
| 531 |
+
"path": str(saved_path),
|
| 532 |
+
"original_filename": file.filename,
|
| 533 |
+
"uploaded_at": datetime.utcnow(),
|
| 534 |
+
"size": len(contents)
|
| 535 |
+
}
|
| 536 |
|
| 537 |
return UploadResponse(
|
| 538 |
+
message="Файл успешно загружен.",
|
| 539 |
+
file_hash=file_hash,
|
| 540 |
+
file_path=str(saved_path),
|
| 541 |
+
original_filename=file.filename,
|
| 542 |
is_new=True
|
| 543 |
)
|
| 544 |
|
| 545 |
+
@app.post("/tasks/auto-encode", response_model=TaskCreateResponse, status_code=202, summary="Запуск задачи auto-encode с логикой VFR", dependencies=[Depends(verify_token)])
|
| 546 |
async def task_auto_encode(request: AutoEncodeRequest, background_tasks: BackgroundTasks):
|
| 547 |
return create_task("auto-encode", request, background_tasks)
|
| 548 |
|
| 549 |
+
@app.post("/tasks/crf-search", response_model=TaskCreateResponse, status_code=202, summary="Запуск задачи crf-search", dependencies=[Depends(verify_token)])
|
| 550 |
async def task_crf_search(request: CrfSearchRequest, background_tasks: BackgroundTasks):
|
| 551 |
return create_task("crf-search", request, background_tasks)
|
| 552 |
|
| 553 |
+
@app.post("/tasks/encode", response_model=TaskCreateResponse, status_code=202, summary="Запуск задачи encode", dependencies=[Depends(verify_token)])
|
| 554 |
async def task_encode(request: EncodeRequest, background_tasks: BackgroundTasks):
|
| 555 |
return create_task("encode", request, background_tasks)
|
| 556 |
|
| 557 |
+
@app.get("/tasks/{task_id}/status", response_model=TaskStatusResponse, summary="Получить статус задачи", dependencies=[Depends(verify_token)])
|
| 558 |
+
def get_task_status(task_id: str = FastApiPath(..., description="ID задачи")):
|
| 559 |
if task_id not in TASKS_DB:
|
| 560 |
+
raise HTTPException(status_code=404, detail="Задача не найдена.")
|
| 561 |
task_info = TASKS_DB[task_id].copy()
|
| 562 |
task_info.pop('process', None)
|
| 563 |
return task_info
|
| 564 |
|
| 565 |
+
@app.get("/tasks/{task_id}/log", summary="Получить полный лог задачи", dependencies=[Depends(verify_token)])
|
| 566 |
+
def get_task_log(task_id: str = FastApiPath(..., description="ID задачи")):
|
| 567 |
if task_id not in TASKS_DB:
|
| 568 |
+
raise HTTPException(status_code=404, detail="Задача не найдена.")
|
| 569 |
log_path = Path(TASKS_DB[task_id]["log_path"])
|
| 570 |
if not log_path.exists():
|
| 571 |
+
return JSONResponse(status_code=404, content={"detail": "Файл лога не найден."})
|
| 572 |
return FileResponse(log_path, media_type="text/plain", filename=log_path.name)
|
| 573 |
|
| 574 |
+
@app.get("/download/{task_id}", summary="Скачать результат кодирования", dependencies=[Depends(verify_token)])
|
| 575 |
+
def download_result(task_id: str = FastApiPath(..., description="ID задачи")):
|
| 576 |
+
if task_id not in TASKS_DB:
|
| 577 |
+
raise HTTPException(status_code=404, detail="Задача не найдена.")
|
| 578 |
+
task = TASKS_DB[task_id]
|
| 579 |
+
if task["status"] != "completed":
|
| 580 |
+
raise HTTPException(status_code=400, detail=f"Задача еще не завершена. Статус: {task['status']}.")
|
| 581 |
+
output_path = task.get("output_path")
|
| 582 |
+
if not output_path or not Path(output_path).exists():
|
| 583 |
+
raise HTTPException(status_code=404, detail="Выходной файл не найден.")
|
| 584 |
|
| 585 |
+
output_path = Path(output_path)
|
| 586 |
+
input_hash = task["input_hash"]
|
| 587 |
+
original_filename = Path(FILES_DB[input_hash]["original_filename"]).stem
|
| 588 |
+
|
| 589 |
+
return FileResponse(output_path, filename=f"{original_filename}_{task['task_type']}_{task_id[:8]}{output_path.suffix}")
|
| 590 |
|
| 591 |
+
# --- Эндпоинты для управления ---
|
| 592 |
|
| 593 |
+
@app.get("/manage", summary="Получить списки задач и файлов (JSON)", dependencies=[Depends(verify_token)])
|
| 594 |
async def get_management_page():
|
| 595 |
+
"""
|
| 596 |
+
Возвращает JSON со списками всех задач и загруженных файлов.
|
| 597 |
+
"""
|
| 598 |
tasks_list = []
|
| 599 |
for task_id, task_data in TASKS_DB.items():
|
| 600 |
task_info = task_data.copy()
|
|
|
|
| 602 |
tasks_list.append(task_info)
|
| 603 |
|
| 604 |
sorted_tasks = sorted(tasks_list, key=lambda x: x['created_at'], reverse=True)
|
|
|
|
| 605 |
|
| 606 |
+
return {"tasks": sorted_tasks, "files": FILES_DB}
|
| 607 |
|
| 608 |
|
| 609 |
+
@app.post("/manage/task/{task_id}/cancel", summary="Прервать выполняющуюся задачу", dependencies=[Depends(verify_token)])
|
| 610 |
+
def cancel_task(task_id: str = FastApiPath(..., description="ID задачи для прерывания")):
|
| 611 |
if task_id not in TASKS_DB:
|
| 612 |
+
raise HTTPException(status_code=404, detail="Задача не найдена.")
|
| 613 |
task = TASKS_DB[task_id]
|
| 614 |
if task['status'] != 'running' or 'process' not in task:
|
| 615 |
+
raise HTTPException(status_code=400, detail="Задачу нельзя прервать (она не выполняется).")
|
| 616 |
|
| 617 |
+
print(f"Прерывание задачи {task_id} (PID: {task['process'].pid})...")
|
| 618 |
task['was_cancelled'] = True
|
| 619 |
os.killpg(os.getpgid(task['process'].pid), signal.SIGTERM)
|
| 620 |
|
| 621 |
+
return {"message": f"Команда на прерывание задачи {task_id} отправлена."}
|
| 622 |
|
| 623 |
+
@app.delete("/manage/task/{task_id}", summary="Удалить задачу и ее файлы", dependencies=[Depends(verify_token)])
|
| 624 |
+
def delete_task(task_id: str = FastApiPath(..., description="ID задачи для удаления")):
|
| 625 |
if task_id not in TASKS_DB:
|
| 626 |
+
raise HTTPException(status_code=404, detail="Задача не найдена.")
|
| 627 |
|
| 628 |
task = TASKS_DB[task_id]
|
| 629 |
if task['status'] == 'running':
|
| 630 |
+
raise HTTPException(status_code=400, detail="Сначала прервите выполняющуюся задачу.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 631 |
|
| 632 |
+
if task.get('output_path') and Path(task['output_path']).exists():
|
| 633 |
+
Path(task['output_path']).unlink(missing_ok=True)
|
| 634 |
if task.get('log_path') and Path(task['log_path']).exists():
|
| 635 |
Path(task['log_path']).unlink(missing_ok=True)
|
| 636 |
|
| 637 |
del TASKS_DB[task_id]
|
| 638 |
|
| 639 |
+
return {"message": f"Задача {task_id} и ее файлы были удалены."}
|
| 640 |
|
| 641 |
+
@app.delete("/manage/file/{file_hash}", summary="Удалить загруженный файл", dependencies=[Depends(verify_token)])
|
| 642 |
+
def delete_file(file_hash: str = FastApiPath(..., description="Хеш файла для удаления")):
|
| 643 |
if file_hash not in FILES_DB:
|
| 644 |
+
raise HTTPException(status_code=404, detail="Файл не найден.")
|
| 645 |
|
| 646 |
for task in TASKS_DB.values():
|
| 647 |
+
if task['input_hash'] == file_hash and task['status'] in ['running', 'pending']:
|
| 648 |
+
raise HTTPException(status_code=400, detail=f"Нельзя удалить файл, так как он используется активной задачей {task['task_id']}.")
|
|
|
|
| 649 |
|
| 650 |
+
file_path = Path(FILES_DB[file_hash]['path'])
|
|
|
|
| 651 |
if file_path.exists():
|
| 652 |
file_path.unlink()
|
| 653 |
|
| 654 |
del FILES_DB[file_hash]
|
| 655 |
|
| 656 |
+
return {"message": f"Файл {file_hash} был удален."}
|
|
|
|
|
|
|
| 657 |
|
|
|