Spaces:
Sleeping
Sleeping
File size: 69,859 Bytes
4b79268 b6f361a 4b79268 961990f 4b79268 b6f361a a528be9 4b79268 961990f b6f361a 4b79268 7c2d809 b6f361a 4b79268 b6f361a 7c2d809 3df5f8f 961990f 1361034 e646192 1361034 89cae6c 4b79268 edad228 4b79268 bbe4867 e646192 bbe4867 e646192 bbe4867 e646192 bbe4867 edad228 961990f b6f361a 2f32611 b6f361a 3964d02 1ca1018 fbb54cc 1ca1018 fbb54cc 1ca1018 fbb54cc 3964d02 fbb54cc 1ca1018 fbb54cc 3964d02 204c25c 3964d02 204c25c 3964d02 b6f361a 88d3d3b 06c75aa 7c2d809 a528be9 7c2d809 a528be9 88d3d3b 7c2d809 961990f 803dc05 961990f d2261c7 961990f 803dc05 d2261c7 803dc05 961990f a528be9 7c2d809 a528be9 7c2d809 06c75aa b6f361a c7cacd1 b6f361a 31ce41a 89cae6c 1ca1018 89cae6c 1ca1018 89cae6c 4b79268 4f70b55 4b79268 364bfae 597ad13 9a058d1 364bfae 3df5f8f 4b79268 364bfae 4b79268 597ad13 364bfae 9a058d1 364bfae 3df5f8f 364bfae 4b79268 364bfae 4b79268 364bfae 4b79268 961990f edad228 0487cd2 961990f 4b79268 edad228 0487cd2 edad228 597ad13 364bfae f2b292d 364bfae 4b79268 364bfae 597ad13 9a058d1 364bfae 3df5f8f 4b79268 961990f 71306bd 961990f 71306bd 961990f 71306bd 961990f 71306bd 961990f 803dc05 961990f 803dc05 961990f 4b79268 edad228 6dd9ac4 edad228 1361034 edad228 6dd9ac4 edad228 4b79268 961990f 4b79268 c7cacd1 4b79268 355b428 4b79268 4af7dd3 063a56d d06062f cf53b59 7e20c47 9a92330 6dd9ac4 3bed55b 961990f 4b79268 06c75aa 1361034 4b79268 908a4e1 b863381 908a4e1 b863381 908a4e1 edad228 961990f edad228 0487cd2 edad228 961990f 803dc05 961990f 1361034 0487cd2 1361034 0487cd2 1361034 d2261c7 e8ba7af 1361034 e8ba7af 1361034 961990f 4b79268 e7c2416 06c75aa 7c2d809 06c75aa 961990f 803dc05 961990f e7c2416 4b79268 e7c2416 355b428 e7c2416 d06062f cf53b59 edad228 9a92330 6dd9ac4 e7c2416 7c2d809 355b428 fbb54cc e7c2416 961990f 4b79268 88d3d3b d9cde51 88d3d3b d9cde51 88d3d3b d9cde51 88d3d3b d9cde51 e646192 d9cde51 88d3d3b d9cde51 e646192 d9cde51 88d3d3b e646192 88d3d3b e646192 bbe4867 e646192 88d3d3b d9cde51 e646192 88d3d3b 4b79268 9d2a95f 7c2d809 4b79268 9d2a95f 4b79268 9d2a95f 7c2d809 9d2a95f 7c2d809 4b79268 9d2a95f 4b79268 597ad13 4b79268 597ad13 4b79268 28435e9 4b79268 4f70b55 28435e9 e845d2d 28435e9 4b79268 597ad13 3df5f8f 364bfae f09a075 364bfae 597ad13 7e20c47 597ad13 9a058d1 7e20c47 9a058d1 355b428 9a058d1 7c2d809 9a058d1 3df5f8f f09a075 3df5f8f a037749 f09a075 a037749 3df5f8f a037749 f09a075 3df5f8f 4f70b55 3df5f8f 4b79268 b6f361a 81fca62 b6f361a fe75e0e c7cacd1 fe75e0e 355b428 b6f361a 2636772 4af7dd3 063a56d d06062f cf53b59 7e20c47 9a92330 6dd9ac4 3bed55b b6f361a 06c75aa 1361034 0658e45 b6f361a 4b79268 7c2d809 b6f361a 2636772 fe75e0e c7cacd1 fe75e0e 355b428 2636772 4af7dd3 ec46cbf d06062f cf53b59 7e20c47 9a92330 6dd9ac4 ec46cbf e7c2416 2636772 803dc05 2636772 4b79268 2636772 4b79268 364bfae 2636772 4b79268 4f70b55 4b79268 b6f361a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 | """FastAPI app: upload UI + SSE-driven pipeline orchestration."""
from __future__ import annotations
import asyncio
import io
import json
import logging
import os
import queue
import secrets
import shutil
import tempfile
import threading
import time
from pathlib import Path
from fastapi import FastAPI, File, Form, HTTPException, Request, UploadFile
from fastapi.responses import HTMLResponse, JSONResponse, Response, StreamingResponse
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from .jobs import Job, JobQuotaExceeded, JobResult, registry
from .pipeline import bundle_zip, render_from_metadata, render_guide, render_review
from .pipeline.filter import filter_instructor_frames
from .pipeline.llm_extract import (
LLMToggles,
LLMTopicError,
SlideAlignError,
align_slides_to_transcript,
extract_topics,
is_valid_video_type,
)
from .pipeline.metadata import CURRENT_SCHEMA_VERSION, PageMetadata, migrate_v1_to_v2
from .pipeline.orchestrator import (
PipelineError,
PipelineInputs,
ProgressEvent,
_apply_topic_segmentation,
run_pipeline,
)
from .pipeline.image_input import (
ImageInputError,
all_have_timestamps,
build_frames_even_split,
build_frames_from_images,
build_frames_from_starts,
build_frames_ordered_by_name,
order_by_name,
)
from .pipeline.scenes import detect_and_extract, video_duration
from .pipeline.transcript import parse_transcript
from .pipeline.types import SceneFrame
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
log = logging.getLogger("videoguidemaker")
BASE_DIR = Path(__file__).resolve().parent
templates = Jinja2Templates(directory=str(BASE_DIR / "templates"))
app = FastAPI(title="The Describatron")
app.mount("/static", StaticFiles(directory=str(BASE_DIR / "static")), name="static")
_SLACK_BYTES = 8 * 1024 * 1024 # multipart framing + the small form fields
def _request_body_ceiling(path: str) -> int | None:
"""Hard upper bound on an acceptable request body for `path`, or None to
skip the early check. Sized per-endpoint from that endpoint's own limits
so a legitimately large images-mode upload isn't wrongly rejected and
/edit isn't given the (much larger) video ceiling. Returns None when the
relevant cap is disabled (an explicit opt-out of size protection).
NB: this only fires when the request declares a Content-Length; a
chunked upload has none, so each upload endpoint additionally bounds
its own read/stream (_save_upload for files, the chunked read in /edit).
"""
if path == "/jobs/from-images":
if MAX_IMAGE_BYTES is None:
return None
return MAX_IMAGE_COUNT * MAX_IMAGE_BYTES + (MAX_TRANSCRIPT_BYTES or 0) + _SLACK_BYTES
if path == "/edit":
if MAX_EDIT_JSON_BYTES is None:
return None
return MAX_EDIT_JSON_BYTES + _SLACK_BYTES
if MAX_VIDEO_BYTES is None:
return None
return MAX_VIDEO_BYTES + (MAX_TRANSCRIPT_BYTES or 0) + _SLACK_BYTES
@app.middleware("http")
async def _security_headers(request: Request, call_next):
# Reject an over-cap upload from its declared Content-Length BEFORE
# Starlette spools the multipart body to a temp file. The per-field
# streaming check in _save_upload still runs (and catches an absent or
# lying Content-Length), but this stops the obvious "declare a 10 GB
# body" case from filling the Space's ephemeral disk first.
if request.method in ("POST", "PUT", "PATCH"):
ceiling = _request_body_ceiling(request.url.path)
declared = request.headers.get("content-length")
if ceiling is not None and declared is not None:
try:
size = int(declared)
except ValueError:
size = -1
if size > ceiling:
return JSONResponse(
{"detail": "Upload exceeds the maximum allowed size."},
status_code=413,
headers={"X-Content-Type-Options": "nosniff"},
)
response = await call_next(request)
# X-Content-Type-Options is safe everywhere β prevents MIME sniffing.
response.headers["X-Content-Type-Options"] = "nosniff"
# X-Frame-Options and CSP are skipped here: HF Spaces proxies the app
# through its own infrastructure and setting these causes "refused to
# connect" failures. Add them at the reverse-proxy / CDN layer instead.
return response
VIDEO_SUFFIXES = {".mp4", ".mov", ".mkv", ".webm", ".m4v", ".avi"}
TRANSCRIPT_SUFFIXES = {".vtt", ".srt"}
IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp"}
# Upper bound on how many slide images one images-mode job accepts. Each
# becomes a segment; a typical deck is well under this. Guards against an
# accidental thousand-file drop.
MAX_IMAGE_COUNT = 200
ALLOWED_FORMATS = {"review", "single", "zip"}
# Cap upload sizes so a no-auth deployment can't be OOM'd / disk-filled
# by an arbitrarily large body. 500 MB covers a ~30-min lecture at typical
# bitrates; transcripts are tiny so 5 MB is plenty.
# Override via env vars locally: MAX_VIDEO_MB=0 disables the cap entirely.
def _size_limit(env_var: str, default_mb: int) -> int | None:
mb = int(os.getenv(env_var, str(default_mb)))
return None if mb == 0 else mb * 1024 * 1024
MAX_VIDEO_BYTES = _size_limit("MAX_VIDEO_MB", 500)
MAX_TRANSCRIPT_BYTES = _size_limit("MAX_TRANSCRIPT_MB", 5)
MAX_IMAGE_BYTES = _size_limit("MAX_IMAGE_MB", 25) # per slide image
# In-memory cache of completed scene previews: token β {dir, video_path, frames_dir, frames, created}
_preview_cache: dict[str, dict] = {}
_preview_cache_lock = threading.Lock()
_PREVIEW_TTL = 1800 # 30 minutes
# Hard cap on cached previews so a flood of preview requests can't pin an
# unbounded amount of video data on disk for the full 30-min TTL. When the
# cap is hit, the oldest entries are evicted (their temp dirs removed).
_MAX_PREVIEW_ENTRIES = 24
def _purge_stale_previews() -> None:
cutoff = time.time() - _PREVIEW_TTL
with _preview_cache_lock:
stale = [t for t, v in list(_preview_cache.items()) if v["created"] < cutoff]
evicted = [_preview_cache.pop(t) for t in stale]
# Enforce the entry cap after the TTL sweep. Evict oldest-first
# until we're back under the limit.
if len(_preview_cache) > _MAX_PREVIEW_ENTRIES:
by_age = sorted(_preview_cache.items(), key=lambda kv: kv[1].get("created", 0.0))
overflow = len(_preview_cache) - _MAX_PREVIEW_ENTRIES
for tok, _ in by_age[:overflow]:
evicted.append(_preview_cache.pop(tok))
for entry in evicted:
shutil.rmtree(entry["dir"], ignore_errors=True)
async def _save_upload(
upload: UploadFile, dest: Path, *, max_bytes: int | None, label: str
) -> None:
"""Stream `upload` to `dest`, aborting (and unlinking) past `max_bytes`."""
size = 0
chunk_size = 1024 * 1024
with dest.open("wb") as f:
while True:
chunk = await upload.read(chunk_size)
if not chunk:
break
size += len(chunk)
if max_bytes is not None and size > max_bytes:
try:
dest.unlink(missing_ok=True)
except OSError:
pass
limit_mb = max_bytes // (1024 * 1024)
raise HTTPException(413, f"{label} exceeds the {limit_mb} MB upload limit.")
f.write(chunk)
def _validate_format(value: str) -> str:
if value not in ALLOWED_FORMATS:
raise HTTPException(
400,
f"Unsupported format {value!r}. Must be one of: {sorted(ALLOWED_FORMATS)}.",
)
return value
def _parse_optional_int(value: str | None) -> int | None:
if value is None:
return None
s = value.strip()
if not s:
return None
try:
return int(s)
except ValueError as exc:
raise HTTPException(400, f"Invalid integer: {value!r}") from exc
@app.get("/", response_class=HTMLResponse)
async def index(request: Request) -> HTMLResponse:
return templates.TemplateResponse(request, "index.html", {})
@app.get("/images", response_class=HTMLResponse)
async def images_page(request: Request) -> HTMLResponse:
return templates.TemplateResponse(request, "images.html", {})
@app.get("/edit", response_class=HTMLResponse)
async def edit_page(request: Request) -> HTMLResponse:
return templates.TemplateResponse(request, "edit.html", {})
# A self-contained editor export embeds the slide images as base64, so the
# JSON can be several MB. Cap generously; well above a normal deck.
MAX_EDIT_JSON_BYTES = _size_limit("MAX_EDIT_MB", 150)
@app.post("/edit", response_class=HTMLResponse)
async def edit_from_json(edits: UploadFile = File(...)) -> HTMLResponse:
"""Rebuild a full editor from a single self-contained edits JSON.
The editor's 'Save edits (JSON)' now embeds each slide as a data URI,
so one file carries everything needed to reconstitute the editor β
no job, no image files. This lets an author hand the JSON to someone
else who opens it here and keeps editing. Returns the editor HTML
directly so the browser navigates straight into it.
"""
# Bounded read: accumulate in β€1 MB chunks and abort the moment the
# body exceeds the cap, instead of buffering the whole (possibly
# multi-GB, or Transfer-Encoding: chunked) upload into memory first.
limit = MAX_EDIT_JSON_BYTES
chunks: list[bytes] = []
total = 0
while True:
chunk = await edits.read(1024 * 1024)
if not chunk:
break
total += len(chunk)
if limit is not None and total > limit:
limit_mb = limit // (1024 * 1024)
raise HTTPException(413, f"Edits file exceeds the {limit_mb} MB limit.")
chunks.append(chunk)
raw = b"".join(chunks)
try:
data = json.loads(raw.decode("utf-8"))
except Exception:
raise HTTPException(400, "That file isn't valid JSON. Upload an edits file saved from the editor.")
if not isinstance(data, dict):
raise HTTPException(400, "The JSON must be an editor export object.")
if int(data.get("schema_version", 1) or 1) < CURRENT_SCHEMA_VERSION:
data = migrate_v1_to_v2(data)
try:
page = PageMetadata.model_validate(data)
except Exception as exc:
raise HTTPException(422, f"Not a valid editor export: {exc}")
if not page.segments:
raise HTTPException(400, "This export has no segments to edit.")
missing = [s.index for s in page.segments if not (s.image_data_uri or "").strip()]
if missing:
raise HTTPException(
400,
"This edits file doesn't embed its slide images, so the editor "
"can't be rebuilt from it alone. Re-save it from the editor's "
"'Save edits (JSON)' button (which now embeds the slides) and "
"upload that file.",
)
# Render against a fresh EMPTY temp dir, not the app's working directory.
# Every segment carries an inlined image_data_uri, so metadata_to_segment
# never reads an image off disk; the empty dir plus basename sanitisation
# in metadata_to_segment also means a crafted audio_filename can't be
# read (no traversal, and nothing to read in an empty dir).
scratch = Path(tempfile.mkdtemp(prefix="vgm_edit_"))
try:
html = render_from_metadata(page, scratch, mode="review")
finally:
shutil.rmtree(scratch, ignore_errors=True)
return HTMLResponse(html)
def _safe_filename(title: str) -> str:
safe = "".join(c if c.isalnum() or c in "-_ " else "-" for c in title).strip()
safe = safe.replace(" ", "-")
return safe or "study-guide"
def _content_disposition(filename: str) -> str:
"""Build a Content-Disposition header value that survives ASGI's
latin-1 header encoding for any filename.
`_safe_filename` keeps Unicode alphanumerics ("δΈ".isalnum() is True),
so a CJK / emoji / accented title reaches here as a non-latin-1 string.
Emitting it raw as filename="β¦" makes Starlette raise UnicodeEncodeError
when it encodes the response headers β a 500 that (for one-shot job
results) also destroys the already-cleared body, so the retry gets
nothing. RFC 6266 solves this: an ASCII-only `filename=` fallback for
legacy clients plus a percent-encoded `filename*=UTF-8''β¦` that modern
browsers prefer and that is pure ASCII on the wire.
"""
from urllib.parse import quote
ascii_fallback = filename.encode("ascii", "replace").decode("ascii").replace('"', "'")
encoded = quote(filename, safe="")
return f"attachment; filename=\"{ascii_fallback}\"; filename*=UTF-8''{encoded}"
def _render_to_result(
*,
format: str,
title: str,
lang: str,
subtitle: str | None,
module: str | None,
eyebrow: str,
meta_lines: list[str] | None,
segments: list,
image_paths: list[Path],
audio_paths: list[Path | None] | None = None,
alts_url_template: str | None = None,
ocr_url_template: str | None = None,
audio_url_template: str | None = None,
zip_url: str | None = None,
) -> JobResult:
audio_paths = audio_paths or [None] * len(segments)
common = dict(
title=title,
segments=segments,
lang=lang,
subtitle=subtitle,
module=module,
meta_lines=meta_lines or None,
eyebrow=eyebrow,
)
if format == "review":
# Don't inline alternates β the editor will lazy-fetch them via
# alts_url_template. Keeps the initial review HTML small.
if alts_url_template:
for seg in segments:
seg.alt_image_data_uris = []
html = render_review(
alts_url_template=alts_url_template,
ocr_url_template=ocr_url_template,
audio_url_template=audio_url_template,
zip_url=zip_url,
**common,
)
return JobResult(body=html.encode("utf-8"), media_type="text/html; charset=utf-8")
inline = format == "single"
if inline:
# Inline audio data URIs alongside images so the single HTML stays
# self-contained when downloaded.
for seg, ap in zip(segments, audio_paths):
if ap and ap.exists():
import base64
seg.audio_data_uri = (
"data:audio/mpeg;base64,"
+ base64.b64encode(ap.read_bytes()).decode("ascii")
)
html = render_guide(inline_images=inline, **common)
if inline:
filename = _safe_filename(title) + ".html"
return JobResult(
body=html.encode("utf-8"),
media_type="text/html; charset=utf-8",
filename=filename,
)
audio_disk_paths = [p for p in audio_paths if p is not None]
zip_bytes = bundle_zip(html, image_paths, audio_paths=audio_disk_paths)
filename = _safe_filename(title) + ".zip"
return JobResult(body=zip_bytes, media_type="application/zip", filename=filename)
def _run_job_in_thread(
job: Job,
inputs: PipelineInputs,
workdir: Path,
*,
format: str,
title: str,
lang: str,
subtitle: str | None,
module: str | None,
preselected_frames: list[SceneFrame] | None = None,
preselected_topic_titles: list[str] | None = None,
preselected_topic_summaries: list[str] | None = None,
preview_dir: Path | None = None,
) -> None:
def progress_cb(ev: ProgressEvent) -> None:
registry.emit(
job,
type="progress",
percent=ev.percent,
stage=ev.stage,
message=ev.message,
)
try:
result = run_pipeline(
inputs,
progress=progress_cb,
preselected_frames=preselected_frames,
preselected_topic_titles=preselected_topic_titles,
preselected_topic_summaries=preselected_topic_summaries,
)
# Snapshot alt-frame bytes into the Job before workdir cleanup so the
# editor can lazy-fetch them via /alts/{job_id}/{filename}.
for f in result.kept_frames:
for p in f.alt_image_paths:
try:
job.alt_image_bytes[p.name] = p.read_bytes()
except OSError:
pass
# Same pattern for sliced segment audio (served via /audio/{id}/{fn}).
# Read from the orchestrator's resolved paths rather than guessing
# the on-disk layout, which has changed across versions.
for seg, ap in zip(result.segments, result.audio_paths):
if not seg.audio_filename or ap is None:
continue
try:
job.audio_bytes[seg.audio_filename] = ap.read_bytes()
except OSError:
pass
job_result = _render_to_result(
format=format,
title=title,
lang=lang,
subtitle=subtitle,
module=module,
eyebrow=result.page.eyebrow,
meta_lines=result.page.meta_lines,
segments=result.segments,
image_paths=[f.image_path for f in result.kept_frames],
audio_paths=result.audio_paths,
alts_url_template=(
f"/alts/{job.id}/{{filename}}" if format == "review" else None
),
ocr_url_template=(
f"/ocr/{job.id}/{{filename}}" if format == "review" else None
),
audio_url_template=(
f"/audio/{job.id}/{{filename}}" if format == "review" else None
),
zip_url=f"/jobs/{job.id}/zip" if format == "review" else None,
)
registry.finish_success(job, job_result)
except PipelineError as exc:
registry.finish_error(job, str(exc))
except Exception as exc: # pragma: no cover (defensive)
log.exception("Job %s failed unexpectedly", job.id)
registry.finish_error(job, f"Unexpected error: {exc}")
finally:
shutil.rmtree(workdir, ignore_errors=True)
if preview_dir:
shutil.rmtree(preview_dir, ignore_errors=True)
@app.post("/scene-preview")
async def scene_preview(
video: UploadFile = File(...),
threshold: float = Form(27.0),
min_gap: float = Form(0.0),
max_frames: str | None = Form(None),
face_threshold: float = Form(0.12),
) -> JSONResponse:
"""Detect and filter scenes, return thumbnails for user selection."""
import base64
from PIL import Image
video_suffix = Path(video.filename or "").suffix.lower()
if video_suffix not in VIDEO_SUFFIXES:
raise HTTPException(400, f"Unsupported video format: {video_suffix or '(none)'}")
workdir = Path(tempfile.mkdtemp(prefix="vgm_prev_"))
try:
video_path = workdir / f"input{video_suffix}"
frames_dir = workdir / "static"
frames_dir.mkdir()
await _save_upload(video, video_path, max_bytes=MAX_VIDEO_BYTES, label="Video")
loop = asyncio.get_running_loop()
max_frames_int = _parse_optional_int(max_frames)
frames = await loop.run_in_executor(
None, lambda: detect_and_extract(
video_path, frames_dir,
threshold=threshold,
min_gap_seconds=min_gap,
max_frames=max_frames_int,
)
)
kept, _ = await loop.run_in_executor(
None, lambda: filter_instructor_frames(frames, face_area_threshold=face_threshold)
)
def _build_thumbnails(frames: list) -> list:
out = []
for f in frames:
img = Image.open(f.image_path)
img.thumbnail((320, 180))
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=72)
b64 = base64.b64encode(buf.getvalue()).decode()
out.append({
"index": f.index,
"start_seconds": f.start_seconds,
"end_seconds": f.end_seconds,
"thumbnail": f"data:image/jpeg;base64,{b64}",
"image_filename": f.image_path.name,
})
return out
result_frames = await loop.run_in_executor(None, lambda: _build_thumbnails(kept))
_purge_stale_previews()
token = secrets.token_hex(16)
with _preview_cache_lock:
_preview_cache[token] = {
"dir": workdir,
"video_path": video_path,
"frames_dir": frames_dir,
"frames": kept,
"created": time.time(),
}
return JSONResponse({"preview_token": token, "frames": result_frames})
except Exception:
shutil.rmtree(workdir, ignore_errors=True)
raise
@app.post("/topic-scene-preview")
async def topic_scene_preview(
video: UploadFile = File(...),
transcript: UploadFile = File(...),
threshold: float = Form(27.0),
min_gap: float = Form(0.0),
max_frames: str | None = Form(None),
face_threshold: float = Form(0.12),
skip_inverted_ocr: bool = Form(False),
anthropic_api_key: str | None = Form(None),
llm_topic_segmentation_with_images: bool = Form(False),
llm_topic_video_type: str = Form("general"),
) -> JSONResponse:
"""Run the topic-segmentation pipeline up through primary-frame
picking, then return one card per topic (title, summary, primary
thumbnail) for the user to pick which to include before the
expensive LLM extraction + audio + render stages.
Caches the post-topic kept frames + parallel title/summary/ocr
lists keyed by a token. /jobs accepts that token plus a list of
selected topic indices and resumes the pipeline from there.
"""
import base64
from PIL import Image
from .pipeline.ocr import ocr_frame as run_ocr
if not anthropic_api_key or not anthropic_api_key.strip():
raise HTTPException(
400,
"Topic preview requires an Anthropic API key β Claude segments the transcript.",
)
video_suffix = Path(video.filename or "").suffix.lower()
if video_suffix not in VIDEO_SUFFIXES:
raise HTTPException(400, f"Unsupported video format: {video_suffix or '(none)'}")
transcript_suffix = Path(transcript.filename or "").suffix.lower()
if transcript_suffix not in TRANSCRIPT_SUFFIXES:
raise HTTPException(400, f"Unsupported transcript format: {transcript_suffix or '(none)'}")
if not is_valid_video_type(llm_topic_video_type):
raise HTTPException(
400,
f"Unknown video type '{llm_topic_video_type}'. "
"Expected one of: general, lecture, lab, tutorial, interview, talk.",
)
workdir = Path(tempfile.mkdtemp(prefix="vgm_topicprev_"))
try:
video_path = workdir / f"input{video_suffix}"
transcript_path = workdir / f"transcript{transcript_suffix}"
frames_dir = workdir / "static"
frames_dir.mkdir()
await _save_upload(video, video_path, max_bytes=MAX_VIDEO_BYTES, label="Video")
await _save_upload(
transcript, transcript_path, max_bytes=MAX_TRANSCRIPT_BYTES, label="Transcript"
)
loop = asyncio.get_running_loop()
max_frames_int = _parse_optional_int(max_frames)
# Stage 1: scene detect + filter (CPU-heavy; off the event loop).
frames = await loop.run_in_executor(
None, lambda: detect_and_extract(
video_path, frames_dir,
threshold=threshold,
min_gap_seconds=min_gap,
max_frames=max_frames_int,
)
)
kept, _ = await loop.run_in_executor(
None, lambda: filter_instructor_frames(frames, face_area_threshold=face_threshold)
)
if not kept:
raise HTTPException(
400,
"No usable visual segments were found. Try lowering scene-change sensitivity "
"or raising the instructor face threshold in Advanced settings.",
)
# Stage 2: transcript parse + full-duration computation. Match
# the orchestrator's behaviour so the topic LLM's time range
# matches what the eventual /jobs run will see.
cues = await loop.run_in_executor(None, lambda: parse_transcript(transcript_path))
if not cues:
raise HTTPException(
400,
"Transcript parsed to zero cues. Topic segmentation needs spoken content.",
)
full_duration = max(
await loop.run_in_executor(None, lambda: video_duration(video_path)),
kept[-1].end_seconds,
)
if kept[-1].end_seconds < full_duration:
kept[-1].end_seconds = full_duration
# Stage 3: OCR per scene β the topic-primary picker uses it as
# the "is this real slide content?" signal.
def _ocr_all() -> list[str]:
out: list[str] = []
for f in kept:
out.append(run_ocr(f.image_path, skip_inverted=skip_inverted_ocr))
return out
scene_ocr_texts = await loop.run_in_executor(None, _ocr_all)
# Stage 4: topic LLM call.
frames_for_topics = (
[(f.start_seconds, f.image_path) for f in kept]
if llm_topic_segmentation_with_images
else None
)
try:
topics = await loop.run_in_executor(
None, lambda: extract_topics(
cues,
total_duration=full_duration,
api_key=anthropic_api_key,
frames=frames_for_topics,
video_type=llm_topic_video_type,
),
)
except LLMTopicError as exc:
raise HTTPException(502, f"Topic segmentation failed: {exc}")
# Stage 5: primary-frame picking per topic.
topic_kept, topic_titles, topic_summaries, topic_ocr = _apply_topic_segmentation(
kept, topics, scene_ocr_texts=scene_ocr_texts
)
if not topic_kept:
raise HTTPException(400, "Topic segmentation produced no segments containing slides.")
# Stage 6: thumbnails for the picker UI.
def _build_topic_cards() -> list[dict]:
cards: list[dict] = []
for i, (sf, title, summary, ocr) in enumerate(
zip(topic_kept, topic_titles, topic_summaries, topic_ocr)
):
img = Image.open(sf.image_path)
img.thumbnail((320, 180))
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=72)
b64 = base64.b64encode(buf.getvalue()).decode()
cards.append({
"index": i,
"title": title,
"summary": summary,
"start_seconds": sf.start_seconds,
"end_seconds": sf.end_seconds,
"thumbnail": f"data:image/jpeg;base64,{b64}",
"image_filename": sf.image_path.name,
"ocr_text": ocr,
"scene_count": 1 + len(sf.alt_image_paths),
})
return cards
result_topics = await loop.run_in_executor(None, _build_topic_cards)
_purge_stale_previews()
token = secrets.token_hex(16)
with _preview_cache_lock:
_preview_cache[token] = {
"dir": workdir,
"video_path": video_path,
"frames_dir": frames_dir,
# The "frames" field is what /jobs expects; we store the
# post-topic kept list here so the existing preselect
# path can pick a subset by index.
"frames": topic_kept,
# Topic-specific extras /jobs reads to override section
# titles and skip the topic LLM call on resume.
"topic_titles": topic_titles,
"topic_summaries": topic_summaries,
"topic_ocr_texts": topic_ocr,
"is_topic_preview": True,
"created": time.time(),
}
return JSONResponse({
"preview_token": token,
"topics": result_topics,
})
except HTTPException:
shutil.rmtree(workdir, ignore_errors=True)
raise
except Exception:
shutil.rmtree(workdir, ignore_errors=True)
raise
@app.post("/jobs")
async def start_job(
title: str = Form(...),
video: UploadFile | None = File(None),
transcript: UploadFile = File(...),
format: str = Form("review"),
threshold: float = Form(27.0),
min_gap: float = Form(0.0),
max_frames: str | None = Form(None),
skip_ocr: bool = Form(False),
skip_inverted_ocr: bool = Form(False),
face_threshold: float = Form(0.12),
lang: str = Form("en"),
subtitle: str | None = Form(None),
module: str | None = Form(None),
anthropic_api_key: str | None = Form(None),
llm_alt_text: bool = Form(False),
llm_math_equations: bool = Form(False),
llm_section_title: bool = Form(False),
llm_ocr_text: bool = Form(False),
llm_description: bool = Form(False),
llm_topic_segmentation: bool = Form(False),
llm_topic_segmentation_with_images: bool = Form(False),
llm_topic_video_type: str = Form("general"),
extract_audio: bool = Form(False),
preview_token: str | None = Form(None),
selected_indices: str | None = Form(None),
) -> JSONResponse:
_validate_format(format)
if not is_valid_video_type(llm_topic_video_type):
raise HTTPException(
400,
f"Unknown video type '{llm_topic_video_type}'. "
"Expected one of: general, lecture, lab, tutorial, interview, talk.",
)
# Preview cache TTL is normally driven by /scene-preview and
# /topic-scene-preview hitting _purge_stale_previews on entry. A
# deployment with sustained /jobs traffic but no fresh previews
# would never trigger cleanup; do an opportunistic purge here so
# stale workdirs can't accumulate beyond TTL.
_purge_stale_previews()
transcript_suffix = Path(transcript.filename or "").suffix.lower()
if transcript_suffix not in TRANSCRIPT_SUFFIXES:
raise HTTPException(400, f"Unsupported transcript format: {transcript_suffix or '(none)'}")
# If the user ticked any AI feature but provided no API key, the run
# would either silently no-op (per-segment toggles) or hard-fail
# halfway through (topic segmentation). Catch it up-front with a
# message that names the fix, so the form doesn't appear to "work"
# while quietly dropping every LLM-driven feature they wanted.
any_llm_requested = (
llm_alt_text or llm_math_equations
or llm_section_title or llm_ocr_text or llm_topic_segmentation
or llm_description
)
if any_llm_requested and not (anthropic_api_key and anthropic_api_key.strip()):
raise HTTPException(
400,
"AI features were requested (e.g. topic segmentation, section titles) "
"but no Anthropic API key was provided. Either add a key in the AI "
"assist section or uncheck the AI features.",
)
# Resolve preview cache entry (if the user went through either
# scene picker β the basic Preview scenes or the topic-aware twin).
preview_entry: dict | None = None
preselected_frames: list[SceneFrame] | None = None
preselected_topic_titles: list[str] | None = None
preselected_topic_summaries: list[str] | None = None
is_topic_preview = False
if preview_token:
with _preview_cache_lock:
preview_entry = _preview_cache.pop(preview_token, None)
if not preview_entry:
raise HTTPException(400, "Scene preview has expired β please preview again.")
# Past this point, preview_entry owns a temp dir on disk. Any
# exception before the worker thread takes ownership (workdir
# try/finally) would orphan that dir. Catch validation errors
# here and clean it up before re-raising.
try:
all_frames: list[SceneFrame] = preview_entry["frames"]
is_topic_preview = bool(preview_entry.get("is_topic_preview"))
if is_topic_preview:
valid_max = len(all_frames) - 1
else:
# Scene preview tracks frames by their original .index
# (post-filter ordering), not positional; valid range =
# the set of those index values.
valid_indices = {f.index for f in all_frames}
if selected_indices:
# Parse and validate up-front so an out-of-range index
# is a 400 rather than a silently-dropped slide the
# user thinks they kept. Negative numbers and non-ints
# raise explicitly instead of being skipped.
raw_indices: list[int] = []
for raw in selected_indices.split(","):
token = raw.strip()
if not token:
continue
try:
n = int(token)
except ValueError:
raise HTTPException(
400,
f"Invalid index '{token}' in selected_indices.",
)
if n < 0:
raise HTTPException(
400,
f"Negative index {n} is not allowed in selected_indices.",
)
raw_indices.append(n)
if is_topic_preview:
for n in raw_indices:
if n > valid_max:
raise HTTPException(
400,
f"Topic index {n} is out of range "
f"(have {len(all_frames)} topics).",
)
else:
bad = [n for n in raw_indices if n not in valid_indices]
if bad:
raise HTTPException(
400,
f"Scene indices {bad} are not in the preview set.",
)
idx_set = set(raw_indices)
else:
idx_set = (
set(range(len(all_frames))) if is_topic_preview else valid_indices
)
if is_topic_preview:
# Topic preview indices are positional (the order topics
# came back in /topic-scene-preview), not the .index field.
topic_titles = preview_entry.get("topic_titles") or []
topic_summaries = preview_entry.get("topic_summaries") or []
preselected_frames = [
f for i, f in enumerate(all_frames) if i in idx_set
]
preselected_topic_titles = [
t for i, t in enumerate(topic_titles) if i in idx_set
]
# Forward summaries too β without this the resumed
# pipeline has no topic-segmentation pass (it was done
# in the preview) so the "In brief:" advance organizer
# would silently fall back to "" for every section.
preselected_topic_summaries = [
s for i, s in enumerate(topic_summaries) if i in idx_set
]
else:
preselected_frames = [f for f in all_frames if f.index in idx_set]
if not preselected_frames:
raise HTTPException(
400,
"No topics selected." if is_topic_preview else "No scenes selected.",
)
except HTTPException as exc:
# A 4xx here is the client's selection being invalid (bad index,
# empty selection) β recoverable. Re-insert the popped preview so
# the user can correct their input and retry WITHOUT paying for
# another topic preview (which costs an LLM call). Only 4xx are
# recoverable; a 5xx means something server-side broke, so the
# preview is dropped like any other hard failure.
if 400 <= exc.status_code < 500:
with _preview_cache_lock:
_preview_cache[preview_token] = preview_entry
preview_entry = None # keep the finally from cleaning it up
raise
shutil.rmtree(preview_entry["dir"], ignore_errors=True)
preview_entry = None
raise
except Exception:
# Cleanup before re-raise so the popped preview workdir
# doesn't leak when validation fails after the cache pop.
# Catch Exception (not BaseException) so KeyboardInterrupt
# and SystemExit still propagate without first running
# cleanup that could mask the signal.
shutil.rmtree(preview_entry["dir"], ignore_errors=True)
preview_entry = None
raise
else:
if not video or not video.filename:
raise HTTPException(400, "A video file is required when not using scene preview.")
video_suffix = Path(video.filename).suffix.lower()
if video_suffix not in VIDEO_SUFFIXES:
raise HTTPException(400, f"Unsupported video format: {video_suffix or '(none)'}")
workdir = Path(tempfile.mkdtemp(prefix="vgm_"))
handed_off = False
try:
transcript_path = workdir / f"transcript{transcript_suffix}"
await _save_upload(
transcript, transcript_path, max_bytes=MAX_TRANSCRIPT_BYTES, label="Transcript"
)
if preview_entry:
video_path = preview_entry["video_path"]
frames_dir = preview_entry["frames_dir"]
else:
video_suffix = Path(video.filename).suffix.lower() # type: ignore[union-attr]
video_path = workdir / f"input{video_suffix}"
frames_dir = workdir / "static"
frames_dir.mkdir(parents=True, exist_ok=True)
await _save_upload(video, video_path, max_bytes=MAX_VIDEO_BYTES, label="Video")
inputs = PipelineInputs(
video_path=video_path,
transcript_path=transcript_path,
frames_dir=frames_dir,
title=title,
subtitle=subtitle or None,
module=module or None,
lang=lang,
threshold=threshold,
min_gap_seconds=min_gap,
max_frames=_parse_optional_int(max_frames),
skip_ocr=skip_ocr,
skip_inverted_ocr=skip_inverted_ocr,
face_threshold=face_threshold,
inline_images=(format in ("single", "review")),
anthropic_api_key=(anthropic_api_key or None),
llm_toggles=LLMToggles(
alt_text=llm_alt_text,
math_equations=llm_math_equations,
section_title=llm_section_title,
ocr_text=llm_ocr_text,
description=llm_description,
# Topic segmentation already happened during the preview;
# don't re-run it in the resume path or we'd pay for the
# call twice and could pick different boundaries than the
# user just curated.
topic_segmentation=llm_topic_segmentation and not is_topic_preview,
topic_segmentation_with_images=llm_topic_segmentation_with_images,
topic_video_type=llm_topic_video_type,
),
extract_audio=extract_audio,
)
try:
job = registry.create()
except JobQuotaExceeded as exc:
raise HTTPException(429, str(exc))
job.ocr_skip_inverted = skip_inverted_ocr
try:
threading.Thread(
target=_run_job_in_thread,
args=(job, inputs, workdir),
kwargs=dict(
format=format,
title=title,
lang=lang,
subtitle=subtitle or None,
module=module or None,
preselected_frames=preselected_frames,
preselected_topic_titles=preselected_topic_titles,
preselected_topic_summaries=preselected_topic_summaries,
preview_dir=preview_entry["dir"] if preview_entry else None,
),
daemon=True,
).start()
except BaseException:
# If the OS refuses the thread, the job would otherwise sit in
# the registry as "running" forever (finished_at stays None, so
# the TTL purge never reclaims it) β a permanently-leaked quota
# slot. Release it before propagating.
registry.pop(job.id)
raise
handed_off = True
finally:
# Anything that fails before the worker thread takes ownership of
# workdir β bad form field, copy error, dataclass validation, even
# an OS thread-start failure β must clean up the temp dir we just
# mkdtemp'd. Once Thread.start() returns successfully, the worker's
# own try/finally handles cleanup.
if not handed_off:
shutil.rmtree(workdir, ignore_errors=True)
if preview_entry:
shutil.rmtree(preview_entry["dir"], ignore_errors=True)
return JSONResponse(
{
"job_id": job.id,
"events_url": f"/events/{job.id}",
"result_url": f"/result/{job.id}",
},
status_code=202,
)
@app.post("/jobs/from-images")
async def start_job_from_images(
title: str = Form(...),
images: list[UploadFile] = File(...),
transcript: UploadFile | None = File(None),
format: str = Form("review"),
subtitle: str | None = Form(None),
module: str | None = Form(None),
lang: str = Form("en"),
skip_ocr: bool = Form(False),
skip_inverted_ocr: bool = Form(False),
anthropic_api_key: str | None = Form(None),
llm_alt_text: bool = Form(False),
llm_math_equations: bool = Form(False),
llm_section_title: bool = Form(False),
llm_ocr_text: bool = Form(False),
llm_description: bool = Form(False),
llm_topic_segmentation: bool = Form(False),
llm_topic_segmentation_with_images: bool = Form(False),
llm_topic_video_type: str = Form("general"),
) -> JSONResponse:
"""Images-input mode: build a study guide from pre-exported slide
images (each named with an HH-MM-SS timestamp) plus one transcript,
with no source video and no scene detection.
Each image becomes a segment anchored at its timestamp; the transcript
is aligned to those anchors by the same downstream pipeline the video
path uses. Deterministic, no large upload, author-curated slides.
"""
if format not in ALLOWED_FORMATS:
raise HTTPException(400, f"Unknown output format: {format}")
# The transcript is OPTIONAL in images mode: with none, the guide is
# built from the slides alone (no narration). FastAPI can hand us an
# empty UploadFile with a blank filename for an omitted field, so treat
# "no filename" as "no transcript".
has_transcript = bool(transcript is not None and (transcript.filename or "").strip())
transcript_suffix = Path(transcript.filename or "").suffix.lower() if has_transcript else ""
if has_transcript and transcript_suffix not in TRANSCRIPT_SUFFIXES:
raise HTTPException(400, f"Unsupported transcript format: {transcript_suffix or '(none)'}")
if not images:
raise HTTPException(400, "At least one slide image is required.")
if len(images) > MAX_IMAGE_COUNT:
raise HTTPException(
400, f"Too many images ({len(images)}); the limit is {MAX_IMAGE_COUNT}."
)
# Same guard as /jobs: AI toggles without a key would silently no-op
# or hard-fail mid-run.
any_llm_requested = (
llm_alt_text or llm_math_equations or llm_section_title
or llm_ocr_text or llm_topic_segmentation or llm_description
)
if any_llm_requested and not (anthropic_api_key and anthropic_api_key.strip()):
raise HTTPException(
400,
"AI features were requested but no Anthropic API key was provided. "
"Add a key or uncheck the AI features.",
)
# Topic segmentation reads the transcript; it can't run without one.
if llm_topic_segmentation and not has_transcript:
raise HTTPException(
400,
"Topic segmentation needs a transcript. Add one, or turn topic "
"segmentation off.",
)
if llm_topic_segmentation and not is_valid_video_type(llm_topic_video_type):
raise HTTPException(
400,
f"Unknown video type '{llm_topic_video_type}'. "
"Expected one of: general, lecture, lab, tutorial, interview, talk.",
)
workdir = Path(tempfile.mkdtemp(prefix="vgm_img_"))
handed_off = False
try:
transcript_path: Path | None = None
transcript_end: float | None = None
transcript_start: float = 0.0
transcript_cues: list = []
if has_transcript:
transcript_path = workdir / f"transcript{transcript_suffix}"
await _save_upload(
transcript, transcript_path, max_bytes=MAX_TRANSCRIPT_BYTES, label="Transcript"
)
cues = parse_transcript(transcript_path)
if not cues:
raise HTTPException(
400,
"Transcript parsed to zero cues. Provide a non-empty VTT/SRT "
"file, or omit the transcript entirely.",
)
transcript_cues = cues
transcript_start = cues[0].start
transcript_end = cues[-1].end
frames_dir = workdir / "static"
frames_dir.mkdir(parents=True, exist_ok=True)
saved_paths: list[Path] = []
seen_names: set[str] = set()
for up in images:
base = Path(up.filename or "").name
suffix = Path(base).suffix.lower()
if suffix not in IMAGE_SUFFIXES:
raise HTTPException(
400,
f"Unsupported image format '{suffix or '(none)'}' for "
f"'{base or '(unnamed)'}'. Use PNG or JPG.",
)
if not base or base in seen_names:
# Guarantee a unique, path-safe on-disk name; keep the
# original (which carries the timestamp) as a prefix so the
# parser still finds HH-MM-SS.
base = f"{len(saved_paths):03d}_{base or 'image' + suffix}"
seen_names.add(base)
dest = frames_dir / base
await _save_upload(up, dest, max_bytes=MAX_IMAGE_BYTES, label="Image")
saved_paths.append(dest)
# Three ways to place the slides on a timeline:
# 1. No transcript β slides just need an order. Sort by filename,
# no timestamps required, and mark the guide untimed so it
# suppresses the (meaningless) time labels.
# 2. Transcript + every filename carries HH-MM-SS β anchor each
# slide at its own filename timestamp (deterministic).
# 3. Transcript but filenames lack timestamps β read each slide's
# on-screen text (local OCR) and ask Claude when the narration
# first discusses it, giving real per-slide start times. If
# that fails (no key, network, bad output), fall back to an
# even split across the transcript's span. Both are "timed".
untimed_flag = not has_transcript
def _build_aligned_frames() -> list[SceneFrame]:
"""Sync (OCR + network) work, run off the event loop."""
from .pipeline.ocr import ocr_frame
ordered = order_by_name(saved_paths)
slide_texts = [
ocr_frame(p, skip_inverted=skip_inverted_ocr) for p in ordered
]
try:
starts = align_slides_to_transcript(
transcript_cues,
slide_texts,
api_key=(anthropic_api_key or None),
)
return build_frames_from_starts(
ordered, starts, transcript_end=transcript_end
)
except SlideAlignError as exc:
log.info("Slide alignment fell back to even split: %s", exc)
return build_frames_even_split(
saved_paths,
transcript_start=transcript_start,
transcript_end=transcript_end,
)
try:
if not has_transcript:
frames = build_frames_ordered_by_name(saved_paths)
elif all_have_timestamps(saved_paths, transcript_end):
frames = build_frames_from_images(
saved_paths, transcript_end=transcript_end
)
elif anthropic_api_key and anthropic_api_key.strip():
# AI alignment needs a key; without one, go straight to the
# even split (no OCR/network to attempt).
loop = asyncio.get_running_loop()
frames = await loop.run_in_executor(None, _build_aligned_frames)
else:
frames = build_frames_even_split(
saved_paths,
transcript_start=transcript_start,
transcript_end=transcript_end,
)
except ImageInputError as exc:
raise HTTPException(400, str(exc))
inputs = PipelineInputs(
video_path=None,
transcript_path=transcript_path,
frames_dir=frames_dir,
title=title,
subtitle=subtitle or None,
module=module or None,
lang=lang,
skip_ocr=skip_ocr,
skip_inverted_ocr=skip_inverted_ocr,
inline_images=(format in ("single", "review")),
anthropic_api_key=(anthropic_api_key or None),
llm_toggles=LLMToggles(
alt_text=llm_alt_text,
math_equations=llm_math_equations,
section_title=llm_section_title,
ocr_text=llm_ocr_text,
description=llm_description,
topic_segmentation=llm_topic_segmentation,
topic_segmentation_with_images=llm_topic_segmentation_with_images,
topic_video_type=llm_topic_video_type,
),
extract_audio=False, # no source video to slice
allow_empty_transcript=True, # transcript optional in images mode
untimed=untimed_flag, # no transcript β order-only, no times
)
try:
job = registry.create()
except JobQuotaExceeded as exc:
raise HTTPException(429, str(exc))
job.ocr_skip_inverted = skip_inverted_ocr
try:
threading.Thread(
target=_run_job_in_thread,
args=(job, inputs, workdir),
kwargs=dict(
format=format,
title=title,
lang=lang,
subtitle=subtitle or None,
module=module or None,
preselected_frames=frames,
),
daemon=True,
).start()
except BaseException:
registry.pop(job.id)
raise
handed_off = True
finally:
if not handed_off:
shutil.rmtree(workdir, ignore_errors=True)
return JSONResponse(
{
"job_id": job.id,
"events_url": f"/events/{job.id}",
"result_url": f"/result/{job.id}",
},
status_code=202,
)
@app.get("/events/{job_id}")
async def stream_events(job_id: str) -> StreamingResponse:
job = registry.get(job_id)
if not job:
raise HTTPException(404, "Unknown job id.")
# Hard cap on how long an SSE consumer will wait without ANY event
# before declaring the job dead. Bounds the worst case where the
# worker thread crashed before queuing a terminal event β without
# this the SSE generator would ping forever.
silence_seconds = 1800
async def gen():
last_event_time = time.time()
while True:
try:
ev = await asyncio.to_thread(job.events.get, True, 15.0)
except queue.Empty:
# Status may have flipped without a queued event
# (defensive); exit cleanly instead of pinging forever.
if job.status in ("done", "error"):
return
# If we haven't seen an event in a long time AND the job
# claims to still be running, treat it as wedged. Re-check
# status right before finish_error to avoid racing a
# finish_success that may have just fired.
if (
time.time() - last_event_time > silence_seconds
and job.status == "running"
and job.finished_at is None
):
registry.finish_error(
job, "Job timed out (no progress in over 30 minutes)."
)
# finish_error queued the terminal event; loop once
# more to pick it up and exit through the normal path.
continue
yield ": ping\n\n"
continue
last_event_time = time.time()
yield f"data: {json.dumps(ev)}\n\n"
if ev.get("type") in ("done", "error"):
return
headers = {
"Cache-Control": "no-cache, no-transform",
"X-Accel-Buffering": "no", # disable proxy buffering for SSE
}
return StreamingResponse(gen(), media_type="text/event-stream", headers=headers)
@app.get("/result/{job_id}")
async def fetch_result(job_id: str) -> Response:
# Keep the job alive after delivery so the editor can lazy-fetch alt
# frames; the job-TTL purge handles eventual cleanup.
job = registry.get(job_id)
if not job:
raise HTTPException(404, "Unknown job id.")
if job.status == "error":
raise HTTPException(422, job.error or "Pipeline error.")
if job.status != "done" or job.result is None:
raise HTTPException(409, "Job not finished.")
if not job.result.body:
# Body was already delivered and freed; alts are still fetchable
# but the rendered HTML/zip itself is one-shot.
raise HTTPException(410, "Result already delivered.")
headers: dict[str, str] = {}
if job.result.filename:
headers["Content-Disposition"] = _content_disposition(job.result.filename)
body = job.result.body
media = job.result.media_type
# Review-mode results have no attachment filename; the upload page
# loads them into an iframe AND lets the user "Open in new tab",
# which re-fetches the same URL. Keep the body for the job's TTL so
# the second GET still works. Single/zip downloads stay one-shot β
# the body for those can be MB-scale and isn't re-requested.
if job.result.filename:
job.result.body = b""
return Response(content=body, media_type=media, headers=headers)
_ALT_FILENAME_RE = __import__("re").compile(r"^scene_\d{4}_alt\d+\.(jpg|jpeg|png)$")
_PRIMARY_FILENAME_RE = __import__("re").compile(r"^scene_\d{4}\.(jpg|jpeg|png)$")
_AUDIO_FILENAME_RE = __import__("re").compile(r"^segment_\d{4}\.mp3$")
_IMAGE_ASSET_EXTS = {".jpg", ".jpeg", ".png", ".webp"}
_AUDIO_ASSET_EXTS = {".mp3"}
def _is_safe_asset_name(name: str, allowed_exts: set[str]) -> bool:
"""A bundle-safe filename: a bare basename (no directory components,
no traversal) with an allowed extension.
The ZIP builder joins these onto a temp dir and reads/writes them, so
they must not be able to escape it. `Path(name).name == name` rejects
anything with a separator or `..` component; the extension whitelist
rejects the rest. This replaces the old scene_NNNN-only patterns, which
wrongly excluded images-mode uploads that keep their original names
(e.g. 'slide 1.jpg')."""
if not isinstance(name, str) or not name:
return False
if Path(name).name != name:
return False
return Path(name).suffix.lower() in allowed_exts
@app.get("/audio/{job_id}/{filename}")
async def fetch_audio(job_id: str, filename: str) -> Response:
if not _AUDIO_FILENAME_RE.match(filename):
raise HTTPException(404, "Unknown audio.")
job = registry.get(job_id)
if not job:
raise HTTPException(404, "Unknown job id.")
data = job.audio_bytes.get(filename)
if data is None:
raise HTTPException(404, "Audio not found.")
return Response(
content=data,
media_type="audio/mpeg",
headers={"Cache-Control": "private, max-age=600"},
)
@app.get("/alts/{job_id}/{filename}")
async def fetch_alt(job_id: str, filename: str) -> Response:
# Filename comes from a URL path, so be strict about its shape β only
# patterns the pipeline produces are accepted, and no traversal chars.
# Topic-based segmentation re-uses other scenes' primary frames as
# alternates, so we also accept the primary scene_NNNN.jpg shape.
if not (
_ALT_FILENAME_RE.match(filename) or _PRIMARY_FILENAME_RE.match(filename)
):
raise HTTPException(404, "Unknown alternate.")
job = registry.get(job_id)
if not job:
raise HTTPException(404, "Unknown job id.")
data = job.alt_image_bytes.get(filename)
if data is None:
raise HTTPException(404, "Alternate not found.")
media = "image/jpeg" if filename.lower().endswith((".jpg", ".jpeg")) else "image/png"
return Response(
content=data,
media_type=media,
headers={"Cache-Control": "private, max-age=600"},
)
@app.get("/ocr/{job_id}/{filename}")
async def fetch_ocr(job_id: str, filename: str) -> Response:
# Lazy OCR for an alternate frame. Tesseract is slow on the HF CPU
# tier, so we skip it during the main pipeline and run it here only
# when the reviewer actually picks an alt. First call writes the
# cached bytes to a temp file and OCRs; subsequent calls return cache.
if not (
_ALT_FILENAME_RE.match(filename) or _PRIMARY_FILENAME_RE.match(filename)
):
raise HTTPException(404, "Unknown alternate.")
job = registry.get(job_id)
if not job:
raise HTTPException(404, "Unknown job id.")
cached = job.alt_ocr_cache.get(filename)
if cached is not None:
return Response(
content=cached,
media_type="text/plain; charset=utf-8",
headers={"Cache-Control": "private, max-age=600"},
)
data = job.alt_image_bytes.get(filename)
if data is None:
raise HTTPException(404, "Alternate not found.")
import tempfile
from pathlib import Path
from fastapi.concurrency import run_in_threadpool
from .pipeline.ocr import ocr_frame
suffix = ".jpg" if filename.lower().endswith((".jpg", ".jpeg")) else ".png"
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
tmp.write(data)
tmp_path = Path(tmp.name)
try:
text = await run_in_threadpool(
lambda: ocr_frame(tmp_path, skip_inverted=job.ocr_skip_inverted)
)
finally:
try:
tmp_path.unlink(missing_ok=True)
except OSError:
pass
# If a parallel /ocr request for the same filename beat us to the cache,
# keep its result rather than overwriting; both ran OCR on identical
# bytes so the texts should match anyway, but using setdefault makes the
# winning text stable for any caller that already read it.
text = job.alt_ocr_cache.setdefault(filename, text)
return Response(
content=text,
media_type="text/plain; charset=utf-8",
headers={"Cache-Control": "private, max-age=600"},
)
_DATA_URI_RE = __import__("re").compile(r"^data:[^;]+;base64,(.+)$", __import__("re").DOTALL)
@app.post("/jobs/{job_id}/zip")
async def fetch_job_zip(job_id: str, request: Request) -> Response:
"""Bundle the current editor state into a downloadable .zip.
The editor POSTs its in-memory state plus the data URIs of every
currently-picked primary frame (alt or original β the editor's main
<img> src is the source of truth). The server decodes those bytes,
pulls audio bytes from the in-memory job, calls the same
render_from_metadata + bundle_zip path the CLI uses, and returns
the bundle. No request body is persisted; the temp dir is deleted
in `finally`.
"""
job = registry.get(job_id)
if not job:
raise HTTPException(404, "Unknown job id.")
try:
body = await request.json()
except Exception:
raise HTTPException(400, "Invalid JSON body.")
state = body.get("state") if isinstance(body, dict) else None
primary_uris = body.get("primary_image_data_uris") if isinstance(body, dict) else None
if not isinstance(state, dict) or not isinstance(primary_uris, dict):
raise HTTPException(400, "Body must be {state, primary_image_data_uris}.")
try:
page = PageMetadata.model_validate(state)
except Exception as exc:
raise HTTPException(422, f"Invalid metadata: {exc}")
import base64
workdir = Path(tempfile.mkdtemp(prefix="vgm_zip_"))
try:
frames_dir = workdir / "static"
frames_dir.mkdir()
# Decode and write the picked primary frame bytes for each segment.
# Filenames must match what the metadata references β only patterns
# the pipeline produces are accepted.
for filename, uri in primary_uris.items():
if not isinstance(filename, str) or not isinstance(uri, str):
continue
if not _is_safe_asset_name(filename, _IMAGE_ASSET_EXTS):
continue
m = _DATA_URI_RE.match(uri)
if not m:
continue
try:
data = base64.b64decode(m.group(1), validate=True)
except Exception:
continue
(frames_dir / filename).write_bytes(data)
# Pull audio bytes from the job (no client upload needed) and
# write them next to the frames so the bundled HTML can link
# via `static/<audio>`.
for fn, data in job.audio_bytes.items():
if not _AUDIO_FILENAME_RE.match(fn):
continue
(frames_dir / fn).write_bytes(data)
# Render with image_dir_name="static" and inline=False; the HTML
# will link relatively to files we just wrote.
try:
html = render_from_metadata(page, frames_dir, mode="guide", image_dir_name="static")
except Exception as exc:
raise HTTPException(500, f"Render failed: {exc}")
# Filenames come from client-supplied state β only accept the
# exact patterns the pipeline produces so a crafted name with
# path separators or `..` can't escape `frames_dir` and pull
# arbitrary readable files into the bundle.
frame_paths = [
frames_dir / s.image_filename
for s in page.segments
if _is_safe_asset_name(s.image_filename, _IMAGE_ASSET_EXTS)
]
audio_paths = [
frames_dir / s.audio_filename
for s in page.segments
if _is_safe_asset_name(s.audio_filename, _AUDIO_ASSET_EXTS)
]
zip_bytes = bundle_zip(
html,
frame_paths,
audio_paths=audio_paths,
)
filename = _safe_filename(page.title) + ".zip"
return Response(
content=zip_bytes,
media_type="application/zip",
headers={"Content-Disposition": _content_disposition(filename)},
)
finally:
shutil.rmtree(workdir, ignore_errors=True)
# Backwards-compat: keep a synchronous /process endpoint for non-browser clients.
@app.post("/process")
async def process(
request: Request,
title: str = Form(...),
video: UploadFile = File(...),
transcript: UploadFile = File(...),
format: str = Form("review"),
threshold: float = Form(27.0),
min_gap: float = Form(0.0),
max_frames: str | None = Form(None),
skip_ocr: bool = Form(False),
skip_inverted_ocr: bool = Form(False),
face_threshold: float = Form(0.12),
lang: str = Form("en"),
subtitle: str | None = Form(None),
module: str | None = Form(None),
anthropic_api_key: str | None = Form(None),
llm_alt_text: bool = Form(False),
llm_math_equations: bool = Form(False),
llm_section_title: bool = Form(False),
llm_ocr_text: bool = Form(False),
llm_description: bool = Form(False),
llm_topic_segmentation: bool = Form(False),
llm_topic_segmentation_with_images: bool = Form(False),
llm_topic_video_type: str = Form("general"),
extract_audio: bool = Form(False),
) -> Response:
_validate_format(format)
if not is_valid_video_type(llm_topic_video_type):
raise HTTPException(
400,
f"Unknown video type '{llm_topic_video_type}'. "
"Expected one of: general, lecture, lab, tutorial, interview, talk.",
)
# Same guard as /jobs: AI features ticked without a key would either
# silently no-op (per-segment toggles) or hard-fail mid-run (topic
# segmentation). Fail fast with a message that names the fix instead
# of returning a guide that quietly dropped everything the caller asked
# for. (/process is the legacy non-browser endpoint; it had no guard.)
any_llm_requested = (
llm_alt_text or llm_math_equations or llm_section_title
or llm_ocr_text or llm_topic_segmentation or llm_description
)
if any_llm_requested and not (anthropic_api_key and anthropic_api_key.strip()):
raise HTTPException(
400,
"AI features were requested but no Anthropic API key was provided. "
"Add a key or drop the AI options.",
)
video_suffix = Path(video.filename or "").suffix.lower()
transcript_suffix = Path(transcript.filename or "").suffix.lower()
if video_suffix not in VIDEO_SUFFIXES:
raise HTTPException(400, f"Unsupported video format: {video_suffix or '(none)'}")
if transcript_suffix not in TRANSCRIPT_SUFFIXES:
raise HTTPException(400, f"Unsupported transcript format: {transcript_suffix or '(none)'}")
workdir = Path(tempfile.mkdtemp(prefix="vgm_"))
try:
video_path = workdir / f"input{video_suffix}"
transcript_path = workdir / f"transcript{transcript_suffix}"
frames_dir = workdir / "static"
await _save_upload(video, video_path, max_bytes=MAX_VIDEO_BYTES, label="Video")
await _save_upload(
transcript, transcript_path, max_bytes=MAX_TRANSCRIPT_BYTES, label="Transcript"
)
inputs = PipelineInputs(
video_path=video_path,
transcript_path=transcript_path,
frames_dir=frames_dir,
title=title,
subtitle=subtitle or None,
module=module or None,
lang=lang,
threshold=threshold,
min_gap_seconds=min_gap,
max_frames=_parse_optional_int(max_frames),
skip_ocr=skip_ocr,
skip_inverted_ocr=skip_inverted_ocr,
face_threshold=face_threshold,
inline_images=(format in ("single", "review")),
anthropic_api_key=(anthropic_api_key or None),
llm_toggles=LLMToggles(
alt_text=llm_alt_text,
math_equations=llm_math_equations,
section_title=llm_section_title,
ocr_text=llm_ocr_text,
description=llm_description,
topic_segmentation=llm_topic_segmentation,
topic_segmentation_with_images=llm_topic_segmentation_with_images,
topic_video_type=llm_topic_video_type,
),
extract_audio=extract_audio,
)
try:
result = await asyncio.to_thread(run_pipeline, inputs)
except PipelineError as exc:
raise HTTPException(422, str(exc))
job_result = _render_to_result(
format=format,
title=title,
lang=lang,
subtitle=subtitle or None,
module=module or None,
eyebrow=result.page.eyebrow,
meta_lines=result.page.meta_lines,
segments=result.segments,
image_paths=[f.image_path for f in result.kept_frames],
audio_paths=result.audio_paths,
)
headers: dict[str, str] = {}
if job_result.filename:
headers["Content-Disposition"] = _content_disposition(job_result.filename)
return Response(content=job_result.body, media_type=job_result.media_type, headers=headers)
finally:
shutil.rmtree(workdir, ignore_errors=True)
|