Spaces:
Runtime error
Runtime error
Refactor VideoTemplateEditor component: remove AI editing features and related state management, streamline template loading, and update styles for improved layout. Enhance CSS for responsive design and visual consistency across various components, including dropdowns, sync stats, and quiz elements. Remove deprecated premium badge styles and adjust layout for chat components.
66de9c8 | import uuid | |
| from fastapi import APIRouter, Depends, HTTPException | |
| from fastapi.concurrency import run_in_threadpool | |
| from pydantic import BaseModel, Field | |
| from sqlalchemy import delete, func, or_, select | |
| from app.auth import Ctx, get_ctx | |
| from app.corpus import INDEX, Corpus, CorpusError | |
| from app.models import Document, Job, NotebookDocument, Source, Upload | |
| from app.pipeline.ingest import IMPORTS_FEED, FeedNotFound, discover_feed, imports_source, rebuild_index | |
| from app.services.jobs import create_job, serialize_job | |
| from app.services.plans import effective_plan, plan_limit_error, post_room | |
| from app.services.storage import storage | |
| router = APIRouter(prefix="/api/sources", tags=["sources"]) | |
| class SourceIn(BaseModel): | |
| url: str = Field(min_length=3, max_length=1000) | |
| class UploadImportIn(BaseModel): | |
| upload_id: uuid.UUID | |
| def serialize_source(source: Source, doc_count: int = 0, locked_count: int = 0) -> dict: | |
| return { | |
| "id": str(source.id), | |
| "feed_url": source.feed_url, | |
| "site_url": source.site_url, | |
| "platform": source.platform, | |
| "title": source.title, | |
| "sync_status": source.sync_status, | |
| "sync_error": source.sync_error, | |
| "last_synced_at": source.last_synced_at.isoformat() if source.last_synced_at else None, | |
| "document_count": doc_count, | |
| # Found but beyond the plan's post limit: listed, not indexed until the workspace upgrades. | |
| "locked_count": locked_count, | |
| "is_imports": source.feed_url == IMPORTS_FEED, | |
| } | |
| def _get(ctx: Ctx, source_id: uuid.UUID) -> Source: | |
| source = ctx.db.scalar(select(Source).where(Source.id == source_id, Source.workspace_id == ctx.workspace.id)) | |
| if not source: | |
| raise HTTPException(404, "Source not found") | |
| return source | |
| def _enqueue_sync(ctx: Ctx, source: Source) -> Job: | |
| source.sync_status = "pending" | |
| return create_job(ctx.db, ctx.workspace.id, "ingest", {"source_id": str(source.id)}) | |
| def _doc_count(ctx: Ctx, source: Source) -> int: | |
| return ctx.db.scalar(select(func.count()).select_from(Document).where(Document.source_id == source.id)) or 0 | |
| async def create_source(body: SourceIn, ctx: Ctx = Depends(get_ctx)): | |
| try: | |
| found = await run_in_threadpool(discover_feed, body.url) | |
| except FeedNotFound as exc: | |
| raise HTTPException(422, {"code": "feed_not_found", "message": str(exc)}) from exc | |
| existing = ctx.db.scalar( | |
| select(Source).where(Source.workspace_id == ctx.workspace.id, Source.feed_url == found.feed_url) | |
| ) | |
| if existing: | |
| job = _enqueue_sync(ctx, existing) | |
| return {"source": serialize_source(existing, _doc_count(ctx, existing)), "job": serialize_job(job)} | |
| plan = effective_plan(ctx.db, ctx.workspace) | |
| count = ctx.db.scalar( | |
| select(func.count()).select_from(Source) | |
| .where(Source.workspace_id == ctx.workspace.id, Source.feed_url != IMPORTS_FEED) | |
| ) | |
| if 0 <= plan.sources <= count: | |
| raise plan_limit_error(plan, "sources", f"Your plan includes {plan.sources} sources.") | |
| source = Source(workspace_id=ctx.workspace.id, feed_url=found.feed_url, platform=found.platform, | |
| site_url=found.site_url, title=found.title) | |
| ctx.db.add(source) | |
| ctx.db.commit() | |
| job = _enqueue_sync(ctx, source) | |
| return {"source": serialize_source(source), "job": serialize_job(job)} | |
| def list_sources(ctx: Ctx = Depends(get_ctx)): | |
| rows = ctx.db.execute( | |
| select(Source, func.count(Document.id)) | |
| .outerjoin(Document, Document.source_id == Source.id) | |
| .where(Source.workspace_id == ctx.workspace.id) | |
| .group_by(Source.id) | |
| .order_by(Source.created_at) | |
| ).all() | |
| locked = dict(ctx.db.execute( | |
| select(Document.source_id, func.count(Document.id)) | |
| .where(Document.workspace_id == ctx.workspace.id, Document.path.is_(None)) | |
| .group_by(Document.source_id) | |
| ).all()) | |
| return [serialize_source(s, n, locked.get(s.id, 0)) for s, n in rows] | |
| def sync_source(source_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): | |
| source = _get(ctx, source_id) | |
| if source.feed_url == IMPORTS_FEED: | |
| raise HTTPException(400, "Imported posts have no feed to sync.") | |
| job = _enqueue_sync(ctx, source) | |
| return {"source": serialize_source(source, _doc_count(ctx, source)), "job": serialize_job(job)} | |
| def remove_documents(ctx: Ctx, docs: list[Document]) -> None: | |
| """Delete documents plus their corpus files and raw HTML, then rebuild INDEX.md.""" | |
| if not docs: | |
| return | |
| paths = {d.path for d in docs if d.path} | |
| for d in docs: | |
| if d.raw_html_key: | |
| storage.delete_prefix(d.raw_html_key.rsplit("/", 1)[0]) | |
| ids = [d.id for d in docs] | |
| ctx.db.execute(delete(NotebookDocument).where(NotebookDocument.document_id.in_(ids))) | |
| ctx.db.execute(delete(Document).where(Document.id.in_(ids))) | |
| ctx.db.flush() | |
| Corpus(ctx.workspace.id).write_files({INDEX: rebuild_index(ctx.db, ctx.workspace.id)}, remove=paths) | |
| def delete_source(source_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): | |
| source = _get(ctx, source_id) | |
| docs = ctx.db.scalars(select(Document).where(Document.source_id == source.id)).all() | |
| remove_documents(ctx, list(docs)) | |
| ctx.db.delete(source) | |
| ctx.db.commit() | |
| return {"ok": True, "removed_posts": len(docs)} | |
| def _require_room(ctx: Ctx) -> None: | |
| """A link or a file adds a post: refused (with the upgrade prompt) once the plan's posts are all indexed.""" | |
| plan = effective_plan(ctx.db, ctx.workspace) | |
| if post_room(ctx.db, ctx.workspace.id, plan) <= 0: | |
| raise plan_limit_error(plan, "indexed_posts", f"Your plan indexes up to {plan.indexed_posts} posts, and they are all " | |
| "used. Remove a post or upgrade to add more.") | |
| def import_url(body: SourceIn, ctx: Ctx = Depends(get_ctx)): | |
| _require_room(ctx) | |
| source = imports_source(ctx.db, ctx.workspace.id) | |
| ctx.db.commit() | |
| job = create_job(ctx.db, ctx.workspace.id, "import_url", {"source_id": str(source.id), "url": body.url}) | |
| return {"source": serialize_source(source, _doc_count(ctx, source)), "job": serialize_job(job)} | |
| def import_upload(body: UploadImportIn, ctx: Ctx = Depends(get_ctx)): | |
| upload = ctx.db.scalar(select(Upload).where(Upload.id == body.upload_id, Upload.workspace_id == ctx.workspace.id)) | |
| if not upload or upload.status != "complete": | |
| raise HTTPException(404, "Upload not found or not finished") | |
| _require_room(ctx) | |
| source = imports_source(ctx.db, ctx.workspace.id) | |
| ctx.db.commit() | |
| job = create_job(ctx.db, ctx.workspace.id, "import_upload", | |
| {"source_id": str(source.id), "upload_id": str(upload.id)}) | |
| return {"source": serialize_source(source, _doc_count(ctx, source)), "job": serialize_job(job)} | |
| documents_router = APIRouter(prefix="/api/documents", tags=["documents"]) | |
| def is_locked(d: Document) -> bool: | |
| """Past the plan's post limit (see ingest.store_locked): listed with title and link only, not indexed.""" | |
| return not d.path and bool((d.metadata_json or {}).get("locked")) | |
| def serialize_document(d: Document, source: Source | None = None) -> dict: | |
| meta = d.metadata_json or {} | |
| return { | |
| "id": str(d.id), | |
| "title": d.title, | |
| "url": d.url, | |
| "path": d.path, | |
| "source_id": str(d.source_id) if d.source_id else None, | |
| "source_title": source.title if source else None, | |
| "published_at": d.published_at.isoformat() if d.published_at else None, | |
| "words": meta.get("words") or len((d.clean_text or "").split()), | |
| "evergreen_score": d.evergreen_score, | |
| "locked": is_locked(d), | |
| } | |
| def list_documents( | |
| ctx: Ctx = Depends(get_ctx), | |
| q: str = "", | |
| source_id: uuid.UUID | None = None, | |
| indexed_only: bool = False, | |
| limit: int = 500, | |
| offset: int = 0, | |
| ): | |
| query = ( | |
| select(Document, Source) | |
| .outerjoin(Source, Source.id == Document.source_id) | |
| .where(Document.workspace_id == ctx.workspace.id) | |
| ) | |
| if q.strip(): | |
| like = f"%{q.strip()}%" | |
| query = query.where(or_(Document.title.ilike(like), Document.clean_text.ilike(like))) | |
| if source_id: | |
| query = query.where(Document.source_id == source_id) | |
| if indexed_only: | |
| query = query.where(Document.path.is_not(None)) | |
| total = ctx.db.scalar(select(func.count()).select_from(query.subquery())) | |
| rows = ctx.db.execute( | |
| query.order_by(Document.published_at.desc().nulls_last(), Document.created_at.desc()) | |
| .offset(max(offset, 0)).limit(min(limit, 5000)) | |
| ).all() | |
| return {"total": total, "items": [serialize_document(d, s) for d, s in rows]} | |
| def get_document(document_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): | |
| doc = ctx.db.scalar(select(Document).where(Document.id == document_id, Document.workspace_id == ctx.workspace.id)) | |
| if not doc: | |
| raise HTTPException(404, "Post not found") | |
| source = ctx.db.get(Source, doc.source_id) if doc.source_id else None | |
| if is_locked(doc): | |
| # Listed, not indexed: there is no text to show until the workspace upgrades. | |
| plan = effective_plan(ctx.db, ctx.workspace) | |
| return {**serialize_document(doc, source), "lines": [f"# {doc.title}"], | |
| "preview": True, "limit": plan.indexed_posts} | |
| lines: list[str] = [] | |
| if doc.path: | |
| corpus = Corpus(ctx.workspace.id) | |
| try: | |
| if not corpus.exists(doc.path): | |
| corpus.sync() | |
| lines = corpus.read_lines(doc.path) | |
| except CorpusError: | |
| lines = [] | |
| if not lines: | |
| lines = [f"# {doc.title}", "", *(doc.clean_text or "").splitlines()] | |
| return {**serialize_document(doc, source), "lines": lines} | |
| def delete_document(document_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): | |
| doc = ctx.db.scalar(select(Document).where(Document.id == document_id, Document.workspace_id == ctx.workspace.id)) | |
| if not doc: | |
| raise HTTPException(404, "Post not found") | |
| remove_documents(ctx, [doc]) | |
| ctx.db.commit() | |
| return {"ok": True} | |