diff --git a/README.md b/README.md index 9e9be6e1a92918ba8a814478725627bd99a93d38..f8008014b85acb5e232f8047336d2eee00f603d3 100644 --- a/README.md +++ b/README.md @@ -93,9 +93,19 @@ cd renderer && npm install && npm start # only needed for vid - With no R2 settings, files live on local disk under `backend/.storage` and are served through signed `/api/storage/local/...` URLs (range requests supported for audio and video seeking). - Audio needs `ELEVENLABS_API_KEY`; everything text based needs `LLM_API_KEY`. -- Launchpad: Bluesky connects with an app password. X and LinkedIn need developer apps; the - redirect URIs are listed in `.env.example` and on the Launchpad page. Substack Notes has no API, - so those posts (and any post without a connected account) arrive as an email reminder at send time. +- Launchpad: posts (text, images, videos from your Library; not audio) go to X and LinkedIn at their + scheduled time through a `publish_post` job. Set up the developer apps and keys as described in + `backend/.env.example`: + - `X_CLIENT_ID` / `X_CLIENT_SECRET`: the X app's **OAuth 2.0 Client ID and Client Secret** (Read and write, + Web App, callback `{API_URL}/api/social/x/callback`, scopes incl. `media.write`). + - `LINKEDIN_CLIENT_ID` / `LINKEDIN_CLIENT_SECRET`: products "Sign In with LinkedIn using OpenID Connect" and + "Share on LinkedIn", redirect `{API_URL}/api/social/linkedin/callback`. + - `API_URL` must be reachable from the internet for the callbacks (an https tunnel locally). + X and LinkedIn posts need a connected account. X tokens are refreshed automatically; LinkedIn connections + last about 60 days. If a connection goes down (expired, revoked on the platform, or disconnected here) its + scheduled posts are **paused** and never attempted; reconnecting the same account resumes the ones still + ahead, and missed ones wait for a new time. Bluesky connects with an app password; Substack Notes has no API, + so those posts arrive as an email reminder at send time. ## What is in the app diff --git a/backend/.env.example b/backend/.env.example new file mode 100644 index 0000000000000000000000000000000000000000..8167bdf3c3710e58d528459d3439773be45cfef4 --- /dev/null +++ b/backend/.env.example @@ -0,0 +1,34 @@ +# Launchpad: posting to X and LinkedIn (copy what you need into .env) + +# Where the app and the API are reached. Both platforms send people back to +# {API_URL}/api/social/{x,linkedin}/callback after they connect an account, so API_URL must be reachable from the +# internet (when running locally, an https tunnel such as ngrok: API_URL=https://.ngrok-free.app). +FRONTEND_URL=http://localhost:5173 +API_URL=http://localhost:8000 + +# X: developer.x.com > your app > Keys and tokens > "OAuth 2.0 Client ID and Client Secret" +# (not the API Key / Bearer Token). Then User authentication settings: +# App permissions: Read and write. Type of app: Web App (confidential client). +# Callback URI: {API_URL}/api/social/x/callback +# Scopes asked for: tweet.read tweet.write users.read offline.access media.write +# (media.write posts images and videos; accounts connected before it was added are asked to reconnect) +# X is switched off for now (LinkedIn only). Set to true to bring it back. +X_ENABLED=false +X_CLIENT_ID= +X_CLIENT_SECRET= + +# LinkedIn: linkedin.com/developers > your app > Auth > "Client ID" and "Primary Client Secret" +# Products: "Sign In with LinkedIn using OpenID Connect" and "Share on LinkedIn". +# Auth > Authorized redirect URLs: {API_URL}/api/social/linkedin/callback +# Scopes asked for: openid profile w_member_social +# LinkedIn tokens last about 60 days; then the account shows Reconnect. +LINKEDIN_CLIENT_ID= +LINKEDIN_CLIENT_SECRET= + +# Connections: X tokens are refreshed automatically (every 10 minutes the worker checks them, and every 6 hours asks +# each platform whether the connection still works). When a connection goes down, its scheduled posts are paused and +# never attempted; reconnecting the same account resumes the ones still ahead. + +# Encrypts the stored social tokens (a Fernet key). Empty: derived from JWT_SECRET. +# Make one with: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" +SOCIAL_TOKEN_KEY= diff --git a/backend/alembic/versions/0020_calendar_kit.py b/backend/alembic/versions/0020_calendar_kit.py new file mode 100644 index 0000000000000000000000000000000000000000..c8ebce7cdf46a6b90f20dbd2b6d92ca8af4953be --- /dev/null +++ b/backend/alembic/versions/0020_calendar_kit.py @@ -0,0 +1,35 @@ +"""calendar_items.kit_id: the Launch Kit a scheduled post was written from, kept apart from its attachment + +Revision ID: 0020_calendar_kit +Revises: 0019_infographics_limit +Create Date: 2026-10-06 +""" +import sqlalchemy as sa +from alembic import op + +revision = "0020_calendar_kit" +down_revision = "0019_infographics_limit" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # 0001 builds from the live models, so a fresh database may already have the column. + if "kit_id" not in {c["name"] for c in sa.inspect(op.get_bind()).get_columns("calendar_items")}: + with op.batch_alter_table("calendar_items") as batch: + batch.add_column(sa.Column("kit_id", sa.Uuid(), nullable=True)) + batch.create_foreign_key("fk_calendar_items_kit_id", "artifacts", ["kit_id"], ["id"], ondelete="SET NULL") + # Posts that carried a Launch Kit as their "attachment": the kit is where they came from, not an attachment. + op.execute("UPDATE calendar_items SET kit_id = artifact_id, artifact_id = NULL " + "WHERE artifact_id IN (SELECT id FROM artifacts WHERE type = 'launch_kit')") + + +def downgrade() -> None: + op.execute("UPDATE calendar_items SET artifact_id = kit_id WHERE artifact_id IS NULL AND kit_id IS NOT NULL") + # The foreign key is named here, or by the database when 0001 made the column from the models. + fks = [fk["name"] for fk in sa.inspect(op.get_bind()).get_foreign_keys("calendar_items") + if fk["constrained_columns"] == ["kit_id"] and fk.get("name")] + with op.batch_alter_table("calendar_items") as batch: + for name in fks: + batch.drop_constraint(name, type_="foreignkey") + batch.drop_column("kit_id") diff --git a/backend/app/config.py b/backend/app/config.py index 6e4b1aa3a715f00247920bddc88ffcf565fde116..7a6efdadb21e4511840402e354a0450b8e79ace7 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -62,7 +62,7 @@ class Settings(BaseSettings): # Folder inside the bucket, so Notestack can share a bucket with other apps. Empty = bucket root. r2_prefix: str = "" r2_presign_ttl_seconds: int = 3600 - max_upload_bytes: int = 200 * 1024 * 1024 + max_upload_bytes: int = 300 * 1024 * 1024 # LLM (LiteLLM model strings; Z.ai GLM by default) llm_model: str = "openai/glm-5.3" @@ -117,6 +117,7 @@ class Settings(BaseSettings): # Social publishing (Launchpad). Redirect URIs: {API_URL}/api/social/{x,linkedin}/callback social_token_key: str = "" # Fernet key; derived from JWT_SECRET when empty + x_enabled: bool = False # X posting is switched off for now (LinkedIn only); set X_ENABLED=true to bring it back x_client_id: str = "" x_client_secret: str = "" linkedin_client_id: str = "" diff --git a/backend/app/infographics/image.py b/backend/app/infographics/image.py index 3e5ee9b37342b705c3a99e61e7fd19098b9ffb77..5f6d15b6c9e563164cec50469ce03d588b830a19 100644 --- a/backend/app/infographics/image.py +++ b/backend/app/infographics/image.py @@ -32,7 +32,8 @@ def chrome() -> str | None: if found := shutil.which(name): return found patterns = ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", - os.path.expanduser("~/.cache/puppeteer/chrome/*/chrome-mac-*/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing")] + os.path.expanduser("~/.cache/puppeteer/chrome/*/chrome-mac-*/" + "Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing")] return next((p for pat in patterns for p in glob.glob(pat)), None) diff --git a/backend/app/infographics/render.py b/backend/app/infographics/render.py index 2684f2471b301a7b3fee3ddfe60743e2cbf9db0c..8158b10d9c99bb50091b1b1b09eb7072d027f407 100644 --- a/backend/app/infographics/render.py +++ b/backend/app/infographics/render.py @@ -5,8 +5,8 @@ import hashlib import random from pathlib import Path -from markupsafe import Markup from jinja2 import Environment, FileSystemLoader, StrictUndefined, select_autoescape +from markupsafe import Markup from app.infographics.content import HEIGHT, WIDTH from app.infographics.themes import DEFAULT_THEME, THEMES, theme_id diff --git a/backend/app/llm/signatures.py b/backend/app/llm/signatures.py index cef6051faf4843c9d828bc4aa63a2e140af6708f..3fe39877242902d36e3956768309913fd4a70a66 100644 --- a/backend/app/llm/signatures.py +++ b/backend/app/llm/signatures.py @@ -451,8 +451,12 @@ class WriteReport(dspy.Signature): "Section 2" or "Details"), and they read in a logical order. Use "### " inside sections, lists, and tables where they help. When the material has numbered lines (FILE blocks), put a [n] marker after each claim it supports and list each n in citations with the path and lines it came from; when it is a chat transcript, - write no markers and no citations. Do not write a quiz, flashcards, practice or review questions into the report unless the reader's instructions - ask for one: those are offered to the reader separately as visuals they can add. Never write about visuals, mind maps, flashcards, quizzes, tables or timelines in the text, and never suggest or describe one, even if the reader's instructions mention them: the app offers those to the reader separately, so write only the report itself. Write in the language requested. + write no markers and no citations. Do not write a quiz, flashcards, practice or review questions into the report + unless the reader's instructions + ask for one: those are offered to the reader separately as visuals they can add. Never write about visuals, mind + maps, flashcards, quizzes, tables or timelines in the text, and never suggest or describe one, even if the + reader's instructions mention them: the app offers those to the reader separately, so write only the report + itself. Write in the language requested. Never use em dashes.""" title: str = dspy.InputField(desc="The notebook, post or chat the material comes from") @@ -483,7 +487,8 @@ class PlanInteractiveReport(dspy.Signature): section that is itself a quiz, flashcards, practice questions or a mind map, because those are visuals, and are suggested through a section's visual instead. Choose 4 to 8 sections in a sensible order, starting with an Introduction and ending with a Summary unless the reader asks for something else. Each - heading is specific and descriptive of what the section covers, never generic like "Details" or "Section 2". Then decide, section by section, where a visual would help the reader and which one. These are only + heading is specific and descriptive of what the section covers, never generic like "Details" or "Section 2". + Then decide, section by section, where a visual would help the reader and which one. These are only suggestions: nothing is built until the reader clicks Add, so suggest the spots where each really fits: a mind_map to show how many linked ideas fit together (at most one, usually near the start or the end), flashcards to learn a set of terms or facts, a quiz to check understanding after a dense section, an infographic for a one page picture of the @@ -502,11 +507,15 @@ class PlanInteractiveReport(dspy.Signature): class WriteReportSection(dspy.Signature): """Write one section of a report, using only the material given. Follow the reader's instructions on style, tone - and length; a section is two to five short paragraphs, or a list or table where that reads better. Do not write quiz questions, flashcards or + and length; a section is two to five short paragraphs, or a list or table where that reads better. Do not write + quiz questions, flashcards or review questions in the section (those are offered to the reader separately). Do not repeat the heading, and do not use "#" or "##" headings; "###" subheadings are fine. When the material has numbered lines (FILE blocks), put a [n] marker after each claim it supports and list each n in citations with the path and lines; when it - is a chat transcript, write no markers and no citations. Never write about visuals, mind maps, flashcards, quizzes, tables or timelines in the text, and never suggest or describe one, even if the reader's instructions mention them: the app offers those to the reader separately, so write only the report itself. Write in the language requested. Never use em dashes.""" + is a chat transcript, write no markers and no citations. Never write about visuals, mind maps, flashcards, + quizzes, tables or timelines in the text, and never suggest or describe one, even if the reader's instructions + mention them: the app offers those to the reader separately, so write only the report itself. Write in the + language requested. Never use em dashes.""" title: str = dspy.InputField() report_title: str = dspy.InputField() diff --git a/backend/app/models/content.py b/backend/app/models/content.py index 6c784e66564fae79776efac4a954c641f223aa30..d68e1837abb67774d95566e49e95a8ad44a37d97 100644 --- a/backend/app/models/content.py +++ b/backend/app/models/content.py @@ -1,3 +1,4 @@ +import logging import uuid from datetime import datetime @@ -14,10 +15,11 @@ from sqlalchemy import ( Text, UniqueConstraint, Uuid, + event, text, true, ) -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, Session, mapped_column, relationship from app.db import Base from app.models.base import IdMixin, TimestampMixin @@ -343,11 +345,46 @@ class Upload(IdMixin, TimestampMixin, Base): status: Mapped[str] = mapped_column(String(20), default="pending") # pending | complete +# Deleting an upload record removes its file from storage too, once the delete is committed (never on a rollback). +# This covers deletes made through the ORM; rows removed straight in the database are caught by the daily orphan +# sweep (app.services.uploads.sweep_orphan_uploads). +_PENDING_FILE_DELETES = "upload_files_to_delete" + + +@event.listens_for(Session, "after_flush") +def _note_deleted_uploads(session: Session, _ctx) -> None: + keys = [obj.key for obj in session.deleted if isinstance(obj, Upload) and obj.key] + if keys: + session.info.setdefault(_PENDING_FILE_DELETES, []).extend(keys) + + +@event.listens_for(Session, "after_commit") +def _delete_upload_files(session: Session) -> None: + keys = session.info.pop(_PENDING_FILE_DELETES, []) + if not keys: + return + from app.services.storage import storage + + for key in keys: + try: + storage.delete(key) + except Exception: # left for the daily sweep + logging.getLogger("notestack.storage").warning("couldn't delete upload file %s", key, exc_info=True) + + +@event.listens_for(Session, "after_rollback") +def _forget_upload_files(session: Session) -> None: + session.info.pop(_PENDING_FILE_DELETES, None) + + class CalendarItem(IdMixin, TimestampMixin, Base): __tablename__ = "calendar_items" workspace_id: Mapped[uuid.UUID] = _ws_fk() + # What the post carries (a video, quote card, carousel...): never a Launch Kit, which is kit_id. artifact_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, ForeignKey("artifacts.id", ondelete="CASCADE")) + # The Launch Kit the post was written from, kept apart from the attachment (nothing of it is uploaded). + kit_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, ForeignKey("artifacts.id", ondelete="SET NULL")) document_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, ForeignKey("documents.id", ondelete="SET NULL")) platform: Mapped[str] = mapped_column(String(30)) # x | linkedin | bluesky | substack_notes scheduled_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True) diff --git a/backend/app/pipeline/launchkit.py b/backend/app/pipeline/launchkit.py index 119358651446a3b6d5d3845e9e531cd0bb386b1d..36092ca75762319319d84e0e3a9ba63302c3508f 100644 --- a/backend/app/pipeline/launchkit.py +++ b/backend/app/pipeline/launchkit.py @@ -1,16 +1,22 @@ """Launch Kit: one post in, everything needed to launch it out, in the writer's voice and grounded in the post's own claims.""" +import logging + from sqlalchemy.orm import Session +from app.config import settings from app.corpus import INDEX, Corpus from app.llm import run from app.llm.signatures import CarouselSlides, ExtractClaims, HookGenerator, PickQuotes, PlatformAdapter, SeoPack from app.models import Artifact, Document, Job from app.pipeline.generate import NothingToDo from app.pipeline.passages import passages_for, tools_for, verify_refs, voice_text +from app.services import renderer from app.services.jobs import create_job, update_job +log = logging.getLogger(__name__) + PLATFORMS = { "x_thread": ("X thread", "5 to 9 posts, each under 270 characters. Post 1 is the hook and stands alone. " "No hashtags except at most one at the end. The last post links to the article with {url}."), @@ -94,8 +100,13 @@ def build_launch_kit(db: Session, job: Job, artifact: Artifact) -> dict: artifact.status = "ready" db.commit() - # Quote cards render as their own artifacts, so a missing renderer does not sink the kit. + # Quote cards render as their own artifacts, so a missing renderer does not sink the kit. With the renderer + # down they would only fail, so skip them: the kit keeps its quotes as text either way. card_ids = [] + if quotes and not renderer.available(): + log.warning("Renderer unreachable at %s: launch kit %s made without quote cards", + settings.renderer_url, artifact.id) + quotes = [] for q in quotes: card = Artifact(workspace_id=ws, document_id=doc.id, type="quote_card", status="pending", content_json={"title": f"Quote: {doc.title}", "parent_id": str(artifact.id), **q}) diff --git a/backend/app/pipeline/report.py b/backend/app/pipeline/report.py index fcb71e18a7c22804340d4685a1c01decd102d5fb..9b8bdf12b882b0f10b0502d37afd11706252204b 100644 --- a/backend/app/pipeline/report.py +++ b/backend/app/pipeline/report.py @@ -10,6 +10,7 @@ import uuid from sqlalchemy.orm import Session +from app.infographics.themes import theme_id from app.llm import run from app.llm.provider import fast_lm, report_lm from app.llm.signatures import ( @@ -22,10 +23,9 @@ from app.models import Artifact, Job from app.pipeline.flashcards import make_cards from app.pipeline.generate import NothingToDo, mind_map_content from app.pipeline.material import Material, load_material -from app.pipeline.quiz import make_questions from app.pipeline.passages import as_citations +from app.pipeline.quiz import make_questions from app.pipeline.research import verify_citations -from app.infographics.themes import theme_id from app.services.artifacts import start_artifact from app.services.jobs import update_job from app.services.renderer import PermanentJobError @@ -319,7 +319,8 @@ def make_block(db: Session, job: Job, report: Artifact, kind: str, brief: str, t if kind == "infographic": # A picture is drawn by the renderer, so it is its own infographic (listed in the Library, counted in the # plan) that the block points at; the report page shows it once it is ready. - child, _ = start_artifact(db, report.workspace_id, "infographic", title=f"Infographic: {brief[:60] or report.content_json.get('title', '')}", + title = f"Infographic: {brief[:60] or report.content_json.get('title', '')}" + child, _ = start_artifact(db, report.workspace_id, "infographic", title=title, notebook_id=report.notebook_id, params={**_source_params(source), "theme": theme_id(theme), "instructions": brief[:600]}) return {"title": f"Infographic: {brief[:60]}", "artifact_id": str(child.id), "theme": theme_id(theme)} diff --git a/backend/app/routers/artifacts.py b/backend/app/routers/artifacts.py index df4193513641cfaacc08d7818938dc975ddf9b5d..a3f0a82e246ddc0a57257dc1109dc18f5d05d1e0 100644 --- a/backend/app/routers/artifacts.py +++ b/backend/app/routers/artifacts.py @@ -8,20 +8,21 @@ from sqlalchemy import func, or_, select from app.auth import Ctx, get_ctx from app.config import settings +from app.infographics.image import ImageUnavailable, render_png +from app.infographics.themes import theme_id from app.models import Artifact, CalendarItem, Chat, Document, Job, Notebook, UserSavedVoice from app.routers.notebooks import ensure_archive_notebook -from app.infographics.image import ImageUnavailable, render_png from app.services.artifacts import latest_jobs, serialize_artifact, start_artifact from app.services.jobs import create_job, serialize_job from app.services.plans import effective_plan, plan_limit_error -from app.infographics.themes import theme_id from app.services.renderer import COMPOSITIONS from app.services.storage import storage from app.services.usage import check_limit router = APIRouter(prefix="/api/artifacts", tags=["artifacts"]) -ArtifactType = Literal["summary", "audio_overview", "video", "quote_card", "carousel", "launch_kit", "mind_map", "quiz", "flashcards", "report", "infographic"] +ArtifactType = Literal["summary", "audio_overview", "video", "quote_card", "carousel", "launch_kit", "mind_map", "quiz", + "flashcards", "report", "infographic"] SOURCE_TYPES = ("quiz", "flashcards", "report", "infographic") # made from posts or chats picked in the dialog @@ -33,7 +34,7 @@ class RenderIn(BaseModel): class GenerateIn(BaseModel): - type: Literal["summary", "audio_overview", "video", "quote_card", "carousel", "launch_kit", "mind_map", "quiz", "flashcards", "report", "infographic"] + type: ArtifactType notebook_id: uuid.UUID | None = None document_id: uuid.UUID | None = None archive: bool = False # no notebook or post picked: use the "All posts" notebook @@ -213,6 +214,8 @@ def list_artifacts( query = select(Artifact).where(Artifact.workspace_id == ctx.workspace.id) if type: query = query.where(Artifact.type.in_(type.split(","))) + else: # files uploaded to post live in the Launchpad's picker, not among the things Notestack made + query = query.where(Artifact.type != "upload") if notebook_id: query = query.where(Artifact.notebook_id == notebook_id) if document_id: @@ -260,9 +263,13 @@ def patch_artifact(artifact_id: uuid.UUID, body: PatchIn, ctx: Ctx = Depends(get content = dict(a.content_json or {}) if body.content is not None: protected = {"slide_keys", "quote_card_ids", "segments", "voices"} + if a.type == "upload": # what the file is, as posting reads it: only its title can change + protected |= {"filename", "content_type", "media", "size_bytes", "duration_s"} content.update({k: v for k, v in body.content.items() if k not in protected}) if body.title is not None: - content["title"] = body.title + if not body.title.strip(): + raise HTTPException(400, "Give it a name.") + content["title"] = body.title.strip() a.content_json = content ctx.db.commit() return serialize_artifact(a, latest_jobs(ctx.db, [a.id]).get(a.id)) @@ -277,6 +284,10 @@ def delete_artifact(artifact_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): remove_video(ctx, a) return {"ok": True} storage.delete_prefix(f"ws/{ctx.workspace.id}/artifacts/{a.id}/") + if a.type == "upload": # a file from the user's computer: its upload record (and file) go too + from app.services.uploads import remove_upload_artifact + + remove_upload_artifact(ctx.db, a) ctx.db.query(CalendarItem).filter(CalendarItem.artifact_id == a.id, CalendarItem.status == "scheduled").delete() ctx.db.delete(a) ctx.db.commit() diff --git a/backend/app/routers/auth.py b/backend/app/routers/auth.py index c78b0a7b7ca8f8a21b718cd4c074eb15a771142c..e6edce56d976b18172227067fda3fdf596b5e607 100644 --- a/backend/app/routers/auth.py +++ b/backend/app/routers/auth.py @@ -36,6 +36,15 @@ def _err(exc: ident.IdentityError | ev.VerificationError, status: int = 400) -> return HTTPException(getattr(exc, "status", status), {"code": exc.code, "message": exc.message}) +def _sent_or_fail(db: Session, sent: bool, email: str, purpose: VerificationPurpose) -> None: + """A code the user never receives is useless: drop it and say so instead of reporting success.""" + if sent: + return + ev.clear_codes(db, email, purpose) + db.commit() + raise HTTPException(502, {"code": "email_failed", "message": "We couldn't send the code. Try again in a moment."}) + + def _serialize_user(user: User) -> dict: return { "id": str(user.id), @@ -279,7 +288,7 @@ def register_start(body: RegisterStartIn, db: Session = Depends(get_db)): except (ident.IdentityError, ev.VerificationError) as exc: raise _err(exc, 429 if getattr(exc, "code", "") == "cooldown" else 400) from exc db.commit() - email_service.send_verification_code(email, issued.code) + _sent_or_fail(db, email_service.send_verification_code(email, issued.code), email, VerificationPurpose.SIGNUP) return {"ok": True} @@ -314,7 +323,7 @@ def register_resend(body: EmailIn, db: Session = Depends(get_db)): except ev.VerificationError as exc: raise _err(exc, 429) from exc db.commit() - email_service.send_verification_code(email, issued.code) + _sent_or_fail(db, email_service.send_verification_code(email, issued.code), email, VerificationPurpose.SIGNUP) return {"ok": True} @@ -338,14 +347,19 @@ def email_login(body: LoginIn, db: Session = Depends(get_db)): @router.post("/password/forgot/start") def forgot_start(body: EmailIn, db: Session = Depends(get_db)): user = ident.find_user(db, body.email) - # Always 200 so the endpoint does not reveal which emails exist. - if user and user.auth_provider == AuthProvider.EMAIL and user.is_active: - try: - issued = ev.issue_code(db, user.email, VerificationPurpose.PASSWORD_RESET) - db.commit() - email_service.send_password_reset_code(user.email, issued.code) - except ev.VerificationError: - pass + # Sign up already tells whether an email is taken, so naming the problem here leaks nothing new. + if not user or user.deleted_at or not user.is_active: + raise HTTPException(404, {"code": "not_found", "message": "No account found with this email. Create one instead."}) + if user.auth_provider != AuthProvider.EMAIL: + raise _err(ident.wrong_provider_error(user)) + try: + issued = ev.issue_code(db, user.email, VerificationPurpose.PASSWORD_RESET) + except ev.VerificationError as exc: + raise _err(exc, 429) from exc + db.commit() + _sent_or_fail( + db, email_service.send_password_reset_code(user.email, issued.code), user.email, VerificationPurpose.PASSWORD_RESET + ) return {"ok": True} diff --git a/backend/app/routers/launchpad.py b/backend/app/routers/launchpad.py index 980b6caa8e55d029f9f0d55aa815df649bb089b1..bfb63008e0f0f3b7237207bd063c020a2e3ced5f 100644 --- a/backend/app/routers/launchpad.py +++ b/backend/app/routers/launchpad.py @@ -1,11 +1,13 @@ """Launchpad: the calendar, connected social accounts (OAuth / app password) and tracked short links.""" import json +import re import secrets import uuid from datetime import UTC, datetime, timedelta from typing import Literal from urllib.parse import urlencode +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.concurrency import run_in_threadpool @@ -17,10 +19,14 @@ from sqlalchemy.orm import Session from app.auth import Ctx, decode_token_full, encode_signed, get_ctx from app.config import settings from app.db import get_db -from app.models import Artifact, CalendarItem, Document, EngagementEvent, SocialAccount, TrackedLink +from app.models import Artifact, CalendarItem, Document, EngagementEvent, SocialAccount, TrackedLink, Upload from app.services import launchpad as lp +from app.services.artifacts import TYPE_LABELS from app.services.crypto import decrypt, encrypt from app.services.social import AUTO_POST, LIMITS, PLATFORM_LABELS, SocialError, bluesky, linkedin, x +from app.services.social.health import pause_account_posts, resume_account_posts +from app.services.social.media import MB, SIZE_LIMITS, UPLOAD_TYPES, check_postable, duration, is_blog2video, media_count, upload_kind +from app.services.storage import storage router = APIRouter(prefix="/api", tags=["launchpad"]) @@ -30,32 +36,104 @@ Platform = Literal["x", "linkedin", "bluesky", "substack_notes"] # Calendar -class ItemIn(BaseModel): +class When(BaseModel): + """When a post goes out: the wall time the user picked in their own time zone (local_time "2026-10-07T11:00" + + timezone "Asia/Karachi", on a half hour), converted to UTC here; or an exact scheduled_at (Post now).""" + scheduled_at: datetime | None = None + local_time: str | None = Field(None, max_length=32) + timezone: str | None = Field(None, max_length=64) + + def utc(self) -> datetime | None: + if self.local_time: + try: + zone = ZoneInfo(self.timezone or "") + except (ZoneInfoNotFoundError, ValueError) as exc: + raise HTTPException(400, {"code": "bad_timezone", "message": "Unknown time zone."}) from exc + try: + wall = datetime.fromisoformat(self.local_time) + except ValueError as exc: + raise HTTPException(400, {"code": "bad_time", "message": "Pick a date and time."}) from exc + if wall.tzinfo or wall.minute not in (0, 30) or wall.second or wall.microsecond: + raise HTTPException(400, {"code": "half_hour_only", "message": "Pick a time on the hour or half hour."}) + return wall.replace(tzinfo=zone).astimezone(UTC) + return _aware(self.scheduled_at) if self.scheduled_at else None + + +class ItemIn(When): platform: Platform - scheduled_at: datetime content: str = Field(min_length=1, max_length=10000) thread: list[str] = Field(default_factory=list, max_length=25) - artifact_id: uuid.UUID | None = None + artifact_id: uuid.UUID | None = None # the attachment (a video, quote card, carousel...), never a Launch Kit + kit_id: uuid.UUID | None = None # the Launch Kit the post was written from (tracked, not attached) document_id: uuid.UUID | None = None social_account_id: uuid.UUID | None = None remind_by_email: bool = False draft: bool = False -class ItemPatch(BaseModel): - scheduled_at: datetime | None = None +class ItemPatch(When): + artifact_id: uuid.UUID | None = None # set or (null) clear what the post carries content: str | None = Field(None, min_length=1, max_length=10000) thread: list[str] | None = None social_account_id: uuid.UUID | None = None remind_by_email: bool | None = None - status: Literal["scheduled", "draft"] | None = None + status: Literal["scheduled", "draft", "paused"] | None = None def _aware(dt: datetime) -> datetime: return dt if dt.tzinfo else dt.replace(tzinfo=UTC) -def serialize_item(item: CalendarItem, account: SocialAccount | None = None, clicks: int = 0) -> dict: +# audio can't be posted; "upload" is a file the user brought from their computer +POSTABLE = ("video", "quote_card", "carousel", "summary", "launch_kit", "mind_map", "upload") + + +def _cite_free(text: str) -> str: + return re.sub(r"\s*\[\d+\]", "", text or "").strip() + + +def _prefill(a: Artifact) -> dict: + """A starting text per platform for a post of this artifact: X as a list of posts (a thread), LinkedIn as one.""" + c = a.content_json or {} + title = c.get("title") or TYPE_LABELS.get(a.type, a.type) + if a.type == "upload": # their own file: the words are theirs to write + return {"x": [""], "linkedin": ""} + if a.type == "launch_kit": + posts = c.get("posts") or {} + x_posts = posts.get("x_thread") or posts.get("x") or [title] + return {"x": x_posts, "linkedin": "\n\n".join(posts.get("linkedin") or x_posts)} + text = { + "summary": _cite_free(c.get("summary") or title), + "quote_card": f'"{c.get("quote")}"' + (f" ({c['source']})" if c.get("source") else "") if c.get("quote") + else title, + "carousel": ((c.get("slides") or [{}])[0].get("heading") or title), + }.get(a.type, title) + return {"x": [text], "linkedin": text} + + +def artifact_brief(a: Artifact) -> dict: + """What the composer and the calendar show of the attached artifact.""" + c = a.content_json or {} + thumb = (a.storage_key if a.type == "quote_card" or upload_kind(a) == "image" + else (c.get("slide_keys") or [a.storage_key])[0] if a.type == "carousel" else None) + return {"id": str(a.id), "type": a.type, "type_label": TYPE_LABELS.get(a.type, a.type), + "title": c.get("title") or TYPE_LABELS.get(a.type, a.type), + "media": upload_kind(a) or ("video" if a.type == "video" else "image" if media_count(a) else "text"), + "media_count": media_count(a), "duration_s": duration(a), + "thumb_url": storage.presign_get(thumb) if thumb else None, + # The user's own file, to preview before posting it. + "view_url": storage.presign_get(a.storage_key) if a.type == "upload" and a.storage_key else None, + # A blog2video video with scenes but no MP4 yet: listed, but it has to be rendered before it can go out. + # Made by the video service (blog2video): it opens in the video editor. + "editable_video": is_blog2video(a), + "needs_render": is_blog2video(a) and not (a.content_json or {}).get("video_url"), + # Its MP4 is being made right now: it can't be scheduled until the render finishes. + "rendering": is_blog2video(a) and a.status == "rendering", + "created_at": a.created_at.isoformat() if a.created_at else None, "prefill": _prefill(a)} + + +def serialize_item(item: CalendarItem, account: SocialAccount | None = None, clicks: int = 0, + artifact: Artifact | None = None, kit: Artifact | None = None) -> dict: return { "id": str(item.id), "platform": item.platform, @@ -65,6 +143,9 @@ def serialize_item(item: CalendarItem, account: SocialAccount | None = None, cli "content": item.content, "thread": item.thread or [], "artifact_id": str(item.artifact_id) if item.artifact_id else None, + "artifact": artifact_brief(artifact) if artifact else None, + "kit_id": str(item.kit_id) if item.kit_id else None, + "kit_title": ((kit.content_json or {}).get("post_title") or (kit.content_json or {}).get("title")) if kit else None, "document_id": str(item.document_id) if item.document_id else None, "social_account_id": str(item.social_account_id) if item.social_account_id else None, "account_handle": account.handle if account else None, @@ -96,6 +177,26 @@ def _account(ctx: Ctx, account_id: uuid.UUID | None, platform: str) -> SocialAcc return account +CONNECTED_ONLY = ("x", "linkedin") # these post only through a live connection (no email fallback) + + +def can_post_media(a: SocialAccount) -> bool: + return {"x": x.can_post_media, "linkedin": linkedin.can_post_media}.get(a.platform, lambda _: False)(a) + + +def _require_connected(platform: str, account: SocialAccount | None, artifact: Artifact | None) -> None: + """X and LinkedIn: scheduling needs a live connection (and one allowed to post media, when media is attached).""" + if platform not in CONNECTED_ONLY: + return + label = PLATFORM_LABELS[platform] + if not account or account.status != "active": + raise HTTPException(409, {"code": "not_connected", + "message": f"Connect {label} to schedule posts there." + if not account else f"Your {label} connection is down. Reconnect {label} first."}) + if artifact and media_count(artifact) and not can_post_media(account): + raise HTTPException(409, {"code": "reconnect", "message": f"Reconnect {label} to post images and videos."}) + + def _validate_lengths(platform: str, posts: list[str]) -> None: limit = LIMITS.get(platform) if platform in ("x", "bluesky") and limit: @@ -110,7 +211,28 @@ def _enrich(ctx: Ctx, items: list[CalendarItem]) -> list[dict]: accounts = {a.id: a for a in ctx.db.scalars( select(SocialAccount).where(SocialAccount.workspace_id == ctx.workspace.id))} clicks = lp.clicks_by_item(ctx.db, [i.id for i in items]) - return [serialize_item(i, accounts.get(i.social_account_id), clicks.get(i.id, 0)) for i in items] + ids = {i.artifact_id for i in items if i.artifact_id} | {i.kit_id for i in items if i.kit_id} + artifacts = {a.id: a for a in ctx.db.scalars(select(Artifact).where(Artifact.id.in_(ids)))} if ids else {} + return [serialize_item(i, accounts.get(i.social_account_id), clicks.get(i.id, 0), artifacts.get(i.artifact_id), + artifacts.get(i.kit_id)) for i in items] + + +def _kit(ctx: Ctx, kit_id: uuid.UUID) -> Artifact: + """This workspace's Launch Kit (what a post was written from).""" + kit = ctx.db.scalar(select(Artifact).where(Artifact.id == kit_id, Artifact.workspace_id == ctx.workspace.id, + Artifact.type == "launch_kit")) + if not kit: + raise HTTPException(404, "Launch Kit not found") + return kit + + +def _postable(ctx: Ctx, artifact_id: uuid.UUID, platform: str) -> Artifact: + """This workspace's artifact, checked for the platform (audio, unfinished, too long for X: 400).""" + a = ctx.db.scalar(select(Artifact).where(Artifact.id == artifact_id, Artifact.workspace_id == ctx.workspace.id)) + if not a: + raise HTTPException(404, "Artifact not found") + check_postable(a, platform) + return a @router.get("/calendar") @@ -129,15 +251,21 @@ def list_items(ctx: Ctx = Depends(get_ctx), start: datetime | None = None, end: @router.post("/calendar") def create_item(body: ItemIn, ctx: Ctx = Depends(get_ctx)): + when = body.utc() + if when is None: + raise HTTPException(422, {"code": "no_time", "message": "Pick when it goes out."}) account = _account(ctx, body.social_account_id, body.platform) _validate_lengths(body.platform, [body.content, *body.thread]) - for model, value in ((Artifact, body.artifact_id), (Document, body.document_id)): - if value and not ctx.db.scalar(select(model.id).where(model.id == value, - model.workspace_id == ctx.workspace.id)): - raise HTTPException(404, f"{model.__name__} not found") + artifact = _postable(ctx, body.artifact_id, body.platform) if body.artifact_id else None + kit = _kit(ctx, body.kit_id) if body.kit_id else None + _require_connected(body.platform, account, artifact) + if body.document_id and not ctx.db.scalar(select(Document.id).where(Document.id == body.document_id, + Document.workspace_id == ctx.workspace.id)): + raise HTTPException(404, "Document not found") item = CalendarItem( - workspace_id=ctx.workspace.id, platform=body.platform, scheduled_at=_aware(body.scheduled_at), + workspace_id=ctx.workspace.id, platform=body.platform, scheduled_at=when, content=body.content, thread=[t for t in body.thread if t.strip()], artifact_id=body.artifact_id, + kit_id=kit.id if kit else None, document_id=body.document_id, social_account_id=account.id if account else None, remind_by_email=body.remind_by_email or body.platform not in AUTO_POST or not account, status="draft" if body.draft else "scheduled", @@ -152,12 +280,15 @@ def update_item(item_id: uuid.UUID, body: ItemPatch, ctx: Ctx = Depends(get_ctx) item = _item(ctx, item_id) if item.status in ("posted", "publishing"): raise HTTPException(409, "This post already went out.") - if body.scheduled_at is not None: - item.scheduled_at = _aware(body.scheduled_at) + when = body.utc() + if when is not None: + item.scheduled_at = when if body.content is not None: item.content = body.content if body.thread is not None: item.thread = [t for t in body.thread if t.strip()] + if "artifact_id" in body.model_fields_set: + item.artifact_id = _postable(ctx, body.artifact_id, item.platform).id if body.artifact_id else None if "social_account_id" in body.model_fields_set: item.social_account_id = (_account(ctx, body.social_account_id, item.platform).id if body.social_account_id else None) @@ -165,10 +296,18 @@ def update_item(item_id: uuid.UUID, body: ItemPatch, ctx: Ctx = Depends(get_ctx) item.remind_by_email = body.remind_by_email if not item.social_account_id or item.platform not in AUTO_POST: item.remind_by_email = True - if body.status is not None: - item.status = body.status - elif item.status in ("failed", "reminded"): - item.status = "scheduled" + account = ctx.db.get(SocialAccount, item.social_account_id) if item.social_account_id else None + artifact = ctx.db.get(Artifact, item.artifact_id) if item.artifact_id else None + if body.status == "paused": + item.status = "paused" # held by hand: never attempted until resumed + elif body.status is not None or item.status in ("failed", "reminded", "paused"): + # Going out (again): only through a live connection, and only at a time still ahead. + _require_connected(item.platform, account, artifact) + if item.status == "paused" and _aware(item.scheduled_at) <= datetime.now(UTC): + raise HTTPException(409, {"code": "missed", "message": "Pick a time that is still ahead."}) + item.status = body.status or "scheduled" + elif item.platform in CONNECTED_ONLY and body.model_fields_set & {"social_account_id", "artifact_id"}: + _require_connected(item.platform, account, artifact) item.error = None item.attempts = 0 _validate_lengths(item.platform, [item.content, *(item.thread or [])]) @@ -192,16 +331,82 @@ async def publish_now(item_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): item.status = "publishing" item.attempts = 0 ctx.db.commit() - await run_in_threadpool(lp.publish_item, ctx.db, item) + if lp.has_media(ctx.db, item): + lp.queue_publish(ctx.db, item) # uploading takes a while: a job does it, the page follows the status + else: + await run_in_threadpool(lp.publish_item, ctx.db, item) return _enrich(ctx, [item])[0] +class UploadIn(BaseModel): + upload_id: uuid.UUID + duration_s: float | None = Field(None, ge=0, le=24 * 3600) # a video's length, read by the browser + + +@router.post("/launchpad/uploads") +def register_upload(body: UploadIn, ctx: Ctx = Depends(get_ctx)): + """A file the user uploaded from their computer (through /api/storage/uploads), made postable: it becomes an + `upload` artifact, attached and posted like anything Notestack made, and offered again in the picker later.""" + upload = ctx.db.scalar(select(Upload).where(Upload.id == body.upload_id, Upload.workspace_id == ctx.workspace.id)) + if not upload: + raise HTTPException(404, "Upload not found") + if upload.status != "complete": + raise HTTPException(409, "This file hasn't finished uploading.") + kind = UPLOAD_TYPES.get(upload.content_type) + if not kind: + raise HTTPException(415, "LinkedIn takes JPG, PNG or GIF images and MP4 videos.") + low, high = SIZE_LIMITS[("linkedin", kind)] + size = upload.size_bytes or 0 + if size > high: + raise HTTPException(413, f"This {kind} is {size / MB:.0f} MB; LinkedIn takes up to {high // MB} MB.") + if size < low: + raise HTTPException(400, f"This {kind} is too small for LinkedIn (at least {low // 1024} KB).") + title = upload.filename.rsplit(".", 1)[0] if "." in upload.filename else upload.filename + a = Artifact(workspace_id=ctx.workspace.id, type="upload", status="ready", storage_key=upload.key, + content_json={"title": title[:200] or "Upload", "filename": upload.filename, + "content_type": upload.content_type, "media": kind, "size_bytes": size, + **({"duration_s": body.duration_s} if body.duration_s else {})}) + ctx.db.add(a) + ctx.db.commit() + return artifact_brief(a) + + +@router.get("/launchpad/postable") +def postable(ctx: Ctx = Depends(get_ctx), q: str = "", limit: int = 60): + """What can be attached to a post: finished videos, quote cards, carousels, summaries, launch kits, mind maps + (not audio), newest first. Videos made but not rendered yet are listed too, flagged needs_render (they can only be + posted once rendered), and so are videos being rendered, flagged rendering.""" + rows = ctx.db.scalars(select(Artifact).where(Artifact.workspace_id == ctx.workspace.id, + Artifact.type.in_(POSTABLE)) + .order_by(Artifact.created_at.desc()).limit(400)) + needle = q.strip().lower() + out = [] + for a in rows: + # A blog2video video counts once blog2video made it (ready), rendered or not; anything else once it is ready. + # A blog2video video mid-render stays listed (flagged rendering). + ready = a.status == "ready" or (is_blog2video(a) and (bool((a.content_json or {}).get("video_url")) + or a.status == "rendering")) + title = ((a.content_json or {}).get("title") or "").lower() + if ready and (not needle or needle in title): + out.append(artifact_brief(a)) + if len(out) >= min(limit, 200): + break + return out + + # Social accounts def serialize_account(a: SocialAccount) -> dict: return {"id": str(a.id), "platform": a.platform, "platform_label": PLATFORM_LABELS[a.platform], "handle": a.handle, "status": a.status, "avatar_url": a.avatar_url, + "can_post_media": can_post_media(a), + # No refresh token and the connection ends within a week (LinkedIn's 60 day tokens). + "expires_soon": bool(a.expires_at and not a.refresh_token + and _aware(a.expires_at) - datetime.now(UTC) < timedelta(days=7)), + "expires_at": _aware(a.expires_at).isoformat() if a.expires_at else None, + # Down (expired, revoked, disconnected), or unable to post images and videos. + "needs_reconnect": a.status != "active" or (a.platform in CONNECTED_ONLY and not can_post_media(a)), "connected_at": a.created_at.isoformat() if a.created_at else None} @@ -222,11 +427,12 @@ def disconnect(account_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): SocialAccount.workspace_id == ctx.workspace.id)) if not account: raise HTTPException(404, "Account not found") - for item in ctx.db.scalars(select(CalendarItem).where(CalendarItem.social_account_id == account.id, - CalendarItem.status == "scheduled")): - item.social_account_id = None - item.remind_by_email = True - ctx.db.delete(account) + # Kept (its posts stay linked to it), tokens gone, posts paused: reconnecting the same account resumes them. + account.status = "revoked" + account.access_token = "" # no usable token kept + account.refresh_token = None + label = PLATFORM_LABELS[account.platform] + pause_account_posts(ctx.db, account, f"{label} disconnected. Reconnect {label} to resume.") ctx.db.commit() return {"ok": True} @@ -247,6 +453,7 @@ def _upsert_account(db: Session, workspace_id: uuid.UUID, platform: str, info: d account.expires_at = datetime.now(UTC) + timedelta(seconds=int(expires_in)) if expires_in else None account.scopes = scopes account.status = "active" + resume_account_posts(db, account) # its paused posts still ahead go back to scheduled db.commit() diff --git a/backend/app/routers/reports.py b/backend/app/routers/reports.py index 04939d341c5248ae28a3cc8e25704c5d4ad16e43..bdadd496e35964edff26e0eabf236d3123e46bb6 100644 --- a/backend/app/routers/reports.py +++ b/backend/app/routers/reports.py @@ -15,8 +15,8 @@ from app.routers.artifacts import get_artifact_or_404 from app.routers.videos import FocusIn, _focus_material from app.services.artifacts import latest_jobs, serialize_artifact from app.services.jobs import create_job -from app.services.usage import check_limit from app.services.report_templates import INTERACTIVE, TEMPLATES +from app.services.usage import check_limit router = APIRouter(prefix="/api/reports", tags=["reports"]) diff --git a/backend/app/routers/storage.py b/backend/app/routers/storage.py index aa22196736e6b304eccca80f6b99802d487fdc94..271832d82fdb523776e9e2d4f572cda64495cf92 100644 --- a/backend/app/routers/storage.py +++ b/backend/app/routers/storage.py @@ -18,6 +18,7 @@ router = APIRouter(prefix="/api/storage", tags=["storage"]) ALLOWED_TYPES = { "image/png", "image/jpeg", + "image/gif", "image/webp", "image/svg+xml", "audio/mpeg", diff --git a/backend/app/services/artifacts.py b/backend/app/services/artifacts.py index 4280f5fe440cb24b8fc2276513f737ed95f13e18..3ea2fdc1f16f87a8e7ec980e422c2fd835f86170 100644 --- a/backend/app/services/artifacts.py +++ b/backend/app/services/artifacts.py @@ -22,6 +22,7 @@ TYPE_LABELS = { "flashcards": "Flashcards", "report": "Report", "infographic": "Infographic", + "upload": "Upload", } diff --git a/backend/app/services/blog2video.py b/backend/app/services/blog2video.py index e89e7aba6292ebf59e68c1db888ebf50a27d6cca..f9b7b9431321d04712cb3633fa1fe94dd1c8eeae 100644 --- a/backend/app/services/blog2video.py +++ b/backend/app/services/blog2video.py @@ -85,7 +85,7 @@ class Denied(B2VError): UNAVAILABLE = "Video creation is temporarily unavailable. Try again later." -NOT_RESPONDING = "The video service is not responding. Try again in a minute." +NOT_RESPONDING = "The video service is being updated. Please try again in a while." def configured() -> bool: diff --git a/backend/app/services/email.py b/backend/app/services/email.py index efcaef61ef5e6c7917800e4998f25c6bd73e89cf..053e1dc3263a57c82a04490aaf6571930402cd85 100644 --- a/backend/app/services/email.py +++ b/backend/app/services/email.py @@ -55,7 +55,8 @@ class ConsoleEmailProvider(BaseEmailProvider): def send(self, email: OutgoingEmail) -> str | None: ConsoleEmailProvider.sent.append(email) - log.info("[email] to=%s subject=%s\n%s", email.to, email.subject, email.text) + # Warning, not info: locally this is the only place a sign up or reset code shows up. + log.warning("[email] to=%s subject=%s\n%s", email.to, email.subject, email.text) return None @@ -264,6 +265,13 @@ class EmailService: def _build() -> EmailService: if settings.email_provider == "resend" and settings.resend_api_key: return EmailService(ResendEmailProvider(settings.resend_api_key)) + if settings.email_provider == "resend": + log.warning("EMAIL_PROVIDER=resend but RESEND_API_KEY is empty: emails are only logged, not sent") + elif settings.email_provider != "console": + log.warning("Unknown EMAIL_PROVIDER=%r (use resend or console): emails are only logged, not sent", + settings.email_provider) + else: + log.warning("EMAIL_PROVIDER=console: emails are only logged, not sent") return EmailService(ConsoleEmailProvider()) diff --git a/backend/app/services/email_verification.py b/backend/app/services/email_verification.py index c82cc2b8e578b5bbb1046fe4e7ecd031850c4139..40c412c7fd7943265dd23a8b64c0c4e7279cd01d 100644 --- a/backend/app/services/email_verification.py +++ b/backend/app/services/email_verification.py @@ -52,6 +52,11 @@ def _latest(db: Session, email: str, purpose: VerificationPurpose) -> EmailVerif ) +def _alive(record: EmailVerificationCode, now: datetime) -> bool: + """A code that can still be entered. Dead codes (expired, out of tries) never hold up a resend.""" + return now <= _aware(record.expires_at) and record.attempts < MAX_ATTEMPTS + + def issue_code( db: Session, email: str, @@ -61,7 +66,7 @@ def issue_code( ) -> IssuedCode: now = datetime.now(UTC) latest = _latest(db, email, purpose) - if latest and now - _aware(latest.created_at) < RESEND_COOLDOWN: + if latest and _alive(latest, now) and now - _aware(latest.created_at) < RESEND_COOLDOWN: raise VerificationError("cooldown", "Please wait a minute before requesting another code.") clear_codes(db, email, purpose) code = f"{secrets.randbelow(1_000_000):06d}" @@ -83,14 +88,20 @@ def check_code(db: Session, email: str, purpose: VerificationPurpose, code: str) record = _latest(db, email, purpose) if not record: raise VerificationError("not_found", "No active code. Request a new one.") + if record.attempts >= MAX_ATTEMPTS: + raise VerificationError("too_many_attempts", "Too many wrong tries. This code has expired. Request a new one.") if datetime.now(UTC) > _aware(record.expires_at): raise VerificationError("expired", "That code expired. Request a new one.") - if record.attempts >= MAX_ATTEMPTS: - raise VerificationError("too_many_attempts", "Too many attempts. Request a new code.") if not hmac.compare_digest(record.code_hash, _hash(email, code.strip())): record.attempts += 1 + left = MAX_ATTEMPTS - record.attempts + if left <= 0: + # Expire rather than consume: sign up resend still needs the pending name and password. + record.expires_at = datetime.now(UTC) + db.commit() + raise VerificationError("too_many_attempts", "Too many wrong tries. This code has expired. Request a new one.") db.commit() - raise VerificationError("invalid", "That code is not right.") + raise VerificationError("invalid", f"That code is not right. {left} {'try' if left == 1 else 'tries'} left.") return record diff --git a/backend/app/services/launchpad.py b/backend/app/services/launchpad.py index 5b1e7c0c0c57451baedb26c57569e9b6ccd4d6b4..6914d06a8f6448ff83035ac9e2feb480800a3b9c 100644 --- a/backend/app/services/launchpad.py +++ b/backend/app/services/launchpad.py @@ -1,9 +1,13 @@ -"""Launchpad: publishes due calendar items, emails reminders for platforms without an API, rewrites links -to tracked short links and pulls engagement back in.""" +"""Launchpad: publishes due calendar items (with the images or video of the artifact they were made from), emails +reminders for platforms without an API, rewrites links to tracked short links and pulls engagement back in. + +Due items are published by a "publish_post" job each, not inside the worker's 30 second tick: a video upload and the +platform's processing can take minutes.""" import logging import re import secrets +import uuid from collections.abc import Callable from datetime import UTC, datetime, timedelta @@ -12,13 +16,17 @@ from sqlalchemy.orm import Session from app.config import settings from app.db import SessionLocal -from app.models import CalendarItem, Document, EngagementEvent, SocialAccount, TrackedLink, User, Workspace +from app.models import Artifact, CalendarItem, Document, EngagementEvent, SocialAccount, TrackedLink, User, Workspace from app.services.email import email_service +from app.services.jobs import create_job from app.services.social import AUTO_POST, PLATFORM_LABELS, SocialError, module +from app.services.social.health import mark_down +from app.services.social.media import media_count, media_for log = logging.getLogger("notestack.launchpad") _URL = re.compile(r"https?://[^\s)\]>]+") MAX_ATTEMPTS = 3 +STUCK_AFTER = timedelta(minutes=30) # longer than a video upload and the platform's processing take def _public_api() -> bool: @@ -58,6 +66,11 @@ def publish_item(db: Session, item: CalendarItem) -> CalendarItem: return item account = db.get(SocialAccount, item.social_account_id) if item.social_account_id else None label = PLATFORM_LABELS.get(item.platform, item.platform) + if item.platform in ("x", "linkedin") and not item.remind_by_email and (not account or account.status != "active"): + # The connection is down: never attempted; it resumes when the account is reconnected. + item.status, item.error = "paused", f"{label} disconnected. Reconnect {label} to resume." + db.commit() + return item if item.platform not in AUTO_POST or item.remind_by_email or not account: to = owner_email(db, item.workspace_id) sent = bool(to) and email_service.send_post_reminder( @@ -71,8 +84,21 @@ def publish_item(db: Session, item: CalendarItem) -> CalendarItem: try: tracked = [track_links(db, item, p) for p in posts] db.flush() - external_id, url = module(item.platform).publish(db, account, tracked) + artifact = db.get(Artifact, item.artifact_id) if item.artifact_id else None + if artifact and media_count(artifact) and item.platform in ("x", "linkedin") \ + and not module(item.platform).can_post_media(account): + # Scheduled before the account lost the right to post media: held, not attempted, until reconnected. + item.status, item.error = "paused", f"Reconnect {label} to post images and videos." + db.commit() + return item + with media_for(artifact, item.platform) as media: # temporary video files go when this ends + external_id, url = module(item.platform).publish(db, account, tracked, media) except SocialError as exc: + if exc.reconnect: # the platform refused the connection: it is down, and its posts pause + mark_down(db, account, "expired", str(exc)) + item.status, item.error = "paused", str(exc)[:1000] + db.commit() + return item _fail_or_retry(db, item, str(exc), exc.permanent) return item except Exception as exc: # network trouble: retry @@ -99,13 +125,34 @@ def _fail_or_retry(db: Session, item: CalendarItem, error: str, permanent: bool) db.commit() +def has_media(db: Session, item: CalendarItem) -> bool: + """The post uploads images or a video (slow: it goes through a job, also for Publish now).""" + artifact = db.get(Artifact, item.artifact_id) if item.artifact_id else None + return bool(artifact) and item.platform in ("x", "linkedin") and media_count(artifact) > 0 + + +def queue_publish(db: Session, item: CalendarItem) -> None: + """Hand the (already claimed, "publishing") item to a worker job.""" + create_job(db, item.workspace_id, "publish_post", {"item_id": str(item.id)}, max_attempts=1) + + +def publish_job(db: Session, item_id: str) -> CalendarItem | None: + """The publish_post job: publish the item if it is still waiting for it.""" + item = db.get(CalendarItem, uuid.UUID(str(item_id))) + if not item or item.status != "publishing": + return item + item.updated_at = datetime.now(UTC) # the stuck limit counts from the job's start, not the claim + db.commit() + return publish_item(db, item) + + def publish_due(session_factory: Callable[[], Session] = SessionLocal) -> int: + """Claim the due items and queue a publish_post job for each. Returns how many were queued.""" now = datetime.now(UTC) - count = 0 with session_factory() as db: - # A tick that died mid publish leaves items in "publishing"; put them back after 10 minutes. + # A job that died mid publish leaves its item in "publishing"; put it back once that has clearly happened. for stuck in db.scalars(select(CalendarItem).where( - CalendarItem.status == "publishing", CalendarItem.updated_at < now - timedelta(minutes=10))): + CalendarItem.status == "publishing", CalendarItem.updated_at < now - STUCK_AFTER)): stuck.status = "scheduled" due = db.scalars( select(CalendarItem).where(CalendarItem.status == "scheduled", CalendarItem.scheduled_at <= now) @@ -115,9 +162,8 @@ def publish_due(session_factory: Callable[[], Session] = SessionLocal) -> int: item.status = "publishing" # claim, so an overlapping tick skips it db.commit() for item in due: - publish_item(db, item) - count += 1 - return count + queue_publish(db, item) + return len(due) def clicks_by_item(db: Session, item_ids: list) -> dict: diff --git a/backend/app/services/renderer.py b/backend/app/services/renderer.py index fcde5587e7589d7b834b8d166c51296cd8ea3045..3d3d8b6fc5ea72b96612bbd61247afc44707804b 100644 --- a/backend/app/services/renderer.py +++ b/backend/app/services/renderer.py @@ -17,6 +17,14 @@ class PermanentJobError(RuntimeError): """Fail the job now; retrying will not help.""" +def available() -> bool: + """Whether the render service answers right now (a quick health check, not a promise for later).""" + try: + return httpx.get(f"{settings.renderer_url}/health", timeout=2).status_code == 200 + except httpx.HTTPError: + return False + + def request_render(job: Job, artifact: Artifact, composition: str, props: dict | None = None, stills: list[dict] | None = None) -> None: """Start a render. `stills` renders one PNG per props dict (carousels) instead of a single output.""" diff --git a/backend/app/services/social/__init__.py b/backend/app/services/social/__init__.py index 53c8b787e90e5e780963bc480da1f41f71e81185..c423932e8fd63923129080a847559a797d0b2a33 100644 --- a/backend/app/services/social/__init__.py +++ b/backend/app/services/social/__init__.py @@ -1,9 +1,10 @@ """Publishing to X, LinkedIn and Bluesky. Each platform module exposes: - publish(db, account, posts: list[str]) -> (external_id, external_url) + publish(db, account, posts: list[str], media: list[Media] | None) -> (external_id, external_url) metrics(db, account, external_id) -> dict -A list with more than one post is published as a thread (reply chain) where the platform supports it. +A list with more than one post is published as a thread (reply chain) where the platform supports it. Media (images or +one video, from social.media) goes on the first post; only X and LinkedIn take it. """ from datetime import UTC, datetime @@ -15,9 +16,13 @@ from app.services.crypto import decrypt, encrypt class SocialError(RuntimeError): - def __init__(self, message: str, permanent: bool = False): + """permanent: don't retry this post. reconnect: the account's connection is gone (expired, revoked, refused): + the account is marked down and its scheduled posts pause until it is reconnected.""" + + def __init__(self, message: str, permanent: bool = False, reconnect: bool = False): super().__init__(message) - self.permanent = permanent + self.permanent = permanent or reconnect + self.reconnect = reconnect PLATFORM_LABELS = {"x": "X", "linkedin": "LinkedIn", "bluesky": "Bluesky", "substack_notes": "Substack Notes"} @@ -49,6 +54,11 @@ def expired(account: SocialAccount) -> bool: return exp.timestamp() - 120 < datetime.now(UTC).timestamp() +def scope_set(scopes: str | None) -> set[str]: + """Granted scopes, whether the platform listed them with spaces or commas.""" + return {s for s in (scopes or "").replace(",", " ").split() if s} + + def module(platform: str): from app.services.social import bluesky, linkedin, x diff --git a/backend/app/services/social/bluesky.py b/backend/app/services/social/bluesky.py index 5d8901a290a874ad2f1c1f00db73cd3a4b36967f..d0565b4ad1fcb65a8cbb99c518779f334f0056bf 100644 --- a/backend/app/services/social/bluesky.py +++ b/backend/app/services/social/bluesky.py @@ -34,7 +34,9 @@ def link_facets(text: str) -> list[dict]: return facets -def publish(db: Session, account: SocialAccount, posts: list[str]) -> tuple[str, str]: +def publish(db: Session, account: SocialAccount, posts: list[str], media: list | None = None) -> tuple[str, str]: + if media: + raise SocialError("Images and videos can only be posted to X and LinkedIn.", permanent=True) session = create_session(account.handle, token(account)) headers = {"Authorization": f"Bearer {session['accessJwt']}"} root = parent = None diff --git a/backend/app/services/social/health.py b/backend/app/services/social/health.py new file mode 100644 index 0000000000000000000000000000000000000000..069f01d7d25fe6e222bece8b1580c47c0800dd47 --- /dev/null +++ b/backend/app/services/social/health.py @@ -0,0 +1,103 @@ +"""Keeping X and LinkedIn connected, and what happens to scheduled posts when a connection goes down. + +A connection that is down (expired, revoked on the platform, or disconnected here) pauses every post scheduled through +it: paused posts are never attempted (publish_due only takes "scheduled" ones). Reconnecting the same account resumes +the ones still ahead; posts whose time passed meanwhile stay paused, for a new time to be picked. + +social_health (every 10 minutes, worker.PERIODIC) refreshes X tokens before they expire, refreshes LinkedIn tokens +where LinkedIn gave a refresh token, marks expired LinkedIn connections down, and every 6 hours asks each platform +once whether the connection still works (it catches access revoked on x.com or linkedin.com).""" + +import logging +import time +import uuid +from collections.abc import Callable +from datetime import UTC, datetime, timedelta + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.db import SessionLocal +from app.models import CalendarItem, SocialAccount +from app.services.social import PLATFORM_LABELS, SocialError, module + +log = logging.getLogger("notestack.social") + +ACTIVE_STATES = ("scheduled", "draft", "failed") # what a dropped connection pauses +LIVENESS_EVERY = 6 * 3600 # seconds between "is this connection still accepted" calls per account +X_REFRESH_AHEAD = timedelta(minutes=15) # X access tokens last 2 hours: refresh before they lapse +LINKEDIN_REFRESH_AHEAD = timedelta(days=7) +_last_checked: dict[uuid.UUID, float] = {} # account id -> monotonic time of its last liveness call + + +def _aware(dt: datetime | None) -> datetime | None: + return dt if dt is None or dt.tzinfo else dt.replace(tzinfo=UTC) + + +def pause_account_posts(db: Session, account: SocialAccount, reason: str) -> int: + """Pause every post still to go out through this account. The caller commits.""" + items = db.scalars(select(CalendarItem).where(CalendarItem.social_account_id == account.id, + CalendarItem.status.in_(ACTIVE_STATES))).all() + for item in items: + item.status, item.error = "paused", reason + return len(items) + + +def mark_down(db: Session, account: SocialAccount, status: str, reason: str | None = None) -> None: + """The connection is gone (expired | revoked): mark it and pause its posts.""" + label = PLATFORM_LABELS.get(account.platform, account.platform) + account.status = status + paused = pause_account_posts(db, account, reason or f"{label} disconnected. Reconnect {label} to resume.") + db.commit() + log.info("%s account %s is %s; paused %d posts", account.platform, account.id, status, paused) + + +def resume_account_posts(db: Session, account: SocialAccount) -> None: + """After a reconnect: paused posts still ahead go back to scheduled; missed ones stay paused. Caller commits.""" + now = datetime.now(UTC) + for item in db.scalars(select(CalendarItem).where(CalendarItem.social_account_id == account.id, + CalendarItem.status == "paused")): + if _aware(item.scheduled_at) > now: + item.status, item.error, item.attempts = "scheduled", None, 0 + else: + item.error = "Missed while disconnected: pick a new time." + + +def _due_for_refresh(account: SocialAccount) -> bool: + expires = _aware(account.expires_at) + if not expires: + return False + ahead = X_REFRESH_AHEAD if account.platform == "x" else LINKEDIN_REFRESH_AHEAD + return expires - ahead <= datetime.now(UTC) + + +def check_account(db: Session, account: SocialAccount) -> None: + """Refresh what is about to expire, mark down what is gone, and now and then ask the platform.""" + mod = module(account.platform) + expires = _aware(account.expires_at) + try: + if _due_for_refresh(account): + if account.refresh_token: + mod.refresh(db, account) + elif expires and expires <= datetime.now(UTC): + raise SocialError("expired", reconnect=True) + if time.monotonic() - _last_checked.get(account.id, -LIVENESS_EVERY) >= LIVENESS_EVERY: + mod.check_alive(db, account) + _last_checked[account.id] = time.monotonic() + except SocialError as exc: + if exc.reconnect: + mark_down(db, account, "revoked" if "no longer accepts" in str(exc) else "expired") + else: + log.warning("%s check for %s: %s", account.platform, account.id, exc) + except Exception: # network trouble: try again next round + log.warning("%s check for %s failed", account.platform, account.id, exc_info=True) + + +def social_health(session_factory: Callable[[], Session] = SessionLocal) -> int: + """Every active X and LinkedIn connection, checked. Returns how many were looked at.""" + with session_factory() as db: + accounts = db.scalars(select(SocialAccount).where(SocialAccount.status == "active", + SocialAccount.platform.in_(("x", "linkedin")))).all() + for account in accounts: + check_account(db, account) + return len(accounts) diff --git a/backend/app/services/social/linkedin.py b/backend/app/services/social/linkedin.py index 2d569e569b8a097ee4eba68f2666e564c065d45f..73fa767847b39fa252dd2f97bbb39e2e4e89653d 100644 --- a/backend/app/services/social/linkedin.py +++ b/backend/app/services/social/linkedin.py @@ -1,6 +1,8 @@ -"""LinkedIn member posts (Share on LinkedIn + Sign In with LinkedIn using OpenID Connect products).""" +"""LinkedIn member posts (Share on LinkedIn + Sign In with LinkedIn using OpenID Connect products), with an image, +several images (multiImage) or a video uploaded first through the Images / Videos APIs.""" import re +import time from urllib.parse import quote, urlencode import httpx @@ -8,12 +10,16 @@ from sqlalchemy.orm import Session from app.config import settings from app.models import SocialAccount -from app.services.social import SocialError, expired, token +from app.services.crypto import decrypt +from app.services.social import SocialError, expired, save_tokens, scope_set, token AUTH_URL = "https://www.linkedin.com/oauth/v2/authorization" TOKEN_URL = "https://www.linkedin.com/oauth/v2/accessToken" SCOPES = "openid profile w_member_social" -VERSION = "202509" # LinkedIn-Version header, YYYYMM; LinkedIn supports each version for a year +VERSION = "202609" # LinkedIn-Version header, YYYYMM; LinkedIn supports each version for a year +REST = "https://api.linkedin.com/rest" +PROCESSING_WAIT = 600 # seconds LinkedIn may take to process a video before we give up +PART_ATTEMPTS = 4 # a video part upload is tried this often (network errors, 5xx) before the post fails def redirect_uri() -> str: @@ -55,11 +61,113 @@ def little_text(text: str) -> str: return _RESERVED.sub(r"\\\1", text) -def publish(db: Session, account: SocialAccount, posts: list[str]) -> tuple[str, str]: +def _call(method: str, url: str, **kw) -> httpx.Response: + """Every request to LinkedIn goes through here (tests replace it).""" + return httpx.request(method, url, timeout=kw.pop("timeout", 60), **kw) + + +def _headers(account: SocialAccount) -> dict: + return {"Authorization": f"Bearer {token(account)}", "LinkedIn-Version": VERSION, + "X-Restli-Protocol-Version": "2.0.0"} + + +def can_post_media(account: SocialAccount) -> bool: + return "w_member_social" in scope_set(account.scopes) + + +def refresh(db: Session, account: SocialAccount) -> None: + """Programmatic refresh, for apps LinkedIn gives refresh tokens to; without one the connection simply expires.""" + if not account.refresh_token: + raise SocialError("The LinkedIn connection expired. Reconnect LinkedIn.", reconnect=True) + resp = _call("POST", TOKEN_URL, data={ + "grant_type": "refresh_token", "refresh_token": decrypt(account.refresh_token), + "client_id": settings.linkedin_client_id, "client_secret": settings.linkedin_client_secret}, timeout=20) + if resp.status_code in (400, 401): + raise SocialError("The LinkedIn connection expired. Reconnect LinkedIn.", reconnect=True) + tokens = _check(resp, "token refresh") + save_tokens(db, account, tokens["access_token"], tokens.get("refresh_token"), tokens.get("expires_in")) + + +def check_alive(db: Session, account: SocialAccount) -> None: + """One cheap call: raises SocialError(reconnect=True) when LinkedIn no longer accepts this connection.""" + resp = _call("GET", "https://api.linkedin.com/v2/userinfo", headers={"Authorization": f"Bearer {token(account)}"}, + timeout=20) + if resp.status_code == 401: + raise SocialError("LinkedIn no longer accepts this connection. Reconnect LinkedIn.", reconnect=True) + + +def _check(resp: httpx.Response, what: str) -> dict: + if resp.status_code == 426 or "NONEXISTENT_VERSION" in resp.text: + raise SocialError(f"LinkedIn retired API version {VERSION}; Notestack needs an update to post.", permanent=True) + if resp.status_code == 401: + raise SocialError("LinkedIn rejected the connection. Reconnect LinkedIn.", reconnect=True) + if resp.status_code >= 400: + raise SocialError(f"LinkedIn {what} {resp.status_code}: {resp.text[:300]}", + permanent=400 <= resp.status_code < 500 and resp.status_code != 429) + return resp.json() if resp.content else {} + + +def _upload_image(account: SocialAccount, m) -> str: + owner = f"urn:li:person:{account.external_id}" + got = _check(_call("POST", f"{REST}/images", params={"action": "initializeUpload"}, headers=_headers(account), + json={"initializeUploadRequest": {"owner": owner}}), "image upload")["value"] + _check(_call("PUT", got["uploadUrl"], content=m.read(), + headers={"Authorization": _headers(account)["Authorization"], "Content-Type": m.content_type}), + "image upload") + return got["image"] + + +def _upload_video(account: SocialAccount, m) -> str: + owner = f"urn:li:person:{account.external_id}" + got = _check(_call("POST", f"{REST}/videos", params={"action": "initializeUpload"}, headers=_headers(account), + json={"initializeUploadRequest": {"owner": owner, "fileSizeBytes": m.size, + "uploadCaptions": False, "uploadThumbnail": False}}), + "video upload")["value"] + etags = [_put_part(part, m) for part in got["uploadInstructions"]] + _check(_call("POST", f"{REST}/videos", params={"action": "finalizeUpload"}, headers=_headers(account), + json={"finalizeUploadRequest": {"video": got["video"], "uploadToken": got.get("uploadToken", ""), + "uploadedPartIds": etags}}), "video upload") + waited = 0 + while True: # usable in a post only once LinkedIn has processed it + state = _check(_call("GET", f"{REST}/videos/{quote(got['video'])}", headers=_headers(account)), + "video status").get("status") + if state == "AVAILABLE": + return got["video"] + if state == "PROCESSING_FAILED": + raise SocialError("LinkedIn could not process the video.", permanent=True) + if waited >= PROCESSING_WAIT: + raise SocialError("LinkedIn took too long to process the video.") + time.sleep(5) + waited += 5 + + +def _put_part(part: dict, m) -> str: + """One part of a video, read from the file. Sent to LinkedIn's upload URL with no bearer token; network errors + and 5xx are retried with backoff. Returns the part's ETag.""" + data = m.read(part["firstByte"], part["lastByte"] + 1) + for attempt in range(1, PART_ATTEMPTS + 1): + try: + resp = _call("PUT", part["uploadUrl"], content=data, headers={"Content-Type": "application/octet-stream"}, + timeout=300) + except httpx.HTTPError as exc: + if attempt == PART_ATTEMPTS: + raise SocialError(f"LinkedIn video upload failed: {exc}") from exc + else: + if resp.status_code in (401, 403): + raise SocialError("LinkedIn's upload session expired; trying again later.") + if resp.status_code < 400: + return resp.headers.get("etag", "").strip('"') + if resp.status_code < 500 or attempt == PART_ATTEMPTS: + raise SocialError(f"LinkedIn video upload {resp.status_code}: {resp.text[:200]}", + permanent=resp.status_code < 500) + time.sleep(2 ** attempt) + raise SocialError("LinkedIn video upload failed.") # not reached + + +def publish(db: Session, account: SocialAccount, posts: list[str], media: list | None = None) -> tuple[str, str]: if expired(account): - account.status = "expired" - db.commit() - raise SocialError("The LinkedIn connection expired. Reconnect the account.", permanent=True) + refresh(db, account) # without a refresh token: reconnect + urns = [(_upload_video if m.kind == "video" else _upload_image)(account, m) for m in media or []] body = { "author": f"urn:li:person:{account.external_id}", "commentary": little_text("\n\n".join(posts)), @@ -68,14 +176,13 @@ def publish(db: Session, account: SocialAccount, posts: list[str]) -> tuple[str, "lifecycleState": "PUBLISHED", "isReshareDisabledByAuthor": False, } - resp = httpx.post("https://api.linkedin.com/rest/posts", json=body, timeout=20, headers={ - "Authorization": f"Bearer {token(account)}", "LinkedIn-Version": VERSION, - "X-Restli-Protocol-Version": "2.0.0", - }) - if resp.status_code == 401: - account.status = "expired" - db.commit() - raise SocialError("LinkedIn rejected the token. Reconnect the account.", permanent=True) + if len(urns) == 1: + body["content"] = {"media": {"id": urns[0]}} + elif urns: + body["content"] = {"multiImage": {"images": [{"id": u} for u in urns]}} + resp = _call("POST", f"{REST}/posts", json=body, timeout=20, headers=_headers(account)) + if resp.status_code in (401, 426) or "NONEXISTENT_VERSION" in resp.text: + _check(resp, "post") # reconnect, or the API version was retired if resp.status_code >= 400: raise SocialError(f"LinkedIn {resp.status_code}: {resp.text[:300]}", permanent=resp.status_code in (400, 403, 422)) diff --git a/backend/app/services/social/media.py b/backend/app/services/social/media.py new file mode 100644 index 0000000000000000000000000000000000000000..36e0825c0c5b412458087a98f4ee3da857307658 --- /dev/null +++ b/backend/app/services/social/media.py @@ -0,0 +1,174 @@ +"""What a scheduled post carries from the artifact it was made from: images (quote cards, carousel slides) or one +video (an audiogram, or a blog2video video), or a file the user uploaded from their computer (an `upload`: one image or +one MP4). Summaries, launch kits and mind maps post as text; audio overviews can't +be posted (X and LinkedIn take no audio files). + +Videos are streamed to a temporary file (never held whole in memory) and read in slices by the uploaders; the files are +removed when the post is done (media_for is a context manager). Sizes are checked before anything is sent.""" + +import contextlib +import os +import tempfile +from collections.abc import Iterator +from dataclasses import dataclass + +import httpx +from fastapi import HTTPException + +from app.models import Artifact +from app.services.social import SocialError + +MAX_IMAGES = {"x": 4, "linkedin": 20} +X_VIDEO_SECONDS = 140 # X's limit for videos from standard accounts (2:20) +TEXT_ONLY = {"summary", "launch_kit", "mind_map"} +MEDIA_PLATFORMS = {"x", "linkedin"} +MB = 1024 * 1024 +# What the user can upload from their computer to post: LinkedIn takes JPG, PNG and GIF images and MP4 videos. +UPLOAD_TYPES = {"image/png": "image", "image/jpeg": "image", "image/gif": "image", "video/mp4": "video"} +# (min, max) bytes per platform and kind, checked before any upload starts +SIZE_LIMITS = { + ("x", "image"): (1, 5 * MB), + ("x", "video"): (1, 512 * MB), + ("linkedin", "image"): (1, 36 * MB), + ("linkedin", "video"): (75 * 1024, 500 * MB), +} + + +@dataclass +class Media: + kind: str # image | video + filename: str + content_type: str + data: bytes | None = None # images: the bytes + path: str | None = None # videos: a temporary file + size: int = 0 + + def __post_init__(self): + if self.data is not None: + self.size = len(self.data) + elif self.path: + self.size = os.path.getsize(self.path) + + def read(self, start: int = 0, end: int | None = None) -> bytes: + """Bytes [start, end) (all of them by default).""" + end = self.size if end is None else min(end, self.size) + if self.data is not None: + return self.data[start:end] + with open(self.path, "rb") as f: + f.seek(start) + return f.read(end - start) + + +def is_blog2video(a: Artifact) -> bool: + return (a.content_json or {}).get("provider") == "blog2video" + + +def media_count(a: Artifact) -> int: + """How many files a post of this artifact carries (before a platform's cap).""" + if a.type == "quote_card": + return 1 + if a.type == "carousel": + return len((a.content_json or {}).get("slide_keys") or ([a.storage_key] if a.storage_key else [])) + if a.type in ("video", "upload"): + return 1 + return 0 + + +def upload_kind(a: Artifact) -> str | None: + """An uploaded file's kind: image | video.""" + return (a.content_json or {}).get("media") if a.type == "upload" else None + + +def duration(a: Artifact) -> float | None: + """A video's length when we know it (audiograms record it; blog2video's is only known to blog2video).""" + value = (a.content_json or {}).get("duration_s") + return float(value) if value else None + + +def check_postable(a: Artifact, platform: str) -> None: + """400 when this artifact can't go out on this platform, so it is refused when scheduled, not at post time.""" + if a.type == "audio_overview": + raise HTTPException(400, "Audio can't be posted: X and LinkedIn don't take audio files.") + if a.type == "launch_kit": # a post is written from a kit (kit_id); the kit itself is never an attachment + raise HTTPException(400, "A launch kit can't be attached. Attach a video, quote card or carousel instead.") + if a.type not in TEXT_ONLY and a.type not in ("quote_card", "carousel", "video", "upload"): + raise HTTPException(400, "This can't be posted.") + if a.type in TEXT_ONLY: + return + if platform not in MEDIA_PLATFORMS: + raise HTTPException(400, "Images and videos can only be posted to X and LinkedIn.") + if a.type == "video" and is_blog2video(a): + if not (a.content_json or {}).get("video_url"): + raise HTTPException(400, "This video isn't finished yet. Post it once it has rendered.") + elif a.status != "ready" or not a.storage_key: + raise HTTPException(400, "This isn't ready yet. Post it once it has finished.") + seconds = duration(a) if a.type == "video" else None + if platform == "x" and seconds and seconds > X_VIDEO_SECONDS: + raise HTTPException(400, f"This video is {int(seconds // 60)}:{int(seconds % 60):02d} long; X takes videos " + "up to 2:20. Post it to LinkedIn, or make a shorter one.") + + +def check_size(m: Media, platform: str) -> None: + """Too big (or, on LinkedIn, too small a video): a clear permanent error before any network call.""" + low, high = SIZE_LIMITS.get((platform, m.kind), (1, 1 << 40)) + label = "X" if platform == "x" else "LinkedIn" + if m.size > high: + raise SocialError(f"This {m.kind} is {m.size / MB:.0f} MB; {label} takes up to {high // MB} MB.", permanent=True) + if m.size < low: + raise SocialError(f"This {m.kind} is too small for {label} (at least {low // 1024} KB).", permanent=True) + + +@contextlib.contextmanager +def media_for(a: Artifact | None, platform: str) -> Iterator[list[Media]]: + """The files to upload with a post of this artifact (none for text-only ones), size checked. Temporary video + files are deleted when the block ends, also on errors.""" + from app.services.storage import storage + + temps: list[str] = [] + try: + out: list[Media] = [] + if a is not None and a.type not in TEXT_ONLY and platform in MEDIA_PLATFORMS: + if a.type == "quote_card": + out = [Media("image", "quote.png", "image/png", data=storage.get_bytes(a.storage_key))] + elif a.type == "carousel": + keys = (a.content_json or {}).get("slide_keys") or [a.storage_key] + out = [Media("image", f"slide-{i + 1}.png", "image/png", data=storage.get_bytes(k)) + for i, k in enumerate(keys[:MAX_IMAGES[platform]])] + elif a.type == "upload" and upload_kind(a) == "image": + c = a.content_json or {} + out = [Media("image", c.get("filename") or "image", c.get("content_type") or "image/png", + data=storage.get_bytes(a.storage_key))] + elif a.type == "upload": + path = _temp_file(temps) + with open(path, "wb") as f: + storage.download_to(a.storage_key, f) + out = [Media("video", (a.content_json or {}).get("filename") or "video.mp4", "video/mp4", path=path)] + elif a.type == "video": + path = _temp_file(temps) + with open(path, "wb") as f: + if is_blog2video(a): + _download(a.content_json["video_url"], f) + else: + storage.download_to(a.storage_key, f) + out = [Media("video", "video.mp4", "video/mp4", path=path)] + for m in out: + check_size(m, platform) + yield out + finally: + for path in temps: + with contextlib.suppress(OSError): + os.unlink(path) + + +def _temp_file(temps: list[str]) -> str: + fd, path = tempfile.mkstemp(prefix="notestack-post-", suffix=".mp4") + os.close(fd) + temps.append(path) + return path + + +def _download(url: str, f) -> None: + with httpx.stream("GET", url, timeout=120, follow_redirects=True) as resp: + resp.raise_for_status() + for chunk in resp.iter_bytes(1024 * 1024): + f.write(chunk) diff --git a/backend/app/services/social/x.py b/backend/app/services/social/x.py index 63bfc306c9a6b3008a507e847ee8eea7fb6dfd94..b1ec4fb2b092091c4198a0740f758fe74b8c4f0d 100644 --- a/backend/app/services/social/x.py +++ b/backend/app/services/social/x.py @@ -1,8 +1,10 @@ -"""X (Twitter) API v2 with OAuth 2.0 user context (PKCE). Threads are reply chains.""" +"""X (Twitter) API v2 with OAuth 2.0 user context (PKCE). Threads are reply chains; images and a video go on the +first post (v2 media upload: one call for an image, initialize / append / finalize / status for a video).""" import base64 import hashlib import secrets +import time from urllib.parse import urlencode import httpx @@ -11,12 +13,14 @@ from sqlalchemy.orm import Session from app.config import settings from app.models import SocialAccount from app.services.crypto import decrypt -from app.services.social import SocialError, expired, save_tokens, token +from app.services.social import SocialError, expired, save_tokens, scope_set, token AUTH_URL = "https://x.com/i/oauth2/authorize" TOKEN_URL = "https://api.x.com/2/oauth2/token" API = "https://api.x.com/2" -SCOPES = "tweet.read tweet.write users.read offline.access" +SCOPES = "tweet.read tweet.write users.read offline.access media.write" +CHUNK = 4 * 1024 * 1024 # video upload segments +PROCESSING_WAIT = 600 # seconds X may take to process a video before we give up def redirect_uri() -> str: @@ -24,7 +28,7 @@ def redirect_uri() -> str: def configured() -> bool: - return bool(settings.x_client_id and settings.x_client_secret) + return bool(settings.x_enabled and settings.x_client_id and settings.x_client_secret) def pkce_pair() -> tuple[str, str]: @@ -59,25 +63,108 @@ def exchange_code(code: str, verifier: str) -> dict: "avatar_url": user.get("profile_image_url")} -def _access(db: Session, account: SocialAccount) -> str: - if expired(account) and account.refresh_token: +def refresh(db: Session, account: SocialAccount) -> None: + """Swap the refresh token for new tokens (X rotates the refresh token too). A refused refresh means the + connection is gone: reconnect.""" + if not account.refresh_token: + raise SocialError("The X connection expired. Reconnect X.", reconnect=True) + try: tokens = _token_request({"grant_type": "refresh_token", "refresh_token": decrypt(account.refresh_token)}) - save_tokens(db, account, tokens["access_token"], tokens.get("refresh_token"), tokens.get("expires_in")) + except SocialError as exc: + if exc.permanent: + raise SocialError("The X connection expired. Reconnect X.", reconnect=True) from exc + raise + save_tokens(db, account, tokens["access_token"], tokens.get("refresh_token"), tokens.get("expires_in")) + + +def _access(db: Session, account: SocialAccount) -> str: + if expired(account): + refresh(db, account) return token(account) -def publish(db: Session, account: SocialAccount, posts: list[str]) -> tuple[str, str]: +def check_alive(db: Session, account: SocialAccount) -> None: + """One cheap call: raises SocialError(reconnect=True) when X no longer accepts this connection.""" + resp = _call("GET", f"{API}/users/me", headers={"Authorization": f"Bearer {_access(db, account)}"}, timeout=20) + if resp.status_code == 401: + raise SocialError("X no longer accepts this connection. Reconnect X.", reconnect=True) + + +def _call(method: str, url: str, **kw) -> httpx.Response: + """Every request to X goes through here (tests replace it).""" + return httpx.request(method, url, timeout=kw.pop("timeout", 60), **kw) + + +def can_post_media(account: SocialAccount) -> bool: + """Accounts connected before media.write was asked for have to reconnect to post images or videos.""" + return "media.write" in scope_set(account.scopes) + + +def _media_error(resp: httpx.Response) -> SocialError: + if resp.status_code in (401, 403): + return SocialError("X refused the upload. Reconnect X to post images and videos.", reconnect=True) + return SocialError(f"X media upload {resp.status_code}: {resp.text[:300]}", + permanent=400 <= resp.status_code < 500 and resp.status_code != 429) + + +def _upload_image(headers: dict, m) -> str: + resp = _call("POST", f"{API}/media/upload", headers=headers, data={"media_category": "tweet_image"}, + files={"media": (m.filename, m.read(), m.content_type)}) + if resp.status_code >= 400: + raise _media_error(resp) + return resp.json()["data"]["id"] + + +def _upload_video(headers: dict, m) -> str: + resp = _call("POST", f"{API}/media/upload/initialize", headers=headers, + json={"media_type": m.content_type, "total_bytes": m.size, "media_category": "tweet_video"}) + if resp.status_code >= 400: + raise _media_error(resp) + media_id = resp.json()["data"]["id"] + for i, start in enumerate(range(0, m.size, CHUNK)): + resp = _call("POST", f"{API}/media/upload/{media_id}/append", headers=headers, data={"segment_index": i}, + files={"media": (m.filename, m.read(start, start + CHUNK), "application/octet-stream")}) + if resp.status_code >= 400: + raise _media_error(resp) + resp = _call("POST", f"{API}/media/upload/{media_id}/finalize", headers=headers) + if resp.status_code >= 400: + raise _media_error(resp) + info = (resp.json().get("data") or {}).get("processing_info") + waited = 0 + while info and info.get("state") in ("pending", "in_progress"): + if waited >= PROCESSING_WAIT: + raise SocialError("X took too long to process the video.") + pause = int(info.get("check_after_secs") or 5) + time.sleep(pause) + waited += pause + resp = _call("GET", f"{API}/media/upload", headers=headers, + params={"command": "STATUS", "media_id": media_id}) + if resp.status_code >= 400: + raise _media_error(resp) + info = (resp.json().get("data") or {}).get("processing_info") + if info and info.get("state") == "failed": + reason = (info.get("error") or {}).get("message") or "X could not process the video" + raise SocialError(f"{reason}. X takes videos up to 2:20.", permanent=True) + return media_id + + +def publish(db: Session, account: SocialAccount, posts: list[str], media: list | None = None) -> tuple[str, str]: headers = {"Authorization": f"Bearer {_access(db, account)}"} + media_ids: list[str] = [] + if media: + if not can_post_media(account): + raise SocialError("Reconnect X to post images and videos.", permanent=True) + media_ids = [(_upload_video if m.kind == "video" else _upload_image)(headers, m) for m in media] first_id, reply_to = None, None for text in posts: body: dict = {"text": text} if reply_to: body["reply"] = {"in_reply_to_tweet_id": reply_to} - resp = httpx.post(f"{API}/tweets", json=body, headers=headers, timeout=20) + elif media_ids: + body["media"] = {"media_ids": media_ids} # the images or video go on the first post + resp = _call("POST", f"{API}/tweets", json=body, headers=headers, timeout=20) if resp.status_code == 401: - account.status = "expired" - db.commit() - raise SocialError("X rejected the token. Reconnect the account.", permanent=True) + raise SocialError("X rejected the connection. Reconnect X.", reconnect=True) if resp.status_code >= 400: raise SocialError(f"X {resp.status_code}: {resp.text[:300]}", permanent=resp.status_code in (400, 403)) reply_to = resp.json()["data"]["id"] diff --git a/backend/app/services/storage.py b/backend/app/services/storage.py index f9a203b017e476b61a8bc086224e9b8ddbcb411a..9a9fa9cc8dfbac62ff7644d172666b6c042ba0d9 100644 --- a/backend/app/services/storage.py +++ b/backend/app/services/storage.py @@ -93,6 +93,10 @@ class Storage: def get_bytes(self, key: str) -> bytes: return self.client.get_object(Bucket=self.bucket, Key=self._k(key))["Body"].read() + def download_to(self, key: str, fileobj) -> None: + """Stream an object into an open binary file (large files, e.g. a video to post, never fully in memory).""" + self.client.download_fileobj(self.bucket, self._k(key), fileobj) + def head(self, key: str) -> dict | None: try: return self.client.head_object(Bucket=self.bucket, Key=self._k(key)) @@ -107,6 +111,14 @@ class Storage: def delete(self, key: str) -> None: self.client.delete_object(Bucket=self.bucket, Key=self._k(key)) + def list_keys(self, prefix: str) -> list[tuple[str, float]]: + """(key, last modified epoch seconds) of every object under the prefix.""" + out = [] + paginator = self.client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=self.bucket, Prefix=self._k(prefix)): + out += [(o["Key"][len(self.prefix):], o["LastModified"].timestamp()) for o in page.get("Contents", [])] + return out + def delete_prefix(self, prefix: str) -> int: deleted = 0 paginator = self.client.get_paginator("list_objects_v2") @@ -177,6 +189,13 @@ class LocalStorage: raise FileNotFoundError(key) return path.read_bytes() + def download_to(self, key: str, fileobj) -> None: + path = self.path(key) + if not path.is_file(): + raise FileNotFoundError(key) + with path.open("rb") as src: + shutil.copyfileobj(src, fileobj) + def head(self, key: str) -> dict | None: path = self.path(key) if not path.is_file(): @@ -189,6 +208,13 @@ class LocalStorage: def delete(self, key: str) -> None: self.path(key).unlink(missing_ok=True) + def list_keys(self, prefix: str) -> list[tuple[str, float]]: + base = self.path(prefix.rstrip("/")) + if not base.is_dir(): + return [] + return [(f"{prefix.rstrip('/')}/{f.relative_to(base).as_posix()}", f.stat().st_mtime) + for f in base.rglob("*") if f.is_file()] + def delete_prefix(self, prefix: str) -> int: base = self.path(prefix.rstrip("/")) if base.is_file(): diff --git a/backend/app/services/uploads.py b/backend/app/services/uploads.py new file mode 100644 index 0000000000000000000000000000000000000000..0e832458b7c188ee797dbb3d3b946ec97aefe059 --- /dev/null +++ b/backend/app/services/uploads.py @@ -0,0 +1,62 @@ +"""Files users upload (ws//uploads//): kept only while their upload record exists. + +Deleting a record through the app removes its file at commit (app.models.content). The daily sweep catches the rest: +files whose record is gone (deleted straight in the database, or a delete that failed) and uploads that were started +but never finished.""" + +import logging +import time +from collections.abc import Callable +from datetime import UTC, datetime, timedelta + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.db import SessionLocal +from app.models import Artifact, Upload, Workspace +from app.services.storage import storage + +log = logging.getLogger("notestack.storage") + +GRACE = timedelta(days=1) # a file or a pending upload younger than this may still be on its way + + +def remove_upload_artifact(db: Session, a: Artifact) -> None: + """An `upload` artifact is being deleted: its upload record goes too (and with it the file), unless another + artifact still posts the same file. The caller commits.""" + if a.type != "upload" or not a.storage_key: + return + shared = db.scalar(select(Artifact.id).where(Artifact.storage_key == a.storage_key, Artifact.id != a.id).limit(1)) + if shared: + return + for up in db.scalars(select(Upload).where(Upload.key == a.storage_key)): + db.delete(up) + + +def sweep_orphan_uploads(session_factory: Callable[[], Session] = SessionLocal) -> int: + """Delete upload files with no upload record, and uploads still pending after a day. Returns files removed.""" + removed = 0 + cutoff = datetime.now(UTC) - GRACE + with session_factory() as db: + # Started but never finished: drop the record (its file, if any, goes at commit). + for up in db.scalars(select(Upload).where(Upload.status == "pending", Upload.created_at < cutoff)): + db.delete(up) + db.commit() + known = set(db.scalars(select(Upload.key))) + for ws_id in db.scalars(select(Workspace.id)): + try: + files = storage.list_keys(f"ws/{ws_id}/uploads/") + except Exception: + log.warning("couldn't list uploads for workspace %s", ws_id, exc_info=True) + continue + for key, modified in files: + if key in known or time.time() - modified < GRACE.total_seconds(): + continue + try: + storage.delete(key) + removed += 1 + except Exception: + log.warning("couldn't delete orphan upload %s", key, exc_info=True) + if removed: + log.info("removed %d upload files with no record", removed) + return removed diff --git a/backend/app/services/usage.py b/backend/app/services/usage.py index c963cf024abacfb787e5b1a436eaecea6cbde181..11afde78aac95c3d5ab4fdeaa366be9525bf96c9 100644 --- a/backend/app/services/usage.py +++ b/backend/app/services/usage.py @@ -60,8 +60,15 @@ def usage_report(db: Session, workspace: Workspace) -> dict: "plan": plan.id, "used": used, "videos_resets_at": videos["resets_at"], - "limits": {"audio_minutes": plan.audio_minutes, "videos": videos["limit"], - "launch_kits": plan.launch_kits, "reports": plan.reports, "infographics": plan.infographics, "sources": plan.sources, "indexed_posts": plan.indexed_posts}, + "limits": { + "audio_minutes": plan.audio_minutes, + "videos": videos["limit"], + "launch_kits": plan.launch_kits, + "reports": plan.reports, + "infographics": plan.infographics, + "sources": plan.sources, + "indexed_posts": plan.indexed_posts, + }, } diff --git a/backend/app/support/scope.py b/backend/app/support/scope.py index aed372586dcec38f868ff9468e2eac5d065c5184..1ebbc5f1e2743a99029ca8ccab2052818f2c148d 100644 --- a/backend/app/support/scope.py +++ b/backend/app/support/scope.py @@ -47,7 +47,10 @@ _ABOUT_MY_DATA = re.compile( """ ) # Words that mean something went wrong with their work; the reply also offers the human form. -_BROKEN = re.compile(r"(?i)\b(?:fail\w*|slow|forever|too\s+long|taking\s+(?:so\s+|way\s+)?long|hang\w*|never\s+(?:finish|load|appear)\w*|error|broken|stuck|crash\w*|not\s+working|won'?t|doesn'?t\s+work|missing|lost)\b") +_BROKEN = re.compile( + r"(?i)\b(?:fail\w*|slow|forever|too\s+long|taking\s+(?:so\s+|way\s+)?long|hang\w*" + r"|never\s+(?:finish|load|appear)\w*|error|broken|stuck|crash\w*|not\s+working|won'?t|doesn'?t\s+work|missing|lost)\b" +) OUT_OF_SCOPE_REPLY = ( "I can help with how Notestack works, but I can't see your notebooks, posts, files or account, so I can't answer " diff --git a/backend/app/worker.py b/backend/app/worker.py index 6111cb09c260a12b76d347f14dbc6a193fc9e69b..6ae8cba7e8d9ca7186ced06b0228ae10227c8b67 100644 --- a/backend/app/worker.py +++ b/backend/app/worker.py @@ -64,7 +64,9 @@ from app.services.jobs import ( from app.services.notestack_voices import save_notestack_voice from app.services.plans import effective_plan from app.services.renderer import PermanentJobError, request_render +from app.services.social.health import social_health from app.services.storage import keys, storage +from app.services.uploads import sweep_orphan_uploads log = logging.getLogger("notestack.worker") @@ -267,11 +269,18 @@ def handle_render(db: Session, job: Job): return HANDED_OFF +def handle_publish_post(db: Session, job: Job): + item = launchpad.publish_job(db, job.params["item_id"]) + return Done({"status": item.status if item else "gone"}, "Published" if item and item.status == "posted" + else f"Post {item.status}" if item else "Post deleted") + + HANDLERS: dict[str, Callable[[Session, Job], object]] = { "ingest": handle_ingest, "import_url": handle_import_url, "import_upload": handle_import_upload, "topics": handle_topics, + "publish_post": handle_publish_post, "ideas": handle_ideas, "memory_update": handle_memory, "chat_memory": handle_chat_memory, @@ -424,6 +433,7 @@ def update_email_batch(session_factory: Callable[[], Session] = SessionLocal) -> def daily_cleanup(session_factory: Callable[[], Session] = SessionLocal) -> None: with session_factory() as db: purge_old_codes(db) + sweep_orphan_uploads(session_factory) # upload files whose record was deleted, and unfinished uploads @dataclass @@ -432,11 +442,27 @@ class Periodic: every_seconds: float fn: Callable[[], object] next_at: float = 0.0 + # Run on the wall clock's multiples of every_seconds (1800: at :00 and :30 UTC), not every_seconds after the last + # run. The first run is still at start-up, to catch up on anything already due. + align: bool = False + + def schedule_next(self, now: float, wall: float | None = None) -> None: + if not self.align: + self.next_at = now + self.every_seconds + return + wall = time.time() if wall is None else wall + # A couple of seconds past the boundary, so a post scheduled exactly on it is already due. + self.next_at = now + (self.every_seconds - wall % self.every_seconds) + ALIGN_GRACE_SECONDS + +ALIGN_GRACE_SECONDS = 2 PERIODIC = [ Periodic("recover_stale", 60, recover_stale), - Periodic("publish_due", 30, launchpad.publish_due), + # Posts are scheduled on half hours (in the user's zone), so they go out on the :00 / :30 run. Zones on a :45 + # offset (Nepal) get slots between runs: those go out at the next run, up to 15 minutes late. + Periodic("publish_due", 1800, launchpad.publish_due, align=True), + Periodic("social_health", 600, social_health), Periodic("sync_engagement", 3600, launchpad.sync_engagement), Periodic("update_email_batch", 300, update_email_batch), Periodic("topics_backfill", 900, topics_backfill), @@ -451,7 +477,7 @@ def run_periodic(now: float) -> None: for task in PERIODIC: if now < task.next_at: continue - task.next_at = now + task.every_seconds + task.schedule_next(now) try: with single_flight(task.name) as got: if got: diff --git a/backend/tests/test_auth.py b/backend/tests/test_auth.py index 96a479d105ba2d1f4b2655afb64bb2d9101979f8..eb5bf3b1307ac381ea94e7424244b1cad1260bec 100644 --- a/backend/tests/test_auth.py +++ b/backend/tests/test_auth.py @@ -35,6 +35,42 @@ def test_wrong_code_and_attempt_limit(client): assert r.json()["detail"]["code"] == "too_many_attempts" +def test_wrong_code_counts_down_and_lockout_allows_resend(client): + client.post("/api/auth/email/register/start", json={"email": "e@f.co", "password": "longenough"}) + if last_code() == "000000": + return + r = client.post("/api/auth/email/register/verify", json={"email": "e@f.co", "code": "000000"}) + assert r.json()["detail"]["message"].endswith("4 tries left.") + for _ in range(4): + r = client.post("/api/auth/email/register/verify", json={"email": "e@f.co", "code": "000000"}) + assert r.json()["detail"]["code"] == "too_many_attempts" + # A dead code does not hold up a new one, and the pending sign up survives the lockout. + assert client.post("/api/auth/email/register/resend", json={"email": "e@f.co"}).status_code == 200 + r = client.post("/api/auth/email/register/verify", json={"email": "e@f.co", "code": last_code()}) + assert r.status_code == 200 + + +def test_email_send_failure_is_reported(client, monkeypatch): + from app.services.email import email_service + + monkeypatch.setattr(email_service, "send_verification_code", lambda to, code: False) + r = client.post("/api/auth/email/register/start", json={"email": "f@g.co", "password": "longenough"}) + assert r.status_code == 502 + assert r.json()["detail"]["code"] == "email_failed" + + +def test_forgot_password_unknown_and_google(client, db_session): + from app.services.auth_identity import resolve_or_create_google_user + + r = client.post("/api/auth/password/forgot/start", json={"email": "nobody@example.com"}) + assert r.status_code == 404 + resolve_or_create_google_user(db_session, email="g2@example.com", google_id="456", name="G", avatar_url=None) + db_session.commit() + r = client.post("/api/auth/password/forgot/start", json={"email": "g2@example.com"}) + assert r.status_code == 409 + assert r.json()["detail"]["code"] == "wrong_provider" + + def test_resend_cooldown(client): client.post("/api/auth/email/register/start", json={"email": "c@d.co", "password": "longenough"}) r = client.post("/api/auth/email/register/resend", json={"email": "c@d.co"}) diff --git a/backend/tests/test_buildout.py b/backend/tests/test_buildout.py index 94263ab3af8e53cc28233d03254e1c285dcea51f..09392ca5cf0d00b003ee94dc2581204334b66e16 100644 --- a/backend/tests/test_buildout.py +++ b/backend/tests/test_buildout.py @@ -542,7 +542,10 @@ def test_launch_kit(client, auth, run_jobs, feed, llm, monkeypatch, db_session): "PickQuotes": {"quotes": [{"quote": "Charging more made my readers take the work seriously.", "source": ref}]}, }) + from app.services import renderer + monkeypatch.setattr(media, "request_render", lambda *a, **k: None) + monkeypatch.setattr(renderer, "available", lambda: True) art = client.post("/api/artifacts/generate", json={"type": "launch_kit", "document_id": doc["id"]}, headers=auth).json() run_jobs() @@ -563,6 +566,15 @@ def test_launch_kit(client, auth, run_jobs, feed, llm, monkeypatch, db_session): listing = client.get("/api/artifacts", params={"type": "launch_kit"}, headers=auth).json() assert listing["total"] == 1 + + # Renderer down: the kit keeps its quotes as text and makes no quote cards that could only fail. + monkeypatch.setattr(renderer, "available", lambda: False) + art = client.post("/api/artifacts/generate", json={"type": "launch_kit", "document_id": doc["id"]}, + headers=auth).json() + run_jobs() + kit = client.get(f"/api/artifacts/{art['id']}", headers=auth).json() + assert kit["status"] == "ready" + assert kit["content"]["quotes"] and kit["content"]["quote_card_ids"] == [] assert client.delete(f"/api/artifacts/{art['id']}", headers=auth).status_code == 200 @@ -580,14 +592,14 @@ def _bluesky(client, auth, monkeypatch): return r.json()["accounts"][0] -def test_schedule_and_publish_bluesky_thread(client, auth, monkeypatch, session_factory, db_session): +def test_schedule_and_publish_bluesky_thread(client, auth, monkeypatch, session_factory, db_session, run_jobs): from app.services import launchpad from app.services.social import bluesky account = _bluesky(client, auth, monkeypatch) published = {} - def publish(db, acct, posts): + def publish(db, acct, posts, media=None): published["posts"] = posts return "at://did:plc:ada/app.bsky.feed.post/abc", "https://bsky.app/profile/ada.bsky.social/post/abc" @@ -602,14 +614,15 @@ def test_schedule_and_publish_bluesky_thread(client, auth, monkeypatch, session_ }, headers=auth).json() assert item["auto_post"] is True - assert launchpad.publish_due(session_factory) == 1 + assert launchpad.publish_due(session_factory) == 1 # queued as a publish_post job + assert [j.kind for j in run_jobs()] == ["publish_post"] assert published["posts"] == ["First", "Second"] got = client.get("/api/calendar", headers=auth).json()[0] assert got["status"] == "posted" and got["external_url"].startswith("https://bsky.app/") assert client.patch(f"/api/calendar/{item['id']}", json={"content": "late"}, headers=auth).status_code == 409 -def test_substack_notes_send_reminder_email(client, auth, session_factory): +def test_substack_notes_send_reminder_email(client, auth, session_factory, run_jobs): from app.services import launchpad ConsoleEmailProvider.sent.clear() @@ -618,17 +631,18 @@ def test_substack_notes_send_reminder_email(client, auth, session_factory): headers=auth).json() assert item["remind_by_email"] is True launchpad.publish_due(session_factory) + run_jobs() assert client.get("/api/calendar", headers=auth).json()[0]["status"] == "reminded" assert ConsoleEmailProvider.sent[-1].subject == "Time to post on Substack Notes" -def test_publish_retries_then_fails(client, auth, monkeypatch, session_factory): +def test_publish_retries_then_fails(client, auth, monkeypatch, session_factory, run_jobs): from app.services import launchpad from app.services.social import SocialError, bluesky account = _bluesky(client, auth, monkeypatch) - def boom(db, acct, posts): + def boom(db, acct, posts, media=None): raise SocialError("Bluesky 500: busy") monkeypatch.setattr(bluesky, "publish", boom) @@ -642,6 +656,7 @@ def test_publish_retries_then_fails(client, auth, monkeypatch, session_factory): db.query(CalendarItem).update({"scheduled_at": datetime.now(UTC) - timedelta(seconds=1)}) db.commit() launchpad.publish_due(session_factory) + run_jobs() assert client.get("/api/calendar", headers=auth).json()[0]["status"] == "failed" @@ -697,6 +712,226 @@ def test_oauth_callback_redirects_with_ticket(client, monkeypatch): assert "error=" in bad.headers["location"] +# Launchpad: posting what was made (images and videos) to X and LinkedIn + + +def _made(db, ws, type_, *, status="ready", key=None, **content): + """An artifact as the pipeline leaves it (its file in storage when it has one).""" + from app.models import Artifact + from app.services.storage import storage + + if key: + storage.put_bytes(key, b"PNG" if key.endswith(".png") else bytes(range(256)) * 400) # a 100 KB "video" + a = Artifact(workspace_id=ws.id, type=type_, status=status, storage_key=key, + content_json={"title": f"A {type_}", **content}) + db.add(a) + db.commit() + return a + + +def _account(db, ws, platform, scopes="tweet.read tweet.write users.read offline.access media.write"): + from app.services.crypto import encrypt + + acct = SocialAccount(workspace_id=ws.id, platform=platform, handle="ada", external_id="p42", + access_token=encrypt("tok"), scopes=scopes, status="active") + db.add(acct) + db.commit() + return acct + + +def _reply(code: int, method: str, url: str, data: dict | None = None, headers: dict | None = None): + return httpx.Response(code, json=data, headers=headers, request=httpx.Request(method, url)) + + +def _schedule(client, auth, artifact, platform, account=None): + r = client.post("/api/calendar", json={ + "platform": platform, "content": "Look at this", "artifact_id": str(artifact.id), + "scheduled_at": (datetime.now(UTC) - timedelta(minutes=1)).isoformat(), + **({"social_account_id": str(account.id)} if account else {})}, headers=auth) + return r + + +def test_postable_lists_what_can_be_posted(client, auth, db_session): + ws = db_session.query(Workspace).first() + quote = _made(db_session, ws, "quote_card", key="ws/q/quotecard.png", quote="Ship it", source="Ada") + _made(db_session, ws, "audio_overview", key="ws/a/audio.mp3") + _made(db_session, ws, "video", status="processing", provider="blog2video") # not rendered yet + done = _made(db_session, ws, "video", status="processing", provider="blog2video", video_url="https://v/x.mp4") + unrendered = _made(db_session, ws, "video", status="ready", provider="blog2video", b2v_status="generated") + _made(db_session, ws, "summary", summary="Raise prices once a year [1].") + got = {a["id"]: a for a in client.get("/api/launchpad/postable", headers=auth).json()} + assert {a["type"] for a in got.values()} == {"quote_card", "video", "summary"} # no audio, no unfinished video + assert str(done.id) in got and got[str(quote.id)]["thumb_url"] and got[str(quote.id)]["media"] == "image" + # Made but not rendered: listed, flagged; the rendered one is not. + assert got[str(unrendered.id)]["needs_render"] is True and got[str(done.id)]["needs_render"] is False + assert got[str(quote.id)]["prefill"] == {"x": ['"Ship it" (Ada)'], "linkedin": '"Ship it" (Ada)'} + summary = next(a for a in got.values() if a["type"] == "summary") + assert summary["prefill"]["linkedin"] == "Raise prices once a year." and summary["media"] == "text" + + +def test_scheduling_refuses_what_cannot_go_out(client, auth, db_session): + ws = db_session.query(Workspace).first() + audio = _made(db_session, ws, "audio_overview", key="ws/a/audio.mp3") + long_video = _made(db_session, ws, "video", key="ws/v/audiogram.mp4", duration_s=200) + quote = _made(db_session, ws, "quote_card", key="ws/q/quotecard.png") + assert _schedule(client, auth, audio, "x").status_code == 400 + r = _schedule(client, auth, long_video, "x") + assert r.status_code == 400 and "2:20" in r.json()["detail"] + linkedin_acct = _account(db_session, ws, "linkedin", scopes="openid profile w_member_social") + assert _schedule(client, auth, long_video, "linkedin", linkedin_acct).status_code == 200 # LinkedIn takes it + assert _schedule(client, auth, quote, "bluesky").status_code == 400 # images: X and LinkedIn only + + +def test_x_posts_images_and_a_video(client, auth, db_session, monkeypatch, session_factory, run_jobs): + from app.services import launchpad + from app.services.social import x + + ws = db_session.query(Workspace).first() + acct = _account(db_session, ws, "x") + calls = [] + + def call(method, url, **kw): + path = url.split("/2/", 1)[1] + calls.append((method, path, kw)) + if path == "media/upload" and method == "POST": + return _reply(200, method, url, {"data": {"id": f"img{len(calls)}"}}) + if path == "media/upload/initialize": + return _reply(200, method, url, {"data": {"id": "vid1"}}) + if path.endswith("/append"): + return _reply(204, method, url) + if path.endswith("/finalize"): + return _reply(200, method, url, {"data": {"processing_info": {"state": "pending", "check_after_secs": 1}}}) + if path == "media/upload": # STATUS + return _reply(200, method, url, {"data": {"processing_info": {"state": "succeeded"}}}) + return _reply(201, method, url, {"data": {"id": "t1"}}) # tweets + + monkeypatch.setattr(x, "_call", call) + monkeypatch.setattr(x.time, "sleep", lambda s: None) + monkeypatch.setattr(x, "CHUNK", 30000) + carousel = _made(db_session, ws, "carousel", key="ws/c/slide-01.png", + slide_keys=[f"ws/c/slide-0{i}.png" for i in range(1, 7)]) + for i in range(1, 7): + from app.services.storage import storage + storage.put_bytes(f"ws/c/slide-0{i}.png", b"PNG") + assert _schedule(client, auth, carousel, "x", acct).status_code == 200 + launchpad.publish_due(session_factory) + run_jobs() + uploads = [c for c in calls if c[1] == "media/upload"] + tweet = next(c for c in calls if c[1] == "tweets") + assert len(uploads) == 4 # X takes 4 images: the first 4 slides + assert tweet[2]["json"]["media"]["media_ids"] == [f"img{i}" for i in range(1, 5)] + assert client.get("/api/calendar", headers=auth).json()[0]["status"] == "posted" + + calls.clear() + video = _made(db_session, ws, "video", key="ws/v/audiogram.mp4", duration_s=30) + _schedule(client, auth, video, "x", acct) + launchpad.publish_due(session_factory) + run_jobs() + paths = [c[1] for c in calls] + assert paths[0] == "media/upload/initialize" and paths.count("media/upload/vid1/append") == 4 # 102400 bytes + assert paths[-3:] == ["media/upload/vid1/finalize", "media/upload", "tweets"] + assert calls[-1][2]["json"]["media"] == {"media_ids": ["vid1"]} + + +def test_x_accounts_from_before_media_must_reconnect(client, auth, db_session, monkeypatch, session_factory, + run_jobs): + from app.services import launchpad + from app.services.social import x + + ws = db_session.query(Workspace).first() + acct = _account(db_session, ws, "x", scopes="tweet.read tweet.write users.read offline.access") + monkeypatch.setattr(x, "_call", lambda *a, **k: pytest.fail("nothing should be sent")) + quote = _made(db_session, ws, "quote_card", key="ws/q/quotecard.png") + r = _schedule(client, auth, quote, "x", acct) # refused up front now + assert r.status_code == 409 and "Reconnect X" in r.json()["detail"]["message"] + got = client.get("/api/social/accounts", headers=auth).json()["accounts"][0] + assert got["needs_reconnect"] is True and got["can_post_media"] is False + # Scheduled earlier, the scope gone since: refused at publish time without a call, as a reconnect. + with session_factory() as db: + db.add(CalendarItem(workspace_id=ws.id, platform="x", content="Hi", artifact_id=quote.id, + social_account_id=acct.id, scheduled_at=datetime.now(UTC) - timedelta(minutes=1))) + db.commit() + launchpad.publish_due(session_factory) + run_jobs() + item = client.get("/api/calendar", headers=auth).json()[0] + assert item["status"] == "paused" and "Reconnect X" in item["error"] + + +def test_linkedin_posts_images_and_a_video(client, auth, db_session, monkeypatch, session_factory, run_jobs): + from app.services import launchpad + from app.services.social import linkedin + + ws = db_session.query(Workspace).first() + acct = _account(db_session, ws, "linkedin", scopes="openid profile w_member_social") + calls = [] + + def call(method, url, **kw): + calls.append((method, url, kw)) + if url.endswith("/rest/images"): + return _reply(200, method, url, {"value": {"uploadUrl": f"https://up/img{len(calls)}", + "image": f"urn:li:image:{len(calls)}"}}) + if url.endswith("/rest/videos") and kw["params"]["action"] == "initializeUpload": + return _reply(200, method, url, {"value": {"video": "urn:li:video:9", "uploadToken": "", + "uploadInstructions": [ + {"uploadUrl": "https://up/v1", "firstByte": 0, "lastByte": 59999}, + {"uploadUrl": "https://up/v2", "firstByte": 60000, + "lastByte": 102399}]}}) + if url.startswith("https://up/"): + return _reply(200, method, url, headers={"etag": f'"e-{url[-2:]}"'}) + if "/rest/videos/" in url: + return _reply(200, method, url, {"status": "AVAILABLE"}) + if url.endswith("/rest/posts"): + return _reply(201, method, url, headers={"x-restli-id": "urn:li:share:1"}) + return _reply(200, method, url) # finalize + + monkeypatch.setattr(linkedin, "_call", call) + quote = _made(db_session, ws, "quote_card", key="ws/q/quotecard.png") + carousel = _made(db_session, ws, "carousel", key="ws/c/slide-01.png", + slide_keys=["ws/q/quotecard.png"] * 3) + video = _made(db_session, ws, "video", key="ws/v/audiogram.mp4", duration_s=200) + bodies = [] + for a in (quote, carousel, video): + calls.clear() + _schedule(client, auth, a, "linkedin", acct) + launchpad.publish_due(session_factory) + run_jobs() + bodies.append(next(c for c in calls if c[1].endswith("/rest/posts"))[2]["json"]["content"]) + if a is video: + puts = [c for c in calls if c[1].startswith("https://up/v")] + assert [len(c[2]["content"]) for c in puts] == [60000, 42400] + fin = next(c for c in calls if c[1].endswith("/rest/videos") and c[2]["params"]["action"] == "finalizeUpload") + assert fin[2]["json"]["finalizeUploadRequest"]["uploadedPartIds"] == ["e-v1", "e-v2"] + assert bodies[0] == {"media": {"id": "urn:li:image:1"}} + assert len(bodies[1]["multiImage"]["images"]) == 3 + assert bodies[2] == {"media": {"id": "urn:li:video:9"}} + assert {i["status"] for i in client.get("/api/calendar", headers=auth).json()} == {"posted"} + + +def test_publish_now_with_media_goes_through_a_job(client, auth, db_session, monkeypatch, run_jobs): + from app.services.social import linkedin + + ws = db_session.query(Workspace).first() + acct = _account(db_session, ws, "linkedin", scopes="openid profile w_member_social") + monkeypatch.setattr(linkedin, "publish", lambda db, a, posts, media=None: ("urn:li:share:2", "https://li/2")) + quote = _made(db_session, ws, "quote_card", key="ws/q/quotecard.png") + item = client.post("/api/calendar", json={"platform": "linkedin", "content": "Hi", "artifact_id": str(quote.id), + "social_account_id": str(acct.id), + "scheduled_at": (datetime.now(UTC) + timedelta(days=1)).isoformat()}, + headers=auth).json() + assert item["artifact"]["type"] == "quote_card" + r = client.post(f"/api/calendar/{item['id']}/publish", headers=auth).json() + assert r["status"] == "publishing" # the upload runs in a job + run_jobs() + assert client.get("/api/calendar", headers=auth).json()[0]["status"] == "posted" + # The attachment can be changed or removed before it goes out. + other = client.post("/api/calendar", json={"platform": "linkedin", "content": "Later", "artifact_id": str(quote.id), + "social_account_id": str(acct.id), + "scheduled_at": (datetime.now(UTC) + timedelta(days=1)).isoformat()}, + headers=auth).json() + r = client.patch(f"/api/calendar/{other['id']}", json={"artifact_id": None}, headers=auth).json() + assert r["artifact"] is None + + # Settings and activity @@ -1587,7 +1822,6 @@ def test_infographic_is_a_page_at_once_and_can_be_shared_by_link(client, auth, r def test_infographic_needs_a_source_and_enough_text_and_obeys_its_limit(client, auth, run_jobs, feed, llm, monkeypatch): - from app.pipeline import infographic pricing, nb = _notebook_with_pricing(client, auth, run_jobs) llm["ExtractIdeas"] = {"ideas": []} diff --git a/backend/tests/test_social_posting.py b/backend/tests/test_social_posting.py new file mode 100644 index 0000000000000000000000000000000000000000..f7b3385b4c59d67750ff7d8adf2bd8b8decea794 --- /dev/null +++ b/backend/tests/test_social_posting.py @@ -0,0 +1,554 @@ +"""Posting to X and LinkedIn: scheduling needs a live connection, a connection that goes down pauses its posts (never +attempted) and reconnecting resumes them, connections are kept fresh, posts go out at their time, and uploads stream +from temporary files with size limits and retries. The platforms are faked at their _call seam.""" + +import os +import uuid +from datetime import UTC, datetime, timedelta + +import httpx +import pytest + +from app.models import Artifact, CalendarItem, SocialAccount, Workspace +from app.services.crypto import encrypt +from tests.conftest import last_code + +X_SCOPES = "tweet.read tweet.write users.read offline.access media.write" +LI_SCOPES = "openid,profile,w_member_social" # LinkedIn lists scopes with commas + + +@pytest.fixture() +def auth(client): + client.post("/api/auth/email/register/start", json={"email": "ada@example.com", "password": "stardust-42", + "name": "Ada Lovelace"}) + r = client.post("/api/auth/email/register/verify", json={"email": "ada@example.com", "code": last_code()}) + return {"Authorization": f"Bearer {r.json()['access_token']}"} + + +@pytest.fixture() +def ws(auth, db_session): + return db_session.query(Workspace).first() + + +@pytest.fixture(autouse=True) +def _fresh_health(monkeypatch): + from app.services.social import health + + monkeypatch.setattr(health, "_last_checked", {}) + + +def account(db, ws, platform="x", scopes=None, expires_in=None, refresh=None, external_id="p42"): + acct = SocialAccount(workspace_id=ws.id, platform=platform, handle="ada", external_id=external_id, + access_token=encrypt("tok"), refresh_token=encrypt(refresh) if refresh else None, + scopes=scopes or (X_SCOPES if platform == "x" else LI_SCOPES), status="active", + expires_at=datetime.now(UTC) + expires_in if expires_in is not None else None) + db.add(acct) + db.commit() + return acct + + +def item(db, ws, acct, minutes=60, status="scheduled", **kw): + it = CalendarItem(workspace_id=ws.id, platform=acct.platform, content="Hello", social_account_id=acct.id, + scheduled_at=datetime.now(UTC) + timedelta(minutes=minutes), status=status, **kw) + db.add(it) + db.commit() + return it + + +def made(db, ws, type_, key=None, data=b"PNG", **content): + from app.services.storage import storage + + if key: + storage.put_bytes(key, data) + a = Artifact(workspace_id=ws.id, type=type_, status="ready", storage_key=key, + content_json={"title": type_, **content}) + db.add(a) + db.commit() + return a + + +def reply(code, method, url, data=None, headers=None): + return httpx.Response(code, json=data, headers=headers, request=httpx.Request(method, url)) + + +def statuses(db, *items): + db.expire_all() + return [db.get(CalendarItem, i.id).status for i in items] + + +# Scheduling needs a live connection + + +def schedule(client, auth, platform, acct=None, **extra): + return client.post("/api/calendar", json={ + "platform": platform, "content": "Hello", "scheduled_at": (datetime.now(UTC) + timedelta(hours=1)).isoformat(), + **({"social_account_id": str(acct.id)} if acct else {}), **extra}, headers=auth) + + +def test_x_and_linkedin_need_a_connected_account(client, auth, ws, db_session): + r = schedule(client, auth, "x") + assert r.status_code == 409 and r.json()["detail"]["code"] == "not_connected" + assert schedule(client, auth, "linkedin").status_code == 409 + down = account(db_session, ws, "linkedin") + down.status = "expired" + db_session.commit() + r = schedule(client, auth, "linkedin", down) + assert r.status_code == 409 and "Reconnect LinkedIn" in r.json()["detail"]["message"] + assert schedule(client, auth, "x", account(db_session, ws, "x")).status_code == 200 + assert schedule(client, auth, "bluesky").status_code == 200 # no API connection needed: an email reminder + + +def test_linkedin_scopes_are_read_with_commas_or_spaces(client, auth, ws, db_session): + acct = account(db_session, ws, "linkedin", scopes="openid profile w_member_social") + quote = made(db_session, ws, "quote_card", key="ws/q/q.png") + assert schedule(client, auth, "linkedin", acct, artifact_id=str(quote.id)).status_code == 200 + got = client.get("/api/social/accounts", headers=auth).json()["accounts"][0] + assert got["can_post_media"] is True and got["needs_reconnect"] is False + + +# A connection that goes down pauses its posts; reconnecting resumes them + + +def test_disconnecting_pauses_its_posts_and_nothing_is_sent(client, auth, ws, db_session, session_factory, + monkeypatch): + from app.services import launchpad + from app.services.social import x + + monkeypatch.setattr(x, "publish", lambda *a, **k: pytest.fail("a paused post was sent")) + acct = account(db_session, ws, "x") + later, due = item(db_session, ws, acct), item(db_session, ws, acct, minutes=-1) + assert client.delete(f"/api/social/accounts/{acct.id}", headers=auth).status_code == 200 + db_session.expire_all() + assert db_session.get(SocialAccount, acct.id).status == "revoked" + assert statuses(db_session, later, due) == ["paused", "paused"] + assert launchpad.publish_due(session_factory) == 0 # paused posts are never claimed + got = client.get("/api/social/accounts", headers=auth).json()["accounts"][0] + assert got["needs_reconnect"] is True + + +def test_a_refused_connection_at_publish_pauses_all_its_posts(client, auth, ws, db_session, session_factory, + run_jobs, monkeypatch): + from app.services import launchpad + from app.services.social import x + + acct = account(db_session, ws, "x") + monkeypatch.setattr(x, "_call", lambda m, url, **k: reply(401, m, url, {"title": "Unauthorized"})) + due, later = item(db_session, ws, acct, minutes=-1), item(db_session, ws, acct, minutes=90) + launchpad.publish_due(session_factory) + run_jobs() + db_session.expire_all() + assert db_session.get(SocialAccount, acct.id).status == "expired" + assert statuses(db_session, due, later) == ["paused", "paused"] + assert "Reconnect X" in db_session.get(CalendarItem, due.id).error + + +def test_reconnecting_resumes_future_posts_and_flags_missed_ones(client, auth, ws, db_session): + import json + + from app.auth import encode_signed + + acct = account(db_session, ws, "x") + ahead, missed = item(db_session, ws, acct, minutes=60), item(db_session, ws, acct, minutes=-30) + client.delete(f"/api/social/accounts/{acct.id}", headers=auth) + info = {"access_token": "new", "refresh_token": "r", "expires_in": 7200, "external_id": "p42", "handle": "ada", + "scope": X_SCOPES} + ticket = encode_signed({"typ": "social_link", "ws": str(ws.id), "platform": "x", + "data": encrypt(json.dumps(info))}, timedelta(minutes=5)) + assert client.post("/api/social/complete", json={"ticket": ticket}, headers=auth).status_code == 200 + db_session.expire_all() + assert db_session.get(SocialAccount, acct.id).status == "active" # the same account, back + assert statuses(db_session, ahead, missed) == ["scheduled", "paused"] + assert "pick a new time" in db_session.get(CalendarItem, missed.id).error + # The missed one goes back out once given a time still ahead. + r = client.patch(f"/api/calendar/{missed.id}", headers=auth, + json={"scheduled_at": (datetime.now(UTC) + timedelta(hours=2)).isoformat()}) + assert r.status_code == 200 and r.json()["status"] == "scheduled" + r = client.patch(f"/api/calendar/{ahead.id}", json={"status": "paused"}, headers=auth) # held by hand + assert r.json()["status"] == "paused" + + +# Keeping connections fresh + + +def test_x_tokens_are_refreshed_before_they_expire(ws, db_session, session_factory, monkeypatch): + from app.services.crypto import decrypt + from app.services.social import health, x + + acct = account(db_session, ws, "x", expires_in=timedelta(minutes=5), refresh="old-refresh") + sent = [] + monkeypatch.setattr(x, "_token_request", lambda data: sent.append(data) or { + "access_token": "fresh", "refresh_token": "rotated", "expires_in": 7200}) + monkeypatch.setattr(x, "_call", lambda m, url, **k: reply(200, m, url, {"data": {"id": "p42"}})) + assert health.social_health(session_factory) == 1 + db_session.expire_all() + acct = db_session.get(SocialAccount, acct.id) + assert sent[0]["grant_type"] == "refresh_token" and decrypt(acct.access_token) == "fresh" + assert decrypt(acct.refresh_token) == "rotated" and acct.status == "active" + + +def test_a_refused_refresh_or_expiry_takes_the_connection_down(ws, db_session, session_factory, monkeypatch): + from app.services.social import SocialError, health, x + + def refused(data): + raise SocialError("invalid_grant", permanent=True) + + monkeypatch.setattr(x, "_token_request", refused) + xa = account(db_session, ws, "x", expires_in=timedelta(minutes=5), refresh="old") + li = account(db_session, ws, "linkedin", expires_in=timedelta(minutes=-1), external_id="li1") # no refresh token + posts = [item(db_session, ws, xa), item(db_session, ws, li)] + health.social_health(session_factory) + db_session.expire_all() + assert [db_session.get(SocialAccount, a.id).status for a in (xa, li)] == ["expired", "expired"] + assert statuses(db_session, *posts) == ["paused", "paused"] + + +def test_access_revoked_on_the_platform_is_noticed(ws, db_session, session_factory, monkeypatch): + from app.services.social import health, linkedin + + acct = account(db_session, ws, "linkedin") + post = item(db_session, ws, acct) + monkeypatch.setattr(linkedin, "_call", lambda m, url, **k: reply(401, m, url, {})) + health.social_health(session_factory) + db_session.expire_all() + assert db_session.get(SocialAccount, acct.id).status == "revoked" + assert statuses(db_session, post) == ["paused"] + + +def test_liveness_is_asked_at_most_every_six_hours(ws, db_session, session_factory, monkeypatch): + from app.services.social import health, x + + account(db_session, ws, "x") + calls = [] + monkeypatch.setattr(x, "_call", lambda m, url, **k: calls.append(url) or reply(200, m, url, {"data": {}})) + health.social_health(session_factory) + health.social_health(session_factory) + assert len(calls) == 1 + + +# On time + + +def test_a_due_post_goes_out_through_its_job(client, auth, ws, db_session, session_factory, run_jobs, monkeypatch): + from app.services import launchpad + from app.services.social import x + + acct = account(db_session, ws, "x") + started = [] + monkeypatch.setattr(x, "publish", lambda db, a, posts, media=None: started.append( + db.get(CalendarItem, due.id).updated_at) or ("t1", "https://x.com/ada/status/t1")) + due = item(db_session, ws, acct, minutes=0) + claimed_at = datetime.now(UTC) + assert launchpad.publish_due(session_factory) == 1 + run_jobs() + assert statuses(db_session, due) == ["posted"] + stamp = started[0] if started[0].tzinfo else started[0].replace(tzinfo=UTC) + assert stamp >= claimed_at - timedelta(seconds=1) # the job touched it as it started + + +# Uploads + + +def test_a_video_is_streamed_to_a_temp_file_that_is_removed(ws, db_session, monkeypatch): + from app.services.social import media + + video = made(db_session, ws, "video", key="ws/v/a.mp4", data=os.urandom(100_000), duration_s=30) + with media.media_for(video, "linkedin") as files: + path = files[0].path + assert os.path.exists(path) and files[0].size == 100_000 and files[0].data is None + assert files[0].read(10, 20) == files[0].read()[10:20] + assert not os.path.exists(path) + with pytest.raises(RuntimeError), media.media_for(video, "x") as files: + path = files[0].path + raise RuntimeError("upload broke") + assert not os.path.exists(path) # also on errors + + +def test_size_limits_fail_before_any_call(ws, db_session, monkeypatch): + from app.services.social import SocialError, media + + tiny = made(db_session, ws, "video", key="ws/v/tiny.mp4", data=b"x" * 1000, duration_s=5) + with pytest.raises(SocialError, match="too small for LinkedIn"), media.media_for(tiny, "linkedin"): + pass + monkeypatch.setitem(media.SIZE_LIMITS, ("x", "video"), (1, 500)) + with pytest.raises(SocialError, match="X takes up to"), media.media_for(tiny, "x"): + pass + + +def _linkedin(monkeypatch, part_codes: list[int]): + """A fake LinkedIn: initializeUpload with 2 parts, part PUTs answering from part_codes, then AVAILABLE.""" + from app.services.social import linkedin + + calls, received = [], {} + codes = iter(part_codes) + + def call(method, url, **kw): + calls.append((method, url, kw)) + if url.endswith("/rest/videos") and kw.get("params", {}).get("action") == "initializeUpload": + size = kw["json"]["initializeUploadRequest"]["fileSizeBytes"] + return reply(200, method, url, {"value": {"video": "urn:li:video:1", "uploadToken": "", "uploadInstructions": [ + {"uploadUrl": "https://up/1", "firstByte": 0, "lastByte": 49_999}, + {"uploadUrl": "https://up/2", "firstByte": 50_000, "lastByte": size - 1}]}}) + if url.startswith("https://up/"): + code = next(codes, 200) + if code < 400: + received[url] = kw["content"] + return reply(code, method, url, {} if code >= 400 else None, headers={"etag": f'"e{url[-1]}"'}) + if "/rest/videos/" in url: + return reply(200, method, url, {"status": "AVAILABLE"}) + if url.endswith("/rest/posts"): + return reply(201, method, url, headers={"x-restli-id": "urn:li:share:9"}) + return reply(200, method, url) + + monkeypatch.setattr(linkedin, "_call", call) + monkeypatch.setattr(linkedin.time, "sleep", lambda s: None) + return calls, received + + +def test_linkedin_video_parts_retry_and_arrive_byte_identical(ws, db_session, monkeypatch): + from app.services.social import linkedin, media + + data = os.urandom(90_000) + acct = account(db_session, ws, "linkedin") + video = made(db_session, ws, "video", key="ws/v/b.mp4", data=data, duration_s=30) + calls, received = _linkedin(monkeypatch, [503, 200, 200]) # the first part fails once + with media.media_for(video, "linkedin") as files: + linkedin.publish(db_session, acct, ["Watch"], files) + assert received["https://up/1"] + received["https://up/2"] == data + part_puts = [c for c in calls if c[1].startswith("https://up/")] + assert len(part_puts) == 3 and all("Authorization" not in c[2].get("headers", {}) for c in part_puts) + + +def test_linkedin_retired_api_version_is_permanent(ws, db_session, monkeypatch): + from app.services.social import SocialError, linkedin + + acct = account(db_session, ws, "linkedin") + monkeypatch.setattr(linkedin, "_call", lambda m, url, **k: reply(426, m, url, {"code": "NONEXISTENT_VERSION"})) + with pytest.raises(SocialError, match="retired API version") as err: + linkedin.publish(db_session, acct, ["Hi"], []) + assert err.value.permanent and not err.value.reconnect + + +def test_x_video_processing_failure_is_permanent(ws, db_session, monkeypatch): + from app.services.social import SocialError, media, x + + acct = account(db_session, ws, "x") + video = made(db_session, ws, "video", key="ws/v/c.mp4", data=os.urandom(100_000), duration_s=30) + + def call(method, url, **kw): + path = url.split("/2/", 1)[1] + if path == "media/upload/initialize": + return reply(200, method, url, {"data": {"id": "v1"}}) + if path.endswith("/append"): + return reply(204, method, url) + if path.endswith("/finalize"): + return reply(200, method, url, {"data": {"processing_info": {"state": "pending", "check_after_secs": 1}}}) + return reply(200, method, url, {"data": {"processing_info": {"state": "failed", + "error": {"message": "Unsupported video"}}}}) + + monkeypatch.setattr(x, "_call", call) + monkeypatch.setattr(x.time, "sleep", lambda s: None) + with pytest.raises(SocialError, match="Unsupported video") as err, media.media_for(video, "x") as files: + x.publish(db_session, acct, ["Watch"], files) + assert err.value.permanent + + +def test_unfinished_blog2video_video_is_refused_when_scheduled(client, auth, ws, db_session): + acct = account(db_session, ws, "x") + video = Artifact(workspace_id=ws.id, type="video", status="processing", + content_json={"title": "Draft", "provider": "blog2video"}) + db_session.add(video) + db_session.commit() + r = schedule(client, auth, "x", acct, artifact_id=str(video.id)) + assert r.status_code == 400 and "isn't finished" in r.json()["detail"] + assert uuid.UUID(str(video.id)) + + +def test_a_video_being_rendered_stays_listed_as_rendering(client, auth, ws, db_session): + video = Artifact(workspace_id=ws.id, type="video", status="rendering", + content_json={"title": "Mid render", "provider": "blog2video"}) + db_session.add(video) + db_session.commit() + listed = {a["id"]: a for a in client.get("/api/launchpad/postable", headers=auth).json()} + item = listed[str(video.id)] + assert item["needs_render"] and item["rendering"] + + +def test_a_kit_is_tracked_apart_and_never_attached(client, auth, ws, db_session): + acct = account(db_session, ws, "x") + kit = made(db_session, ws, "launch_kit", post_title="The Boring Portfolio", posts={"x_thread": ["One", "Two"]}) + video = made(db_session, ws, "video", key="ws/v/k.mp4", data=os.urandom(100_000), duration_s=30) + r = schedule(client, auth, "x", acct, artifact_id=str(kit.id)) + assert r.status_code == 400 and "can't be attached" in r.json()["detail"] + r = schedule(client, auth, "x", acct, kit_id=str(kit.id), artifact_id=str(video.id)) + assert r.status_code == 200 + got = r.json() + assert got["kit_id"] == str(kit.id) and got["kit_title"] == "The Boring Portfolio" + assert got["artifact"]["type"] == "video" # the attachment is the video, not the kit + assert schedule(client, auth, "x", acct, kit_id=str(video.id)).status_code == 404 # only a Launch Kit is a kit + + +def test_a_time_picked_in_the_users_zone_is_stored_in_utc(client, auth, ws, db_session): + acct = account(db_session, ws, "linkedin", scopes=LI_SCOPES) + year = datetime.now(UTC).year + 1 + r = schedule(client, auth, "linkedin", acct, local_time=f"{year}-10-07T11:00", timezone="Asia/Karachi") + assert r.status_code == 200, r.text + assert datetime.fromisoformat(r.json()["scheduled_at"]) == datetime(year, 10, 7, 6, 0, tzinfo=UTC) + # New York is UTC-4 in summer and UTC-5 in winter: the same wall time lands an hour apart. + summer = schedule(client, auth, "linkedin", acct, local_time=f"{year}-07-01T09:30", timezone="America/New_York") + winter = schedule(client, auth, "linkedin", acct, local_time=f"{year}-12-01T09:30", timezone="America/New_York") + assert datetime.fromisoformat(summer.json()["scheduled_at"]) == datetime(year, 7, 1, 13, 30, tzinfo=UTC) + assert datetime.fromisoformat(winter.json()["scheduled_at"]) == datetime(year, 12, 1, 14, 30, tzinfo=UTC) + # Moving it keeps to the user's zone too. + item_id = r.json()["id"] + moved = client.patch(f"/api/calendar/{item_id}", json={"local_time": f"{year}-10-08T00:30", "timezone": "Asia/Karachi"}, + headers=auth) + assert datetime.fromisoformat(moved.json()["scheduled_at"]) == datetime(year, 10, 7, 19, 30, tzinfo=UTC) + + +def test_only_half_hours_in_a_known_zone_are_taken(client, auth, ws, db_session): + acct = account(db_session, ws, "linkedin", scopes=LI_SCOPES) + year = datetime.now(UTC).year + 1 + off = schedule(client, auth, "linkedin", acct, local_time=f"{year}-10-07T11:15", timezone="Asia/Karachi") + assert off.status_code == 400 and off.json()["detail"]["code"] == "half_hour_only" + bad = schedule(client, auth, "linkedin", acct, local_time=f"{year}-10-07T11:00", timezone="Mars/Olympus") + assert bad.status_code == 400 and bad.json()["detail"]["code"] == "bad_timezone" + assert schedule(client, auth, "linkedin", acct).status_code == 200 # an exact scheduled_at (Post now) still works + + +def test_the_publisher_runs_on_the_half_hour(): + from app.worker import ALIGN_GRACE_SECONDS, Periodic + + task = Periodic("publish_due", 1800, lambda: None, align=True) + wall = datetime(2026, 10, 7, 6, 10, tzinfo=UTC).timestamp() # 06:10 UTC: the next run is at 06:30 + task.schedule_next(100.0, wall) + assert task.next_at == 100.0 + 20 * 60 + ALIGN_GRACE_SECONDS + + +def uploaded(db, ws, filename, content_type, data, status="complete"): + """A file the user uploaded through /api/storage/uploads, in storage and recorded.""" + from app.models import Upload, User + from app.services.storage import keys, storage + + upload_id = uuid.uuid4() + key = keys.upload(ws.id, upload_id, filename) + storage.put_bytes(key, data) + up = Upload(id=upload_id, workspace_id=ws.id, user_id=db.query(User).first().id, key=key, filename=filename, + content_type=content_type, size_bytes=len(data), status=status) + db.add(up) + db.commit() + return up + + +def test_a_file_from_the_computer_is_attached_and_posted(client, auth, ws, db_session): + from app.services.social import media + + acct = account(db_session, ws, "linkedin") + png = uploaded(db_session, ws, "chart.png", "image/png", b"\x89PNG-chart") + r = client.post("/api/launchpad/uploads", json={"upload_id": str(png.id)}, headers=auth) + assert r.status_code == 200, r.text + brief = r.json() + assert brief["type"] == "upload" and brief["media"] == "image" and brief["title"] == "chart" + assert brief["thumb_url"] and brief["prefill"]["linkedin"] == "" + listed = {a["id"] for a in client.get("/api/launchpad/postable", headers=auth).json()} + assert brief["id"] in listed + assert schedule(client, auth, "linkedin", acct, artifact_id=brief["id"]).status_code == 200 + art = db_session.get(Artifact, uuid.UUID(brief["id"])) + with media.media_for(art, "linkedin") as files: + assert [(f.kind, f.content_type, f.read()) for f in files] == [("image", "image/png", b"\x89PNG-chart")] + # Not one of the things Notestack made: kept out of the general artifact list. + assert brief["id"] not in {a["id"] for a in client.get("/api/artifacts", headers=auth).json()["items"]} + + +def test_an_uploaded_video_goes_out_from_a_temporary_file(client, auth, ws, db_session): + from app.services.social import media + + mp4 = uploaded(db_session, ws, "clip.mp4", "video/mp4", b"v" * (80 * 1024)) + r = client.post("/api/launchpad/uploads", json={"upload_id": str(mp4.id), "duration_s": 42}, headers=auth) + assert r.status_code == 200 and r.json()["media"] == "video" and r.json()["duration_s"] == 42 + art = db_session.get(Artifact, uuid.UUID(r.json()["id"])) + with media.media_for(art, "linkedin") as files: + assert files[0].kind == "video" and files[0].path and files[0].size == 80 * 1024 + + +def test_only_files_linkedin_takes_can_be_uploaded(client, auth, ws, db_session): + def register(up): + return client.post("/api/launchpad/uploads", json={"upload_id": str(up.id)}, headers=auth) + + assert register(uploaded(db_session, ws, "doc.pdf", "application/pdf", b"%PDF")).status_code == 415 + assert register(uploaded(db_session, ws, "half.png", "image/png", b"x", status="pending")).status_code == 409 + big = uploaded(db_session, ws, "huge.png", "image/png", b"x") + big.size_bytes = 40 * 1024 * 1024 + db_session.commit() + assert register(big).status_code == 413 + assert client.post("/api/launchpad/uploads", json={"upload_id": str(uuid.uuid4())}, headers=auth).status_code == 404 + + +def test_deleting_an_upload_record_removes_its_file(db_session, ws): + from app.services.storage import storage + + up = uploaded(db_session, ws, "gone.png", "image/png", b"PNG") + kept = uploaded(db_session, ws, "kept.png", "image/png", b"PNG") + db_session.delete(kept) + db_session.rollback() # a delete that never commits leaves the file alone + assert storage.exists(kept.key) + db_session.delete(up) + db_session.commit() + assert not storage.exists(up.key) + + +def test_deleting_an_uploaded_file_from_the_library_removes_record_and_file(client, auth, ws, db_session): + from app.models import Upload + from app.services.storage import storage + + up = uploaded(db_session, ws, "pic.png", "image/png", b"PNG") + art = client.post("/api/launchpad/uploads", json={"upload_id": str(up.id)}, headers=auth).json() + assert client.delete(f"/api/artifacts/{art['id']}", headers=auth).status_code == 200 + db_session.expire_all() + assert db_session.get(Upload, up.id) is None and not storage.exists(up.key) + + +def test_the_daily_sweep_removes_files_with_no_record(db_session, ws, session_factory): + from app.models import Upload + from app.services.storage import keys, storage + from app.services.uploads import sweep_orphan_uploads + + old = datetime.now(UTC).timestamp() - 3 * 24 * 3600 + orphan = keys.upload(ws.id, uuid.uuid4(), "orphan.png") + fresh = keys.upload(ws.id, uuid.uuid4(), "fresh.png") # may still be on its way: kept + storage.put_bytes(orphan, b"x") + storage.put_bytes(fresh, b"x") + os.utime(storage.path(orphan), (old, old)) + known = uploaded(db_session, ws, "known.png", "image/png", b"x") + os.utime(storage.path(known.key), (old, old)) + stale = uploaded(db_session, ws, "stale.png", "image/png", b"x", status="pending") + stale.created_at = datetime.now(UTC) - timedelta(days=2) + db_session.commit() + stale_id, stale_key = stale.id, stale.key + + assert sweep_orphan_uploads(session_factory) == 1 + assert not storage.exists(orphan) and storage.exists(fresh) and storage.exists(known.key) + db_session.expire_all() + assert db_session.get(Upload, stale_id) is None and not storage.exists(stale_key) + + +def test_an_uploaded_file_can_be_renamed_but_not_retyped(client, auth, ws, db_session): + up = uploaded(db_session, ws, "IMG_2041.png", "image/png", b"PNG") + art = client.post("/api/launchpad/uploads", json={"upload_id": str(up.id)}, headers=auth).json() + r = client.patch(f"/api/artifacts/{art['id']}", json={"title": " Q3 chart ", + "content": {"content_type": "video/mp4", "media": "video"}}, + headers=auth) + assert r.status_code == 200 + listed = {a["id"]: a for a in client.get("/api/launchpad/postable", headers=auth).json()}[art["id"]] + assert listed["title"] == "Q3 chart" and listed["media"] == "image" + assert client.patch(f"/api/artifacts/{art['id']}", json={"title": " "}, headers=auth).status_code == 400 + + +def test_uploaded_files_carry_a_preview_link(client, auth, ws, db_session): + png = uploaded(db_session, ws, "a.png", "image/png", b"PNG") + mp4 = uploaded(db_session, ws, "b.mp4", "video/mp4", b"v" * (80 * 1024)) + for up in (png, mp4): + brief = client.post("/api/launchpad/uploads", json={"upload_id": str(up.id)}, headers=auth).json() + assert brief["view_url"] + kit = made(db_session, ws, "launch_kit", post_title="Kit", posts={"linkedin": ["Hi"]}) + listed = {a["id"]: a for a in client.get("/api/launchpad/postable", headers=auth).json()} + assert listed[str(kit.id)]["view_url"] is None diff --git a/backend/tests/test_videos.py b/backend/tests/test_videos.py index 640925c660f2b98649f3520c4c1c1113b598b6bc..8c8f69d820fb491e9adbede86fa4123a7306cf36 100644 --- a/backend/tests/test_videos.py +++ b/backend/tests/test_videos.py @@ -440,7 +440,7 @@ def test_slow_catalog_is_a_clean_error_not_a_500(client, b2v, owner, db_session) b2v.queue("GET", "/api/templates", (0, httpx.ReadTimeout("slow"))) r = client.post("/api/videos", json={"url": "https://ada.example.com/p/a", "template": "geometric"}, headers=headers) - assert r.status_code == 502 and "not responding" in r.json()["detail"] + assert r.status_code == 502 and "being updated" in r.json()["detail"] assert not creates(b2v) and used(db_session, ws) == 0 diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 7c91d8842b7399a814b2713f1b0bf2afffc683ee..12596db265a46fd9767c9fa4d3a8ed490d55260f 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -8,8 +8,8 @@ import AuthPage from "./pages/AuthPage"; import { Blog, BlogPostPage } from "./pages/Blog"; import Landing from "./pages/Landing"; import Archive from "./pages/Archive"; -import Launchpad from "./pages/Launchpad"; -import LaunchKit from "./pages/LaunchKit"; +import Launchpad, { ScheduleLaunch } from "./pages/Launchpad"; +import LaunchKitRedirect, { LaunchKitPage, LaunchKits } from "./pages/LaunchKit"; import MissionControl from "./pages/MissionControl"; import Notebooks from "./pages/Notebooks"; import NotebookLMAlternative from "./pages/NotebookLMAlternative"; @@ -70,8 +70,11 @@ export default function App() { } /> } /> } /> - } /> + } /> } /> + } /> + } /> + } /> } /> } /> } /> diff --git a/frontend/src/api/auth.ts b/frontend/src/api/auth.ts index 328a02e110693c3cc59d88f13a1d94c8c6a51296..d187284555291bd9d04a8cf7ac16ed83f699adcb 100644 --- a/frontend/src/api/auth.ts +++ b/frontend/src/api/auth.ts @@ -34,6 +34,8 @@ export const authApi = { forgotStart: (email: string) => post<{ ok: true }>("/api/auth/password/forgot/start", { email }), forgotComplete: (email: string, code: string, password: string) => post("/api/auth/password/forgot/complete", { email, code, password }), - logout: () => post<{ ok: true }>("/api/auth/logout"), + /** Takes the token explicitly: sign out clears local tokens before revoking on the server. */ + logout: (access: string) => + api<{ ok: true }>("/api/auth/logout", { method: "POST", headers: { Authorization: `Bearer ${access}` } }, false), deleteAccount: () => post<{ ok: true }>("/api/auth/delete-account"), }; diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 538412ead9f50a95be73274726ca0fc213d2845c..5fbd87d96a53b7a74130d651141f0a080df7609e 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -17,6 +17,9 @@ export class ApiError extends Error { /** Fired on every 402 plan_limit so the upgrade popup opens wherever the request came from. */ export const PLAN_LIMIT_EVENT = "ns:plan-limit"; +/** Fired when a signed in request gets a 401 that a refresh cannot fix, so the app signs the user out. */ +export const SESSION_EXPIRED_EVENT = "ns:session-expired"; + export type PlanLimitDetail = { message: string; kind?: string; plan?: string; upgrade_to?: string | null }; function read(key: string): string | null { @@ -93,9 +96,14 @@ async function request(path: string, init: RequestInit, retry: boolean): Promise if (init.body && !(init.body instanceof FormData) && !headers.has("Content-Type")) { headers.set("Content-Type", "application/json"); } - if (tokens.access) headers.set("Authorization", `Bearer ${tokens.access}`); + const signedIn = Boolean(tokens.access); + if (signedIn) headers.set("Authorization", `Bearer ${tokens.access}`); const res = await fetch(`${BASE}${path}`, { ...init, headers }); if (res.status === 401 && retry && (await tryRefresh())) return request(path, init, false); + // Auth endpoints answer 401 for a wrong password and similar; only a dead session elsewhere signs out. + if (res.status === 401 && signedIn && retry && !path.startsWith("/api/auth/")) { + window.dispatchEvent(new Event(SESSION_EXPIRED_EVENT)); + } if (!res.ok) throw await parseError(res); return res; } @@ -147,7 +155,9 @@ const TYPES_BY_EXT: Record = { jpg: "image/jpeg", jpeg: "image/jpeg", webp: "image/webp", + gif: "image/gif", svg: "image/svg+xml", + mp4: "video/mp4", }; /** Browsers leave File.type empty for some extensions (.md on Windows) and use vendor names for others. */ diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts index a3f0aac0922a840a3a0f6f52496d33955116f009..c76997197c5216ac8d2072b23b59080934e805d3 100644 --- a/frontend/src/api/endpoints.ts +++ b/frontend/src/api/endpoints.ts @@ -22,6 +22,7 @@ import type { WorkspaceChat, Page, Platform, + PostableArtifact, Quota, ResurfaceItem, Settings, @@ -202,10 +203,16 @@ export const voiceApi = { export type ItemBody = { platform: Platform; - scheduled_at: string; + /** An exact instant (Post now). Scheduled posts send local_time + timezone instead. */ + scheduled_at?: string; + /** The half hour picked on the user's own clock, "2026-10-07T11:00", read in `timezone` (IANA) by the backend. */ + local_time?: string; + timezone?: string; content: string; thread?: string[]; artifact_id?: string | null; + /** The Launch Kit the post was written from (tracked, not attached). */ + kit_id?: string | null; document_id?: string | null; social_account_id?: string | null; remind_by_email?: boolean; @@ -225,6 +232,13 @@ export const launchpadApi = { startOAuth: (platform: "x" | "linkedin") => post<{ url: string }>(`/api/social/${platform}/start`), completeOAuth: (ticket: string) => post("/api/social/complete", { ticket }), disconnect: (id: string) => del(`/api/social/accounts/${id}`), + /** What can be attached to a post (finished videos, quote cards, carousels, summaries, launch kits; not audio). */ + postable: (q = "") => api(`/api/launchpad/postable${qs({ q: q || undefined })}`), + /** A file from the user's computer, sent to storage: make it postable (an `upload` in the picker). */ + uploadToPost: async (file: File, duration_s?: number) => { + const { upload_id } = await uploadFile(file); + return post("/api/launchpad/uploads", { upload_id, duration_s }); + }, }; export const resurfaceApi = { diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index cee62b1b6c71032a5c9bcbff6e688e5d4d2a94b5..d0436abb571be84abebee03bb2233c8e649c8a9b 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -89,7 +89,7 @@ export type AnswerFeedback = { rating: "up" | "down"; reasons: string[]; comment export type WorkspaceChat = ChatSummary & { notebook_id: string; notebook_title: string }; export type ChatMessage = { id: string; role: "user" | "assistant"; text: string; citations: Citation[]; feedback?: AnswerFeedback | null }; -export type ArtifactType = "summary" | "audio_overview" | "video" | "quote_card" | "carousel" | "launch_kit" | "mind_map" | "quiz" | "flashcards" | "report" | "infographic"; +export type ArtifactType = "summary" | "audio_overview" | "video" | "quote_card" | "carousel" | "launch_kit" | "mind_map" | "quiz" | "flashcards" | "report" | "infographic"| "upload"; export type QuizQuestionType = "multiple_choice" | "multiple_select" | "fill_blank" | "short_answer"; export type QuizQuestion = { @@ -314,15 +314,44 @@ export type Quota = { export type Platform = "x" | "linkedin" | "bluesky" | "substack_notes"; +/** Something made that a post can carry: images (quote card, carousel slides), a video, or text only. */ +export type PostableArtifact = { + id: string; + type: ArtifactType; + type_label: string; + title: string; + media: "image" | "video" | "text"; + media_count: number; + duration_s: number | null; + thumb_url: string | null; + /** A video made but not rendered to an MP4 yet: it can be posted once rendered (in the video editor). */ + needs_render: boolean; + /** Its MP4 is being rendered: it can be scheduled once that's done. */ + rendering: boolean; + /** A Notestack (blog2video) video: it opens in the video editor. */ + editable_video: boolean; + /** An uploaded file's link, to preview it (null for anything else). */ + view_url: string | null; + created_at: string | null; + /** A starting text: X as a thread (list of posts), LinkedIn as one post. */ + prefill: { x: string[]; linkedin: string }; +}; + export type CalendarItem = { id: string; platform: Platform; platform_label: string; scheduled_at: string; - status: "scheduled" | "publishing" | "posted" | "reminded" | "failed" | "draft"; + /** paused: its account's connection is down (or held by hand); never attempted until resumed. */ + status: "scheduled" | "publishing" | "posted" | "reminded" | "failed" | "draft" | "paused"; content: string; thread: string[]; artifact_id: string | null; + /** What the post carries (its images or video go out with it). */ + artifact: PostableArtifact | null; + /** The Launch Kit the post was written from (tracked apart; never an attachment). */ + kit_id: string | null; + kit_title: string | null; document_id: string | null; social_account_id: string | null; account_handle: string | null; @@ -343,6 +372,12 @@ export type SocialAccount = { status: string; avatar_url: string | null; connected_at: string | null; + /** Down (expired, revoked, disconnected), or not allowed to post images and videos. */ + needs_reconnect: boolean; + can_post_media: boolean; + /** No refresh token and the connection ends within a week (LinkedIn's 60 day tokens). */ + expires_soon: boolean; + expires_at: string | null; }; export type SocialAccounts = { diff --git a/frontend/src/components/AppShell.tsx b/frontend/src/components/AppShell.tsx index 8427824e1980eb602fc0b67652139796778c57e1..771af4f5666d55e7ed4537f77c4d3fbe74789abc 100644 --- a/frontend/src/components/AppShell.tsx +++ b/frontend/src/components/AppShell.tsx @@ -19,7 +19,7 @@ const NAV: { to: string; label: string; icon: typeof PlanetIcon; end?: boolean; { to: "/app/notebooks", label: "Notebooks", icon: PlanetIcon }, { to: "/app/sources", label: "Sources", icon: SatelliteDishIcon, also: ["/app/map"] }, { to: "/app/archive", label: "Library", icon: LibraryIcon, also: ["/app/videos"] }, - { to: "/app/launchpad", label: "Launchpad", icon: LaunchpadIcon, also: ["/app/resurface"] }, + { to: "/app/launchpad", label: "Launchpad", icon: LaunchpadIcon, also: ["/app/resurface", "/app/launch-kit"] }, ]; const COLLAPSED_KEY = "ns_sidebar_collapsed"; @@ -33,7 +33,7 @@ function readCollapsed(): boolean { } export default function AppShell() { - const { user, loading, logout } = useAuth(); + const { user, loading, signingOut, logout } = useAuth(); const location = useLocation(); const notebookFocus = /^\/app\/notebooks\/[^/]+$/.test(location.pathname); const [collapsed, setCollapsed] = useState(readCollapsed); @@ -47,10 +47,7 @@ export default function AppShell() { return !c; }); if (loading) return
; - if (!user) { - const next = new URLSearchParams({ next: location.pathname + location.search }); - return ; - } + if (!user) return ; return ( @@ -103,7 +100,7 @@ export default function AppShell() {
{user.email} -
diff --git a/frontend/src/components/ArtifactCard.tsx b/frontend/src/components/ArtifactCard.tsx index bd192a2b6f414ac74b92229d7a965fdba236d6a5..034077ffbc2ca9f5756a9dbb042236399adb8183 100644 --- a/frontend/src/components/ArtifactCard.tsx +++ b/frontend/src/components/ArtifactCard.tsx @@ -202,7 +202,7 @@ export function Body({ artifact, onCite }: { artifact: Artifact; onCite?: (c: Ci

{c.claims?.length ?? 0} claims, {c.hooks?.length ?? 0} hooks, posts for 4 platforms, SEO pack and carousel.

- + Open Launch Kit diff --git a/frontend/src/components/ScheduleModal.tsx b/frontend/src/components/ScheduleModal.tsx index feacb6d03be113f845e656355a8efc10ffb01ee5..51e72b1f2941ea8487d0b0de14b11d8c6d13d09a 100644 --- a/frontend/src/components/ScheduleModal.tsx +++ b/frontend/src/components/ScheduleModal.tsx @@ -1,6 +1,9 @@ import { useEffect, useState, type FormEvent } from "react"; +import { useNavigate } from "react-router-dom"; import { launchpadApi } from "../api/endpoints"; -import type { CalendarItem, Platform, SocialAccount } from "../api/types"; +import type { CalendarItem, Platform, PostableArtifact, SocialAccounts } from "../api/types"; +import { AttachmentChip, AttachPicker } from "./launchpad/Attachment"; +import { nextSlot, SlotPicker, toWall, userTimeZone, wallToDate } from "./launchpad/SlotPicker"; import { errorMessage, Modal } from "./ui"; export const PLATFORMS: { id: Platform; label: string; limit: number; auto: boolean }[] = [ @@ -10,22 +13,27 @@ export const PLATFORMS: { id: Platform; label: string; limit: number; auto: bool { id: "substack_notes", label: "Substack Notes", limit: 2000, auto: false }, ]; -export function toLocalInput(date: Date): string { - const pad = (n: number) => String(n).padStart(2, "0"); - return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}T${pad(date.getHours())}:${pad(date.getMinutes())}`; -} +/** Only X and LinkedIn take images and videos; the others get the text. */ +const MEDIA_PLATFORMS: Platform[] = ["x", "linkedin"]; +const THREADED: Platform[] = ["x", "bluesky"]; +/** These post only through a live connection: no email fallback. */ +const CONNECTED_ONLY: Platform[] = ["x", "linkedin"]; +/** What the composer offers for now: LinkedIn only (X posting is switched off). */ +const COMPOSER_PLATFORMS: Platform[] = ["linkedin"]; -export function nextSlot(): Date { - const d = new Date(Date.now() + 60 * 60 * 1000); - d.setMinutes(0, 0, 0); - return d; -} +/** The attachment's starting text for a platform: a thread for X and Bluesky, one post otherwise. */ +const prefillFor = (p: Platform, a: PostableArtifact): string[] => + THREADED.includes(p) ? [...a.prefill.x] : [a.prefill.linkedin]; -/** Schedules one post (or a thread) on the Launchpad. Posts auto publish when an account is connected. */ +/** The composer: one post to any of X, LinkedIn, Bluesky and Substack Notes (one calendar item each), with + * something from the Library attached (its images or video go to X and LinkedIn). Scheduled or posted now. */ export function ScheduleModal({ platform: initialPlatform, posts, artifactId, + artifact, + kitId, + kitTitle, documentId, onClose, onScheduled, @@ -33,134 +41,253 @@ export function ScheduleModal({ platform: Platform; posts: string[]; artifactId?: string | null; + /** The attachment, when the caller already has it (else it is looked up from artifactId). */ + artifact?: PostableArtifact | null; + /** The Launch Kit the post is written from: tracked with it, never attached. */ + kitId?: string | null; + kitTitle?: string | null; documentId?: string | null; onClose: () => void; onScheduled?: (item: CalendarItem) => void; }) { - const [platform, setPlatform] = useState(initialPlatform); - const [texts, setTexts] = useState(posts.length ? posts : [""]); - const [when, setWhen] = useState(toLocalInput(nextSlot())); - const [accounts, setAccounts] = useState([]); - const [accountId, setAccountId] = useState(""); + // One platform per post. A caller asking for one the composer doesn't offer starts on LinkedIn with that text. + const first: Platform = COMPOSER_PLATFORMS.includes(initialPlatform) ? initialPlatform : "linkedin"; + const [selected, setSelected] = useState([first]); + const [texts, setTexts] = useState>>(() => ({ + [first]: posts.length ? posts : artifact ? prefillFor(first, artifact) : [""], + })); + const [attached, setAttached] = useState(artifact?.type === "launch_kit" ? null : artifact ?? null); + const navigate = useNavigate(); + const [picking, setPicking] = useState(false); + const [when, setWhen] = useState(() => toWall(nextSlot())); + const [accounts, setAccounts] = useState(null); + // The connections are checked before anything says one is missing: loading isn't "not connected". + const [accountsState, setAccountsState] = useState<"loading" | "ready" | "error">("loading"); + const loadAccounts = () => { + setAccountsState("loading"); + launchpadApi.accounts().then((a) => { + setAccounts(a); + setAccountsState("ready"); + }, () => setAccountsState("error")); + }; const [remind, setRemind] = useState(false); - const [busy, setBusy] = useState(false); + const [busy, setBusy] = useState<"schedule" | "now" | null>(null); const [error, setError] = useState(null); - const meta = PLATFORMS.find((p) => p.id === platform)!; - const threadable = platform === "x" || platform === "bluesky"; useEffect(() => { - launchpadApi.accounts().then((a) => setAccounts(a.accounts), () => setAccounts([])); + loadAccounts(); + if (artifactId && !artifact) { + launchpadApi.postable().then((list) => { + const found = list.find((a) => a.id === artifactId); + if (found && found.type !== "launch_kit") attach(found); // a kit is never an attachment + }, () => undefined); + } + // eslint-disable-next-line react-hooks/exhaustive-deps }, []); - const options = accounts.filter((a) => a.platform === platform && a.status === "active"); - useEffect(() => { - setAccountId(options[0]?.id ?? ""); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [platform, accounts.length]); + const accountFor = (p: Platform) => + accounts?.accounts.find((a) => a.platform === p && a.status === "active") ?? null; + const carriesMedia = !!attached && attached.media !== "text"; + /** Why a ticked X / LinkedIn can't be scheduled yet (no connection, connection down, can't post media). */ + const blocker = (p: Platform): string | null => { + if (!CONNECTED_ONLY.includes(p) || accountsState !== "ready") return null; + const label = PLATFORMS.find((x) => x.id === p)!.label; + const any = accounts?.accounts.find((a) => a.platform === p); + const live = accountFor(p); + if (!live) return any ? `Your ${label} connection is down. Reconnect ${label}.` : `Connect ${label} to post there.`; + if (carriesMedia && !live.can_post_media) return `Reconnect ${label} to post images and videos.`; + return null; + }; + const blocked = selected.map(blocker).filter(Boolean) as string[]; + const checking = accountsState !== "ready" && selected.some((p) => CONNECTED_ONLY.includes(p)); + const remindable = selected.some((p) => !CONNECTED_ONLY.includes(p)); + + /** Switch to another platform (one per post); it starts from the attachment's text, or the text written so far. */ + const pick = (p: Platform) => { + if (selected.includes(p)) return; + setSelected([p]); + if (!texts[p]?.some((t) => t.trim())) { + const from = selected.map((s) => texts[s] ?? []).find((t) => t.some((x) => x.trim())) ?? [""]; + const start = attached ? prefillFor(p, attached) : THREADED.includes(p) ? from : [from.join("\n\n")]; + setTexts((cur) => ({ ...cur, [p]: start })); + } + }; + + /** Attach: empty texts take its suggested text. */ + const attach = (a: PostableArtifact) => { + setAttached(a); + setPicking(false); + setTexts((cur) => { + const next = { ...cur }; + for (const p of selected) if (!next[p]?.some((t) => t.trim())) next[p] = prefillFor(p, a); + return next; + }); + }; + + const setText = (p: Platform, i: number, value: string) => + setTexts((cur) => ({ ...cur, [p]: (cur[p] ?? [""]).map((t, j) => (j === i ? value : t)) })); + + const connect = async (p: "x" | "linkedin") => { + try { + window.location.href = (await launchpadApi.startOAuth(p)).url; + } catch (e) { + setError(errorMessage(e)); + } + }; - const submit = async (e: FormEvent) => { - e.preventDefault(); - setBusy(true); + const submit = async (now: boolean, e?: FormEvent) => { + e?.preventDefault(); + if (!selected.length) return setError("Pick where to post."); + if (blocked.length) return setError(blocked[0]); + if (!now && wallToDate(when) <= new Date()) return setError("Pick a time that is still ahead."); + setBusy(now ? "now" : "schedule"); setError(null); - const clean = texts.map((t) => t.trim()).filter(Boolean); - const merged = threadable ? clean : [clean.join("\n\n")]; + const done: string[] = []; try { - const item = await launchpadApi.create({ - platform, - scheduled_at: new Date(when).toISOString(), - content: merged[0] ?? "", - thread: merged.slice(1), - artifact_id: artifactId ?? null, - document_id: documentId ?? null, - social_account_id: meta.auto && accountId ? accountId : null, - remind_by_email: remind || !meta.auto || !accountId, - }); - onScheduled?.(item); + for (const p of PLATFORMS.map((x) => x.id).filter((id) => selected.includes(id))) { + const meta = PLATFORMS.find((x) => x.id === p)!; + const clean = (texts[p] ?? []).map((t) => t.trim()).filter(Boolean); + const merged = THREADED.includes(p) ? clean : [clean.join("\n\n")]; + const account = meta.auto ? accountFor(p) : null; + const item = await launchpadApi.create({ + platform: p, + // Post now: this instant. Scheduled: the half hour picked, in the user's zone (the backend makes it UTC). + ...(now ? { scheduled_at: new Date().toISOString() } : { local_time: when, timezone: userTimeZone() }), + content: merged[0] ?? "", + thread: merged.slice(1), + // Images and videos only go to X and LinkedIn; text attachments go anywhere. + artifact_id: attached && (!carriesMedia || MEDIA_PLATFORMS.includes(p)) ? attached.id : null, + document_id: documentId ?? null, + kit_id: kitId ?? null, + social_account_id: account?.id ?? null, + remind_by_email: CONNECTED_ONLY.includes(p) ? false : remind || !account, + }); + const out = now && item.auto_post ? await launchpadApi.publishNow(item.id) : item; + onScheduled?.(out); + done.push(meta.label); + } onClose(); } catch (err) { - setError(errorMessage(err)); + setError(done.length ? `${done.join(" and ")} done; then: ${errorMessage(err)}` : errorMessage(err)); } finally { - setBusy(false); + setBusy(null); } }; return ( - -
-
-