added blogs

.env.example

@@ -7,6 +7,13 @@ JWT_SECRET=replace-with-64-random-chars
 INTERNAL_TOKEN=replace-with-random-token
 BILLING_ENABLED=false
 
+# ---- Database (also backs the job queue)
+# Local: a SQLite file, created by `alembic upgrade head` run from backend/.
+# Prod (Neon, Supabase, RDS...): Postgres with the psycopg driver, e.g.
+#   DATABASE_URL=postgresql+psycopg://user:pass@host:5432/dbname?sslmode=require
+# docker-compose ignores this and points at its own Postgres container.
+DATABASE_URL=sqlite:///./notestack.db
+
 # ---- Google sign in (OAuth client, type Web; add http://localhost:5173 to authorized JS origins)
 # Client ID only: popup sign in (Google Identity Services), no redirect URIs needed.
 # Client ID + secret: redirect flow with server side code exchange. Add the redirect URI
@@ -23,6 +30,13 @@ FROM_EMAIL=Notestack <hello@notestack.ai>
 ALERTS_EMAIL=arslan@firebird-technologies.com
 UNSUBSCRIBE_SECRET=replace-me
 
+# ---- Local dev conveniences
+# The API runs the job worker in process when ENV=development (set false to use python -m app.worker).
+RUN_WORKER_IN_API=
+# auto = local disk (LOCAL_STORAGE_DIR) when no R2 account or endpoint is set, else R2
+STORAGE_BACKEND=auto
+LOCAL_STORAGE_DIR=.storage
+
 # ---- Storage: Cloudflare R2
 # Prod: set account id + keys, leave R2_ENDPOINT_URL empty.
 # Local: docker-compose points R2_ENDPOINT_URL at MinIO (keys minioadmin/minioadmin).
@@ -53,5 +67,21 @@ WORKER_CONCURRENCY=4
 WORKER_POLL_SECONDS=1
 JOB_STALE_SECONDS=900
 
-# ---- ElevenLabs (Phase 1 audio)
+# ---- ElevenLabs (audio overviews, video narration, voice cloning)
 ELEVENLABS_API_KEY=
+ELEVENLABS_MODEL=eleven_multilingual_v2
+ELEVENLABS_VOICE_A=21m00Tcm4TlvDq8ikWAM
+ELEVENLABS_VOICE_B=pNInz6obpgDQGcFMaJgB
+
+# ---- Launchpad social publishing
+# Bluesky needs nothing here (users connect with an app password).
+# X: developer app with OAuth 2.0, type Web App, scopes tweet.read tweet.write users.read offline.access.
+#   Callback URI: {API_URL}/api/social/x/callback  e.g. http://localhost:8000/api/social/x/callback
+# LinkedIn: app with the "Sign In with LinkedIn using OpenID Connect" and "Share on LinkedIn" products.
+#   Redirect URL: {API_URL}/api/social/linkedin/callback
+X_CLIENT_ID=
+X_CLIENT_SECRET=
+LINKEDIN_CLIENT_ID=
+LINKEDIN_CLIENT_SECRET=
+# Fernet key for stored social tokens; derived from JWT_SECRET when empty.
+SOCIAL_TOKEN_KEY=

.gitignore

@@ -12,3 +12,11 @@ __pycache__/
 out/
 renderer/build/
 .corpus-cache/
+
+# Local SQLite database
+*.db
+*.db-wal
+*.db-shm
+
+# Local object storage (used when no R2 is configured)
+.storage/

README.md

@@ -4,7 +4,7 @@ Your archive, in orbit. Turn a Substack (or any feed) into a grounded research n
 overviews, videos and launch kits, all traced back to what the writer actually wrote.
 
 - Design doc: [docs/DESIGN.md](docs/DESIGN.md)
-- Phase 0 task list: [docs/PHASE0.md](docs/PHASE0.md)
+- Foundations task list: [docs/PHASE0.md](docs/PHASE0.md)
 
 ## Layout
 
@@ -27,16 +27,41 @@ docker compose up --build
   `127.0.0.1 minio` to your hosts file for browser uploads to work locally.
 - With `EMAIL_PROVIDER=console`, verification codes print in the `api` logs.
 
-Without Docker:
+Without Docker (one API process does everything):
 
 ```bash
 cd backend && python -m venv .venv && .venv/Scripts/pip install -r requirements-dev.txt
-alembic upgrade head && uvicorn app.main:app --reload     # needs Postgres only
-python -m app.worker                                       # second terminal
+alembic upgrade head && uvicorn app.main:app --reload     # SQLite at backend/notestack.db
 cd frontend && npm install && npm run dev
-cd renderer && npm install && npm start                    # or: npm run studio
+cd renderer && npm install && npm start                    # only needed for videos, quote cards, carousels
 ```
 
+- With `ENV=development` the API runs the job worker in process (`RUN_WORKER_IN_API`), so syncing,
+  topic mapping, audio and launch kits work without a second terminal. In production run
+  `python -m app.worker` separately and set `RUN_WORKER_IN_API=false`.
+- With no R2 settings, files live on local disk under `backend/.storage` and are served through
+  signed `/api/storage/local/...` URLs (range requests supported for audio and video seeking).
+- Audio needs `ELEVENLABS_API_KEY`; everything text based needs `LLM_API_KEY`.
+- Launchpad: Bluesky connects with an app password. X and LinkedIn need developer apps; the
+  redirect URIs are listed in `.env.example` and on the Launchpad page. Substack Notes has no API,
+  so those posts (and any post without a connected account) arrive as an email reminder at send time.
+
+## What is in the app
+
+| Menu | Does |
+| --- | --- |
+| Mission Control | Pre-flight checklist, jobs in flight, sources, notebooks, recent launches, upcoming posts |
+| Notebooks | Group posts, chat with cited answers (history kept), summaries, audio overviews, videos, quote cards |
+| Sources | Connect feeds (Substack archive included), import one URL, upload md/txt/html/pdf, read posts by line |
+| Topic map | LLM tagged topics per post, merged into a constellation; make a notebook from any topic |
+| Voice profile | Writing voice built from your posts (editable), host voices, consented voice cloning |
+| Video and audio | Audio overviews, 9:16 shorts, 16:9 explainers, 1:1 audiograms, quote cards |
+| Launch Kit | Hooks, X thread, LinkedIn, Substack Notes, Bluesky, SEO pack, carousel, quote cards, all editable |
+| Launchpad | Calendar and agenda, auto posting to X, LinkedIn and Bluesky, email reminders, tracked links, engagement |
+| Archive | Every generated artifact with filters, players, downloads, retry and delete |
+| Resurfacing | Evergreen scores and reshare angles for old posts, on this day, straight into a kit or the calendar |
+| Settings | Profile, workspace, brand kit, privacy, plan usage meters, connections, sign out everywhere, delete |
+
 ## Key choices
 
 | Concern | Where |

backend/alembic/env.py

@@ -24,7 +24,9 @@ def run_migrations_online() -> None:
     connectable = engine_from_config(config.get_section(config.config_ini_section), prefix="sqlalchemy.",
                                      poolclass=pool.NullPool)
     with connectable.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata)
+        # Batch mode rebuilds tables for ALTERs that SQLite cannot do in place (local dev only).
+        context.configure(connection=connection, target_metadata=target_metadata,
+                          render_as_batch=connection.dialect.name == "sqlite")
         with context.begin_transaction():
             context.run_migrations()
 

backend/alembic/versions/0002_build_out.py

@@ -0,0 +1,94 @@
+"""build out: social accounts, topics, launchpad fields, resurfacing, host voices
+
+Revision ID: 0002_build_out
+Revises: 0001_initial
+Create Date: 2026-09-27
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0002_build_out"
+down_revision = "0001_initial"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # 0001 builds from the live models, so a fresh database already has everything below.
+    # Only apply what is missing, which makes this safe on both fresh and existing databases.
+    insp = sa.inspect(op.get_bind())
+    tables = set(insp.get_table_names())
+    if "social_accounts" in tables and "evergreen_score" in {c["name"] for c in insp.get_columns("documents")}:
+        return
+    op.create_table(
+        "social_accounts",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("workspace_id", sa.Uuid(), sa.ForeignKey("workspaces.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("platform", sa.String(20), nullable=False),
+        sa.Column("handle", sa.String(200), nullable=False),
+        sa.Column("external_id", sa.String(200), nullable=False),
+        sa.Column("access_token", sa.Text(), nullable=False),
+        sa.Column("refresh_token", sa.Text()),
+        sa.Column("expires_at", sa.DateTime(timezone=True)),
+        sa.Column("scopes", sa.String(500)),
+        sa.Column("status", sa.String(20), nullable=False, server_default="active"),
+        sa.Column("avatar_url", sa.String(1000)),
+        sa.UniqueConstraint("workspace_id", "platform", "external_id"),
+    )
+    op.create_table(
+        "topics",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("workspace_id", sa.Uuid(), sa.ForeignKey("workspaces.id", ondelete="CASCADE"), nullable=False, index=True),
+        sa.Column("name", sa.String(200), nullable=False),
+        sa.Column("slug", sa.String(200), nullable=False),
+        sa.Column("summary", sa.Text()),
+        sa.Column("post_count", sa.Integer(), nullable=False, server_default="0"),
+        sa.UniqueConstraint("workspace_id", "slug"),
+    )
+    op.create_table(
+        "document_topics",
+        sa.Column("document_id", sa.Uuid(), sa.ForeignKey("documents.id", ondelete="CASCADE"), primary_key=True),
+        sa.Column("topic_id", sa.Uuid(), sa.ForeignKey("topics.id", ondelete="CASCADE"), primary_key=True),
+        sa.Column("weight", sa.Float(), nullable=False, server_default="1"),
+    )
+    with op.batch_alter_table("documents") as b:
+        b.add_column(sa.Column("evergreen_score", sa.Float()))
+        b.add_column(sa.Column("last_resurfaced_at", sa.DateTime(timezone=True)))
+    with op.batch_alter_table("voice_profiles") as b:
+        b.add_column(sa.Column("host_voices", sa.JSON(), nullable=False, server_default="{}"))
+    with op.batch_alter_table("calendar_items") as b:
+        b.alter_column("artifact_id", existing_type=sa.Uuid(), nullable=True)
+        b.add_column(sa.Column("document_id", sa.Uuid(), sa.ForeignKey(
+            "documents.id", name="fk_calendar_items_document_id", ondelete="SET NULL")))
+        b.add_column(sa.Column("content", sa.Text(), nullable=False, server_default=""))
+        b.add_column(sa.Column("thread", sa.JSON(), nullable=False, server_default="[]"))
+        b.add_column(sa.Column("social_account_id", sa.Uuid(), sa.ForeignKey(
+            "social_accounts.id", name="fk_calendar_items_social_account_id", ondelete="SET NULL")))
+        b.add_column(sa.Column("remind_by_email", sa.Boolean(), nullable=False, server_default=sa.false()))
+        b.add_column(sa.Column("external_id", sa.String(200)))
+        b.add_column(sa.Column("external_url", sa.String(1000)))
+        b.add_column(sa.Column("posted_at", sa.DateTime(timezone=True)))
+        b.add_column(sa.Column("error", sa.Text()))
+        b.add_column(sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"))
+        b.add_column(sa.Column("metrics", sa.JSON(), nullable=False, server_default="{}"))
+        b.create_index("ix_calendar_items_scheduled_at", ["scheduled_at"])
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("calendar_items") as b:
+        b.drop_index("ix_calendar_items_scheduled_at")
+        for col in ("metrics", "attempts", "error", "posted_at", "external_url", "external_id", "remind_by_email",
+                    "social_account_id", "thread", "content", "document_id"):
+            b.drop_column(col)
+    with op.batch_alter_table("voice_profiles") as b:
+        b.drop_column("host_voices")
+    with op.batch_alter_table("documents") as b:
+        b.drop_column("last_resurfaced_at")
+        b.drop_column("evergreen_score")
+    op.drop_table("document_topics")
+    op.drop_table("topics")
+    op.drop_table("social_accounts")

backend/app/config.py

@@ -17,12 +17,17 @@ class Settings(BaseSettings):
     api_url: str = "http://localhost:8000"
     cors_origins: str = "http://localhost:5173"
 
-    database_url: str = "postgresql+psycopg://notestack:notestack@localhost:5432/notestack"
+    # Local default is a SQLite file (relative to the working directory, normally backend/).
+    # Prod and docker-compose use Postgres: postgresql+psycopg://user:pass@host:5432/db
+    database_url: str = "sqlite:///./notestack.db"
 
     # Worker (Postgres backed queue, see app/worker.py)
     worker_concurrency: int = 4
     worker_poll_seconds: float = 1.0
     job_stale_seconds: int = 900  # a running job with no heartbeat for this long is retried
+    # Run the worker loop inside the API process (local dev: one command, no second terminal).
+    # None = on in development, off elsewhere.
+    run_worker_in_api: bool | None = None
 
     # Auth
     jwt_secret: str = "change-me"
@@ -43,7 +48,9 @@ class Settings(BaseSettings):
     unsubscribe_secret: str = "change-me-too"
     update_email_send_hour: int = 9
 
-    # Storage (Cloudflare R2, S3 compatible)
+    # Storage: auto = local disk when no R2 account or endpoint is set, else R2 (S3 compatible)
+    storage_backend: str = "auto"  # auto | r2 | local
+    local_storage_dir: str = ".storage"
     r2_account_id: str = ""
     r2_access_key_id: str = ""
     r2_secret_access_key: str = ""
@@ -68,8 +75,18 @@ class Settings(BaseSettings):
     corpus_cache_dir: str = ".corpus-cache"
     research_max_steps: int = 10
 
-    # ElevenLabs
+    # ElevenLabs (premade voices by default; a consented clone can replace host A)
     elevenlabs_api_key: str = ""
+    elevenlabs_model: str = "eleven_multilingual_v2"
+    elevenlabs_voice_a: str = "21m00Tcm4TlvDq8ikWAM"  # Rachel
+    elevenlabs_voice_b: str = "pNInz6obpgDQGcFMaJgB"  # Adam
+
+    # Social publishing (Launchpad). Redirect URIs: {API_URL}/api/social/{x,linkedin}/callback
+    social_token_key: str = ""  # Fernet key; derived from JWT_SECRET when empty
+    x_client_id: str = ""
+    x_client_secret: str = ""
+    linkedin_client_id: str = ""
+    linkedin_client_secret: str = ""
 
     # Renderer
     renderer_url: str = "http://localhost:3100"
@@ -78,6 +95,16 @@ class Settings(BaseSettings):
     # Billing
     billing_enabled: bool = False
 
+    @property
+    def worker_in_api(self) -> bool:
+        return self.env == "development" if self.run_worker_in_api is None else self.run_worker_in_api
+
+    @property
+    def use_local_storage(self) -> bool:
+        if self.storage_backend != "auto":
+            return self.storage_backend == "local"
+        return not (self.r2_account_id or self.r2_endpoint_url)
+
     @property
     def r2_endpoint(self) -> str:
         if self.r2_endpoint_url:

backend/app/db.py

@@ -1,6 +1,6 @@
 from collections.abc import Iterator
 
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, event
 from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
 
 from app.config import settings
@@ -10,8 +10,20 @@ class Base(DeclarativeBase):
     pass
 
 
-_connect_args = {"check_same_thread": False} if settings.database_url.startswith("sqlite") else {}
+_sqlite = settings.database_url.startswith("sqlite")
+_connect_args = {"check_same_thread": False, "timeout": 30} if _sqlite else {}
 engine = create_engine(settings.database_url, pool_pre_ping=True, connect_args=_connect_args)
+
+if _sqlite:
+    # Local dev: the API and the worker share one file. WAL lets reads run during a write, the busy
+    # timeout makes a second writer wait instead of failing, and SQLite needs FKs switched on.
+    @event.listens_for(engine, "connect")
+    def _sqlite_pragmas(dbapi_conn, _record):
+        cur = dbapi_conn.cursor()
+        cur.execute("PRAGMA journal_mode=WAL")
+        cur.execute("PRAGMA busy_timeout=30000")
+        cur.execute("PRAGMA foreign_keys=ON")
+        cur.close()
 SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
 
 

backend/app/llm/run.py

@@ -0,0 +1,48 @@
+"""One door for every generation call: runs the signature, meters tokens, writes a trace, and strips
+em dashes from the outputs. Tests replace `predict` to return canned outputs."""
+
+import uuid
+
+import dspy
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.llm.postprocess import clean_output
+from app.llm.provider import main_lm, provider_name, track_usage
+from app.models import DspyTrace, Job, Workspace
+from app.services.jobs import record_usage
+
+
+def _dump(value):
+    if hasattr(value, "model_dump"):
+        return value.model_dump()
+    if isinstance(value, list):
+        return [_dump(v) for v in value]
+    return value
+
+
+def predict(
+    signature: type[dspy.Signature],
+    *,
+    db: Session,
+    workspace_id: uuid.UUID,
+    job: Job | None = None,
+    lm: dspy.LM | None = None,
+    **inputs,
+) -> dict:
+    """Run `signature` and return its outputs as plain dicts/lists (pydantic models dumped)."""
+    lm = lm or main_lm()
+    with track_usage(lm) as usage, dspy.context(lm=lm):
+        pred = dspy.Predict(signature)(**inputs)
+    outputs = {name: clean_output(_dump(pred[name])) for name in signature.output_fields}
+    if usage.total:
+        record_usage(db, workspace_id=workspace_id, kind="llm", provider=provider_name(), model=settings.llm_model,
+                     quantity=usage.total, unit="tokens", cost_usd=usage.cost_usd, job=job)
+    workspace = db.get(Workspace, workspace_id)
+    db.add(DspyTrace(
+        workspace_id=workspace_id, module=signature.__name__, model=lm.model,
+        inputs={k: (v if len(str(v)) < 4000 else str(v)[:4000]) for k, v in inputs.items()},
+        outputs=outputs, training_allowed=bool(workspace and workspace.training_opt_in),
+    ))
+    db.commit()
+    return outputs

backend/app/llm/signatures.py

@@ -1,5 +1,5 @@
-"""Typed DSPy signatures with Pydantic outputs. Phase 1 modules are implemented; later ones are
-declared now so the output contract is fixed early."""
+"""Typed DSPy signatures with Pydantic outputs. Every module that produces facts cites SourceRefs,
+which are verified against the corpus before anything reaches the writer."""
 
 from typing import Literal
 
@@ -61,6 +61,41 @@ class Hook(BaseModel):
     rationale: str
 
 
+class TopicTag(BaseModel):
+    name: str = Field(description="Short topic name, 1 to 4 words, Title Case")
+    weight: float = Field(ge=0, le=1, description="How central the topic is to the post")
+
+
+class TopicGroup(BaseModel):
+    canonical: str = Field(description="The best name for the merged topic")
+    members: list[str] = Field(description="Every input name that means the same topic")
+    summary: str = Field(description="One sentence on what the writer says about it")
+
+
+class InternalLink(BaseModel):
+    path: str
+    anchor_text: str
+    reason: str
+
+
+class SeoPackOut(BaseModel):
+    title_options: list[str]
+    meta_description: str = Field(description="At most 155 characters")
+    slug: str
+    keywords: list[str]
+    internal_links: list[InternalLink] = Field(default_factory=list)
+
+
+class Slide(BaseModel):
+    heading: str
+    body: str
+
+
+class QuotePick(BaseModel):
+    quote: str = Field(description="Exact words from the passage, at most 220 characters")
+    source: SourceRef
+
+
 # Signatures
 
 
@@ -146,3 +181,71 @@ class PlatformAdapter(dspy.Signature):
     platform: Literal["x_thread", "linkedin", "substack_notes", "bluesky"] = dspy.InputField()
     platform_rules: str = dspy.InputField()
     posts: list[str] = dspy.OutputField()
+
+
+class SummarizeNotebook(dspy.Signature):
+    """Summarize what these posts say as a whole: the main arguments, how they connect and where the
+    writer changed their mind. Every factual sentence ends with a [n] marker and a citation to the
+    lines it came from. Plain, warm prose. Never use em dashes."""
+
+    title: str = dspy.InputField()
+    passages: list[str] = dspy.InputField(desc="Posts with path and numbered lines")
+    summary: str = dspy.OutputField(desc="3 to 6 paragraphs with [n] markers")
+    themes: list[str] = dspy.OutputField(desc="3 to 7 short theme names")
+    citations: list[FileCitation] = dspy.OutputField()
+
+
+class ExtractTopics(dspy.Signature):
+    """Name the topics this post is about, the way a reader would search for them."""
+
+    title: str = dspy.InputField()
+    text: str = dspy.InputField()
+    known_topics: list[str] = dspy.InputField(desc="Reuse one of these names when it fits")
+    topics: list[TopicTag] = dspy.OutputField(desc="3 to 8 topics")
+
+
+class ConsolidateTopics(dspy.Signature):
+    """Merge topic names that mean the same thing (plural/singular, synonyms, broader wording) and
+    write a one sentence summary per merged topic from the post titles given."""
+
+    topics: list[str] = dspy.InputField(desc="name (post count): sample post titles")
+    groups: list[TopicGroup] = dspy.OutputField()
+
+
+class SeoPack(dspy.Signature):
+    """Write search metadata for the post in the writer's voice and suggest links to their other
+    posts where a reader would want them. No em dashes."""
+
+    title: str = dspy.InputField()
+    passages: list[str] = dspy.InputField()
+    related_posts: str = dspy.InputField(desc="Other posts in the archive as date | path | title")
+    seo: SeoPackOut = dspy.OutputField()
+
+
+class CarouselSlides(dspy.Signature):
+    """Turn the post's key claims into a 6 to 9 slide carousel. Slide 1 is a hook, the last is a call to
+    read the post. Short, punchy, in the writer's voice. No em dashes."""
+
+    passages: list[str] = dspy.InputField()
+    voice_profile: str = dspy.InputField()
+    slides: list[Slide] = dspy.OutputField()
+
+
+class PickQuotes(dspy.Signature):
+    """Pick the most quotable lines, copied exactly from the passages."""
+
+    passages: list[str] = dspy.InputField()
+    n: int = dspy.InputField()
+    quotes: list[QuotePick] = dspy.OutputField()
+
+
+class EvergreenScore(dspy.Signature):
+    """Judge how well this post would land if reshared today. Evergreen means the ideas are not tied to
+    news, dates or a moment. Suggest a fresh angle for resharing it. No em dashes."""
+
+    title: str = dspy.InputField()
+    published: str = dspy.InputField()
+    excerpt: str = dspy.InputField()
+    score: float = dspy.OutputField(desc="0 (dated) to 1 (timeless)")
+    reason: str = dspy.OutputField()
+    angle: str = dspy.OutputField(desc="One line hook for resharing now")

backend/app/main.py

@@ -1,4 +1,5 @@
 import logging
+import threading
 import time
 import uuid
 from contextlib import asynccontextmanager
@@ -8,7 +9,22 @@ from fastapi.middleware.cors import CORSMiddleware
 
 from app.config import settings
 from app.llm.provider import configure_default
-from app.routers import artifacts, auth, billing, jobs, notebooks, sources, storage, unsubscribe
+from app.routers import (
+    artifacts,
+    auth,
+    billing,
+    jobs,
+    launchpad,
+    notebooks,
+    resurface,
+    sources,
+    storage,
+    topics,
+    unsubscribe,
+    voice,
+)
+from app.routers import settings as settings_router
+from app.services.storage import storage as store
 
 log = logging.getLogger("notestack")
 
@@ -17,14 +33,19 @@ log = logging.getLogger("notestack")
 async def lifespan(app: FastAPI):
     logging.basicConfig(level=logging.INFO)
     configure_default()
-    if settings.env == "development" and settings.r2_endpoint_url:
+    if settings.env == "development" and (settings.r2_endpoint_url or settings.use_local_storage):
         try:
-            from app.services.storage import storage as store
-
-            store.ensure_bucket()  # MinIO locally
+            store.ensure_bucket()  # MinIO or the local storage folder
         except Exception:
             log.warning("could not ensure local bucket", exc_info=True)
+    stop = threading.Event()
+    if settings.worker_in_api:
+        from app.worker import start_background
+
+        start_background(stop)
+        log.info("worker running inside the API (RUN_WORKER_IN_API)")
     yield
+    stop.set()
 
 
 app = FastAPI(title="Notestack API", version="0.1.0", lifespan=lifespan)
@@ -53,6 +74,10 @@ def health():
     return {"ok": True, "llm": settings.llm_model, "billing_enabled": settings.billing_enabled}
 
 
-for r in (auth, billing, storage, sources, notebooks, artifacts, jobs, unsubscribe):
+for r in (auth, billing, storage, sources, notebooks, artifacts, jobs, unsubscribe, topics, voice, resurface,
+          launchpad, settings_router):
     app.include_router(r.router)
 app.include_router(sources.documents_router)
+app.include_router(launchpad.links_router)
+if settings.use_local_storage:
+    app.include_router(storage.local_router)

backend/app/models/__init__.py

@@ -4,13 +4,16 @@ from app.models.content import (
     Chat,
     Citation,
     Document,
+    DocumentTopic,
     DspyTrace,
     EngagementEvent,
     Job,
     Message,
     Notebook,
     NotebookDocument,
+    SocialAccount,
     Source,
+    Topic,
     TrackedLink,
     Upload,
     UsageEvent,
@@ -27,6 +30,7 @@ __all__ = [
     "Chat",
     "Citation",
     "Document",
+    "DocumentTopic",
     "DspyTrace",
     "EmailVerificationCode",
     "EngagementEvent",
@@ -34,8 +38,10 @@ __all__ = [
     "Message",
     "Notebook",
     "NotebookDocument",
+    "SocialAccount",
     "Source",
     "Subscription",
+    "Topic",
     "TrackedLink",
     "UpdateEmail",
     "UpdateEmailSend",

backend/app/models/content.py

@@ -53,6 +53,9 @@ class Document(IdMixin, TimestampMixin, Base):
     clean_text: Mapped[str] = mapped_column(Text, default="")
     content_hash: Mapped[str | None] = mapped_column(String(64))
     metadata_json: Mapped[dict] = mapped_column(JSON, default=dict)
+    # Resurfacing: 0..1 from the EvergreenScore module (reason and angle live in metadata_json)
+    evergreen_score: Mapped[float | None] = mapped_column(Float)
+    last_resurfaced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
 
 
 class Notebook(IdMixin, TimestampMixin, Base):
@@ -83,6 +86,7 @@ class VoiceProfile(IdMixin, TimestampMixin, Base):
     )
     profile_json: Mapped[dict] = mapped_column(JSON, default=dict)
     sample_doc_ids: Mapped[list] = mapped_column(JSON, default=list)
+    host_voices: Mapped[dict] = mapped_column(JSON, default=dict)  # {"host_a": voice_id, "host_b": voice_id}
 
 
 class VoiceConsent(IdMixin, TimestampMixin, Base):
@@ -188,10 +192,63 @@ class CalendarItem(IdMixin, TimestampMixin, Base):
     __tablename__ = "calendar_items"
 
     workspace_id: Mapped[uuid.UUID] = _ws_fk()
-    artifact_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("artifacts.id", ondelete="CASCADE"))
-    platform: Mapped[str] = mapped_column(String(30))
-    scheduled_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+    artifact_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, ForeignKey("artifacts.id", ondelete="CASCADE"))
+    document_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, ForeignKey("documents.id", ondelete="SET NULL"))
+    platform: Mapped[str] = mapped_column(String(30))  # x | linkedin | bluesky | substack_notes
+    scheduled_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
+    # scheduled | publishing | posted | reminded | failed | draft
     status: Mapped[str] = mapped_column(String(20), default="scheduled")
+    content: Mapped[str] = mapped_column(Text, default="")
+    thread: Mapped[list] = mapped_column(JSON, default=list)  # extra posts after the first, for threads
+    social_account_id: Mapped[uuid.UUID | None] = mapped_column(
+        Uuid, ForeignKey("social_accounts.id", ondelete="SET NULL")
+    )
+    remind_by_email: Mapped[bool] = mapped_column(Boolean, default=False)
+    external_id: Mapped[str | None] = mapped_column(String(200))
+    external_url: Mapped[str | None] = mapped_column(String(1000))
+    posted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    error: Mapped[str | None] = mapped_column(Text)
+    attempts: Mapped[int] = mapped_column(Integer, default=0)
+    metrics: Mapped[dict] = mapped_column(JSON, default=dict)  # likes, reposts, replies, impressions
+
+
+class SocialAccount(IdMixin, TimestampMixin, Base):
+    """A connected X, LinkedIn or Bluesky account. Tokens are Fernet encrypted (app/services/crypto.py)."""
+
+    __tablename__ = "social_accounts"
+    __table_args__ = (UniqueConstraint("workspace_id", "platform", "external_id"),)
+
+    workspace_id: Mapped[uuid.UUID] = _ws_fk()
+    platform: Mapped[str] = mapped_column(String(20))
+    handle: Mapped[str] = mapped_column(String(200))
+    external_id: Mapped[str] = mapped_column(String(200))
+    access_token: Mapped[str] = mapped_column(Text)
+    refresh_token: Mapped[str | None] = mapped_column(Text)
+    expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    scopes: Mapped[str | None] = mapped_column(String(500))
+    status: Mapped[str] = mapped_column(String(20), default="active")  # active | expired | revoked
+    avatar_url: Mapped[str | None] = mapped_column(String(1000))
+
+
+class Topic(IdMixin, TimestampMixin, Base):
+    __tablename__ = "topics"
+    __table_args__ = (UniqueConstraint("workspace_id", "slug"),)
+
+    workspace_id: Mapped[uuid.UUID] = _ws_fk()
+    name: Mapped[str] = mapped_column(String(200))
+    slug: Mapped[str] = mapped_column(String(200))
+    summary: Mapped[str | None] = mapped_column(Text)
+    post_count: Mapped[int] = mapped_column(Integer, default=0)
+
+
+class DocumentTopic(Base):
+    __tablename__ = "document_topics"
+
+    document_id: Mapped[uuid.UUID] = mapped_column(
+        Uuid, ForeignKey("documents.id", ondelete="CASCADE"), primary_key=True
+    )
+    topic_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("topics.id", ondelete="CASCADE"), primary_key=True)
+    weight: Mapped[float] = mapped_column(Float, default=1.0)
 
 
 class TrackedLink(IdMixin, TimestampMixin, Base):

backend/app/pipeline/generate.py

@@ -0,0 +1,182 @@
+"""Text generation jobs over the corpus: notebook summaries, topic map, voice profile, evergreen scores."""
+
+import uuid
+from collections import Counter, defaultdict
+from datetime import UTC, datetime
+
+from sqlalchemy import delete, select
+from sqlalchemy.orm import Session
+
+from app.corpus import Corpus, slugify
+from app.llm import run
+from app.llm.provider import fast_lm
+from app.llm.signatures import BuildVoiceProfile, ConsolidateTopics, EvergreenScore, ExtractTopics, SummarizeNotebook
+from app.models import Artifact, DocumentTopic, Job, Notebook, Topic, VoiceProfile
+from app.pipeline.passages import as_citations, notebook_docs, passages_for, tools_for, workspace_docs
+from app.pipeline.research import verify_citations
+from app.services.jobs import update_job
+
+
+class NothingToDo(ValueError):
+    pass
+
+
+# Notebook summary
+
+
+def summarize_notebook(db: Session, job: Job, artifact: Artifact) -> dict:
+    nb = db.get(Notebook, artifact.notebook_id)
+    docs = notebook_docs(db, nb.id)
+    if not docs:
+        raise NothingToDo("This notebook has no posts yet.")
+    corpus = Corpus(nb.workspace_id)
+    update_job(db, job, progress=0.1, message=f"Reading {len(docs)} posts")
+    passages = passages_for(corpus, docs)
+    update_job(db, job, progress=0.3, message="Writing the summary")
+    out = run.predict(SummarizeNotebook, db=db, workspace_id=nb.workspace_id, job=job,
+                      title=nb.title, passages=passages)
+    tools = tools_for(corpus, docs)
+    text, cites = verify_citations(tools, out.get("summary") or "", as_citations(out.get("citations")))
+    by_path = {d.path: d for d in docs}
+    content = {
+        "title": f"Summary: {nb.title}",
+        "summary": text,
+        "themes": out.get("themes") or [],
+        "citations": [
+            {"marker": c.marker, "path": c.path, "line_start": c.line_start, "line_end": c.line_end,
+             "span": c.quote, "title": by_path[c.path].title, "url": by_path[c.path].url,
+             "document_id": str(by_path[c.path].id)}
+            for c in cites if c.path in by_path
+        ],
+    }
+    artifact.content_json = content
+    artifact.status = "ready"
+    nb.summary = text
+    db.commit()
+    return {"artifact_id": str(artifact.id)}
+
+
+# Topic map
+
+
+def extract_topics(db: Session, job: Job, workspace_id: uuid.UUID, doc_ids: list[str] | None = None) -> dict:
+    """Tag posts that have no topics yet (or the given ones), then rebuild the workspace topic table."""
+    docs = workspace_docs(db, workspace_id, [uuid.UUID(i) for i in doc_ids] if doc_ids else None)
+    todo = [d for d in docs if doc_ids or "topics" not in (d.metadata_json or {})]
+    known = sorted({t["name"] for d in docs for t in (d.metadata_json or {}).get("topics", [])})[:200]
+    lm = fast_lm()
+    for i, d in enumerate(todo):
+        update_job(db, job, progress=0.05 + 0.75 * i / max(len(todo), 1), message=f"Mapping {d.title[:70]}")
+        try:
+            out = run.predict(ExtractTopics, db=db, workspace_id=workspace_id, job=job, lm=lm,
+                              title=d.title, text=(d.clean_text or "")[:12000], known_topics=known)
+        except Exception:
+            continue
+        tags = [t for t in (out.get("topics") or []) if t.get("name")][:8]
+        d.metadata_json = {**(d.metadata_json or {}), "topics": tags}
+        known = sorted(set(known) | {t["name"] for t in tags})[:200]
+        db.commit()
+    update_job(db, job, progress=0.85, message="Drawing constellations")
+    count = rebuild_topics(db, job, workspace_id)
+    return {"tagged": len(todo), "topics": count}
+
+
+def rebuild_topics(db: Session, job: Job | None, workspace_id: uuid.UUID) -> int:
+    docs = workspace_docs(db, workspace_id)
+    counts: Counter[str] = Counter()
+    titles: dict[str, list[str]] = defaultdict(list)
+    for d in docs:
+        for t in (d.metadata_json or {}).get("topics", []):
+            counts[t["name"]] += 1
+            if len(titles[t["name"]]) < 3:
+                titles[t["name"]].append(d.title)
+    if not counts:
+        return 0
+    canonical = {name: name for name in counts}
+    summaries: dict[str, str] = {}
+    listing = [f"{n} ({c}): {'; '.join(titles[n])}" for n, c in counts.most_common(150)]
+    try:
+        out = run.predict(ConsolidateTopics, db=db, workspace_id=workspace_id, job=job, lm=fast_lm(), topics=listing)
+        for g in out.get("groups") or []:
+            name = (g.get("canonical") or "").strip()
+            if not name:
+                continue
+            summaries[name] = g.get("summary") or ""
+            for m in g.get("members") or []:
+                if m in canonical:
+                    canonical[m] = name
+    except Exception:
+        pass  # keep raw names; the map still works
+
+    db.execute(delete(DocumentTopic).where(DocumentTopic.document_id.in_([d.id for d in docs])))
+    db.execute(delete(Topic).where(Topic.workspace_id == workspace_id))
+    db.flush()
+    topics: dict[str, Topic] = {}
+    for d in docs:
+        weights: dict[str, float] = {}
+        for t in (d.metadata_json or {}).get("topics", []):
+            name = canonical.get(t["name"], t["name"])
+            weights[name] = max(weights.get(name, 0), float(t.get("weight") or 0.5))
+        for name, weight in weights.items():
+            slug = slugify(name, 120)
+            topic = topics.get(slug)
+            if not topic:
+                topic = Topic(workspace_id=workspace_id, name=name, slug=slug, summary=summaries.get(name))
+                db.add(topic)
+                db.flush()
+                topics[slug] = topic
+            topic.post_count += 1
+            db.add(DocumentTopic(document_id=d.id, topic_id=topic.id, weight=weight))
+    db.commit()
+    return len(topics)
+
+
+# Voice profile
+
+
+def build_voice_profile(db: Session, job: Job, workspace_id: uuid.UUID, doc_ids: list[str]) -> dict:
+    docs = workspace_docs(db, workspace_id, [uuid.UUID(i) for i in doc_ids] if doc_ids else None)
+    if not doc_ids:
+        docs = sorted(docs, key=lambda d: len(d.clean_text or ""), reverse=True)[:8]
+    if not docs:
+        raise NothingToDo("Add some posts before building a voice profile.")
+    update_job(db, job, progress=0.2, message=f"Listening to {len(docs)} posts")
+    samples = [f"# {d.title}\n\n{(d.clean_text or '')[:6000]}" for d in docs[:10]]
+    out = run.predict(BuildVoiceProfile, db=db, workspace_id=workspace_id, job=job, samples=samples)
+    vp = db.scalar(select(VoiceProfile).where(VoiceProfile.workspace_id == workspace_id))
+    if not vp:
+        vp = VoiceProfile(workspace_id=workspace_id)
+        db.add(vp)
+    vp.profile_json = out.get("profile") or {}
+    vp.sample_doc_ids = [str(d.id) for d in docs[:10]]
+    db.commit()
+    return {"profile": vp.profile_json}
+
+
+# Resurfacing
+
+
+def score_evergreen(db: Session, job: Job, workspace_id: uuid.UUID, limit: int = 60) -> dict:
+    docs = [d for d in workspace_docs(db, workspace_id) if d.evergreen_score is None][:limit]
+    lm = fast_lm()
+    scored = 0
+    for i, d in enumerate(docs):
+        update_job(db, job, progress=0.05 + 0.9 * i / max(len(docs), 1), message=f"Weighing {d.title[:70]}")
+        try:
+            out = run.predict(
+                EvergreenScore, db=db, workspace_id=workspace_id, job=job, lm=lm, title=d.title,
+                published=d.published_at.date().isoformat() if d.published_at else "unknown",
+                excerpt=(d.clean_text or "")[:2500],
+            )
+        except Exception:
+            continue
+        try:
+            d.evergreen_score = max(0.0, min(1.0, float(out.get("score") or 0)))
+        except (TypeError, ValueError):
+            continue
+        d.metadata_json = {**(d.metadata_json or {}), "evergreen_reason": out.get("reason") or "",
+                           "reshare_angle": out.get("angle") or "",
+                           "scored_at": datetime.now(UTC).isoformat()}
+        scored += 1
+        db.commit()
+    return {"scored": scored}

backend/app/pipeline/ingest.py

@@ -1,13 +1,22 @@
-"""Ingestion: feed -> raw HTML to R2 -> clean markdown files in the workspace corpus -> INDEX.md."""
+"""Ingestion: feed -> raw HTML to storage -> clean markdown files in the workspace corpus -> INDEX.md.
+
+Three ways in, one pipeline out:
+- feeds (Substack, Ghost, Medium, any RSS/Atom), with Substack's archive API for posts past the feed
+- one article by URL
+- an uploaded file (markdown, text, HTML or PDF)
+All of them end in `store_entries`, which writes documents, raw HTML and corpus files.
+"""
 
 import hashlib
+import io
 import logging
 import re
+import time
 import uuid
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from time import mktime
-from urllib.parse import urlparse
+from urllib.parse import urljoin, urlparse
 
 import feedparser
 import httpx
@@ -23,13 +32,25 @@ from app.services.storage import keys, storage
 log = logging.getLogger(__name__)
 
 _DROP = re.compile(r"(subscribe|share this post|leave a comment|thanks for reading)", re.I)
+UA = {"User-Agent": "Mozilla/5.0 (compatible; NotestackBot/0.2; +https://notestack.ai)"}
+IMPORTS_FEED = "notestack:imports"
 
 
-def normalize_feed_url(url: str) -> tuple[str, str]:
-    """Accept 'name.substack.com', a Substack post URL, or any feed URL. Returns (feed_url, platform)."""
+class FeedNotFound(ValueError):
+    pass
+
+
+# Feed discovery
+
+
+def _with_scheme(url: str) -> str:
     url = url.strip()
-    if not url.startswith(("http://", "https://")):
-        url = f"https://{url}"
+    return url if url.startswith(("http://", "https://")) else f"https://{url}"
+
+
+def normalize_feed_url(url: str) -> tuple[str, str]:
+    """Best guess without network: accept 'name.substack.com', a post URL, or any feed URL."""
+    url = _with_scheme(url)
     parsed = urlparse(url)
     host = parsed.netloc.lower()
     if host.endswith("substack.com") or "/p/" in parsed.path:
@@ -41,12 +62,82 @@ def normalize_feed_url(url: str) -> tuple[str, str]:
     return f"{parsed.scheme}://{host}/feed", "rss"
 
 
+def _parse_feed(url: str, timeout: float = 15) -> tuple[feedparser.FeedParserDict | None, httpx.Response | None]:
+    try:
+        resp = httpx.get(url, follow_redirects=True, timeout=timeout, headers=UA)
+    except httpx.HTTPError:
+        return None, None
+    if resp.status_code >= 400:
+        return None, resp
+    parsed = feedparser.parse(resp.content)
+    if parsed.get("version") and (parsed.entries or parsed.feed.get("title")):
+        return parsed, resp
+    return None, resp
+
+
+@dataclass
+class DiscoveredFeed:
+    feed_url: str
+    platform: str
+    title: str | None
+    site_url: str
+
+
+def discover_feed(url: str) -> DiscoveredFeed:
+    """Find a working feed for whatever the writer pasted, or raise FeedNotFound with a clear reason."""
+    url = _with_scheme(url)
+    guess, platform = normalize_feed_url(url)
+    parsed_url = urlparse(url)
+    site = f"{parsed_url.scheme}://{parsed_url.netloc}"
+    candidates = [guess, url, f"{site}/feed", f"{site}/rss", f"{site}/rss.xml", f"{site}/feed.xml", f"{site}/atom.xml"]
+    seen: set[str] = set()
+    last: httpx.Response | None = None
+    for candidate in candidates:
+        if candidate in seen:
+            continue
+        seen.add(candidate)
+        parsed, resp = _parse_feed(candidate)
+        last = resp or last
+        if parsed:
+            return DiscoveredFeed(candidate, _platform(candidate, parsed, platform), parsed.feed.get("title"), site)
+        if resp is not None and resp.status_code < 400 and "html" in resp.headers.get("content-type", ""):
+            # A web page: look for <link rel="alternate" type="application/rss+xml">.
+            soup = BeautifulSoup(resp.text, "html.parser")
+            for link in soup.find_all("link", rel="alternate"):
+                if "rss" in (link.get("type") or "") or "atom" in (link.get("type") or ""):
+                    href = urljoin(str(resp.url), link.get("href", ""))
+                    if href and href not in seen:
+                        seen.add(href)
+                        found, _ = _parse_feed(href)
+                        if found:
+                            return DiscoveredFeed(href, _platform(href, found, platform), found.feed.get("title"), site)
+    if last is not None and last.status_code == 404:
+        raise FeedNotFound(f"{parsed_url.netloc} returned 404. Check the address in your browser.")
+    raise FeedNotFound("We couldn't find a feed at that address.")
+
+
+def _platform(feed_url: str, parsed, guess: str) -> str:
+    generator = (parsed.feed.get("generator") or "").lower()
+    if "substack" in feed_url or "substack" in generator:
+        return "substack"
+    if "ghost" in generator:
+        return "ghost"
+    if "medium.com" in feed_url:
+        return "medium"
+    if "wordpress" in generator:
+        return "wordpress"
+    return guess if guess != "rss" else "rss"
+
+
+# Text extraction
+
+
 def html_to_sections(html: str) -> list[tuple[str | None, str]]:
     soup = BeautifulSoup(html, "html.parser")
-    for tag in soup(["script", "style", "nav", "footer", "form", "button", "iframe"]):
+    for tag in soup(["script", "style", "nav", "footer", "form", "button", "iframe", "noscript", "svg"]):
         tag.decompose()
     sections: list[tuple[str | None, list[str]]] = [(None, [])]
-    for el in soup.find_all(["h1", "h2", "h3", "p", "li", "blockquote"]):
+    for el in soup.find_all(["h1", "h2", "h3", "p", "li", "blockquote", "pre"]):
         text = el.get_text(" ", strip=True)
         if not text or (len(text) < 120 and _DROP.search(text)):
             continue
@@ -57,16 +148,94 @@ def html_to_sections(html: str) -> list[tuple[str | None, str]]:
     return [(h, "\n\n".join(p)) for h, p in sections if p]
 
 
+def markdown_to_sections(text: str) -> list[tuple[str | None, str]]:
+    sections: list[tuple[str | None, list[str]]] = [(None, [])]
+    para: list[str] = []
+
+    def flush() -> None:
+        if para:
+            sections[-1][1].append(" ".join(para).strip())
+            para.clear()
+
+    for line in text.splitlines():
+        stripped = line.strip()
+        heading = re.match(r"^#{1,3}\s+(.*)", stripped)
+        if heading:
+            flush()
+            sections.append((heading.group(1).strip(), []))
+        elif not stripped:
+            flush()
+        else:
+            para.append(stripped)
+    flush()
+    return [(h, "\n\n".join(p)) for h, p in sections if p]
+
+
+def pdf_to_sections(data: bytes) -> list[tuple[str | None, str]]:
+    from pypdf import PdfReader
+
+    reader = PdfReader(io.BytesIO(data))
+    text = "\n\n".join((page.extract_text() or "") for page in reader.pages)
+    # PDF text keeps hard line breaks inside paragraphs; blank lines separate paragraphs.
+    paragraphs = [re.sub(r"\s*\n\s*", " ", p).strip() for p in re.split(r"\n\s*\n", text)]
+    return [(None, "\n\n".join(p for p in paragraphs if p))] if any(paragraphs) else []
+
+
+def _meta(soup: BeautifulSoup, *names: str) -> str | None:
+    for name in names:
+        tag = soup.find("meta", attrs={"property": name}) or soup.find("meta", attrs={"name": name})
+        if tag and tag.get("content"):
+            return tag["content"].strip()
+    return None
+
+
+def _parse_date(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
+        return dt if dt.tzinfo else dt.replace(tzinfo=UTC)
+    except ValueError:
+        return None
+
+
 @dataclass
 class FeedEntry:
     title: str
     url: str
     published_at: datetime | None
     html: str
+    sections: list[tuple[str | None, str]] | None = None  # set when the source is not HTML
+
+
+def extract_article(url: str) -> FeedEntry:
+    """Fetch one web page and keep the article body."""
+    url = _with_scheme(url)
+    resp = httpx.get(url, follow_redirects=True, timeout=30, headers=UA)
+    if resp.status_code >= 400:
+        raise FeedNotFound(f"That page returned {resp.status_code}.")
+    soup = BeautifulSoup(resp.text, "html.parser")
+    title = _meta(soup, "og:title", "twitter:title") or (soup.title.get_text(strip=True) if soup.title else url)
+    published = _parse_date(_meta(soup, "article:published_time", "og:published_time", "date"))
+    if not published and (t := soup.find("time", datetime=True)):
+        published = _parse_date(t["datetime"])
+    body = soup.find("article") or soup.find("main") or soup.find(class_=re.compile("post-content|entry-content|body"))
+    if body is None:
+        # Fall back to the element holding the most paragraph text.
+        candidates = soup.find_all(["div", "section"])
+        body = max(candidates, key=lambda el: sum(len(p.get_text()) for p in el.find_all("p", recursive=False)),
+                   default=soup.body or soup)
+    html = str(body)
+    if not html_to_sections(html):
+        raise FeedNotFound("We could not find article text on that page.")
+    return FeedEntry(title=title[:1000], url=str(resp.url), published_at=published, html=html)
+
+
+# Feed fetching
 
 
 def fetch_feed(feed_url: str, limit: int) -> tuple[str | None, list[FeedEntry]]:
-    resp = httpx.get(feed_url, follow_redirects=True, timeout=30, headers={"User-Agent": "NotestackBot/0.1"})
+    resp = httpx.get(feed_url, follow_redirects=True, timeout=30, headers=UA)
     resp.raise_for_status()
     parsed = feedparser.parse(resp.content)
     entries = []
@@ -84,7 +253,45 @@ def fetch_feed(feed_url: str, limit: int) -> tuple[str | None, list[FeedEntry]]:
     return parsed.feed.get("title"), entries
 
 
+def fetch_substack_archive(site: str, known_urls: set[str], limit: int, on_page=None) -> list[FeedEntry]:
+    """Posts beyond the ~20 the RSS feed carries, through Substack's public archive API."""
+    out: list[FeedEntry] = []
+    offset = 0
+    with httpx.Client(follow_redirects=True, timeout=30, headers=UA) as client:
+        while len(out) + len(known_urls) < limit:
+            resp = client.get(f"{site}/api/v1/archive", params={"sort": "new", "offset": offset, "limit": 50})
+            if resp.status_code >= 400:
+                break
+            page = resp.json()
+            if not page:
+                break
+            offset += len(page)
+            for item in page:
+                url = item.get("canonical_url") or f"{site}/p/{item.get('slug')}"
+                if url in known_urls or item.get("type") not in (None, "newsletter", "thread"):
+                    continue
+                if len(out) + len(known_urls) >= limit:
+                    break
+                post = client.get(f"{site}/api/v1/posts/{item['slug']}")
+                if post.status_code >= 400:
+                    continue
+                body = post.json().get("body_html") or item.get("truncated_body_text") or ""
+                if not body:
+                    continue
+                out.append(FeedEntry(title=item.get("title") or "Untitled", url=url,
+                                     published_at=_parse_date(item.get("post_date")), html=body))
+                time.sleep(0.2)  # be polite to Substack
+            if on_page:
+                on_page(len(out))
+    return out
+
+
+# Storing
+
+
 def source_slug(source: Source) -> str:
+    if source.feed_url == IMPORTS_FEED:
+        return "imports"
     return slugify(urlparse(source.site_url or source.feed_url).netloc or str(source.id), 50)
 
 
@@ -93,34 +300,46 @@ def rebuild_index(db: Session, workspace_id: uuid.UUID) -> str:
         select(Document).where(Document.workspace_id == workspace_id, Document.path.is_not(None))
     ).all()
     rows = sorted(
-        (index_line(d.path, d.title, d.published_at, d.metadata_json.get("headings", [])) for d in docs),
+        (index_line(d.path, d.title, d.published_at, (d.metadata_json or {}).get("headings", [])) for d in docs),
         reverse=True,
     )
     header = "# Archive index\n\ndate | path | title | section headings\n\n"
     return header + "\n".join(rows) + "\n"
 
 
-def ingest_source(db: Session, source: Source, job: Job, max_posts: int) -> dict:
-    source.sync_status = "syncing"
-    update_job(db, job, status="running", progress=0.02, message="Contacting your feed")
+def imports_source(db: Session, workspace_id: uuid.UUID) -> Source:
+    """Single articles and uploaded files live under one built in source that does not count to limits."""
+    source = db.scalar(select(Source).where(Source.workspace_id == workspace_id, Source.feed_url == IMPORTS_FEED))
+    if not source:
+        source = Source(workspace_id=workspace_id, feed_url=IMPORTS_FEED, platform="imports",
+                        title="Imported posts and files", sync_status="ok")
+        db.add(source)
+        db.flush()
+    return source
 
-    title, entries = fetch_feed(source.feed_url, max_posts)
-    source.title = source.title or title
+
+def store_entries(
+    db: Session, source: Source, entries: list[FeedEntry], progress=None
+) -> tuple[int, int, list[uuid.UUID]]:
+    """Write documents, raw HTML and corpus files. Returns (indexed, skipped, changed document ids)."""
     corpus = Corpus(source.workspace_id)
     folder = source_slug(source)
-    total = max(len(entries), 1)
     indexed = skipped = 0
+    changed: list[uuid.UUID] = []
     files: dict[str, str] = {}
-
-    taken = set(
-        db.scalars(select(Document.path).where(Document.workspace_id == source.workspace_id)).all()
-    )
+    taken = set(db.scalars(select(Document.path).where(Document.workspace_id == source.workspace_id)).all())
+    total = max(len(entries), 1)
     for i, entry in enumerate(entries):
-        content_hash = hashlib.sha256(entry.html.encode()).hexdigest()
+        payload = entry.html or "\n\n".join(t for _, t in (entry.sections or []))
+        content_hash = hashlib.sha256(payload.encode()).hexdigest()
         doc = db.scalar(select(Document).where(Document.source_id == source.id, Document.url == entry.url))
         if doc and doc.content_hash == content_hash and doc.path:
             skipped += 1
             continue
+        sections = entry.sections if entry.sections is not None else html_to_sections(entry.html)
+        if not sections:
+            skipped += 1
+            continue
         if not doc:
             doc = Document(id=uuid.uuid4(), workspace_id=source.workspace_id, source_id=source.id, url=entry.url)
             db.add(doc)
@@ -135,25 +354,76 @@ def ingest_source(db: Session, source: Source, job: Job, max_posts: int) -> dict
         doc.title = entry.title
         doc.published_at = entry.published_at
         doc.content_hash = content_hash
-        doc.raw_html_key = storage.put_text(
-            keys.raw_html(source.workspace_id, source.id, doc.id), entry.html, "text/html; charset=utf-8"
-        )
-        sections = html_to_sections(entry.html)
+        if entry.html:
+            doc.raw_html_key = storage.put_text(
+                keys.raw_html(source.workspace_id, source.id, doc.id), entry.html, "text/html; charset=utf-8"
+            )
         doc.clean_text = "\n\n".join(f"## {h}\n{t}" if h else t for h, t in sections)
-        doc.metadata_json = {**(doc.metadata_json or {}), "headings": [h for h, _ in sections if h]}
+        doc.metadata_json = {
+            **(doc.metadata_json or {}),
+            "headings": [h for h, _ in sections if h],
+            "words": len(doc.clean_text.split()),
+        }
         files[doc.path] = render_post(
             title=entry.title, url=entry.url, published_at=entry.published_at, source=folder, sections=sections
         )
         indexed += 1
+        changed.append(doc.id)
         db.commit()
-        update_job(db, job, progress=0.05 + 0.85 * (i + 1) / total, message=f"{entry.title[:80]} is in orbit")
-
-    update_job(db, job, progress=0.93, message="Writing your archive index")
+        if progress:
+            progress(i + 1, total, entry.title)
+    db.flush()
     files[INDEX] = rebuild_index(db, source.workspace_id)
     corpus.write_files(files)
+    db.commit()
+    return indexed, skipped, changed
+
+
+def ingest_source(db: Session, source: Source, job: Job, max_posts: int) -> dict:
+    source.sync_status = "syncing"
+    update_job(db, job, status="running", progress=0.02, message="Contacting your feed")
+
+    title, entries = fetch_feed(source.feed_url, max_posts)
+    source.title = source.title or title
+    if source.platform == "substack" and len(entries) < max_posts:
+        site = f"{urlparse(source.feed_url).scheme}://{urlparse(source.feed_url).netloc}"
+        update_job(db, job, progress=0.05, message="Reading your Substack archive")
+        try:
+            entries += fetch_substack_archive(
+                site, {e.url for e in entries}, max_posts,
+                on_page=lambda n: update_job(db, job, message=f"Found {len(entries) + n} posts in the archive"),
+            )
+        except (httpx.HTTPError, ValueError):
+            log.warning("substack archive fetch failed for %s", site, exc_info=True)
+
+    def progress(done: int, total: int, post_title: str) -> None:
+        update_job(db, job, progress=0.1 + 0.8 * done / total, message=f"{post_title[:80]} is in orbit")
+
+    update_job(db, job, progress=0.1, message=f"Indexing {len(entries)} posts")
+    indexed, skipped, changed = store_entries(db, source, entries, progress)
 
     source.sync_status = "ok"
     source.sync_error = None
     source.last_synced_at = datetime.now(UTC)
     db.commit()
-    return {"indexed": indexed, "skipped": skipped, "found": len(entries)}
+    return {"indexed": indexed, "skipped": skipped, "found": len(entries), "changed": [str(i) for i in changed]}
+
+
+def entry_from_upload(filename: str, content_type: str, data: bytes) -> FeedEntry:
+    name = filename.rsplit("/", 1)[-1]
+    stem = re.sub(r"\.[A-Za-z0-9]+$", "", name).replace("-", " ").replace("_", " ").strip() or "Untitled"
+    lower = name.lower()
+    url = f"upload://{name}"
+    if content_type == "application/pdf" or lower.endswith(".pdf"):
+        return FeedEntry(title=stem, url=url, published_at=None, html="", sections=pdf_to_sections(data))
+    text = data.decode("utf-8", errors="replace")
+    if content_type == "text/html" or lower.endswith((".html", ".htm")):
+        soup = BeautifulSoup(text, "html.parser")
+        title = soup.title.get_text(strip=True) if soup.title else stem
+        return FeedEntry(title=title, url=url, published_at=None, html=text)
+    sections = markdown_to_sections(text)
+    first_heading = re.search(r"^#\s+(.+)$", text, re.M)
+    title = first_heading.group(1).strip() if first_heading else stem
+    if sections and sections[0][0] == title:
+        sections[0] = (None, sections[0][1])
+    return FeedEntry(title=title, url=url, published_at=None, html="", sections=sections)

backend/app/pipeline/launchkit.py

@@ -0,0 +1,108 @@
+"""Launch Kit: one post in, everything needed to launch it out, in the writer's voice and grounded in
+the post's own claims."""
+
+from sqlalchemy.orm import Session
+
+from app.corpus import INDEX, Corpus
+from app.llm import run
+from app.llm.signatures import CarouselSlides, ExtractClaims, HookGenerator, PickQuotes, PlatformAdapter, SeoPack
+from app.models import Artifact, Document, Job
+from app.pipeline.generate import NothingToDo
+from app.pipeline.passages import passages_for, tools_for, verify_refs, voice_text
+from app.services.jobs import create_job, update_job
+
+PLATFORMS = {
+    "x_thread": ("X thread", "5 to 9 posts, each under 270 characters. Post 1 is the hook and stands alone. "
+                 "No hashtags except at most one at the end. The last post links to the article with {url}."),
+    "linkedin": ("LinkedIn", "One post, 900 to 1,500 characters. Short paragraphs of 1 to 2 lines. Open with a "
+                 "concrete line, not a question. End with the link {url} and one question for comments."),
+    "substack_notes": ("Substack Notes", "3 separate notes, each 1 to 4 sentences, conversational, each able "
+                       "to stand alone. One of them links to {url}."),
+    "bluesky": ("Bluesky", "A thread of 3 to 6 posts, each under 290 characters. Plain, human, no hashtags. "
+                "The last post links to {url}."),
+}
+
+
+def build_launch_kit(db: Session, job: Job, artifact: Artifact) -> dict:
+    doc = db.get(Document, artifact.document_id) if artifact.document_id else None
+    if not doc or not doc.path:
+        raise NothingToDo("Pick a post for the Launch Kit.")
+    ws = artifact.workspace_id
+    corpus = Corpus(ws)
+    passages = passages_for(corpus, [doc], budget_chars=50_000)
+    tools = tools_for(corpus, [doc])
+    voice = voice_text(db, ws)
+    url = doc.url if doc.url.startswith("http") else ""
+
+    def step(progress: float, message: str) -> None:
+        update_job(db, job, progress=progress, message=message)
+
+    step(0.08, "Pulling out the key claims")
+    claims_out = run.predict(ExtractClaims, db=db, workspace_id=ws, job=job, passages=passages)
+    claims = []
+    for c in claims_out.get("claims") or []:
+        refs = verify_refs(tools, c.get("sources") or [])
+        if refs and c.get("claim"):
+            claims.append({"claim": c["claim"], "sources": refs})
+    if not claims:
+        raise RuntimeError("No verifiable claims found in this post")
+    claim_inputs = [{"claim": c["claim"], "sources": [{k: s[k] for k in ("path", "line_start", "line_end")}
+                                                      for s in c["sources"]]} for c in claims]
+
+    step(0.2, "Writing hooks")
+    hooks = run.predict(HookGenerator, db=db, workspace_id=ws, job=job, passages=passages,
+                        voice_profile=voice, n=8).get("hooks") or []
+    hooks = sorted(hooks, key=lambda h: -(h.get("strength") or 0))
+
+    posts: dict[str, list[str]] = {}
+    for i, (platform, (label, rules)) in enumerate(PLATFORMS.items()):
+        step(0.3 + 0.1 * i, f"Adapting for {label}")
+        out = run.predict(PlatformAdapter, db=db, workspace_id=ws, job=job, claims=claim_inputs, voice_profile=voice,
+                          platform=platform, platform_rules=rules.format(url=url or "the post"))
+        posts[platform] = [p for p in (out.get("posts") or []) if p.strip()]
+
+    step(0.72, "Building the SEO pack")
+    related = []
+    if corpus.exists(INDEX):
+        related = [ln for ln in corpus.read_lines(INDEX) if "|" in ln and doc.path not in ln][:150]
+    seo = run.predict(SeoPack, db=db, workspace_id=ws, job=job, title=doc.title, passages=passages,
+                      related_posts="\n".join(related)).get("seo") or {}
+
+    step(0.82, "Designing the carousel")
+    slides = run.predict(CarouselSlides, db=db, workspace_id=ws, job=job, passages=passages,
+                         voice_profile=voice).get("slides") or []
+
+    step(0.9, "Choosing quotes")
+    quotes = []
+    for pick in run.predict(PickQuotes, db=db, workspace_id=ws, job=job, passages=passages, n=3).get("quotes") or []:
+        refs = verify_refs(tools, [pick.get("source") or {}])
+        if refs and pick.get("quote"):
+            quotes.append({"quote": pick["quote"][:380], "source": refs[0]})
+
+    artifact.content_json = {
+        **(artifact.content_json or {}),
+        "title": f"Launch Kit: {doc.title}",
+        "post_title": doc.title,
+        "post_url": doc.url,
+        "claims": claims,
+        "hooks": hooks,
+        "posts": posts,
+        "seo": seo,
+        "carousel": slides,
+        "quotes": quotes,
+    }
+    artifact.status = "ready"
+    db.commit()
+
+    # Quote cards render as their own artifacts, so a missing renderer does not sink the kit.
+    card_ids = []
+    for q in quotes:
+        card = Artifact(workspace_id=ws, document_id=doc.id, type="quote_card", status="pending",
+                        content_json={"title": f"Quote: {doc.title}", "parent_id": str(artifact.id), **q})
+        db.add(card)
+        db.flush()
+        create_job(db, ws, "quote_card", {"artifact_id": str(card.id)}, artifact_id=card.id, max_attempts=1)
+        card_ids.append(str(card.id))
+    artifact.content_json = {**artifact.content_json, "quote_card_ids": card_ids}
+    db.commit()
+    return {"artifact_id": str(artifact.id), "claims": len(claims)}

backend/app/pipeline/media.py

@@ -0,0 +1,260 @@
+"""Audio overviews (two hosts, ElevenLabs), videos and stills (Remotion renderer)."""
+
+import uuid
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.corpus import Corpus
+from app.llm import run
+from app.llm.signatures import PickQuotes, PodcastScript, VideoStoryboard
+from app.models import Artifact, Document, Job, Source, User, VoiceProfile, Workspace
+from app.pipeline.generate import NothingToDo
+from app.pipeline.passages import notebook_docs, passages_for, tools_for, verify_refs
+from app.services import tts
+from app.services.jobs import record_usage, update_job
+from app.services.renderer import PermanentJobError, brand_props, request_render
+from app.services.storage import keys, storage
+
+SCENE_TYPES = {"title", "section", "pull_quote", "number", "outro"}
+VIDEO_STYLES = {
+    "short": ("ShortVertical", "9:16"),
+    "explainer": ("ExplainerLong", "16:9"),
+}
+
+
+def artifact_docs(db: Session, artifact: Artifact) -> list[Document]:
+    if artifact.document_id:
+        doc = db.get(Document, artifact.document_id)
+        return [doc] if doc and doc.path else []
+    if artifact.notebook_id:
+        return notebook_docs(db, artifact.notebook_id)
+    return []
+
+
+def host_voices(db: Session, workspace_id: uuid.UUID) -> tuple[str, str]:
+    vp = db.scalar(select(VoiceProfile).where(VoiceProfile.workspace_id == workspace_id))
+    voices = (vp.host_voices if vp else None) or {}
+    return voices.get("host_a") or settings.elevenlabs_voice_a, voices.get("host_b") or settings.elevenlabs_voice_b
+
+
+def _require_tts() -> None:
+    if not settings.elevenlabs_api_key:
+        raise PermanentJobError("Audio needs ELEVENLABS_API_KEY in .env")
+
+
+# Audio overview
+
+
+def audio_overview(db: Session, job: Job, artifact: Artifact) -> dict:
+    _require_tts()
+    params = job.params
+    docs = artifact_docs(db, artifact)
+    if not docs:
+        raise NothingToDo("Add posts before making an audio overview.")
+    corpus = Corpus(artifact.workspace_id)
+    update_job(db, job, progress=0.05, message=f"Reading {len(docs)} posts")
+    passages = passages_for(corpus, docs, budget_chars=90_000)
+    update_job(db, job, progress=0.15, message="Writing the conversation")
+    out = run.predict(PodcastScript, db=db, workspace_id=artifact.workspace_id, job=job, passages=passages,
+                      format=params.get("format", "deep_dive"), target_minutes=int(params.get("minutes", 6)))
+    lines = [ln for ln in (out.get("lines") or []) if (ln.get("text") or "").strip()]
+    if not lines:
+        raise RuntimeError("The script came back empty")
+    tools = tools_for(corpus, docs)
+    voice_a, voice_b = host_voices(db, artifact.workspace_id)
+    audio = bytearray()
+    segments = []
+    t = 0.0
+    for i, line in enumerate(lines):
+        voice = voice_a if line.get("speaker") == "host_a" else voice_b
+        clip = tts.synthesize(artifact.workspace_id, line["text"], voice)
+        seconds = tts.mp3_seconds(clip)
+        segments.append({"speaker": line.get("speaker", "host_a"), "text": line["text"],
+                         "start": round(t, 2), "end": round(t + seconds, 2),
+                         "sources": verify_refs(tools, line.get("sources") or [])})
+        audio += clip
+        t += seconds
+        update_job(db, job, progress=0.2 + 0.75 * (i + 1) / len(lines),
+                   message=f"Recording line {i + 1} of {len(lines)}")
+    key = keys.artifact(artifact.workspace_id, artifact.id, "audio", "mp3")
+    storage.put_bytes(key, bytes(audio), "audio/mpeg")
+    artifact.storage_key = key
+    artifact.status = "ready"
+    artifact.content_json = {
+        **(artifact.content_json or {}),
+        "format": params.get("format", "deep_dive"),
+        "duration_s": round(t, 1),
+        "segments": segments,
+        "voices": {"host_a": voice_a, "host_b": voice_b},
+    }
+    db.commit()
+    record_usage(db, workspace_id=artifact.workspace_id, kind="tts", provider="elevenlabs",
+                 quantity=round(t, 1), unit="seconds", job=job)
+    return {"artifact_id": str(artifact.id), "duration_s": round(t, 1)}
+
+
+# Video
+
+
+def _clamp(value, lo: float, hi: float) -> float:
+    try:
+        return max(lo, min(hi, float(value)))
+    except (TypeError, ValueError):
+        return lo
+
+
+def make_video(db: Session, job: Job, artifact: Artifact) -> None:
+    style = job.params.get("style", "short")
+    if style == "audiogram":
+        return make_audiogram(db, job, artifact)
+    composition, aspect = VIDEO_STYLES[style]
+    docs = artifact_docs(db, artifact)
+    if not docs:
+        raise NothingToDo("Add posts before making a video.")
+    corpus = Corpus(artifact.workspace_id)
+    workspace = db.get(Workspace, artifact.workspace_id)
+    update_job(db, job, progress=0.05, message="Storyboarding")
+    passages = passages_for(corpus, docs, budget_chars=60_000)
+    out = run.predict(VideoStoryboard, db=db, workspace_id=artifact.workspace_id, job=job,
+                      passages=passages, aspect=aspect)
+    scenes = [s for s in (out.get("scenes") or []) if (s.get("on_screen_text") or "").strip()]
+    if not scenes:
+        raise RuntimeError("The storyboard came back empty")
+    max_scenes = 8 if style == "short" else 14
+    scenes = scenes[:max_scenes]
+
+    captions: list[dict] = []
+    audio = bytearray()
+    cursor = 0.0
+    voice_a, _ = host_voices(db, artifact.workspace_id)
+    out_scenes = []
+    for i, s in enumerate(scenes):
+        narration = (s.get("narration") or "").strip()
+        duration = _clamp(s.get("duration_hint_s"), 2.0, 20.0)
+        if narration and settings.elevenlabs_api_key:
+            update_job(db, job, progress=0.15 + 0.5 * i / len(scenes), message=f"Narrating scene {i + 1}")
+            clip, words = tts.synthesize_with_timestamps(artifact.workspace_id, narration, voice_a)
+            seconds = tts.mp3_seconds(clip)
+            # Pad with silence so each clip starts exactly when its scene does.
+            audio += _silence(cursor - tts.mp3_seconds(bytes(audio)))
+            offset = int(cursor * 1000)
+            captions += [{**w, "startMs": w["startMs"] + offset, "endMs": w["endMs"] + offset} for w in words]
+            audio += clip
+            duration = max(duration, seconds + 0.3)
+        cursor += duration
+        scene_type = s.get("type") if s.get("type") in SCENE_TYPES else "section"
+        out_scenes.append({"type": scene_type, "onScreenText": s["on_screen_text"][:300],
+                           "narration": narration, "durationS": round(_clamp(duration, 0.5, 60), 2)})
+
+    props: dict = {"scenes": out_scenes, "captions": captions, "brand": brand_props(workspace)}
+    title = docs[0].title if len(docs) == 1 else (artifact.content_json or {}).get("title", "Notestack")
+    if composition == "ShortVertical":
+        hook = next((sc["onScreenText"] for sc in out_scenes if sc["type"] == "title"), title)
+        props["hook"] = hook[:120]
+    else:
+        props["title"] = title[:140]
+    if audio:
+        key = keys.artifact(artifact.workspace_id, artifact.id, "narration", "mp3")
+        storage.put_bytes(key, bytes(audio), "audio/mpeg")
+        props["audioUrl"] = storage.presign_get(key, ttl=6 * 3600)
+    artifact.content_json = {**(artifact.content_json or {}), "style": style, "composition": composition,
+                             "scenes": out_scenes, "duration_s": round(cursor, 1)}
+    artifact.status = "rendering"
+    db.commit()
+    update_job(db, job, progress=0.7, message="Handing off to the renderer")
+    request_render(job, artifact, composition, props)
+    record_usage(db, workspace_id=artifact.workspace_id, kind="video", provider="remotion",
+                 quantity=round(cursor, 1), unit="seconds", job=job)
+
+
+_SILENT_FRAME = None
+
+
+def _silence(seconds: float) -> bytes:
+    """MPEG1 Layer III 128 kbps 44.1 kHz silent frames (417 bytes, ~26 ms each)."""
+    global _SILENT_FRAME
+    if _SILENT_FRAME is None:
+        _SILENT_FRAME = bytes([0xFF, 0xFB, 0x90, 0x64]) + bytes(413)
+    frames = int(seconds / 0.026122)
+    return _SILENT_FRAME * max(frames, 0)
+
+
+def make_audiogram(db: Session, job: Job, artifact: Artifact) -> None:
+    source = db.get(Artifact, uuid.UUID(job.params["audio_artifact_id"]))
+    if not source or source.type != "audio_overview" or not source.storage_key:
+        raise PermanentJobError("Pick a finished audio overview for the audiogram.")
+    workspace = db.get(Workspace, artifact.workspace_id)
+    content = source.content_json or {}
+    duration = float(content.get("duration_s") or 30)
+    segments = [{"speaker": s["speaker"], "text": s["text"], "startMs": int(s["start"] * 1000),
+                 "endMs": int(s["end"] * 1000)} for s in content.get("segments", [])]
+    props = {"title": (content.get("title") or "Audio overview")[:140],
+             "audioUrl": storage.presign_get(source.storage_key, ttl=6 * 3600),
+             "durationS": round(min(duration, 600), 2), "segments": segments, "brand": brand_props(workspace)}
+    artifact.content_json = {**(artifact.content_json or {}), "style": "audiogram", "composition": "AudiogramSquare",
+                             "audio_artifact_id": str(source.id), "duration_s": props["durationS"]}
+    artifact.status = "rendering"
+    db.commit()
+    update_job(db, job, progress=0.1, message="Handing off to the renderer")
+    request_render(job, artifact, "AudiogramSquare", props)
+    record_usage(db, workspace_id=artifact.workspace_id, kind="video", provider="remotion",
+                 quantity=props["durationS"], unit="seconds", job=job)
+
+
+# Stills
+
+
+def author_line(db: Session, workspace_id: uuid.UUID) -> str:
+    ws = db.get(Workspace, workspace_id)
+    owner = db.get(User, ws.owner_id) if ws else None
+    return (ws.brand_json or {}).get("name") or (owner.name if owner and owner.name else "") or "Notestack"
+
+
+def make_quote_card(db: Session, job: Job, artifact: Artifact) -> None:
+    content = artifact.content_json or {}
+    workspace = db.get(Workspace, artifact.workspace_id)
+    if not content.get("quote"):
+        docs = artifact_docs(db, artifact)
+        if not docs:
+            raise NothingToDo("Pick a post or notebook for the quote card.")
+        corpus = Corpus(artifact.workspace_id)
+        out = run.predict(PickQuotes, db=db, workspace_id=artifact.workspace_id, job=job,
+                          passages=passages_for(corpus, docs, budget_chars=40_000), n=3)
+        tools = tools_for(corpus, docs)
+        for pick in out.get("quotes") or []:
+            refs = verify_refs(tools, [pick.get("source") or {}])
+            if refs and pick.get("quote"):
+                content = {**content, "quote": pick["quote"][:380], "source": refs[0]}
+                break
+        else:
+            raise RuntimeError("No verifiable quote found")
+    source = content.get("source") or {}
+    doc = db.scalar(select(Document).where(Document.workspace_id == artifact.workspace_id,
+                                           Document.path == source.get("path")))
+    publication = None
+    if doc and doc.source_id:
+        src = db.get(Source, doc.source_id)
+        publication = src.title if src else None
+    props = {"quote": content["quote"][:380], "author": author_line(db, artifact.workspace_id),
+             "brand": brand_props(workspace)}
+    if publication or (doc and doc.title):
+        props["publication"] = (publication or doc.title)[:120]
+    artifact.content_json = {**content, "composition": "QuoteCard"}
+    artifact.status = "rendering"
+    db.commit()
+    request_render(job, artifact, "QuoteCard", props)
+
+
+def render_carousel(db: Session, job: Job, artifact: Artifact) -> None:
+    slides = (artifact.content_json or {}).get("slides") or []
+    if not slides:
+        raise PermanentJobError("This carousel has no slides.")
+    workspace = db.get(Workspace, artifact.workspace_id)
+    brand = brand_props(workspace)
+    stills = [{"heading": s.get("heading", "")[:120], "body": s.get("body", "")[:500], "index": i + 1,
+               "total": len(slides), "brand": brand} for i, s in enumerate(slides)]
+    artifact.status = "rendering"
+    db.commit()
+    request_render(job, artifact, "CarouselSlide", stills=stills)

backend/app/pipeline/passages.py

@@ -0,0 +1,98 @@
+"""Feeding posts to whole-document modules: numbered passages in, verified SourceRefs out."""
+
+import uuid
+from types import SimpleNamespace
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.corpus import Corpus, CorpusError, ScopedTools
+from app.models import Document, NotebookDocument, VoiceProfile
+
+DEFAULT_BUDGET = 120_000  # characters across all passages; GLM-5.3 has room, this keeps cost sane
+
+
+def notebook_docs(db: Session, notebook_id: uuid.UUID) -> list[Document]:
+    return list(db.scalars(
+        select(Document)
+        .join(NotebookDocument, NotebookDocument.document_id == Document.id)
+        .where(NotebookDocument.notebook_id == notebook_id, Document.path.is_not(None))
+        .order_by(Document.published_at.desc())
+    ).all())
+
+
+def workspace_docs(db: Session, workspace_id: uuid.UUID, ids: list[uuid.UUID] | None = None) -> list[Document]:
+    q = select(Document).where(Document.workspace_id == workspace_id, Document.path.is_not(None))
+    if ids is not None:
+        q = q.where(Document.id.in_(ids))
+    return list(db.scalars(q.order_by(Document.published_at.desc())).all())
+
+
+def passages_for(corpus: Corpus, docs: list[Document], budget_chars: int = DEFAULT_BUDGET) -> list[str]:
+    """One string per post: a header line then `n| text` numbered lines (same numbering as read()).
+    The budget is split evenly, so one long post cannot crowd out the rest."""
+    corpus.sync()
+    per_doc = max(budget_chars // max(len(docs), 1), 1500)
+    out = []
+    for d in docs:
+        try:
+            lines = corpus.read_lines(d.path)
+        except CorpusError:
+            continue
+        body, used = [], 0
+        for n, line in enumerate(lines, start=1):
+            if not line.strip():
+                continue
+            entry = f"{n}| {line}"
+            if used + len(entry) > per_doc:
+                body.append("[... post continues]")
+                break
+            body.append(entry)
+            used += len(entry) + 1
+        out.append(f"FILE {d.path} ({d.title})\n" + "\n".join(body))
+    return out
+
+
+def tools_for(corpus: Corpus, docs: list[Document]) -> ScopedTools:
+    return ScopedTools(corpus, {d.path: d.title for d in docs})
+
+
+def verify_refs(tools: ScopedTools, refs: list[dict]) -> list[dict]:
+    """Keep refs that point at real lines of allowed files; attach the quoted text."""
+    out = []
+    for r in refs or []:
+        try:
+            start, end = int(r["line_start"]), int(r["line_end"])
+        except (KeyError, TypeError, ValueError):
+            continue
+        quote = tools.quote(r.get("path", ""), start, end)
+        if quote:
+            out.append({"path": r["path"], "line_start": start, "line_end": max(start, end),
+                        "title": tools.allowed.get(r["path"]), "quote": quote[:600]})
+    return out
+
+
+def as_citations(items: list[dict]) -> list[SimpleNamespace]:
+    """Dicts from predict() in the shape research.verify_citations expects."""
+    out = []
+    for c in items or []:
+        try:
+            out.append(SimpleNamespace(marker=int(c["marker"]), path=c["path"],
+                                       line_start=int(c["line_start"]), line_end=int(c["line_end"])))
+        except (KeyError, TypeError, ValueError):
+            continue
+    return out
+
+
+def voice_text(db: Session, workspace_id: uuid.UUID) -> str:
+    vp = db.scalar(select(VoiceProfile).where(VoiceProfile.workspace_id == workspace_id))
+    if not vp or not vp.profile_json:
+        return "No voice profile yet: write clearly, warmly and concretely."
+    p = vp.profile_json
+    parts = [p.get("summary", "")]
+    for key in ("tone", "vocabulary", "structure_habits", "openings", "avoid"):
+        if p.get(key):
+            parts.append(f"{key.replace('_', ' ')}: {', '.join(p[key])}")
+    if p.get("sentence_length"):
+        parts.append(f"sentence length: {p['sentence_length']}")
+    return "\n".join(x for x in parts if x)

backend/app/routers/artifacts.py

@@ -2,79 +2,205 @@ import uuid
 from typing import Literal
 
 from fastapi import APIRouter, Depends, HTTPException
-from pydantic import BaseModel
-from sqlalchemy import select
+from pydantic import BaseModel, Field
+from sqlalchemy import func, or_, select
 
 from app.auth import Ctx, get_ctx
-from app.models import Artifact
+from app.models import Artifact, CalendarItem, Document, Job, Notebook
+from app.services.artifacts import latest_jobs, serialize_artifact, start_artifact
 from app.services.jobs import create_job, serialize_job
+from app.services.renderer import COMPOSITIONS
 from app.services.storage import storage
+from app.services.usage import check_limit
 
 router = APIRouter(prefix="/api/artifacts", tags=["artifacts"])
 
-COMPOSITIONS = {"ShortVertical": "mp4", "QuoteCard": "png"}
-
-
-class ArtifactIn(BaseModel):
-    type: Literal["summary", "audio_overview", "video", "thread", "linkedin", "notes", "quote_card", "carousel", "seo"]
-    notebook_id: uuid.UUID | None = None
-    document_id: uuid.UUID | None = None
-    content: dict = {}
+ArtifactType = Literal["summary", "audio_overview", "video", "quote_card", "carousel", "launch_kit"]
 
 
 class RenderIn(BaseModel):
-    composition: Literal["ShortVertical", "QuoteCard"]
+    composition: Literal["ShortVertical", "ExplainerLong", "AudiogramSquare", "QuoteCard", "CarouselSlide"]
     props: dict
 
 
-def _serialize(a: Artifact) -> dict:
-    return {
-        "id": str(a.id),
-        "type": a.type,
-        "status": a.status,
-        "content": a.content_json,
-        "storage_key": a.storage_key,
-        "url": storage.presign_get(a.storage_key) if a.storage_key else None,
-    }
+class GenerateIn(BaseModel):
+    type: Literal["summary", "audio_overview", "video", "quote_card", "carousel", "launch_kit"]
+    notebook_id: uuid.UUID | None = None
+    document_id: uuid.UUID | None = None
+    # audio_overview
+    format: Literal["deep_dive", "brief", "debate"] = "deep_dive"
+    minutes: int = Field(6, ge=1, le=30)
+    # video
+    style: Literal["short", "explainer", "audiogram"] = "short"
+    audio_artifact_id: uuid.UUID | None = None
+    # carousel
+    slides: list[dict] | None = None
+    parent_id: uuid.UUID | None = None
+
 
+class PatchIn(BaseModel):
+    title: str | None = Field(None, max_length=300)
+    content: dict | None = None
 
-def _get(ctx: Ctx, artifact_id: uuid.UUID) -> Artifact:
+
+def get_artifact_or_404(ctx: Ctx, artifact_id: uuid.UUID) -> Artifact:
     a = ctx.db.scalar(select(Artifact).where(Artifact.id == artifact_id, Artifact.workspace_id == ctx.workspace.id))
     if not a:
         raise HTTPException(404, "Artifact not found")
     return a
 
 
-@router.post("")
-def create_artifact(body: ArtifactIn, ctx: Ctx = Depends(get_ctx)):
-    a = Artifact(
-        workspace_id=ctx.workspace.id,
-        type=body.type,
-        notebook_id=body.notebook_id,
-        document_id=body.document_id,
-        content_json=body.content,
-        status="draft",
-    )
-    ctx.db.add(a)
-    ctx.db.commit()
-    return _serialize(a)
+def _target_title(ctx: Ctx, notebook_id: uuid.UUID | None, document_id: uuid.UUID | None) -> str:
+    if document_id:
+        doc = ctx.db.scalar(select(Document).where(Document.id == document_id,
+                                                   Document.workspace_id == ctx.workspace.id))
+        if not doc:
+            raise HTTPException(404, "Post not found")
+        return doc.title
+    if notebook_id:
+        nb = ctx.db.scalar(select(Notebook).where(Notebook.id == notebook_id,
+                                                  Notebook.workspace_id == ctx.workspace.id))
+        if not nb:
+            raise HTTPException(404, "Notebook not found")
+        return nb.title
+    raise HTTPException(400, "Pick a notebook or a post")
+
+
+@router.post("/generate")
+def generate(body: GenerateIn, ctx: Ctx = Depends(get_ctx)):
+    """One entry point for every generated artifact."""
+    params: dict = {}
+    content: dict = {}
+    if body.type == "video" and body.style == "audiogram":
+        if not body.audio_artifact_id:
+            raise HTTPException(400, "Pick an audio overview for the audiogram")
+        source = get_artifact_or_404(ctx, body.audio_artifact_id)
+        seconds = float((source.content_json or {}).get("duration_s", 60))
+        check_limit(ctx.db, ctx.workspace, "video_minutes", seconds / 60)
+        title = f"Audiogram: {(source.content_json or {}).get('title', 'Audio overview')}"
+        params = {"style": "audiogram", "audio_artifact_id": str(source.id)}
+        artifact, job = start_artifact(ctx.db, ctx.workspace.id, "video", title=title, notebook_id=source.notebook_id,
+                                       document_id=source.document_id, params=params, job_kind="video")
+        return serialize_artifact(artifact, job)
+    if body.type == "carousel":
+        if not body.slides:
+            raise HTTPException(400, "A carousel needs slides")
+        content = {"slides": body.slides[:12], "parent_id": str(body.parent_id) if body.parent_id else None}
+    target = _target_title(ctx, body.notebook_id, body.document_id)
+    if body.type == "audio_overview":
+        check_limit(ctx.db, ctx.workspace, "audio_minutes", body.minutes)
+        params = {"format": body.format, "minutes": body.minutes}
+        title = f"Audio overview: {target}"
+    elif body.type == "video":
+        check_limit(ctx.db, ctx.workspace, "video_minutes", 1 if body.style == "short" else 3)
+        params = {"style": body.style}
+        title = f"{'Short' if body.style == 'short' else 'Explainer'} video: {target}"
+    elif body.type == "launch_kit":
+        if not body.document_id:
+            raise HTTPException(400, "A Launch Kit is made from one post")
+        check_limit(ctx.db, ctx.workspace, "launch_kits", 1)
+        title = f"Launch Kit: {target}"
+    else:
+        title = {"summary": "Summary", "quote_card": "Quote", "carousel": "Carousel"}[body.type] + f": {target}"
+    artifact, job = start_artifact(ctx.db, ctx.workspace.id, body.type, title=title, notebook_id=body.notebook_id,
+                                   document_id=body.document_id, content=content, params=params,
+                                   max_attempts=1 if body.type in {"quote_card", "carousel"} else 2)
+    return serialize_artifact(artifact, job)
+
+
+@router.get("")
+def list_artifacts(
+    ctx: Ctx = Depends(get_ctx),
+    type: str | None = None,
+    notebook_id: uuid.UUID | None = None,
+    document_id: uuid.UUID | None = None,
+    status: str | None = None,
+    q: str = "",
+    limit: int = 60,
+    offset: int = 0,
+):
+    query = select(Artifact).where(Artifact.workspace_id == ctx.workspace.id)
+    if type:
+        query = query.where(Artifact.type.in_(type.split(",")))
+    if notebook_id:
+        query = query.where(Artifact.notebook_id == notebook_id)
+    if document_id:
+        query = query.where(Artifact.document_id == document_id)
+    if status:
+        query = query.where(Artifact.status.in_(status.split(",")))
+    if q.strip():
+        # Titles live in JSON; match on the linked post or notebook title instead of JSON operators.
+        like = f"%{q.strip()}%"
+        docs = select(Document.id).where(Document.workspace_id == ctx.workspace.id, Document.title.ilike(like))
+        nbs = select(Notebook.id).where(Notebook.workspace_id == ctx.workspace.id, Notebook.title.ilike(like))
+        query = query.where(or_(Artifact.document_id.in_(docs), Artifact.notebook_id.in_(nbs)))
+    total = ctx.db.scalar(select(func.count()).select_from(query.subquery()))
+    rows = ctx.db.scalars(query.order_by(Artifact.created_at.desc()).offset(max(offset, 0)).limit(min(limit, 200)))
+    rows = list(rows.all())
+    jobs = latest_jobs(ctx.db, [a.id for a in rows])
+    return {"total": total, "items": [serialize_artifact(a, jobs.get(a.id)) for a in rows]}
 
 
 @router.get("/{artifact_id}")
 def get_artifact(artifact_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
-    return _serialize(_get(ctx, artifact_id))
+    a = get_artifact_or_404(ctx, artifact_id)
+    return serialize_artifact(a, latest_jobs(ctx.db, [a.id]).get(a.id))
+
+
+@router.patch("/{artifact_id}")
+def patch_artifact(artifact_id: uuid.UUID, body: PatchIn, ctx: Ctx = Depends(get_ctx)):
+    a = get_artifact_or_404(ctx, artifact_id)
+    content = dict(a.content_json or {})
+    if body.content is not None:
+        protected = {"slide_keys", "quote_card_ids", "segments", "voices"}
+        content.update({k: v for k, v in body.content.items() if k not in protected})
+    if body.title is not None:
+        content["title"] = body.title
+    a.content_json = content
+    ctx.db.commit()
+    return serialize_artifact(a, latest_jobs(ctx.db, [a.id]).get(a.id))
+
+
+@router.delete("/{artifact_id}")
+def delete_artifact(artifact_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    a = get_artifact_or_404(ctx, artifact_id)
+    storage.delete_prefix(f"ws/{ctx.workspace.id}/artifacts/{a.id}/")
+    ctx.db.query(CalendarItem).filter(CalendarItem.artifact_id == a.id, CalendarItem.status == "scheduled").delete()
+    ctx.db.delete(a)
+    ctx.db.commit()
+    return {"ok": True}
+
+
+@router.post("/{artifact_id}/retry")
+def retry_artifact(artifact_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    a = get_artifact_or_404(ctx, artifact_id)
+    last = latest_jobs(ctx.db, [a.id]).get(a.id)
+    if not last:
+        raise HTTPException(400, "Nothing to retry")
+    if last.status not in {"failed", "done"}:
+        raise HTTPException(409, "Still running")
+    a.status = "pending"
+    content = dict(a.content_json or {})
+    content.pop("error", None)
+    a.content_json = content
+    job = create_job(ctx.db, ctx.workspace.id, last.kind, last.params, artifact_id=a.id, max_attempts=last.max_attempts)
+    return serialize_artifact(a, job)
 
 
 @router.post("/{artifact_id}/render")
 def render_artifact(artifact_id: uuid.UUID, body: RenderIn, ctx: Ctx = Depends(get_ctx)):
-    a = _get(ctx, artifact_id)
+    a = get_artifact_or_404(ctx, artifact_id)
+    if body.composition not in COMPOSITIONS:
+        raise HTTPException(400, "Unknown composition")
     a.status = "rendering"
-    job = create_job(
-        ctx.db,
-        ctx.workspace.id,
-        "render",
-        {"artifact_id": str(a.id), "composition": body.composition, "props": body.props,
-         "format": COMPOSITIONS[body.composition]},
-        artifact_id=a.id,
-    )
+    job = create_job(ctx.db, ctx.workspace.id, "render",
+                     {"artifact_id": str(a.id), "composition": body.composition, "props": body.props},
+                     artifact_id=a.id, max_attempts=1)
     return serialize_job(job)
+
+
+@router.get("/{artifact_id}/jobs")
+def artifact_jobs(artifact_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    a = get_artifact_or_404(ctx, artifact_id)
+    jobs = ctx.db.scalars(select(Job).where(Job.artifact_id == a.id).order_by(Job.created_at.desc())).all()
+    return [serialize_job(j) for j in jobs]

backend/app/routers/jobs.py

@@ -22,6 +22,17 @@ def _get(ctx: Ctx, job_id: uuid.UUID) -> Job:
     return job
 
 
+@router.get("/jobs")
+def list_jobs(ctx: Ctx = Depends(get_ctx), active: bool = False, kind: str | None = None, limit: int = 20):
+    q = select(Job).where(Job.workspace_id == ctx.workspace.id)
+    if active:
+        q = q.where(Job.status.in_(["queued", "running"]))
+    if kind:
+        q = q.where(Job.kind.in_(kind.split(",")))
+    jobs = ctx.db.scalars(q.order_by(Job.created_at.desc()).limit(min(limit, 100))).all()
+    return [serialize_job(j) for j in jobs]
+
+
 @router.get("/jobs/{job_id}")
 def get_job(job_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
     return serialize_job(_get(ctx, job_id))
@@ -45,6 +56,7 @@ class ProgressIn(BaseModel):
     error: str | None = None
     render_seconds: float | None = None
     storage_key: str | None = None
+    storage_keys: list[str] | None = None  # carousels: one PNG per slide
 
 
 @router.post("/internal/jobs/{job_id}/progress", include_in_schema=False)
@@ -59,9 +71,16 @@ def renderer_progress(
     job = db.get(Job, job_id)
     if not job:
         raise HTTPException(404, "Job not found")
-    if body.status == "done" and job.artifact_id:
-        artifact = db.get(Artifact, job.artifact_id)
-        if artifact and body.storage_key:
+    artifact = db.get(Artifact, job.artifact_id) if job.artifact_id else None
+    if body.status == "failed" and artifact:
+        artifact.status = "failed"
+        artifact.content_json = {**(artifact.content_json or {}), "error": (body.error or "Render failed")[:500]}
+    if body.status == "done" and artifact:
+        if body.storage_keys:
+            artifact.content_json = {**(artifact.content_json or {}), "slide_keys": body.storage_keys}
+            artifact.storage_key = body.storage_keys[0]
+            artifact.status = "ready"
+        elif body.storage_key:
             artifact.storage_key = body.storage_key
             artifact.status = "ready"
         if body.render_seconds:

backend/app/routers/launchpad.py

@@ -0,0 +1,360 @@
+"""Launchpad: the calendar, connected social accounts (OAuth / app password) and tracked short links."""
+
+import json
+import secrets
+import uuid
+from datetime import UTC, datetime, timedelta
+from typing import Literal
+from urllib.parse import urlencode
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi.concurrency import run_in_threadpool
+from fastapi.responses import RedirectResponse
+from pydantic import BaseModel, Field
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.auth import Ctx, decode_token_full, encode_signed, get_ctx
+from app.config import settings
+from app.db import get_db
+from app.models import Artifact, CalendarItem, Document, EngagementEvent, SocialAccount, TrackedLink
+from app.services import launchpad as lp
+from app.services.crypto import decrypt, encrypt
+from app.services.social import AUTO_POST, LIMITS, PLATFORM_LABELS, SocialError, bluesky, linkedin, x
+
+router = APIRouter(prefix="/api", tags=["launchpad"])
+
+Platform = Literal["x", "linkedin", "bluesky", "substack_notes"]
+
+
+# Calendar
+
+
+class ItemIn(BaseModel):
+    platform: Platform
+    scheduled_at: datetime
+    content: str = Field(min_length=1, max_length=10000)
+    thread: list[str] = Field(default_factory=list, max_length=25)
+    artifact_id: uuid.UUID | None = None
+    document_id: uuid.UUID | None = None
+    social_account_id: uuid.UUID | None = None
+    remind_by_email: bool = False
+    draft: bool = False
+
+
+class ItemPatch(BaseModel):
+    scheduled_at: datetime | None = None
+    content: str | None = Field(None, min_length=1, max_length=10000)
+    thread: list[str] | None = None
+    social_account_id: uuid.UUID | None = None
+    remind_by_email: bool | None = None
+    status: Literal["scheduled", "draft"] | None = None
+
+
+def _aware(dt: datetime) -> datetime:
+    return dt if dt.tzinfo else dt.replace(tzinfo=UTC)
+
+
+def serialize_item(item: CalendarItem, account: SocialAccount | None = None, clicks: int = 0) -> dict:
+    return {
+        "id": str(item.id),
+        "platform": item.platform,
+        "platform_label": PLATFORM_LABELS.get(item.platform, item.platform),
+        "scheduled_at": _aware(item.scheduled_at).isoformat(),
+        "status": item.status,
+        "content": item.content,
+        "thread": item.thread or [],
+        "artifact_id": str(item.artifact_id) if item.artifact_id else None,
+        "document_id": str(item.document_id) if item.document_id else None,
+        "social_account_id": str(item.social_account_id) if item.social_account_id else None,
+        "account_handle": account.handle if account else None,
+        "remind_by_email": item.remind_by_email,
+        "auto_post": item.platform in AUTO_POST and bool(item.social_account_id) and not item.remind_by_email,
+        "external_url": item.external_url,
+        "posted_at": _aware(item.posted_at).isoformat() if item.posted_at else None,
+        "error": item.error,
+        "metrics": item.metrics or {},
+        "clicks": clicks,
+    }
+
+
+def _item(ctx: Ctx, item_id: uuid.UUID) -> CalendarItem:
+    item = ctx.db.scalar(select(CalendarItem).where(CalendarItem.id == item_id,
+                                                    CalendarItem.workspace_id == ctx.workspace.id))
+    if not item:
+        raise HTTPException(404, "Calendar item not found")
+    return item
+
+
+def _account(ctx: Ctx, account_id: uuid.UUID | None, platform: str) -> SocialAccount | None:
+    if not account_id:
+        return None
+    account = ctx.db.scalar(select(SocialAccount).where(SocialAccount.id == account_id,
+                                                        SocialAccount.workspace_id == ctx.workspace.id))
+    if not account or account.platform != platform:
+        raise HTTPException(400, "That account does not match the platform")
+    return account
+
+
+def _validate_lengths(platform: str, posts: list[str]) -> None:
+    limit = LIMITS.get(platform)
+    if platform in ("x", "bluesky") and limit:
+        for i, p in enumerate(posts, start=1):
+            if len(p) > limit:
+                raise HTTPException(422, {"code": "too_long",
+                                          "message": f"Post {i} is {len(p)} characters; {PLATFORM_LABELS[platform]} "
+                                                     f"allows {limit}."})
+
+
+def _enrich(ctx: Ctx, items: list[CalendarItem]) -> list[dict]:
+    accounts = {a.id: a for a in ctx.db.scalars(
+        select(SocialAccount).where(SocialAccount.workspace_id == ctx.workspace.id))}
+    clicks = lp.clicks_by_item(ctx.db, [i.id for i in items])
+    return [serialize_item(i, accounts.get(i.social_account_id), clicks.get(i.id, 0)) for i in items]
+
+
+@router.get("/calendar")
+def list_items(ctx: Ctx = Depends(get_ctx), start: datetime | None = None, end: datetime | None = None,
+               status: str | None = None, limit: int = 500):
+    q = select(CalendarItem).where(CalendarItem.workspace_id == ctx.workspace.id)
+    if start:
+        q = q.where(CalendarItem.scheduled_at >= start)
+    if end:
+        q = q.where(CalendarItem.scheduled_at < end)
+    if status:
+        q = q.where(CalendarItem.status.in_(status.split(",")))
+    items = list(ctx.db.scalars(q.order_by(CalendarItem.scheduled_at).limit(min(limit, 1000))).all())
+    return _enrich(ctx, items)
+
+
+@router.post("/calendar")
+def create_item(body: ItemIn, ctx: Ctx = Depends(get_ctx)):
+    account = _account(ctx, body.social_account_id, body.platform)
+    _validate_lengths(body.platform, [body.content, *body.thread])
+    for model, value in ((Artifact, body.artifact_id), (Document, body.document_id)):
+        if value and not ctx.db.scalar(select(model.id).where(model.id == value,
+                                                              model.workspace_id == ctx.workspace.id)):
+            raise HTTPException(404, f"{model.__name__} not found")
+    item = CalendarItem(
+        workspace_id=ctx.workspace.id, platform=body.platform, scheduled_at=_aware(body.scheduled_at),
+        content=body.content, thread=[t for t in body.thread if t.strip()], artifact_id=body.artifact_id,
+        document_id=body.document_id, social_account_id=account.id if account else None,
+        remind_by_email=body.remind_by_email or body.platform not in AUTO_POST or not account,
+        status="draft" if body.draft else "scheduled",
+    )
+    ctx.db.add(item)
+    ctx.db.commit()
+    return _enrich(ctx, [item])[0]
+
+
+@router.patch("/calendar/{item_id}")
+def update_item(item_id: uuid.UUID, body: ItemPatch, ctx: Ctx = Depends(get_ctx)):
+    item = _item(ctx, item_id)
+    if item.status in ("posted", "publishing"):
+        raise HTTPException(409, "This post already went out.")
+    if body.scheduled_at is not None:
+        item.scheduled_at = _aware(body.scheduled_at)
+    if body.content is not None:
+        item.content = body.content
+    if body.thread is not None:
+        item.thread = [t for t in body.thread if t.strip()]
+    if "social_account_id" in body.model_fields_set:
+        item.social_account_id = (_account(ctx, body.social_account_id, item.platform).id
+                                  if body.social_account_id else None)
+    if body.remind_by_email is not None:
+        item.remind_by_email = body.remind_by_email
+    if not item.social_account_id or item.platform not in AUTO_POST:
+        item.remind_by_email = True
+    if body.status is not None:
+        item.status = body.status
+    elif item.status in ("failed", "reminded"):
+        item.status = "scheduled"
+    item.error = None
+    item.attempts = 0
+    _validate_lengths(item.platform, [item.content, *(item.thread or [])])
+    ctx.db.commit()
+    return _enrich(ctx, [item])[0]
+
+
+@router.delete("/calendar/{item_id}")
+def delete_item(item_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    item = _item(ctx, item_id)
+    ctx.db.delete(item)
+    ctx.db.commit()
+    return {"ok": True}
+
+
+@router.post("/calendar/{item_id}/publish")
+async def publish_now(item_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    item = _item(ctx, item_id)
+    if item.status in ("posted", "publishing"):
+        raise HTTPException(409, "This post already went out.")
+    item.status = "publishing"
+    item.attempts = 0
+    ctx.db.commit()
+    await run_in_threadpool(lp.publish_item, ctx.db, item)
+    return _enrich(ctx, [item])[0]
+
+
+# Social accounts
+
+
+def serialize_account(a: SocialAccount) -> dict:
+    return {"id": str(a.id), "platform": a.platform, "platform_label": PLATFORM_LABELS[a.platform],
+            "handle": a.handle, "status": a.status, "avatar_url": a.avatar_url,
+            "connected_at": a.created_at.isoformat() if a.created_at else None}
+
+
+@router.get("/social/accounts")
+def list_accounts(ctx: Ctx = Depends(get_ctx)):
+    accounts = ctx.db.scalars(select(SocialAccount).where(SocialAccount.workspace_id == ctx.workspace.id)
+                              .order_by(SocialAccount.created_at)).all()
+    return {
+        "accounts": [serialize_account(a) for a in accounts],
+        "available": {"x": x.configured(), "linkedin": linkedin.configured(), "bluesky": True},
+        "redirect_uris": {"x": x.redirect_uri(), "linkedin": linkedin.redirect_uri()},
+    }
+
+
+@router.delete("/social/accounts/{account_id}")
+def disconnect(account_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    account = ctx.db.scalar(select(SocialAccount).where(SocialAccount.id == account_id,
+                                                        SocialAccount.workspace_id == ctx.workspace.id))
+    if not account:
+        raise HTTPException(404, "Account not found")
+    for item in ctx.db.scalars(select(CalendarItem).where(CalendarItem.social_account_id == account.id,
+                                                          CalendarItem.status == "scheduled")):
+        item.social_account_id = None
+        item.remind_by_email = True
+    ctx.db.delete(account)
+    ctx.db.commit()
+    return {"ok": True}
+
+
+def _upsert_account(db: Session, workspace_id: uuid.UUID, platform: str, info: dict, secret: str,
+                    refresh: str | None = None, expires_in: int | None = None, scopes: str | None = None) -> None:
+    account = db.scalar(select(SocialAccount).where(SocialAccount.workspace_id == workspace_id,
+                                                    SocialAccount.platform == platform,
+                                                    SocialAccount.external_id == info["external_id"]))
+    if not account:
+        account = SocialAccount(workspace_id=workspace_id, platform=platform, external_id=info["external_id"],
+                                handle=info["handle"], access_token="")
+        db.add(account)
+    account.handle = info["handle"]
+    account.avatar_url = info.get("avatar_url")
+    account.access_token = encrypt(secret)
+    account.refresh_token = encrypt(refresh) if refresh else account.refresh_token
+    account.expires_at = datetime.now(UTC) + timedelta(seconds=int(expires_in)) if expires_in else None
+    account.scopes = scopes
+    account.status = "active"
+    db.commit()
+
+
+class BlueskyIn(BaseModel):
+    handle: str = Field(min_length=3, max_length=200)
+    app_password: str = Field(min_length=8, max_length=100)
+
+
+@router.post("/social/bluesky")
+async def connect_bluesky(body: BlueskyIn, ctx: Ctx = Depends(get_ctx)):
+    try:
+        session = await run_in_threadpool(bluesky.create_session, body.handle, body.app_password)
+    except SocialError as exc:
+        raise HTTPException(400, {"code": "bluesky_auth", "message": str(exc)}) from exc
+    _upsert_account(ctx.db, ctx.workspace.id, "bluesky",
+                    {"external_id": session["did"], "handle": session.get("handle", body.handle)}, body.app_password)
+    return list_accounts(ctx)
+
+
+# OAuth (X, LinkedIn). The signed state carries the workspace and PKCE verifier; the callback hands a
+# short lived encrypted ticket back to the SPA, which completes the link as the signed in user.
+
+STATE_TTL = timedelta(minutes=10)
+
+
+def _launchpad_redirect(**params: str) -> RedirectResponse:
+    return RedirectResponse(f"{settings.frontend_url.rstrip('/')}/app/launchpad?{urlencode(params)}", 302)
+
+
+@router.post("/social/{platform}/start")
+def oauth_start(platform: Literal["x", "linkedin"], ctx: Ctx = Depends(get_ctx)):
+    """Returns the provider URL. The SPA calls this with its bearer token, then navigates to `url`."""
+    mod = x if platform == "x" else linkedin
+    if not mod.configured():
+        raise HTTPException(503, {"code": "not_configured",
+                                  "message": f"Set {platform.upper()}_CLIENT_ID and {platform.upper()}_CLIENT_SECRET."})
+    nonce = secrets.token_urlsafe(24)
+    payload = {"typ": "social_state", "nonce": nonce, "ws": str(ctx.workspace.id), "platform": platform}
+    if platform == "x":
+        verifier, challenge = x.pkce_pair()
+        payload["verifier"] = verifier
+        state = encode_signed(payload, STATE_TTL)
+        url = x.authorize_url(state, challenge)
+    else:
+        state = encode_signed(payload, STATE_TTL)
+        url = linkedin.authorize_url(state)
+    return {"url": url, "nonce": nonce}
+
+
+@router.get("/social/{platform}/callback")
+def oauth_callback(platform: Literal["x", "linkedin"], code: str | None = None, state: str | None = None,
+                   error: str | None = None, db: Session = Depends(get_db)):
+    label = PLATFORM_LABELS[platform]
+    if error or not code or not state:
+        return _launchpad_redirect(error=f"{label} connection was cancelled.")
+    try:
+        saved = decode_token_full(state, expected_type="social_state")
+    except HTTPException:
+        return _launchpad_redirect(error=f"{label} connection expired. Try again.")
+    if saved.get("platform") != platform:
+        return _launchpad_redirect(error=f"{label} connection failed.")
+    try:
+        if platform == "x":
+            info = x.exchange_code(code, saved["verifier"])
+        else:
+            info = linkedin.exchange_code(code)
+    except Exception:
+        return _launchpad_redirect(error=f"{label} connection failed. Try again.")
+    # Do not link here: this request carries no session. The signed in SPA finishes the link, and only
+    # for the workspace that started it, so a forwarded authorize link cannot attach someone's account
+    # to another workspace.
+    ticket = encode_signed({"typ": "social_link", "ws": saved["ws"], "platform": platform,
+                            "data": encrypt(json.dumps(info))}, timedelta(minutes=10))
+    return _launchpad_redirect(link=ticket, platform=platform)
+
+
+class LinkIn(BaseModel):
+    ticket: str
+
+
+@router.post("/social/complete")
+def oauth_complete(body: LinkIn, ctx: Ctx = Depends(get_ctx)):
+    try:
+        saved = decode_token_full(body.ticket, expected_type="social_link")
+    except HTTPException as exc:
+        raise HTTPException(400, {"code": "expired", "message": "That connection expired. Try again."}) from exc
+    if saved.get("ws") != str(ctx.workspace.id):
+        raise HTTPException(403, {"code": "wrong_workspace", "message": "This connection was started elsewhere."})
+    info = json.loads(decrypt(saved["data"]) or "{}")
+    if not info.get("access_token"):
+        raise HTTPException(400, {"code": "expired", "message": "That connection expired. Try again."})
+    _upsert_account(ctx.db, ctx.workspace.id, saved["platform"], info, info["access_token"],
+                    info.get("refresh_token"), info.get("expires_in"), info.get("scope"))
+    return list_accounts(ctx)
+
+
+# Tracked links (public)
+
+links_router = APIRouter(tags=["links"])
+
+
+@links_router.get("/l/{slug}")
+def follow_link(slug: str, request: Request, db: Session = Depends(get_db)):
+    link = db.scalar(select(TrackedLink).where(TrackedLink.slug == slug))
+    if not link:
+        raise HTTPException(404, "Link not found")
+    db.add(EngagementEvent(workspace_id=link.workspace_id, tracked_link_id=link.id, kind="click",
+                           metadata_json={"referer": request.headers.get("referer", "")[:300],
+                                          "ua": request.headers.get("user-agent", "")[:200]}))
+    db.commit()
+    return RedirectResponse(link.target_url, status_code=302)

backend/app/routers/notebooks.py

@@ -1,18 +1,21 @@
 import asyncio
 import json
 import uuid
+from datetime import UTC, datetime
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import StreamingResponse
 from pydantic import BaseModel, Field
-from sqlalchemy import select
+from sqlalchemy import delete, func, select
 
 from app.auth import Ctx, get_ctx
 from app.config import settings
 from app.corpus import Corpus
 from app.llm.provider import provider_name
-from app.models import Chat, Citation, Document, Message, Notebook, NotebookDocument
+from app.models import Artifact, Chat, Citation, Document, DocumentTopic, Message, Notebook, NotebookDocument, Topic
 from app.pipeline.research import research
+from app.routers.sources import serialize_document
+from app.services.artifacts import latest_jobs, serialize_artifact
 from app.services.jobs import record_usage
 
 router = APIRouter(prefix="/api/notebooks", tags=["notebooks"])
@@ -21,6 +24,12 @@ router = APIRouter(prefix="/api/notebooks", tags=["notebooks"])
 class NotebookIn(BaseModel):
     title: str = Field(min_length=1, max_length=300)
     description: str | None = None
+    document_ids: list[uuid.UUID] = []
+
+
+class NotebookPatch(BaseModel):
+    title: str | None = Field(None, min_length=1, max_length=300)
+    description: str | None = None
 
 
 class DocumentsIn(BaseModel):
@@ -41,12 +50,38 @@ def _get(ctx: Ctx, notebook_id: uuid.UUID) -> Notebook:
     return nb
 
 
+def _add_docs(ctx: Ctx, nb: Notebook, ids: list[uuid.UUID]) -> int:
+    valid = set(ctx.db.scalars(
+        select(Document.id).where(Document.id.in_(ids), Document.workspace_id == ctx.workspace.id)
+    ).all())
+    existing = set(ctx.db.scalars(select(NotebookDocument.document_id).where(NotebookDocument.notebook_id == nb.id)))
+    for doc_id in valid - existing:
+        ctx.db.add(NotebookDocument(notebook_id=nb.id, document_id=doc_id))
+    return len(valid - existing)
+
+
 @router.post("")
 def create_notebook(body: NotebookIn, ctx: Ctx = Depends(get_ctx)):
-    nb = Notebook(workspace_id=ctx.workspace.id, title=body.title, description=body.description)
+    nb = Notebook(workspace_id=ctx.workspace.id, title=body.title.strip(), description=body.description)
+    ctx.db.add(nb)
+    ctx.db.flush()
+    added = _add_docs(ctx, nb, body.document_ids)
+    ctx.db.commit()
+    return {"id": str(nb.id), "title": nb.title, "added": added}
+
+
+@router.post("/from-topic/{topic_id}")
+def notebook_from_topic(topic_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    topic = ctx.db.scalar(select(Topic).where(Topic.id == topic_id, Topic.workspace_id == ctx.workspace.id))
+    if not topic:
+        raise HTTPException(404, "Topic not found")
+    ids = list(ctx.db.scalars(select(DocumentTopic.document_id).where(DocumentTopic.topic_id == topic.id)))
+    nb = Notebook(workspace_id=ctx.workspace.id, title=topic.name, description=topic.summary)
     ctx.db.add(nb)
+    ctx.db.flush()
+    added = _add_docs(ctx, nb, ids)
     ctx.db.commit()
-    return {"id": str(nb.id), "title": nb.title}
+    return {"id": str(nb.id), "title": nb.title, "added": added}
 
 
 @router.get("")
@@ -54,7 +89,24 @@ def list_notebooks(ctx: Ctx = Depends(get_ctx)):
     rows = ctx.db.scalars(
         select(Notebook).where(Notebook.workspace_id == ctx.workspace.id).order_by(Notebook.updated_at.desc())
     ).all()
-    return [{"id": str(n.id), "title": n.title, "description": n.description} for n in rows]
+    ids = [n.id for n in rows]
+    doc_counts = dict(ctx.db.execute(
+        select(NotebookDocument.notebook_id, func.count()).where(NotebookDocument.notebook_id.in_(ids))
+        .group_by(NotebookDocument.notebook_id)
+    ).all()) if ids else {}
+    chat_counts = dict(ctx.db.execute(
+        select(Chat.notebook_id, func.count()).where(Chat.notebook_id.in_(ids)).group_by(Chat.notebook_id)
+    ).all()) if ids else {}
+    artifact_counts = dict(ctx.db.execute(
+        select(Artifact.notebook_id, func.count()).where(Artifact.notebook_id.in_(ids)).group_by(Artifact.notebook_id)
+    ).all()) if ids else {}
+    return [
+        {"id": str(n.id), "title": n.title, "description": n.description, "summary": n.summary,
+         "document_count": doc_counts.get(n.id, 0), "chat_count": chat_counts.get(n.id, 0),
+         "artifact_count": artifact_counts.get(n.id, 0),
+         "updated_at": n.updated_at.isoformat() if n.updated_at else None}
+        for n in rows
+    ]
 
 
 @router.get("/{notebook_id}")
@@ -71,29 +123,96 @@ def get_notebook(notebook_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
         "title": nb.title,
         "description": nb.description,
         "summary": nb.summary,
-        "documents": [
-            {"id": str(d.id), "title": d.title, "url": d.url,
-             "published_at": d.published_at.isoformat() if d.published_at else None}
-            for d in docs
-        ],
+        "documents": [serialize_document(d) for d in docs],
     }
 
 
+@router.patch("/{notebook_id}")
+def patch_notebook(notebook_id: uuid.UUID, body: NotebookPatch, ctx: Ctx = Depends(get_ctx)):
+    nb = _get(ctx, notebook_id)
+    if body.title is not None:
+        nb.title = body.title.strip()
+    if body.description is not None:
+        nb.description = body.description
+    ctx.db.commit()
+    return {"id": str(nb.id), "title": nb.title, "description": nb.description}
+
+
+@router.delete("/{notebook_id}")
+def delete_notebook(notebook_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    nb = _get(ctx, notebook_id)
+    ctx.db.execute(delete(NotebookDocument).where(NotebookDocument.notebook_id == nb.id))
+    ctx.db.delete(nb)
+    ctx.db.commit()
+    return {"ok": True}
+
+
 @router.post("/{notebook_id}/documents")
 def add_documents(notebook_id: uuid.UUID, body: DocumentsIn, ctx: Ctx = Depends(get_ctx)):
     nb = _get(ctx, notebook_id)
-    valid = set(
-        ctx.db.scalars(
-            select(Document.id).where(Document.id.in_(body.document_ids), Document.workspace_id == ctx.workspace.id)
-        ).all()
-    )
-    existing = set(
-        ctx.db.scalars(select(NotebookDocument.document_id).where(NotebookDocument.notebook_id == nb.id)).all()
-    )
-    for doc_id in valid - existing:
-        ctx.db.add(NotebookDocument(notebook_id=nb.id, document_id=doc_id))
+    added = _add_docs(ctx, nb, body.document_ids)
+    ctx.db.commit()
+    return {"added": added}
+
+
+@router.delete("/{notebook_id}/documents/{document_id}")
+def remove_document(notebook_id: uuid.UUID, document_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    nb = _get(ctx, notebook_id)
+    ctx.db.execute(delete(NotebookDocument).where(NotebookDocument.notebook_id == nb.id,
+                                                  NotebookDocument.document_id == document_id))
+    ctx.db.commit()
+    return {"ok": True}
+
+
+@router.get("/{notebook_id}/artifacts")
+def notebook_artifacts(notebook_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    nb = _get(ctx, notebook_id)
+    rows = list(ctx.db.scalars(
+        select(Artifact).where(Artifact.notebook_id == nb.id).order_by(Artifact.created_at.desc())
+    ).all())
+    jobs = latest_jobs(ctx.db, [a.id for a in rows])
+    return [serialize_artifact(a, jobs.get(a.id)) for a in rows]
+
+
+@router.get("/{notebook_id}/chats")
+def list_chats(notebook_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    nb = _get(ctx, notebook_id)
+    chats = ctx.db.scalars(select(Chat).where(Chat.notebook_id == nb.id).order_by(Chat.updated_at.desc())).all()
+    return [{"id": str(c.id), "title": c.title, "updated_at": c.updated_at.isoformat() if c.updated_at else None}
+            for c in chats]
+
+
+def _get_chat(ctx: Ctx, chat_id: uuid.UUID) -> Chat:
+    chat = ctx.db.scalar(select(Chat).where(Chat.id == chat_id, Chat.workspace_id == ctx.workspace.id))
+    if not chat:
+        raise HTTPException(404, "Chat not found")
+    return chat
+
+
+@router.get("/chats/{chat_id}/messages")
+def chat_messages(chat_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    chat = _get_chat(ctx, chat_id)
+    msgs = ctx.db.scalars(select(Message).where(Message.chat_id == chat.id).order_by(Message.created_at)).all()
+    doc_ids = {c.document_id for m in msgs for c in m.citations}
+    docs = {d.id: d for d in ctx.db.scalars(select(Document).where(Document.id.in_(doc_ids)))} if doc_ids else {}
+    out = []
+    for m in msgs:
+        cites = []
+        for c in sorted(m.citations, key=lambda c: c.marker):
+            d = docs.get(c.document_id)
+            cites.append({"marker": c.marker, "document_id": str(c.document_id), "title": d.title if d else "",
+                          "url": d.url if d else "", "path": c.path, "line_start": c.line_start,
+                          "line_end": c.line_end, "span": c.span or ""})
+        out.append({"id": str(m.id), "role": m.role, "text": m.content, "citations": cites})
+    return out
+
+
+@router.delete("/chats/{chat_id}")
+def delete_chat(chat_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    chat = _get_chat(ctx, chat_id)
+    ctx.db.delete(chat)
     ctx.db.commit()
-    return {"added": len(valid - existing)}
+    return {"ok": True}
 
 
 def _sse(event: str, data: dict) -> str:
@@ -112,6 +231,7 @@ async def chat(notebook_id: uuid.UUID, body: ChatIn, ctx: Ctx = Depends(get_ctx)
         chat_row = Chat(workspace_id=ctx.workspace.id, notebook_id=nb.id, title=body.question[:120])
         db.add(chat_row)
         db.flush()
+    chat_row.updated_at = datetime.now(UTC)  # keeps the history list newest first
     db.add(Message(chat_id=chat_row.id, role="user", content=body.question))
     db.commit()
 

backend/app/routers/resurface.py

@@ -0,0 +1,53 @@
+from datetime import UTC, datetime, timedelta
+
+from fastapi import APIRouter, Depends
+from sqlalchemy import select
+
+from app.auth import Ctx, get_ctx
+from app.models import Document, Source
+from app.routers.sources import serialize_document
+from app.services.jobs import create_job, serialize_job
+
+router = APIRouter(prefix="/api/resurface", tags=["resurface"])
+
+COOLDOWN = timedelta(days=90)
+
+
+def _aware(dt: datetime | None) -> datetime | None:
+    return dt if dt is None or dt.tzinfo else dt.replace(tzinfo=UTC)
+
+
+@router.get("")
+def suggestions(ctx: Ctx = Depends(get_ctx), limit: int = 20):
+    now = datetime.now(UTC)
+    rows = ctx.db.execute(
+        select(Document, Source).outerjoin(Source, Source.id == Document.source_id)
+        .where(Document.workspace_id == ctx.workspace.id, Document.path.is_not(None))
+    ).all()
+    evergreen, on_this_day = [], []
+    unscored = 0
+    for d, s in rows:
+        published = _aware(d.published_at)
+        resurfaced = _aware(d.last_resurfaced_at)
+        meta = d.metadata_json or {}
+        item = {**serialize_document(d, s), "reason": meta.get("evergreen_reason"), "angle": meta.get("reshare_angle"),
+                "last_resurfaced_at": resurfaced.isoformat() if resurfaced else None}
+        if published and published.month == now.month and published.day == now.day and published.year < now.year:
+            on_this_day.append({**item, "years_ago": now.year - published.year})
+        if d.evergreen_score is None:
+            unscored += 1
+            continue
+        if resurfaced and now - resurfaced < COOLDOWN:
+            continue
+        age_days = (now - published).days if published else 365
+        # Old, timeless and not recently reshared ranks highest; brand new posts do not need resurfacing.
+        age_factor = min(age_days / 365, 2.0) if age_days > 30 else 0.1
+        evergreen.append({**item, "rank": round(d.evergreen_score * (0.5 + age_factor / 2), 3)})
+    evergreen.sort(key=lambda x: -x["rank"])
+    return {"evergreen": evergreen[: min(limit, 100)], "on_this_day": on_this_day, "unscored": unscored,
+            "total_posts": len(rows)}
+
+
+@router.post("/scan")
+def scan(ctx: Ctx = Depends(get_ctx)):
+    return serialize_job(create_job(ctx.db, ctx.workspace.id, "resurface_scan", {}, max_attempts=2))

backend/app/routers/settings.py

@@ -0,0 +1,84 @@
+import re
+
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel, Field
+from sqlalchemy import select
+
+from app.auth import Ctx, get_ctx
+from app.config import settings as app_settings
+from app.models import Upload
+from app.services.plans import effective_plan, plan_dict
+from app.services.storage import storage
+from app.services.usage import usage_report
+
+router = APIRouter(prefix="/api/settings", tags=["settings"])
+
+
+class SettingsIn(BaseModel):
+    name: str | None = Field(None, max_length=200)
+    workspace_name: str | None = Field(None, min_length=1, max_length=200)
+    brand_name: str | None = Field(None, max_length=120)
+    brand_accent: str | None = None
+    logo_upload_id: str | None = None  # "" removes the logo
+    training_opt_in: bool | None = None
+    email_unsubscribed: bool | None = None
+
+
+def _serialize(ctx: Ctx) -> dict:
+    brand = ctx.workspace.brand_json or {}
+    return {
+        "user": {"name": ctx.user.name, "email": ctx.user.email, "auth_provider": ctx.user.auth_provider.value,
+                 "email_unsubscribed": ctx.user.email_unsubscribed},
+        "workspace": {"id": str(ctx.workspace.id), "name": ctx.workspace.name,
+                      "training_opt_in": ctx.workspace.training_opt_in},
+        "brand": {"name": brand.get("name"), "accent": brand.get("accent") or "#217cff",
+                  "logo_url": storage.presign_get(brand["logo_key"]) if brand.get("logo_key") else None},
+        "plan": plan_dict(effective_plan(ctx.db, ctx.workspace)),
+        "billing_enabled": app_settings.billing_enabled,
+        "usage": usage_report(ctx.db, ctx.workspace),
+        "integrations": {"llm": bool(app_settings.llm_api_key), "elevenlabs": bool(app_settings.elevenlabs_api_key),
+                         "x": bool(app_settings.x_client_id), "linkedin": bool(app_settings.linkedin_client_id),
+                         "storage": "local" if app_settings.use_local_storage else "r2",
+                         "renderer": app_settings.renderer_url},
+    }
+
+
+@router.get("")
+def get_settings(ctx: Ctx = Depends(get_ctx)):
+    return _serialize(ctx)
+
+
+@router.patch("")
+def update_settings(body: SettingsIn, ctx: Ctx = Depends(get_ctx)):
+    if body.name is not None:
+        ctx.user.name = body.name.strip() or None
+    if body.workspace_name is not None:
+        ctx.workspace.name = body.workspace_name.strip()
+    if body.training_opt_in is not None:
+        ctx.workspace.training_opt_in = body.training_opt_in
+    if body.email_unsubscribed is not None:
+        ctx.user.email_unsubscribed = body.email_unsubscribed
+    brand = dict(ctx.workspace.brand_json or {})
+    if body.brand_name is not None:
+        brand["name"] = body.brand_name.strip() or None
+    if body.brand_accent is not None:
+        if not re.fullmatch(r"#[0-9a-fA-F]{6}", body.brand_accent):
+            raise HTTPException(422, "Use a hex color like #217cff")
+        brand["accent"] = body.brand_accent.lower()
+    if body.logo_upload_id is not None:
+        if body.logo_upload_id == "":
+            brand.pop("logo_key", None)
+        else:
+            upload = ctx.db.scalar(select(Upload).where(Upload.id == body.logo_upload_id,
+                                                        Upload.workspace_id == ctx.workspace.id))
+            if not upload or upload.status != "complete" or not upload.content_type.startswith("image/"):
+                raise HTTPException(400, "Upload an image first")
+            brand["logo_key"] = upload.key
+    ctx.workspace.brand_json = brand
+    ctx.db.commit()
+    return _serialize(ctx)
+
+
+@router.get("/usage")
+def usage(ctx: Ctx = Depends(get_ctx)):
+    return usage_report(ctx.db, ctx.workspace)

backend/app/routers/sources.py

@@ -1,57 +1,85 @@
 import uuid
 
 from fastapi import APIRouter, Depends, HTTPException
-from pydantic import BaseModel
-from sqlalchemy import func, select
+from fastapi.concurrency import run_in_threadpool
+from pydantic import BaseModel, Field
+from sqlalchemy import delete, func, or_, select
 
 from app.auth import Ctx, get_ctx
-from app.models import Document, Job, Source
-from app.pipeline.ingest import normalize_feed_url
+from app.corpus import INDEX, Corpus, CorpusError
+from app.models import Document, Job, NotebookDocument, Source, Upload
+from app.pipeline.ingest import IMPORTS_FEED, FeedNotFound, discover_feed, imports_source, rebuild_index
 from app.services.jobs import create_job, serialize_job
 from app.services.plans import effective_plan
+from app.services.storage import storage
 
 router = APIRouter(prefix="/api/sources", tags=["sources"])
 
 
 class SourceIn(BaseModel):
-    url: str
+    url: str = Field(min_length=3, max_length=1000)
 
 
-def _serialize(source: Source, doc_count: int = 0) -> dict:
+class UploadImportIn(BaseModel):
+    upload_id: uuid.UUID
+
+
+def serialize_source(source: Source, doc_count: int = 0) -> dict:
     return {
         "id": str(source.id),
         "feed_url": source.feed_url,
+        "site_url": source.site_url,
         "platform": source.platform,
         "title": source.title,
         "sync_status": source.sync_status,
         "sync_error": source.sync_error,
         "last_synced_at": source.last_synced_at.isoformat() if source.last_synced_at else None,
         "document_count": doc_count,
+        "is_imports": source.feed_url == IMPORTS_FEED,
     }
 
 
+def _get(ctx: Ctx, source_id: uuid.UUID) -> Source:
+    source = ctx.db.scalar(select(Source).where(Source.id == source_id, Source.workspace_id == ctx.workspace.id))
+    if not source:
+        raise HTTPException(404, "Source not found")
+    return source
+
+
 def _enqueue_sync(ctx: Ctx, source: Source) -> Job:
+    source.sync_status = "pending"
     return create_job(ctx.db, ctx.workspace.id, "ingest", {"source_id": str(source.id)})
 
 
+def _doc_count(ctx: Ctx, source: Source) -> int:
+    return ctx.db.scalar(select(func.count()).select_from(Document).where(Document.source_id == source.id)) or 0
+
+
 @router.post("")
-def create_source(body: SourceIn, ctx: Ctx = Depends(get_ctx)):
-    feed_url, platform = normalize_feed_url(body.url)
+async def create_source(body: SourceIn, ctx: Ctx = Depends(get_ctx)):
+    try:
+        found = await run_in_threadpool(discover_feed, body.url)
+    except FeedNotFound as exc:
+        raise HTTPException(422, {"code": "feed_not_found", "message": str(exc)}) from exc
     existing = ctx.db.scalar(
-        select(Source).where(Source.workspace_id == ctx.workspace.id, Source.feed_url == feed_url)
+        select(Source).where(Source.workspace_id == ctx.workspace.id, Source.feed_url == found.feed_url)
     )
     if existing:
         job = _enqueue_sync(ctx, existing)
-        return {"source": _serialize(existing), "job": serialize_job(job)}
+        return {"source": serialize_source(existing, _doc_count(ctx, existing)), "job": serialize_job(job)}
     plan = effective_plan(ctx.db, ctx.workspace)
-    count = ctx.db.scalar(select(func.count()).select_from(Source).where(Source.workspace_id == ctx.workspace.id))
+    count = ctx.db.scalar(
+        select(func.count()).select_from(Source)
+        .where(Source.workspace_id == ctx.workspace.id, Source.feed_url != IMPORTS_FEED)
+    )
     if count >= plan.sources:
         raise HTTPException(402, {"code": "plan_limit", "message": f"Your plan includes {plan.sources} sources."})
-    source = Source(workspace_id=ctx.workspace.id, feed_url=feed_url, platform=platform, site_url=body.url)
+    source = Source(workspace_id=ctx.workspace.id, feed_url=found.feed_url, platform=found.platform,
+                    site_url=found.site_url, title=found.title)
     ctx.db.add(source)
     ctx.db.commit()
     job = _enqueue_sync(ctx, source)
-    return {"source": _serialize(source), "job": serialize_job(job)}
+    return {"source": serialize_source(source), "job": serialize_job(job)}
 
 
 @router.get("")
@@ -63,33 +91,132 @@ def list_sources(ctx: Ctx = Depends(get_ctx)):
         .group_by(Source.id)
         .order_by(Source.created_at)
     ).all()
-    return [_serialize(s, n) for s, n in rows]
+    return [serialize_source(s, n) for s, n in rows]
 
 
-@router.get("/{source_id}/sync")
+@router.post("/{source_id}/sync")
 def sync_source(source_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
-    source = ctx.db.scalar(
-        select(Source).where(Source.id == source_id, Source.workspace_id == ctx.workspace.id)
-    )
-    if not source:
-        raise HTTPException(404, "Source not found")
+    source = _get(ctx, source_id)
+    if source.feed_url == IMPORTS_FEED:
+        raise HTTPException(400, "Imported posts have no feed to sync.")
     job = _enqueue_sync(ctx, source)
-    return {"source": _serialize(source), "job": serialize_job(job)}
+    return {"source": serialize_source(source, _doc_count(ctx, source)), "job": serialize_job(job)}
+
+
+def remove_documents(ctx: Ctx, docs: list[Document]) -> None:
+    """Delete documents plus their corpus files and raw HTML, then rebuild INDEX.md."""
+    if not docs:
+        return
+    paths = {d.path for d in docs if d.path}
+    for d in docs:
+        if d.raw_html_key:
+            storage.delete_prefix(d.raw_html_key.rsplit("/", 1)[0])
+    ids = [d.id for d in docs]
+    ctx.db.execute(delete(NotebookDocument).where(NotebookDocument.document_id.in_(ids)))
+    ctx.db.execute(delete(Document).where(Document.id.in_(ids)))
+    ctx.db.flush()
+    Corpus(ctx.workspace.id).write_files({INDEX: rebuild_index(ctx.db, ctx.workspace.id)}, remove=paths)
+
+
+@router.delete("/{source_id}")
+def delete_source(source_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    source = _get(ctx, source_id)
+    docs = ctx.db.scalars(select(Document).where(Document.source_id == source.id)).all()
+    remove_documents(ctx, list(docs))
+    ctx.db.delete(source)
+    ctx.db.commit()
+    return {"ok": True, "removed_posts": len(docs)}
+
+
+@router.post("/url")
+def import_url(body: SourceIn, ctx: Ctx = Depends(get_ctx)):
+    source = imports_source(ctx.db, ctx.workspace.id)
+    ctx.db.commit()
+    job = create_job(ctx.db, ctx.workspace.id, "import_url", {"source_id": str(source.id), "url": body.url})
+    return {"source": serialize_source(source, _doc_count(ctx, source)), "job": serialize_job(job)}
+
+
+@router.post("/upload")
+def import_upload(body: UploadImportIn, ctx: Ctx = Depends(get_ctx)):
+    upload = ctx.db.scalar(select(Upload).where(Upload.id == body.upload_id, Upload.workspace_id == ctx.workspace.id))
+    if not upload or upload.status != "complete":
+        raise HTTPException(404, "Upload not found or not finished")
+    source = imports_source(ctx.db, ctx.workspace.id)
+    ctx.db.commit()
+    job = create_job(ctx.db, ctx.workspace.id, "import_upload",
+                     {"source_id": str(source.id), "upload_id": str(upload.id)})
+    return {"source": serialize_source(source, _doc_count(ctx, source)), "job": serialize_job(job)}
 
 
 documents_router = APIRouter(prefix="/api/documents", tags=["documents"])
 
 
+def serialize_document(d: Document, source: Source | None = None) -> dict:
+    meta = d.metadata_json or {}
+    return {
+        "id": str(d.id),
+        "title": d.title,
+        "url": d.url,
+        "path": d.path,
+        "source_id": str(d.source_id) if d.source_id else None,
+        "source_title": source.title if source else None,
+        "published_at": d.published_at.isoformat() if d.published_at else None,
+        "words": meta.get("words") or len((d.clean_text or "").split()),
+        "evergreen_score": d.evergreen_score,
+    }
+
+
 @documents_router.get("")
-def list_documents(ctx: Ctx = Depends(get_ctx), limit: int = 500):
-    docs = ctx.db.scalars(
-        select(Document)
+def list_documents(
+    ctx: Ctx = Depends(get_ctx),
+    q: str = "",
+    source_id: uuid.UUID | None = None,
+    limit: int = 500,
+    offset: int = 0,
+):
+    query = (
+        select(Document, Source)
+        .outerjoin(Source, Source.id == Document.source_id)
         .where(Document.workspace_id == ctx.workspace.id)
-        .order_by(Document.published_at.desc())
-        .limit(min(limit, 5000))
+    )
+    if q.strip():
+        like = f"%{q.strip()}%"
+        query = query.where(or_(Document.title.ilike(like), Document.clean_text.ilike(like)))
+    if source_id:
+        query = query.where(Document.source_id == source_id)
+    total = ctx.db.scalar(select(func.count()).select_from(query.subquery()))
+    rows = ctx.db.execute(
+        query.order_by(Document.published_at.desc().nulls_last(), Document.created_at.desc())
+        .offset(max(offset, 0)).limit(min(limit, 5000))
     ).all()
-    return [
-        {"id": str(d.id), "title": d.title, "url": d.url,
-         "published_at": d.published_at.isoformat() if d.published_at else None}
-        for d in docs
-    ]
+    return {"total": total, "items": [serialize_document(d, s) for d, s in rows]}
+
+
+@documents_router.get("/{document_id}")
+def get_document(document_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    doc = ctx.db.scalar(select(Document).where(Document.id == document_id, Document.workspace_id == ctx.workspace.id))
+    if not doc:
+        raise HTTPException(404, "Post not found")
+    source = ctx.db.get(Source, doc.source_id) if doc.source_id else None
+    lines: list[str] = []
+    if doc.path:
+        corpus = Corpus(ctx.workspace.id)
+        try:
+            if not corpus.exists(doc.path):
+                corpus.sync()
+            lines = corpus.read_lines(doc.path)
+        except CorpusError:
+            lines = []
+    if not lines:
+        lines = [f"# {doc.title}", "", *(doc.clean_text or "").splitlines()]
+    return {**serialize_document(doc, source), "lines": lines}
+
+
+@documents_router.delete("/{document_id}")
+def delete_document(document_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    doc = ctx.db.scalar(select(Document).where(Document.id == document_id, Document.workspace_id == ctx.workspace.id))
+    if not doc:
+        raise HTTPException(404, "Post not found")
+    remove_documents(ctx, [doc])
+    ctx.db.commit()
+    return {"ok": True}

backend/app/routers/storage.py

@@ -1,16 +1,17 @@
 """Browser uploads go straight to R2 with presigned PUTs; reads redirect to presigned GETs."""
 
+import mimetypes
 import uuid
 
-from fastapi import APIRouter, Depends, HTTPException
-from fastapi.responses import RedirectResponse
+from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi.responses import FileResponse, RedirectResponse
 from pydantic import BaseModel, Field
 from sqlalchemy import select
 
 from app.auth import Ctx, get_ctx
 from app.config import settings
 from app.models import Upload
-from app.services.storage import keys, storage, workspace_owns_key
+from app.services.storage import LocalStorage, keys, safe_filename, storage, verify_local_sig, workspace_owns_key
 
 router = APIRouter(prefix="/api/storage", tags=["storage"])
 
@@ -23,6 +24,7 @@ ALLOWED_TYPES = {
     "audio/wav",
     "audio/webm",
     "audio/mp4",
+    "audio/ogg",
     "video/mp4",
     "application/pdf",
     "text/markdown",
@@ -89,3 +91,38 @@ def read_object(key: str, download: bool = False, ctx: Ctx = Depends(get_ctx)):
         raise HTTPException(404, "Not found")
     name = key.rsplit("/", 1)[-1] if download else None
     return RedirectResponse(storage.presign_get(key, download_name=name), status_code=302)
+
+
+# Local disk backend: the "presigned" URLs LocalStorage hands out point here.
+
+local_router = APIRouter(prefix="/api/storage/local", tags=["storage"], include_in_schema=False)
+
+
+def _local_path(method: str, key: str, exp: int, sig: str):
+    if not isinstance(storage, LocalStorage) or not verify_local_sig(method, key, exp, sig):
+        raise HTTPException(403, "Invalid or expired link")
+    try:
+        return storage.path(key)
+    except ValueError as exc:
+        raise HTTPException(404, "Not found") from exc
+
+
+@local_router.get("/{key:path}")
+def local_get(key: str, exp: int, sig: str, download: str | None = None):
+    path = _local_path("GET", key, exp, sig)
+    if not path.is_file():
+        raise HTTPException(404, "Not found")
+    media_type = mimetypes.guess_type(key)[0] or "application/octet-stream"
+    # FileResponse answers Range requests, so audio and video can seek.
+    return FileResponse(path, media_type=media_type, filename=safe_filename(download) if download else None,
+                        content_disposition_type="attachment" if download else "inline")
+
+
+@local_router.put("/{key:path}")
+async def local_put(key: str, exp: int, sig: str, request: Request):
+    _local_path("PUT", key, exp, sig)
+    body = await request.body()
+    if len(body) > settings.max_upload_bytes:
+        raise HTTPException(413, "File too large")
+    storage.put_bytes(key, body)
+    return {"ok": True, "size_bytes": len(body)}

backend/app/routers/topics.py

@@ -0,0 +1,71 @@
+import uuid
+from collections import Counter
+from itertools import combinations
+
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy import select
+
+from app.auth import Ctx, get_ctx
+from app.models import Document, DocumentTopic, Topic
+from app.routers.sources import serialize_document
+from app.services.jobs import create_job, serialize_job
+
+router = APIRouter(prefix="/api/topics", tags=["topics"])
+
+
+@router.get("")
+def topic_map(ctx: Ctx = Depends(get_ctx), min_posts: int = 1, limit: int = 80):
+    """Nodes are topics sized by post count; edges join topics that share posts."""
+    topics = ctx.db.scalars(
+        select(Topic).where(Topic.workspace_id == ctx.workspace.id, Topic.post_count >= min_posts)
+        .order_by(Topic.post_count.desc()).limit(min(limit, 300))
+    ).all()
+    ids = {t.id for t in topics}
+    links = ctx.db.execute(
+        select(DocumentTopic.document_id, DocumentTopic.topic_id).where(DocumentTopic.topic_id.in_(ids))
+    ).all() if ids else []
+    by_doc: dict[uuid.UUID, list[uuid.UUID]] = {}
+    for doc_id, topic_id in links:
+        by_doc.setdefault(doc_id, []).append(topic_id)
+    edges: Counter[tuple[str, str]] = Counter()
+    for tids in by_doc.values():
+        for a, b in combinations(sorted(str(t) for t in tids), 2):
+            edges[(a, b)] += 1
+    untagged = ctx.db.scalar(
+        select(Document.id).where(Document.workspace_id == ctx.workspace.id, Document.path.is_not(None),
+                                  Document.id.not_in(select(DocumentTopic.document_id))).limit(1)
+    )
+    return {
+        "nodes": [{"id": str(t.id), "name": t.name, "summary": t.summary, "post_count": t.post_count}
+                  for t in topics],
+        "edges": [{"source": a, "target": b, "weight": w} for (a, b), w in edges.most_common(400)],
+        "has_untagged_posts": untagged is not None,
+    }
+
+
+@router.get("/{topic_id}")
+def topic_detail(topic_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
+    topic = ctx.db.scalar(select(Topic).where(Topic.id == topic_id, Topic.workspace_id == ctx.workspace.id))
+    if not topic:
+        raise HTTPException(404, "Topic not found")
+    rows = ctx.db.execute(
+        select(Document, DocumentTopic.weight)
+        .join(DocumentTopic, DocumentTopic.document_id == Document.id)
+        .where(DocumentTopic.topic_id == topic.id)
+        .order_by(DocumentTopic.weight.desc(), Document.published_at.desc())
+    ).all()
+    return {"id": str(topic.id), "name": topic.name, "summary": topic.summary, "post_count": topic.post_count,
+            "posts": [{**serialize_document(d), "weight": w} for d, w in rows]}
+
+
+@router.post("/rebuild")
+def rebuild(ctx: Ctx = Depends(get_ctx), full: bool = False):
+    params = {}
+    if full:
+        # Clear per post tags so every post is mapped again.
+        for d in ctx.db.scalars(select(Document).where(Document.workspace_id == ctx.workspace.id)):
+            if d.metadata_json and "topics" in d.metadata_json:
+                d.metadata_json = {k: v for k, v in d.metadata_json.items() if k != "topics"}
+        ctx.db.commit()
+    job = create_job(ctx.db, ctx.workspace.id, "topics", params, max_attempts=2)
+    return serialize_job(job)

backend/app/routers/voice.py

@@ -0,0 +1,144 @@
+import uuid
+from datetime import UTC, datetime
+
+from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi.concurrency import run_in_threadpool
+from pydantic import BaseModel, Field
+from sqlalchemy import select
+
+from app.auth import Ctx, get_ctx
+from app.config import settings
+from app.models import Upload, VoiceConsent, VoiceProfile
+from app.services import tts
+from app.services.jobs import create_job, serialize_job
+from app.services.plans import effective_plan
+from app.services.storage import storage
+
+router = APIRouter(prefix="/api/voice", tags=["voice"])
+
+CONSENT_TEXT = (
+    "I confirm this recording is my own voice. I allow Notestack to create a synthetic copy of it with "
+    "ElevenLabs, only for audio and video made in my workspace. I can revoke this at any time, which deletes "
+    "the synthetic voice."
+)
+
+
+class BuildIn(BaseModel):
+    document_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10)
+
+
+class ProfileIn(BaseModel):
+    profile: dict | None = None
+    host_voices: dict[str, str] | None = None
+
+
+class ConsentIn(BaseModel):
+    upload_id: uuid.UUID
+    agreed: bool
+    consent_text: str
+
+
+def _profile(ctx: Ctx) -> VoiceProfile | None:
+    return ctx.db.scalar(select(VoiceProfile).where(VoiceProfile.workspace_id == ctx.workspace.id))
+
+
+def _consent(ctx: Ctx) -> VoiceConsent | None:
+    return ctx.db.scalar(
+        select(VoiceConsent).where(VoiceConsent.workspace_id == ctx.workspace.id, VoiceConsent.revoked_at.is_(None))
+        .order_by(VoiceConsent.created_at.desc())
+    )
+
+
+def _serialize(ctx: Ctx) -> dict:
+    vp = _profile(ctx)
+    consent = _consent(ctx)
+    plan = effective_plan(ctx.db, ctx.workspace)
+    voices = (vp.host_voices if vp else None) or {}
+    return {
+        "profile": vp.profile_json if vp else None,
+        "sample_doc_ids": vp.sample_doc_ids if vp else [],
+        "updated_at": vp.updated_at.isoformat() if vp and vp.updated_at else None,
+        "host_voices": {"host_a": voices.get("host_a") or settings.elevenlabs_voice_a,
+                        "host_b": voices.get("host_b") or settings.elevenlabs_voice_b},
+        "clone": {
+            "allowed": plan.voice_cloning,
+            "consent_text": CONSENT_TEXT,
+            "status": ("ready" if consent and consent.elevenlabs_voice_id else "processing" if consent else "none"),
+            "voice_id": consent.elevenlabs_voice_id if consent else None,
+            "created_at": consent.created_at.isoformat() if consent else None,
+        },
+        "tts_configured": bool(settings.elevenlabs_api_key),
+    }
+
+
+@router.get("")
+def get_voice(ctx: Ctx = Depends(get_ctx)):
+    return _serialize(ctx)
+
+
+@router.put("")
+def update_voice(body: ProfileIn, ctx: Ctx = Depends(get_ctx)):
+    vp = _profile(ctx)
+    if not vp:
+        vp = VoiceProfile(workspace_id=ctx.workspace.id)
+        ctx.db.add(vp)
+    if body.profile is not None:
+        vp.profile_json = body.profile
+    if body.host_voices is not None:
+        vp.host_voices = {k: v for k, v in body.host_voices.items() if k in ("host_a", "host_b") and v}
+    ctx.db.commit()
+    return _serialize(ctx)
+
+
+@router.post("/build")
+def build(body: BuildIn, ctx: Ctx = Depends(get_ctx)):
+    job = create_job(ctx.db, ctx.workspace.id, "voice_profile",
+                     {"document_ids": [str(i) for i in body.document_ids]}, max_attempts=2)
+    return serialize_job(job)
+
+
+@router.get("/voices")
+async def voices(ctx: Ctx = Depends(get_ctx)):
+    items = await run_in_threadpool(tts.list_voices)
+    consent = _consent(ctx)
+    if consent and consent.elevenlabs_voice_id and not any(v["voice_id"] == consent.elevenlabs_voice_id for v in items):
+        items = [{"voice_id": consent.elevenlabs_voice_id, "name": "My voice", "category": "cloned"}, *items]
+    return items
+
+
+@router.post("/consent")
+def give_consent(body: ConsentIn, request: Request, ctx: Ctx = Depends(get_ctx)):
+    if not effective_plan(ctx.db, ctx.workspace).voice_cloning:
+        raise HTTPException(402, {"code": "plan_limit", "message": "Voice cloning is on the Writer and Studio plans."})
+    if not body.agreed or body.consent_text.strip() != CONSENT_TEXT:
+        raise HTTPException(400, "Please read and accept the consent statement.")
+    if not settings.elevenlabs_api_key:
+        raise HTTPException(503, "Voice cloning needs ELEVENLABS_API_KEY in .env")
+    upload = ctx.db.scalar(select(Upload).where(Upload.id == body.upload_id, Upload.workspace_id == ctx.workspace.id))
+    if not upload or upload.status != "complete" or not upload.content_type.startswith("audio/"):
+        raise HTTPException(400, "Upload an audio recording first.")
+    consent = VoiceConsent(workspace_id=ctx.workspace.id, user_id=ctx.user.id, sample_key=upload.key,
+                           consent_text=CONSENT_TEXT, ip_address=request.client.host if request.client else None)
+    ctx.db.add(consent)
+    ctx.db.commit()
+    job = create_job(ctx.db, ctx.workspace.id, "voice_clone", {"consent_id": str(consent.id)}, max_attempts=1)
+    return {"job": serialize_job(job), **_serialize(ctx)}
+
+
+@router.delete("/consent")
+async def revoke_consent(ctx: Ctx = Depends(get_ctx)):
+    consent = _consent(ctx)
+    if not consent:
+        return _serialize(ctx)
+    if consent.elevenlabs_voice_id and settings.elevenlabs_api_key:
+        try:
+            await run_in_threadpool(tts.delete_voice, consent.elevenlabs_voice_id)
+        except Exception:
+            pass
+    consent.revoked_at = datetime.now(UTC)
+    storage.delete(consent.sample_key)
+    vp = _profile(ctx)
+    if vp and vp.host_voices:
+        vp.host_voices = {k: v for k, v in vp.host_voices.items() if v != consent.elevenlabs_voice_id}
+    ctx.db.commit()
+    return _serialize(ctx)

backend/app/services/artifacts.py

@@ -0,0 +1,76 @@
+"""Artifacts are every generated thing (summaries, audio, video, launch kits, stills). Creating one always
+means: artifact row + queued job, returned together so the UI can stream progress."""
+
+import uuid
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.models import Artifact, Job
+from app.services.jobs import create_job, serialize_job
+from app.services.storage import storage
+
+TYPE_LABELS = {
+    "summary": "Summary",
+    "audio_overview": "Audio overview",
+    "video": "Video",
+    "quote_card": "Quote card",
+    "carousel": "Carousel",
+    "launch_kit": "Launch Kit",
+}
+
+
+def start_artifact(
+    db: Session,
+    workspace_id: uuid.UUID,
+    type_: str,
+    *,
+    title: str,
+    notebook_id: uuid.UUID | None = None,
+    document_id: uuid.UUID | None = None,
+    content: dict | None = None,
+    job_kind: str | None = None,
+    params: dict | None = None,
+    max_attempts: int = 2,
+) -> tuple[Artifact, Job]:
+    artifact = Artifact(workspace_id=workspace_id, type=type_, notebook_id=notebook_id, document_id=document_id,
+                        status="pending", content_json={"title": title, **(content or {})})
+    db.add(artifact)
+    db.commit()
+    job = create_job(db, workspace_id, job_kind or type_, {"artifact_id": str(artifact.id), **(params or {})},
+                     artifact_id=artifact.id, max_attempts=max_attempts)
+    return artifact, job
+
+
+def latest_jobs(db: Session, artifact_ids: list[uuid.UUID]) -> dict[uuid.UUID, Job]:
+    if not artifact_ids:
+        return {}
+    jobs = db.scalars(select(Job).where(Job.artifact_id.in_(artifact_ids)).order_by(Job.created_at)).all()
+    return {j.artifact_id: j for j in jobs}  # later rows win, so this is the newest job per artifact
+
+
+def serialize_artifact(a: Artifact, job: Job | None = None) -> dict:
+    content = a.content_json or {}
+    slides = content.get("slide_keys") or []
+    return {
+        "id": str(a.id),
+        "type": a.type,
+        "type_label": TYPE_LABELS.get(a.type, a.type),
+        "title": content.get("title") or TYPE_LABELS.get(a.type, a.type),
+        "status": a.status,
+        "notebook_id": str(a.notebook_id) if a.notebook_id else None,
+        "document_id": str(a.document_id) if a.document_id else None,
+        "content": content,
+        "storage_key": a.storage_key,
+        "url": storage.presign_get(a.storage_key) if a.storage_key else None,
+        "download_url": storage.presign_get(a.storage_key, download_name=_download_name(a)) if a.storage_key else None,
+        "slide_urls": [storage.presign_get(k) for k in slides],
+        "created_at": a.created_at.isoformat() if a.created_at else None,
+        "job": serialize_job(job) if job else None,
+    }
+
+
+def _download_name(a: Artifact) -> str:
+    title = (a.content_json or {}).get("title") or a.type
+    ext = (a.storage_key or "").rsplit(".", 1)[-1]
+    return f"{title[:60]}.{ext}"

backend/app/services/crypto.py

@@ -0,0 +1,31 @@
+"""Encrypts third party tokens at rest (social accounts). Fernet key from SOCIAL_TOKEN_KEY, or derived
+from JWT_SECRET so local dev needs no extra setting."""
+
+import base64
+import hashlib
+from functools import lru_cache
+
+from cryptography.fernet import Fernet, InvalidToken
+
+from app.config import settings
+
+
+@lru_cache
+def _fernet() -> Fernet:
+    key = settings.social_token_key
+    if not key:
+        key = base64.urlsafe_b64encode(hashlib.sha256(f"social:{settings.jwt_secret}".encode()).digest()).decode()
+    return Fernet(key.encode())
+
+
+def encrypt(value: str | None) -> str | None:
+    return _fernet().encrypt(value.encode()).decode() if value else None
+
+
+def decrypt(value: str | None) -> str | None:
+    if not value:
+        return None
+    try:
+        return _fernet().decrypt(value.encode()).decode()
+    except InvalidToken:
+        return None

backend/app/services/email.py

@@ -165,6 +165,21 @@ class EmailService:
             )
         )
 
+    def send_post_reminder(self, to: str, platform: str, content: str, url: str) -> bool:
+        body = (
+            f"Your {html.escape(platform)} post is scheduled for now. Copy it below and post it."
+            f"<pre style='white-space:pre-wrap;background:rgba(255,255,255,0.06);padding:16px;border-radius:8px'>"
+            f"{html.escape(content)}</pre>" + _button("Open Launchpad", url)
+        )
+        return self._send(
+            OutgoingEmail(
+                to=to,
+                subject=f"Time to post on {platform}",
+                html=_layout("Launch window open", body),
+                text=f"Time to post on {platform}:\n\n{content}\n\nLaunchpad: {url}",
+            )
+        )
+
     # Internal alerts
 
     def send_alert(self, subject: str, text: str) -> bool:

backend/app/services/jobs.py

@@ -29,6 +29,7 @@ def serialize_job(job: Job) -> dict:
     return {
         "id": str(job.id),
         "kind": job.kind,
+        "params": job.params,
         "status": job.status,
         "progress": job.progress,
         "message": job.message,

backend/app/services/launchpad.py

@@ -0,0 +1,156 @@
+"""Launchpad: publishes due calendar items, emails reminders for platforms without an API, rewrites links
+to tracked short links and pulls engagement back in."""
+
+import logging
+import re
+import secrets
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.db import SessionLocal
+from app.models import CalendarItem, Document, EngagementEvent, SocialAccount, TrackedLink, User, Workspace
+from app.services.email import email_service
+from app.services.social import AUTO_POST, PLATFORM_LABELS, SocialError, module
+
+log = logging.getLogger("notestack.launchpad")
+_URL = re.compile(r"https?://[^\s)\]>]+")
+MAX_ATTEMPTS = 3
+
+
+def _public_api() -> bool:
+    return not re.search(r"//(localhost|127\.0\.0\.1|api)(:|/|$)", settings.api_url)
+
+
+def track_links(db: Session, item: CalendarItem, text: str) -> str:
+    """Swap outbound links for {API_URL}/l/{slug} so clicks are counted. Skipped when the API is not public."""
+    if not _public_api():
+        return text
+    base = f"{settings.api_url.rstrip('/')}/l/"
+
+    def swap(m: re.Match) -> str:
+        url = m.group(0)
+        if url.startswith(base):
+            return url
+        link = TrackedLink(workspace_id=item.workspace_id, slug=secrets.token_urlsafe(5)[:7], target_url=url,
+                           calendar_item_id=item.id)
+        db.add(link)
+        return base + link.slug
+
+    return _URL.sub(swap, text)
+
+
+def owner_email(db: Session, workspace_id) -> str | None:
+    ws = db.get(Workspace, workspace_id)
+    user = db.get(User, ws.owner_id) if ws else None
+    return user.email if user and user.is_active else None
+
+
+def publish_item(db: Session, item: CalendarItem) -> CalendarItem:
+    """Publish or remind now. Leaves the item posted, reminded, failed, or scheduled again for a retry."""
+    posts = [p for p in [item.content, *(item.thread or [])] if p and p.strip()]
+    if not posts:
+        item.status, item.error = "failed", "Nothing to post."
+        db.commit()
+        return item
+    account = db.get(SocialAccount, item.social_account_id) if item.social_account_id else None
+    label = PLATFORM_LABELS.get(item.platform, item.platform)
+    if item.platform not in AUTO_POST or item.remind_by_email or not account:
+        to = owner_email(db, item.workspace_id)
+        sent = bool(to) and email_service.send_post_reminder(
+            to, label, "\n\n".join(posts), f"{settings.frontend_url}/app/launchpad?item={item.id}")
+        item.status = "reminded" if sent else "failed"
+        item.error = None if sent else "Could not send the reminder email."
+        item.posted_at = datetime.now(UTC)
+        db.commit()
+        return item
+    item.attempts += 1
+    try:
+        tracked = [track_links(db, item, p) for p in posts]
+        db.flush()
+        external_id, url = module(item.platform).publish(db, account, tracked)
+    except SocialError as exc:
+        _fail_or_retry(db, item, str(exc), exc.permanent)
+        return item
+    except Exception as exc:  # network trouble: retry
+        log.exception("publish failed for %s", item.id)
+        _fail_or_retry(db, item, f"{type(exc).__name__}: {exc}", False)
+        return item
+    item.status, item.error = "posted", None
+    item.external_id, item.external_url, item.posted_at = external_id, url, datetime.now(UTC)
+    if item.document_id:
+        doc = db.get(Document, item.document_id)
+        if doc:
+            doc.last_resurfaced_at = item.posted_at
+    db.commit()
+    return item
+
+
+def _fail_or_retry(db: Session, item: CalendarItem, error: str, permanent: bool) -> None:
+    item.error = error[:1000]
+    if permanent or item.attempts >= MAX_ATTEMPTS:
+        item.status = "failed"
+    else:
+        item.status = "scheduled"
+        item.scheduled_at = datetime.now(UTC) + timedelta(minutes=5 * item.attempts)
+    db.commit()
+
+
+def publish_due(session_factory: Callable[[], Session] = SessionLocal) -> int:
+    now = datetime.now(UTC)
+    count = 0
+    with session_factory() as db:
+        # A tick that died mid publish leaves items in "publishing"; put them back after 10 minutes.
+        for stuck in db.scalars(select(CalendarItem).where(
+                CalendarItem.status == "publishing", CalendarItem.updated_at < now - timedelta(minutes=10))):
+            stuck.status = "scheduled"
+        due = db.scalars(
+            select(CalendarItem).where(CalendarItem.status == "scheduled", CalendarItem.scheduled_at <= now)
+            .order_by(CalendarItem.scheduled_at).limit(50)
+        ).all()
+        for item in due:
+            item.status = "publishing"  # claim, so an overlapping tick skips it
+        db.commit()
+        for item in due:
+            publish_item(db, item)
+            count += 1
+    return count
+
+
+def clicks_by_item(db: Session, item_ids: list) -> dict:
+    if not item_ids:
+        return {}
+    rows = db.execute(
+        select(TrackedLink.calendar_item_id, func.count(EngagementEvent.id))
+        .join(EngagementEvent, EngagementEvent.tracked_link_id == TrackedLink.id)
+        .where(TrackedLink.calendar_item_id.in_(item_ids), EngagementEvent.kind == "click")
+        .group_by(TrackedLink.calendar_item_id)
+    ).all()
+    return {item_id: n for item_id, n in rows}
+
+
+def sync_engagement(session_factory: Callable[[], Session] = SessionLocal) -> int:
+    since = datetime.now(UTC) - timedelta(days=30)
+    updated = 0
+    with session_factory() as db:
+        items = db.scalars(
+            select(CalendarItem).where(CalendarItem.status == "posted", CalendarItem.posted_at >= since,
+                                       CalendarItem.external_id.is_not(None))
+        ).all()
+        for item in items:
+            account = db.get(SocialAccount, item.social_account_id) if item.social_account_id else None
+            if not account:
+                continue
+            try:
+                data = module(item.platform).metrics(db, account, item.external_id)
+            except Exception:
+                log.warning("metrics failed for %s", item.id, exc_info=True)
+                continue
+            if data:
+                item.metrics = {**(item.metrics or {}), **data, "synced_at": datetime.now(UTC).isoformat()}
+                updated += 1
+        db.commit()
+    return updated

backend/app/services/renderer.py

@@ -0,0 +1,65 @@
+"""Hands a render to the Remotion service with presigned upload URLs (it never holds storage keys)."""
+
+import httpx
+
+from app.config import settings
+from app.models import Artifact, Job
+from app.services.storage import keys, storage
+
+COMPOSITIONS = {
+    "ShortVertical": "mp4",
+    "ExplainerLong": "mp4",
+    "AudiogramSquare": "mp4",
+    "QuoteCard": "png",
+    "CarouselSlide": "png",
+}
+
+
+class PermanentJobError(RuntimeError):
+    """Fail the job now; retrying will not help."""
+
+
+def request_render(job: Job, artifact: Artifact, composition: str, props: dict | None = None,
+                   stills: list[dict] | None = None) -> None:
+    """Start a render. `stills` renders one PNG per props dict (carousels) instead of a single output."""
+    fmt = COMPOSITIONS[composition]
+    content_type = "image/png" if fmt == "png" else "video/mp4"
+    body: dict = {
+        "jobId": str(job.id),
+        "compositionId": composition,
+        "props": props or {},
+        "format": fmt,
+        "uploadContentType": content_type,
+        "callbackUrl": f"{settings.api_url}/api/internal/jobs/{job.id}/progress",
+    }
+    if stills:
+        outputs = []
+        for i, still_props in enumerate(stills, start=1):
+            key = keys.artifact(artifact.workspace_id, artifact.id, f"slide-{i:02d}", fmt)
+            outputs.append({"props": still_props, "uploadUrl": storage.presign_put(key, content_type, ttl=6 * 3600),
+                            "storageKey": key})
+        body.update(stills=outputs, props=stills[0], uploadUrl=outputs[0]["uploadUrl"],
+                    storageKey=outputs[0]["storageKey"])
+    else:
+        key = keys.artifact(artifact.workspace_id, artifact.id, composition.lower(), fmt)
+        body.update(uploadUrl=storage.presign_put(key, content_type, ttl=6 * 3600), storageKey=key)
+    try:
+        resp = httpx.post(f"{settings.renderer_url}/render", headers={"x-internal-token": settings.internal_token},
+                          json=body, timeout=30)
+    except httpx.ConnectError as exc:
+        raise PermanentJobError("Renderer is not running. Start it with: cd renderer && npm start") from exc
+    if resp.status_code == 400:
+        raise PermanentJobError(f"Renderer rejected the props: {resp.text[:400]}")
+    resp.raise_for_status()
+
+
+def brand_props(workspace, *, ttl: int = 6 * 3600) -> dict:
+    brand = workspace.brand_json or {}
+    out: dict = {}
+    if brand.get("accent"):
+        out["accent"] = brand["accent"]
+    if brand.get("name"):
+        out["name"] = brand["name"]
+    if brand.get("logo_key"):
+        out["logoUrl"] = storage.presign_get(brand["logo_key"], ttl=ttl)
+    return out

backend/app/services/social/__init__.py

@@ -0,0 +1,55 @@
+"""Publishing to X, LinkedIn and Bluesky. Each platform module exposes:
+
+    publish(db, account, posts: list[str]) -> (external_id, external_url)
+    metrics(db, account, external_id) -> dict
+
+A list with more than one post is published as a thread (reply chain) where the platform supports it.
+"""
+
+from datetime import UTC, datetime
+
+from sqlalchemy.orm import Session
+
+from app.models import SocialAccount
+from app.services.crypto import decrypt, encrypt
+
+
+class SocialError(RuntimeError):
+    def __init__(self, message: str, permanent: bool = False):
+        super().__init__(message)
+        self.permanent = permanent
+
+
+PLATFORM_LABELS = {"x": "X", "linkedin": "LinkedIn", "bluesky": "Bluesky", "substack_notes": "Substack Notes"}
+AUTO_POST = {"x", "linkedin", "bluesky"}
+LIMITS = {"x": 280, "bluesky": 300, "linkedin": 3000, "substack_notes": 2000}
+
+
+def token(account: SocialAccount) -> str:
+    value = decrypt(account.access_token)
+    if not value:
+        raise SocialError("Stored credentials could not be read. Reconnect the account.", permanent=True)
+    return value
+
+
+def save_tokens(db: Session, account: SocialAccount, access: str, refresh: str | None, expires_in: int | None) -> None:
+    account.access_token = encrypt(access)
+    if refresh:
+        account.refresh_token = encrypt(refresh)
+    if expires_in:
+        account.expires_at = datetime.fromtimestamp(datetime.now(UTC).timestamp() + int(expires_in), UTC)
+    account.status = "active"
+    db.commit()
+
+
+def expired(account: SocialAccount) -> bool:
+    if not account.expires_at:
+        return False
+    exp = account.expires_at if account.expires_at.tzinfo else account.expires_at.replace(tzinfo=UTC)
+    return exp.timestamp() - 120 < datetime.now(UTC).timestamp()
+
+
+def module(platform: str):
+    from app.services.social import bluesky, linkedin, x
+
+    return {"x": x, "linkedin": linkedin, "bluesky": bluesky}[platform]

backend/app/services/social/bluesky.py

@@ -0,0 +1,67 @@
+"""Bluesky over AT Protocol with an app password (bsky.app > Settings > App passwords)."""
+
+import re
+from datetime import UTC, datetime
+
+import httpx
+from sqlalchemy.orm import Session
+
+from app.models import SocialAccount
+from app.services.social import SocialError, token
+
+PDS = "https://bsky.social/xrpc"
+PUBLIC = "https://public.api.bsky.app/xrpc"
+_URL = re.compile(r"https?://[^\s)\]]+")
+
+
+def create_session(handle: str, app_password: str) -> dict:
+    resp = httpx.post(f"{PDS}/com.atproto.server.createSession",
+                      json={"identifier": handle.lstrip("@"), "password": app_password}, timeout=20)
+    if resp.status_code in (400, 401):
+        raise SocialError("Bluesky did not accept that handle and app password.", permanent=True)
+    resp.raise_for_status()
+    return resp.json()
+
+
+def link_facets(text: str) -> list[dict]:
+    """Links must be declared as facets with UTF-8 byte offsets to be clickable."""
+    facets = []
+    for m in _URL.finditer(text):
+        start = len(text[: m.start()].encode())
+        end = start + len(m.group(0).encode())
+        facets.append({"index": {"byteStart": start, "byteEnd": end},
+                       "features": [{"$type": "app.bsky.richtext.facet#link", "uri": m.group(0)}]})
+    return facets
+
+
+def publish(db: Session, account: SocialAccount, posts: list[str]) -> tuple[str, str]:
+    session = create_session(account.handle, token(account))
+    headers = {"Authorization": f"Bearer {session['accessJwt']}"}
+    root = parent = None
+    for text in posts:
+        record: dict = {"$type": "app.bsky.feed.post", "text": text,
+                        "createdAt": datetime.now(UTC).isoformat().replace("+00:00", "Z")}
+        facets = link_facets(text)
+        if facets:
+            record["facets"] = facets
+        if parent:
+            record["reply"] = {"root": root, "parent": parent}
+        resp = httpx.post(f"{PDS}/com.atproto.repo.createRecord", headers=headers, timeout=20,
+                          json={"repo": session["did"], "collection": "app.bsky.feed.post", "record": record})
+        if resp.status_code >= 400:
+            raise SocialError(f"Bluesky {resp.status_code}: {resp.text[:300]}", permanent=resp.status_code == 400)
+        ref = {"uri": resp.json()["uri"], "cid": resp.json()["cid"]}
+        root = root or ref
+        parent = ref
+    rkey = root["uri"].rsplit("/", 1)[-1]
+    return root["uri"], f"https://bsky.app/profile/{session.get('handle', account.handle)}/post/{rkey}"
+
+
+def metrics(db: Session, account: SocialAccount, external_id: str) -> dict:
+    resp = httpx.get(f"{PUBLIC}/app.bsky.feed.getPosts", params={"uris": external_id}, timeout=20)
+    if resp.status_code >= 400:
+        return {}
+    posts = resp.json().get("posts") or [{}]
+    p = posts[0]
+    return {"likes": p.get("likeCount", 0), "reposts": p.get("repostCount", 0),
+            "replies": p.get("replyCount", 0), "quotes": p.get("quoteCount", 0)}

backend/app/services/social/linkedin.py

@@ -0,0 +1,87 @@
+"""LinkedIn member posts (Share on LinkedIn + Sign In with LinkedIn using OpenID Connect products)."""
+
+import re
+from urllib.parse import quote, urlencode
+
+import httpx
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.models import SocialAccount
+from app.services.social import SocialError, expired, token
+
+AUTH_URL = "https://www.linkedin.com/oauth/v2/authorization"
+TOKEN_URL = "https://www.linkedin.com/oauth/v2/accessToken"
+SCOPES = "openid profile w_member_social"
+VERSION = "202509"  # LinkedIn-Version header, YYYYMM; LinkedIn supports each version for a year
+
+
+def redirect_uri() -> str:
+    return f"{settings.api_url.rstrip('/')}/api/social/linkedin/callback"
+
+
+def configured() -> bool:
+    return bool(settings.linkedin_client_id and settings.linkedin_client_secret)
+
+
+def authorize_url(state: str) -> str:
+    return AUTH_URL + "?" + urlencode({
+        "response_type": "code", "client_id": settings.linkedin_client_id, "redirect_uri": redirect_uri(),
+        "state": state, "scope": SCOPES,
+    })
+
+
+def exchange_code(code: str) -> dict:
+    resp = httpx.post(TOKEN_URL, data={
+        "grant_type": "authorization_code", "code": code, "redirect_uri": redirect_uri(),
+        "client_id": settings.linkedin_client_id, "client_secret": settings.linkedin_client_secret,
+    }, timeout=20)
+    if resp.status_code >= 400:
+        raise SocialError(f"LinkedIn token request failed: {resp.text[:300]}", permanent=True)
+    tokens = resp.json()
+    me = httpx.get("https://api.linkedin.com/v2/userinfo",
+                   headers={"Authorization": f"Bearer {tokens['access_token']}"}, timeout=20)
+    me.raise_for_status()
+    info = me.json()
+    return {**tokens, "external_id": info["sub"], "handle": info.get("name") or info["sub"],
+            "avatar_url": info.get("picture")}
+
+
+_RESERVED = re.compile(r"([\\|{}@\[\]()<>#*_~])")
+
+
+def little_text(text: str) -> str:
+    """LinkedIn's commentary field uses 'little text' markup; reserved characters must be escaped."""
+    return _RESERVED.sub(r"\\\1", text)
+
+
+def publish(db: Session, account: SocialAccount, posts: list[str]) -> tuple[str, str]:
+    if expired(account):
+        account.status = "expired"
+        db.commit()
+        raise SocialError("The LinkedIn connection expired. Reconnect the account.", permanent=True)
+    body = {
+        "author": f"urn:li:person:{account.external_id}",
+        "commentary": little_text("\n\n".join(posts)),
+        "visibility": "PUBLIC",
+        "distribution": {"feedDistribution": "MAIN_FEED", "targetEntities": [], "thirdPartyDistributionChannels": []},
+        "lifecycleState": "PUBLISHED",
+        "isReshareDisabledByAuthor": False,
+    }
+    resp = httpx.post("https://api.linkedin.com/rest/posts", json=body, timeout=20, headers={
+        "Authorization": f"Bearer {token(account)}", "LinkedIn-Version": VERSION,
+        "X-Restli-Protocol-Version": "2.0.0",
+    })
+    if resp.status_code == 401:
+        account.status = "expired"
+        db.commit()
+        raise SocialError("LinkedIn rejected the token. Reconnect the account.", permanent=True)
+    if resp.status_code >= 400:
+        raise SocialError(f"LinkedIn {resp.status_code}: {resp.text[:300]}",
+                          permanent=resp.status_code in (400, 403, 422))
+    urn = resp.headers.get("x-restli-id") or resp.headers.get("x-linkedin-id") or ""
+    return urn, f"https://www.linkedin.com/feed/update/{quote(urn)}/" if urn else ""
+
+
+def metrics(db: Session, account: SocialAccount, external_id: str) -> dict:
+    return {}  # member post analytics need the Community Management API, which personal apps rarely have

backend/app/services/social/x.py

@@ -0,0 +1,95 @@
+"""X (Twitter) API v2 with OAuth 2.0 user context (PKCE). Threads are reply chains."""
+
+import base64
+import hashlib
+import secrets
+from urllib.parse import urlencode
+
+import httpx
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.models import SocialAccount
+from app.services.crypto import decrypt
+from app.services.social import SocialError, expired, save_tokens, token
+
+AUTH_URL = "https://x.com/i/oauth2/authorize"
+TOKEN_URL = "https://api.x.com/2/oauth2/token"
+API = "https://api.x.com/2"
+SCOPES = "tweet.read tweet.write users.read offline.access"
+
+
+def redirect_uri() -> str:
+    return f"{settings.api_url.rstrip('/')}/api/social/x/callback"
+
+
+def configured() -> bool:
+    return bool(settings.x_client_id and settings.x_client_secret)
+
+
+def pkce_pair() -> tuple[str, str]:
+    verifier = secrets.token_urlsafe(64)[:96]
+    challenge = base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).decode().rstrip("=")
+    return verifier, challenge
+
+
+def authorize_url(state: str, challenge: str) -> str:
+    return AUTH_URL + "?" + urlencode({
+        "response_type": "code", "client_id": settings.x_client_id, "redirect_uri": redirect_uri(),
+        "scope": SCOPES, "state": state, "code_challenge": challenge, "code_challenge_method": "S256",
+    })
+
+
+def _token_request(data: dict) -> dict:
+    resp = httpx.post(TOKEN_URL, data={**data, "client_id": settings.x_client_id},
+                      auth=(settings.x_client_id, settings.x_client_secret), timeout=20)
+    if resp.status_code >= 400:
+        raise SocialError(f"X token request failed: {resp.text[:300]}", permanent=resp.status_code in (400, 401))
+    return resp.json()
+
+
+def exchange_code(code: str, verifier: str) -> dict:
+    tokens = _token_request({"grant_type": "authorization_code", "code": code, "redirect_uri": redirect_uri(),
+                             "code_verifier": verifier})
+    me = httpx.get(f"{API}/users/me", params={"user.fields": "profile_image_url"},
+                   headers={"Authorization": f"Bearer {tokens['access_token']}"}, timeout=20)
+    me.raise_for_status()
+    user = me.json()["data"]
+    return {**tokens, "external_id": user["id"], "handle": user["username"],
+            "avatar_url": user.get("profile_image_url")}
+
+
+def _access(db: Session, account: SocialAccount) -> str:
+    if expired(account) and account.refresh_token:
+        tokens = _token_request({"grant_type": "refresh_token", "refresh_token": decrypt(account.refresh_token)})
+        save_tokens(db, account, tokens["access_token"], tokens.get("refresh_token"), tokens.get("expires_in"))
+    return token(account)
+
+
+def publish(db: Session, account: SocialAccount, posts: list[str]) -> tuple[str, str]:
+    headers = {"Authorization": f"Bearer {_access(db, account)}"}
+    first_id, reply_to = None, None
+    for text in posts:
+        body: dict = {"text": text}
+        if reply_to:
+            body["reply"] = {"in_reply_to_tweet_id": reply_to}
+        resp = httpx.post(f"{API}/tweets", json=body, headers=headers, timeout=20)
+        if resp.status_code == 401:
+            account.status = "expired"
+            db.commit()
+            raise SocialError("X rejected the token. Reconnect the account.", permanent=True)
+        if resp.status_code >= 400:
+            raise SocialError(f"X {resp.status_code}: {resp.text[:300]}", permanent=resp.status_code in (400, 403))
+        reply_to = resp.json()["data"]["id"]
+        first_id = first_id or reply_to
+    return first_id, f"https://x.com/{account.handle}/status/{first_id}"
+
+
+def metrics(db: Session, account: SocialAccount, external_id: str) -> dict:
+    resp = httpx.get(f"{API}/tweets", params={"ids": external_id, "tweet.fields": "public_metrics"},
+                     headers={"Authorization": f"Bearer {_access(db, account)}"}, timeout=20)
+    if resp.status_code >= 400:
+        return {}
+    data = (resp.json().get("data") or [{}])[0].get("public_metrics") or {}
+    return {"likes": data.get("like_count", 0), "reposts": data.get("retweet_count", 0),
+            "replies": data.get("reply_count", 0), "impressions": data.get("impression_count", 0)}

backend/app/services/storage.py

@@ -1,13 +1,20 @@
-"""Cloudflare R2 storage (S3 API). Postgres stores keys; bytes live here.
+"""Object storage. Postgres stores keys; bytes live here.
 
-Locally, point R2_ENDPOINT_URL at MinIO and nothing else changes.
+Production: Cloudflare R2 (S3 API). docker-compose: MinIO via R2_ENDPOINT_URL. Plain local dev with
+no R2 settings: LocalStorage, a folder on disk whose presigned URLs are HMAC signed API routes, so
+every caller (browser uploads, the renderer, media players) works unchanged.
 """
 
 import hashlib
+import hmac
 import mimetypes
 import re
+import shutil
+import time
 import uuid
 from functools import cached_property
+from pathlib import Path
+from urllib.parse import quote, urlencode
 
 import boto3
 from botocore.client import Config
@@ -127,4 +134,85 @@ class Storage:
         return f"{settings.r2_public_base_url.rstrip('/')}/{key}"
 
 
-storage = Storage()
+def _local_sig(method: str, key: str, exp: int) -> str:
+    msg = f"{method.upper()}:{key}:{exp}".encode()
+    return hmac.new(settings.jwt_secret.encode(), msg, hashlib.sha256).hexdigest()
+
+
+def verify_local_sig(method: str, key: str, exp: int, sig: str) -> bool:
+    return exp >= time.time() and hmac.compare_digest(_local_sig(method, key, exp), sig)
+
+
+class LocalStorage:
+    """Disk backed stand in for R2 with the same interface."""
+
+    def __init__(self, root: str | None = None):
+        self.root = Path(root or settings.local_storage_dir).resolve()
+        self.bucket = "local"
+
+    def path(self, key: str) -> Path:
+        if ".." in key.split("/") or key.startswith("/"):
+            raise ValueError(f"Invalid key: {key!r}")
+        return self.root / key
+
+    def ensure_bucket(self) -> None:
+        self.root.mkdir(parents=True, exist_ok=True)
+
+    def put_bytes(self, key: str, data: bytes, content_type: str | None = None) -> str:
+        path = self.path(key)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_bytes(data)
+        return key
+
+    def put_text(self, key: str, text: str, content_type: str = "text/plain; charset=utf-8") -> str:
+        return self.put_bytes(key, text.encode("utf-8"), content_type)
+
+    def get_bytes(self, key: str) -> bytes:
+        path = self.path(key)
+        if not path.is_file():
+            raise FileNotFoundError(key)
+        return path.read_bytes()
+
+    def head(self, key: str) -> dict | None:
+        path = self.path(key)
+        if not path.is_file():
+            return None
+        return {"ContentLength": path.stat().st_size, "ContentType": mimetypes.guess_type(key)[0]}
+
+    def exists(self, key: str) -> bool:
+        return self.path(key).is_file()
+
+    def delete(self, key: str) -> None:
+        self.path(key).unlink(missing_ok=True)
+
+    def delete_prefix(self, prefix: str) -> int:
+        base = self.path(prefix.rstrip("/"))
+        if base.is_file():
+            base.unlink()
+            return 1
+        if not base.is_dir():
+            return 0
+        count = sum(1 for f in base.rglob("*") if f.is_file())
+        shutil.rmtree(base, ignore_errors=True)
+        return count
+
+    def _signed(self, method: str, key: str, ttl: int | None, extra: dict | None = None) -> str:
+        exp = int(time.time()) + (ttl or settings.r2_presign_ttl_seconds)
+        params = {"exp": exp, "sig": _local_sig(method, key, exp), **(extra or {})}
+        return f"{settings.api_url.rstrip('/')}/api/storage/local/{quote(key)}?{urlencode(params)}"
+
+    def presign_get(self, key: str, ttl: int | None = None, download_name: str | None = None) -> str:
+        return self._signed("GET", key, ttl, {"download": safe_filename(download_name)} if download_name else None)
+
+    def presign_put(self, key: str, content_type: str, ttl: int | None = None) -> str:
+        return self._signed("PUT", key, ttl)
+
+    def public_url(self, key: str) -> str | None:
+        return None
+
+
+def make_storage() -> Storage | LocalStorage:
+    return LocalStorage() if settings.use_local_storage else Storage()
+
+
+storage = make_storage()

backend/app/services/tts.py

@@ -0,0 +1,124 @@
+"""ElevenLabs text to speech, cached in storage by (script, voice, settings)."""
+
+import base64
+import json
+
+import httpx
+
+from app.config import settings
+from app.services.storage import keys, storage
+
+API = "https://api.elevenlabs.io/v1"
+OUTPUT = "mp3_44100_128"
+BITRATE = 128_000
+
+PREMADE = [
+    {"voice_id": "21m00Tcm4TlvDq8ikWAM", "name": "Rachel", "category": "premade"},
+    {"voice_id": "pNInz6obpgDQGcFMaJgB", "name": "Adam", "category": "premade"},
+    {"voice_id": "EXAVITQu4vr4xnSDxMaL", "name": "Bella", "category": "premade"},
+    {"voice_id": "ErXwobaYiN019PkySvjV", "name": "Antoni", "category": "premade"},
+    {"voice_id": "AZnzlk1XvdvUeBnXmlld", "name": "Domi", "category": "premade"},
+    {"voice_id": "TxGEqnHWrfWFTfGW9XjX", "name": "Josh", "category": "premade"},
+]
+
+
+class TTSError(RuntimeError):
+    pass
+
+
+def _headers() -> dict:
+    if not settings.elevenlabs_api_key:
+        raise TTSError("ELEVENLABS_API_KEY is not set")
+    return {"xi-api-key": settings.elevenlabs_api_key}
+
+
+def mp3_seconds(data: bytes) -> float:
+    return len(data) * 8 / BITRATE
+
+
+def _voice_settings() -> dict:
+    return {"stability": 0.45, "similarity_boost": 0.8, "style": 0.2, "use_speaker_boost": True}
+
+
+def _cache_key(workspace_id, text: str, voice_id: str, variant: str) -> str:
+    return keys.tts_cache(workspace_id, text, voice_id,
+                          json.dumps(_voice_settings(), sort_keys=True) + settings.elevenlabs_model + variant)
+
+
+def synthesize(workspace_id, text: str, voice_id: str) -> bytes:
+    key = _cache_key(workspace_id, text, voice_id, OUTPUT)
+    if storage.exists(key):
+        return storage.get_bytes(key)
+    body = {"text": text, "model_id": settings.elevenlabs_model, "voice_settings": _voice_settings()}
+    resp = httpx.post(f"{API}/text-to-speech/{voice_id}", params={"output_format": OUTPUT},
+                      headers=_headers(), json=body, timeout=120)
+    if resp.status_code >= 400:
+        raise TTSError(f"ElevenLabs {resp.status_code}: {resp.text[:300]}")
+    storage.put_bytes(key, resp.content, "audio/mpeg")
+    return resp.content
+
+
+def synthesize_with_timestamps(workspace_id, text: str, voice_id: str) -> tuple[bytes, list[dict]]:
+    """Audio plus word timings [{text, startMs, endMs}] for captions."""
+    key = _cache_key(workspace_id, text, voice_id, "timestamps") + ".json"
+    if storage.exists(key):
+        data = json.loads(storage.get_bytes(key))
+    else:
+        body = {"text": text, "model_id": settings.elevenlabs_model, "voice_settings": _voice_settings()}
+        resp = httpx.post(f"{API}/text-to-speech/{voice_id}/with-timestamps", params={"output_format": OUTPUT},
+                          headers=_headers(), json=body, timeout=120)
+        if resp.status_code >= 400:
+            raise TTSError(f"ElevenLabs {resp.status_code}: {resp.text[:300]}")
+        data = resp.json()
+        storage.put_bytes(key, json.dumps(data).encode(), "application/json")
+    audio = base64.b64decode(data["audio_base64"])
+    return audio, words_from_alignment(data.get("alignment") or {})
+
+
+def words_from_alignment(alignment: dict) -> list[dict]:
+    chars = alignment.get("characters") or []
+    starts = alignment.get("character_start_times_seconds") or []
+    ends = alignment.get("character_end_times_seconds") or []
+    words: list[dict] = []
+    current, start, last_end = "", 0.0, 0.0
+    for ch, s, e in zip(chars, starts, ends, strict=False):
+        if ch.isspace():
+            if current:
+                words.append({"text": current, "startMs": int(start * 1000), "endMs": int(last_end * 1000)})
+            current = ""
+            continue
+        if not current:
+            start = s
+        current += ch
+        last_end = e
+    if current:
+        words.append({"text": current, "startMs": int(start * 1000), "endMs": int(last_end * 1000)})
+    return words
+
+
+def list_voices() -> list[dict]:
+    if not settings.elevenlabs_api_key:
+        return PREMADE
+    try:
+        resp = httpx.get(f"{API}/voices", headers=_headers(), timeout=20)
+        resp.raise_for_status()
+    except httpx.HTTPError:
+        return PREMADE
+    return [
+        {"voice_id": v["voice_id"], "name": v["name"], "category": v.get("category"),
+         "preview_url": v.get("preview_url")}
+        for v in resp.json().get("voices", [])
+    ]
+
+
+def clone_voice(name: str, sample: bytes, filename: str, content_type: str) -> str:
+    resp = httpx.post(f"{API}/voices/add", headers=_headers(), timeout=120,
+                      data={"name": name, "description": "Notestack consented voice clone"},
+                      files={"files": (filename, sample, content_type)})
+    if resp.status_code >= 400:
+        raise TTSError(f"ElevenLabs {resp.status_code}: {resp.text[:300]}")
+    return resp.json()["voice_id"]
+
+
+def delete_voice(voice_id: str) -> None:
+    httpx.delete(f"{API}/voices/{voice_id}", headers=_headers(), timeout=30)

backend/app/services/usage.py

@@ -0,0 +1,64 @@
+"""Monthly usage against plan limits, from usage_events."""
+
+from datetime import UTC, datetime
+
+from fastapi import HTTPException
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from app.models import Artifact, UsageEvent, Workspace
+from app.services.plans import Plan, effective_plan
+
+
+def month_start(now: datetime | None = None) -> datetime:
+    now = now or datetime.now(UTC)
+    return now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+
+
+def month_usage(db: Session, workspace_id) -> dict:
+    since = month_start()
+
+    def total(kind: str, unit: str) -> float:
+        return float(db.scalar(
+            select(func.coalesce(func.sum(UsageEvent.quantity), 0)).where(
+                UsageEvent.workspace_id == workspace_id, UsageEvent.kind == kind,
+                UsageEvent.unit == unit, UsageEvent.created_at >= since,
+            )
+        ) or 0)
+
+    kits = db.scalar(
+        select(func.count()).select_from(Artifact).where(
+            Artifact.workspace_id == workspace_id, Artifact.type == "launch_kit", Artifact.created_at >= since
+        )
+    ) or 0
+    return {
+        "audio_minutes": round(total("tts", "seconds") / 60, 1),
+        "video_minutes": round(total("video", "seconds") / 60, 1),
+        "launch_kits": kits,
+        "llm_tokens": int(total("llm", "tokens")),
+        "since": since.isoformat(),
+    }
+
+
+def usage_report(db: Session, workspace: Workspace) -> dict:
+    plan = effective_plan(db, workspace)
+    used = month_usage(db, workspace.id)
+    return {
+        "plan": plan.id,
+        "used": used,
+        "limits": {"audio_minutes": plan.audio_minutes, "video_minutes": plan.video_minutes,
+                   "launch_kits": plan.launch_kits, "sources": plan.sources, "indexed_posts": plan.indexed_posts},
+    }
+
+
+def check_limit(db: Session, workspace: Workspace, kind: str, amount: float = 1) -> Plan:
+    """Raise 402 plan_limit when this request would go over the monthly allowance."""
+    plan = effective_plan(db, workspace)
+    used = month_usage(db, workspace.id)
+    limit = {"audio_minutes": plan.audio_minutes, "video_minutes": plan.video_minutes,
+             "launch_kits": plan.launch_kits}[kind]
+    if limit >= 0 and used[kind] + amount > limit:
+        label = kind.replace("_", " ")
+        raise HTTPException(402, {"code": "plan_limit",
+                                  "message": f"This would go over your {limit} {label} this month."})
+    return plan

backend/app/worker.py

@@ -34,16 +34,22 @@ from app.models import (
     Subscription,
     UpdateEmail,
     UpdateEmailSend,
+    Upload,
     User,
+    VoiceConsent,
     Workspace,
     WorkspaceMember,
 )
-from app.pipeline.ingest import ingest_source
+from app.pipeline import generate, launchkit, media
+from app.pipeline.generate import NothingToDo
+from app.pipeline.ingest import FeedNotFound, entry_from_upload, extract_article, ingest_source, store_entries
+from app.services import launchpad, tts
 from app.services.email import email_service
 from app.services.email_verification import purge_old_codes
-from app.services.jobs import claim_next, retry_or_fail, stale_jobs, update_job
+from app.services.jobs import claim_next, create_job, retry_or_fail, stale_jobs, update_job
 from app.services.plans import effective_plan
-from app.services.storage import keys, storage
+from app.services.renderer import PermanentJobError, request_render
+from app.services.storage import storage
 
 log = logging.getLogger("notestack.worker")
 
@@ -60,55 +66,140 @@ HANDED_OFF = object()  # the job continues elsewhere (renderer) and reports back
 # Handlers: (db, job) -> Done | HANDED_OFF. Raise to trigger a retry.
 
 
+def _after_import(db: Session, job: Job, workspace_id: uuid.UUID, changed: list[str]) -> None:
+    """New or changed posts get mapped into the topic constellation automatically."""
+    if changed and settings.llm_api_key:
+        create_job(db, workspace_id, "topics", {"document_ids": changed}, max_attempts=2)
+
+
 def handle_ingest(db: Session, job: Job):
     source = db.get(Source, uuid.UUID(job.params["source_id"]))
     if not source:
         return Done({"error": "source deleted"}, "Source no longer exists")
     plan = effective_plan(db, db.get(Workspace, source.workspace_id))
     try:
-        result = ingest_source(db, source, job, max_posts=min(plan.indexed_posts, 200))
+        result = ingest_source(db, source, job, max_posts=plan.indexed_posts)
     except Exception as exc:
         db.rollback()
         source.sync_status = "error"
         source.sync_error = str(exc)[:500]
         db.commit()
+        if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code in (401, 403, 404, 410):
+            raise PermanentJobError(f"The feed returned {exc.response.status_code}.") from exc
         raise
-    return Done(result, "All posts in orbit")
+    _after_import(db, job, source.workspace_id, result.get("changed", []))
+    return Done(result, f"All posts in orbit: {result['indexed']} new or updated")
+
+
+def handle_import_url(db: Session, job: Job):
+    source = db.get(Source, uuid.UUID(job.params["source_id"]))
+    update_job(db, job, progress=0.1, message="Fetching the article")
+    try:
+        entry = extract_article(job.params["url"])
+    except FeedNotFound as exc:
+        raise PermanentJobError(str(exc)) from exc
+    indexed, _, changed = store_entries(db, source, [entry])
+    _after_import(db, job, source.workspace_id, [str(i) for i in changed])
+    return Done({"indexed": indexed, "document_ids": [str(i) for i in changed]}, f"{entry.title[:80]} is in orbit")
+
+
+def handle_import_upload(db: Session, job: Job):
+    source = db.get(Source, uuid.UUID(job.params["source_id"]))
+    upload = db.get(Upload, uuid.UUID(job.params["upload_id"]))
+    update_job(db, job, progress=0.1, message=f"Reading {upload.filename}")
+    entry = entry_from_upload(upload.filename, upload.content_type, storage.get_bytes(upload.key))
+    if not entry.sections and not entry.html:
+        raise PermanentJobError("We could not read any text in that file.")
+    indexed, _, changed = store_entries(db, source, [entry])
+    _after_import(db, job, source.workspace_id, [str(i) for i in changed])
+    return Done({"indexed": indexed, "document_ids": [str(i) for i in changed]}, f"{entry.title[:80]} is in orbit")
+
+
+def handle_topics(db: Session, job: Job):
+    return Done(generate.extract_topics(db, job, job.workspace_id, job.params.get("document_ids")), "Topic map ready")
+
+
+def handle_voice_profile(db: Session, job: Job):
+    try:
+        result = generate.build_voice_profile(db, job, job.workspace_id, job.params.get("document_ids") or [])
+    except NothingToDo as exc:
+        raise PermanentJobError(str(exc)) from exc
+    return Done(result, "Voice profile ready")
+
+
+def handle_voice_clone(db: Session, job: Job):
+    consent = db.get(VoiceConsent, uuid.UUID(job.params["consent_id"]))
+    if not consent or consent.revoked_at:
+        return Done({}, "Consent was revoked")
+    update_job(db, job, progress=0.2, message="Sending your sample to ElevenLabs")
+    head = storage.head(consent.sample_key) or {}
+    try:
+        voice_id = tts.clone_voice(f"Notestack {consent.workspace_id}", storage.get_bytes(consent.sample_key),
+                                   consent.sample_key.rsplit("/", 1)[-1], head.get("ContentType") or "audio/webm")
+    except tts.TTSError as exc:
+        raise PermanentJobError(str(exc)) from exc
+    consent.elevenlabs_voice_id = voice_id
+    db.commit()
+    return Done({"voice_id": voice_id}, "Your voice is ready")
+
+
+def handle_resurface(db: Session, job: Job):
+    return Done(generate.score_evergreen(db, job, job.workspace_id), "Evergreen scores updated")
+
+
+def _artifact_handler(fn: Callable[[Session, Job, Artifact], object], done_message: str):
+    def handler(db: Session, job: Job):
+        artifact = db.get(Artifact, uuid.UUID(job.params["artifact_id"]))
+        if not artifact:
+            return Done({}, "Artifact was deleted")
+        artifact.status = "generating"
+        db.commit()
+        try:
+            result = fn(db, job, artifact)
+        except (NothingToDo, tts.TTSError) as exc:
+            raise PermanentJobError(str(exc)) from exc
+        if result is None:
+            return HANDED_OFF  # the renderer reports back through /api/internal/jobs/{id}/progress
+        return Done(result if isinstance(result, dict) else {}, done_message)
+
+    return handler
 
 
 def handle_render(db: Session, job: Job):
-    """Hand off to the Remotion renderer with a presigned PUT so it never holds R2 credentials."""
-    p = job.params
-    artifact = db.get(Artifact, uuid.UUID(p["artifact_id"]))
-    ext = "png" if p["format"] == "png" else "mp4"
-    key = keys.artifact(artifact.workspace_id, artifact.id, p["composition"].lower(), ext)
-    content_type = "image/png" if ext == "png" else "video/mp4"
+    """Direct render of caller supplied props (POST /api/artifacts/{id}/render)."""
+    artifact = db.get(Artifact, uuid.UUID(job.params["artifact_id"]))
     update_job(db, job, progress=0.01, message="T-minus: preparing render")
-    resp = httpx.post(
-        f"{settings.renderer_url}/render",
-        headers={"x-internal-token": settings.internal_token},
-        json={
-            "jobId": str(job.id),
-            "compositionId": p["composition"],
-            "props": p["props"],
-            "format": p["format"],
-            "uploadUrl": storage.presign_put(key, content_type, ttl=6 * 3600),
-            "uploadContentType": content_type,
-            "callbackUrl": f"{settings.api_url}/api/internal/jobs/{job.id}/progress",
-            "storageKey": key,
-        },
-        timeout=30,
-    )
-    resp.raise_for_status()
+    request_render(job, artifact, job.params["composition"], job.params.get("props") or {})
     return HANDED_OFF
 
 
 HANDLERS: dict[str, Callable[[Session, Job], object]] = {
     "ingest": handle_ingest,
+    "import_url": handle_import_url,
+    "import_upload": handle_import_upload,
+    "topics": handle_topics,
+    "voice_profile": handle_voice_profile,
+    "voice_clone": handle_voice_clone,
+    "resurface_scan": handle_resurface,
+    "summary": _artifact_handler(generate.summarize_notebook, "Summary ready"),
+    "audio_overview": _artifact_handler(media.audio_overview, "Audio overview ready"),
+    "video": _artifact_handler(media.make_video, "Video ready"),
+    "quote_card": _artifact_handler(media.make_quote_card, "Quote card ready"),
+    "carousel": _artifact_handler(media.render_carousel, "Carousel ready"),
+    "launch_kit": _artifact_handler(launchkit.build_launch_kit, "Launch Kit ready"),
     "render": handle_render,
 }
 
 
+def mark_artifact_failed(db: Session, job: Job, error: str) -> None:
+    if job.artifact_id:
+        artifact = db.get(Artifact, job.artifact_id)
+        if artifact:
+            artifact.status = "failed"
+            artifact.content_json = {**(artifact.content_json or {}), "error": error[:500]}
+            db.commit()
+
+
 def run_job(
     job_id: uuid.UUID,
     session_factory: Callable[[], Session] = SessionLocal,
@@ -125,11 +216,18 @@ def run_job(
             return
         try:
             outcome = handler(db, job)
+        except PermanentJobError as exc:
+            db.rollback()
+            job = db.get(Job, job_id)
+            update_job(db, job, status="failed", error=str(exc), message=str(exc)[:500])
+            mark_artifact_failed(db, job, str(exc))
+            return
         except Exception as exc:
             log.exception("job %s (%s) attempt %s failed", job.id, job.kind, job.attempts)
             db.rollback()
             job = db.get(Job, job_id)
-            retry_or_fail(db, job, f"{type(exc).__name__}: {exc}")
+            if not retry_or_fail(db, job, f"{type(exc).__name__}: {exc}"):
+                mark_artifact_failed(db, job, f"{type(exc).__name__}: {exc}")
             return
         if outcome is HANDED_OFF:
             return
@@ -219,6 +317,8 @@ class Periodic:
 
 PERIODIC = [
     Periodic("recover_stale", 60, recover_stale),
+    Periodic("publish_due", 30, launchpad.publish_due),
+    Periodic("sync_engagement", 3600, launchpad.sync_engagement),
     Periodic("update_email_batch", 300, update_email_batch),
     Periodic("daily_cleanup", 24 * 3600, daily_cleanup),
 ]
@@ -240,13 +340,9 @@ def run_periodic(now: float) -> None:
 # Main loop
 
 
-def main() -> None:
-    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
-    worker_id = f"{socket.gethostname()}:{os.getpid()}"
-    stop = threading.Event()
-    for sig in (signal.SIGINT, signal.SIGTERM):
-        signal.signal(sig, lambda *_: stop.set())
-
+def run_loop(stop: threading.Event, worker_id: str | None = None) -> None:
+    """Claim and run jobs until `stop` is set. Used by the CLI and, in development, by the API."""
+    worker_id = worker_id or f"{socket.gethostname()}:{os.getpid()}"
     pool = ThreadPoolExecutor(max_workers=settings.worker_concurrency, thread_name_prefix="job")
     running: set[Future] = set()
     log.info("worker %s up, concurrency=%s", worker_id, settings.worker_concurrency)
@@ -273,5 +369,19 @@ def main() -> None:
     pool.shutdown(wait=True)
 
 
+def start_background(stop: threading.Event) -> threading.Thread:
+    thread = threading.Thread(target=run_loop, args=(stop, f"api:{os.getpid()}"), name="worker", daemon=True)
+    thread.start()
+    return thread
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
+    stop = threading.Event()
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        signal.signal(sig, lambda *_: stop.set())
+    run_loop(stop)
+
+
 if __name__ == "__main__":
     main()

backend/requirements.txt

@@ -16,3 +16,4 @@ httpx==0.28.*
 dspy==2.6.*
 feedparser==6.*
 beautifulsoup4==4.*
+pypdf==5.*

backend/tests/conftest.py

@@ -1,6 +1,13 @@
 import os
+import tempfile
 
 os.environ.setdefault("DATABASE_URL", "sqlite://")
+os.environ["STORAGE_BACKEND"] = "local"
+os.environ["LOCAL_STORAGE_DIR"] = tempfile.mkdtemp(prefix="notestack-storage-")
+os.environ["CORPUS_CACHE_DIR"] = tempfile.mkdtemp(prefix="notestack-corpus-")
+os.environ["RUN_WORKER_IN_API"] = "false"
+os.environ["LLM_API_KEY"] = ""
+os.environ["ELEVENLABS_API_KEY"] = ""
 os.environ.setdefault("EMAIL_PROVIDER", "console")
 os.environ.setdefault("JWT_SECRET", "test-secret-that-is-at-least-32-bytes-long")
 os.environ.setdefault("BILLING_ENABLED", "false")
@@ -18,15 +25,41 @@ from app.services.email import ConsoleEmailProvider  # noqa: E402
 
 
 @pytest.fixture()
-def db_session():
+def session_factory():
     engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
     Base.metadata.create_all(engine)
-    Session = sessionmaker(bind=engine, expire_on_commit=False)
-    session = Session()
+    return sessionmaker(bind=engine, expire_on_commit=False)
+
+
+@pytest.fixture()
+def db_session(session_factory):
+    session = session_factory()
     yield session
     session.close()
 
 
+@pytest.fixture()
+def run_jobs(session_factory, db_session):
+    """Drain the queue the way the worker does, in process. Returns the jobs that ran."""
+    from app.models import Job
+    from app.services.jobs import claim_next
+    from app.worker import run_job
+
+    def drain(limit: int = 50) -> list:
+        ran = []
+        for _ in range(limit):
+            with session_factory() as db:
+                job_id = claim_next(db, "test")
+            if not job_id:
+                break
+            run_job(job_id, session_factory=session_factory)
+            ran.append(job_id)
+        db_session.expire_all()
+        return [db_session.get(Job, j) for j in ran]
+
+    return drain
+
+
 @pytest.fixture()
 def client(db_session):
     app.dependency_overrides[get_db] = lambda: db_session

backend/tests/test_buildout.py

@@ -0,0 +1,612 @@
+"""End to end coverage for sources, notebooks, generation, Launchpad and settings. External services
+(feeds, LLM, ElevenLabs, renderer, social APIs) are stubbed at their module seams."""
+
+import time
+import uuid
+from datetime import UTC, datetime, timedelta
+
+import httpx
+import pytest
+
+from app.models import (
+    Artifact,
+    CalendarItem,
+    Document,
+    EngagementEvent,
+    Job,
+    SocialAccount,
+    Source,
+    TrackedLink,
+    UsageEvent,
+    Workspace,
+)
+from app.services.email import ConsoleEmailProvider
+from tests.conftest import last_code
+
+FEED = """<?xml version="1.0"?>
+<rss version="2.0"><channel><title>Ada Writes</title><link>https://ada.example.com</link>
+<item><title>On Pricing</title><link>https://ada.example.com/p/on-pricing</link>
+<pubDate>Mon, 03 Mar 2025 10:00:00 GMT</pubDate>
+<description><![CDATA[<h2>Why price high</h2><p>Charging more made my readers take the work seriously.</p>
+<p>The paid tier doubled after I raised the price to ten dollars.</p>]]></description></item>
+<item><title>Writing Every Day</title><link>https://ada.example.com/p/writing-every-day</link>
+<pubDate>Tue, 04 Jun 2024 10:00:00 GMT</pubDate>
+<description><![CDATA[<p>I write five hundred words before breakfast.</p>
+<p>Habits beat inspiration.</p>]]></description>
+</item></channel></rss>"""
+
+
+class FakeResponse:
+    def __init__(self, status: int = 200, text: str = "", content_type: str = "application/rss+xml",
+                 url: str = "", json_data=None):
+        self.status_code = status
+        self.text = text
+        self.content = text.encode()
+        self.headers = {"content-type": content_type}
+        self.url = url
+        self._json = json_data
+
+    def json(self):
+        return self._json
+
+    def raise_for_status(self):
+        if self.status_code >= 400:
+            raise httpx.HTTPStatusError("bad", request=None, response=self)
+
+
+def fake_web(routes: dict):
+    def get(url, *args, **kwargs):
+        for prefix, resp in routes.items():
+            if url.startswith(prefix):
+                resp.url = url
+                return resp
+        return FakeResponse(404, "not found", "text/html", url)
+
+    return get
+
+
+@pytest.fixture()
+def auth(client):
+    client.post("/api/auth/email/register/start", json={"email": "ada@example.com", "password": "stardust-42",
+                                                        "name": "Ada Lovelace"})
+    r = client.post("/api/auth/email/register/verify", json={"email": "ada@example.com", "code": last_code()})
+    return {"Authorization": f"Bearer {r.json()['access_token']}"}
+
+
+@pytest.fixture()
+def feed(monkeypatch):
+    monkeypatch.setattr(httpx, "get", fake_web({"https://ada.example.com/feed": FakeResponse(200, FEED)}))
+
+
+@pytest.fixture()
+def llm(monkeypatch):
+    """Canned outputs per signature. Tests override entries in the returned dict."""
+    from app.llm import run
+
+    outputs: dict = {}
+    calls: list[str] = []
+
+    def predict(signature, *, db, workspace_id, job=None, lm=None, **inputs):
+        name = signature.__name__
+        calls.append(name)
+        value = outputs[name]
+        return value(**inputs) if callable(value) else value
+
+    monkeypatch.setattr(run, "predict", predict)
+    outputs["_calls"] = calls
+    return outputs
+
+
+def connect(client, auth, run_jobs):
+    r = client.post("/api/sources", json={"url": "ada.example.com"}, headers=auth)
+    assert r.status_code == 200, r.text
+    jobs = run_jobs()
+    assert jobs[0].status == "done", jobs[0].error
+    return r.json()["source"]
+
+
+def docs(client, auth):
+    return client.get("/api/documents", headers=auth).json()["items"]
+
+
+# Storage
+
+
+def test_local_storage_signed_urls(client):
+    from app.services.storage import storage
+
+    storage.put_bytes("ws/x/file.txt", b"0123456789")
+    url = storage.presign_get("ws/x/file.txt").split("8000", 1)[1]
+    assert client.get(url).content == b"0123456789"
+    ranged = client.get(url, headers={"Range": "bytes=2-4"})
+    assert ranged.status_code == 206 and ranged.content == b"234"
+    assert client.get(url.replace("sig=", "sig=0")).status_code == 403
+    expired = storage._signed("GET", "ws/x/file.txt", -10)
+    assert client.get(expired.split("8000", 1)[1]).status_code == 403
+    put = storage.presign_put("ws/x/new.bin", "application/octet-stream").split("8000", 1)[1]
+    assert client.put(put, content=b"abc").status_code == 200
+    assert storage.get_bytes("ws/x/new.bin") == b"abc"
+    assert client.get(url.replace("file.txt", "new.bin")).status_code == 403  # signature is per key
+
+
+# Sources
+
+
+def test_connect_rejects_missing_feed(client, auth, monkeypatch):
+    monkeypatch.setattr(httpx, "get", fake_web({}))
+    r = client.post("/api/sources", json={"url": "nobody.substack.com"}, headers=auth)
+    assert r.status_code == 422
+    assert r.json()["detail"]["code"] == "feed_not_found"
+
+
+def test_connect_ingest_read_and_delete(client, auth, run_jobs, feed, db_session):
+    source = connect(client, auth, run_jobs)
+    assert source["title"] == "Ada Writes"
+    items = docs(client, auth)
+    assert {d["title"] for d in items} == {"On Pricing", "Writing Every Day"}
+    listed = client.get("/api/sources", headers=auth).json()
+    assert listed[0]["document_count"] == 2 and listed[0]["sync_status"] == "ok"
+
+    post = client.get(f"/api/documents/{items[0]['id']}", headers=auth).json()
+    assert post["lines"][0] == "---" and any("paid tier" in ln or "breakfast" in ln for ln in post["lines"])
+    assert client.get("/api/documents", params={"q": "breakfast"}, headers=auth).json()["total"] == 1
+
+    # Re-sync is a no-op for unchanged posts.
+    client.post(f"/api/sources/{source['id']}/sync", headers=auth)
+    assert run_jobs()[0].result["skipped"] == 2
+
+    r = client.delete(f"/api/sources/{source['id']}", headers=auth)
+    assert r.json()["removed_posts"] == 2
+    assert docs(client, auth) == []
+
+
+def test_import_url_and_markdown_upload(client, auth, run_jobs, monkeypatch):
+    article = FakeResponse(200, "<html><head><title>Loose Post</title>"
+                                "<meta property='article:published_time' content='2024-01-02T00:00:00Z'></head>"
+                                "<body><article><h1>Loose Post</h1><p>Standalone essay text lives here.</p>"
+                                "</article></body></html>", "text/html")
+    monkeypatch.setattr(httpx, "get", fake_web({"https://elsewhere.example.com/essay": article}))
+    r = client.post("/api/sources/url", json={"url": "https://elsewhere.example.com/essay"}, headers=auth)
+    assert r.status_code == 200
+    assert run_jobs()[0].status == "done"
+
+    start = client.post("/api/storage/uploads", json={"filename": "notes.md", "content_type": "text/markdown",
+                                                      "size_bytes": 60}, headers=auth).json()
+    client.put(start["upload_url"].split("8000", 1)[1], content=b"# Field Notes\n\n## Day one\nIt rained all day.\n")
+    client.post(f"/api/storage/uploads/{start['upload_id']}/complete", headers=auth)
+    r = client.post("/api/sources/upload", json={"upload_id": start["upload_id"]}, headers=auth)
+    assert run_jobs()[0].status == "done", r.text
+    titles = {d["title"] for d in docs(client, auth)}
+    assert titles == {"Loose Post", "Field Notes"}
+    sources = client.get("/api/sources", headers=auth).json()
+    assert len(sources) == 1 and sources[0]["is_imports"]
+
+
+# Notebooks
+
+
+def test_notebook_crud_chat_history_and_summary(client, auth, run_jobs, feed, llm, db_session):
+    connect(client, auth, run_jobs)
+    items = docs(client, auth)
+    pricing = next(d for d in items if d["title"] == "On Pricing")
+    nb = client.post("/api/notebooks", json={"title": "Money", "document_ids": [pricing["id"]]}, headers=auth).json()
+    assert nb["added"] == 1
+    assert client.patch(f"/api/notebooks/{nb['id']}", json={"title": "Pricing"}, headers=auth).json()["title"] == \
+        "Pricing"
+    listed = client.get("/api/notebooks", headers=auth).json()
+    assert listed[0]["document_count"] == 1
+
+    path = pricing["path"]
+    llm["SummarizeNotebook"] = {
+        "summary": "Raising prices worked [1]. Also the moon is cheese [2].",
+        "themes": ["Pricing"],
+        "citations": [{"marker": 1, "path": path, "line_start": 12, "line_end": 14},
+                      {"marker": 2, "path": "sources/fake.md", "line_start": 1, "line_end": 2}],
+    }
+    art = client.post("/api/artifacts/generate", json={"type": "summary", "notebook_id": nb["id"]},
+                      headers=auth).json()
+    assert art["job"]["status"] == "queued"
+    run_jobs()
+    done = client.get(f"/api/artifacts/{art['id']}", headers=auth).json()
+    assert done["status"] == "ready"
+    assert len(done["content"]["citations"]) == 1  # the invented citation was dropped
+    assert "[2]" not in done["content"]["summary"]
+    assert client.get(f"/api/notebooks/{nb['id']}", headers=auth).json()["summary"].startswith("Raising")
+
+    # Chat history endpoints
+    from app.models import Chat, Message
+
+    chat = Chat(workspace_id=db_session.get(Source, uuid.UUID(pricing["source_id"])).workspace_id,
+                notebook_id=uuid.UUID(nb["id"]), title="q")
+    db_session.add(chat)
+    db_session.flush()
+    db_session.add(Message(chat_id=chat.id, role="user", content="Why price high?"))
+    db_session.commit()
+    chats = client.get(f"/api/notebooks/{nb['id']}/chats", headers=auth).json()
+    assert chats[0]["id"] == str(chat.id)
+    assert client.get(f"/api/notebooks/chats/{chat.id}/messages", headers=auth).json()[0]["text"] == "Why price high?"
+
+    client.delete(f"/api/notebooks/{nb['id']}/documents/{pricing['id']}", headers=auth)
+    assert client.get(f"/api/notebooks/{nb['id']}", headers=auth).json()["documents"] == []
+    assert client.delete(f"/api/notebooks/{nb['id']}", headers=auth).status_code == 200
+
+
+def test_summary_of_empty_notebook_fails_cleanly(client, auth, run_jobs, llm):
+    nb = client.post("/api/notebooks", json={"title": "Empty"}, headers=auth).json()
+    art = client.post("/api/artifacts/generate", json={"type": "summary", "notebook_id": nb["id"]},
+                      headers=auth).json()
+    job = run_jobs()[0]
+    assert job.status == "failed" and job.attempts == 1  # permanent, no retries
+    assert client.get(f"/api/artifacts/{art['id']}", headers=auth).json()["status"] == "failed"
+
+
+# Topics, voice, resurfacing
+
+
+def test_topic_map_and_notebook_from_topic(client, auth, run_jobs, feed, llm):
+    connect(client, auth, run_jobs)
+    llm["ExtractTopics"] = lambda title, **_: {"topics": [
+        {"name": "Pricing" if "Pricing" in title else "Habits", "weight": 0.9},
+        {"name": "Writing Business", "weight": 0.5},
+        {"name": "Writing business", "weight": 0.4},
+    ]}
+    llm["ConsolidateTopics"] = {"groups": [
+        {"canonical": "Writing Business", "members": ["Writing Business", "Writing business"],
+         "summary": "Making writing pay."},
+    ]}
+    client.post("/api/topics/rebuild", headers=auth)
+    assert run_jobs()[0].status == "done"
+    graph = client.get("/api/topics", headers=auth).json()
+    names = {n["name"]: n for n in graph["nodes"]}
+    assert set(names) == {"Pricing", "Habits", "Writing Business"}
+    assert names["Writing Business"]["post_count"] == 2
+    assert len(graph["edges"]) == 2
+    detail = client.get(f"/api/topics/{names['Writing Business']['id']}", headers=auth).json()
+    assert len(detail["posts"]) == 2
+    nb = client.post(f"/api/notebooks/from-topic/{names['Pricing']['id']}", headers=auth).json()
+    assert nb["added"] == 1
+
+
+def test_voice_profile_build_and_edit(client, auth, run_jobs, feed, llm):
+    connect(client, auth, run_jobs)
+    profile = {"tone": ["warm"], "sentence_length": "short", "vocabulary": ["readers"], "structure_habits": [],
+               "openings": ["A number"], "avoid": ["jargon"], "summary": "Plain and direct."}
+    llm["BuildVoiceProfile"] = {"profile": profile}
+    client.post("/api/voice/build", json={"document_ids": []}, headers=auth)
+    assert run_jobs()[0].status == "done"
+    voice = client.get("/api/voice", headers=auth).json()
+    assert voice["profile"]["summary"] == "Plain and direct."
+    r = client.put("/api/voice", json={"host_voices": {"host_a": "abc", "host_b": "def"}}, headers=auth)
+    assert r.json()["host_voices"] == {"host_a": "abc", "host_b": "def"}
+    bad = client.post("/api/voice/consent", json={"upload_id": str(uuid.uuid4()), "agreed": True,
+                                                  "consent_text": "nope"}, headers=auth)
+    assert bad.status_code == 400
+
+
+def test_resurface_scan_and_suggestions(client, auth, run_jobs, feed, llm):
+    connect(client, auth, run_jobs)
+    llm["EvergreenScore"] = lambda title, **_: {"score": 0.9 if "Writing" in title else 0.2,
+                                                "reason": "Timeless habit advice", "angle": "Still true"}
+    client.post("/api/resurface/scan", headers=auth)
+    assert run_jobs()[0].status == "done"
+    data = client.get("/api/resurface", headers=auth).json()
+    assert data["unscored"] == 0
+    assert data["evergreen"][0]["title"] == "Writing Every Day"
+    assert data["evergreen"][0]["angle"] == "Still true"
+
+
+# Audio, video, launch kit
+
+
+def test_audio_overview_and_limits(client, auth, run_jobs, feed, llm, monkeypatch, db_session):
+    from app.config import settings
+    from app.services import tts
+
+    connect(client, auth, run_jobs)
+    doc = docs(client, auth)[0]
+    monkeypatch.setattr(settings, "elevenlabs_api_key", "test")
+    monkeypatch.setattr(tts, "synthesize", lambda ws, text, voice: b"\x00" * 16000)  # 1 second at 128 kbps
+    llm["PodcastScript"] = {"lines": [
+        {"speaker": "host_a", "text": "Welcome in.", "sources": []},
+        {"speaker": "host_b", "text": "Prices went up.", "sources": [{"path": doc["path"], "line_start": 12,
+                                                                      "line_end": 13}]},
+    ]}
+    art = client.post("/api/artifacts/generate", json={"type": "audio_overview", "document_id": doc["id"],
+                                                       "minutes": 5}, headers=auth).json()
+    run_jobs()
+    done = client.get(f"/api/artifacts/{art['id']}", headers=auth).json()
+    assert done["status"] == "ready", done["content"]
+    assert done["content"]["duration_s"] == 2.0
+    assert done["url"] and client.get(done["url"].split("8000", 1)[1]).content == b"\x00" * 32000
+
+    ws_id = db_session.get(Document, uuid.UUID(doc["id"])).workspace_id
+    db_session.add(UsageEvent(workspace_id=ws_id, kind="tts", provider="elevenlabs", quantity=240 * 60,
+                              unit="seconds"))
+    db_session.commit()
+    r = client.post("/api/artifacts/generate", json={"type": "audio_overview", "document_id": doc["id"],
+                                                     "minutes": 5}, headers=auth)
+    assert r.status_code == 402 and r.json()["detail"]["code"] == "plan_limit"
+
+
+def test_video_hands_off_and_renderer_callback(client, auth, run_jobs, feed, llm, monkeypatch):
+    from app.pipeline import media
+
+    connect(client, auth, run_jobs)
+    doc = docs(client, auth)[0]
+    sent = {}
+    monkeypatch.setattr(media, "request_render", lambda job, artifact, comp, props=None, stills=None:
+                        sent.update(comp=comp, props=props, job=str(job.id)))
+    llm["VideoStoryboard"] = {"scenes": [
+        {"type": "title", "on_screen_text": "Charge more", "narration": "", "duration_hint_s": 2.5, "visual": ""},
+        {"type": "section", "on_screen_text": "Readers valued it", "narration": "", "duration_hint_s": 3,
+         "visual": ""},
+    ]}
+    art = client.post("/api/artifacts/generate", json={"type": "video", "document_id": doc["id"], "style": "short"},
+                      headers=auth).json()
+    job = run_jobs()[0]
+    assert job.status == "running"  # handed off to the renderer
+    assert sent["comp"] == "ShortVertical" and sent["props"]["hook"] == "Charge more"
+    from app.config import settings
+
+    r = client.post(f"/api/internal/jobs/{sent['job']}/progress", headers={"x-internal-token": settings.internal_token},
+                    json={"status": "done", "progress": 1, "storage_key": "ws/x/video.mp4", "render_seconds": 12})
+    assert r.status_code == 200
+    assert client.get(f"/api/artifacts/{art['id']}", headers=auth).json()["status"] == "ready"
+
+
+def test_renderer_down_fails_without_retry(client, auth, run_jobs, feed, llm, monkeypatch):
+    connect(client, auth, run_jobs)
+    doc = docs(client, auth)[0]
+
+    def refuse(*args, **kwargs):
+        raise httpx.ConnectError("refused")
+
+    monkeypatch.setattr(httpx, "post", refuse)
+    llm["PickQuotes"] = lambda passages, **_: {"quotes": [{"quote": "Charging more made my readers take the work "
+                                                                    "seriously.",
+                                                           "source": {"path": doc["path"], "line_start": 1,
+                                                                      "line_end": 20}}]}
+    art = client.post("/api/artifacts/generate", json={"type": "quote_card", "document_id": doc["id"]},
+                      headers=auth).json()
+    job = run_jobs()[0]
+    assert job.status == "failed" and "Renderer is not running" in job.error
+    assert client.get(f"/api/artifacts/{art['id']}", headers=auth).json()["status"] == "failed"
+
+
+def test_launch_kit(client, auth, run_jobs, feed, llm, monkeypatch, db_session):
+    from app.pipeline import media
+
+    connect(client, auth, run_jobs)
+    doc = next(d for d in docs(client, auth) if d["title"] == "On Pricing")
+    ref = {"path": doc["path"], "line_start": 12, "line_end": 14}
+    llm.update({
+        "ExtractClaims": {"claims": [{"claim": "Higher prices signal value", "sources": [ref]},
+                                     {"claim": "Unsupported", "sources": [{**ref, "line_start": 999}]}]},
+        "HookGenerator": {"hooks": [{"text": "I doubled my price.", "strength": 0.9, "rationale": "number"}]},
+        "PlatformAdapter": lambda platform, **_: {"posts": [f"{platform} post one", f"{platform} post two"]},
+        "SeoPack": {"seo": {"title_options": ["On Pricing"], "meta_description": "Why I charge more.",
+                            "slug": "on-pricing", "keywords": ["pricing"], "internal_links": []}},
+        "CarouselSlides": {"slides": [{"heading": "Charge more", "body": "It works."}]},
+        "PickQuotes": {"quotes": [{"quote": "Charging more made my readers take the work seriously.",
+                                   "source": ref}]},
+    })
+    monkeypatch.setattr(media, "request_render", lambda *a, **k: None)
+    art = client.post("/api/artifacts/generate", json={"type": "launch_kit", "document_id": doc["id"]},
+                      headers=auth).json()
+    run_jobs()
+    kit = client.get(f"/api/artifacts/{art['id']}", headers=auth).json()
+    assert kit["status"] == "ready", kit["content"]
+    c = kit["content"]
+    assert len(c["claims"]) == 1  # the unverifiable claim was dropped
+    assert set(c["posts"]) == {"x_thread", "linkedin", "substack_notes", "bluesky"}
+    assert len(c["quote_card_ids"]) == 1
+    card = client.get(f"/api/artifacts/{c['quote_card_ids'][0]}", headers=auth).json()
+    assert card["status"] == "rendering"  # its own job ran and handed off
+
+    edited = client.patch(f"/api/artifacts/{art['id']}", json={"content": {"posts": {**c["posts"],
+                                                                                    "linkedin": ["Edited"]}}},
+                          headers=auth).json()
+    assert edited["content"]["posts"]["linkedin"] == ["Edited"]
+    assert edited["content"]["quote_card_ids"] == c["quote_card_ids"]
+
+    listing = client.get("/api/artifacts", params={"type": "launch_kit"}, headers=auth).json()
+    assert listing["total"] == 1
+    assert client.delete(f"/api/artifacts/{art['id']}", headers=auth).status_code == 200
+
+
+# Launchpad
+
+
+def _bluesky(client, auth, monkeypatch):
+    from app.services.social import bluesky
+
+    monkeypatch.setattr(bluesky, "create_session", lambda handle, pw: {"did": "did:plc:ada", "handle": handle,
+                                                                       "accessJwt": "jwt"})
+    r = client.post("/api/social/bluesky", json={"handle": "ada.bsky.social", "app_password": "abcd-efgh-ijkl"},
+                    headers=auth)
+    assert r.status_code == 200, r.text
+    return r.json()["accounts"][0]
+
+
+def test_schedule_and_publish_bluesky_thread(client, auth, monkeypatch, session_factory, db_session):
+    from app.services import launchpad
+    from app.services.social import bluesky
+
+    account = _bluesky(client, auth, monkeypatch)
+    published = {}
+
+    def publish(db, acct, posts):
+        published["posts"] = posts
+        return "at://did:plc:ada/app.bsky.feed.post/abc", "https://bsky.app/profile/ada.bsky.social/post/abc"
+
+    monkeypatch.setattr(bluesky, "publish", publish)
+    too_long = client.post("/api/calendar", json={"platform": "bluesky", "content": "x" * 400,
+                                                  "scheduled_at": datetime.now(UTC).isoformat(),
+                                                  "social_account_id": account["id"]}, headers=auth)
+    assert too_long.status_code == 422
+    item = client.post("/api/calendar", json={
+        "platform": "bluesky", "content": "First", "thread": ["Second", " "],
+        "scheduled_at": (datetime.now(UTC) - timedelta(minutes=1)).isoformat(), "social_account_id": account["id"],
+    }, headers=auth).json()
+    assert item["auto_post"] is True
+
+    assert launchpad.publish_due(session_factory) == 1
+    assert published["posts"] == ["First", "Second"]
+    got = client.get("/api/calendar", headers=auth).json()[0]
+    assert got["status"] == "posted" and got["external_url"].startswith("https://bsky.app/")
+    assert client.patch(f"/api/calendar/{item['id']}", json={"content": "late"}, headers=auth).status_code == 409
+
+
+def test_substack_notes_send_reminder_email(client, auth, session_factory):
+    from app.services import launchpad
+
+    ConsoleEmailProvider.sent.clear()
+    item = client.post("/api/calendar", json={"platform": "substack_notes", "content": "A note",
+                                              "scheduled_at": (datetime.now(UTC) - timedelta(seconds=5)).isoformat()},
+                       headers=auth).json()
+    assert item["remind_by_email"] is True
+    launchpad.publish_due(session_factory)
+    assert client.get("/api/calendar", headers=auth).json()[0]["status"] == "reminded"
+    assert ConsoleEmailProvider.sent[-1].subject == "Time to post on Substack Notes"
+
+
+def test_publish_retries_then_fails(client, auth, monkeypatch, session_factory):
+    from app.services import launchpad
+    from app.services.social import SocialError, bluesky
+
+    account = _bluesky(client, auth, monkeypatch)
+
+    def boom(db, acct, posts):
+        raise SocialError("Bluesky 500: busy")
+
+    monkeypatch.setattr(bluesky, "publish", boom)
+    item = client.post("/api/calendar", json={"platform": "bluesky", "content": "Hi", "social_account_id":
+                                              account["id"], "scheduled_at": datetime.now(UTC).isoformat()},
+                       headers=auth).json()
+    r = client.post(f"/api/calendar/{item['id']}/publish", headers=auth).json()
+    assert r["status"] == "scheduled" and "busy" in r["error"]  # retry queued
+    for _ in range(2):
+        with session_factory() as db:
+            db.query(CalendarItem).update({"scheduled_at": datetime.now(UTC) - timedelta(seconds=1)})
+            db.commit()
+        launchpad.publish_due(session_factory)
+    assert client.get("/api/calendar", headers=auth).json()[0]["status"] == "failed"
+
+
+def test_tracked_link_counts_clicks(client, auth, db_session):
+    ws = db_session.query(Workspace).first()
+    item = CalendarItem(workspace_id=ws.id, platform="x", scheduled_at=datetime.now(UTC), content="x",
+                        status="posted")
+    db_session.add(item)
+    db_session.flush()
+    db_session.add(TrackedLink(workspace_id=ws.id, slug="abc1234", target_url="https://ada.example.com/p/x",
+                               calendar_item_id=item.id))
+    db_session.commit()
+    r = client.get("/l/abc1234", follow_redirects=False)
+    assert r.status_code == 302 and r.headers["location"] == "https://ada.example.com/p/x"
+    assert db_session.query(EngagementEvent).count() == 1
+    assert client.get("/api/calendar", headers=auth).json()[0]["clicks"] == 1
+    assert client.get("/l/nope", follow_redirects=False).status_code == 404
+
+
+def test_oauth_link_is_bound_to_the_starting_workspace(client, auth, monkeypatch, db_session):
+    import json
+
+    from app.auth import encode_signed
+    from app.services.crypto import encrypt
+
+    ws = db_session.query(Workspace).first()
+    info = {"access_token": "tok", "refresh_token": "ref", "expires_in": 7200, "external_id": "42",
+            "handle": "ada", "scope": "tweet.write"}
+
+    def ticket(ws_id):
+        return encode_signed({"typ": "social_link", "ws": str(ws_id), "platform": "x",
+                              "data": encrypt(json.dumps(info))}, timedelta(minutes=5))
+
+    assert client.post("/api/social/complete", json={"ticket": ticket(uuid.uuid4())}, headers=auth).status_code == 403
+    r = client.post("/api/social/complete", json={"ticket": ticket(ws.id)}, headers=auth)
+    assert r.status_code == 200
+    acct = db_session.query(SocialAccount).one()
+    assert acct.platform == "x" and acct.access_token != "tok"  # stored encrypted
+    assert client.post("/api/social/x/start", headers=auth).status_code == 503  # no client id configured
+
+
+def test_oauth_callback_redirects_with_ticket(client, monkeypatch):
+    from app.auth import encode_signed
+    from app.services.social import linkedin
+
+    monkeypatch.setattr(linkedin, "exchange_code", lambda code: {"access_token": "t", "external_id": "p",
+                                                                 "handle": "Ada", "expires_in": 60})
+    state = encode_signed({"typ": "social_state", "nonce": "n", "ws": str(uuid.uuid4()), "platform": "linkedin"},
+                          timedelta(minutes=5))
+    r = client.get("/api/social/linkedin/callback", params={"code": "c", "state": state}, follow_redirects=False)
+    assert r.status_code == 302 and "link=" in r.headers["location"] and "/app/launchpad" in r.headers["location"]
+    bad = client.get("/api/social/linkedin/callback", params={"code": "c", "state": "junk"}, follow_redirects=False)
+    assert "error=" in bad.headers["location"]
+
+
+# Settings and activity
+
+
+def test_settings_and_usage(client, auth):
+    s = client.get("/api/settings", headers=auth).json()
+    assert s["user"]["name"] == "Ada Lovelace" and s["usage"]["used"]["launch_kits"] == 0
+    r = client.patch("/api/settings", json={"name": "Ada L", "workspace_name": "Ada HQ", "brand_accent": "#FF0000",
+                                            "training_opt_in": True}, headers=auth).json()
+    assert r["user"]["name"] == "Ada L" and r["workspace"]["name"] == "Ada HQ"
+    assert r["brand"]["accent"] == "#ff0000" and r["workspace"]["training_opt_in"] is True
+    assert client.patch("/api/settings", json={"brand_accent": "red"}, headers=auth).status_code == 422
+
+
+def test_active_jobs_listing(client, auth, feed):
+    client.post("/api/sources", json={"url": "ada.example.com"}, headers=auth)
+    active = client.get("/api/jobs", params={"active": 1}, headers=auth).json()
+    assert active[0]["kind"] == "ingest" and active[0]["status"] == "queued"
+
+
+def test_worker_loop_runs_in_background(tmp_path, monkeypatch):
+    """The embedded worker thread (RUN_WORKER_IN_API) drains jobs without a second process."""
+    import threading
+
+    from sqlalchemy import create_engine
+    from sqlalchemy.orm import sessionmaker
+
+    from app import worker
+    from app.db import Base
+    from app.services.jobs import create_job
+
+    # A real file database: the in-memory test engine shares one connection across threads.
+    engine = create_engine(f"sqlite:///{tmp_path / 'w.db'}", connect_args={"check_same_thread": False})
+    Base.metadata.create_all(engine)
+    session_factory = sessionmaker(bind=engine, expire_on_commit=False)
+
+    with session_factory() as db:
+        ws = Workspace(name="w", owner_id=uuid.uuid4())
+        db.add(ws)
+        db.commit()
+        job = create_job(db, ws.id, "noop")
+    monkeypatch.setattr(worker, "SessionLocal", session_factory)
+    monkeypatch.setitem(worker.HANDLERS, "noop", lambda db, j: worker.Done({"ok": True}))
+    monkeypatch.setattr(worker, "PERIODIC", [])
+    real_run_job = worker.run_job
+    monkeypatch.setattr(worker, "run_job", lambda job_id: real_run_job(job_id, session_factory=session_factory))
+    stop = threading.Event()
+    thread = worker.start_background(stop)
+    deadline = time.time() + 5
+    status = None
+    while time.time() < deadline:
+        with session_factory() as db:
+            status = db.get(Job, job.id).status
+        if status == "done":
+            break
+        time.sleep(0.05)
+    stop.set()
+    thread.join(timeout=5)
+    assert status == "done"
+
+
+def test_artifact_ownership(client, auth, db_session):
+    other = Artifact(workspace_id=uuid.uuid4(), type="summary", status="ready", content_json={})
+    db_session.add(other)
+    db_session.commit()
+    assert client.get(f"/api/artifacts/{other.id}", headers=auth).status_code == 404

frontend/src/App.tsx

@@ -1,5 +1,5 @@
 import { Route, Routes } from "react-router-dom";
-import AppShell, { ComingSoon } from "./components/AppShell";
+import AppShell from "./components/AppShell";
 import { PublicFooter, PublicNav } from "./components/PublicChrome";
 import PricingTiers from "./components/PricingTiers";
 import SkyCanvas from "./components/SkyCanvas";
@@ -7,8 +7,19 @@ import AuthCallback from "./pages/AuthCallback";
 import AuthPage from "./pages/AuthPage";
 import { Blog, BlogPostPage } from "./pages/Blog";
 import Landing from "./pages/Landing";
+import Archive from "./pages/Archive";
+import Launchpad from "./pages/Launchpad";
+import LaunchKit from "./pages/LaunchKit";
 import MissionControl from "./pages/MissionControl";
+import Notebooks from "./pages/Notebooks";
 import NotebookView from "./pages/NotebookView";
+import Resurface from "./pages/Resurface";
+import Settings from "./pages/Settings";
+import Sources from "./pages/Sources";
+import Studio from "./pages/Studio";
+import TopicMap from "./pages/TopicMap";
+import VoiceProfile from "./pages/VoiceProfile";
+import Welcome from "./pages/Welcome";
 
 function PricingPage() {
   return (
@@ -34,19 +45,21 @@ export default function App() {
       <Route path="/blogs/:slug" element={<BlogPostPage />} />
       <Route path="/auth" element={<AuthPage />} />
       <Route path="/auth/callback" element={<AuthCallback />} />
+      <Route path="/welcome" element={<Welcome />} />
       <Route path="/app" element={<AppShell />}>
         <Route index element={<MissionControl />} />
-        <Route path="notebooks" element={<MissionControl />} />
+        <Route path="notebooks" element={<Notebooks />} />
         <Route path="notebooks/:id" element={<NotebookView />} />
-        <Route path="sources" element={<MissionControl />} />
-        <Route path="map" element={<ComingSoon title="Topic map" phase="Phase 2" />} />
-        <Route path="voice" element={<ComingSoon title="Voice profile" phase="Phase 1" />} />
-        <Route path="studio" element={<ComingSoon title="Video and audio studio" phase="Phase 2" />} />
-        <Route path="launch-kit" element={<ComingSoon title="Launch Kit" phase="Phase 2" />} />
-        <Route path="launchpad" element={<ComingSoon title="Launchpad" phase="Phase 3" />} />
-        <Route path="archive" element={<ComingSoon title="Archive" phase="Phase 2" />} />
-        <Route path="resurface" element={<ComingSoon title="Resurfacing" phase="Phase 3" />} />
-        <Route path="settings" element={<ComingSoon title="Settings and billing" phase="Phase 1" />} />
+        <Route path="sources" element={<Sources />} />
+        <Route path="map" element={<TopicMap />} />
+        <Route path="voice" element={<VoiceProfile />} />
+        <Route path="studio" element={<Studio />} />
+        <Route path="launch-kit" element={<LaunchKit />} />
+        <Route path="launchpad" element={<Launchpad />} />
+        <Route path="archive" element={<Archive />} />
+        <Route path="resurface" element={<Resurface />} />
+        <Route path="settings" element={<Settings />} />
+        <Route path="*" element={<MissionControl />} />
       </Route>
       <Route path="*" element={<Landing />} />
     </Routes>

frontend/src/api/auth.ts

@@ -35,4 +35,5 @@ export const authApi = {
   forgotComplete: (email: string, code: string, password: string) =>
     post<LoginResult>("/api/auth/password/forgot/complete", { email, code, password }),
   logout: () => post<{ ok: true }>("/api/auth/logout"),
+  deleteAccount: () => post<{ ok: true }>("/api/auth/delete-account"),
 };

frontend/src/api/client.ts

@@ -88,11 +88,53 @@ export async function api<T>(path: string, init: RequestInit = {}, retry = true)
 export const post = <T>(path: string, body?: unknown) =>
   api<T>(path, { method: "POST", body: body === undefined ? undefined : JSON.stringify(body) });
 
-/** Direct browser upload to Cloudflare R2 through a presigned PUT. */
+export const patch = <T>(path: string, body: unknown) => api<T>(path, { method: "PATCH", body: JSON.stringify(body) });
+
+export const put = <T>(path: string, body: unknown) => api<T>(path, { method: "PUT", body: JSON.stringify(body) });
+
+export const del = <T = { ok: true }>(path: string) => api<T>(path, { method: "DELETE" });
+
+/** Query string from an object, skipping empty values. */
+export function qs(params: Record<string, string | number | boolean | null | undefined>): string {
+  const q = new URLSearchParams();
+  for (const [k, v] of Object.entries(params)) if (v !== undefined && v !== null && v !== "") q.set(k, String(v));
+  const out = q.toString();
+  return out ? `?${out}` : "";
+}
+
+/** Direct browser upload to storage (R2, or the API's local disk store) through a presigned PUT. */
+const TYPES_BY_EXT: Record<string, string> = {
+  md: "text/markdown",
+  markdown: "text/markdown",
+  txt: "text/plain",
+  html: "text/html",
+  htm: "text/html",
+  pdf: "application/pdf",
+  mp3: "audio/mpeg",
+  m4a: "audio/mp4",
+  wav: "audio/wav",
+  webm: "audio/webm",
+  ogg: "audio/ogg",
+  png: "image/png",
+  jpg: "image/jpeg",
+  jpeg: "image/jpeg",
+  webp: "image/webp",
+  svg: "image/svg+xml",
+};
+
+/** Browsers leave File.type empty for some extensions (.md on Windows) and use vendor names for others. */
+function contentType(file: File): string {
+  const ext = file.name.split(".").pop()?.toLowerCase() ?? "";
+  const base = file.type.split(";")[0];
+  if (base === "audio/x-m4a") return "audio/mp4";
+  if (base === "audio/x-wav") return "audio/wav";
+  return base || TYPES_BY_EXT[ext] || "application/octet-stream";
+}
+
 export async function uploadFile(file: File): Promise<{ upload_id: string; key: string }> {
   const start = await post<{ upload_id: string; upload_url: string; headers: Record<string, string> }>(
     "/api/storage/uploads",
-    { filename: file.name, content_type: file.type, size_bytes: file.size },
+    { filename: file.name, content_type: contentType(file), size_bytes: file.size },
   );
   const put = await fetch(start.upload_url, { method: "PUT", headers: start.headers, body: file });
   if (!put.ok) throw new ApiError(put.status, "Upload to storage failed");

frontend/src/api/endpoints.ts

@@ -0,0 +1,161 @@
+import { api, del, patch, post, put, qs, uploadFile } from "./client";
+import type {
+  Artifact,
+  ArtifactType,
+  CalendarItem,
+  ChatMessage,
+  ChatSummary,
+  Doc,
+  DocDetail,
+  Job,
+  Notebook,
+  NotebookSummary,
+  Page,
+  Platform,
+  ResurfaceItem,
+  Settings,
+  SocialAccounts,
+  Source,
+  TopicDetail,
+  TopicMap,
+  Usage,
+  Voice,
+  VoiceProfileData,
+  VoiceState,
+} from "./types";
+
+type SourceJob = { source: Source; job: Job };
+
+export const sourcesApi = {
+  list: () => api<Source[]>("/api/sources"),
+  connect: (url: string) => post<SourceJob>("/api/sources", { url }),
+  importUrl: (url: string) => post<SourceJob>("/api/sources/url", { url }),
+  importFile: async (file: File) => {
+    const { upload_id } = await uploadFile(file);
+    return post<SourceJob>("/api/sources/upload", { upload_id });
+  },
+  sync: (id: string) => post<SourceJob>(`/api/sources/${id}/sync`),
+  remove: (id: string) => del<{ ok: true; removed_posts: number }>(`/api/sources/${id}`),
+};
+
+export const docsApi = {
+  list: (p: { q?: string; source_id?: string; limit?: number; offset?: number } = {}) =>
+    api<Page<Doc>>(`/api/documents${qs(p)}`),
+  get: (id: string) => api<DocDetail>(`/api/documents/${id}`),
+  remove: (id: string) => del(`/api/documents/${id}`),
+};
+
+export const notebooksApi = {
+  list: () => api<NotebookSummary[]>("/api/notebooks"),
+  get: (id: string) => api<Notebook>(`/api/notebooks/${id}`),
+  create: (title: string, document_ids: string[] = [], description?: string) =>
+    post<{ id: string; title: string; added: number }>("/api/notebooks", { title, document_ids, description }),
+  fromTopic: (topicId: string) => post<{ id: string; title: string }>(`/api/notebooks/from-topic/${topicId}`),
+  update: (id: string, body: { title?: string; description?: string }) => patch(`/api/notebooks/${id}`, body),
+  remove: (id: string) => del(`/api/notebooks/${id}`),
+  addDocs: (id: string, document_ids: string[]) =>
+    post<{ added: number }>(`/api/notebooks/${id}/documents`, { document_ids }),
+  removeDoc: (id: string, docId: string) => del(`/api/notebooks/${id}/documents/${docId}`),
+  artifacts: (id: string) => api<Artifact[]>(`/api/notebooks/${id}/artifacts`),
+  chats: (id: string) => api<ChatSummary[]>(`/api/notebooks/${id}/chats`),
+  messages: (chatId: string) => api<ChatMessage[]>(`/api/notebooks/chats/${chatId}/messages`),
+  removeChat: (chatId: string) => del(`/api/notebooks/chats/${chatId}`),
+};
+
+export type GenerateBody = {
+  type: ArtifactType;
+  notebook_id?: string;
+  document_id?: string;
+  format?: "deep_dive" | "brief" | "debate";
+  minutes?: number;
+  style?: "short" | "explainer" | "audiogram";
+  audio_artifact_id?: string;
+  slides?: { heading: string; body: string }[];
+  parent_id?: string;
+};
+
+export const artifactsApi = {
+  list: (p: { type?: string; notebook_id?: string; document_id?: string; status?: string; q?: string;
+              limit?: number; offset?: number } = {}) => api<Page<Artifact>>(`/api/artifacts${qs(p)}`),
+  get: (id: string) => api<Artifact>(`/api/artifacts/${id}`),
+  generate: (body: GenerateBody) => post<Artifact>("/api/artifacts/generate", body),
+  update: (id: string, body: { title?: string; content?: Record<string, unknown> }) =>
+    patch<Artifact>(`/api/artifacts/${id}`, body),
+  remove: (id: string) => del(`/api/artifacts/${id}`),
+  retry: (id: string) => post<Artifact>(`/api/artifacts/${id}/retry`),
+};
+
+export const jobsApi = {
+  active: () => api<Job[]>("/api/jobs?active=1"),
+  recent: (kind?: string) => api<Job[]>(`/api/jobs${qs({ kind, limit: 10 })}`),
+  get: (id: string) => api<Job>(`/api/jobs/${id}`),
+};
+
+export const topicsApi = {
+  map: () => api<TopicMap>("/api/topics"),
+  get: (id: string) => api<TopicDetail>(`/api/topics/${id}`),
+  rebuild: (full = false) => post<Job>(`/api/topics/rebuild${qs({ full })}`),
+};
+
+export const voiceApi = {
+  get: () => api<VoiceState>("/api/voice"),
+  update: (body: { profile?: VoiceProfileData; host_voices?: { host_a: string; host_b: string } }) =>
+    put<VoiceState>("/api/voice", body),
+  build: (document_ids: string[]) => post<Job>("/api/voice/build", { document_ids }),
+  voices: () => api<Voice[]>("/api/voice/voices"),
+  consent: async (file: File, consent_text: string) => {
+    const { upload_id } = await uploadFile(file);
+    return post<VoiceState & { job: Job }>("/api/voice/consent", { upload_id, agreed: true, consent_text });
+  },
+  revoke: () => del<VoiceState>("/api/voice/consent"),
+};
+
+export type ItemBody = {
+  platform: Platform;
+  scheduled_at: string;
+  content: string;
+  thread?: string[];
+  artifact_id?: string | null;
+  document_id?: string | null;
+  social_account_id?: string | null;
+  remind_by_email?: boolean;
+  draft?: boolean;
+};
+
+export const launchpadApi = {
+  items: (p: { start?: string; end?: string; status?: string } = {}) => api<CalendarItem[]>(`/api/calendar${qs(p)}`),
+  create: (body: ItemBody) => post<CalendarItem>("/api/calendar", body),
+  update: (id: string, body: Partial<ItemBody> & { status?: "scheduled" | "draft" }) =>
+    patch<CalendarItem>(`/api/calendar/${id}`, body),
+  remove: (id: string) => del(`/api/calendar/${id}`),
+  publishNow: (id: string) => post<CalendarItem>(`/api/calendar/${id}/publish`),
+  accounts: () => api<SocialAccounts>("/api/social/accounts"),
+  connectBluesky: (handle: string, app_password: string) =>
+    post<SocialAccounts>("/api/social/bluesky", { handle, app_password }),
+  startOAuth: (platform: "x" | "linkedin") => post<{ url: string }>(`/api/social/${platform}/start`),
+  completeOAuth: (ticket: string) => post<SocialAccounts>("/api/social/complete", { ticket }),
+  disconnect: (id: string) => del(`/api/social/accounts/${id}`),
+};
+
+export const resurfaceApi = {
+  get: () => api<{ evergreen: ResurfaceItem[]; on_this_day: ResurfaceItem[]; unscored: number; total_posts: number }>(
+    "/api/resurface",
+  ),
+  scan: () => post<Job>("/api/resurface/scan"),
+};
+
+export type SettingsPatch = {
+  name?: string;
+  workspace_name?: string;
+  brand_name?: string;
+  brand_accent?: string;
+  logo_upload_id?: string;
+  training_opt_in?: boolean;
+  email_unsubscribed?: boolean;
+};
+
+export const settingsApi = {
+  get: () => api<Settings>("/api/settings"),
+  update: (body: SettingsPatch) => patch<Settings>("/api/settings", body),
+  usage: () => api<Usage>("/api/settings/usage"),
+};

frontend/src/api/types.ts

@@ -0,0 +1,230 @@
+export type JobStatus = "queued" | "running" | "done" | "failed";
+
+export type Job = {
+  id: string;
+  kind: string;
+  params: Record<string, unknown>;
+  status: JobStatus;
+  progress: number;
+  message: string | null;
+  error: string | null;
+  result: Record<string, unknown>;
+  artifact_id: string | null;
+  attempts: number;
+  created_at: string | null;
+  finished_at: string | null;
+};
+
+export type Source = {
+  id: string;
+  feed_url: string;
+  site_url: string | null;
+  platform: string;
+  title: string | null;
+  sync_status: "pending" | "syncing" | "ok" | "error";
+  sync_error: string | null;
+  last_synced_at: string | null;
+  document_count: number;
+  is_imports: boolean;
+};
+
+export type Doc = {
+  id: string;
+  title: string;
+  url: string;
+  path: string | null;
+  source_id: string | null;
+  source_title: string | null;
+  published_at: string | null;
+  words: number;
+  evergreen_score: number | null;
+};
+
+export type DocDetail = Doc & { lines: string[] };
+
+export type Page<T> = { total: number; items: T[] };
+
+export type NotebookSummary = {
+  id: string;
+  title: string;
+  description: string | null;
+  summary: string | null;
+  document_count: number;
+  chat_count: number;
+  artifact_count: number;
+  updated_at: string | null;
+};
+
+export type Notebook = {
+  id: string;
+  title: string;
+  description: string | null;
+  summary: string | null;
+  documents: Doc[];
+};
+
+export type Citation = {
+  marker: number;
+  document_id?: string;
+  title: string;
+  url: string;
+  path: string;
+  line_start: number;
+  line_end: number;
+  span: string;
+};
+
+export type ChatSummary = { id: string; title: string | null; updated_at: string | null };
+export type ChatMessage = { id: string; role: "user" | "assistant"; text: string; citations: Citation[] };
+
+export type ArtifactType = "summary" | "audio_overview" | "video" | "quote_card" | "carousel" | "launch_kit";
+
+export type SourceRef = { path: string; line_start: number; line_end: number; title?: string | null; quote?: string };
+
+export type Segment = { speaker: string; text: string; start: number; end: number; sources: SourceRef[] };
+
+export type LaunchKitContent = {
+  title: string;
+  post_title: string;
+  post_url: string;
+  claims: { claim: string; sources: SourceRef[] }[];
+  hooks: { text: string; strength: number; rationale: string }[];
+  posts: Record<"x_thread" | "linkedin" | "substack_notes" | "bluesky", string[]>;
+  seo: {
+    title_options?: string[];
+    meta_description?: string;
+    slug?: string;
+    keywords?: string[];
+    internal_links?: { path: string; anchor_text: string; reason: string }[];
+  };
+  carousel: { heading: string; body: string }[];
+  quotes: { quote: string; source: SourceRef }[];
+  quote_card_ids?: string[];
+};
+
+export type Artifact = {
+  id: string;
+  type: ArtifactType;
+  type_label: string;
+  title: string;
+  status: "pending" | "generating" | "rendering" | "ready" | "failed" | "draft";
+  notebook_id: string | null;
+  document_id: string | null;
+  content: Record<string, any>;
+  storage_key: string | null;
+  url: string | null;
+  download_url: string | null;
+  slide_urls: string[];
+  created_at: string | null;
+  job: Job | null;
+};
+
+export type TopicNode = { id: string; name: string; summary: string | null; post_count: number };
+export type TopicMap = {
+  nodes: TopicNode[];
+  edges: { source: string; target: string; weight: number }[];
+  has_untagged_posts: boolean;
+};
+export type TopicDetail = TopicNode & { posts: (Doc & { weight: number })[] };
+
+export type VoiceProfileData = {
+  tone: string[];
+  sentence_length: "short" | "medium" | "long" | "varied";
+  vocabulary: string[];
+  structure_habits: string[];
+  openings: string[];
+  avoid: string[];
+  summary: string;
+};
+
+export type VoiceState = {
+  profile: VoiceProfileData | null;
+  sample_doc_ids: string[];
+  updated_at: string | null;
+  host_voices: { host_a: string; host_b: string };
+  clone: {
+    allowed: boolean;
+    consent_text: string;
+    status: "none" | "processing" | "ready";
+    voice_id: string | null;
+    created_at: string | null;
+  };
+  tts_configured: boolean;
+};
+
+export type Voice = { voice_id: string; name: string; category?: string | null; preview_url?: string | null };
+
+export type Platform = "x" | "linkedin" | "bluesky" | "substack_notes";
+
+export type CalendarItem = {
+  id: string;
+  platform: Platform;
+  platform_label: string;
+  scheduled_at: string;
+  status: "scheduled" | "publishing" | "posted" | "reminded" | "failed" | "draft";
+  content: string;
+  thread: string[];
+  artifact_id: string | null;
+  document_id: string | null;
+  social_account_id: string | null;
+  account_handle: string | null;
+  remind_by_email: boolean;
+  auto_post: boolean;
+  external_url: string | null;
+  posted_at: string | null;
+  error: string | null;
+  metrics: Record<string, number | string>;
+  clicks: number;
+};
+
+export type SocialAccount = {
+  id: string;
+  platform: "x" | "linkedin" | "bluesky";
+  platform_label: string;
+  handle: string;
+  status: string;
+  avatar_url: string | null;
+  connected_at: string | null;
+};
+
+export type SocialAccounts = {
+  accounts: SocialAccount[];
+  available: { x: boolean; linkedin: boolean; bluesky: boolean };
+  redirect_uris: { x: string; linkedin: string };
+};
+
+export type ResurfaceItem = Doc & {
+  reason: string | null;
+  angle: string | null;
+  last_resurfaced_at: string | null;
+  rank?: number;
+  years_ago?: number;
+};
+
+export type Plan = {
+  id: string;
+  name: string;
+  sources: number;
+  indexed_posts: number;
+  audio_minutes: number;
+  video_minutes: number;
+  launch_kits: number;
+  voice_cloning: boolean;
+  brand_kit: boolean;
+};
+
+export type Usage = {
+  plan: string;
+  used: { audio_minutes: number; video_minutes: number; launch_kits: number; llm_tokens: number; since: string };
+  limits: { audio_minutes: number; video_minutes: number; launch_kits: number; sources: number; indexed_posts: number };
+};
+
+export type Settings = {
+  user: { name: string | null; email: string; auth_provider: string; email_unsubscribed: boolean };
+  workspace: { id: string; name: string; training_opt_in: boolean };
+  brand: { name: string | null; accent: string; logo_url: string | null };
+  plan: Plan;
+  billing_enabled: boolean;
+  usage: Usage;
+  integrations: { llm: boolean; elevenlabs: boolean; x: boolean; linkedin: boolean; storage: string; renderer: string };
+};

frontend/src/components/AppShell.tsx

@@ -1,3 +1,4 @@
+import { useState } from "react";
 import { NavLink, Outlet, Navigate, useLocation } from "react-router-dom";
 import { useAuth } from "../hooks/useAuth";
 import Logo from "./Logo";
@@ -31,9 +32,29 @@ const NAV = [
   { to: "/app/settings", label: "Settings", icon: OrbitIcon },
 ];
 
+const COLLAPSED_KEY = "ns_sidebar_collapsed";
+
+function readCollapsed(): boolean {
+  try {
+    return localStorage.getItem(COLLAPSED_KEY) === "1";
+  } catch {
+    return false;
+  }
+}
+
 export default function AppShell() {
   const { user, loading, logout } = useAuth();
   const location = useLocation();
+  const [collapsed, setCollapsed] = useState(readCollapsed);
+  const toggle = () =>
+    setCollapsed((c) => {
+      try {
+        localStorage.setItem(COLLAPSED_KEY, c ? "0" : "1");
+      } catch {
+        /* private mode: remember for this visit only */
+      }
+      return !c;
+    });
   if (loading) return <div className="boot"><div className="orbit-loader"><span /></div></div>;
   if (!user) {
     const next = new URLSearchParams({ next: location.pathname + location.search });
@@ -41,23 +62,39 @@ export default function AppShell() {
   }
 
   return (
-    <div className="shell">
+    <div className={`shell${collapsed ? " collapsed" : ""}`}>
       <SkyCanvas intensity={0.35} />
       <aside className="sidebar">
-        <NavLink to="/" className="sidebar-brand">
-          <Logo size={28} />
-        </NavLink>
+        <div className="sidebar-top">
+          <NavLink to="/" className="sidebar-brand">
+            <Logo size={28} />
+          </NavLink>
+          <button
+            type="button"
+            className="icon-btn sidebar-toggle"
+            onClick={toggle}
+            aria-label={collapsed ? "Expand sidebar" : "Collapse sidebar"}
+            aria-expanded={!collapsed}
+            title={collapsed ? "Expand sidebar" : "Collapse sidebar"}
+          >
+            <svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true">
+              <rect x="3.5" y="4.5" width="17" height="15" rx="2.5" />
+              <path d="M9 4.5v15" />
+              <path d={collapsed ? "M13 10l2 2-2 2" : "M15 10l-2 2 2 2"} />
+            </svg>
+          </button>
+        </div>
         <nav aria-label="App">
           {NAV.map(({ to, label, icon: Icon, end }) => (
-            <NavLink key={to} to={to} end={end} className={({ isActive }) => `side-link${isActive ? " active" : ""}`}>
+            <NavLink key={to} to={to} end={end} title={collapsed ? label : undefined} className={({ isActive }) => `side-link${isActive ? " active" : ""}`}>
               <Icon />
-              <span>{label}</span>
+              <span className="side-label">{label}</span>
             </NavLink>
           ))}
         </nav>
         <div className="sidebar-user">
-          <span className="mono muted">{user.email}</span>
-          <button className="link-btn" onClick={logout}>
+          <span className="mono muted side-label">{user.email}</span>
+          <button className="link-btn" onClick={logout} title="Sign out">
             Sign out
           </button>
         </div>
@@ -68,25 +105,3 @@ export default function AppShell() {
     </div>
   );
 }
-
-export function ComingSoon({ title, phase }: { title: string; phase: string }) {
-  return (
-    <div className="empty-state">
-      <svg width="120" height="90" viewBox="0 0 120 90" fill="none" aria-hidden="true" className="empty-art">
-        <circle cx="84" cy="30" r="16" stroke="currentColor" strokeWidth="1.5" />
-        <circle cx="78" cy="26" r="3" stroke="currentColor" strokeWidth="1.5" opacity="0.5" />
-        <circle cx="90" cy="36" r="2" stroke="currentColor" strokeWidth="1.5" opacity="0.5" />
-        <path d="M20 70l14-8 6 10-14 8z" stroke="currentColor" strokeWidth="1.5" />
-        <path d="M34 62l10-6M26 80l-6 6" stroke="currentColor" strokeWidth="1.5" />
-        <circle cx="12" cy="20" r="1" fill="currentColor" />
-        <circle cx="50" cy="12" r="1" fill="currentColor" />
-        <circle cx="108" cy="72" r="1" fill="currentColor" />
-      </svg>
-      <h1>{title}</h1>
-      <p className="muted">This part of the station comes online in {phase}.</p>
-      <NavLink to="/app" className="btn">
-        Back to Mission Control
-      </NavLink>
-    </div>
-  );
-}