diff --git a/.env.example b/.env.example index a7c8cb8cc2f591eed585343731678058bfcab8e6..ba72b741b657a4ebb97cd250d8df7da0f775a8c2 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,13 @@ JWT_SECRET=replace-with-64-random-chars INTERNAL_TOKEN=replace-with-random-token BILLING_ENABLED=false +# ---- Database (also backs the job queue) +# Local: a SQLite file, created by `alembic upgrade head` run from backend/. +# Prod (Neon, Supabase, RDS...): Postgres with the psycopg driver, e.g. +# DATABASE_URL=postgresql+psycopg://user:pass@host:5432/dbname?sslmode=require +# docker-compose ignores this and points at its own Postgres container. +DATABASE_URL=sqlite:///./notestack.db + # ---- Google sign in (OAuth client, type Web; add http://localhost:5173 to authorized JS origins) # Client ID only: popup sign in (Google Identity Services), no redirect URIs needed. # Client ID + secret: redirect flow with server side code exchange. Add the redirect URI @@ -23,6 +30,13 @@ FROM_EMAIL=Notestack ALERTS_EMAIL=arslan@firebird-technologies.com UNSUBSCRIBE_SECRET=replace-me +# ---- Local dev conveniences +# The API runs the job worker in process when ENV=development (set false to use python -m app.worker). +RUN_WORKER_IN_API= +# auto = local disk (LOCAL_STORAGE_DIR) when no R2 account or endpoint is set, else R2 +STORAGE_BACKEND=auto +LOCAL_STORAGE_DIR=.storage + # ---- Storage: Cloudflare R2 # Prod: set account id + keys, leave R2_ENDPOINT_URL empty. # Local: docker-compose points R2_ENDPOINT_URL at MinIO (keys minioadmin/minioadmin). @@ -53,5 +67,21 @@ WORKER_CONCURRENCY=4 WORKER_POLL_SECONDS=1 JOB_STALE_SECONDS=900 -# ---- ElevenLabs (Phase 1 audio) +# ---- ElevenLabs (audio overviews, video narration, voice cloning) ELEVENLABS_API_KEY= +ELEVENLABS_MODEL=eleven_multilingual_v2 +ELEVENLABS_VOICE_A=21m00Tcm4TlvDq8ikWAM +ELEVENLABS_VOICE_B=pNInz6obpgDQGcFMaJgB + +# ---- Launchpad social publishing +# Bluesky needs nothing here (users connect with an app password). +# X: developer app with OAuth 2.0, type Web App, scopes tweet.read tweet.write users.read offline.access. +# Callback URI: {API_URL}/api/social/x/callback e.g. http://localhost:8000/api/social/x/callback +# LinkedIn: app with the "Sign In with LinkedIn using OpenID Connect" and "Share on LinkedIn" products. +# Redirect URL: {API_URL}/api/social/linkedin/callback +X_CLIENT_ID= +X_CLIENT_SECRET= +LINKEDIN_CLIENT_ID= +LINKEDIN_CLIENT_SECRET= +# Fernet key for stored social tokens; derived from JWT_SECRET when empty. +SOCIAL_TOKEN_KEY= diff --git a/.gitignore b/.gitignore index 53fc86656f7b0d96f606122f52062d0b9ea8caae..eaa7ea3f71f9582c5195edbd8ac67680ebdeee95 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,11 @@ __pycache__/ out/ renderer/build/ .corpus-cache/ + +# Local SQLite database +*.db +*.db-wal +*.db-shm + +# Local object storage (used when no R2 is configured) +.storage/ diff --git a/README.md b/README.md index b5013ec969f695597175475838454a4f6ae2052a..04af9d421ca8954f756a7328ce934b959b092576 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Your archive, in orbit. Turn a Substack (or any feed) into a grounded research n overviews, videos and launch kits, all traced back to what the writer actually wrote. - Design doc: [docs/DESIGN.md](docs/DESIGN.md) -- Phase 0 task list: [docs/PHASE0.md](docs/PHASE0.md) +- Foundations task list: [docs/PHASE0.md](docs/PHASE0.md) ## Layout @@ -27,16 +27,41 @@ docker compose up --build `127.0.0.1 minio` to your hosts file for browser uploads to work locally. - With `EMAIL_PROVIDER=console`, verification codes print in the `api` logs. -Without Docker: +Without Docker (one API process does everything): ```bash cd backend && python -m venv .venv && .venv/Scripts/pip install -r requirements-dev.txt -alembic upgrade head && uvicorn app.main:app --reload # needs Postgres only -python -m app.worker # second terminal +alembic upgrade head && uvicorn app.main:app --reload # SQLite at backend/notestack.db cd frontend && npm install && npm run dev -cd renderer && npm install && npm start # or: npm run studio +cd renderer && npm install && npm start # only needed for videos, quote cards, carousels ``` +- With `ENV=development` the API runs the job worker in process (`RUN_WORKER_IN_API`), so syncing, + topic mapping, audio and launch kits work without a second terminal. In production run + `python -m app.worker` separately and set `RUN_WORKER_IN_API=false`. +- With no R2 settings, files live on local disk under `backend/.storage` and are served through + signed `/api/storage/local/...` URLs (range requests supported for audio and video seeking). +- Audio needs `ELEVENLABS_API_KEY`; everything text based needs `LLM_API_KEY`. +- Launchpad: Bluesky connects with an app password. X and LinkedIn need developer apps; the + redirect URIs are listed in `.env.example` and on the Launchpad page. Substack Notes has no API, + so those posts (and any post without a connected account) arrive as an email reminder at send time. + +## What is in the app + +| Menu | Does | +| --- | --- | +| Mission Control | Pre-flight checklist, jobs in flight, sources, notebooks, recent launches, upcoming posts | +| Notebooks | Group posts, chat with cited answers (history kept), summaries, audio overviews, videos, quote cards | +| Sources | Connect feeds (Substack archive included), import one URL, upload md/txt/html/pdf, read posts by line | +| Topic map | LLM tagged topics per post, merged into a constellation; make a notebook from any topic | +| Voice profile | Writing voice built from your posts (editable), host voices, consented voice cloning | +| Video and audio | Audio overviews, 9:16 shorts, 16:9 explainers, 1:1 audiograms, quote cards | +| Launch Kit | Hooks, X thread, LinkedIn, Substack Notes, Bluesky, SEO pack, carousel, quote cards, all editable | +| Launchpad | Calendar and agenda, auto posting to X, LinkedIn and Bluesky, email reminders, tracked links, engagement | +| Archive | Every generated artifact with filters, players, downloads, retry and delete | +| Resurfacing | Evergreen scores and reshare angles for old posts, on this day, straight into a kit or the calendar | +| Settings | Profile, workspace, brand kit, privacy, plan usage meters, connections, sign out everywhere, delete | + ## Key choices | Concern | Where | diff --git a/backend/alembic/env.py b/backend/alembic/env.py index 91c9973b74237303de9f3ed7b265fc56953234cf..337ee007343f3f928e66ec59d62ae8da3072b3de 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -24,7 +24,9 @@ def run_migrations_online() -> None: connectable = engine_from_config(config.get_section(config.config_ini_section), prefix="sqlalchemy.", poolclass=pool.NullPool) with connectable.connect() as connection: - context.configure(connection=connection, target_metadata=target_metadata) + # Batch mode rebuilds tables for ALTERs that SQLite cannot do in place (local dev only). + context.configure(connection=connection, target_metadata=target_metadata, + render_as_batch=connection.dialect.name == "sqlite") with context.begin_transaction(): context.run_migrations() diff --git a/backend/alembic/versions/0002_build_out.py b/backend/alembic/versions/0002_build_out.py new file mode 100644 index 0000000000000000000000000000000000000000..70d6aebdbcd2ad5a00a99a22cd839a739fcc5526 --- /dev/null +++ b/backend/alembic/versions/0002_build_out.py @@ -0,0 +1,94 @@ +"""build out: social accounts, topics, launchpad fields, resurfacing, host voices + +Revision ID: 0002_build_out +Revises: 0001_initial +Create Date: 2026-09-27 +""" +import sqlalchemy as sa +from alembic import op + +revision = "0002_build_out" +down_revision = "0001_initial" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # 0001 builds from the live models, so a fresh database already has everything below. + # Only apply what is missing, which makes this safe on both fresh and existing databases. + insp = sa.inspect(op.get_bind()) + tables = set(insp.get_table_names()) + if "social_accounts" in tables and "evergreen_score" in {c["name"] for c in insp.get_columns("documents")}: + return + op.create_table( + "social_accounts", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("workspace_id", sa.Uuid(), sa.ForeignKey("workspaces.id", ondelete="CASCADE"), nullable=False, index=True), + sa.Column("platform", sa.String(20), nullable=False), + sa.Column("handle", sa.String(200), nullable=False), + sa.Column("external_id", sa.String(200), nullable=False), + sa.Column("access_token", sa.Text(), nullable=False), + sa.Column("refresh_token", sa.Text()), + sa.Column("expires_at", sa.DateTime(timezone=True)), + sa.Column("scopes", sa.String(500)), + sa.Column("status", sa.String(20), nullable=False, server_default="active"), + sa.Column("avatar_url", sa.String(1000)), + sa.UniqueConstraint("workspace_id", "platform", "external_id"), + ) + op.create_table( + "topics", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("workspace_id", sa.Uuid(), sa.ForeignKey("workspaces.id", ondelete="CASCADE"), nullable=False, index=True), + sa.Column("name", sa.String(200), nullable=False), + sa.Column("slug", sa.String(200), nullable=False), + sa.Column("summary", sa.Text()), + sa.Column("post_count", sa.Integer(), nullable=False, server_default="0"), + sa.UniqueConstraint("workspace_id", "slug"), + ) + op.create_table( + "document_topics", + sa.Column("document_id", sa.Uuid(), sa.ForeignKey("documents.id", ondelete="CASCADE"), primary_key=True), + sa.Column("topic_id", sa.Uuid(), sa.ForeignKey("topics.id", ondelete="CASCADE"), primary_key=True), + sa.Column("weight", sa.Float(), nullable=False, server_default="1"), + ) + with op.batch_alter_table("documents") as b: + b.add_column(sa.Column("evergreen_score", sa.Float())) + b.add_column(sa.Column("last_resurfaced_at", sa.DateTime(timezone=True))) + with op.batch_alter_table("voice_profiles") as b: + b.add_column(sa.Column("host_voices", sa.JSON(), nullable=False, server_default="{}")) + with op.batch_alter_table("calendar_items") as b: + b.alter_column("artifact_id", existing_type=sa.Uuid(), nullable=True) + b.add_column(sa.Column("document_id", sa.Uuid(), sa.ForeignKey( + "documents.id", name="fk_calendar_items_document_id", ondelete="SET NULL"))) + b.add_column(sa.Column("content", sa.Text(), nullable=False, server_default="")) + b.add_column(sa.Column("thread", sa.JSON(), nullable=False, server_default="[]")) + b.add_column(sa.Column("social_account_id", sa.Uuid(), sa.ForeignKey( + "social_accounts.id", name="fk_calendar_items_social_account_id", ondelete="SET NULL"))) + b.add_column(sa.Column("remind_by_email", sa.Boolean(), nullable=False, server_default=sa.false())) + b.add_column(sa.Column("external_id", sa.String(200))) + b.add_column(sa.Column("external_url", sa.String(1000))) + b.add_column(sa.Column("posted_at", sa.DateTime(timezone=True))) + b.add_column(sa.Column("error", sa.Text())) + b.add_column(sa.Column("attempts", sa.Integer(), nullable=False, server_default="0")) + b.add_column(sa.Column("metrics", sa.JSON(), nullable=False, server_default="{}")) + b.create_index("ix_calendar_items_scheduled_at", ["scheduled_at"]) + + +def downgrade() -> None: + with op.batch_alter_table("calendar_items") as b: + b.drop_index("ix_calendar_items_scheduled_at") + for col in ("metrics", "attempts", "error", "posted_at", "external_url", "external_id", "remind_by_email", + "social_account_id", "thread", "content", "document_id"): + b.drop_column(col) + with op.batch_alter_table("voice_profiles") as b: + b.drop_column("host_voices") + with op.batch_alter_table("documents") as b: + b.drop_column("last_resurfaced_at") + b.drop_column("evergreen_score") + op.drop_table("document_topics") + op.drop_table("topics") + op.drop_table("social_accounts") diff --git a/backend/app/config.py b/backend/app/config.py index 348c8aa1f8340243499dea7cc6bd3e368d68f749..5a2a75ce86fdd484641d996dacfc95b05bec9fbd 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -17,12 +17,17 @@ class Settings(BaseSettings): api_url: str = "http://localhost:8000" cors_origins: str = "http://localhost:5173" - database_url: str = "postgresql+psycopg://notestack:notestack@localhost:5432/notestack" + # Local default is a SQLite file (relative to the working directory, normally backend/). + # Prod and docker-compose use Postgres: postgresql+psycopg://user:pass@host:5432/db + database_url: str = "sqlite:///./notestack.db" # Worker (Postgres backed queue, see app/worker.py) worker_concurrency: int = 4 worker_poll_seconds: float = 1.0 job_stale_seconds: int = 900 # a running job with no heartbeat for this long is retried + # Run the worker loop inside the API process (local dev: one command, no second terminal). + # None = on in development, off elsewhere. + run_worker_in_api: bool | None = None # Auth jwt_secret: str = "change-me" @@ -43,7 +48,9 @@ class Settings(BaseSettings): unsubscribe_secret: str = "change-me-too" update_email_send_hour: int = 9 - # Storage (Cloudflare R2, S3 compatible) + # Storage: auto = local disk when no R2 account or endpoint is set, else R2 (S3 compatible) + storage_backend: str = "auto" # auto | r2 | local + local_storage_dir: str = ".storage" r2_account_id: str = "" r2_access_key_id: str = "" r2_secret_access_key: str = "" @@ -68,8 +75,18 @@ class Settings(BaseSettings): corpus_cache_dir: str = ".corpus-cache" research_max_steps: int = 10 - # ElevenLabs + # ElevenLabs (premade voices by default; a consented clone can replace host A) elevenlabs_api_key: str = "" + elevenlabs_model: str = "eleven_multilingual_v2" + elevenlabs_voice_a: str = "21m00Tcm4TlvDq8ikWAM" # Rachel + elevenlabs_voice_b: str = "pNInz6obpgDQGcFMaJgB" # Adam + + # Social publishing (Launchpad). Redirect URIs: {API_URL}/api/social/{x,linkedin}/callback + social_token_key: str = "" # Fernet key; derived from JWT_SECRET when empty + x_client_id: str = "" + x_client_secret: str = "" + linkedin_client_id: str = "" + linkedin_client_secret: str = "" # Renderer renderer_url: str = "http://localhost:3100" @@ -78,6 +95,16 @@ class Settings(BaseSettings): # Billing billing_enabled: bool = False + @property + def worker_in_api(self) -> bool: + return self.env == "development" if self.run_worker_in_api is None else self.run_worker_in_api + + @property + def use_local_storage(self) -> bool: + if self.storage_backend != "auto": + return self.storage_backend == "local" + return not (self.r2_account_id or self.r2_endpoint_url) + @property def r2_endpoint(self) -> str: if self.r2_endpoint_url: diff --git a/backend/app/db.py b/backend/app/db.py index 9806460568ad635b72853c9ddc20622326554d6b..3c1480edbdab906ca7052c0292def6f1fbc038f4 100644 --- a/backend/app/db.py +++ b/backend/app/db.py @@ -1,6 +1,6 @@ from collections.abc import Iterator -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker from app.config import settings @@ -10,8 +10,20 @@ class Base(DeclarativeBase): pass -_connect_args = {"check_same_thread": False} if settings.database_url.startswith("sqlite") else {} +_sqlite = settings.database_url.startswith("sqlite") +_connect_args = {"check_same_thread": False, "timeout": 30} if _sqlite else {} engine = create_engine(settings.database_url, pool_pre_ping=True, connect_args=_connect_args) + +if _sqlite: + # Local dev: the API and the worker share one file. WAL lets reads run during a write, the busy + # timeout makes a second writer wait instead of failing, and SQLite needs FKs switched on. + @event.listens_for(engine, "connect") + def _sqlite_pragmas(dbapi_conn, _record): + cur = dbapi_conn.cursor() + cur.execute("PRAGMA journal_mode=WAL") + cur.execute("PRAGMA busy_timeout=30000") + cur.execute("PRAGMA foreign_keys=ON") + cur.close() SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False) diff --git a/backend/app/llm/run.py b/backend/app/llm/run.py new file mode 100644 index 0000000000000000000000000000000000000000..08501c475d0484e6d64469c9d676e1bfeb76c3eb --- /dev/null +++ b/backend/app/llm/run.py @@ -0,0 +1,48 @@ +"""One door for every generation call: runs the signature, meters tokens, writes a trace, and strips +em dashes from the outputs. Tests replace `predict` to return canned outputs.""" + +import uuid + +import dspy +from sqlalchemy.orm import Session + +from app.config import settings +from app.llm.postprocess import clean_output +from app.llm.provider import main_lm, provider_name, track_usage +from app.models import DspyTrace, Job, Workspace +from app.services.jobs import record_usage + + +def _dump(value): + if hasattr(value, "model_dump"): + return value.model_dump() + if isinstance(value, list): + return [_dump(v) for v in value] + return value + + +def predict( + signature: type[dspy.Signature], + *, + db: Session, + workspace_id: uuid.UUID, + job: Job | None = None, + lm: dspy.LM | None = None, + **inputs, +) -> dict: + """Run `signature` and return its outputs as plain dicts/lists (pydantic models dumped).""" + lm = lm or main_lm() + with track_usage(lm) as usage, dspy.context(lm=lm): + pred = dspy.Predict(signature)(**inputs) + outputs = {name: clean_output(_dump(pred[name])) for name in signature.output_fields} + if usage.total: + record_usage(db, workspace_id=workspace_id, kind="llm", provider=provider_name(), model=settings.llm_model, + quantity=usage.total, unit="tokens", cost_usd=usage.cost_usd, job=job) + workspace = db.get(Workspace, workspace_id) + db.add(DspyTrace( + workspace_id=workspace_id, module=signature.__name__, model=lm.model, + inputs={k: (v if len(str(v)) < 4000 else str(v)[:4000]) for k, v in inputs.items()}, + outputs=outputs, training_allowed=bool(workspace and workspace.training_opt_in), + )) + db.commit() + return outputs diff --git a/backend/app/llm/signatures.py b/backend/app/llm/signatures.py index 5a882cb2265a6133b6a10cebc7303c97c155b31e..f4a0e07b8acd0a39d9b19e1b0f5be85503d6b933 100644 --- a/backend/app/llm/signatures.py +++ b/backend/app/llm/signatures.py @@ -1,5 +1,5 @@ -"""Typed DSPy signatures with Pydantic outputs. Phase 1 modules are implemented; later ones are -declared now so the output contract is fixed early.""" +"""Typed DSPy signatures with Pydantic outputs. Every module that produces facts cites SourceRefs, +which are verified against the corpus before anything reaches the writer.""" from typing import Literal @@ -61,6 +61,41 @@ class Hook(BaseModel): rationale: str +class TopicTag(BaseModel): + name: str = Field(description="Short topic name, 1 to 4 words, Title Case") + weight: float = Field(ge=0, le=1, description="How central the topic is to the post") + + +class TopicGroup(BaseModel): + canonical: str = Field(description="The best name for the merged topic") + members: list[str] = Field(description="Every input name that means the same topic") + summary: str = Field(description="One sentence on what the writer says about it") + + +class InternalLink(BaseModel): + path: str + anchor_text: str + reason: str + + +class SeoPackOut(BaseModel): + title_options: list[str] + meta_description: str = Field(description="At most 155 characters") + slug: str + keywords: list[str] + internal_links: list[InternalLink] = Field(default_factory=list) + + +class Slide(BaseModel): + heading: str + body: str + + +class QuotePick(BaseModel): + quote: str = Field(description="Exact words from the passage, at most 220 characters") + source: SourceRef + + # Signatures @@ -146,3 +181,71 @@ class PlatformAdapter(dspy.Signature): platform: Literal["x_thread", "linkedin", "substack_notes", "bluesky"] = dspy.InputField() platform_rules: str = dspy.InputField() posts: list[str] = dspy.OutputField() + + +class SummarizeNotebook(dspy.Signature): + """Summarize what these posts say as a whole: the main arguments, how they connect and where the + writer changed their mind. Every factual sentence ends with a [n] marker and a citation to the + lines it came from. Plain, warm prose. Never use em dashes.""" + + title: str = dspy.InputField() + passages: list[str] = dspy.InputField(desc="Posts with path and numbered lines") + summary: str = dspy.OutputField(desc="3 to 6 paragraphs with [n] markers") + themes: list[str] = dspy.OutputField(desc="3 to 7 short theme names") + citations: list[FileCitation] = dspy.OutputField() + + +class ExtractTopics(dspy.Signature): + """Name the topics this post is about, the way a reader would search for them.""" + + title: str = dspy.InputField() + text: str = dspy.InputField() + known_topics: list[str] = dspy.InputField(desc="Reuse one of these names when it fits") + topics: list[TopicTag] = dspy.OutputField(desc="3 to 8 topics") + + +class ConsolidateTopics(dspy.Signature): + """Merge topic names that mean the same thing (plural/singular, synonyms, broader wording) and + write a one sentence summary per merged topic from the post titles given.""" + + topics: list[str] = dspy.InputField(desc="name (post count): sample post titles") + groups: list[TopicGroup] = dspy.OutputField() + + +class SeoPack(dspy.Signature): + """Write search metadata for the post in the writer's voice and suggest links to their other + posts where a reader would want them. No em dashes.""" + + title: str = dspy.InputField() + passages: list[str] = dspy.InputField() + related_posts: str = dspy.InputField(desc="Other posts in the archive as date | path | title") + seo: SeoPackOut = dspy.OutputField() + + +class CarouselSlides(dspy.Signature): + """Turn the post's key claims into a 6 to 9 slide carousel. Slide 1 is a hook, the last is a call to + read the post. Short, punchy, in the writer's voice. No em dashes.""" + + passages: list[str] = dspy.InputField() + voice_profile: str = dspy.InputField() + slides: list[Slide] = dspy.OutputField() + + +class PickQuotes(dspy.Signature): + """Pick the most quotable lines, copied exactly from the passages.""" + + passages: list[str] = dspy.InputField() + n: int = dspy.InputField() + quotes: list[QuotePick] = dspy.OutputField() + + +class EvergreenScore(dspy.Signature): + """Judge how well this post would land if reshared today. Evergreen means the ideas are not tied to + news, dates or a moment. Suggest a fresh angle for resharing it. No em dashes.""" + + title: str = dspy.InputField() + published: str = dspy.InputField() + excerpt: str = dspy.InputField() + score: float = dspy.OutputField(desc="0 (dated) to 1 (timeless)") + reason: str = dspy.OutputField() + angle: str = dspy.OutputField(desc="One line hook for resharing now") diff --git a/backend/app/main.py b/backend/app/main.py index f30f9cefbdcd2fe6bc46211626d7d139800884bc..d1f722090fd467aa16000ae5afdc6dbc746eb395 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,4 +1,5 @@ import logging +import threading import time import uuid from contextlib import asynccontextmanager @@ -8,7 +9,22 @@ from fastapi.middleware.cors import CORSMiddleware from app.config import settings from app.llm.provider import configure_default -from app.routers import artifacts, auth, billing, jobs, notebooks, sources, storage, unsubscribe +from app.routers import ( + artifacts, + auth, + billing, + jobs, + launchpad, + notebooks, + resurface, + sources, + storage, + topics, + unsubscribe, + voice, +) +from app.routers import settings as settings_router +from app.services.storage import storage as store log = logging.getLogger("notestack") @@ -17,14 +33,19 @@ log = logging.getLogger("notestack") async def lifespan(app: FastAPI): logging.basicConfig(level=logging.INFO) configure_default() - if settings.env == "development" and settings.r2_endpoint_url: + if settings.env == "development" and (settings.r2_endpoint_url or settings.use_local_storage): try: - from app.services.storage import storage as store - - store.ensure_bucket() # MinIO locally + store.ensure_bucket() # MinIO or the local storage folder except Exception: log.warning("could not ensure local bucket", exc_info=True) + stop = threading.Event() + if settings.worker_in_api: + from app.worker import start_background + + start_background(stop) + log.info("worker running inside the API (RUN_WORKER_IN_API)") yield + stop.set() app = FastAPI(title="Notestack API", version="0.1.0", lifespan=lifespan) @@ -53,6 +74,10 @@ def health(): return {"ok": True, "llm": settings.llm_model, "billing_enabled": settings.billing_enabled} -for r in (auth, billing, storage, sources, notebooks, artifacts, jobs, unsubscribe): +for r in (auth, billing, storage, sources, notebooks, artifacts, jobs, unsubscribe, topics, voice, resurface, + launchpad, settings_router): app.include_router(r.router) app.include_router(sources.documents_router) +app.include_router(launchpad.links_router) +if settings.use_local_storage: + app.include_router(storage.local_router) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 4ffdf5749e2d6e1c2d42d057bbbece4dff6a5c8b..d3fecfef77f27c2b9d8fe276f05134e5a5a23de1 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -4,13 +4,16 @@ from app.models.content import ( Chat, Citation, Document, + DocumentTopic, DspyTrace, EngagementEvent, Job, Message, Notebook, NotebookDocument, + SocialAccount, Source, + Topic, TrackedLink, Upload, UsageEvent, @@ -27,6 +30,7 @@ __all__ = [ "Chat", "Citation", "Document", + "DocumentTopic", "DspyTrace", "EmailVerificationCode", "EngagementEvent", @@ -34,8 +38,10 @@ __all__ = [ "Message", "Notebook", "NotebookDocument", + "SocialAccount", "Source", "Subscription", + "Topic", "TrackedLink", "UpdateEmail", "UpdateEmailSend", diff --git a/backend/app/models/content.py b/backend/app/models/content.py index 1b11bd85c1ff0912feda7ac9ce8f80301fe0c11d..80953435fad301fffb96552a70e3da568f4b906f 100644 --- a/backend/app/models/content.py +++ b/backend/app/models/content.py @@ -53,6 +53,9 @@ class Document(IdMixin, TimestampMixin, Base): clean_text: Mapped[str] = mapped_column(Text, default="") content_hash: Mapped[str | None] = mapped_column(String(64)) metadata_json: Mapped[dict] = mapped_column(JSON, default=dict) + # Resurfacing: 0..1 from the EvergreenScore module (reason and angle live in metadata_json) + evergreen_score: Mapped[float | None] = mapped_column(Float) + last_resurfaced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) class Notebook(IdMixin, TimestampMixin, Base): @@ -83,6 +86,7 @@ class VoiceProfile(IdMixin, TimestampMixin, Base): ) profile_json: Mapped[dict] = mapped_column(JSON, default=dict) sample_doc_ids: Mapped[list] = mapped_column(JSON, default=list) + host_voices: Mapped[dict] = mapped_column(JSON, default=dict) # {"host_a": voice_id, "host_b": voice_id} class VoiceConsent(IdMixin, TimestampMixin, Base): @@ -188,10 +192,63 @@ class CalendarItem(IdMixin, TimestampMixin, Base): __tablename__ = "calendar_items" workspace_id: Mapped[uuid.UUID] = _ws_fk() - artifact_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("artifacts.id", ondelete="CASCADE")) - platform: Mapped[str] = mapped_column(String(30)) - scheduled_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + artifact_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, ForeignKey("artifacts.id", ondelete="CASCADE")) + document_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, ForeignKey("documents.id", ondelete="SET NULL")) + platform: Mapped[str] = mapped_column(String(30)) # x | linkedin | bluesky | substack_notes + scheduled_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True) + # scheduled | publishing | posted | reminded | failed | draft status: Mapped[str] = mapped_column(String(20), default="scheduled") + content: Mapped[str] = mapped_column(Text, default="") + thread: Mapped[list] = mapped_column(JSON, default=list) # extra posts after the first, for threads + social_account_id: Mapped[uuid.UUID | None] = mapped_column( + Uuid, ForeignKey("social_accounts.id", ondelete="SET NULL") + ) + remind_by_email: Mapped[bool] = mapped_column(Boolean, default=False) + external_id: Mapped[str | None] = mapped_column(String(200)) + external_url: Mapped[str | None] = mapped_column(String(1000)) + posted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + error: Mapped[str | None] = mapped_column(Text) + attempts: Mapped[int] = mapped_column(Integer, default=0) + metrics: Mapped[dict] = mapped_column(JSON, default=dict) # likes, reposts, replies, impressions + + +class SocialAccount(IdMixin, TimestampMixin, Base): + """A connected X, LinkedIn or Bluesky account. Tokens are Fernet encrypted (app/services/crypto.py).""" + + __tablename__ = "social_accounts" + __table_args__ = (UniqueConstraint("workspace_id", "platform", "external_id"),) + + workspace_id: Mapped[uuid.UUID] = _ws_fk() + platform: Mapped[str] = mapped_column(String(20)) + handle: Mapped[str] = mapped_column(String(200)) + external_id: Mapped[str] = mapped_column(String(200)) + access_token: Mapped[str] = mapped_column(Text) + refresh_token: Mapped[str | None] = mapped_column(Text) + expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + scopes: Mapped[str | None] = mapped_column(String(500)) + status: Mapped[str] = mapped_column(String(20), default="active") # active | expired | revoked + avatar_url: Mapped[str | None] = mapped_column(String(1000)) + + +class Topic(IdMixin, TimestampMixin, Base): + __tablename__ = "topics" + __table_args__ = (UniqueConstraint("workspace_id", "slug"),) + + workspace_id: Mapped[uuid.UUID] = _ws_fk() + name: Mapped[str] = mapped_column(String(200)) + slug: Mapped[str] = mapped_column(String(200)) + summary: Mapped[str | None] = mapped_column(Text) + post_count: Mapped[int] = mapped_column(Integer, default=0) + + +class DocumentTopic(Base): + __tablename__ = "document_topics" + + document_id: Mapped[uuid.UUID] = mapped_column( + Uuid, ForeignKey("documents.id", ondelete="CASCADE"), primary_key=True + ) + topic_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("topics.id", ondelete="CASCADE"), primary_key=True) + weight: Mapped[float] = mapped_column(Float, default=1.0) class TrackedLink(IdMixin, TimestampMixin, Base): diff --git a/backend/app/pipeline/generate.py b/backend/app/pipeline/generate.py new file mode 100644 index 0000000000000000000000000000000000000000..e4f816dab38ff8bdbda820d01f6c4291fad26863 --- /dev/null +++ b/backend/app/pipeline/generate.py @@ -0,0 +1,182 @@ +"""Text generation jobs over the corpus: notebook summaries, topic map, voice profile, evergreen scores.""" + +import uuid +from collections import Counter, defaultdict +from datetime import UTC, datetime + +from sqlalchemy import delete, select +from sqlalchemy.orm import Session + +from app.corpus import Corpus, slugify +from app.llm import run +from app.llm.provider import fast_lm +from app.llm.signatures import BuildVoiceProfile, ConsolidateTopics, EvergreenScore, ExtractTopics, SummarizeNotebook +from app.models import Artifact, DocumentTopic, Job, Notebook, Topic, VoiceProfile +from app.pipeline.passages import as_citations, notebook_docs, passages_for, tools_for, workspace_docs +from app.pipeline.research import verify_citations +from app.services.jobs import update_job + + +class NothingToDo(ValueError): + pass + + +# Notebook summary + + +def summarize_notebook(db: Session, job: Job, artifact: Artifact) -> dict: + nb = db.get(Notebook, artifact.notebook_id) + docs = notebook_docs(db, nb.id) + if not docs: + raise NothingToDo("This notebook has no posts yet.") + corpus = Corpus(nb.workspace_id) + update_job(db, job, progress=0.1, message=f"Reading {len(docs)} posts") + passages = passages_for(corpus, docs) + update_job(db, job, progress=0.3, message="Writing the summary") + out = run.predict(SummarizeNotebook, db=db, workspace_id=nb.workspace_id, job=job, + title=nb.title, passages=passages) + tools = tools_for(corpus, docs) + text, cites = verify_citations(tools, out.get("summary") or "", as_citations(out.get("citations"))) + by_path = {d.path: d for d in docs} + content = { + "title": f"Summary: {nb.title}", + "summary": text, + "themes": out.get("themes") or [], + "citations": [ + {"marker": c.marker, "path": c.path, "line_start": c.line_start, "line_end": c.line_end, + "span": c.quote, "title": by_path[c.path].title, "url": by_path[c.path].url, + "document_id": str(by_path[c.path].id)} + for c in cites if c.path in by_path + ], + } + artifact.content_json = content + artifact.status = "ready" + nb.summary = text + db.commit() + return {"artifact_id": str(artifact.id)} + + +# Topic map + + +def extract_topics(db: Session, job: Job, workspace_id: uuid.UUID, doc_ids: list[str] | None = None) -> dict: + """Tag posts that have no topics yet (or the given ones), then rebuild the workspace topic table.""" + docs = workspace_docs(db, workspace_id, [uuid.UUID(i) for i in doc_ids] if doc_ids else None) + todo = [d for d in docs if doc_ids or "topics" not in (d.metadata_json or {})] + known = sorted({t["name"] for d in docs for t in (d.metadata_json or {}).get("topics", [])})[:200] + lm = fast_lm() + for i, d in enumerate(todo): + update_job(db, job, progress=0.05 + 0.75 * i / max(len(todo), 1), message=f"Mapping {d.title[:70]}") + try: + out = run.predict(ExtractTopics, db=db, workspace_id=workspace_id, job=job, lm=lm, + title=d.title, text=(d.clean_text or "")[:12000], known_topics=known) + except Exception: + continue + tags = [t for t in (out.get("topics") or []) if t.get("name")][:8] + d.metadata_json = {**(d.metadata_json or {}), "topics": tags} + known = sorted(set(known) | {t["name"] for t in tags})[:200] + db.commit() + update_job(db, job, progress=0.85, message="Drawing constellations") + count = rebuild_topics(db, job, workspace_id) + return {"tagged": len(todo), "topics": count} + + +def rebuild_topics(db: Session, job: Job | None, workspace_id: uuid.UUID) -> int: + docs = workspace_docs(db, workspace_id) + counts: Counter[str] = Counter() + titles: dict[str, list[str]] = defaultdict(list) + for d in docs: + for t in (d.metadata_json or {}).get("topics", []): + counts[t["name"]] += 1 + if len(titles[t["name"]]) < 3: + titles[t["name"]].append(d.title) + if not counts: + return 0 + canonical = {name: name for name in counts} + summaries: dict[str, str] = {} + listing = [f"{n} ({c}): {'; '.join(titles[n])}" for n, c in counts.most_common(150)] + try: + out = run.predict(ConsolidateTopics, db=db, workspace_id=workspace_id, job=job, lm=fast_lm(), topics=listing) + for g in out.get("groups") or []: + name = (g.get("canonical") or "").strip() + if not name: + continue + summaries[name] = g.get("summary") or "" + for m in g.get("members") or []: + if m in canonical: + canonical[m] = name + except Exception: + pass # keep raw names; the map still works + + db.execute(delete(DocumentTopic).where(DocumentTopic.document_id.in_([d.id for d in docs]))) + db.execute(delete(Topic).where(Topic.workspace_id == workspace_id)) + db.flush() + topics: dict[str, Topic] = {} + for d in docs: + weights: dict[str, float] = {} + for t in (d.metadata_json or {}).get("topics", []): + name = canonical.get(t["name"], t["name"]) + weights[name] = max(weights.get(name, 0), float(t.get("weight") or 0.5)) + for name, weight in weights.items(): + slug = slugify(name, 120) + topic = topics.get(slug) + if not topic: + topic = Topic(workspace_id=workspace_id, name=name, slug=slug, summary=summaries.get(name)) + db.add(topic) + db.flush() + topics[slug] = topic + topic.post_count += 1 + db.add(DocumentTopic(document_id=d.id, topic_id=topic.id, weight=weight)) + db.commit() + return len(topics) + + +# Voice profile + + +def build_voice_profile(db: Session, job: Job, workspace_id: uuid.UUID, doc_ids: list[str]) -> dict: + docs = workspace_docs(db, workspace_id, [uuid.UUID(i) for i in doc_ids] if doc_ids else None) + if not doc_ids: + docs = sorted(docs, key=lambda d: len(d.clean_text or ""), reverse=True)[:8] + if not docs: + raise NothingToDo("Add some posts before building a voice profile.") + update_job(db, job, progress=0.2, message=f"Listening to {len(docs)} posts") + samples = [f"# {d.title}\n\n{(d.clean_text or '')[:6000]}" for d in docs[:10]] + out = run.predict(BuildVoiceProfile, db=db, workspace_id=workspace_id, job=job, samples=samples) + vp = db.scalar(select(VoiceProfile).where(VoiceProfile.workspace_id == workspace_id)) + if not vp: + vp = VoiceProfile(workspace_id=workspace_id) + db.add(vp) + vp.profile_json = out.get("profile") or {} + vp.sample_doc_ids = [str(d.id) for d in docs[:10]] + db.commit() + return {"profile": vp.profile_json} + + +# Resurfacing + + +def score_evergreen(db: Session, job: Job, workspace_id: uuid.UUID, limit: int = 60) -> dict: + docs = [d for d in workspace_docs(db, workspace_id) if d.evergreen_score is None][:limit] + lm = fast_lm() + scored = 0 + for i, d in enumerate(docs): + update_job(db, job, progress=0.05 + 0.9 * i / max(len(docs), 1), message=f"Weighing {d.title[:70]}") + try: + out = run.predict( + EvergreenScore, db=db, workspace_id=workspace_id, job=job, lm=lm, title=d.title, + published=d.published_at.date().isoformat() if d.published_at else "unknown", + excerpt=(d.clean_text or "")[:2500], + ) + except Exception: + continue + try: + d.evergreen_score = max(0.0, min(1.0, float(out.get("score") or 0))) + except (TypeError, ValueError): + continue + d.metadata_json = {**(d.metadata_json or {}), "evergreen_reason": out.get("reason") or "", + "reshare_angle": out.get("angle") or "", + "scored_at": datetime.now(UTC).isoformat()} + scored += 1 + db.commit() + return {"scored": scored} diff --git a/backend/app/pipeline/ingest.py b/backend/app/pipeline/ingest.py index 59dfc7871b657d6a760256088ee1781d3136aa36..9ccbbedc514072a863e64a80ade56befa7383f54 100644 --- a/backend/app/pipeline/ingest.py +++ b/backend/app/pipeline/ingest.py @@ -1,13 +1,22 @@ -"""Ingestion: feed -> raw HTML to R2 -> clean markdown files in the workspace corpus -> INDEX.md.""" +"""Ingestion: feed -> raw HTML to storage -> clean markdown files in the workspace corpus -> INDEX.md. + +Three ways in, one pipeline out: +- feeds (Substack, Ghost, Medium, any RSS/Atom), with Substack's archive API for posts past the feed +- one article by URL +- an uploaded file (markdown, text, HTML or PDF) +All of them end in `store_entries`, which writes documents, raw HTML and corpus files. +""" import hashlib +import io import logging import re +import time import uuid from dataclasses import dataclass from datetime import UTC, datetime from time import mktime -from urllib.parse import urlparse +from urllib.parse import urljoin, urlparse import feedparser import httpx @@ -23,13 +32,25 @@ from app.services.storage import keys, storage log = logging.getLogger(__name__) _DROP = re.compile(r"(subscribe|share this post|leave a comment|thanks for reading)", re.I) +UA = {"User-Agent": "Mozilla/5.0 (compatible; NotestackBot/0.2; +https://notestack.ai)"} +IMPORTS_FEED = "notestack:imports" -def normalize_feed_url(url: str) -> tuple[str, str]: - """Accept 'name.substack.com', a Substack post URL, or any feed URL. Returns (feed_url, platform).""" +class FeedNotFound(ValueError): + pass + + +# Feed discovery + + +def _with_scheme(url: str) -> str: url = url.strip() - if not url.startswith(("http://", "https://")): - url = f"https://{url}" + return url if url.startswith(("http://", "https://")) else f"https://{url}" + + +def normalize_feed_url(url: str) -> tuple[str, str]: + """Best guess without network: accept 'name.substack.com', a post URL, or any feed URL.""" + url = _with_scheme(url) parsed = urlparse(url) host = parsed.netloc.lower() if host.endswith("substack.com") or "/p/" in parsed.path: @@ -41,12 +62,82 @@ def normalize_feed_url(url: str) -> tuple[str, str]: return f"{parsed.scheme}://{host}/feed", "rss" +def _parse_feed(url: str, timeout: float = 15) -> tuple[feedparser.FeedParserDict | None, httpx.Response | None]: + try: + resp = httpx.get(url, follow_redirects=True, timeout=timeout, headers=UA) + except httpx.HTTPError: + return None, None + if resp.status_code >= 400: + return None, resp + parsed = feedparser.parse(resp.content) + if parsed.get("version") and (parsed.entries or parsed.feed.get("title")): + return parsed, resp + return None, resp + + +@dataclass +class DiscoveredFeed: + feed_url: str + platform: str + title: str | None + site_url: str + + +def discover_feed(url: str) -> DiscoveredFeed: + """Find a working feed for whatever the writer pasted, or raise FeedNotFound with a clear reason.""" + url = _with_scheme(url) + guess, platform = normalize_feed_url(url) + parsed_url = urlparse(url) + site = f"{parsed_url.scheme}://{parsed_url.netloc}" + candidates = [guess, url, f"{site}/feed", f"{site}/rss", f"{site}/rss.xml", f"{site}/feed.xml", f"{site}/atom.xml"] + seen: set[str] = set() + last: httpx.Response | None = None + for candidate in candidates: + if candidate in seen: + continue + seen.add(candidate) + parsed, resp = _parse_feed(candidate) + last = resp or last + if parsed: + return DiscoveredFeed(candidate, _platform(candidate, parsed, platform), parsed.feed.get("title"), site) + if resp is not None and resp.status_code < 400 and "html" in resp.headers.get("content-type", ""): + # A web page: look for . + soup = BeautifulSoup(resp.text, "html.parser") + for link in soup.find_all("link", rel="alternate"): + if "rss" in (link.get("type") or "") or "atom" in (link.get("type") or ""): + href = urljoin(str(resp.url), link.get("href", "")) + if href and href not in seen: + seen.add(href) + found, _ = _parse_feed(href) + if found: + return DiscoveredFeed(href, _platform(href, found, platform), found.feed.get("title"), site) + if last is not None and last.status_code == 404: + raise FeedNotFound(f"{parsed_url.netloc} returned 404. Check the address in your browser.") + raise FeedNotFound("We couldn't find a feed at that address.") + + +def _platform(feed_url: str, parsed, guess: str) -> str: + generator = (parsed.feed.get("generator") or "").lower() + if "substack" in feed_url or "substack" in generator: + return "substack" + if "ghost" in generator: + return "ghost" + if "medium.com" in feed_url: + return "medium" + if "wordpress" in generator: + return "wordpress" + return guess if guess != "rss" else "rss" + + +# Text extraction + + def html_to_sections(html: str) -> list[tuple[str | None, str]]: soup = BeautifulSoup(html, "html.parser") - for tag in soup(["script", "style", "nav", "footer", "form", "button", "iframe"]): + for tag in soup(["script", "style", "nav", "footer", "form", "button", "iframe", "noscript", "svg"]): tag.decompose() sections: list[tuple[str | None, list[str]]] = [(None, [])] - for el in soup.find_all(["h1", "h2", "h3", "p", "li", "blockquote"]): + for el in soup.find_all(["h1", "h2", "h3", "p", "li", "blockquote", "pre"]): text = el.get_text(" ", strip=True) if not text or (len(text) < 120 and _DROP.search(text)): continue @@ -57,16 +148,94 @@ def html_to_sections(html: str) -> list[tuple[str | None, str]]: return [(h, "\n\n".join(p)) for h, p in sections if p] +def markdown_to_sections(text: str) -> list[tuple[str | None, str]]: + sections: list[tuple[str | None, list[str]]] = [(None, [])] + para: list[str] = [] + + def flush() -> None: + if para: + sections[-1][1].append(" ".join(para).strip()) + para.clear() + + for line in text.splitlines(): + stripped = line.strip() + heading = re.match(r"^#{1,3}\s+(.*)", stripped) + if heading: + flush() + sections.append((heading.group(1).strip(), [])) + elif not stripped: + flush() + else: + para.append(stripped) + flush() + return [(h, "\n\n".join(p)) for h, p in sections if p] + + +def pdf_to_sections(data: bytes) -> list[tuple[str | None, str]]: + from pypdf import PdfReader + + reader = PdfReader(io.BytesIO(data)) + text = "\n\n".join((page.extract_text() or "") for page in reader.pages) + # PDF text keeps hard line breaks inside paragraphs; blank lines separate paragraphs. + paragraphs = [re.sub(r"\s*\n\s*", " ", p).strip() for p in re.split(r"\n\s*\n", text)] + return [(None, "\n\n".join(p for p in paragraphs if p))] if any(paragraphs) else [] + + +def _meta(soup: BeautifulSoup, *names: str) -> str | None: + for name in names: + tag = soup.find("meta", attrs={"property": name}) or soup.find("meta", attrs={"name": name}) + if tag and tag.get("content"): + return tag["content"].strip() + return None + + +def _parse_date(value: str | None) -> datetime | None: + if not value: + return None + try: + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + return dt if dt.tzinfo else dt.replace(tzinfo=UTC) + except ValueError: + return None + + @dataclass class FeedEntry: title: str url: str published_at: datetime | None html: str + sections: list[tuple[str | None, str]] | None = None # set when the source is not HTML + + +def extract_article(url: str) -> FeedEntry: + """Fetch one web page and keep the article body.""" + url = _with_scheme(url) + resp = httpx.get(url, follow_redirects=True, timeout=30, headers=UA) + if resp.status_code >= 400: + raise FeedNotFound(f"That page returned {resp.status_code}.") + soup = BeautifulSoup(resp.text, "html.parser") + title = _meta(soup, "og:title", "twitter:title") or (soup.title.get_text(strip=True) if soup.title else url) + published = _parse_date(_meta(soup, "article:published_time", "og:published_time", "date")) + if not published and (t := soup.find("time", datetime=True)): + published = _parse_date(t["datetime"]) + body = soup.find("article") or soup.find("main") or soup.find(class_=re.compile("post-content|entry-content|body")) + if body is None: + # Fall back to the element holding the most paragraph text. + candidates = soup.find_all(["div", "section"]) + body = max(candidates, key=lambda el: sum(len(p.get_text()) for p in el.find_all("p", recursive=False)), + default=soup.body or soup) + html = str(body) + if not html_to_sections(html): + raise FeedNotFound("We could not find article text on that page.") + return FeedEntry(title=title[:1000], url=str(resp.url), published_at=published, html=html) + + +# Feed fetching def fetch_feed(feed_url: str, limit: int) -> tuple[str | None, list[FeedEntry]]: - resp = httpx.get(feed_url, follow_redirects=True, timeout=30, headers={"User-Agent": "NotestackBot/0.1"}) + resp = httpx.get(feed_url, follow_redirects=True, timeout=30, headers=UA) resp.raise_for_status() parsed = feedparser.parse(resp.content) entries = [] @@ -84,7 +253,45 @@ def fetch_feed(feed_url: str, limit: int) -> tuple[str | None, list[FeedEntry]]: return parsed.feed.get("title"), entries +def fetch_substack_archive(site: str, known_urls: set[str], limit: int, on_page=None) -> list[FeedEntry]: + """Posts beyond the ~20 the RSS feed carries, through Substack's public archive API.""" + out: list[FeedEntry] = [] + offset = 0 + with httpx.Client(follow_redirects=True, timeout=30, headers=UA) as client: + while len(out) + len(known_urls) < limit: + resp = client.get(f"{site}/api/v1/archive", params={"sort": "new", "offset": offset, "limit": 50}) + if resp.status_code >= 400: + break + page = resp.json() + if not page: + break + offset += len(page) + for item in page: + url = item.get("canonical_url") or f"{site}/p/{item.get('slug')}" + if url in known_urls or item.get("type") not in (None, "newsletter", "thread"): + continue + if len(out) + len(known_urls) >= limit: + break + post = client.get(f"{site}/api/v1/posts/{item['slug']}") + if post.status_code >= 400: + continue + body = post.json().get("body_html") or item.get("truncated_body_text") or "" + if not body: + continue + out.append(FeedEntry(title=item.get("title") or "Untitled", url=url, + published_at=_parse_date(item.get("post_date")), html=body)) + time.sleep(0.2) # be polite to Substack + if on_page: + on_page(len(out)) + return out + + +# Storing + + def source_slug(source: Source) -> str: + if source.feed_url == IMPORTS_FEED: + return "imports" return slugify(urlparse(source.site_url or source.feed_url).netloc or str(source.id), 50) @@ -93,34 +300,46 @@ def rebuild_index(db: Session, workspace_id: uuid.UUID) -> str: select(Document).where(Document.workspace_id == workspace_id, Document.path.is_not(None)) ).all() rows = sorted( - (index_line(d.path, d.title, d.published_at, d.metadata_json.get("headings", [])) for d in docs), + (index_line(d.path, d.title, d.published_at, (d.metadata_json or {}).get("headings", [])) for d in docs), reverse=True, ) header = "# Archive index\n\ndate | path | title | section headings\n\n" return header + "\n".join(rows) + "\n" -def ingest_source(db: Session, source: Source, job: Job, max_posts: int) -> dict: - source.sync_status = "syncing" - update_job(db, job, status="running", progress=0.02, message="Contacting your feed") +def imports_source(db: Session, workspace_id: uuid.UUID) -> Source: + """Single articles and uploaded files live under one built in source that does not count to limits.""" + source = db.scalar(select(Source).where(Source.workspace_id == workspace_id, Source.feed_url == IMPORTS_FEED)) + if not source: + source = Source(workspace_id=workspace_id, feed_url=IMPORTS_FEED, platform="imports", + title="Imported posts and files", sync_status="ok") + db.add(source) + db.flush() + return source - title, entries = fetch_feed(source.feed_url, max_posts) - source.title = source.title or title + +def store_entries( + db: Session, source: Source, entries: list[FeedEntry], progress=None +) -> tuple[int, int, list[uuid.UUID]]: + """Write documents, raw HTML and corpus files. Returns (indexed, skipped, changed document ids).""" corpus = Corpus(source.workspace_id) folder = source_slug(source) - total = max(len(entries), 1) indexed = skipped = 0 + changed: list[uuid.UUID] = [] files: dict[str, str] = {} - - taken = set( - db.scalars(select(Document.path).where(Document.workspace_id == source.workspace_id)).all() - ) + taken = set(db.scalars(select(Document.path).where(Document.workspace_id == source.workspace_id)).all()) + total = max(len(entries), 1) for i, entry in enumerate(entries): - content_hash = hashlib.sha256(entry.html.encode()).hexdigest() + payload = entry.html or "\n\n".join(t for _, t in (entry.sections or [])) + content_hash = hashlib.sha256(payload.encode()).hexdigest() doc = db.scalar(select(Document).where(Document.source_id == source.id, Document.url == entry.url)) if doc and doc.content_hash == content_hash and doc.path: skipped += 1 continue + sections = entry.sections if entry.sections is not None else html_to_sections(entry.html) + if not sections: + skipped += 1 + continue if not doc: doc = Document(id=uuid.uuid4(), workspace_id=source.workspace_id, source_id=source.id, url=entry.url) db.add(doc) @@ -135,25 +354,76 @@ def ingest_source(db: Session, source: Source, job: Job, max_posts: int) -> dict doc.title = entry.title doc.published_at = entry.published_at doc.content_hash = content_hash - doc.raw_html_key = storage.put_text( - keys.raw_html(source.workspace_id, source.id, doc.id), entry.html, "text/html; charset=utf-8" - ) - sections = html_to_sections(entry.html) + if entry.html: + doc.raw_html_key = storage.put_text( + keys.raw_html(source.workspace_id, source.id, doc.id), entry.html, "text/html; charset=utf-8" + ) doc.clean_text = "\n\n".join(f"## {h}\n{t}" if h else t for h, t in sections) - doc.metadata_json = {**(doc.metadata_json or {}), "headings": [h for h, _ in sections if h]} + doc.metadata_json = { + **(doc.metadata_json or {}), + "headings": [h for h, _ in sections if h], + "words": len(doc.clean_text.split()), + } files[doc.path] = render_post( title=entry.title, url=entry.url, published_at=entry.published_at, source=folder, sections=sections ) indexed += 1 + changed.append(doc.id) db.commit() - update_job(db, job, progress=0.05 + 0.85 * (i + 1) / total, message=f"{entry.title[:80]} is in orbit") - - update_job(db, job, progress=0.93, message="Writing your archive index") + if progress: + progress(i + 1, total, entry.title) + db.flush() files[INDEX] = rebuild_index(db, source.workspace_id) corpus.write_files(files) + db.commit() + return indexed, skipped, changed + + +def ingest_source(db: Session, source: Source, job: Job, max_posts: int) -> dict: + source.sync_status = "syncing" + update_job(db, job, status="running", progress=0.02, message="Contacting your feed") + + title, entries = fetch_feed(source.feed_url, max_posts) + source.title = source.title or title + if source.platform == "substack" and len(entries) < max_posts: + site = f"{urlparse(source.feed_url).scheme}://{urlparse(source.feed_url).netloc}" + update_job(db, job, progress=0.05, message="Reading your Substack archive") + try: + entries += fetch_substack_archive( + site, {e.url for e in entries}, max_posts, + on_page=lambda n: update_job(db, job, message=f"Found {len(entries) + n} posts in the archive"), + ) + except (httpx.HTTPError, ValueError): + log.warning("substack archive fetch failed for %s", site, exc_info=True) + + def progress(done: int, total: int, post_title: str) -> None: + update_job(db, job, progress=0.1 + 0.8 * done / total, message=f"{post_title[:80]} is in orbit") + + update_job(db, job, progress=0.1, message=f"Indexing {len(entries)} posts") + indexed, skipped, changed = store_entries(db, source, entries, progress) source.sync_status = "ok" source.sync_error = None source.last_synced_at = datetime.now(UTC) db.commit() - return {"indexed": indexed, "skipped": skipped, "found": len(entries)} + return {"indexed": indexed, "skipped": skipped, "found": len(entries), "changed": [str(i) for i in changed]} + + +def entry_from_upload(filename: str, content_type: str, data: bytes) -> FeedEntry: + name = filename.rsplit("/", 1)[-1] + stem = re.sub(r"\.[A-Za-z0-9]+$", "", name).replace("-", " ").replace("_", " ").strip() or "Untitled" + lower = name.lower() + url = f"upload://{name}" + if content_type == "application/pdf" or lower.endswith(".pdf"): + return FeedEntry(title=stem, url=url, published_at=None, html="", sections=pdf_to_sections(data)) + text = data.decode("utf-8", errors="replace") + if content_type == "text/html" or lower.endswith((".html", ".htm")): + soup = BeautifulSoup(text, "html.parser") + title = soup.title.get_text(strip=True) if soup.title else stem + return FeedEntry(title=title, url=url, published_at=None, html=text) + sections = markdown_to_sections(text) + first_heading = re.search(r"^#\s+(.+)$", text, re.M) + title = first_heading.group(1).strip() if first_heading else stem + if sections and sections[0][0] == title: + sections[0] = (None, sections[0][1]) + return FeedEntry(title=title, url=url, published_at=None, html="", sections=sections) diff --git a/backend/app/pipeline/launchkit.py b/backend/app/pipeline/launchkit.py new file mode 100644 index 0000000000000000000000000000000000000000..119358651446a3b6d5d3845e9e531cd0bb386b1d --- /dev/null +++ b/backend/app/pipeline/launchkit.py @@ -0,0 +1,108 @@ +"""Launch Kit: one post in, everything needed to launch it out, in the writer's voice and grounded in +the post's own claims.""" + +from sqlalchemy.orm import Session + +from app.corpus import INDEX, Corpus +from app.llm import run +from app.llm.signatures import CarouselSlides, ExtractClaims, HookGenerator, PickQuotes, PlatformAdapter, SeoPack +from app.models import Artifact, Document, Job +from app.pipeline.generate import NothingToDo +from app.pipeline.passages import passages_for, tools_for, verify_refs, voice_text +from app.services.jobs import create_job, update_job + +PLATFORMS = { + "x_thread": ("X thread", "5 to 9 posts, each under 270 characters. Post 1 is the hook and stands alone. " + "No hashtags except at most one at the end. The last post links to the article with {url}."), + "linkedin": ("LinkedIn", "One post, 900 to 1,500 characters. Short paragraphs of 1 to 2 lines. Open with a " + "concrete line, not a question. End with the link {url} and one question for comments."), + "substack_notes": ("Substack Notes", "3 separate notes, each 1 to 4 sentences, conversational, each able " + "to stand alone. One of them links to {url}."), + "bluesky": ("Bluesky", "A thread of 3 to 6 posts, each under 290 characters. Plain, human, no hashtags. " + "The last post links to {url}."), +} + + +def build_launch_kit(db: Session, job: Job, artifact: Artifact) -> dict: + doc = db.get(Document, artifact.document_id) if artifact.document_id else None + if not doc or not doc.path: + raise NothingToDo("Pick a post for the Launch Kit.") + ws = artifact.workspace_id + corpus = Corpus(ws) + passages = passages_for(corpus, [doc], budget_chars=50_000) + tools = tools_for(corpus, [doc]) + voice = voice_text(db, ws) + url = doc.url if doc.url.startswith("http") else "" + + def step(progress: float, message: str) -> None: + update_job(db, job, progress=progress, message=message) + + step(0.08, "Pulling out the key claims") + claims_out = run.predict(ExtractClaims, db=db, workspace_id=ws, job=job, passages=passages) + claims = [] + for c in claims_out.get("claims") or []: + refs = verify_refs(tools, c.get("sources") or []) + if refs and c.get("claim"): + claims.append({"claim": c["claim"], "sources": refs}) + if not claims: + raise RuntimeError("No verifiable claims found in this post") + claim_inputs = [{"claim": c["claim"], "sources": [{k: s[k] for k in ("path", "line_start", "line_end")} + for s in c["sources"]]} for c in claims] + + step(0.2, "Writing hooks") + hooks = run.predict(HookGenerator, db=db, workspace_id=ws, job=job, passages=passages, + voice_profile=voice, n=8).get("hooks") or [] + hooks = sorted(hooks, key=lambda h: -(h.get("strength") or 0)) + + posts: dict[str, list[str]] = {} + for i, (platform, (label, rules)) in enumerate(PLATFORMS.items()): + step(0.3 + 0.1 * i, f"Adapting for {label}") + out = run.predict(PlatformAdapter, db=db, workspace_id=ws, job=job, claims=claim_inputs, voice_profile=voice, + platform=platform, platform_rules=rules.format(url=url or "the post")) + posts[platform] = [p for p in (out.get("posts") or []) if p.strip()] + + step(0.72, "Building the SEO pack") + related = [] + if corpus.exists(INDEX): + related = [ln for ln in corpus.read_lines(INDEX) if "|" in ln and doc.path not in ln][:150] + seo = run.predict(SeoPack, db=db, workspace_id=ws, job=job, title=doc.title, passages=passages, + related_posts="\n".join(related)).get("seo") or {} + + step(0.82, "Designing the carousel") + slides = run.predict(CarouselSlides, db=db, workspace_id=ws, job=job, passages=passages, + voice_profile=voice).get("slides") or [] + + step(0.9, "Choosing quotes") + quotes = [] + for pick in run.predict(PickQuotes, db=db, workspace_id=ws, job=job, passages=passages, n=3).get("quotes") or []: + refs = verify_refs(tools, [pick.get("source") or {}]) + if refs and pick.get("quote"): + quotes.append({"quote": pick["quote"][:380], "source": refs[0]}) + + artifact.content_json = { + **(artifact.content_json or {}), + "title": f"Launch Kit: {doc.title}", + "post_title": doc.title, + "post_url": doc.url, + "claims": claims, + "hooks": hooks, + "posts": posts, + "seo": seo, + "carousel": slides, + "quotes": quotes, + } + artifact.status = "ready" + db.commit() + + # Quote cards render as their own artifacts, so a missing renderer does not sink the kit. + card_ids = [] + for q in quotes: + card = Artifact(workspace_id=ws, document_id=doc.id, type="quote_card", status="pending", + content_json={"title": f"Quote: {doc.title}", "parent_id": str(artifact.id), **q}) + db.add(card) + db.flush() + create_job(db, ws, "quote_card", {"artifact_id": str(card.id)}, artifact_id=card.id, max_attempts=1) + card_ids.append(str(card.id)) + artifact.content_json = {**artifact.content_json, "quote_card_ids": card_ids} + db.commit() + return {"artifact_id": str(artifact.id), "claims": len(claims)} diff --git a/backend/app/pipeline/media.py b/backend/app/pipeline/media.py new file mode 100644 index 0000000000000000000000000000000000000000..afacd336e0a6860f00f5120429ff5ab731902ec7 --- /dev/null +++ b/backend/app/pipeline/media.py @@ -0,0 +1,260 @@ +"""Audio overviews (two hosts, ElevenLabs), videos and stills (Remotion renderer).""" + +import uuid + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.config import settings +from app.corpus import Corpus +from app.llm import run +from app.llm.signatures import PickQuotes, PodcastScript, VideoStoryboard +from app.models import Artifact, Document, Job, Source, User, VoiceProfile, Workspace +from app.pipeline.generate import NothingToDo +from app.pipeline.passages import notebook_docs, passages_for, tools_for, verify_refs +from app.services import tts +from app.services.jobs import record_usage, update_job +from app.services.renderer import PermanentJobError, brand_props, request_render +from app.services.storage import keys, storage + +SCENE_TYPES = {"title", "section", "pull_quote", "number", "outro"} +VIDEO_STYLES = { + "short": ("ShortVertical", "9:16"), + "explainer": ("ExplainerLong", "16:9"), +} + + +def artifact_docs(db: Session, artifact: Artifact) -> list[Document]: + if artifact.document_id: + doc = db.get(Document, artifact.document_id) + return [doc] if doc and doc.path else [] + if artifact.notebook_id: + return notebook_docs(db, artifact.notebook_id) + return [] + + +def host_voices(db: Session, workspace_id: uuid.UUID) -> tuple[str, str]: + vp = db.scalar(select(VoiceProfile).where(VoiceProfile.workspace_id == workspace_id)) + voices = (vp.host_voices if vp else None) or {} + return voices.get("host_a") or settings.elevenlabs_voice_a, voices.get("host_b") or settings.elevenlabs_voice_b + + +def _require_tts() -> None: + if not settings.elevenlabs_api_key: + raise PermanentJobError("Audio needs ELEVENLABS_API_KEY in .env") + + +# Audio overview + + +def audio_overview(db: Session, job: Job, artifact: Artifact) -> dict: + _require_tts() + params = job.params + docs = artifact_docs(db, artifact) + if not docs: + raise NothingToDo("Add posts before making an audio overview.") + corpus = Corpus(artifact.workspace_id) + update_job(db, job, progress=0.05, message=f"Reading {len(docs)} posts") + passages = passages_for(corpus, docs, budget_chars=90_000) + update_job(db, job, progress=0.15, message="Writing the conversation") + out = run.predict(PodcastScript, db=db, workspace_id=artifact.workspace_id, job=job, passages=passages, + format=params.get("format", "deep_dive"), target_minutes=int(params.get("minutes", 6))) + lines = [ln for ln in (out.get("lines") or []) if (ln.get("text") or "").strip()] + if not lines: + raise RuntimeError("The script came back empty") + tools = tools_for(corpus, docs) + voice_a, voice_b = host_voices(db, artifact.workspace_id) + audio = bytearray() + segments = [] + t = 0.0 + for i, line in enumerate(lines): + voice = voice_a if line.get("speaker") == "host_a" else voice_b + clip = tts.synthesize(artifact.workspace_id, line["text"], voice) + seconds = tts.mp3_seconds(clip) + segments.append({"speaker": line.get("speaker", "host_a"), "text": line["text"], + "start": round(t, 2), "end": round(t + seconds, 2), + "sources": verify_refs(tools, line.get("sources") or [])}) + audio += clip + t += seconds + update_job(db, job, progress=0.2 + 0.75 * (i + 1) / len(lines), + message=f"Recording line {i + 1} of {len(lines)}") + key = keys.artifact(artifact.workspace_id, artifact.id, "audio", "mp3") + storage.put_bytes(key, bytes(audio), "audio/mpeg") + artifact.storage_key = key + artifact.status = "ready" + artifact.content_json = { + **(artifact.content_json or {}), + "format": params.get("format", "deep_dive"), + "duration_s": round(t, 1), + "segments": segments, + "voices": {"host_a": voice_a, "host_b": voice_b}, + } + db.commit() + record_usage(db, workspace_id=artifact.workspace_id, kind="tts", provider="elevenlabs", + quantity=round(t, 1), unit="seconds", job=job) + return {"artifact_id": str(artifact.id), "duration_s": round(t, 1)} + + +# Video + + +def _clamp(value, lo: float, hi: float) -> float: + try: + return max(lo, min(hi, float(value))) + except (TypeError, ValueError): + return lo + + +def make_video(db: Session, job: Job, artifact: Artifact) -> None: + style = job.params.get("style", "short") + if style == "audiogram": + return make_audiogram(db, job, artifact) + composition, aspect = VIDEO_STYLES[style] + docs = artifact_docs(db, artifact) + if not docs: + raise NothingToDo("Add posts before making a video.") + corpus = Corpus(artifact.workspace_id) + workspace = db.get(Workspace, artifact.workspace_id) + update_job(db, job, progress=0.05, message="Storyboarding") + passages = passages_for(corpus, docs, budget_chars=60_000) + out = run.predict(VideoStoryboard, db=db, workspace_id=artifact.workspace_id, job=job, + passages=passages, aspect=aspect) + scenes = [s for s in (out.get("scenes") or []) if (s.get("on_screen_text") or "").strip()] + if not scenes: + raise RuntimeError("The storyboard came back empty") + max_scenes = 8 if style == "short" else 14 + scenes = scenes[:max_scenes] + + captions: list[dict] = [] + audio = bytearray() + cursor = 0.0 + voice_a, _ = host_voices(db, artifact.workspace_id) + out_scenes = [] + for i, s in enumerate(scenes): + narration = (s.get("narration") or "").strip() + duration = _clamp(s.get("duration_hint_s"), 2.0, 20.0) + if narration and settings.elevenlabs_api_key: + update_job(db, job, progress=0.15 + 0.5 * i / len(scenes), message=f"Narrating scene {i + 1}") + clip, words = tts.synthesize_with_timestamps(artifact.workspace_id, narration, voice_a) + seconds = tts.mp3_seconds(clip) + # Pad with silence so each clip starts exactly when its scene does. + audio += _silence(cursor - tts.mp3_seconds(bytes(audio))) + offset = int(cursor * 1000) + captions += [{**w, "startMs": w["startMs"] + offset, "endMs": w["endMs"] + offset} for w in words] + audio += clip + duration = max(duration, seconds + 0.3) + cursor += duration + scene_type = s.get("type") if s.get("type") in SCENE_TYPES else "section" + out_scenes.append({"type": scene_type, "onScreenText": s["on_screen_text"][:300], + "narration": narration, "durationS": round(_clamp(duration, 0.5, 60), 2)}) + + props: dict = {"scenes": out_scenes, "captions": captions, "brand": brand_props(workspace)} + title = docs[0].title if len(docs) == 1 else (artifact.content_json or {}).get("title", "Notestack") + if composition == "ShortVertical": + hook = next((sc["onScreenText"] for sc in out_scenes if sc["type"] == "title"), title) + props["hook"] = hook[:120] + else: + props["title"] = title[:140] + if audio: + key = keys.artifact(artifact.workspace_id, artifact.id, "narration", "mp3") + storage.put_bytes(key, bytes(audio), "audio/mpeg") + props["audioUrl"] = storage.presign_get(key, ttl=6 * 3600) + artifact.content_json = {**(artifact.content_json or {}), "style": style, "composition": composition, + "scenes": out_scenes, "duration_s": round(cursor, 1)} + artifact.status = "rendering" + db.commit() + update_job(db, job, progress=0.7, message="Handing off to the renderer") + request_render(job, artifact, composition, props) + record_usage(db, workspace_id=artifact.workspace_id, kind="video", provider="remotion", + quantity=round(cursor, 1), unit="seconds", job=job) + + +_SILENT_FRAME = None + + +def _silence(seconds: float) -> bytes: + """MPEG1 Layer III 128 kbps 44.1 kHz silent frames (417 bytes, ~26 ms each).""" + global _SILENT_FRAME + if _SILENT_FRAME is None: + _SILENT_FRAME = bytes([0xFF, 0xFB, 0x90, 0x64]) + bytes(413) + frames = int(seconds / 0.026122) + return _SILENT_FRAME * max(frames, 0) + + +def make_audiogram(db: Session, job: Job, artifact: Artifact) -> None: + source = db.get(Artifact, uuid.UUID(job.params["audio_artifact_id"])) + if not source or source.type != "audio_overview" or not source.storage_key: + raise PermanentJobError("Pick a finished audio overview for the audiogram.") + workspace = db.get(Workspace, artifact.workspace_id) + content = source.content_json or {} + duration = float(content.get("duration_s") or 30) + segments = [{"speaker": s["speaker"], "text": s["text"], "startMs": int(s["start"] * 1000), + "endMs": int(s["end"] * 1000)} for s in content.get("segments", [])] + props = {"title": (content.get("title") or "Audio overview")[:140], + "audioUrl": storage.presign_get(source.storage_key, ttl=6 * 3600), + "durationS": round(min(duration, 600), 2), "segments": segments, "brand": brand_props(workspace)} + artifact.content_json = {**(artifact.content_json or {}), "style": "audiogram", "composition": "AudiogramSquare", + "audio_artifact_id": str(source.id), "duration_s": props["durationS"]} + artifact.status = "rendering" + db.commit() + update_job(db, job, progress=0.1, message="Handing off to the renderer") + request_render(job, artifact, "AudiogramSquare", props) + record_usage(db, workspace_id=artifact.workspace_id, kind="video", provider="remotion", + quantity=props["durationS"], unit="seconds", job=job) + + +# Stills + + +def author_line(db: Session, workspace_id: uuid.UUID) -> str: + ws = db.get(Workspace, workspace_id) + owner = db.get(User, ws.owner_id) if ws else None + return (ws.brand_json or {}).get("name") or (owner.name if owner and owner.name else "") or "Notestack" + + +def make_quote_card(db: Session, job: Job, artifact: Artifact) -> None: + content = artifact.content_json or {} + workspace = db.get(Workspace, artifact.workspace_id) + if not content.get("quote"): + docs = artifact_docs(db, artifact) + if not docs: + raise NothingToDo("Pick a post or notebook for the quote card.") + corpus = Corpus(artifact.workspace_id) + out = run.predict(PickQuotes, db=db, workspace_id=artifact.workspace_id, job=job, + passages=passages_for(corpus, docs, budget_chars=40_000), n=3) + tools = tools_for(corpus, docs) + for pick in out.get("quotes") or []: + refs = verify_refs(tools, [pick.get("source") or {}]) + if refs and pick.get("quote"): + content = {**content, "quote": pick["quote"][:380], "source": refs[0]} + break + else: + raise RuntimeError("No verifiable quote found") + source = content.get("source") or {} + doc = db.scalar(select(Document).where(Document.workspace_id == artifact.workspace_id, + Document.path == source.get("path"))) + publication = None + if doc and doc.source_id: + src = db.get(Source, doc.source_id) + publication = src.title if src else None + props = {"quote": content["quote"][:380], "author": author_line(db, artifact.workspace_id), + "brand": brand_props(workspace)} + if publication or (doc and doc.title): + props["publication"] = (publication or doc.title)[:120] + artifact.content_json = {**content, "composition": "QuoteCard"} + artifact.status = "rendering" + db.commit() + request_render(job, artifact, "QuoteCard", props) + + +def render_carousel(db: Session, job: Job, artifact: Artifact) -> None: + slides = (artifact.content_json or {}).get("slides") or [] + if not slides: + raise PermanentJobError("This carousel has no slides.") + workspace = db.get(Workspace, artifact.workspace_id) + brand = brand_props(workspace) + stills = [{"heading": s.get("heading", "")[:120], "body": s.get("body", "")[:500], "index": i + 1, + "total": len(slides), "brand": brand} for i, s in enumerate(slides)] + artifact.status = "rendering" + db.commit() + request_render(job, artifact, "CarouselSlide", stills=stills) diff --git a/backend/app/pipeline/passages.py b/backend/app/pipeline/passages.py new file mode 100644 index 0000000000000000000000000000000000000000..a7c654248d376ab56c8638e90a89b6cc7a31dfd4 --- /dev/null +++ b/backend/app/pipeline/passages.py @@ -0,0 +1,98 @@ +"""Feeding posts to whole-document modules: numbered passages in, verified SourceRefs out.""" + +import uuid +from types import SimpleNamespace + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.corpus import Corpus, CorpusError, ScopedTools +from app.models import Document, NotebookDocument, VoiceProfile + +DEFAULT_BUDGET = 120_000 # characters across all passages; GLM-5.3 has room, this keeps cost sane + + +def notebook_docs(db: Session, notebook_id: uuid.UUID) -> list[Document]: + return list(db.scalars( + select(Document) + .join(NotebookDocument, NotebookDocument.document_id == Document.id) + .where(NotebookDocument.notebook_id == notebook_id, Document.path.is_not(None)) + .order_by(Document.published_at.desc()) + ).all()) + + +def workspace_docs(db: Session, workspace_id: uuid.UUID, ids: list[uuid.UUID] | None = None) -> list[Document]: + q = select(Document).where(Document.workspace_id == workspace_id, Document.path.is_not(None)) + if ids is not None: + q = q.where(Document.id.in_(ids)) + return list(db.scalars(q.order_by(Document.published_at.desc())).all()) + + +def passages_for(corpus: Corpus, docs: list[Document], budget_chars: int = DEFAULT_BUDGET) -> list[str]: + """One string per post: a header line then `n| text` numbered lines (same numbering as read()). + The budget is split evenly, so one long post cannot crowd out the rest.""" + corpus.sync() + per_doc = max(budget_chars // max(len(docs), 1), 1500) + out = [] + for d in docs: + try: + lines = corpus.read_lines(d.path) + except CorpusError: + continue + body, used = [], 0 + for n, line in enumerate(lines, start=1): + if not line.strip(): + continue + entry = f"{n}| {line}" + if used + len(entry) > per_doc: + body.append("[... post continues]") + break + body.append(entry) + used += len(entry) + 1 + out.append(f"FILE {d.path} ({d.title})\n" + "\n".join(body)) + return out + + +def tools_for(corpus: Corpus, docs: list[Document]) -> ScopedTools: + return ScopedTools(corpus, {d.path: d.title for d in docs}) + + +def verify_refs(tools: ScopedTools, refs: list[dict]) -> list[dict]: + """Keep refs that point at real lines of allowed files; attach the quoted text.""" + out = [] + for r in refs or []: + try: + start, end = int(r["line_start"]), int(r["line_end"]) + except (KeyError, TypeError, ValueError): + continue + quote = tools.quote(r.get("path", ""), start, end) + if quote: + out.append({"path": r["path"], "line_start": start, "line_end": max(start, end), + "title": tools.allowed.get(r["path"]), "quote": quote[:600]}) + return out + + +def as_citations(items: list[dict]) -> list[SimpleNamespace]: + """Dicts from predict() in the shape research.verify_citations expects.""" + out = [] + for c in items or []: + try: + out.append(SimpleNamespace(marker=int(c["marker"]), path=c["path"], + line_start=int(c["line_start"]), line_end=int(c["line_end"]))) + except (KeyError, TypeError, ValueError): + continue + return out + + +def voice_text(db: Session, workspace_id: uuid.UUID) -> str: + vp = db.scalar(select(VoiceProfile).where(VoiceProfile.workspace_id == workspace_id)) + if not vp or not vp.profile_json: + return "No voice profile yet: write clearly, warmly and concretely." + p = vp.profile_json + parts = [p.get("summary", "")] + for key in ("tone", "vocabulary", "structure_habits", "openings", "avoid"): + if p.get(key): + parts.append(f"{key.replace('_', ' ')}: {', '.join(p[key])}") + if p.get("sentence_length"): + parts.append(f"sentence length: {p['sentence_length']}") + return "\n".join(x for x in parts if x) diff --git a/backend/app/routers/artifacts.py b/backend/app/routers/artifacts.py index 180eeecee0293c290593965903e2916122385264..f32d1befb8456fb31d18bbd9b820ef5a5a4896ee 100644 --- a/backend/app/routers/artifacts.py +++ b/backend/app/routers/artifacts.py @@ -2,79 +2,205 @@ import uuid from typing import Literal from fastapi import APIRouter, Depends, HTTPException -from pydantic import BaseModel -from sqlalchemy import select +from pydantic import BaseModel, Field +from sqlalchemy import func, or_, select from app.auth import Ctx, get_ctx -from app.models import Artifact +from app.models import Artifact, CalendarItem, Document, Job, Notebook +from app.services.artifacts import latest_jobs, serialize_artifact, start_artifact from app.services.jobs import create_job, serialize_job +from app.services.renderer import COMPOSITIONS from app.services.storage import storage +from app.services.usage import check_limit router = APIRouter(prefix="/api/artifacts", tags=["artifacts"]) -COMPOSITIONS = {"ShortVertical": "mp4", "QuoteCard": "png"} - - -class ArtifactIn(BaseModel): - type: Literal["summary", "audio_overview", "video", "thread", "linkedin", "notes", "quote_card", "carousel", "seo"] - notebook_id: uuid.UUID | None = None - document_id: uuid.UUID | None = None - content: dict = {} +ArtifactType = Literal["summary", "audio_overview", "video", "quote_card", "carousel", "launch_kit"] class RenderIn(BaseModel): - composition: Literal["ShortVertical", "QuoteCard"] + composition: Literal["ShortVertical", "ExplainerLong", "AudiogramSquare", "QuoteCard", "CarouselSlide"] props: dict -def _serialize(a: Artifact) -> dict: - return { - "id": str(a.id), - "type": a.type, - "status": a.status, - "content": a.content_json, - "storage_key": a.storage_key, - "url": storage.presign_get(a.storage_key) if a.storage_key else None, - } +class GenerateIn(BaseModel): + type: Literal["summary", "audio_overview", "video", "quote_card", "carousel", "launch_kit"] + notebook_id: uuid.UUID | None = None + document_id: uuid.UUID | None = None + # audio_overview + format: Literal["deep_dive", "brief", "debate"] = "deep_dive" + minutes: int = Field(6, ge=1, le=30) + # video + style: Literal["short", "explainer", "audiogram"] = "short" + audio_artifact_id: uuid.UUID | None = None + # carousel + slides: list[dict] | None = None + parent_id: uuid.UUID | None = None + +class PatchIn(BaseModel): + title: str | None = Field(None, max_length=300) + content: dict | None = None -def _get(ctx: Ctx, artifact_id: uuid.UUID) -> Artifact: + +def get_artifact_or_404(ctx: Ctx, artifact_id: uuid.UUID) -> Artifact: a = ctx.db.scalar(select(Artifact).where(Artifact.id == artifact_id, Artifact.workspace_id == ctx.workspace.id)) if not a: raise HTTPException(404, "Artifact not found") return a -@router.post("") -def create_artifact(body: ArtifactIn, ctx: Ctx = Depends(get_ctx)): - a = Artifact( - workspace_id=ctx.workspace.id, - type=body.type, - notebook_id=body.notebook_id, - document_id=body.document_id, - content_json=body.content, - status="draft", - ) - ctx.db.add(a) - ctx.db.commit() - return _serialize(a) +def _target_title(ctx: Ctx, notebook_id: uuid.UUID | None, document_id: uuid.UUID | None) -> str: + if document_id: + doc = ctx.db.scalar(select(Document).where(Document.id == document_id, + Document.workspace_id == ctx.workspace.id)) + if not doc: + raise HTTPException(404, "Post not found") + return doc.title + if notebook_id: + nb = ctx.db.scalar(select(Notebook).where(Notebook.id == notebook_id, + Notebook.workspace_id == ctx.workspace.id)) + if not nb: + raise HTTPException(404, "Notebook not found") + return nb.title + raise HTTPException(400, "Pick a notebook or a post") + + +@router.post("/generate") +def generate(body: GenerateIn, ctx: Ctx = Depends(get_ctx)): + """One entry point for every generated artifact.""" + params: dict = {} + content: dict = {} + if body.type == "video" and body.style == "audiogram": + if not body.audio_artifact_id: + raise HTTPException(400, "Pick an audio overview for the audiogram") + source = get_artifact_or_404(ctx, body.audio_artifact_id) + seconds = float((source.content_json or {}).get("duration_s", 60)) + check_limit(ctx.db, ctx.workspace, "video_minutes", seconds / 60) + title = f"Audiogram: {(source.content_json or {}).get('title', 'Audio overview')}" + params = {"style": "audiogram", "audio_artifact_id": str(source.id)} + artifact, job = start_artifact(ctx.db, ctx.workspace.id, "video", title=title, notebook_id=source.notebook_id, + document_id=source.document_id, params=params, job_kind="video") + return serialize_artifact(artifact, job) + if body.type == "carousel": + if not body.slides: + raise HTTPException(400, "A carousel needs slides") + content = {"slides": body.slides[:12], "parent_id": str(body.parent_id) if body.parent_id else None} + target = _target_title(ctx, body.notebook_id, body.document_id) + if body.type == "audio_overview": + check_limit(ctx.db, ctx.workspace, "audio_minutes", body.minutes) + params = {"format": body.format, "minutes": body.minutes} + title = f"Audio overview: {target}" + elif body.type == "video": + check_limit(ctx.db, ctx.workspace, "video_minutes", 1 if body.style == "short" else 3) + params = {"style": body.style} + title = f"{'Short' if body.style == 'short' else 'Explainer'} video: {target}" + elif body.type == "launch_kit": + if not body.document_id: + raise HTTPException(400, "A Launch Kit is made from one post") + check_limit(ctx.db, ctx.workspace, "launch_kits", 1) + title = f"Launch Kit: {target}" + else: + title = {"summary": "Summary", "quote_card": "Quote", "carousel": "Carousel"}[body.type] + f": {target}" + artifact, job = start_artifact(ctx.db, ctx.workspace.id, body.type, title=title, notebook_id=body.notebook_id, + document_id=body.document_id, content=content, params=params, + max_attempts=1 if body.type in {"quote_card", "carousel"} else 2) + return serialize_artifact(artifact, job) + + +@router.get("") +def list_artifacts( + ctx: Ctx = Depends(get_ctx), + type: str | None = None, + notebook_id: uuid.UUID | None = None, + document_id: uuid.UUID | None = None, + status: str | None = None, + q: str = "", + limit: int = 60, + offset: int = 0, +): + query = select(Artifact).where(Artifact.workspace_id == ctx.workspace.id) + if type: + query = query.where(Artifact.type.in_(type.split(","))) + if notebook_id: + query = query.where(Artifact.notebook_id == notebook_id) + if document_id: + query = query.where(Artifact.document_id == document_id) + if status: + query = query.where(Artifact.status.in_(status.split(","))) + if q.strip(): + # Titles live in JSON; match on the linked post or notebook title instead of JSON operators. + like = f"%{q.strip()}%" + docs = select(Document.id).where(Document.workspace_id == ctx.workspace.id, Document.title.ilike(like)) + nbs = select(Notebook.id).where(Notebook.workspace_id == ctx.workspace.id, Notebook.title.ilike(like)) + query = query.where(or_(Artifact.document_id.in_(docs), Artifact.notebook_id.in_(nbs))) + total = ctx.db.scalar(select(func.count()).select_from(query.subquery())) + rows = ctx.db.scalars(query.order_by(Artifact.created_at.desc()).offset(max(offset, 0)).limit(min(limit, 200))) + rows = list(rows.all()) + jobs = latest_jobs(ctx.db, [a.id for a in rows]) + return {"total": total, "items": [serialize_artifact(a, jobs.get(a.id)) for a in rows]} @router.get("/{artifact_id}") def get_artifact(artifact_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): - return _serialize(_get(ctx, artifact_id)) + a = get_artifact_or_404(ctx, artifact_id) + return serialize_artifact(a, latest_jobs(ctx.db, [a.id]).get(a.id)) + + +@router.patch("/{artifact_id}") +def patch_artifact(artifact_id: uuid.UUID, body: PatchIn, ctx: Ctx = Depends(get_ctx)): + a = get_artifact_or_404(ctx, artifact_id) + content = dict(a.content_json or {}) + if body.content is not None: + protected = {"slide_keys", "quote_card_ids", "segments", "voices"} + content.update({k: v for k, v in body.content.items() if k not in protected}) + if body.title is not None: + content["title"] = body.title + a.content_json = content + ctx.db.commit() + return serialize_artifact(a, latest_jobs(ctx.db, [a.id]).get(a.id)) + + +@router.delete("/{artifact_id}") +def delete_artifact(artifact_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + a = get_artifact_or_404(ctx, artifact_id) + storage.delete_prefix(f"ws/{ctx.workspace.id}/artifacts/{a.id}/") + ctx.db.query(CalendarItem).filter(CalendarItem.artifact_id == a.id, CalendarItem.status == "scheduled").delete() + ctx.db.delete(a) + ctx.db.commit() + return {"ok": True} + + +@router.post("/{artifact_id}/retry") +def retry_artifact(artifact_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + a = get_artifact_or_404(ctx, artifact_id) + last = latest_jobs(ctx.db, [a.id]).get(a.id) + if not last: + raise HTTPException(400, "Nothing to retry") + if last.status not in {"failed", "done"}: + raise HTTPException(409, "Still running") + a.status = "pending" + content = dict(a.content_json or {}) + content.pop("error", None) + a.content_json = content + job = create_job(ctx.db, ctx.workspace.id, last.kind, last.params, artifact_id=a.id, max_attempts=last.max_attempts) + return serialize_artifact(a, job) @router.post("/{artifact_id}/render") def render_artifact(artifact_id: uuid.UUID, body: RenderIn, ctx: Ctx = Depends(get_ctx)): - a = _get(ctx, artifact_id) + a = get_artifact_or_404(ctx, artifact_id) + if body.composition not in COMPOSITIONS: + raise HTTPException(400, "Unknown composition") a.status = "rendering" - job = create_job( - ctx.db, - ctx.workspace.id, - "render", - {"artifact_id": str(a.id), "composition": body.composition, "props": body.props, - "format": COMPOSITIONS[body.composition]}, - artifact_id=a.id, - ) + job = create_job(ctx.db, ctx.workspace.id, "render", + {"artifact_id": str(a.id), "composition": body.composition, "props": body.props}, + artifact_id=a.id, max_attempts=1) return serialize_job(job) + + +@router.get("/{artifact_id}/jobs") +def artifact_jobs(artifact_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + a = get_artifact_or_404(ctx, artifact_id) + jobs = ctx.db.scalars(select(Job).where(Job.artifact_id == a.id).order_by(Job.created_at.desc())).all() + return [serialize_job(j) for j in jobs] diff --git a/backend/app/routers/jobs.py b/backend/app/routers/jobs.py index af851fa4443514a0b115f97e12b31ac9f7893e80..9aabd867dc100dc6bb94f834793d0ff6f3d191e3 100644 --- a/backend/app/routers/jobs.py +++ b/backend/app/routers/jobs.py @@ -22,6 +22,17 @@ def _get(ctx: Ctx, job_id: uuid.UUID) -> Job: return job +@router.get("/jobs") +def list_jobs(ctx: Ctx = Depends(get_ctx), active: bool = False, kind: str | None = None, limit: int = 20): + q = select(Job).where(Job.workspace_id == ctx.workspace.id) + if active: + q = q.where(Job.status.in_(["queued", "running"])) + if kind: + q = q.where(Job.kind.in_(kind.split(","))) + jobs = ctx.db.scalars(q.order_by(Job.created_at.desc()).limit(min(limit, 100))).all() + return [serialize_job(j) for j in jobs] + + @router.get("/jobs/{job_id}") def get_job(job_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): return serialize_job(_get(ctx, job_id)) @@ -45,6 +56,7 @@ class ProgressIn(BaseModel): error: str | None = None render_seconds: float | None = None storage_key: str | None = None + storage_keys: list[str] | None = None # carousels: one PNG per slide @router.post("/internal/jobs/{job_id}/progress", include_in_schema=False) @@ -59,9 +71,16 @@ def renderer_progress( job = db.get(Job, job_id) if not job: raise HTTPException(404, "Job not found") - if body.status == "done" and job.artifact_id: - artifact = db.get(Artifact, job.artifact_id) - if artifact and body.storage_key: + artifact = db.get(Artifact, job.artifact_id) if job.artifact_id else None + if body.status == "failed" and artifact: + artifact.status = "failed" + artifact.content_json = {**(artifact.content_json or {}), "error": (body.error or "Render failed")[:500]} + if body.status == "done" and artifact: + if body.storage_keys: + artifact.content_json = {**(artifact.content_json or {}), "slide_keys": body.storage_keys} + artifact.storage_key = body.storage_keys[0] + artifact.status = "ready" + elif body.storage_key: artifact.storage_key = body.storage_key artifact.status = "ready" if body.render_seconds: diff --git a/backend/app/routers/launchpad.py b/backend/app/routers/launchpad.py new file mode 100644 index 0000000000000000000000000000000000000000..980b6caa8e55d029f9f0d55aa815df649bb089b1 --- /dev/null +++ b/backend/app/routers/launchpad.py @@ -0,0 +1,360 @@ +"""Launchpad: the calendar, connected social accounts (OAuth / app password) and tracked short links.""" + +import json +import secrets +import uuid +from datetime import UTC, datetime, timedelta +from typing import Literal +from urllib.parse import urlencode + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.concurrency import run_in_threadpool +from fastapi.responses import RedirectResponse +from pydantic import BaseModel, Field +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.auth import Ctx, decode_token_full, encode_signed, get_ctx +from app.config import settings +from app.db import get_db +from app.models import Artifact, CalendarItem, Document, EngagementEvent, SocialAccount, TrackedLink +from app.services import launchpad as lp +from app.services.crypto import decrypt, encrypt +from app.services.social import AUTO_POST, LIMITS, PLATFORM_LABELS, SocialError, bluesky, linkedin, x + +router = APIRouter(prefix="/api", tags=["launchpad"]) + +Platform = Literal["x", "linkedin", "bluesky", "substack_notes"] + + +# Calendar + + +class ItemIn(BaseModel): + platform: Platform + scheduled_at: datetime + content: str = Field(min_length=1, max_length=10000) + thread: list[str] = Field(default_factory=list, max_length=25) + artifact_id: uuid.UUID | None = None + document_id: uuid.UUID | None = None + social_account_id: uuid.UUID | None = None + remind_by_email: bool = False + draft: bool = False + + +class ItemPatch(BaseModel): + scheduled_at: datetime | None = None + content: str | None = Field(None, min_length=1, max_length=10000) + thread: list[str] | None = None + social_account_id: uuid.UUID | None = None + remind_by_email: bool | None = None + status: Literal["scheduled", "draft"] | None = None + + +def _aware(dt: datetime) -> datetime: + return dt if dt.tzinfo else dt.replace(tzinfo=UTC) + + +def serialize_item(item: CalendarItem, account: SocialAccount | None = None, clicks: int = 0) -> dict: + return { + "id": str(item.id), + "platform": item.platform, + "platform_label": PLATFORM_LABELS.get(item.platform, item.platform), + "scheduled_at": _aware(item.scheduled_at).isoformat(), + "status": item.status, + "content": item.content, + "thread": item.thread or [], + "artifact_id": str(item.artifact_id) if item.artifact_id else None, + "document_id": str(item.document_id) if item.document_id else None, + "social_account_id": str(item.social_account_id) if item.social_account_id else None, + "account_handle": account.handle if account else None, + "remind_by_email": item.remind_by_email, + "auto_post": item.platform in AUTO_POST and bool(item.social_account_id) and not item.remind_by_email, + "external_url": item.external_url, + "posted_at": _aware(item.posted_at).isoformat() if item.posted_at else None, + "error": item.error, + "metrics": item.metrics or {}, + "clicks": clicks, + } + + +def _item(ctx: Ctx, item_id: uuid.UUID) -> CalendarItem: + item = ctx.db.scalar(select(CalendarItem).where(CalendarItem.id == item_id, + CalendarItem.workspace_id == ctx.workspace.id)) + if not item: + raise HTTPException(404, "Calendar item not found") + return item + + +def _account(ctx: Ctx, account_id: uuid.UUID | None, platform: str) -> SocialAccount | None: + if not account_id: + return None + account = ctx.db.scalar(select(SocialAccount).where(SocialAccount.id == account_id, + SocialAccount.workspace_id == ctx.workspace.id)) + if not account or account.platform != platform: + raise HTTPException(400, "That account does not match the platform") + return account + + +def _validate_lengths(platform: str, posts: list[str]) -> None: + limit = LIMITS.get(platform) + if platform in ("x", "bluesky") and limit: + for i, p in enumerate(posts, start=1): + if len(p) > limit: + raise HTTPException(422, {"code": "too_long", + "message": f"Post {i} is {len(p)} characters; {PLATFORM_LABELS[platform]} " + f"allows {limit}."}) + + +def _enrich(ctx: Ctx, items: list[CalendarItem]) -> list[dict]: + accounts = {a.id: a for a in ctx.db.scalars( + select(SocialAccount).where(SocialAccount.workspace_id == ctx.workspace.id))} + clicks = lp.clicks_by_item(ctx.db, [i.id for i in items]) + return [serialize_item(i, accounts.get(i.social_account_id), clicks.get(i.id, 0)) for i in items] + + +@router.get("/calendar") +def list_items(ctx: Ctx = Depends(get_ctx), start: datetime | None = None, end: datetime | None = None, + status: str | None = None, limit: int = 500): + q = select(CalendarItem).where(CalendarItem.workspace_id == ctx.workspace.id) + if start: + q = q.where(CalendarItem.scheduled_at >= start) + if end: + q = q.where(CalendarItem.scheduled_at < end) + if status: + q = q.where(CalendarItem.status.in_(status.split(","))) + items = list(ctx.db.scalars(q.order_by(CalendarItem.scheduled_at).limit(min(limit, 1000))).all()) + return _enrich(ctx, items) + + +@router.post("/calendar") +def create_item(body: ItemIn, ctx: Ctx = Depends(get_ctx)): + account = _account(ctx, body.social_account_id, body.platform) + _validate_lengths(body.platform, [body.content, *body.thread]) + for model, value in ((Artifact, body.artifact_id), (Document, body.document_id)): + if value and not ctx.db.scalar(select(model.id).where(model.id == value, + model.workspace_id == ctx.workspace.id)): + raise HTTPException(404, f"{model.__name__} not found") + item = CalendarItem( + workspace_id=ctx.workspace.id, platform=body.platform, scheduled_at=_aware(body.scheduled_at), + content=body.content, thread=[t for t in body.thread if t.strip()], artifact_id=body.artifact_id, + document_id=body.document_id, social_account_id=account.id if account else None, + remind_by_email=body.remind_by_email or body.platform not in AUTO_POST or not account, + status="draft" if body.draft else "scheduled", + ) + ctx.db.add(item) + ctx.db.commit() + return _enrich(ctx, [item])[0] + + +@router.patch("/calendar/{item_id}") +def update_item(item_id: uuid.UUID, body: ItemPatch, ctx: Ctx = Depends(get_ctx)): + item = _item(ctx, item_id) + if item.status in ("posted", "publishing"): + raise HTTPException(409, "This post already went out.") + if body.scheduled_at is not None: + item.scheduled_at = _aware(body.scheduled_at) + if body.content is not None: + item.content = body.content + if body.thread is not None: + item.thread = [t for t in body.thread if t.strip()] + if "social_account_id" in body.model_fields_set: + item.social_account_id = (_account(ctx, body.social_account_id, item.platform).id + if body.social_account_id else None) + if body.remind_by_email is not None: + item.remind_by_email = body.remind_by_email + if not item.social_account_id or item.platform not in AUTO_POST: + item.remind_by_email = True + if body.status is not None: + item.status = body.status + elif item.status in ("failed", "reminded"): + item.status = "scheduled" + item.error = None + item.attempts = 0 + _validate_lengths(item.platform, [item.content, *(item.thread or [])]) + ctx.db.commit() + return _enrich(ctx, [item])[0] + + +@router.delete("/calendar/{item_id}") +def delete_item(item_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + item = _item(ctx, item_id) + ctx.db.delete(item) + ctx.db.commit() + return {"ok": True} + + +@router.post("/calendar/{item_id}/publish") +async def publish_now(item_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + item = _item(ctx, item_id) + if item.status in ("posted", "publishing"): + raise HTTPException(409, "This post already went out.") + item.status = "publishing" + item.attempts = 0 + ctx.db.commit() + await run_in_threadpool(lp.publish_item, ctx.db, item) + return _enrich(ctx, [item])[0] + + +# Social accounts + + +def serialize_account(a: SocialAccount) -> dict: + return {"id": str(a.id), "platform": a.platform, "platform_label": PLATFORM_LABELS[a.platform], + "handle": a.handle, "status": a.status, "avatar_url": a.avatar_url, + "connected_at": a.created_at.isoformat() if a.created_at else None} + + +@router.get("/social/accounts") +def list_accounts(ctx: Ctx = Depends(get_ctx)): + accounts = ctx.db.scalars(select(SocialAccount).where(SocialAccount.workspace_id == ctx.workspace.id) + .order_by(SocialAccount.created_at)).all() + return { + "accounts": [serialize_account(a) for a in accounts], + "available": {"x": x.configured(), "linkedin": linkedin.configured(), "bluesky": True}, + "redirect_uris": {"x": x.redirect_uri(), "linkedin": linkedin.redirect_uri()}, + } + + +@router.delete("/social/accounts/{account_id}") +def disconnect(account_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + account = ctx.db.scalar(select(SocialAccount).where(SocialAccount.id == account_id, + SocialAccount.workspace_id == ctx.workspace.id)) + if not account: + raise HTTPException(404, "Account not found") + for item in ctx.db.scalars(select(CalendarItem).where(CalendarItem.social_account_id == account.id, + CalendarItem.status == "scheduled")): + item.social_account_id = None + item.remind_by_email = True + ctx.db.delete(account) + ctx.db.commit() + return {"ok": True} + + +def _upsert_account(db: Session, workspace_id: uuid.UUID, platform: str, info: dict, secret: str, + refresh: str | None = None, expires_in: int | None = None, scopes: str | None = None) -> None: + account = db.scalar(select(SocialAccount).where(SocialAccount.workspace_id == workspace_id, + SocialAccount.platform == platform, + SocialAccount.external_id == info["external_id"])) + if not account: + account = SocialAccount(workspace_id=workspace_id, platform=platform, external_id=info["external_id"], + handle=info["handle"], access_token="") + db.add(account) + account.handle = info["handle"] + account.avatar_url = info.get("avatar_url") + account.access_token = encrypt(secret) + account.refresh_token = encrypt(refresh) if refresh else account.refresh_token + account.expires_at = datetime.now(UTC) + timedelta(seconds=int(expires_in)) if expires_in else None + account.scopes = scopes + account.status = "active" + db.commit() + + +class BlueskyIn(BaseModel): + handle: str = Field(min_length=3, max_length=200) + app_password: str = Field(min_length=8, max_length=100) + + +@router.post("/social/bluesky") +async def connect_bluesky(body: BlueskyIn, ctx: Ctx = Depends(get_ctx)): + try: + session = await run_in_threadpool(bluesky.create_session, body.handle, body.app_password) + except SocialError as exc: + raise HTTPException(400, {"code": "bluesky_auth", "message": str(exc)}) from exc + _upsert_account(ctx.db, ctx.workspace.id, "bluesky", + {"external_id": session["did"], "handle": session.get("handle", body.handle)}, body.app_password) + return list_accounts(ctx) + + +# OAuth (X, LinkedIn). The signed state carries the workspace and PKCE verifier; the callback hands a +# short lived encrypted ticket back to the SPA, which completes the link as the signed in user. + +STATE_TTL = timedelta(minutes=10) + + +def _launchpad_redirect(**params: str) -> RedirectResponse: + return RedirectResponse(f"{settings.frontend_url.rstrip('/')}/app/launchpad?{urlencode(params)}", 302) + + +@router.post("/social/{platform}/start") +def oauth_start(platform: Literal["x", "linkedin"], ctx: Ctx = Depends(get_ctx)): + """Returns the provider URL. The SPA calls this with its bearer token, then navigates to `url`.""" + mod = x if platform == "x" else linkedin + if not mod.configured(): + raise HTTPException(503, {"code": "not_configured", + "message": f"Set {platform.upper()}_CLIENT_ID and {platform.upper()}_CLIENT_SECRET."}) + nonce = secrets.token_urlsafe(24) + payload = {"typ": "social_state", "nonce": nonce, "ws": str(ctx.workspace.id), "platform": platform} + if platform == "x": + verifier, challenge = x.pkce_pair() + payload["verifier"] = verifier + state = encode_signed(payload, STATE_TTL) + url = x.authorize_url(state, challenge) + else: + state = encode_signed(payload, STATE_TTL) + url = linkedin.authorize_url(state) + return {"url": url, "nonce": nonce} + + +@router.get("/social/{platform}/callback") +def oauth_callback(platform: Literal["x", "linkedin"], code: str | None = None, state: str | None = None, + error: str | None = None, db: Session = Depends(get_db)): + label = PLATFORM_LABELS[platform] + if error or not code or not state: + return _launchpad_redirect(error=f"{label} connection was cancelled.") + try: + saved = decode_token_full(state, expected_type="social_state") + except HTTPException: + return _launchpad_redirect(error=f"{label} connection expired. Try again.") + if saved.get("platform") != platform: + return _launchpad_redirect(error=f"{label} connection failed.") + try: + if platform == "x": + info = x.exchange_code(code, saved["verifier"]) + else: + info = linkedin.exchange_code(code) + except Exception: + return _launchpad_redirect(error=f"{label} connection failed. Try again.") + # Do not link here: this request carries no session. The signed in SPA finishes the link, and only + # for the workspace that started it, so a forwarded authorize link cannot attach someone's account + # to another workspace. + ticket = encode_signed({"typ": "social_link", "ws": saved["ws"], "platform": platform, + "data": encrypt(json.dumps(info))}, timedelta(minutes=10)) + return _launchpad_redirect(link=ticket, platform=platform) + + +class LinkIn(BaseModel): + ticket: str + + +@router.post("/social/complete") +def oauth_complete(body: LinkIn, ctx: Ctx = Depends(get_ctx)): + try: + saved = decode_token_full(body.ticket, expected_type="social_link") + except HTTPException as exc: + raise HTTPException(400, {"code": "expired", "message": "That connection expired. Try again."}) from exc + if saved.get("ws") != str(ctx.workspace.id): + raise HTTPException(403, {"code": "wrong_workspace", "message": "This connection was started elsewhere."}) + info = json.loads(decrypt(saved["data"]) or "{}") + if not info.get("access_token"): + raise HTTPException(400, {"code": "expired", "message": "That connection expired. Try again."}) + _upsert_account(ctx.db, ctx.workspace.id, saved["platform"], info, info["access_token"], + info.get("refresh_token"), info.get("expires_in"), info.get("scope")) + return list_accounts(ctx) + + +# Tracked links (public) + +links_router = APIRouter(tags=["links"]) + + +@links_router.get("/l/{slug}") +def follow_link(slug: str, request: Request, db: Session = Depends(get_db)): + link = db.scalar(select(TrackedLink).where(TrackedLink.slug == slug)) + if not link: + raise HTTPException(404, "Link not found") + db.add(EngagementEvent(workspace_id=link.workspace_id, tracked_link_id=link.id, kind="click", + metadata_json={"referer": request.headers.get("referer", "")[:300], + "ua": request.headers.get("user-agent", "")[:200]})) + db.commit() + return RedirectResponse(link.target_url, status_code=302) diff --git a/backend/app/routers/notebooks.py b/backend/app/routers/notebooks.py index 4e882f016a9528451c59f0ccf2f660f43e1f19fa..da444eb3531ff153fea1163cdc73437624108159 100644 --- a/backend/app/routers/notebooks.py +++ b/backend/app/routers/notebooks.py @@ -1,18 +1,21 @@ import asyncio import json import uuid +from datetime import UTC, datetime from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field -from sqlalchemy import select +from sqlalchemy import delete, func, select from app.auth import Ctx, get_ctx from app.config import settings from app.corpus import Corpus from app.llm.provider import provider_name -from app.models import Chat, Citation, Document, Message, Notebook, NotebookDocument +from app.models import Artifact, Chat, Citation, Document, DocumentTopic, Message, Notebook, NotebookDocument, Topic from app.pipeline.research import research +from app.routers.sources import serialize_document +from app.services.artifacts import latest_jobs, serialize_artifact from app.services.jobs import record_usage router = APIRouter(prefix="/api/notebooks", tags=["notebooks"]) @@ -21,6 +24,12 @@ router = APIRouter(prefix="/api/notebooks", tags=["notebooks"]) class NotebookIn(BaseModel): title: str = Field(min_length=1, max_length=300) description: str | None = None + document_ids: list[uuid.UUID] = [] + + +class NotebookPatch(BaseModel): + title: str | None = Field(None, min_length=1, max_length=300) + description: str | None = None class DocumentsIn(BaseModel): @@ -41,12 +50,38 @@ def _get(ctx: Ctx, notebook_id: uuid.UUID) -> Notebook: return nb +def _add_docs(ctx: Ctx, nb: Notebook, ids: list[uuid.UUID]) -> int: + valid = set(ctx.db.scalars( + select(Document.id).where(Document.id.in_(ids), Document.workspace_id == ctx.workspace.id) + ).all()) + existing = set(ctx.db.scalars(select(NotebookDocument.document_id).where(NotebookDocument.notebook_id == nb.id))) + for doc_id in valid - existing: + ctx.db.add(NotebookDocument(notebook_id=nb.id, document_id=doc_id)) + return len(valid - existing) + + @router.post("") def create_notebook(body: NotebookIn, ctx: Ctx = Depends(get_ctx)): - nb = Notebook(workspace_id=ctx.workspace.id, title=body.title, description=body.description) + nb = Notebook(workspace_id=ctx.workspace.id, title=body.title.strip(), description=body.description) + ctx.db.add(nb) + ctx.db.flush() + added = _add_docs(ctx, nb, body.document_ids) + ctx.db.commit() + return {"id": str(nb.id), "title": nb.title, "added": added} + + +@router.post("/from-topic/{topic_id}") +def notebook_from_topic(topic_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + topic = ctx.db.scalar(select(Topic).where(Topic.id == topic_id, Topic.workspace_id == ctx.workspace.id)) + if not topic: + raise HTTPException(404, "Topic not found") + ids = list(ctx.db.scalars(select(DocumentTopic.document_id).where(DocumentTopic.topic_id == topic.id))) + nb = Notebook(workspace_id=ctx.workspace.id, title=topic.name, description=topic.summary) ctx.db.add(nb) + ctx.db.flush() + added = _add_docs(ctx, nb, ids) ctx.db.commit() - return {"id": str(nb.id), "title": nb.title} + return {"id": str(nb.id), "title": nb.title, "added": added} @router.get("") @@ -54,7 +89,24 @@ def list_notebooks(ctx: Ctx = Depends(get_ctx)): rows = ctx.db.scalars( select(Notebook).where(Notebook.workspace_id == ctx.workspace.id).order_by(Notebook.updated_at.desc()) ).all() - return [{"id": str(n.id), "title": n.title, "description": n.description} for n in rows] + ids = [n.id for n in rows] + doc_counts = dict(ctx.db.execute( + select(NotebookDocument.notebook_id, func.count()).where(NotebookDocument.notebook_id.in_(ids)) + .group_by(NotebookDocument.notebook_id) + ).all()) if ids else {} + chat_counts = dict(ctx.db.execute( + select(Chat.notebook_id, func.count()).where(Chat.notebook_id.in_(ids)).group_by(Chat.notebook_id) + ).all()) if ids else {} + artifact_counts = dict(ctx.db.execute( + select(Artifact.notebook_id, func.count()).where(Artifact.notebook_id.in_(ids)).group_by(Artifact.notebook_id) + ).all()) if ids else {} + return [ + {"id": str(n.id), "title": n.title, "description": n.description, "summary": n.summary, + "document_count": doc_counts.get(n.id, 0), "chat_count": chat_counts.get(n.id, 0), + "artifact_count": artifact_counts.get(n.id, 0), + "updated_at": n.updated_at.isoformat() if n.updated_at else None} + for n in rows + ] @router.get("/{notebook_id}") @@ -71,29 +123,96 @@ def get_notebook(notebook_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): "title": nb.title, "description": nb.description, "summary": nb.summary, - "documents": [ - {"id": str(d.id), "title": d.title, "url": d.url, - "published_at": d.published_at.isoformat() if d.published_at else None} - for d in docs - ], + "documents": [serialize_document(d) for d in docs], } +@router.patch("/{notebook_id}") +def patch_notebook(notebook_id: uuid.UUID, body: NotebookPatch, ctx: Ctx = Depends(get_ctx)): + nb = _get(ctx, notebook_id) + if body.title is not None: + nb.title = body.title.strip() + if body.description is not None: + nb.description = body.description + ctx.db.commit() + return {"id": str(nb.id), "title": nb.title, "description": nb.description} + + +@router.delete("/{notebook_id}") +def delete_notebook(notebook_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + nb = _get(ctx, notebook_id) + ctx.db.execute(delete(NotebookDocument).where(NotebookDocument.notebook_id == nb.id)) + ctx.db.delete(nb) + ctx.db.commit() + return {"ok": True} + + @router.post("/{notebook_id}/documents") def add_documents(notebook_id: uuid.UUID, body: DocumentsIn, ctx: Ctx = Depends(get_ctx)): nb = _get(ctx, notebook_id) - valid = set( - ctx.db.scalars( - select(Document.id).where(Document.id.in_(body.document_ids), Document.workspace_id == ctx.workspace.id) - ).all() - ) - existing = set( - ctx.db.scalars(select(NotebookDocument.document_id).where(NotebookDocument.notebook_id == nb.id)).all() - ) - for doc_id in valid - existing: - ctx.db.add(NotebookDocument(notebook_id=nb.id, document_id=doc_id)) + added = _add_docs(ctx, nb, body.document_ids) + ctx.db.commit() + return {"added": added} + + +@router.delete("/{notebook_id}/documents/{document_id}") +def remove_document(notebook_id: uuid.UUID, document_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + nb = _get(ctx, notebook_id) + ctx.db.execute(delete(NotebookDocument).where(NotebookDocument.notebook_id == nb.id, + NotebookDocument.document_id == document_id)) + ctx.db.commit() + return {"ok": True} + + +@router.get("/{notebook_id}/artifacts") +def notebook_artifacts(notebook_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + nb = _get(ctx, notebook_id) + rows = list(ctx.db.scalars( + select(Artifact).where(Artifact.notebook_id == nb.id).order_by(Artifact.created_at.desc()) + ).all()) + jobs = latest_jobs(ctx.db, [a.id for a in rows]) + return [serialize_artifact(a, jobs.get(a.id)) for a in rows] + + +@router.get("/{notebook_id}/chats") +def list_chats(notebook_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + nb = _get(ctx, notebook_id) + chats = ctx.db.scalars(select(Chat).where(Chat.notebook_id == nb.id).order_by(Chat.updated_at.desc())).all() + return [{"id": str(c.id), "title": c.title, "updated_at": c.updated_at.isoformat() if c.updated_at else None} + for c in chats] + + +def _get_chat(ctx: Ctx, chat_id: uuid.UUID) -> Chat: + chat = ctx.db.scalar(select(Chat).where(Chat.id == chat_id, Chat.workspace_id == ctx.workspace.id)) + if not chat: + raise HTTPException(404, "Chat not found") + return chat + + +@router.get("/chats/{chat_id}/messages") +def chat_messages(chat_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + chat = _get_chat(ctx, chat_id) + msgs = ctx.db.scalars(select(Message).where(Message.chat_id == chat.id).order_by(Message.created_at)).all() + doc_ids = {c.document_id for m in msgs for c in m.citations} + docs = {d.id: d for d in ctx.db.scalars(select(Document).where(Document.id.in_(doc_ids)))} if doc_ids else {} + out = [] + for m in msgs: + cites = [] + for c in sorted(m.citations, key=lambda c: c.marker): + d = docs.get(c.document_id) + cites.append({"marker": c.marker, "document_id": str(c.document_id), "title": d.title if d else "", + "url": d.url if d else "", "path": c.path, "line_start": c.line_start, + "line_end": c.line_end, "span": c.span or ""}) + out.append({"id": str(m.id), "role": m.role, "text": m.content, "citations": cites}) + return out + + +@router.delete("/chats/{chat_id}") +def delete_chat(chat_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + chat = _get_chat(ctx, chat_id) + ctx.db.delete(chat) ctx.db.commit() - return {"added": len(valid - existing)} + return {"ok": True} def _sse(event: str, data: dict) -> str: @@ -112,6 +231,7 @@ async def chat(notebook_id: uuid.UUID, body: ChatIn, ctx: Ctx = Depends(get_ctx) chat_row = Chat(workspace_id=ctx.workspace.id, notebook_id=nb.id, title=body.question[:120]) db.add(chat_row) db.flush() + chat_row.updated_at = datetime.now(UTC) # keeps the history list newest first db.add(Message(chat_id=chat_row.id, role="user", content=body.question)) db.commit() diff --git a/backend/app/routers/resurface.py b/backend/app/routers/resurface.py new file mode 100644 index 0000000000000000000000000000000000000000..dcdc9952ebf0357649ff2e5812c92558015d1492 --- /dev/null +++ b/backend/app/routers/resurface.py @@ -0,0 +1,53 @@ +from datetime import UTC, datetime, timedelta + +from fastapi import APIRouter, Depends +from sqlalchemy import select + +from app.auth import Ctx, get_ctx +from app.models import Document, Source +from app.routers.sources import serialize_document +from app.services.jobs import create_job, serialize_job + +router = APIRouter(prefix="/api/resurface", tags=["resurface"]) + +COOLDOWN = timedelta(days=90) + + +def _aware(dt: datetime | None) -> datetime | None: + return dt if dt is None or dt.tzinfo else dt.replace(tzinfo=UTC) + + +@router.get("") +def suggestions(ctx: Ctx = Depends(get_ctx), limit: int = 20): + now = datetime.now(UTC) + rows = ctx.db.execute( + select(Document, Source).outerjoin(Source, Source.id == Document.source_id) + .where(Document.workspace_id == ctx.workspace.id, Document.path.is_not(None)) + ).all() + evergreen, on_this_day = [], [] + unscored = 0 + for d, s in rows: + published = _aware(d.published_at) + resurfaced = _aware(d.last_resurfaced_at) + meta = d.metadata_json or {} + item = {**serialize_document(d, s), "reason": meta.get("evergreen_reason"), "angle": meta.get("reshare_angle"), + "last_resurfaced_at": resurfaced.isoformat() if resurfaced else None} + if published and published.month == now.month and published.day == now.day and published.year < now.year: + on_this_day.append({**item, "years_ago": now.year - published.year}) + if d.evergreen_score is None: + unscored += 1 + continue + if resurfaced and now - resurfaced < COOLDOWN: + continue + age_days = (now - published).days if published else 365 + # Old, timeless and not recently reshared ranks highest; brand new posts do not need resurfacing. + age_factor = min(age_days / 365, 2.0) if age_days > 30 else 0.1 + evergreen.append({**item, "rank": round(d.evergreen_score * (0.5 + age_factor / 2), 3)}) + evergreen.sort(key=lambda x: -x["rank"]) + return {"evergreen": evergreen[: min(limit, 100)], "on_this_day": on_this_day, "unscored": unscored, + "total_posts": len(rows)} + + +@router.post("/scan") +def scan(ctx: Ctx = Depends(get_ctx)): + return serialize_job(create_job(ctx.db, ctx.workspace.id, "resurface_scan", {}, max_attempts=2)) diff --git a/backend/app/routers/settings.py b/backend/app/routers/settings.py new file mode 100644 index 0000000000000000000000000000000000000000..be3c260885358df91cdb95b49d321d20f8ccdf03 --- /dev/null +++ b/backend/app/routers/settings.py @@ -0,0 +1,84 @@ +import re + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field +from sqlalchemy import select + +from app.auth import Ctx, get_ctx +from app.config import settings as app_settings +from app.models import Upload +from app.services.plans import effective_plan, plan_dict +from app.services.storage import storage +from app.services.usage import usage_report + +router = APIRouter(prefix="/api/settings", tags=["settings"]) + + +class SettingsIn(BaseModel): + name: str | None = Field(None, max_length=200) + workspace_name: str | None = Field(None, min_length=1, max_length=200) + brand_name: str | None = Field(None, max_length=120) + brand_accent: str | None = None + logo_upload_id: str | None = None # "" removes the logo + training_opt_in: bool | None = None + email_unsubscribed: bool | None = None + + +def _serialize(ctx: Ctx) -> dict: + brand = ctx.workspace.brand_json or {} + return { + "user": {"name": ctx.user.name, "email": ctx.user.email, "auth_provider": ctx.user.auth_provider.value, + "email_unsubscribed": ctx.user.email_unsubscribed}, + "workspace": {"id": str(ctx.workspace.id), "name": ctx.workspace.name, + "training_opt_in": ctx.workspace.training_opt_in}, + "brand": {"name": brand.get("name"), "accent": brand.get("accent") or "#217cff", + "logo_url": storage.presign_get(brand["logo_key"]) if brand.get("logo_key") else None}, + "plan": plan_dict(effective_plan(ctx.db, ctx.workspace)), + "billing_enabled": app_settings.billing_enabled, + "usage": usage_report(ctx.db, ctx.workspace), + "integrations": {"llm": bool(app_settings.llm_api_key), "elevenlabs": bool(app_settings.elevenlabs_api_key), + "x": bool(app_settings.x_client_id), "linkedin": bool(app_settings.linkedin_client_id), + "storage": "local" if app_settings.use_local_storage else "r2", + "renderer": app_settings.renderer_url}, + } + + +@router.get("") +def get_settings(ctx: Ctx = Depends(get_ctx)): + return _serialize(ctx) + + +@router.patch("") +def update_settings(body: SettingsIn, ctx: Ctx = Depends(get_ctx)): + if body.name is not None: + ctx.user.name = body.name.strip() or None + if body.workspace_name is not None: + ctx.workspace.name = body.workspace_name.strip() + if body.training_opt_in is not None: + ctx.workspace.training_opt_in = body.training_opt_in + if body.email_unsubscribed is not None: + ctx.user.email_unsubscribed = body.email_unsubscribed + brand = dict(ctx.workspace.brand_json or {}) + if body.brand_name is not None: + brand["name"] = body.brand_name.strip() or None + if body.brand_accent is not None: + if not re.fullmatch(r"#[0-9a-fA-F]{6}", body.brand_accent): + raise HTTPException(422, "Use a hex color like #217cff") + brand["accent"] = body.brand_accent.lower() + if body.logo_upload_id is not None: + if body.logo_upload_id == "": + brand.pop("logo_key", None) + else: + upload = ctx.db.scalar(select(Upload).where(Upload.id == body.logo_upload_id, + Upload.workspace_id == ctx.workspace.id)) + if not upload or upload.status != "complete" or not upload.content_type.startswith("image/"): + raise HTTPException(400, "Upload an image first") + brand["logo_key"] = upload.key + ctx.workspace.brand_json = brand + ctx.db.commit() + return _serialize(ctx) + + +@router.get("/usage") +def usage(ctx: Ctx = Depends(get_ctx)): + return usage_report(ctx.db, ctx.workspace) diff --git a/backend/app/routers/sources.py b/backend/app/routers/sources.py index 6cb5c593300eeba7d56b1b2070ad9cf02ca83d02..57e17f79b3073e26fce7d3d167c31e4738636966 100644 --- a/backend/app/routers/sources.py +++ b/backend/app/routers/sources.py @@ -1,57 +1,85 @@ import uuid from fastapi import APIRouter, Depends, HTTPException -from pydantic import BaseModel -from sqlalchemy import func, select +from fastapi.concurrency import run_in_threadpool +from pydantic import BaseModel, Field +from sqlalchemy import delete, func, or_, select from app.auth import Ctx, get_ctx -from app.models import Document, Job, Source -from app.pipeline.ingest import normalize_feed_url +from app.corpus import INDEX, Corpus, CorpusError +from app.models import Document, Job, NotebookDocument, Source, Upload +from app.pipeline.ingest import IMPORTS_FEED, FeedNotFound, discover_feed, imports_source, rebuild_index from app.services.jobs import create_job, serialize_job from app.services.plans import effective_plan +from app.services.storage import storage router = APIRouter(prefix="/api/sources", tags=["sources"]) class SourceIn(BaseModel): - url: str + url: str = Field(min_length=3, max_length=1000) -def _serialize(source: Source, doc_count: int = 0) -> dict: +class UploadImportIn(BaseModel): + upload_id: uuid.UUID + + +def serialize_source(source: Source, doc_count: int = 0) -> dict: return { "id": str(source.id), "feed_url": source.feed_url, + "site_url": source.site_url, "platform": source.platform, "title": source.title, "sync_status": source.sync_status, "sync_error": source.sync_error, "last_synced_at": source.last_synced_at.isoformat() if source.last_synced_at else None, "document_count": doc_count, + "is_imports": source.feed_url == IMPORTS_FEED, } +def _get(ctx: Ctx, source_id: uuid.UUID) -> Source: + source = ctx.db.scalar(select(Source).where(Source.id == source_id, Source.workspace_id == ctx.workspace.id)) + if not source: + raise HTTPException(404, "Source not found") + return source + + def _enqueue_sync(ctx: Ctx, source: Source) -> Job: + source.sync_status = "pending" return create_job(ctx.db, ctx.workspace.id, "ingest", {"source_id": str(source.id)}) +def _doc_count(ctx: Ctx, source: Source) -> int: + return ctx.db.scalar(select(func.count()).select_from(Document).where(Document.source_id == source.id)) or 0 + + @router.post("") -def create_source(body: SourceIn, ctx: Ctx = Depends(get_ctx)): - feed_url, platform = normalize_feed_url(body.url) +async def create_source(body: SourceIn, ctx: Ctx = Depends(get_ctx)): + try: + found = await run_in_threadpool(discover_feed, body.url) + except FeedNotFound as exc: + raise HTTPException(422, {"code": "feed_not_found", "message": str(exc)}) from exc existing = ctx.db.scalar( - select(Source).where(Source.workspace_id == ctx.workspace.id, Source.feed_url == feed_url) + select(Source).where(Source.workspace_id == ctx.workspace.id, Source.feed_url == found.feed_url) ) if existing: job = _enqueue_sync(ctx, existing) - return {"source": _serialize(existing), "job": serialize_job(job)} + return {"source": serialize_source(existing, _doc_count(ctx, existing)), "job": serialize_job(job)} plan = effective_plan(ctx.db, ctx.workspace) - count = ctx.db.scalar(select(func.count()).select_from(Source).where(Source.workspace_id == ctx.workspace.id)) + count = ctx.db.scalar( + select(func.count()).select_from(Source) + .where(Source.workspace_id == ctx.workspace.id, Source.feed_url != IMPORTS_FEED) + ) if count >= plan.sources: raise HTTPException(402, {"code": "plan_limit", "message": f"Your plan includes {plan.sources} sources."}) - source = Source(workspace_id=ctx.workspace.id, feed_url=feed_url, platform=platform, site_url=body.url) + source = Source(workspace_id=ctx.workspace.id, feed_url=found.feed_url, platform=found.platform, + site_url=found.site_url, title=found.title) ctx.db.add(source) ctx.db.commit() job = _enqueue_sync(ctx, source) - return {"source": _serialize(source), "job": serialize_job(job)} + return {"source": serialize_source(source), "job": serialize_job(job)} @router.get("") @@ -63,33 +91,132 @@ def list_sources(ctx: Ctx = Depends(get_ctx)): .group_by(Source.id) .order_by(Source.created_at) ).all() - return [_serialize(s, n) for s, n in rows] + return [serialize_source(s, n) for s, n in rows] -@router.get("/{source_id}/sync") +@router.post("/{source_id}/sync") def sync_source(source_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): - source = ctx.db.scalar( - select(Source).where(Source.id == source_id, Source.workspace_id == ctx.workspace.id) - ) - if not source: - raise HTTPException(404, "Source not found") + source = _get(ctx, source_id) + if source.feed_url == IMPORTS_FEED: + raise HTTPException(400, "Imported posts have no feed to sync.") job = _enqueue_sync(ctx, source) - return {"source": _serialize(source), "job": serialize_job(job)} + return {"source": serialize_source(source, _doc_count(ctx, source)), "job": serialize_job(job)} + + +def remove_documents(ctx: Ctx, docs: list[Document]) -> None: + """Delete documents plus their corpus files and raw HTML, then rebuild INDEX.md.""" + if not docs: + return + paths = {d.path for d in docs if d.path} + for d in docs: + if d.raw_html_key: + storage.delete_prefix(d.raw_html_key.rsplit("/", 1)[0]) + ids = [d.id for d in docs] + ctx.db.execute(delete(NotebookDocument).where(NotebookDocument.document_id.in_(ids))) + ctx.db.execute(delete(Document).where(Document.id.in_(ids))) + ctx.db.flush() + Corpus(ctx.workspace.id).write_files({INDEX: rebuild_index(ctx.db, ctx.workspace.id)}, remove=paths) + + +@router.delete("/{source_id}") +def delete_source(source_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + source = _get(ctx, source_id) + docs = ctx.db.scalars(select(Document).where(Document.source_id == source.id)).all() + remove_documents(ctx, list(docs)) + ctx.db.delete(source) + ctx.db.commit() + return {"ok": True, "removed_posts": len(docs)} + + +@router.post("/url") +def import_url(body: SourceIn, ctx: Ctx = Depends(get_ctx)): + source = imports_source(ctx.db, ctx.workspace.id) + ctx.db.commit() + job = create_job(ctx.db, ctx.workspace.id, "import_url", {"source_id": str(source.id), "url": body.url}) + return {"source": serialize_source(source, _doc_count(ctx, source)), "job": serialize_job(job)} + + +@router.post("/upload") +def import_upload(body: UploadImportIn, ctx: Ctx = Depends(get_ctx)): + upload = ctx.db.scalar(select(Upload).where(Upload.id == body.upload_id, Upload.workspace_id == ctx.workspace.id)) + if not upload or upload.status != "complete": + raise HTTPException(404, "Upload not found or not finished") + source = imports_source(ctx.db, ctx.workspace.id) + ctx.db.commit() + job = create_job(ctx.db, ctx.workspace.id, "import_upload", + {"source_id": str(source.id), "upload_id": str(upload.id)}) + return {"source": serialize_source(source, _doc_count(ctx, source)), "job": serialize_job(job)} documents_router = APIRouter(prefix="/api/documents", tags=["documents"]) +def serialize_document(d: Document, source: Source | None = None) -> dict: + meta = d.metadata_json or {} + return { + "id": str(d.id), + "title": d.title, + "url": d.url, + "path": d.path, + "source_id": str(d.source_id) if d.source_id else None, + "source_title": source.title if source else None, + "published_at": d.published_at.isoformat() if d.published_at else None, + "words": meta.get("words") or len((d.clean_text or "").split()), + "evergreen_score": d.evergreen_score, + } + + @documents_router.get("") -def list_documents(ctx: Ctx = Depends(get_ctx), limit: int = 500): - docs = ctx.db.scalars( - select(Document) +def list_documents( + ctx: Ctx = Depends(get_ctx), + q: str = "", + source_id: uuid.UUID | None = None, + limit: int = 500, + offset: int = 0, +): + query = ( + select(Document, Source) + .outerjoin(Source, Source.id == Document.source_id) .where(Document.workspace_id == ctx.workspace.id) - .order_by(Document.published_at.desc()) - .limit(min(limit, 5000)) + ) + if q.strip(): + like = f"%{q.strip()}%" + query = query.where(or_(Document.title.ilike(like), Document.clean_text.ilike(like))) + if source_id: + query = query.where(Document.source_id == source_id) + total = ctx.db.scalar(select(func.count()).select_from(query.subquery())) + rows = ctx.db.execute( + query.order_by(Document.published_at.desc().nulls_last(), Document.created_at.desc()) + .offset(max(offset, 0)).limit(min(limit, 5000)) ).all() - return [ - {"id": str(d.id), "title": d.title, "url": d.url, - "published_at": d.published_at.isoformat() if d.published_at else None} - for d in docs - ] + return {"total": total, "items": [serialize_document(d, s) for d, s in rows]} + + +@documents_router.get("/{document_id}") +def get_document(document_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + doc = ctx.db.scalar(select(Document).where(Document.id == document_id, Document.workspace_id == ctx.workspace.id)) + if not doc: + raise HTTPException(404, "Post not found") + source = ctx.db.get(Source, doc.source_id) if doc.source_id else None + lines: list[str] = [] + if doc.path: + corpus = Corpus(ctx.workspace.id) + try: + if not corpus.exists(doc.path): + corpus.sync() + lines = corpus.read_lines(doc.path) + except CorpusError: + lines = [] + if not lines: + lines = [f"# {doc.title}", "", *(doc.clean_text or "").splitlines()] + return {**serialize_document(doc, source), "lines": lines} + + +@documents_router.delete("/{document_id}") +def delete_document(document_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + doc = ctx.db.scalar(select(Document).where(Document.id == document_id, Document.workspace_id == ctx.workspace.id)) + if not doc: + raise HTTPException(404, "Post not found") + remove_documents(ctx, [doc]) + ctx.db.commit() + return {"ok": True} diff --git a/backend/app/routers/storage.py b/backend/app/routers/storage.py index ae399f209e0c00f8c3e5ed358fc14cdb2f974c23..ce3e1cad0f5e5a121169e8302f0a7830210a661d 100644 --- a/backend/app/routers/storage.py +++ b/backend/app/routers/storage.py @@ -1,16 +1,17 @@ """Browser uploads go straight to R2 with presigned PUTs; reads redirect to presigned GETs.""" +import mimetypes import uuid -from fastapi import APIRouter, Depends, HTTPException -from fastapi.responses import RedirectResponse +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import FileResponse, RedirectResponse from pydantic import BaseModel, Field from sqlalchemy import select from app.auth import Ctx, get_ctx from app.config import settings from app.models import Upload -from app.services.storage import keys, storage, workspace_owns_key +from app.services.storage import LocalStorage, keys, safe_filename, storage, verify_local_sig, workspace_owns_key router = APIRouter(prefix="/api/storage", tags=["storage"]) @@ -23,6 +24,7 @@ ALLOWED_TYPES = { "audio/wav", "audio/webm", "audio/mp4", + "audio/ogg", "video/mp4", "application/pdf", "text/markdown", @@ -89,3 +91,38 @@ def read_object(key: str, download: bool = False, ctx: Ctx = Depends(get_ctx)): raise HTTPException(404, "Not found") name = key.rsplit("/", 1)[-1] if download else None return RedirectResponse(storage.presign_get(key, download_name=name), status_code=302) + + +# Local disk backend: the "presigned" URLs LocalStorage hands out point here. + +local_router = APIRouter(prefix="/api/storage/local", tags=["storage"], include_in_schema=False) + + +def _local_path(method: str, key: str, exp: int, sig: str): + if not isinstance(storage, LocalStorage) or not verify_local_sig(method, key, exp, sig): + raise HTTPException(403, "Invalid or expired link") + try: + return storage.path(key) + except ValueError as exc: + raise HTTPException(404, "Not found") from exc + + +@local_router.get("/{key:path}") +def local_get(key: str, exp: int, sig: str, download: str | None = None): + path = _local_path("GET", key, exp, sig) + if not path.is_file(): + raise HTTPException(404, "Not found") + media_type = mimetypes.guess_type(key)[0] or "application/octet-stream" + # FileResponse answers Range requests, so audio and video can seek. + return FileResponse(path, media_type=media_type, filename=safe_filename(download) if download else None, + content_disposition_type="attachment" if download else "inline") + + +@local_router.put("/{key:path}") +async def local_put(key: str, exp: int, sig: str, request: Request): + _local_path("PUT", key, exp, sig) + body = await request.body() + if len(body) > settings.max_upload_bytes: + raise HTTPException(413, "File too large") + storage.put_bytes(key, body) + return {"ok": True, "size_bytes": len(body)} diff --git a/backend/app/routers/topics.py b/backend/app/routers/topics.py new file mode 100644 index 0000000000000000000000000000000000000000..6f6397d7662f575d8e82e937c5a925adf7a48535 --- /dev/null +++ b/backend/app/routers/topics.py @@ -0,0 +1,71 @@ +import uuid +from collections import Counter +from itertools import combinations + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy import select + +from app.auth import Ctx, get_ctx +from app.models import Document, DocumentTopic, Topic +from app.routers.sources import serialize_document +from app.services.jobs import create_job, serialize_job + +router = APIRouter(prefix="/api/topics", tags=["topics"]) + + +@router.get("") +def topic_map(ctx: Ctx = Depends(get_ctx), min_posts: int = 1, limit: int = 80): + """Nodes are topics sized by post count; edges join topics that share posts.""" + topics = ctx.db.scalars( + select(Topic).where(Topic.workspace_id == ctx.workspace.id, Topic.post_count >= min_posts) + .order_by(Topic.post_count.desc()).limit(min(limit, 300)) + ).all() + ids = {t.id for t in topics} + links = ctx.db.execute( + select(DocumentTopic.document_id, DocumentTopic.topic_id).where(DocumentTopic.topic_id.in_(ids)) + ).all() if ids else [] + by_doc: dict[uuid.UUID, list[uuid.UUID]] = {} + for doc_id, topic_id in links: + by_doc.setdefault(doc_id, []).append(topic_id) + edges: Counter[tuple[str, str]] = Counter() + for tids in by_doc.values(): + for a, b in combinations(sorted(str(t) for t in tids), 2): + edges[(a, b)] += 1 + untagged = ctx.db.scalar( + select(Document.id).where(Document.workspace_id == ctx.workspace.id, Document.path.is_not(None), + Document.id.not_in(select(DocumentTopic.document_id))).limit(1) + ) + return { + "nodes": [{"id": str(t.id), "name": t.name, "summary": t.summary, "post_count": t.post_count} + for t in topics], + "edges": [{"source": a, "target": b, "weight": w} for (a, b), w in edges.most_common(400)], + "has_untagged_posts": untagged is not None, + } + + +@router.get("/{topic_id}") +def topic_detail(topic_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): + topic = ctx.db.scalar(select(Topic).where(Topic.id == topic_id, Topic.workspace_id == ctx.workspace.id)) + if not topic: + raise HTTPException(404, "Topic not found") + rows = ctx.db.execute( + select(Document, DocumentTopic.weight) + .join(DocumentTopic, DocumentTopic.document_id == Document.id) + .where(DocumentTopic.topic_id == topic.id) + .order_by(DocumentTopic.weight.desc(), Document.published_at.desc()) + ).all() + return {"id": str(topic.id), "name": topic.name, "summary": topic.summary, "post_count": topic.post_count, + "posts": [{**serialize_document(d), "weight": w} for d, w in rows]} + + +@router.post("/rebuild") +def rebuild(ctx: Ctx = Depends(get_ctx), full: bool = False): + params = {} + if full: + # Clear per post tags so every post is mapped again. + for d in ctx.db.scalars(select(Document).where(Document.workspace_id == ctx.workspace.id)): + if d.metadata_json and "topics" in d.metadata_json: + d.metadata_json = {k: v for k, v in d.metadata_json.items() if k != "topics"} + ctx.db.commit() + job = create_job(ctx.db, ctx.workspace.id, "topics", params, max_attempts=2) + return serialize_job(job) diff --git a/backend/app/routers/voice.py b/backend/app/routers/voice.py new file mode 100644 index 0000000000000000000000000000000000000000..c0f95faa9452d72cac84836df06203bc1902729c --- /dev/null +++ b/backend/app/routers/voice.py @@ -0,0 +1,144 @@ +import uuid +from datetime import UTC, datetime + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.concurrency import run_in_threadpool +from pydantic import BaseModel, Field +from sqlalchemy import select + +from app.auth import Ctx, get_ctx +from app.config import settings +from app.models import Upload, VoiceConsent, VoiceProfile +from app.services import tts +from app.services.jobs import create_job, serialize_job +from app.services.plans import effective_plan +from app.services.storage import storage + +router = APIRouter(prefix="/api/voice", tags=["voice"]) + +CONSENT_TEXT = ( + "I confirm this recording is my own voice. I allow Notestack to create a synthetic copy of it with " + "ElevenLabs, only for audio and video made in my workspace. I can revoke this at any time, which deletes " + "the synthetic voice." +) + + +class BuildIn(BaseModel): + document_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10) + + +class ProfileIn(BaseModel): + profile: dict | None = None + host_voices: dict[str, str] | None = None + + +class ConsentIn(BaseModel): + upload_id: uuid.UUID + agreed: bool + consent_text: str + + +def _profile(ctx: Ctx) -> VoiceProfile | None: + return ctx.db.scalar(select(VoiceProfile).where(VoiceProfile.workspace_id == ctx.workspace.id)) + + +def _consent(ctx: Ctx) -> VoiceConsent | None: + return ctx.db.scalar( + select(VoiceConsent).where(VoiceConsent.workspace_id == ctx.workspace.id, VoiceConsent.revoked_at.is_(None)) + .order_by(VoiceConsent.created_at.desc()) + ) + + +def _serialize(ctx: Ctx) -> dict: + vp = _profile(ctx) + consent = _consent(ctx) + plan = effective_plan(ctx.db, ctx.workspace) + voices = (vp.host_voices if vp else None) or {} + return { + "profile": vp.profile_json if vp else None, + "sample_doc_ids": vp.sample_doc_ids if vp else [], + "updated_at": vp.updated_at.isoformat() if vp and vp.updated_at else None, + "host_voices": {"host_a": voices.get("host_a") or settings.elevenlabs_voice_a, + "host_b": voices.get("host_b") or settings.elevenlabs_voice_b}, + "clone": { + "allowed": plan.voice_cloning, + "consent_text": CONSENT_TEXT, + "status": ("ready" if consent and consent.elevenlabs_voice_id else "processing" if consent else "none"), + "voice_id": consent.elevenlabs_voice_id if consent else None, + "created_at": consent.created_at.isoformat() if consent else None, + }, + "tts_configured": bool(settings.elevenlabs_api_key), + } + + +@router.get("") +def get_voice(ctx: Ctx = Depends(get_ctx)): + return _serialize(ctx) + + +@router.put("") +def update_voice(body: ProfileIn, ctx: Ctx = Depends(get_ctx)): + vp = _profile(ctx) + if not vp: + vp = VoiceProfile(workspace_id=ctx.workspace.id) + ctx.db.add(vp) + if body.profile is not None: + vp.profile_json = body.profile + if body.host_voices is not None: + vp.host_voices = {k: v for k, v in body.host_voices.items() if k in ("host_a", "host_b") and v} + ctx.db.commit() + return _serialize(ctx) + + +@router.post("/build") +def build(body: BuildIn, ctx: Ctx = Depends(get_ctx)): + job = create_job(ctx.db, ctx.workspace.id, "voice_profile", + {"document_ids": [str(i) for i in body.document_ids]}, max_attempts=2) + return serialize_job(job) + + +@router.get("/voices") +async def voices(ctx: Ctx = Depends(get_ctx)): + items = await run_in_threadpool(tts.list_voices) + consent = _consent(ctx) + if consent and consent.elevenlabs_voice_id and not any(v["voice_id"] == consent.elevenlabs_voice_id for v in items): + items = [{"voice_id": consent.elevenlabs_voice_id, "name": "My voice", "category": "cloned"}, *items] + return items + + +@router.post("/consent") +def give_consent(body: ConsentIn, request: Request, ctx: Ctx = Depends(get_ctx)): + if not effective_plan(ctx.db, ctx.workspace).voice_cloning: + raise HTTPException(402, {"code": "plan_limit", "message": "Voice cloning is on the Writer and Studio plans."}) + if not body.agreed or body.consent_text.strip() != CONSENT_TEXT: + raise HTTPException(400, "Please read and accept the consent statement.") + if not settings.elevenlabs_api_key: + raise HTTPException(503, "Voice cloning needs ELEVENLABS_API_KEY in .env") + upload = ctx.db.scalar(select(Upload).where(Upload.id == body.upload_id, Upload.workspace_id == ctx.workspace.id)) + if not upload or upload.status != "complete" or not upload.content_type.startswith("audio/"): + raise HTTPException(400, "Upload an audio recording first.") + consent = VoiceConsent(workspace_id=ctx.workspace.id, user_id=ctx.user.id, sample_key=upload.key, + consent_text=CONSENT_TEXT, ip_address=request.client.host if request.client else None) + ctx.db.add(consent) + ctx.db.commit() + job = create_job(ctx.db, ctx.workspace.id, "voice_clone", {"consent_id": str(consent.id)}, max_attempts=1) + return {"job": serialize_job(job), **_serialize(ctx)} + + +@router.delete("/consent") +async def revoke_consent(ctx: Ctx = Depends(get_ctx)): + consent = _consent(ctx) + if not consent: + return _serialize(ctx) + if consent.elevenlabs_voice_id and settings.elevenlabs_api_key: + try: + await run_in_threadpool(tts.delete_voice, consent.elevenlabs_voice_id) + except Exception: + pass + consent.revoked_at = datetime.now(UTC) + storage.delete(consent.sample_key) + vp = _profile(ctx) + if vp and vp.host_voices: + vp.host_voices = {k: v for k, v in vp.host_voices.items() if v != consent.elevenlabs_voice_id} + ctx.db.commit() + return _serialize(ctx) diff --git a/backend/app/services/artifacts.py b/backend/app/services/artifacts.py new file mode 100644 index 0000000000000000000000000000000000000000..6de95d28c0aaa736185c8ec418057118918437f2 --- /dev/null +++ b/backend/app/services/artifacts.py @@ -0,0 +1,76 @@ +"""Artifacts are every generated thing (summaries, audio, video, launch kits, stills). Creating one always +means: artifact row + queued job, returned together so the UI can stream progress.""" + +import uuid + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.models import Artifact, Job +from app.services.jobs import create_job, serialize_job +from app.services.storage import storage + +TYPE_LABELS = { + "summary": "Summary", + "audio_overview": "Audio overview", + "video": "Video", + "quote_card": "Quote card", + "carousel": "Carousel", + "launch_kit": "Launch Kit", +} + + +def start_artifact( + db: Session, + workspace_id: uuid.UUID, + type_: str, + *, + title: str, + notebook_id: uuid.UUID | None = None, + document_id: uuid.UUID | None = None, + content: dict | None = None, + job_kind: str | None = None, + params: dict | None = None, + max_attempts: int = 2, +) -> tuple[Artifact, Job]: + artifact = Artifact(workspace_id=workspace_id, type=type_, notebook_id=notebook_id, document_id=document_id, + status="pending", content_json={"title": title, **(content or {})}) + db.add(artifact) + db.commit() + job = create_job(db, workspace_id, job_kind or type_, {"artifact_id": str(artifact.id), **(params or {})}, + artifact_id=artifact.id, max_attempts=max_attempts) + return artifact, job + + +def latest_jobs(db: Session, artifact_ids: list[uuid.UUID]) -> dict[uuid.UUID, Job]: + if not artifact_ids: + return {} + jobs = db.scalars(select(Job).where(Job.artifact_id.in_(artifact_ids)).order_by(Job.created_at)).all() + return {j.artifact_id: j for j in jobs} # later rows win, so this is the newest job per artifact + + +def serialize_artifact(a: Artifact, job: Job | None = None) -> dict: + content = a.content_json or {} + slides = content.get("slide_keys") or [] + return { + "id": str(a.id), + "type": a.type, + "type_label": TYPE_LABELS.get(a.type, a.type), + "title": content.get("title") or TYPE_LABELS.get(a.type, a.type), + "status": a.status, + "notebook_id": str(a.notebook_id) if a.notebook_id else None, + "document_id": str(a.document_id) if a.document_id else None, + "content": content, + "storage_key": a.storage_key, + "url": storage.presign_get(a.storage_key) if a.storage_key else None, + "download_url": storage.presign_get(a.storage_key, download_name=_download_name(a)) if a.storage_key else None, + "slide_urls": [storage.presign_get(k) for k in slides], + "created_at": a.created_at.isoformat() if a.created_at else None, + "job": serialize_job(job) if job else None, + } + + +def _download_name(a: Artifact) -> str: + title = (a.content_json or {}).get("title") or a.type + ext = (a.storage_key or "").rsplit(".", 1)[-1] + return f"{title[:60]}.{ext}" diff --git a/backend/app/services/crypto.py b/backend/app/services/crypto.py new file mode 100644 index 0000000000000000000000000000000000000000..2360039024beae1b2d83579a1ec2265bd09dba05 --- /dev/null +++ b/backend/app/services/crypto.py @@ -0,0 +1,31 @@ +"""Encrypts third party tokens at rest (social accounts). Fernet key from SOCIAL_TOKEN_KEY, or derived +from JWT_SECRET so local dev needs no extra setting.""" + +import base64 +import hashlib +from functools import lru_cache + +from cryptography.fernet import Fernet, InvalidToken + +from app.config import settings + + +@lru_cache +def _fernet() -> Fernet: + key = settings.social_token_key + if not key: + key = base64.urlsafe_b64encode(hashlib.sha256(f"social:{settings.jwt_secret}".encode()).digest()).decode() + return Fernet(key.encode()) + + +def encrypt(value: str | None) -> str | None: + return _fernet().encrypt(value.encode()).decode() if value else None + + +def decrypt(value: str | None) -> str | None: + if not value: + return None + try: + return _fernet().decrypt(value.encode()).decode() + except InvalidToken: + return None diff --git a/backend/app/services/email.py b/backend/app/services/email.py index 14198dd6d228805b7f2c2070def0bfe99a914d88..f239bc61e95026bfb8aaaa74f88192378ca18614 100644 --- a/backend/app/services/email.py +++ b/backend/app/services/email.py @@ -165,6 +165,21 @@ class EmailService: ) ) + def send_post_reminder(self, to: str, platform: str, content: str, url: str) -> bool: + body = ( + f"Your {html.escape(platform)} post is scheduled for now. Copy it below and post it." + f"
"
+            f"{html.escape(content)}
" + _button("Open Launchpad", url) + ) + return self._send( + OutgoingEmail( + to=to, + subject=f"Time to post on {platform}", + html=_layout("Launch window open", body), + text=f"Time to post on {platform}:\n\n{content}\n\nLaunchpad: {url}", + ) + ) + # Internal alerts def send_alert(self, subject: str, text: str) -> bool: diff --git a/backend/app/services/jobs.py b/backend/app/services/jobs.py index 40e769a08fa7647e4203df677261efeaa1b06a4b..c5e7e7aec1c7756d89279934e0ee04be840e8031 100644 --- a/backend/app/services/jobs.py +++ b/backend/app/services/jobs.py @@ -29,6 +29,7 @@ def serialize_job(job: Job) -> dict: return { "id": str(job.id), "kind": job.kind, + "params": job.params, "status": job.status, "progress": job.progress, "message": job.message, diff --git a/backend/app/services/launchpad.py b/backend/app/services/launchpad.py new file mode 100644 index 0000000000000000000000000000000000000000..5b1e7c0c0c57451baedb26c57569e9b6ccd4d6b4 --- /dev/null +++ b/backend/app/services/launchpad.py @@ -0,0 +1,156 @@ +"""Launchpad: publishes due calendar items, emails reminders for platforms without an API, rewrites links +to tracked short links and pulls engagement back in.""" + +import logging +import re +import secrets +from collections.abc import Callable +from datetime import UTC, datetime, timedelta + +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from app.config import settings +from app.db import SessionLocal +from app.models import CalendarItem, Document, EngagementEvent, SocialAccount, TrackedLink, User, Workspace +from app.services.email import email_service +from app.services.social import AUTO_POST, PLATFORM_LABELS, SocialError, module + +log = logging.getLogger("notestack.launchpad") +_URL = re.compile(r"https?://[^\s)\]>]+") +MAX_ATTEMPTS = 3 + + +def _public_api() -> bool: + return not re.search(r"//(localhost|127\.0\.0\.1|api)(:|/|$)", settings.api_url) + + +def track_links(db: Session, item: CalendarItem, text: str) -> str: + """Swap outbound links for {API_URL}/l/{slug} so clicks are counted. Skipped when the API is not public.""" + if not _public_api(): + return text + base = f"{settings.api_url.rstrip('/')}/l/" + + def swap(m: re.Match) -> str: + url = m.group(0) + if url.startswith(base): + return url + link = TrackedLink(workspace_id=item.workspace_id, slug=secrets.token_urlsafe(5)[:7], target_url=url, + calendar_item_id=item.id) + db.add(link) + return base + link.slug + + return _URL.sub(swap, text) + + +def owner_email(db: Session, workspace_id) -> str | None: + ws = db.get(Workspace, workspace_id) + user = db.get(User, ws.owner_id) if ws else None + return user.email if user and user.is_active else None + + +def publish_item(db: Session, item: CalendarItem) -> CalendarItem: + """Publish or remind now. Leaves the item posted, reminded, failed, or scheduled again for a retry.""" + posts = [p for p in [item.content, *(item.thread or [])] if p and p.strip()] + if not posts: + item.status, item.error = "failed", "Nothing to post." + db.commit() + return item + account = db.get(SocialAccount, item.social_account_id) if item.social_account_id else None + label = PLATFORM_LABELS.get(item.platform, item.platform) + if item.platform not in AUTO_POST or item.remind_by_email or not account: + to = owner_email(db, item.workspace_id) + sent = bool(to) and email_service.send_post_reminder( + to, label, "\n\n".join(posts), f"{settings.frontend_url}/app/launchpad?item={item.id}") + item.status = "reminded" if sent else "failed" + item.error = None if sent else "Could not send the reminder email." + item.posted_at = datetime.now(UTC) + db.commit() + return item + item.attempts += 1 + try: + tracked = [track_links(db, item, p) for p in posts] + db.flush() + external_id, url = module(item.platform).publish(db, account, tracked) + except SocialError as exc: + _fail_or_retry(db, item, str(exc), exc.permanent) + return item + except Exception as exc: # network trouble: retry + log.exception("publish failed for %s", item.id) + _fail_or_retry(db, item, f"{type(exc).__name__}: {exc}", False) + return item + item.status, item.error = "posted", None + item.external_id, item.external_url, item.posted_at = external_id, url, datetime.now(UTC) + if item.document_id: + doc = db.get(Document, item.document_id) + if doc: + doc.last_resurfaced_at = item.posted_at + db.commit() + return item + + +def _fail_or_retry(db: Session, item: CalendarItem, error: str, permanent: bool) -> None: + item.error = error[:1000] + if permanent or item.attempts >= MAX_ATTEMPTS: + item.status = "failed" + else: + item.status = "scheduled" + item.scheduled_at = datetime.now(UTC) + timedelta(minutes=5 * item.attempts) + db.commit() + + +def publish_due(session_factory: Callable[[], Session] = SessionLocal) -> int: + now = datetime.now(UTC) + count = 0 + with session_factory() as db: + # A tick that died mid publish leaves items in "publishing"; put them back after 10 minutes. + for stuck in db.scalars(select(CalendarItem).where( + CalendarItem.status == "publishing", CalendarItem.updated_at < now - timedelta(minutes=10))): + stuck.status = "scheduled" + due = db.scalars( + select(CalendarItem).where(CalendarItem.status == "scheduled", CalendarItem.scheduled_at <= now) + .order_by(CalendarItem.scheduled_at).limit(50) + ).all() + for item in due: + item.status = "publishing" # claim, so an overlapping tick skips it + db.commit() + for item in due: + publish_item(db, item) + count += 1 + return count + + +def clicks_by_item(db: Session, item_ids: list) -> dict: + if not item_ids: + return {} + rows = db.execute( + select(TrackedLink.calendar_item_id, func.count(EngagementEvent.id)) + .join(EngagementEvent, EngagementEvent.tracked_link_id == TrackedLink.id) + .where(TrackedLink.calendar_item_id.in_(item_ids), EngagementEvent.kind == "click") + .group_by(TrackedLink.calendar_item_id) + ).all() + return {item_id: n for item_id, n in rows} + + +def sync_engagement(session_factory: Callable[[], Session] = SessionLocal) -> int: + since = datetime.now(UTC) - timedelta(days=30) + updated = 0 + with session_factory() as db: + items = db.scalars( + select(CalendarItem).where(CalendarItem.status == "posted", CalendarItem.posted_at >= since, + CalendarItem.external_id.is_not(None)) + ).all() + for item in items: + account = db.get(SocialAccount, item.social_account_id) if item.social_account_id else None + if not account: + continue + try: + data = module(item.platform).metrics(db, account, item.external_id) + except Exception: + log.warning("metrics failed for %s", item.id, exc_info=True) + continue + if data: + item.metrics = {**(item.metrics or {}), **data, "synced_at": datetime.now(UTC).isoformat()} + updated += 1 + db.commit() + return updated diff --git a/backend/app/services/renderer.py b/backend/app/services/renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..76bb62830337e61b4f734e9de7e13228e6025a39 --- /dev/null +++ b/backend/app/services/renderer.py @@ -0,0 +1,65 @@ +"""Hands a render to the Remotion service with presigned upload URLs (it never holds storage keys).""" + +import httpx + +from app.config import settings +from app.models import Artifact, Job +from app.services.storage import keys, storage + +COMPOSITIONS = { + "ShortVertical": "mp4", + "ExplainerLong": "mp4", + "AudiogramSquare": "mp4", + "QuoteCard": "png", + "CarouselSlide": "png", +} + + +class PermanentJobError(RuntimeError): + """Fail the job now; retrying will not help.""" + + +def request_render(job: Job, artifact: Artifact, composition: str, props: dict | None = None, + stills: list[dict] | None = None) -> None: + """Start a render. `stills` renders one PNG per props dict (carousels) instead of a single output.""" + fmt = COMPOSITIONS[composition] + content_type = "image/png" if fmt == "png" else "video/mp4" + body: dict = { + "jobId": str(job.id), + "compositionId": composition, + "props": props or {}, + "format": fmt, + "uploadContentType": content_type, + "callbackUrl": f"{settings.api_url}/api/internal/jobs/{job.id}/progress", + } + if stills: + outputs = [] + for i, still_props in enumerate(stills, start=1): + key = keys.artifact(artifact.workspace_id, artifact.id, f"slide-{i:02d}", fmt) + outputs.append({"props": still_props, "uploadUrl": storage.presign_put(key, content_type, ttl=6 * 3600), + "storageKey": key}) + body.update(stills=outputs, props=stills[0], uploadUrl=outputs[0]["uploadUrl"], + storageKey=outputs[0]["storageKey"]) + else: + key = keys.artifact(artifact.workspace_id, artifact.id, composition.lower(), fmt) + body.update(uploadUrl=storage.presign_put(key, content_type, ttl=6 * 3600), storageKey=key) + try: + resp = httpx.post(f"{settings.renderer_url}/render", headers={"x-internal-token": settings.internal_token}, + json=body, timeout=30) + except httpx.ConnectError as exc: + raise PermanentJobError("Renderer is not running. Start it with: cd renderer && npm start") from exc + if resp.status_code == 400: + raise PermanentJobError(f"Renderer rejected the props: {resp.text[:400]}") + resp.raise_for_status() + + +def brand_props(workspace, *, ttl: int = 6 * 3600) -> dict: + brand = workspace.brand_json or {} + out: dict = {} + if brand.get("accent"): + out["accent"] = brand["accent"] + if brand.get("name"): + out["name"] = brand["name"] + if brand.get("logo_key"): + out["logoUrl"] = storage.presign_get(brand["logo_key"], ttl=ttl) + return out diff --git a/backend/app/services/social/__init__.py b/backend/app/services/social/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..53c8b787e90e5e780963bc480da1f41f71e81185 --- /dev/null +++ b/backend/app/services/social/__init__.py @@ -0,0 +1,55 @@ +"""Publishing to X, LinkedIn and Bluesky. Each platform module exposes: + + publish(db, account, posts: list[str]) -> (external_id, external_url) + metrics(db, account, external_id) -> dict + +A list with more than one post is published as a thread (reply chain) where the platform supports it. +""" + +from datetime import UTC, datetime + +from sqlalchemy.orm import Session + +from app.models import SocialAccount +from app.services.crypto import decrypt, encrypt + + +class SocialError(RuntimeError): + def __init__(self, message: str, permanent: bool = False): + super().__init__(message) + self.permanent = permanent + + +PLATFORM_LABELS = {"x": "X", "linkedin": "LinkedIn", "bluesky": "Bluesky", "substack_notes": "Substack Notes"} +AUTO_POST = {"x", "linkedin", "bluesky"} +LIMITS = {"x": 280, "bluesky": 300, "linkedin": 3000, "substack_notes": 2000} + + +def token(account: SocialAccount) -> str: + value = decrypt(account.access_token) + if not value: + raise SocialError("Stored credentials could not be read. Reconnect the account.", permanent=True) + return value + + +def save_tokens(db: Session, account: SocialAccount, access: str, refresh: str | None, expires_in: int | None) -> None: + account.access_token = encrypt(access) + if refresh: + account.refresh_token = encrypt(refresh) + if expires_in: + account.expires_at = datetime.fromtimestamp(datetime.now(UTC).timestamp() + int(expires_in), UTC) + account.status = "active" + db.commit() + + +def expired(account: SocialAccount) -> bool: + if not account.expires_at: + return False + exp = account.expires_at if account.expires_at.tzinfo else account.expires_at.replace(tzinfo=UTC) + return exp.timestamp() - 120 < datetime.now(UTC).timestamp() + + +def module(platform: str): + from app.services.social import bluesky, linkedin, x + + return {"x": x, "linkedin": linkedin, "bluesky": bluesky}[platform] diff --git a/backend/app/services/social/bluesky.py b/backend/app/services/social/bluesky.py new file mode 100644 index 0000000000000000000000000000000000000000..5d8901a290a874ad2f1c1f00db73cd3a4b36967f --- /dev/null +++ b/backend/app/services/social/bluesky.py @@ -0,0 +1,67 @@ +"""Bluesky over AT Protocol with an app password (bsky.app > Settings > App passwords).""" + +import re +from datetime import UTC, datetime + +import httpx +from sqlalchemy.orm import Session + +from app.models import SocialAccount +from app.services.social import SocialError, token + +PDS = "https://bsky.social/xrpc" +PUBLIC = "https://public.api.bsky.app/xrpc" +_URL = re.compile(r"https?://[^\s)\]]+") + + +def create_session(handle: str, app_password: str) -> dict: + resp = httpx.post(f"{PDS}/com.atproto.server.createSession", + json={"identifier": handle.lstrip("@"), "password": app_password}, timeout=20) + if resp.status_code in (400, 401): + raise SocialError("Bluesky did not accept that handle and app password.", permanent=True) + resp.raise_for_status() + return resp.json() + + +def link_facets(text: str) -> list[dict]: + """Links must be declared as facets with UTF-8 byte offsets to be clickable.""" + facets = [] + for m in _URL.finditer(text): + start = len(text[: m.start()].encode()) + end = start + len(m.group(0).encode()) + facets.append({"index": {"byteStart": start, "byteEnd": end}, + "features": [{"$type": "app.bsky.richtext.facet#link", "uri": m.group(0)}]}) + return facets + + +def publish(db: Session, account: SocialAccount, posts: list[str]) -> tuple[str, str]: + session = create_session(account.handle, token(account)) + headers = {"Authorization": f"Bearer {session['accessJwt']}"} + root = parent = None + for text in posts: + record: dict = {"$type": "app.bsky.feed.post", "text": text, + "createdAt": datetime.now(UTC).isoformat().replace("+00:00", "Z")} + facets = link_facets(text) + if facets: + record["facets"] = facets + if parent: + record["reply"] = {"root": root, "parent": parent} + resp = httpx.post(f"{PDS}/com.atproto.repo.createRecord", headers=headers, timeout=20, + json={"repo": session["did"], "collection": "app.bsky.feed.post", "record": record}) + if resp.status_code >= 400: + raise SocialError(f"Bluesky {resp.status_code}: {resp.text[:300]}", permanent=resp.status_code == 400) + ref = {"uri": resp.json()["uri"], "cid": resp.json()["cid"]} + root = root or ref + parent = ref + rkey = root["uri"].rsplit("/", 1)[-1] + return root["uri"], f"https://bsky.app/profile/{session.get('handle', account.handle)}/post/{rkey}" + + +def metrics(db: Session, account: SocialAccount, external_id: str) -> dict: + resp = httpx.get(f"{PUBLIC}/app.bsky.feed.getPosts", params={"uris": external_id}, timeout=20) + if resp.status_code >= 400: + return {} + posts = resp.json().get("posts") or [{}] + p = posts[0] + return {"likes": p.get("likeCount", 0), "reposts": p.get("repostCount", 0), + "replies": p.get("replyCount", 0), "quotes": p.get("quoteCount", 0)} diff --git a/backend/app/services/social/linkedin.py b/backend/app/services/social/linkedin.py new file mode 100644 index 0000000000000000000000000000000000000000..2d569e569b8a097ee4eba68f2666e564c065d45f --- /dev/null +++ b/backend/app/services/social/linkedin.py @@ -0,0 +1,87 @@ +"""LinkedIn member posts (Share on LinkedIn + Sign In with LinkedIn using OpenID Connect products).""" + +import re +from urllib.parse import quote, urlencode + +import httpx +from sqlalchemy.orm import Session + +from app.config import settings +from app.models import SocialAccount +from app.services.social import SocialError, expired, token + +AUTH_URL = "https://www.linkedin.com/oauth/v2/authorization" +TOKEN_URL = "https://www.linkedin.com/oauth/v2/accessToken" +SCOPES = "openid profile w_member_social" +VERSION = "202509" # LinkedIn-Version header, YYYYMM; LinkedIn supports each version for a year + + +def redirect_uri() -> str: + return f"{settings.api_url.rstrip('/')}/api/social/linkedin/callback" + + +def configured() -> bool: + return bool(settings.linkedin_client_id and settings.linkedin_client_secret) + + +def authorize_url(state: str) -> str: + return AUTH_URL + "?" + urlencode({ + "response_type": "code", "client_id": settings.linkedin_client_id, "redirect_uri": redirect_uri(), + "state": state, "scope": SCOPES, + }) + + +def exchange_code(code: str) -> dict: + resp = httpx.post(TOKEN_URL, data={ + "grant_type": "authorization_code", "code": code, "redirect_uri": redirect_uri(), + "client_id": settings.linkedin_client_id, "client_secret": settings.linkedin_client_secret, + }, timeout=20) + if resp.status_code >= 400: + raise SocialError(f"LinkedIn token request failed: {resp.text[:300]}", permanent=True) + tokens = resp.json() + me = httpx.get("https://api.linkedin.com/v2/userinfo", + headers={"Authorization": f"Bearer {tokens['access_token']}"}, timeout=20) + me.raise_for_status() + info = me.json() + return {**tokens, "external_id": info["sub"], "handle": info.get("name") or info["sub"], + "avatar_url": info.get("picture")} + + +_RESERVED = re.compile(r"([\\|{}@\[\]()<>#*_~])") + + +def little_text(text: str) -> str: + """LinkedIn's commentary field uses 'little text' markup; reserved characters must be escaped.""" + return _RESERVED.sub(r"\\\1", text) + + +def publish(db: Session, account: SocialAccount, posts: list[str]) -> tuple[str, str]: + if expired(account): + account.status = "expired" + db.commit() + raise SocialError("The LinkedIn connection expired. Reconnect the account.", permanent=True) + body = { + "author": f"urn:li:person:{account.external_id}", + "commentary": little_text("\n\n".join(posts)), + "visibility": "PUBLIC", + "distribution": {"feedDistribution": "MAIN_FEED", "targetEntities": [], "thirdPartyDistributionChannels": []}, + "lifecycleState": "PUBLISHED", + "isReshareDisabledByAuthor": False, + } + resp = httpx.post("https://api.linkedin.com/rest/posts", json=body, timeout=20, headers={ + "Authorization": f"Bearer {token(account)}", "LinkedIn-Version": VERSION, + "X-Restli-Protocol-Version": "2.0.0", + }) + if resp.status_code == 401: + account.status = "expired" + db.commit() + raise SocialError("LinkedIn rejected the token. Reconnect the account.", permanent=True) + if resp.status_code >= 400: + raise SocialError(f"LinkedIn {resp.status_code}: {resp.text[:300]}", + permanent=resp.status_code in (400, 403, 422)) + urn = resp.headers.get("x-restli-id") or resp.headers.get("x-linkedin-id") or "" + return urn, f"https://www.linkedin.com/feed/update/{quote(urn)}/" if urn else "" + + +def metrics(db: Session, account: SocialAccount, external_id: str) -> dict: + return {} # member post analytics need the Community Management API, which personal apps rarely have diff --git a/backend/app/services/social/x.py b/backend/app/services/social/x.py new file mode 100644 index 0000000000000000000000000000000000000000..63bfc306c9a6b3008a507e847ee8eea7fb6dfd94 --- /dev/null +++ b/backend/app/services/social/x.py @@ -0,0 +1,95 @@ +"""X (Twitter) API v2 with OAuth 2.0 user context (PKCE). Threads are reply chains.""" + +import base64 +import hashlib +import secrets +from urllib.parse import urlencode + +import httpx +from sqlalchemy.orm import Session + +from app.config import settings +from app.models import SocialAccount +from app.services.crypto import decrypt +from app.services.social import SocialError, expired, save_tokens, token + +AUTH_URL = "https://x.com/i/oauth2/authorize" +TOKEN_URL = "https://api.x.com/2/oauth2/token" +API = "https://api.x.com/2" +SCOPES = "tweet.read tweet.write users.read offline.access" + + +def redirect_uri() -> str: + return f"{settings.api_url.rstrip('/')}/api/social/x/callback" + + +def configured() -> bool: + return bool(settings.x_client_id and settings.x_client_secret) + + +def pkce_pair() -> tuple[str, str]: + verifier = secrets.token_urlsafe(64)[:96] + challenge = base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).decode().rstrip("=") + return verifier, challenge + + +def authorize_url(state: str, challenge: str) -> str: + return AUTH_URL + "?" + urlencode({ + "response_type": "code", "client_id": settings.x_client_id, "redirect_uri": redirect_uri(), + "scope": SCOPES, "state": state, "code_challenge": challenge, "code_challenge_method": "S256", + }) + + +def _token_request(data: dict) -> dict: + resp = httpx.post(TOKEN_URL, data={**data, "client_id": settings.x_client_id}, + auth=(settings.x_client_id, settings.x_client_secret), timeout=20) + if resp.status_code >= 400: + raise SocialError(f"X token request failed: {resp.text[:300]}", permanent=resp.status_code in (400, 401)) + return resp.json() + + +def exchange_code(code: str, verifier: str) -> dict: + tokens = _token_request({"grant_type": "authorization_code", "code": code, "redirect_uri": redirect_uri(), + "code_verifier": verifier}) + me = httpx.get(f"{API}/users/me", params={"user.fields": "profile_image_url"}, + headers={"Authorization": f"Bearer {tokens['access_token']}"}, timeout=20) + me.raise_for_status() + user = me.json()["data"] + return {**tokens, "external_id": user["id"], "handle": user["username"], + "avatar_url": user.get("profile_image_url")} + + +def _access(db: Session, account: SocialAccount) -> str: + if expired(account) and account.refresh_token: + tokens = _token_request({"grant_type": "refresh_token", "refresh_token": decrypt(account.refresh_token)}) + save_tokens(db, account, tokens["access_token"], tokens.get("refresh_token"), tokens.get("expires_in")) + return token(account) + + +def publish(db: Session, account: SocialAccount, posts: list[str]) -> tuple[str, str]: + headers = {"Authorization": f"Bearer {_access(db, account)}"} + first_id, reply_to = None, None + for text in posts: + body: dict = {"text": text} + if reply_to: + body["reply"] = {"in_reply_to_tweet_id": reply_to} + resp = httpx.post(f"{API}/tweets", json=body, headers=headers, timeout=20) + if resp.status_code == 401: + account.status = "expired" + db.commit() + raise SocialError("X rejected the token. Reconnect the account.", permanent=True) + if resp.status_code >= 400: + raise SocialError(f"X {resp.status_code}: {resp.text[:300]}", permanent=resp.status_code in (400, 403)) + reply_to = resp.json()["data"]["id"] + first_id = first_id or reply_to + return first_id, f"https://x.com/{account.handle}/status/{first_id}" + + +def metrics(db: Session, account: SocialAccount, external_id: str) -> dict: + resp = httpx.get(f"{API}/tweets", params={"ids": external_id, "tweet.fields": "public_metrics"}, + headers={"Authorization": f"Bearer {_access(db, account)}"}, timeout=20) + if resp.status_code >= 400: + return {} + data = (resp.json().get("data") or [{}])[0].get("public_metrics") or {} + return {"likes": data.get("like_count", 0), "reposts": data.get("retweet_count", 0), + "replies": data.get("reply_count", 0), "impressions": data.get("impression_count", 0)} diff --git a/backend/app/services/storage.py b/backend/app/services/storage.py index f5c9bd42d495dd88bfd144715b5771f29ed69547..47ee940fec34bad33649169a1d966b2726b403a5 100644 --- a/backend/app/services/storage.py +++ b/backend/app/services/storage.py @@ -1,13 +1,20 @@ -"""Cloudflare R2 storage (S3 API). Postgres stores keys; bytes live here. +"""Object storage. Postgres stores keys; bytes live here. -Locally, point R2_ENDPOINT_URL at MinIO and nothing else changes. +Production: Cloudflare R2 (S3 API). docker-compose: MinIO via R2_ENDPOINT_URL. Plain local dev with +no R2 settings: LocalStorage, a folder on disk whose presigned URLs are HMAC signed API routes, so +every caller (browser uploads, the renderer, media players) works unchanged. """ import hashlib +import hmac import mimetypes import re +import shutil +import time import uuid from functools import cached_property +from pathlib import Path +from urllib.parse import quote, urlencode import boto3 from botocore.client import Config @@ -127,4 +134,85 @@ class Storage: return f"{settings.r2_public_base_url.rstrip('/')}/{key}" -storage = Storage() +def _local_sig(method: str, key: str, exp: int) -> str: + msg = f"{method.upper()}:{key}:{exp}".encode() + return hmac.new(settings.jwt_secret.encode(), msg, hashlib.sha256).hexdigest() + + +def verify_local_sig(method: str, key: str, exp: int, sig: str) -> bool: + return exp >= time.time() and hmac.compare_digest(_local_sig(method, key, exp), sig) + + +class LocalStorage: + """Disk backed stand in for R2 with the same interface.""" + + def __init__(self, root: str | None = None): + self.root = Path(root or settings.local_storage_dir).resolve() + self.bucket = "local" + + def path(self, key: str) -> Path: + if ".." in key.split("/") or key.startswith("/"): + raise ValueError(f"Invalid key: {key!r}") + return self.root / key + + def ensure_bucket(self) -> None: + self.root.mkdir(parents=True, exist_ok=True) + + def put_bytes(self, key: str, data: bytes, content_type: str | None = None) -> str: + path = self.path(key) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return key + + def put_text(self, key: str, text: str, content_type: str = "text/plain; charset=utf-8") -> str: + return self.put_bytes(key, text.encode("utf-8"), content_type) + + def get_bytes(self, key: str) -> bytes: + path = self.path(key) + if not path.is_file(): + raise FileNotFoundError(key) + return path.read_bytes() + + def head(self, key: str) -> dict | None: + path = self.path(key) + if not path.is_file(): + return None + return {"ContentLength": path.stat().st_size, "ContentType": mimetypes.guess_type(key)[0]} + + def exists(self, key: str) -> bool: + return self.path(key).is_file() + + def delete(self, key: str) -> None: + self.path(key).unlink(missing_ok=True) + + def delete_prefix(self, prefix: str) -> int: + base = self.path(prefix.rstrip("/")) + if base.is_file(): + base.unlink() + return 1 + if not base.is_dir(): + return 0 + count = sum(1 for f in base.rglob("*") if f.is_file()) + shutil.rmtree(base, ignore_errors=True) + return count + + def _signed(self, method: str, key: str, ttl: int | None, extra: dict | None = None) -> str: + exp = int(time.time()) + (ttl or settings.r2_presign_ttl_seconds) + params = {"exp": exp, "sig": _local_sig(method, key, exp), **(extra or {})} + return f"{settings.api_url.rstrip('/')}/api/storage/local/{quote(key)}?{urlencode(params)}" + + def presign_get(self, key: str, ttl: int | None = None, download_name: str | None = None) -> str: + return self._signed("GET", key, ttl, {"download": safe_filename(download_name)} if download_name else None) + + def presign_put(self, key: str, content_type: str, ttl: int | None = None) -> str: + return self._signed("PUT", key, ttl) + + def public_url(self, key: str) -> str | None: + return None + + +def make_storage() -> Storage | LocalStorage: + return LocalStorage() if settings.use_local_storage else Storage() + + +storage = make_storage() diff --git a/backend/app/services/tts.py b/backend/app/services/tts.py new file mode 100644 index 0000000000000000000000000000000000000000..7482a09a2c4f7223fb1c79c108928719d5fce678 --- /dev/null +++ b/backend/app/services/tts.py @@ -0,0 +1,124 @@ +"""ElevenLabs text to speech, cached in storage by (script, voice, settings).""" + +import base64 +import json + +import httpx + +from app.config import settings +from app.services.storage import keys, storage + +API = "https://api.elevenlabs.io/v1" +OUTPUT = "mp3_44100_128" +BITRATE = 128_000 + +PREMADE = [ + {"voice_id": "21m00Tcm4TlvDq8ikWAM", "name": "Rachel", "category": "premade"}, + {"voice_id": "pNInz6obpgDQGcFMaJgB", "name": "Adam", "category": "premade"}, + {"voice_id": "EXAVITQu4vr4xnSDxMaL", "name": "Bella", "category": "premade"}, + {"voice_id": "ErXwobaYiN019PkySvjV", "name": "Antoni", "category": "premade"}, + {"voice_id": "AZnzlk1XvdvUeBnXmlld", "name": "Domi", "category": "premade"}, + {"voice_id": "TxGEqnHWrfWFTfGW9XjX", "name": "Josh", "category": "premade"}, +] + + +class TTSError(RuntimeError): + pass + + +def _headers() -> dict: + if not settings.elevenlabs_api_key: + raise TTSError("ELEVENLABS_API_KEY is not set") + return {"xi-api-key": settings.elevenlabs_api_key} + + +def mp3_seconds(data: bytes) -> float: + return len(data) * 8 / BITRATE + + +def _voice_settings() -> dict: + return {"stability": 0.45, "similarity_boost": 0.8, "style": 0.2, "use_speaker_boost": True} + + +def _cache_key(workspace_id, text: str, voice_id: str, variant: str) -> str: + return keys.tts_cache(workspace_id, text, voice_id, + json.dumps(_voice_settings(), sort_keys=True) + settings.elevenlabs_model + variant) + + +def synthesize(workspace_id, text: str, voice_id: str) -> bytes: + key = _cache_key(workspace_id, text, voice_id, OUTPUT) + if storage.exists(key): + return storage.get_bytes(key) + body = {"text": text, "model_id": settings.elevenlabs_model, "voice_settings": _voice_settings()} + resp = httpx.post(f"{API}/text-to-speech/{voice_id}", params={"output_format": OUTPUT}, + headers=_headers(), json=body, timeout=120) + if resp.status_code >= 400: + raise TTSError(f"ElevenLabs {resp.status_code}: {resp.text[:300]}") + storage.put_bytes(key, resp.content, "audio/mpeg") + return resp.content + + +def synthesize_with_timestamps(workspace_id, text: str, voice_id: str) -> tuple[bytes, list[dict]]: + """Audio plus word timings [{text, startMs, endMs}] for captions.""" + key = _cache_key(workspace_id, text, voice_id, "timestamps") + ".json" + if storage.exists(key): + data = json.loads(storage.get_bytes(key)) + else: + body = {"text": text, "model_id": settings.elevenlabs_model, "voice_settings": _voice_settings()} + resp = httpx.post(f"{API}/text-to-speech/{voice_id}/with-timestamps", params={"output_format": OUTPUT}, + headers=_headers(), json=body, timeout=120) + if resp.status_code >= 400: + raise TTSError(f"ElevenLabs {resp.status_code}: {resp.text[:300]}") + data = resp.json() + storage.put_bytes(key, json.dumps(data).encode(), "application/json") + audio = base64.b64decode(data["audio_base64"]) + return audio, words_from_alignment(data.get("alignment") or {}) + + +def words_from_alignment(alignment: dict) -> list[dict]: + chars = alignment.get("characters") or [] + starts = alignment.get("character_start_times_seconds") or [] + ends = alignment.get("character_end_times_seconds") or [] + words: list[dict] = [] + current, start, last_end = "", 0.0, 0.0 + for ch, s, e in zip(chars, starts, ends, strict=False): + if ch.isspace(): + if current: + words.append({"text": current, "startMs": int(start * 1000), "endMs": int(last_end * 1000)}) + current = "" + continue + if not current: + start = s + current += ch + last_end = e + if current: + words.append({"text": current, "startMs": int(start * 1000), "endMs": int(last_end * 1000)}) + return words + + +def list_voices() -> list[dict]: + if not settings.elevenlabs_api_key: + return PREMADE + try: + resp = httpx.get(f"{API}/voices", headers=_headers(), timeout=20) + resp.raise_for_status() + except httpx.HTTPError: + return PREMADE + return [ + {"voice_id": v["voice_id"], "name": v["name"], "category": v.get("category"), + "preview_url": v.get("preview_url")} + for v in resp.json().get("voices", []) + ] + + +def clone_voice(name: str, sample: bytes, filename: str, content_type: str) -> str: + resp = httpx.post(f"{API}/voices/add", headers=_headers(), timeout=120, + data={"name": name, "description": "Notestack consented voice clone"}, + files={"files": (filename, sample, content_type)}) + if resp.status_code >= 400: + raise TTSError(f"ElevenLabs {resp.status_code}: {resp.text[:300]}") + return resp.json()["voice_id"] + + +def delete_voice(voice_id: str) -> None: + httpx.delete(f"{API}/voices/{voice_id}", headers=_headers(), timeout=30) diff --git a/backend/app/services/usage.py b/backend/app/services/usage.py new file mode 100644 index 0000000000000000000000000000000000000000..c47d2fefbee2d01df320f8e19765dd703c276003 --- /dev/null +++ b/backend/app/services/usage.py @@ -0,0 +1,64 @@ +"""Monthly usage against plan limits, from usage_events.""" + +from datetime import UTC, datetime + +from fastapi import HTTPException +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from app.models import Artifact, UsageEvent, Workspace +from app.services.plans import Plan, effective_plan + + +def month_start(now: datetime | None = None) -> datetime: + now = now or datetime.now(UTC) + return now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + + +def month_usage(db: Session, workspace_id) -> dict: + since = month_start() + + def total(kind: str, unit: str) -> float: + return float(db.scalar( + select(func.coalesce(func.sum(UsageEvent.quantity), 0)).where( + UsageEvent.workspace_id == workspace_id, UsageEvent.kind == kind, + UsageEvent.unit == unit, UsageEvent.created_at >= since, + ) + ) or 0) + + kits = db.scalar( + select(func.count()).select_from(Artifact).where( + Artifact.workspace_id == workspace_id, Artifact.type == "launch_kit", Artifact.created_at >= since + ) + ) or 0 + return { + "audio_minutes": round(total("tts", "seconds") / 60, 1), + "video_minutes": round(total("video", "seconds") / 60, 1), + "launch_kits": kits, + "llm_tokens": int(total("llm", "tokens")), + "since": since.isoformat(), + } + + +def usage_report(db: Session, workspace: Workspace) -> dict: + plan = effective_plan(db, workspace) + used = month_usage(db, workspace.id) + return { + "plan": plan.id, + "used": used, + "limits": {"audio_minutes": plan.audio_minutes, "video_minutes": plan.video_minutes, + "launch_kits": plan.launch_kits, "sources": plan.sources, "indexed_posts": plan.indexed_posts}, + } + + +def check_limit(db: Session, workspace: Workspace, kind: str, amount: float = 1) -> Plan: + """Raise 402 plan_limit when this request would go over the monthly allowance.""" + plan = effective_plan(db, workspace) + used = month_usage(db, workspace.id) + limit = {"audio_minutes": plan.audio_minutes, "video_minutes": plan.video_minutes, + "launch_kits": plan.launch_kits}[kind] + if limit >= 0 and used[kind] + amount > limit: + label = kind.replace("_", " ") + raise HTTPException(402, {"code": "plan_limit", + "message": f"This would go over your {limit} {label} this month."}) + return plan diff --git a/backend/app/worker.py b/backend/app/worker.py index 0eaac6e635fa063854e6d23ffd36ffdf2b736923..3cd92b2b4e4498d88b9b49250f51cd716ee02e0e 100644 --- a/backend/app/worker.py +++ b/backend/app/worker.py @@ -34,16 +34,22 @@ from app.models import ( Subscription, UpdateEmail, UpdateEmailSend, + Upload, User, + VoiceConsent, Workspace, WorkspaceMember, ) -from app.pipeline.ingest import ingest_source +from app.pipeline import generate, launchkit, media +from app.pipeline.generate import NothingToDo +from app.pipeline.ingest import FeedNotFound, entry_from_upload, extract_article, ingest_source, store_entries +from app.services import launchpad, tts from app.services.email import email_service from app.services.email_verification import purge_old_codes -from app.services.jobs import claim_next, retry_or_fail, stale_jobs, update_job +from app.services.jobs import claim_next, create_job, retry_or_fail, stale_jobs, update_job from app.services.plans import effective_plan -from app.services.storage import keys, storage +from app.services.renderer import PermanentJobError, request_render +from app.services.storage import storage log = logging.getLogger("notestack.worker") @@ -60,55 +66,140 @@ HANDED_OFF = object() # the job continues elsewhere (renderer) and reports back # Handlers: (db, job) -> Done | HANDED_OFF. Raise to trigger a retry. +def _after_import(db: Session, job: Job, workspace_id: uuid.UUID, changed: list[str]) -> None: + """New or changed posts get mapped into the topic constellation automatically.""" + if changed and settings.llm_api_key: + create_job(db, workspace_id, "topics", {"document_ids": changed}, max_attempts=2) + + def handle_ingest(db: Session, job: Job): source = db.get(Source, uuid.UUID(job.params["source_id"])) if not source: return Done({"error": "source deleted"}, "Source no longer exists") plan = effective_plan(db, db.get(Workspace, source.workspace_id)) try: - result = ingest_source(db, source, job, max_posts=min(plan.indexed_posts, 200)) + result = ingest_source(db, source, job, max_posts=plan.indexed_posts) except Exception as exc: db.rollback() source.sync_status = "error" source.sync_error = str(exc)[:500] db.commit() + if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code in (401, 403, 404, 410): + raise PermanentJobError(f"The feed returned {exc.response.status_code}.") from exc raise - return Done(result, "All posts in orbit") + _after_import(db, job, source.workspace_id, result.get("changed", [])) + return Done(result, f"All posts in orbit: {result['indexed']} new or updated") + + +def handle_import_url(db: Session, job: Job): + source = db.get(Source, uuid.UUID(job.params["source_id"])) + update_job(db, job, progress=0.1, message="Fetching the article") + try: + entry = extract_article(job.params["url"]) + except FeedNotFound as exc: + raise PermanentJobError(str(exc)) from exc + indexed, _, changed = store_entries(db, source, [entry]) + _after_import(db, job, source.workspace_id, [str(i) for i in changed]) + return Done({"indexed": indexed, "document_ids": [str(i) for i in changed]}, f"{entry.title[:80]} is in orbit") + + +def handle_import_upload(db: Session, job: Job): + source = db.get(Source, uuid.UUID(job.params["source_id"])) + upload = db.get(Upload, uuid.UUID(job.params["upload_id"])) + update_job(db, job, progress=0.1, message=f"Reading {upload.filename}") + entry = entry_from_upload(upload.filename, upload.content_type, storage.get_bytes(upload.key)) + if not entry.sections and not entry.html: + raise PermanentJobError("We could not read any text in that file.") + indexed, _, changed = store_entries(db, source, [entry]) + _after_import(db, job, source.workspace_id, [str(i) for i in changed]) + return Done({"indexed": indexed, "document_ids": [str(i) for i in changed]}, f"{entry.title[:80]} is in orbit") + + +def handle_topics(db: Session, job: Job): + return Done(generate.extract_topics(db, job, job.workspace_id, job.params.get("document_ids")), "Topic map ready") + + +def handle_voice_profile(db: Session, job: Job): + try: + result = generate.build_voice_profile(db, job, job.workspace_id, job.params.get("document_ids") or []) + except NothingToDo as exc: + raise PermanentJobError(str(exc)) from exc + return Done(result, "Voice profile ready") + + +def handle_voice_clone(db: Session, job: Job): + consent = db.get(VoiceConsent, uuid.UUID(job.params["consent_id"])) + if not consent or consent.revoked_at: + return Done({}, "Consent was revoked") + update_job(db, job, progress=0.2, message="Sending your sample to ElevenLabs") + head = storage.head(consent.sample_key) or {} + try: + voice_id = tts.clone_voice(f"Notestack {consent.workspace_id}", storage.get_bytes(consent.sample_key), + consent.sample_key.rsplit("/", 1)[-1], head.get("ContentType") or "audio/webm") + except tts.TTSError as exc: + raise PermanentJobError(str(exc)) from exc + consent.elevenlabs_voice_id = voice_id + db.commit() + return Done({"voice_id": voice_id}, "Your voice is ready") + + +def handle_resurface(db: Session, job: Job): + return Done(generate.score_evergreen(db, job, job.workspace_id), "Evergreen scores updated") + + +def _artifact_handler(fn: Callable[[Session, Job, Artifact], object], done_message: str): + def handler(db: Session, job: Job): + artifact = db.get(Artifact, uuid.UUID(job.params["artifact_id"])) + if not artifact: + return Done({}, "Artifact was deleted") + artifact.status = "generating" + db.commit() + try: + result = fn(db, job, artifact) + except (NothingToDo, tts.TTSError) as exc: + raise PermanentJobError(str(exc)) from exc + if result is None: + return HANDED_OFF # the renderer reports back through /api/internal/jobs/{id}/progress + return Done(result if isinstance(result, dict) else {}, done_message) + + return handler def handle_render(db: Session, job: Job): - """Hand off to the Remotion renderer with a presigned PUT so it never holds R2 credentials.""" - p = job.params - artifact = db.get(Artifact, uuid.UUID(p["artifact_id"])) - ext = "png" if p["format"] == "png" else "mp4" - key = keys.artifact(artifact.workspace_id, artifact.id, p["composition"].lower(), ext) - content_type = "image/png" if ext == "png" else "video/mp4" + """Direct render of caller supplied props (POST /api/artifacts/{id}/render).""" + artifact = db.get(Artifact, uuid.UUID(job.params["artifact_id"])) update_job(db, job, progress=0.01, message="T-minus: preparing render") - resp = httpx.post( - f"{settings.renderer_url}/render", - headers={"x-internal-token": settings.internal_token}, - json={ - "jobId": str(job.id), - "compositionId": p["composition"], - "props": p["props"], - "format": p["format"], - "uploadUrl": storage.presign_put(key, content_type, ttl=6 * 3600), - "uploadContentType": content_type, - "callbackUrl": f"{settings.api_url}/api/internal/jobs/{job.id}/progress", - "storageKey": key, - }, - timeout=30, - ) - resp.raise_for_status() + request_render(job, artifact, job.params["composition"], job.params.get("props") or {}) return HANDED_OFF HANDLERS: dict[str, Callable[[Session, Job], object]] = { "ingest": handle_ingest, + "import_url": handle_import_url, + "import_upload": handle_import_upload, + "topics": handle_topics, + "voice_profile": handle_voice_profile, + "voice_clone": handle_voice_clone, + "resurface_scan": handle_resurface, + "summary": _artifact_handler(generate.summarize_notebook, "Summary ready"), + "audio_overview": _artifact_handler(media.audio_overview, "Audio overview ready"), + "video": _artifact_handler(media.make_video, "Video ready"), + "quote_card": _artifact_handler(media.make_quote_card, "Quote card ready"), + "carousel": _artifact_handler(media.render_carousel, "Carousel ready"), + "launch_kit": _artifact_handler(launchkit.build_launch_kit, "Launch Kit ready"), "render": handle_render, } +def mark_artifact_failed(db: Session, job: Job, error: str) -> None: + if job.artifact_id: + artifact = db.get(Artifact, job.artifact_id) + if artifact: + artifact.status = "failed" + artifact.content_json = {**(artifact.content_json or {}), "error": error[:500]} + db.commit() + + def run_job( job_id: uuid.UUID, session_factory: Callable[[], Session] = SessionLocal, @@ -125,11 +216,18 @@ def run_job( return try: outcome = handler(db, job) + except PermanentJobError as exc: + db.rollback() + job = db.get(Job, job_id) + update_job(db, job, status="failed", error=str(exc), message=str(exc)[:500]) + mark_artifact_failed(db, job, str(exc)) + return except Exception as exc: log.exception("job %s (%s) attempt %s failed", job.id, job.kind, job.attempts) db.rollback() job = db.get(Job, job_id) - retry_or_fail(db, job, f"{type(exc).__name__}: {exc}") + if not retry_or_fail(db, job, f"{type(exc).__name__}: {exc}"): + mark_artifact_failed(db, job, f"{type(exc).__name__}: {exc}") return if outcome is HANDED_OFF: return @@ -219,6 +317,8 @@ class Periodic: PERIODIC = [ Periodic("recover_stale", 60, recover_stale), + Periodic("publish_due", 30, launchpad.publish_due), + Periodic("sync_engagement", 3600, launchpad.sync_engagement), Periodic("update_email_batch", 300, update_email_batch), Periodic("daily_cleanup", 24 * 3600, daily_cleanup), ] @@ -240,13 +340,9 @@ def run_periodic(now: float) -> None: # Main loop -def main() -> None: - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") - worker_id = f"{socket.gethostname()}:{os.getpid()}" - stop = threading.Event() - for sig in (signal.SIGINT, signal.SIGTERM): - signal.signal(sig, lambda *_: stop.set()) - +def run_loop(stop: threading.Event, worker_id: str | None = None) -> None: + """Claim and run jobs until `stop` is set. Used by the CLI and, in development, by the API.""" + worker_id = worker_id or f"{socket.gethostname()}:{os.getpid()}" pool = ThreadPoolExecutor(max_workers=settings.worker_concurrency, thread_name_prefix="job") running: set[Future] = set() log.info("worker %s up, concurrency=%s", worker_id, settings.worker_concurrency) @@ -273,5 +369,19 @@ def main() -> None: pool.shutdown(wait=True) +def start_background(stop: threading.Event) -> threading.Thread: + thread = threading.Thread(target=run_loop, args=(stop, f"api:{os.getpid()}"), name="worker", daemon=True) + thread.start() + return thread + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + stop = threading.Event() + for sig in (signal.SIGINT, signal.SIGTERM): + signal.signal(sig, lambda *_: stop.set()) + run_loop(stop) + + if __name__ == "__main__": main() diff --git a/backend/requirements.txt b/backend/requirements.txt index b327de342f9cb1fb7c4bace6135528c88d201618..69ad491084d143f2b65629a270414ff168659651 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -16,3 +16,4 @@ httpx==0.28.* dspy==2.6.* feedparser==6.* beautifulsoup4==4.* +pypdf==5.* diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 6a46bf2d51671c6c3a9b6680c22e508221c22135..fcdae4ca74a60844ac7e6d84be165e9ee93559c7 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -1,6 +1,13 @@ import os +import tempfile os.environ.setdefault("DATABASE_URL", "sqlite://") +os.environ["STORAGE_BACKEND"] = "local" +os.environ["LOCAL_STORAGE_DIR"] = tempfile.mkdtemp(prefix="notestack-storage-") +os.environ["CORPUS_CACHE_DIR"] = tempfile.mkdtemp(prefix="notestack-corpus-") +os.environ["RUN_WORKER_IN_API"] = "false" +os.environ["LLM_API_KEY"] = "" +os.environ["ELEVENLABS_API_KEY"] = "" os.environ.setdefault("EMAIL_PROVIDER", "console") os.environ.setdefault("JWT_SECRET", "test-secret-that-is-at-least-32-bytes-long") os.environ.setdefault("BILLING_ENABLED", "false") @@ -18,15 +25,41 @@ from app.services.email import ConsoleEmailProvider # noqa: E402 @pytest.fixture() -def db_session(): +def session_factory(): engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool) Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine, expire_on_commit=False) - session = Session() + return sessionmaker(bind=engine, expire_on_commit=False) + + +@pytest.fixture() +def db_session(session_factory): + session = session_factory() yield session session.close() +@pytest.fixture() +def run_jobs(session_factory, db_session): + """Drain the queue the way the worker does, in process. Returns the jobs that ran.""" + from app.models import Job + from app.services.jobs import claim_next + from app.worker import run_job + + def drain(limit: int = 50) -> list: + ran = [] + for _ in range(limit): + with session_factory() as db: + job_id = claim_next(db, "test") + if not job_id: + break + run_job(job_id, session_factory=session_factory) + ran.append(job_id) + db_session.expire_all() + return [db_session.get(Job, j) for j in ran] + + return drain + + @pytest.fixture() def client(db_session): app.dependency_overrides[get_db] = lambda: db_session diff --git a/backend/tests/test_buildout.py b/backend/tests/test_buildout.py new file mode 100644 index 0000000000000000000000000000000000000000..122ae72db072a2b7bda6c35f91f1da80a98bedda --- /dev/null +++ b/backend/tests/test_buildout.py @@ -0,0 +1,612 @@ +"""End to end coverage for sources, notebooks, generation, Launchpad and settings. External services +(feeds, LLM, ElevenLabs, renderer, social APIs) are stubbed at their module seams.""" + +import time +import uuid +from datetime import UTC, datetime, timedelta + +import httpx +import pytest + +from app.models import ( + Artifact, + CalendarItem, + Document, + EngagementEvent, + Job, + SocialAccount, + Source, + TrackedLink, + UsageEvent, + Workspace, +) +from app.services.email import ConsoleEmailProvider +from tests.conftest import last_code + +FEED = """ +Ada Writeshttps://ada.example.com +On Pricinghttps://ada.example.com/p/on-pricing +Mon, 03 Mar 2025 10:00:00 GMT +Why price high

Charging more made my readers take the work seriously.

+

The paid tier doubled after I raised the price to ten dollars.

]]> +Writing Every Dayhttps://ada.example.com/p/writing-every-day +Tue, 04 Jun 2024 10:00:00 GMT +I write five hundred words before breakfast.

+

Habits beat inspiration.

]]> +""" + + +class FakeResponse: + def __init__(self, status: int = 200, text: str = "", content_type: str = "application/rss+xml", + url: str = "", json_data=None): + self.status_code = status + self.text = text + self.content = text.encode() + self.headers = {"content-type": content_type} + self.url = url + self._json = json_data + + def json(self): + return self._json + + def raise_for_status(self): + if self.status_code >= 400: + raise httpx.HTTPStatusError("bad", request=None, response=self) + + +def fake_web(routes: dict): + def get(url, *args, **kwargs): + for prefix, resp in routes.items(): + if url.startswith(prefix): + resp.url = url + return resp + return FakeResponse(404, "not found", "text/html", url) + + return get + + +@pytest.fixture() +def auth(client): + client.post("/api/auth/email/register/start", json={"email": "ada@example.com", "password": "stardust-42", + "name": "Ada Lovelace"}) + r = client.post("/api/auth/email/register/verify", json={"email": "ada@example.com", "code": last_code()}) + return {"Authorization": f"Bearer {r.json()['access_token']}"} + + +@pytest.fixture() +def feed(monkeypatch): + monkeypatch.setattr(httpx, "get", fake_web({"https://ada.example.com/feed": FakeResponse(200, FEED)})) + + +@pytest.fixture() +def llm(monkeypatch): + """Canned outputs per signature. Tests override entries in the returned dict.""" + from app.llm import run + + outputs: dict = {} + calls: list[str] = [] + + def predict(signature, *, db, workspace_id, job=None, lm=None, **inputs): + name = signature.__name__ + calls.append(name) + value = outputs[name] + return value(**inputs) if callable(value) else value + + monkeypatch.setattr(run, "predict", predict) + outputs["_calls"] = calls + return outputs + + +def connect(client, auth, run_jobs): + r = client.post("/api/sources", json={"url": "ada.example.com"}, headers=auth) + assert r.status_code == 200, r.text + jobs = run_jobs() + assert jobs[0].status == "done", jobs[0].error + return r.json()["source"] + + +def docs(client, auth): + return client.get("/api/documents", headers=auth).json()["items"] + + +# Storage + + +def test_local_storage_signed_urls(client): + from app.services.storage import storage + + storage.put_bytes("ws/x/file.txt", b"0123456789") + url = storage.presign_get("ws/x/file.txt").split("8000", 1)[1] + assert client.get(url).content == b"0123456789" + ranged = client.get(url, headers={"Range": "bytes=2-4"}) + assert ranged.status_code == 206 and ranged.content == b"234" + assert client.get(url.replace("sig=", "sig=0")).status_code == 403 + expired = storage._signed("GET", "ws/x/file.txt", -10) + assert client.get(expired.split("8000", 1)[1]).status_code == 403 + put = storage.presign_put("ws/x/new.bin", "application/octet-stream").split("8000", 1)[1] + assert client.put(put, content=b"abc").status_code == 200 + assert storage.get_bytes("ws/x/new.bin") == b"abc" + assert client.get(url.replace("file.txt", "new.bin")).status_code == 403 # signature is per key + + +# Sources + + +def test_connect_rejects_missing_feed(client, auth, monkeypatch): + monkeypatch.setattr(httpx, "get", fake_web({})) + r = client.post("/api/sources", json={"url": "nobody.substack.com"}, headers=auth) + assert r.status_code == 422 + assert r.json()["detail"]["code"] == "feed_not_found" + + +def test_connect_ingest_read_and_delete(client, auth, run_jobs, feed, db_session): + source = connect(client, auth, run_jobs) + assert source["title"] == "Ada Writes" + items = docs(client, auth) + assert {d["title"] for d in items} == {"On Pricing", "Writing Every Day"} + listed = client.get("/api/sources", headers=auth).json() + assert listed[0]["document_count"] == 2 and listed[0]["sync_status"] == "ok" + + post = client.get(f"/api/documents/{items[0]['id']}", headers=auth).json() + assert post["lines"][0] == "---" and any("paid tier" in ln or "breakfast" in ln for ln in post["lines"]) + assert client.get("/api/documents", params={"q": "breakfast"}, headers=auth).json()["total"] == 1 + + # Re-sync is a no-op for unchanged posts. + client.post(f"/api/sources/{source['id']}/sync", headers=auth) + assert run_jobs()[0].result["skipped"] == 2 + + r = client.delete(f"/api/sources/{source['id']}", headers=auth) + assert r.json()["removed_posts"] == 2 + assert docs(client, auth) == [] + + +def test_import_url_and_markdown_upload(client, auth, run_jobs, monkeypatch): + article = FakeResponse(200, "Loose Post" + "" + "

Loose Post

Standalone essay text lives here.

" + "
", "text/html") + monkeypatch.setattr(httpx, "get", fake_web({"https://elsewhere.example.com/essay": article})) + r = client.post("/api/sources/url", json={"url": "https://elsewhere.example.com/essay"}, headers=auth) + assert r.status_code == 200 + assert run_jobs()[0].status == "done" + + start = client.post("/api/storage/uploads", json={"filename": "notes.md", "content_type": "text/markdown", + "size_bytes": 60}, headers=auth).json() + client.put(start["upload_url"].split("8000", 1)[1], content=b"# Field Notes\n\n## Day one\nIt rained all day.\n") + client.post(f"/api/storage/uploads/{start['upload_id']}/complete", headers=auth) + r = client.post("/api/sources/upload", json={"upload_id": start["upload_id"]}, headers=auth) + assert run_jobs()[0].status == "done", r.text + titles = {d["title"] for d in docs(client, auth)} + assert titles == {"Loose Post", "Field Notes"} + sources = client.get("/api/sources", headers=auth).json() + assert len(sources) == 1 and sources[0]["is_imports"] + + +# Notebooks + + +def test_notebook_crud_chat_history_and_summary(client, auth, run_jobs, feed, llm, db_session): + connect(client, auth, run_jobs) + items = docs(client, auth) + pricing = next(d for d in items if d["title"] == "On Pricing") + nb = client.post("/api/notebooks", json={"title": "Money", "document_ids": [pricing["id"]]}, headers=auth).json() + assert nb["added"] == 1 + assert client.patch(f"/api/notebooks/{nb['id']}", json={"title": "Pricing"}, headers=auth).json()["title"] == \ + "Pricing" + listed = client.get("/api/notebooks", headers=auth).json() + assert listed[0]["document_count"] == 1 + + path = pricing["path"] + llm["SummarizeNotebook"] = { + "summary": "Raising prices worked [1]. Also the moon is cheese [2].", + "themes": ["Pricing"], + "citations": [{"marker": 1, "path": path, "line_start": 12, "line_end": 14}, + {"marker": 2, "path": "sources/fake.md", "line_start": 1, "line_end": 2}], + } + art = client.post("/api/artifacts/generate", json={"type": "summary", "notebook_id": nb["id"]}, + headers=auth).json() + assert art["job"]["status"] == "queued" + run_jobs() + done = client.get(f"/api/artifacts/{art['id']}", headers=auth).json() + assert done["status"] == "ready" + assert len(done["content"]["citations"]) == 1 # the invented citation was dropped + assert "[2]" not in done["content"]["summary"] + assert client.get(f"/api/notebooks/{nb['id']}", headers=auth).json()["summary"].startswith("Raising") + + # Chat history endpoints + from app.models import Chat, Message + + chat = Chat(workspace_id=db_session.get(Source, uuid.UUID(pricing["source_id"])).workspace_id, + notebook_id=uuid.UUID(nb["id"]), title="q") + db_session.add(chat) + db_session.flush() + db_session.add(Message(chat_id=chat.id, role="user", content="Why price high?")) + db_session.commit() + chats = client.get(f"/api/notebooks/{nb['id']}/chats", headers=auth).json() + assert chats[0]["id"] == str(chat.id) + assert client.get(f"/api/notebooks/chats/{chat.id}/messages", headers=auth).json()[0]["text"] == "Why price high?" + + client.delete(f"/api/notebooks/{nb['id']}/documents/{pricing['id']}", headers=auth) + assert client.get(f"/api/notebooks/{nb['id']}", headers=auth).json()["documents"] == [] + assert client.delete(f"/api/notebooks/{nb['id']}", headers=auth).status_code == 200 + + +def test_summary_of_empty_notebook_fails_cleanly(client, auth, run_jobs, llm): + nb = client.post("/api/notebooks", json={"title": "Empty"}, headers=auth).json() + art = client.post("/api/artifacts/generate", json={"type": "summary", "notebook_id": nb["id"]}, + headers=auth).json() + job = run_jobs()[0] + assert job.status == "failed" and job.attempts == 1 # permanent, no retries + assert client.get(f"/api/artifacts/{art['id']}", headers=auth).json()["status"] == "failed" + + +# Topics, voice, resurfacing + + +def test_topic_map_and_notebook_from_topic(client, auth, run_jobs, feed, llm): + connect(client, auth, run_jobs) + llm["ExtractTopics"] = lambda title, **_: {"topics": [ + {"name": "Pricing" if "Pricing" in title else "Habits", "weight": 0.9}, + {"name": "Writing Business", "weight": 0.5}, + {"name": "Writing business", "weight": 0.4}, + ]} + llm["ConsolidateTopics"] = {"groups": [ + {"canonical": "Writing Business", "members": ["Writing Business", "Writing business"], + "summary": "Making writing pay."}, + ]} + client.post("/api/topics/rebuild", headers=auth) + assert run_jobs()[0].status == "done" + graph = client.get("/api/topics", headers=auth).json() + names = {n["name"]: n for n in graph["nodes"]} + assert set(names) == {"Pricing", "Habits", "Writing Business"} + assert names["Writing Business"]["post_count"] == 2 + assert len(graph["edges"]) == 2 + detail = client.get(f"/api/topics/{names['Writing Business']['id']}", headers=auth).json() + assert len(detail["posts"]) == 2 + nb = client.post(f"/api/notebooks/from-topic/{names['Pricing']['id']}", headers=auth).json() + assert nb["added"] == 1 + + +def test_voice_profile_build_and_edit(client, auth, run_jobs, feed, llm): + connect(client, auth, run_jobs) + profile = {"tone": ["warm"], "sentence_length": "short", "vocabulary": ["readers"], "structure_habits": [], + "openings": ["A number"], "avoid": ["jargon"], "summary": "Plain and direct."} + llm["BuildVoiceProfile"] = {"profile": profile} + client.post("/api/voice/build", json={"document_ids": []}, headers=auth) + assert run_jobs()[0].status == "done" + voice = client.get("/api/voice", headers=auth).json() + assert voice["profile"]["summary"] == "Plain and direct." + r = client.put("/api/voice", json={"host_voices": {"host_a": "abc", "host_b": "def"}}, headers=auth) + assert r.json()["host_voices"] == {"host_a": "abc", "host_b": "def"} + bad = client.post("/api/voice/consent", json={"upload_id": str(uuid.uuid4()), "agreed": True, + "consent_text": "nope"}, headers=auth) + assert bad.status_code == 400 + + +def test_resurface_scan_and_suggestions(client, auth, run_jobs, feed, llm): + connect(client, auth, run_jobs) + llm["EvergreenScore"] = lambda title, **_: {"score": 0.9 if "Writing" in title else 0.2, + "reason": "Timeless habit advice", "angle": "Still true"} + client.post("/api/resurface/scan", headers=auth) + assert run_jobs()[0].status == "done" + data = client.get("/api/resurface", headers=auth).json() + assert data["unscored"] == 0 + assert data["evergreen"][0]["title"] == "Writing Every Day" + assert data["evergreen"][0]["angle"] == "Still true" + + +# Audio, video, launch kit + + +def test_audio_overview_and_limits(client, auth, run_jobs, feed, llm, monkeypatch, db_session): + from app.config import settings + from app.services import tts + + connect(client, auth, run_jobs) + doc = docs(client, auth)[0] + monkeypatch.setattr(settings, "elevenlabs_api_key", "test") + monkeypatch.setattr(tts, "synthesize", lambda ws, text, voice: b"\x00" * 16000) # 1 second at 128 kbps + llm["PodcastScript"] = {"lines": [ + {"speaker": "host_a", "text": "Welcome in.", "sources": []}, + {"speaker": "host_b", "text": "Prices went up.", "sources": [{"path": doc["path"], "line_start": 12, + "line_end": 13}]}, + ]} + art = client.post("/api/artifacts/generate", json={"type": "audio_overview", "document_id": doc["id"], + "minutes": 5}, headers=auth).json() + run_jobs() + done = client.get(f"/api/artifacts/{art['id']}", headers=auth).json() + assert done["status"] == "ready", done["content"] + assert done["content"]["duration_s"] == 2.0 + assert done["url"] and client.get(done["url"].split("8000", 1)[1]).content == b"\x00" * 32000 + + ws_id = db_session.get(Document, uuid.UUID(doc["id"])).workspace_id + db_session.add(UsageEvent(workspace_id=ws_id, kind="tts", provider="elevenlabs", quantity=240 * 60, + unit="seconds")) + db_session.commit() + r = client.post("/api/artifacts/generate", json={"type": "audio_overview", "document_id": doc["id"], + "minutes": 5}, headers=auth) + assert r.status_code == 402 and r.json()["detail"]["code"] == "plan_limit" + + +def test_video_hands_off_and_renderer_callback(client, auth, run_jobs, feed, llm, monkeypatch): + from app.pipeline import media + + connect(client, auth, run_jobs) + doc = docs(client, auth)[0] + sent = {} + monkeypatch.setattr(media, "request_render", lambda job, artifact, comp, props=None, stills=None: + sent.update(comp=comp, props=props, job=str(job.id))) + llm["VideoStoryboard"] = {"scenes": [ + {"type": "title", "on_screen_text": "Charge more", "narration": "", "duration_hint_s": 2.5, "visual": ""}, + {"type": "section", "on_screen_text": "Readers valued it", "narration": "", "duration_hint_s": 3, + "visual": ""}, + ]} + art = client.post("/api/artifacts/generate", json={"type": "video", "document_id": doc["id"], "style": "short"}, + headers=auth).json() + job = run_jobs()[0] + assert job.status == "running" # handed off to the renderer + assert sent["comp"] == "ShortVertical" and sent["props"]["hook"] == "Charge more" + from app.config import settings + + r = client.post(f"/api/internal/jobs/{sent['job']}/progress", headers={"x-internal-token": settings.internal_token}, + json={"status": "done", "progress": 1, "storage_key": "ws/x/video.mp4", "render_seconds": 12}) + assert r.status_code == 200 + assert client.get(f"/api/artifacts/{art['id']}", headers=auth).json()["status"] == "ready" + + +def test_renderer_down_fails_without_retry(client, auth, run_jobs, feed, llm, monkeypatch): + connect(client, auth, run_jobs) + doc = docs(client, auth)[0] + + def refuse(*args, **kwargs): + raise httpx.ConnectError("refused") + + monkeypatch.setattr(httpx, "post", refuse) + llm["PickQuotes"] = lambda passages, **_: {"quotes": [{"quote": "Charging more made my readers take the work " + "seriously.", + "source": {"path": doc["path"], "line_start": 1, + "line_end": 20}}]} + art = client.post("/api/artifacts/generate", json={"type": "quote_card", "document_id": doc["id"]}, + headers=auth).json() + job = run_jobs()[0] + assert job.status == "failed" and "Renderer is not running" in job.error + assert client.get(f"/api/artifacts/{art['id']}", headers=auth).json()["status"] == "failed" + + +def test_launch_kit(client, auth, run_jobs, feed, llm, monkeypatch, db_session): + from app.pipeline import media + + connect(client, auth, run_jobs) + doc = next(d for d in docs(client, auth) if d["title"] == "On Pricing") + ref = {"path": doc["path"], "line_start": 12, "line_end": 14} + llm.update({ + "ExtractClaims": {"claims": [{"claim": "Higher prices signal value", "sources": [ref]}, + {"claim": "Unsupported", "sources": [{**ref, "line_start": 999}]}]}, + "HookGenerator": {"hooks": [{"text": "I doubled my price.", "strength": 0.9, "rationale": "number"}]}, + "PlatformAdapter": lambda platform, **_: {"posts": [f"{platform} post one", f"{platform} post two"]}, + "SeoPack": {"seo": {"title_options": ["On Pricing"], "meta_description": "Why I charge more.", + "slug": "on-pricing", "keywords": ["pricing"], "internal_links": []}}, + "CarouselSlides": {"slides": [{"heading": "Charge more", "body": "It works."}]}, + "PickQuotes": {"quotes": [{"quote": "Charging more made my readers take the work seriously.", + "source": ref}]}, + }) + monkeypatch.setattr(media, "request_render", lambda *a, **k: None) + art = client.post("/api/artifacts/generate", json={"type": "launch_kit", "document_id": doc["id"]}, + headers=auth).json() + run_jobs() + kit = client.get(f"/api/artifacts/{art['id']}", headers=auth).json() + assert kit["status"] == "ready", kit["content"] + c = kit["content"] + assert len(c["claims"]) == 1 # the unverifiable claim was dropped + assert set(c["posts"]) == {"x_thread", "linkedin", "substack_notes", "bluesky"} + assert len(c["quote_card_ids"]) == 1 + card = client.get(f"/api/artifacts/{c['quote_card_ids'][0]}", headers=auth).json() + assert card["status"] == "rendering" # its own job ran and handed off + + edited = client.patch(f"/api/artifacts/{art['id']}", json={"content": {"posts": {**c["posts"], + "linkedin": ["Edited"]}}}, + headers=auth).json() + assert edited["content"]["posts"]["linkedin"] == ["Edited"] + assert edited["content"]["quote_card_ids"] == c["quote_card_ids"] + + listing = client.get("/api/artifacts", params={"type": "launch_kit"}, headers=auth).json() + assert listing["total"] == 1 + assert client.delete(f"/api/artifacts/{art['id']}", headers=auth).status_code == 200 + + +# Launchpad + + +def _bluesky(client, auth, monkeypatch): + from app.services.social import bluesky + + monkeypatch.setattr(bluesky, "create_session", lambda handle, pw: {"did": "did:plc:ada", "handle": handle, + "accessJwt": "jwt"}) + r = client.post("/api/social/bluesky", json={"handle": "ada.bsky.social", "app_password": "abcd-efgh-ijkl"}, + headers=auth) + assert r.status_code == 200, r.text + return r.json()["accounts"][0] + + +def test_schedule_and_publish_bluesky_thread(client, auth, monkeypatch, session_factory, db_session): + from app.services import launchpad + from app.services.social import bluesky + + account = _bluesky(client, auth, monkeypatch) + published = {} + + def publish(db, acct, posts): + published["posts"] = posts + return "at://did:plc:ada/app.bsky.feed.post/abc", "https://bsky.app/profile/ada.bsky.social/post/abc" + + monkeypatch.setattr(bluesky, "publish", publish) + too_long = client.post("/api/calendar", json={"platform": "bluesky", "content": "x" * 400, + "scheduled_at": datetime.now(UTC).isoformat(), + "social_account_id": account["id"]}, headers=auth) + assert too_long.status_code == 422 + item = client.post("/api/calendar", json={ + "platform": "bluesky", "content": "First", "thread": ["Second", " "], + "scheduled_at": (datetime.now(UTC) - timedelta(minutes=1)).isoformat(), "social_account_id": account["id"], + }, headers=auth).json() + assert item["auto_post"] is True + + assert launchpad.publish_due(session_factory) == 1 + assert published["posts"] == ["First", "Second"] + got = client.get("/api/calendar", headers=auth).json()[0] + assert got["status"] == "posted" and got["external_url"].startswith("https://bsky.app/") + assert client.patch(f"/api/calendar/{item['id']}", json={"content": "late"}, headers=auth).status_code == 409 + + +def test_substack_notes_send_reminder_email(client, auth, session_factory): + from app.services import launchpad + + ConsoleEmailProvider.sent.clear() + item = client.post("/api/calendar", json={"platform": "substack_notes", "content": "A note", + "scheduled_at": (datetime.now(UTC) - timedelta(seconds=5)).isoformat()}, + headers=auth).json() + assert item["remind_by_email"] is True + launchpad.publish_due(session_factory) + assert client.get("/api/calendar", headers=auth).json()[0]["status"] == "reminded" + assert ConsoleEmailProvider.sent[-1].subject == "Time to post on Substack Notes" + + +def test_publish_retries_then_fails(client, auth, monkeypatch, session_factory): + from app.services import launchpad + from app.services.social import SocialError, bluesky + + account = _bluesky(client, auth, monkeypatch) + + def boom(db, acct, posts): + raise SocialError("Bluesky 500: busy") + + monkeypatch.setattr(bluesky, "publish", boom) + item = client.post("/api/calendar", json={"platform": "bluesky", "content": "Hi", "social_account_id": + account["id"], "scheduled_at": datetime.now(UTC).isoformat()}, + headers=auth).json() + r = client.post(f"/api/calendar/{item['id']}/publish", headers=auth).json() + assert r["status"] == "scheduled" and "busy" in r["error"] # retry queued + for _ in range(2): + with session_factory() as db: + db.query(CalendarItem).update({"scheduled_at": datetime.now(UTC) - timedelta(seconds=1)}) + db.commit() + launchpad.publish_due(session_factory) + assert client.get("/api/calendar", headers=auth).json()[0]["status"] == "failed" + + +def test_tracked_link_counts_clicks(client, auth, db_session): + ws = db_session.query(Workspace).first() + item = CalendarItem(workspace_id=ws.id, platform="x", scheduled_at=datetime.now(UTC), content="x", + status="posted") + db_session.add(item) + db_session.flush() + db_session.add(TrackedLink(workspace_id=ws.id, slug="abc1234", target_url="https://ada.example.com/p/x", + calendar_item_id=item.id)) + db_session.commit() + r = client.get("/l/abc1234", follow_redirects=False) + assert r.status_code == 302 and r.headers["location"] == "https://ada.example.com/p/x" + assert db_session.query(EngagementEvent).count() == 1 + assert client.get("/api/calendar", headers=auth).json()[0]["clicks"] == 1 + assert client.get("/l/nope", follow_redirects=False).status_code == 404 + + +def test_oauth_link_is_bound_to_the_starting_workspace(client, auth, monkeypatch, db_session): + import json + + from app.auth import encode_signed + from app.services.crypto import encrypt + + ws = db_session.query(Workspace).first() + info = {"access_token": "tok", "refresh_token": "ref", "expires_in": 7200, "external_id": "42", + "handle": "ada", "scope": "tweet.write"} + + def ticket(ws_id): + return encode_signed({"typ": "social_link", "ws": str(ws_id), "platform": "x", + "data": encrypt(json.dumps(info))}, timedelta(minutes=5)) + + assert client.post("/api/social/complete", json={"ticket": ticket(uuid.uuid4())}, headers=auth).status_code == 403 + r = client.post("/api/social/complete", json={"ticket": ticket(ws.id)}, headers=auth) + assert r.status_code == 200 + acct = db_session.query(SocialAccount).one() + assert acct.platform == "x" and acct.access_token != "tok" # stored encrypted + assert client.post("/api/social/x/start", headers=auth).status_code == 503 # no client id configured + + +def test_oauth_callback_redirects_with_ticket(client, monkeypatch): + from app.auth import encode_signed + from app.services.social import linkedin + + monkeypatch.setattr(linkedin, "exchange_code", lambda code: {"access_token": "t", "external_id": "p", + "handle": "Ada", "expires_in": 60}) + state = encode_signed({"typ": "social_state", "nonce": "n", "ws": str(uuid.uuid4()), "platform": "linkedin"}, + timedelta(minutes=5)) + r = client.get("/api/social/linkedin/callback", params={"code": "c", "state": state}, follow_redirects=False) + assert r.status_code == 302 and "link=" in r.headers["location"] and "/app/launchpad" in r.headers["location"] + bad = client.get("/api/social/linkedin/callback", params={"code": "c", "state": "junk"}, follow_redirects=False) + assert "error=" in bad.headers["location"] + + +# Settings and activity + + +def test_settings_and_usage(client, auth): + s = client.get("/api/settings", headers=auth).json() + assert s["user"]["name"] == "Ada Lovelace" and s["usage"]["used"]["launch_kits"] == 0 + r = client.patch("/api/settings", json={"name": "Ada L", "workspace_name": "Ada HQ", "brand_accent": "#FF0000", + "training_opt_in": True}, headers=auth).json() + assert r["user"]["name"] == "Ada L" and r["workspace"]["name"] == "Ada HQ" + assert r["brand"]["accent"] == "#ff0000" and r["workspace"]["training_opt_in"] is True + assert client.patch("/api/settings", json={"brand_accent": "red"}, headers=auth).status_code == 422 + + +def test_active_jobs_listing(client, auth, feed): + client.post("/api/sources", json={"url": "ada.example.com"}, headers=auth) + active = client.get("/api/jobs", params={"active": 1}, headers=auth).json() + assert active[0]["kind"] == "ingest" and active[0]["status"] == "queued" + + +def test_worker_loop_runs_in_background(tmp_path, monkeypatch): + """The embedded worker thread (RUN_WORKER_IN_API) drains jobs without a second process.""" + import threading + + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + from app import worker + from app.db import Base + from app.services.jobs import create_job + + # A real file database: the in-memory test engine shares one connection across threads. + engine = create_engine(f"sqlite:///{tmp_path / 'w.db'}", connect_args={"check_same_thread": False}) + Base.metadata.create_all(engine) + session_factory = sessionmaker(bind=engine, expire_on_commit=False) + + with session_factory() as db: + ws = Workspace(name="w", owner_id=uuid.uuid4()) + db.add(ws) + db.commit() + job = create_job(db, ws.id, "noop") + monkeypatch.setattr(worker, "SessionLocal", session_factory) + monkeypatch.setitem(worker.HANDLERS, "noop", lambda db, j: worker.Done({"ok": True})) + monkeypatch.setattr(worker, "PERIODIC", []) + real_run_job = worker.run_job + monkeypatch.setattr(worker, "run_job", lambda job_id: real_run_job(job_id, session_factory=session_factory)) + stop = threading.Event() + thread = worker.start_background(stop) + deadline = time.time() + 5 + status = None + while time.time() < deadline: + with session_factory() as db: + status = db.get(Job, job.id).status + if status == "done": + break + time.sleep(0.05) + stop.set() + thread.join(timeout=5) + assert status == "done" + + +def test_artifact_ownership(client, auth, db_session): + other = Artifact(workspace_id=uuid.uuid4(), type="summary", status="ready", content_json={}) + db_session.add(other) + db_session.commit() + assert client.get(f"/api/artifacts/{other.id}", headers=auth).status_code == 404 diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 4e9124a484bcc8457443e861256bb03095264998..6a68164a096c73b32cb0afd2ba35479f4617417a 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,5 +1,5 @@ import { Route, Routes } from "react-router-dom"; -import AppShell, { ComingSoon } from "./components/AppShell"; +import AppShell from "./components/AppShell"; import { PublicFooter, PublicNav } from "./components/PublicChrome"; import PricingTiers from "./components/PricingTiers"; import SkyCanvas from "./components/SkyCanvas"; @@ -7,8 +7,19 @@ import AuthCallback from "./pages/AuthCallback"; import AuthPage from "./pages/AuthPage"; import { Blog, BlogPostPage } from "./pages/Blog"; import Landing from "./pages/Landing"; +import Archive from "./pages/Archive"; +import Launchpad from "./pages/Launchpad"; +import LaunchKit from "./pages/LaunchKit"; import MissionControl from "./pages/MissionControl"; +import Notebooks from "./pages/Notebooks"; import NotebookView from "./pages/NotebookView"; +import Resurface from "./pages/Resurface"; +import Settings from "./pages/Settings"; +import Sources from "./pages/Sources"; +import Studio from "./pages/Studio"; +import TopicMap from "./pages/TopicMap"; +import VoiceProfile from "./pages/VoiceProfile"; +import Welcome from "./pages/Welcome"; function PricingPage() { return ( @@ -34,19 +45,21 @@ export default function App() { } /> } /> } /> + } /> }> } /> - } /> + } /> } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> - } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> + } /> } /> diff --git a/frontend/src/api/auth.ts b/frontend/src/api/auth.ts index a55d92db6d8ed2b5c7c6a2bc9740354bffef7b2b..328a02e110693c3cc59d88f13a1d94c8c6a51296 100644 --- a/frontend/src/api/auth.ts +++ b/frontend/src/api/auth.ts @@ -35,4 +35,5 @@ export const authApi = { forgotComplete: (email: string, code: string, password: string) => post("/api/auth/password/forgot/complete", { email, code, password }), logout: () => post<{ ok: true }>("/api/auth/logout"), + deleteAccount: () => post<{ ok: true }>("/api/auth/delete-account"), }; diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index f144ccf115e3a18faba701cb490eb0ef2ff74c49..1d24607277e899fa5bdecf3c3460d71bc06ede16 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -88,11 +88,53 @@ export async function api(path: string, init: RequestInit = {}, retry = true) export const post = (path: string, body?: unknown) => api(path, { method: "POST", body: body === undefined ? undefined : JSON.stringify(body) }); -/** Direct browser upload to Cloudflare R2 through a presigned PUT. */ +export const patch = (path: string, body: unknown) => api(path, { method: "PATCH", body: JSON.stringify(body) }); + +export const put = (path: string, body: unknown) => api(path, { method: "PUT", body: JSON.stringify(body) }); + +export const del = (path: string) => api(path, { method: "DELETE" }); + +/** Query string from an object, skipping empty values. */ +export function qs(params: Record): string { + const q = new URLSearchParams(); + for (const [k, v] of Object.entries(params)) if (v !== undefined && v !== null && v !== "") q.set(k, String(v)); + const out = q.toString(); + return out ? `?${out}` : ""; +} + +/** Direct browser upload to storage (R2, or the API's local disk store) through a presigned PUT. */ +const TYPES_BY_EXT: Record = { + md: "text/markdown", + markdown: "text/markdown", + txt: "text/plain", + html: "text/html", + htm: "text/html", + pdf: "application/pdf", + mp3: "audio/mpeg", + m4a: "audio/mp4", + wav: "audio/wav", + webm: "audio/webm", + ogg: "audio/ogg", + png: "image/png", + jpg: "image/jpeg", + jpeg: "image/jpeg", + webp: "image/webp", + svg: "image/svg+xml", +}; + +/** Browsers leave File.type empty for some extensions (.md on Windows) and use vendor names for others. */ +function contentType(file: File): string { + const ext = file.name.split(".").pop()?.toLowerCase() ?? ""; + const base = file.type.split(";")[0]; + if (base === "audio/x-m4a") return "audio/mp4"; + if (base === "audio/x-wav") return "audio/wav"; + return base || TYPES_BY_EXT[ext] || "application/octet-stream"; +} + export async function uploadFile(file: File): Promise<{ upload_id: string; key: string }> { const start = await post<{ upload_id: string; upload_url: string; headers: Record }>( "/api/storage/uploads", - { filename: file.name, content_type: file.type, size_bytes: file.size }, + { filename: file.name, content_type: contentType(file), size_bytes: file.size }, ); const put = await fetch(start.upload_url, { method: "PUT", headers: start.headers, body: file }); if (!put.ok) throw new ApiError(put.status, "Upload to storage failed"); diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts new file mode 100644 index 0000000000000000000000000000000000000000..966397ab8fc2f4ee9fbb1efec8092ed72375c36f --- /dev/null +++ b/frontend/src/api/endpoints.ts @@ -0,0 +1,161 @@ +import { api, del, patch, post, put, qs, uploadFile } from "./client"; +import type { + Artifact, + ArtifactType, + CalendarItem, + ChatMessage, + ChatSummary, + Doc, + DocDetail, + Job, + Notebook, + NotebookSummary, + Page, + Platform, + ResurfaceItem, + Settings, + SocialAccounts, + Source, + TopicDetail, + TopicMap, + Usage, + Voice, + VoiceProfileData, + VoiceState, +} from "./types"; + +type SourceJob = { source: Source; job: Job }; + +export const sourcesApi = { + list: () => api("/api/sources"), + connect: (url: string) => post("/api/sources", { url }), + importUrl: (url: string) => post("/api/sources/url", { url }), + importFile: async (file: File) => { + const { upload_id } = await uploadFile(file); + return post("/api/sources/upload", { upload_id }); + }, + sync: (id: string) => post(`/api/sources/${id}/sync`), + remove: (id: string) => del<{ ok: true; removed_posts: number }>(`/api/sources/${id}`), +}; + +export const docsApi = { + list: (p: { q?: string; source_id?: string; limit?: number; offset?: number } = {}) => + api>(`/api/documents${qs(p)}`), + get: (id: string) => api(`/api/documents/${id}`), + remove: (id: string) => del(`/api/documents/${id}`), +}; + +export const notebooksApi = { + list: () => api("/api/notebooks"), + get: (id: string) => api(`/api/notebooks/${id}`), + create: (title: string, document_ids: string[] = [], description?: string) => + post<{ id: string; title: string; added: number }>("/api/notebooks", { title, document_ids, description }), + fromTopic: (topicId: string) => post<{ id: string; title: string }>(`/api/notebooks/from-topic/${topicId}`), + update: (id: string, body: { title?: string; description?: string }) => patch(`/api/notebooks/${id}`, body), + remove: (id: string) => del(`/api/notebooks/${id}`), + addDocs: (id: string, document_ids: string[]) => + post<{ added: number }>(`/api/notebooks/${id}/documents`, { document_ids }), + removeDoc: (id: string, docId: string) => del(`/api/notebooks/${id}/documents/${docId}`), + artifacts: (id: string) => api(`/api/notebooks/${id}/artifacts`), + chats: (id: string) => api(`/api/notebooks/${id}/chats`), + messages: (chatId: string) => api(`/api/notebooks/chats/${chatId}/messages`), + removeChat: (chatId: string) => del(`/api/notebooks/chats/${chatId}`), +}; + +export type GenerateBody = { + type: ArtifactType; + notebook_id?: string; + document_id?: string; + format?: "deep_dive" | "brief" | "debate"; + minutes?: number; + style?: "short" | "explainer" | "audiogram"; + audio_artifact_id?: string; + slides?: { heading: string; body: string }[]; + parent_id?: string; +}; + +export const artifactsApi = { + list: (p: { type?: string; notebook_id?: string; document_id?: string; status?: string; q?: string; + limit?: number; offset?: number } = {}) => api>(`/api/artifacts${qs(p)}`), + get: (id: string) => api(`/api/artifacts/${id}`), + generate: (body: GenerateBody) => post("/api/artifacts/generate", body), + update: (id: string, body: { title?: string; content?: Record }) => + patch(`/api/artifacts/${id}`, body), + remove: (id: string) => del(`/api/artifacts/${id}`), + retry: (id: string) => post(`/api/artifacts/${id}/retry`), +}; + +export const jobsApi = { + active: () => api("/api/jobs?active=1"), + recent: (kind?: string) => api(`/api/jobs${qs({ kind, limit: 10 })}`), + get: (id: string) => api(`/api/jobs/${id}`), +}; + +export const topicsApi = { + map: () => api("/api/topics"), + get: (id: string) => api(`/api/topics/${id}`), + rebuild: (full = false) => post(`/api/topics/rebuild${qs({ full })}`), +}; + +export const voiceApi = { + get: () => api("/api/voice"), + update: (body: { profile?: VoiceProfileData; host_voices?: { host_a: string; host_b: string } }) => + put("/api/voice", body), + build: (document_ids: string[]) => post("/api/voice/build", { document_ids }), + voices: () => api("/api/voice/voices"), + consent: async (file: File, consent_text: string) => { + const { upload_id } = await uploadFile(file); + return post("/api/voice/consent", { upload_id, agreed: true, consent_text }); + }, + revoke: () => del("/api/voice/consent"), +}; + +export type ItemBody = { + platform: Platform; + scheduled_at: string; + content: string; + thread?: string[]; + artifact_id?: string | null; + document_id?: string | null; + social_account_id?: string | null; + remind_by_email?: boolean; + draft?: boolean; +}; + +export const launchpadApi = { + items: (p: { start?: string; end?: string; status?: string } = {}) => api(`/api/calendar${qs(p)}`), + create: (body: ItemBody) => post("/api/calendar", body), + update: (id: string, body: Partial & { status?: "scheduled" | "draft" }) => + patch(`/api/calendar/${id}`, body), + remove: (id: string) => del(`/api/calendar/${id}`), + publishNow: (id: string) => post(`/api/calendar/${id}/publish`), + accounts: () => api("/api/social/accounts"), + connectBluesky: (handle: string, app_password: string) => + post("/api/social/bluesky", { handle, app_password }), + startOAuth: (platform: "x" | "linkedin") => post<{ url: string }>(`/api/social/${platform}/start`), + completeOAuth: (ticket: string) => post("/api/social/complete", { ticket }), + disconnect: (id: string) => del(`/api/social/accounts/${id}`), +}; + +export const resurfaceApi = { + get: () => api<{ evergreen: ResurfaceItem[]; on_this_day: ResurfaceItem[]; unscored: number; total_posts: number }>( + "/api/resurface", + ), + scan: () => post("/api/resurface/scan"), +}; + +export type SettingsPatch = { + name?: string; + workspace_name?: string; + brand_name?: string; + brand_accent?: string; + logo_upload_id?: string; + training_opt_in?: boolean; + email_unsubscribed?: boolean; +}; + +export const settingsApi = { + get: () => api("/api/settings"), + update: (body: SettingsPatch) => patch("/api/settings", body), + usage: () => api("/api/settings/usage"), +}; diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts new file mode 100644 index 0000000000000000000000000000000000000000..f24a9f191b5c716fbe2f86e738557895c1969c65 --- /dev/null +++ b/frontend/src/api/types.ts @@ -0,0 +1,230 @@ +export type JobStatus = "queued" | "running" | "done" | "failed"; + +export type Job = { + id: string; + kind: string; + params: Record; + status: JobStatus; + progress: number; + message: string | null; + error: string | null; + result: Record; + artifact_id: string | null; + attempts: number; + created_at: string | null; + finished_at: string | null; +}; + +export type Source = { + id: string; + feed_url: string; + site_url: string | null; + platform: string; + title: string | null; + sync_status: "pending" | "syncing" | "ok" | "error"; + sync_error: string | null; + last_synced_at: string | null; + document_count: number; + is_imports: boolean; +}; + +export type Doc = { + id: string; + title: string; + url: string; + path: string | null; + source_id: string | null; + source_title: string | null; + published_at: string | null; + words: number; + evergreen_score: number | null; +}; + +export type DocDetail = Doc & { lines: string[] }; + +export type Page = { total: number; items: T[] }; + +export type NotebookSummary = { + id: string; + title: string; + description: string | null; + summary: string | null; + document_count: number; + chat_count: number; + artifact_count: number; + updated_at: string | null; +}; + +export type Notebook = { + id: string; + title: string; + description: string | null; + summary: string | null; + documents: Doc[]; +}; + +export type Citation = { + marker: number; + document_id?: string; + title: string; + url: string; + path: string; + line_start: number; + line_end: number; + span: string; +}; + +export type ChatSummary = { id: string; title: string | null; updated_at: string | null }; +export type ChatMessage = { id: string; role: "user" | "assistant"; text: string; citations: Citation[] }; + +export type ArtifactType = "summary" | "audio_overview" | "video" | "quote_card" | "carousel" | "launch_kit"; + +export type SourceRef = { path: string; line_start: number; line_end: number; title?: string | null; quote?: string }; + +export type Segment = { speaker: string; text: string; start: number; end: number; sources: SourceRef[] }; + +export type LaunchKitContent = { + title: string; + post_title: string; + post_url: string; + claims: { claim: string; sources: SourceRef[] }[]; + hooks: { text: string; strength: number; rationale: string }[]; + posts: Record<"x_thread" | "linkedin" | "substack_notes" | "bluesky", string[]>; + seo: { + title_options?: string[]; + meta_description?: string; + slug?: string; + keywords?: string[]; + internal_links?: { path: string; anchor_text: string; reason: string }[]; + }; + carousel: { heading: string; body: string }[]; + quotes: { quote: string; source: SourceRef }[]; + quote_card_ids?: string[]; +}; + +export type Artifact = { + id: string; + type: ArtifactType; + type_label: string; + title: string; + status: "pending" | "generating" | "rendering" | "ready" | "failed" | "draft"; + notebook_id: string | null; + document_id: string | null; + content: Record; + storage_key: string | null; + url: string | null; + download_url: string | null; + slide_urls: string[]; + created_at: string | null; + job: Job | null; +}; + +export type TopicNode = { id: string; name: string; summary: string | null; post_count: number }; +export type TopicMap = { + nodes: TopicNode[]; + edges: { source: string; target: string; weight: number }[]; + has_untagged_posts: boolean; +}; +export type TopicDetail = TopicNode & { posts: (Doc & { weight: number })[] }; + +export type VoiceProfileData = { + tone: string[]; + sentence_length: "short" | "medium" | "long" | "varied"; + vocabulary: string[]; + structure_habits: string[]; + openings: string[]; + avoid: string[]; + summary: string; +}; + +export type VoiceState = { + profile: VoiceProfileData | null; + sample_doc_ids: string[]; + updated_at: string | null; + host_voices: { host_a: string; host_b: string }; + clone: { + allowed: boolean; + consent_text: string; + status: "none" | "processing" | "ready"; + voice_id: string | null; + created_at: string | null; + }; + tts_configured: boolean; +}; + +export type Voice = { voice_id: string; name: string; category?: string | null; preview_url?: string | null }; + +export type Platform = "x" | "linkedin" | "bluesky" | "substack_notes"; + +export type CalendarItem = { + id: string; + platform: Platform; + platform_label: string; + scheduled_at: string; + status: "scheduled" | "publishing" | "posted" | "reminded" | "failed" | "draft"; + content: string; + thread: string[]; + artifact_id: string | null; + document_id: string | null; + social_account_id: string | null; + account_handle: string | null; + remind_by_email: boolean; + auto_post: boolean; + external_url: string | null; + posted_at: string | null; + error: string | null; + metrics: Record; + clicks: number; +}; + +export type SocialAccount = { + id: string; + platform: "x" | "linkedin" | "bluesky"; + platform_label: string; + handle: string; + status: string; + avatar_url: string | null; + connected_at: string | null; +}; + +export type SocialAccounts = { + accounts: SocialAccount[]; + available: { x: boolean; linkedin: boolean; bluesky: boolean }; + redirect_uris: { x: string; linkedin: string }; +}; + +export type ResurfaceItem = Doc & { + reason: string | null; + angle: string | null; + last_resurfaced_at: string | null; + rank?: number; + years_ago?: number; +}; + +export type Plan = { + id: string; + name: string; + sources: number; + indexed_posts: number; + audio_minutes: number; + video_minutes: number; + launch_kits: number; + voice_cloning: boolean; + brand_kit: boolean; +}; + +export type Usage = { + plan: string; + used: { audio_minutes: number; video_minutes: number; launch_kits: number; llm_tokens: number; since: string }; + limits: { audio_minutes: number; video_minutes: number; launch_kits: number; sources: number; indexed_posts: number }; +}; + +export type Settings = { + user: { name: string | null; email: string; auth_provider: string; email_unsubscribed: boolean }; + workspace: { id: string; name: string; training_opt_in: boolean }; + brand: { name: string | null; accent: string; logo_url: string | null }; + plan: Plan; + billing_enabled: boolean; + usage: Usage; + integrations: { llm: boolean; elevenlabs: boolean; x: boolean; linkedin: boolean; storage: string; renderer: string }; +}; diff --git a/frontend/src/components/AppShell.tsx b/frontend/src/components/AppShell.tsx index df808abe1b9c9f7d537cab6c212d0b6d3d872253..e96a24e4493036b93d7587f2a4ee357ca4476e4c 100644 --- a/frontend/src/components/AppShell.tsx +++ b/frontend/src/components/AppShell.tsx @@ -1,3 +1,4 @@ +import { useState } from "react"; import { NavLink, Outlet, Navigate, useLocation } from "react-router-dom"; import { useAuth } from "../hooks/useAuth"; import Logo from "./Logo"; @@ -31,9 +32,29 @@ const NAV = [ { to: "/app/settings", label: "Settings", icon: OrbitIcon }, ]; +const COLLAPSED_KEY = "ns_sidebar_collapsed"; + +function readCollapsed(): boolean { + try { + return localStorage.getItem(COLLAPSED_KEY) === "1"; + } catch { + return false; + } +} + export default function AppShell() { const { user, loading, logout } = useAuth(); const location = useLocation(); + const [collapsed, setCollapsed] = useState(readCollapsed); + const toggle = () => + setCollapsed((c) => { + try { + localStorage.setItem(COLLAPSED_KEY, c ? "0" : "1"); + } catch { + /* private mode: remember for this visit only */ + } + return !c; + }); if (loading) return
; if (!user) { const next = new URLSearchParams({ next: location.pathname + location.search }); @@ -41,23 +62,39 @@ export default function AppShell() { } return ( -
+
); } - -export function ComingSoon({ title, phase }: { title: string; phase: string }) { - return ( -
- -

{title}

-

This part of the station comes online in {phase}.

- - Back to Mission Control - -
- ); -} diff --git a/frontend/src/components/ArtifactCard.tsx b/frontend/src/components/ArtifactCard.tsx new file mode 100644 index 0000000000000000000000000000000000000000..f3864c7f945e058f5980cfd779e5b6969cee7c45 --- /dev/null +++ b/frontend/src/components/ArtifactCard.tsx @@ -0,0 +1,233 @@ +import { useState, type ReactNode } from "react"; +import { Link } from "react-router-dom"; +import { artifactsApi } from "../api/endpoints"; +import type { Artifact, Citation, Segment } from "../api/types"; +import { useJob } from "../hooks/useJob"; +import { ConfirmButton, errorMessage, formatDate, formatDuration, JobProgress, StatusPill } from "./ui"; + +/** Answer text with [n] markers turned into buttons that open the cited passage. */ +export function CitedText({ text, citations, onCite }: { text: string; citations: Citation[]; onCite?: (c: Citation) => void }) { + const byMarker = new Map(citations.map((c) => [c.marker, c])); + const parts = text.split(/(\[\d+\])/g); + return ( + <> + {parts.map((part, i) => { + const m = part.match(/^\[(\d+)\]$/); + const cite = m ? byMarker.get(Number(m[1])) : undefined; + if (!cite) return {part}; + return ( + + ); + })} + + ); +} + +export function CitationList({ citations, onCite }: { citations: Citation[]; onCite?: (c: Citation) => void }) { + if (!citations.length) return null; + return ( +
    + {citations.map((c) => ( +
  1. + {" "} + + lines {c.line_start} + {c.line_end > c.line_start ? ` to ${c.line_end}` : ""} + +

    {c.span.length > 220 ? `${c.span.slice(0, 220)}...` : c.span}

    +
    2. + ))} +
+ ); +} + +function Transcript({ segments, onSeek }: { segments: Segment[]; onSeek: (t: number) => void }) { + return ( +
    + {segments.map((s, i) => ( +
  1. + + {s.speaker === "host_b" ? "B" : "A"} + {s.text} + {s.sources.length > 0 && [{s.sources.map((r) => r.title ?? r.path).join("; ")}]} +
    2. + ))} +
+ ); +} + +function Body({ artifact, onCite }: { artifact: Artifact; onCite?: (c: Citation) => void }) { + const [showTranscript, setShowTranscript] = useState(false); + const [audioEl, setAudioEl] = useState(null); + const c = artifact.content; + if (artifact.status !== "ready") return null; + switch (artifact.type) { + case "summary": + return ( +
+ {(c.themes as string[] | undefined)?.length ? ( +
+ {(c.themes as string[]).map((t) => ( + + {t} + + ))} +
+ ) : null} + {String(c.summary ?? "") + .split(/\n\n+/) + .map((p, i) => ( +

+ +

+ ))} + +
+ ); + case "audio_overview": + return ( +
+ {artifact.url &&
+ ); + case "video": + return artifact.url ? ( +
+
+ ) : null; + case "quote_card": + return artifact.url ? ( +
+ {c.quote +
+ ) : null; + case "carousel": + return ( +
+ {artifact.slide_urls.map((u, i) => ( + + {`Slide + + ))} +
+ ); + case "launch_kit": + return ( +
+

+ {c.claims?.length ?? 0} claims, {c.hooks?.length ?? 0} hooks, posts for 4 platforms, SEO pack and carousel. +

+ + Open Launch Kit + +
+ ); + default: + return null; + } +} + +/** Any generated artifact: live progress while it runs, then the right player or preview. */ +export function ArtifactCard({ + artifact: initial, + onCite, + onRemoved, + onChanged, + actions, +}: { + artifact: Artifact; + onCite?: (c: Citation) => void; + onRemoved?: (id: string) => void; + onChanged?: (a: Artifact) => void; + actions?: (a: Artifact) => ReactNode; +}) { + const [artifact, setArtifact] = useState(initial); + const [error, setError] = useState(null); + const job = useJob(artifact.job, () => { + artifactsApi.get(artifact.id).then((a) => { + setArtifact(a); + onChanged?.(a); + }); + }); + const running = job && (job.status === "queued" || job.status === "running"); + const failedMessage = artifact.status === "failed" ? (artifact.content.error as string | undefined) ?? job?.error : null; + + return ( +
+
+
+

{artifact.type_label}

+

{artifact.title}

+

+ {formatDate(artifact.created_at, true)} + {artifact.content.duration_s ? ` ยท ${formatDuration(artifact.content.duration_s)}` : ""} +

+
+ +
+ {running && job && } + {failedMessage &&

{failedMessage}

} + + {error &&

{error}

} +
+ {actions?.(artifact)} + {artifact.download_url && ( + + Download + + )} + {artifact.status === "failed" && ( + + )} + {onRemoved && ( + { + await artifactsApi.remove(artifact.id); + onRemoved(artifact.id); + }} + > + Delete + + )} +
+
+ ); +} diff --git a/frontend/src/components/DocPicker.tsx b/frontend/src/components/DocPicker.tsx new file mode 100644 index 0000000000000000000000000000000000000000..90aac79c82f5e69d450bafab35b8267c21ab0788 --- /dev/null +++ b/frontend/src/components/DocPicker.tsx @@ -0,0 +1,81 @@ +import { useEffect, useMemo, useState } from "react"; +import { docsApi } from "../api/endpoints"; +import type { Doc } from "../api/types"; +import { formatDate, Loading } from "./ui"; + +/** Searchable checklist of posts. `single` turns it into a radio list. */ +export function DocPicker({ + selected, + onChange, + single = false, + exclude = [], + max, +}: { + selected: string[]; + onChange: (ids: string[]) => void; + single?: boolean; + exclude?: string[]; + max?: number; +}) { + const [docs, setDocs] = useState(null); + const [q, setQ] = useState(""); + + useEffect(() => { + docsApi.list({ limit: 5000 }).then((p) => setDocs(p.items)); + }, []); + + const visible = useMemo(() => { + const skip = new Set(exclude); + const needle = q.trim().toLowerCase(); + return (docs ?? []).filter( + (d) => !skip.has(d.id) && (!needle || d.title.toLowerCase().includes(needle) || (d.source_title ?? "").toLowerCase().includes(needle)), + ); + }, [docs, q, exclude]); + + if (!docs) return ; + if (docs.length === 0) return

No posts yet. Connect a source first.

; + + const toggle = (id: string) => { + if (single) return onChange([id]); + if (selected.includes(id)) return onChange(selected.filter((x) => x !== id)); + if (max && selected.length >= max) return; + onChange([...selected, id]); + }; + + return ( +
+
+ setQ(e.target.value)} aria-label="Search posts" /> + {!single && ( +
+ {selected.length} selected{max ? ` of ${max}` : ""} + {!max && ( + + )} + {selected.length > 0 && ( + + )} +
+ )} +
+
    + {visible.map((d) => ( +
  • + +
    • + ))} + {visible.length === 0 &&
  • No posts match.
    • } +
+
+ ); +} diff --git a/frontend/src/components/Reader.tsx b/frontend/src/components/Reader.tsx new file mode 100644 index 0000000000000000000000000000000000000000..a1ab985ba532f7b5ca749320ab2e4edf2b686fba --- /dev/null +++ b/frontend/src/components/Reader.tsx @@ -0,0 +1,76 @@ +import { useEffect, useRef, useState } from "react"; +import { docsApi } from "../api/endpoints"; +import type { DocDetail } from "../api/types"; +import { Drawer, formatDate, Loading } from "./ui"; + +/** Reads a post from the corpus with the exact line numbers citations point at. */ +export function Reader({ + documentId, + highlight, + onClose, +}: { + documentId: string; + highlight?: { start: number; end: number }; + onClose: () => void; +}) { + const [doc, setDoc] = useState(null); + const [error, setError] = useState(null); + const first = useRef(null); + + useEffect(() => { + setDoc(null); + docsApi.get(documentId).then(setDoc, () => setError("Could not open this post.")); + }, [documentId]); + + useEffect(() => { + first.current?.scrollIntoView({ block: "center" }); + }, [doc, highlight]); + + // Skip the front matter block for display, but keep real line numbers. + let bodyStart = 0; + if (doc?.lines[0] === "---") { + const close = doc.lines.indexOf("---", 1); + bodyStart = close > 0 ? close + 1 : 0; + } + + return ( + + {!doc && !error && } + {error &&

{error}

} + {doc && ( +
+

{doc.source_title ?? "Post"}

+

+ {formatDate(doc.published_at)} ยท {doc.words} words + {doc.url.startsWith("http") && ( + <> + {" ยท "} + + Open original + + + )} +

+
+ {doc.lines.slice(bodyStart).map((line, i) => { + const n = i + bodyStart + 1; + const hit = highlight && n >= highlight.start && n <= highlight.end; + if (!line.trim() && !hit) return
; + const heading = line.match(/^(#{1,3})\s+(.*)/); + return ( +
+ {n} + {heading ? ( + {heading[2]} + ) : ( + {line} + )} +
+ ); + })} +
+
+ )} + + ); +} diff --git a/frontend/src/components/ScheduleModal.tsx b/frontend/src/components/ScheduleModal.tsx new file mode 100644 index 0000000000000000000000000000000000000000..feacb6d03be113f845e656355a8efc10ffb01ee5 --- /dev/null +++ b/frontend/src/components/ScheduleModal.tsx @@ -0,0 +1,166 @@ +import { useEffect, useState, type FormEvent } from "react"; +import { launchpadApi } from "../api/endpoints"; +import type { CalendarItem, Platform, SocialAccount } from "../api/types"; +import { errorMessage, Modal } from "./ui"; + +export const PLATFORMS: { id: Platform; label: string; limit: number; auto: boolean }[] = [ + { id: "x", label: "X", limit: 280, auto: true }, + { id: "linkedin", label: "LinkedIn", limit: 3000, auto: true }, + { id: "bluesky", label: "Bluesky", limit: 300, auto: true }, + { id: "substack_notes", label: "Substack Notes", limit: 2000, auto: false }, +]; + +export function toLocalInput(date: Date): string { + const pad = (n: number) => String(n).padStart(2, "0"); + return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}T${pad(date.getHours())}:${pad(date.getMinutes())}`; +} + +export function nextSlot(): Date { + const d = new Date(Date.now() + 60 * 60 * 1000); + d.setMinutes(0, 0, 0); + return d; +} + +/** Schedules one post (or a thread) on the Launchpad. Posts auto publish when an account is connected. */ +export function ScheduleModal({ + platform: initialPlatform, + posts, + artifactId, + documentId, + onClose, + onScheduled, +}: { + platform: Platform; + posts: string[]; + artifactId?: string | null; + documentId?: string | null; + onClose: () => void; + onScheduled?: (item: CalendarItem) => void; +}) { + const [platform, setPlatform] = useState(initialPlatform); + const [texts, setTexts] = useState(posts.length ? posts : [""]); + const [when, setWhen] = useState(toLocalInput(nextSlot())); + const [accounts, setAccounts] = useState([]); + const [accountId, setAccountId] = useState(""); + const [remind, setRemind] = useState(false); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const meta = PLATFORMS.find((p) => p.id === platform)!; + const threadable = platform === "x" || platform === "bluesky"; + + useEffect(() => { + launchpadApi.accounts().then((a) => setAccounts(a.accounts), () => setAccounts([])); + }, []); + + const options = accounts.filter((a) => a.platform === platform && a.status === "active"); + useEffect(() => { + setAccountId(options[0]?.id ?? ""); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [platform, accounts.length]); + + const submit = async (e: FormEvent) => { + e.preventDefault(); + setBusy(true); + setError(null); + const clean = texts.map((t) => t.trim()).filter(Boolean); + const merged = threadable ? clean : [clean.join("\n\n")]; + try { + const item = await launchpadApi.create({ + platform, + scheduled_at: new Date(when).toISOString(), + content: merged[0] ?? "", + thread: merged.slice(1), + artifact_id: artifactId ?? null, + document_id: documentId ?? null, + social_account_id: meta.auto && accountId ? accountId : null, + remind_by_email: remind || !meta.auto || !accountId, + }); + onScheduled?.(item); + onClose(); + } catch (err) { + setError(errorMessage(err)); + } finally { + setBusy(false); + } + }; + + return ( + +
+
+ + + {meta.auto && ( + + )} +
+ {!meta.auto &&

Substack has no posting API, so we email you the note at send time, ready to paste.

} + {meta.auto && !options.length && ( +

+ No {meta.label} account connected. Connect one on the Launchpad to auto post; until then you get an email reminder. +

+ )} + {texts.map((t, i) => ( +