${esc(post.heroEyebrow)}
${esc(post.title)}
${esc(post.heroDescription)}
${mission}${sections}${faqHtml(faq)}diff --git a/backend/alembic/versions/0004_archive_notebook.py b/backend/alembic/versions/0004_archive_notebook.py new file mode 100644 index 0000000000000000000000000000000000000000..bfe13009872aea2e01fa2ece1067746f70800e9d --- /dev/null +++ b/backend/alembic/versions/0004_archive_notebook.py @@ -0,0 +1,24 @@ +"""notebooks.is_archive: the workspace's always-present "All posts" notebook + +Revision ID: 0004_archive_notebook +Revises: 0003_billing_events +Create Date: 2026-09-28 +""" +import sqlalchemy as sa +from alembic import op + +revision = "0004_archive_notebook" +down_revision = "0003_billing_events" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # 0001 builds from the live models, so a fresh database may already have the column. + columns = {c["name"] for c in sa.inspect(op.get_bind()).get_columns("notebooks")} + if "is_archive" not in columns: + op.add_column("notebooks", sa.Column("is_archive", sa.Boolean(), nullable=False, server_default=sa.false())) + + +def downgrade() -> None: + op.drop_column("notebooks", "is_archive") diff --git a/backend/app/models/content.py b/backend/app/models/content.py index 80953435fad301fffb96552a70e3da568f4b906f..dff2ab28d94b6d96f4de0321975a569fa0297466 100644 --- a/backend/app/models/content.py +++ b/backend/app/models/content.py @@ -65,6 +65,8 @@ class Notebook(IdMixin, TimestampMixin, Base): title: Mapped[str] = mapped_column(String(300)) description: Mapped[str | None] = mapped_column(Text) summary: Mapped[str | None] = mapped_column(Text) + # The workspace's "All posts" notebook: created on demand, kept in step with every indexed post. + is_archive: Mapped[bool] = mapped_column(Boolean, default=False) class NotebookDocument(Base): diff --git a/backend/app/pipeline/ingest.py b/backend/app/pipeline/ingest.py index 563f512dfbfc70d77c160cdffd3278c6a81bfd3b..d38f3aea14616f0b23c6dc8462234002058b16df 100644 --- a/backend/app/pipeline/ingest.py +++ b/backend/app/pipeline/ingest.py @@ -418,8 +418,15 @@ def _plain_links(url: str) -> list[str]: return links -def crawl_site(url: str, limit: int, on_found=None, on_post=None) -> list[FeedEntry]: - """Find the posts on a site with no feed and read each one.""" +def _title_from_url(u: str) -> str: + slug = urlparse(u).path.rstrip("/").rsplit("/", 1)[-1] + slug = re.sub(r"\.[a-z0-9]+$", "", slug) + return slug.replace("-", " ").replace("_", " ").strip().capitalize() or urlparse(u).netloc + + +def crawl_site(url: str, limit: int, on_found=None, on_post=None, read_limit: int | None = None) -> list[FeedEntry]: + """Find the posts on a site with no feed and read each one. Past read_limit, pages are only listed (title from the + URL, no html): they show the writer what the rest of their archive holds without paying to read it.""" if firecrawl.enabled(): links = firecrawl.map_site(url, max(limit * 4, 100)) else: @@ -437,20 +444,24 @@ def crawl_site(url: str, limit: int, on_found=None, on_post=None) -> list[FeedEn except (FeedNotFound, httpx.HTTPError, firecrawl.FirecrawlError): return None + to_read = urls if read_limit is None else urls[:read_limit] entries: list[FeedEntry] = [] with ThreadPoolExecutor(max_workers=4) as pool: - for i, entry in enumerate(pool.map(read, urls)): + for i, entry in enumerate(pool.map(read, to_read)): if entry: entries.append(entry) if on_post: - on_post(i + 1, len(urls)) + on_post(i + 1, len(to_read)) + entries += [FeedEntry(title=_title_from_url(u), url=u, published_at=None, html="") for u in urls[len(to_read):]] if not entries: raise FeedNotFound(f"We found {len(urls)} pages on {urlparse(url).netloc} but none had article text.") return entries -def fetch_substack_archive(site: str, known_urls: set[str], limit: int, on_page=None) -> list[FeedEntry]: - """Posts beyond the ~20 the RSS feed carries, through Substack's public archive API.""" +def fetch_substack_archive(site: str, known_urls: set[str], limit: int, on_page=None, + body_budget: int | None = None) -> list[FeedEntry]: + """Posts beyond the ~20 the RSS feed carries, through Substack's public archive API. Only the first body_budget + posts get their body fetched; the rest are listed from the archive metadata alone (title, date, link).""" out: list[FeedEntry] = [] offset = 0 with httpx.Client(follow_redirects=True, timeout=30, headers=UA) as client: @@ -468,6 +479,10 @@ def fetch_substack_archive(site: str, known_urls: set[str], limit: int, on_page= continue if len(out) + len(known_urls) >= limit: break + if body_budget is not None and len(out) >= body_budget: + out.append(FeedEntry(title=item.get("title") or "Untitled", url=url, + published_at=_parse_date(item.get("post_date")), html="")) + continue post = client.get(f"{site}/api/v1/posts/{item['slug']}") if post.status_code >= 400: continue @@ -556,7 +571,7 @@ def store_entries( ) doc.clean_text = "\n\n".join(f"## {h}\n{t}" if h else t for h, t in sections) doc.metadata_json = { - **(doc.metadata_json or {}), + **{k: v for k, v in (doc.metadata_json or {}).items() if k != "locked"}, "headings": [h for h, _ in sections if h], "words": len(doc.clean_text.split()), } @@ -575,44 +590,95 @@ def store_entries( return indexed, skipped, changed +# A sync discovers up to SCAN_CAP posts (or the plan's own limit, if higher) but only indexes the plan's +# indexed_posts. The rest are listed, not indexed: title, date and link only, no text and no corpus file. They exist +# to show the writer how much of their archive is waiting behind an upgrade. Upgrading re-syncs and indexes them. +SCAN_CAP = 200 + + +def _newest_first(entries: list[FeedEntry]) -> list[FeedEntry]: + """Dated posts newest first; undated ones (site crawls) keep their discovery order after them.""" + dated = sorted((e for e in entries if e.published_at), key=lambda e: e.published_at.timestamp(), reverse=True) + return dated + [e for e in entries if not e.published_at] + + +def store_locked(db: Session, source: Source, entries: list[FeedEntry]) -> tuple[int, set[str]]: + """List posts beyond the plan limit without indexing them: title, date and link only. Returns (count, corpus paths + to remove): a post that was indexed before (a newer post pushed it out, or the plan went down) loses its file.""" + locked = 0 + remove: set[str] = set() + for entry in entries: + doc = db.scalar(select(Document).where(Document.source_id == source.id, Document.url == entry.url)) + if not doc: + doc = Document(id=uuid.uuid4(), workspace_id=source.workspace_id, source_id=source.id, url=entry.url) + db.add(doc) + if doc.path: + remove.add(doc.path) + doc.path = None + doc.title = entry.title + doc.published_at = entry.published_at + doc.content_hash = None # indexed from scratch once the plan allows it + doc.clean_text = "" + doc.metadata_json = {"locked": True} + locked += 1 + db.commit() + return locked, remove + + def ingest_source(db: Session, source: Source, job: Job, max_posts: int) -> dict: source.sync_status = "syncing" update_job(db, job, status="running", progress=0.02, message="Contacting your feed") + fetch_limit = max(SCAN_CAP, max_posts) if source.feed_url.startswith(SITE_PREFIX): update_job(db, job, message="Mapping your site") entries = crawl_site( - source.feed_url.removeprefix(SITE_PREFIX), max_posts, + source.feed_url.removeprefix(SITE_PREFIX), fetch_limit, on_found=lambda n: update_job(db, job, progress=0.05, message=f"Found {n} pages, reading them"), on_post=lambda done, total: update_job(db, job, progress=0.05 + 0.05 * done / total, message=f"Read {done} of {total} pages"), + read_limit=max_posts, ) title = None else: - title, entries = fetch_feed(source.feed_url, max_posts) + title, entries = fetch_feed(source.feed_url, fetch_limit) source.title = source.title or title - if source.platform == "substack" and len(entries) < max_posts: + if source.platform == "substack" and len(entries) < fetch_limit: site = f"{urlparse(source.feed_url).scheme}://{urlparse(source.feed_url).netloc}" update_job(db, job, progress=0.05, message="Reading your Substack archive") try: entries += fetch_substack_archive( - site, {e.url for e in entries}, max_posts, + site, {e.url for e in entries}, fetch_limit, on_page=lambda n: update_job(db, job, message=f"Found {len(entries) + n} posts in the archive"), + body_budget=max(0, max_posts - len(entries)), ) except (httpx.HTTPError, ValueError): log.warning("substack archive fetch failed for %s", site, exc_info=True) + entries = _newest_first(entries) + available, beyond = entries[:max_posts], entries[max_posts:] + def progress(done: int, total: int, post_title: str) -> None: - update_job(db, job, progress=0.1 + 0.8 * done / total, message=f"{post_title[:80]} is in orbit") + update_job(db, job, progress=0.1 + 0.7 * done / total, message=f"{post_title[:80]} is in orbit") + + update_job(db, job, progress=0.1, message=f"Indexing {len(available)} posts") + indexed, skipped, changed = store_entries(db, source, available, progress) - update_job(db, job, progress=0.1, message=f"Indexing {len(entries)} posts") - indexed, skipped, changed = store_entries(db, source, entries, progress) + locked = 0 + if beyond: + update_job(db, job, progress=0.85, + message=f"Found {len(beyond)} more posts. Your plan indexes the latest {max_posts}") + locked, remove = store_locked(db, source, beyond) + if remove: + Corpus(source.workspace_id).write_files({INDEX: rebuild_index(db, source.workspace_id)}, remove=remove) source.sync_status = "ok" source.sync_error = None source.last_synced_at = datetime.now(UTC) db.commit() - return {"indexed": indexed, "skipped": skipped, "found": len(entries), "changed": [str(i) for i in changed]} + return {"indexed": indexed, "skipped": skipped, "found": len(entries), "limit": max_posts, + "capped": locked > 0, "locked": locked, "available": len(available), + "changed": [str(i) for i in changed]} def entry_from_upload(filename: str, content_type: str, data: bytes) -> FeedEntry: diff --git a/backend/app/routers/artifacts.py b/backend/app/routers/artifacts.py index f32d1befb8456fb31d18bbd9b820ef5a5a4896ee..58ad74cc9448b6013802d86ed86795bc967c5210 100644 --- a/backend/app/routers/artifacts.py +++ b/backend/app/routers/artifacts.py @@ -7,10 +7,12 @@ from sqlalchemy import func, or_, select from app.auth import Ctx, get_ctx from app.models import Artifact, CalendarItem, Document, Job, Notebook +from app.routers.notebooks import ensure_archive_notebook from app.services.artifacts import latest_jobs, serialize_artifact, start_artifact from app.services.jobs import create_job, serialize_job from app.services.renderer import COMPOSITIONS from app.services.storage import storage +from app.services.plans import effective_plan, plan_limit_error from app.services.usage import check_limit router = APIRouter(prefix="/api/artifacts", tags=["artifacts"]) @@ -27,6 +29,7 @@ class GenerateIn(BaseModel): type: Literal["summary", "audio_overview", "video", "quote_card", "carousel", "launch_kit"] notebook_id: uuid.UUID | None = None document_id: uuid.UUID | None = None + archive: bool = False # no notebook or post picked: use the "All posts" notebook # audio_overview format: Literal["deep_dive", "brief", "debate"] = "deep_dive" minutes: int = Field(6, ge=1, le=30) @@ -56,6 +59,11 @@ def _target_title(ctx: Ctx, notebook_id: uuid.UUID | None, document_id: uuid.UUI Document.workspace_id == ctx.workspace.id)) if not doc: raise HTTPException(404, "Post not found") + if not doc.path: + plan = effective_plan(ctx.db, ctx.workspace) + raise plan_limit_error(plan, "indexed_posts", + f"This post is not indexed. Your plan indexes your latest {plan.indexed_posts} " + "posts; upgrade to use your whole archive.") return doc.title if notebook_id: nb = ctx.db.scalar(select(Notebook).where(Notebook.id == notebook_id, @@ -69,6 +77,8 @@ def _target_title(ctx: Ctx, notebook_id: uuid.UUID | None, document_id: uuid.UUI @router.post("/generate") def generate(body: GenerateIn, ctx: Ctx = Depends(get_ctx)): """One entry point for every generated artifact.""" + if body.archive and not body.notebook_id and not body.document_id: + body.notebook_id = ensure_archive_notebook(ctx).id params: dict = {} content: dict = {} if body.type == "video" and body.style == "audiogram": diff --git a/backend/app/routers/notebooks.py b/backend/app/routers/notebooks.py index bc7f41e782f2e36b423a0181f814dbc2d3aaaed3..977f65c06574fd144a595179676abbb48a3eeefc 100644 --- a/backend/app/routers/notebooks.py +++ b/backend/app/routers/notebooks.py @@ -52,7 +52,9 @@ def _get(ctx: Ctx, notebook_id: uuid.UUID) -> Notebook: def _add_docs(ctx: Ctx, nb: Notebook, ids: list[uuid.UUID]) -> int: valid = set(ctx.db.scalars( - select(Document.id).where(Document.id.in_(ids), Document.workspace_id == ctx.workspace.id) + # Locked posts (beyond the plan's post limit) have no corpus file and cannot join a notebook. + select(Document.id).where(Document.id.in_(ids), Document.workspace_id == ctx.workspace.id, + Document.path.is_not(None)) ).all()) existing = set(ctx.db.scalars(select(NotebookDocument.document_id).where(NotebookDocument.notebook_id == nb.id))) for doc_id in valid - existing: @@ -70,6 +72,29 @@ def create_notebook(body: NotebookIn, ctx: Ctx = Depends(get_ctx)): return {"id": str(nb.id), "title": nb.title, "added": added} +ARCHIVE_TITLE = "All posts" + + +def ensure_archive_notebook(ctx: Ctx) -> Notebook: + """The workspace's "All posts" notebook, topped up with every indexed post (locked posts are skipped).""" + nb = ctx.db.scalar(select(Notebook).where(Notebook.workspace_id == ctx.workspace.id, Notebook.is_archive.is_(True))) + if not nb: + nb = Notebook(workspace_id=ctx.workspace.id, title=ARCHIVE_TITLE, is_archive=True, + description="Every indexed post in your archive, kept up to date.") + ctx.db.add(nb) + ctx.db.flush() + ids = list(ctx.db.scalars(select(Document.id).where(Document.workspace_id == ctx.workspace.id))) + _add_docs(ctx, nb, ids) + ctx.db.commit() + return nb + + +@router.post("/archive") +def archive_notebook(ctx: Ctx = Depends(get_ctx)): + nb = ensure_archive_notebook(ctx) + return {"id": str(nb.id), "title": nb.title} + + @router.post("/from-topic/{topic_id}") def notebook_from_topic(topic_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): topic = ctx.db.scalar(select(Topic).where(Topic.id == topic_id, Topic.workspace_id == ctx.workspace.id)) @@ -87,7 +112,8 @@ def notebook_from_topic(topic_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): @router.get("") def list_notebooks(ctx: Ctx = Depends(get_ctx)): rows = ctx.db.scalars( - select(Notebook).where(Notebook.workspace_id == ctx.workspace.id).order_by(Notebook.updated_at.desc()) + select(Notebook).where(Notebook.workspace_id == ctx.workspace.id) + .order_by(Notebook.is_archive.desc(), Notebook.updated_at.desc()) ).all() ids = [n.id for n in rows] doc_counts = dict(ctx.db.execute( @@ -103,7 +129,7 @@ def list_notebooks(ctx: Ctx = Depends(get_ctx)): return [ {"id": str(n.id), "title": n.title, "description": n.description, "summary": n.summary, "document_count": doc_counts.get(n.id, 0), "chat_count": chat_counts.get(n.id, 0), - "artifact_count": artifact_counts.get(n.id, 0), + "artifact_count": artifact_counts.get(n.id, 0), "is_archive": n.is_archive, "updated_at": n.updated_at.isoformat() if n.updated_at else None} for n in rows ] @@ -112,6 +138,8 @@ def list_notebooks(ctx: Ctx = Depends(get_ctx)): @router.get("/{notebook_id}") def get_notebook(notebook_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): nb = _get(ctx, notebook_id) + if nb.is_archive: # posts synced since it was made join on open + ensure_archive_notebook(ctx) docs = ctx.db.scalars( select(Document) .join(NotebookDocument, NotebookDocument.document_id == Document.id) @@ -123,6 +151,7 @@ def get_notebook(notebook_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): "title": nb.title, "description": nb.description, "summary": nb.summary, + "is_archive": nb.is_archive, "documents": [serialize_document(d) for d in docs], } diff --git a/backend/app/routers/sources.py b/backend/app/routers/sources.py index 702db44e4224d6633abb9b3add4648d76a24bd79..575228765aa571f10d1280ac1d0b9889e7051112 100644 --- a/backend/app/routers/sources.py +++ b/backend/app/routers/sources.py @@ -24,7 +24,7 @@ class UploadImportIn(BaseModel): upload_id: uuid.UUID -def serialize_source(source: Source, doc_count: int = 0) -> dict: +def serialize_source(source: Source, doc_count: int = 0, locked_count: int = 0) -> dict: return { "id": str(source.id), "feed_url": source.feed_url, @@ -35,6 +35,8 @@ def serialize_source(source: Source, doc_count: int = 0) -> dict: "sync_error": source.sync_error, "last_synced_at": source.last_synced_at.isoformat() if source.last_synced_at else None, "document_count": doc_count, + # Found but beyond the plan's post limit: listed, not indexed until the workspace upgrades. + "locked_count": locked_count, "is_imports": source.feed_url == IMPORTS_FEED, } @@ -91,7 +93,12 @@ def list_sources(ctx: Ctx = Depends(get_ctx)): .group_by(Source.id) .order_by(Source.created_at) ).all() - return [serialize_source(s, n) for s, n in rows] + locked = dict(ctx.db.execute( + select(Document.source_id, func.count(Document.id)) + .where(Document.workspace_id == ctx.workspace.id, Document.path.is_(None)) + .group_by(Document.source_id) + ).all()) + return [serialize_source(s, n, locked.get(s.id, 0)) for s, n in rows] @router.post("/{source_id}/sync") @@ -151,6 +158,11 @@ def import_upload(body: UploadImportIn, ctx: Ctx = Depends(get_ctx)): documents_router = APIRouter(prefix="/api/documents", tags=["documents"]) +def is_locked(d: Document) -> bool: + """Past the plan's post limit (see ingest.store_locked): listed with title and link only, not indexed.""" + return not d.path and bool((d.metadata_json or {}).get("locked")) + + def serialize_document(d: Document, source: Source | None = None) -> dict: meta = d.metadata_json or {} return { @@ -163,6 +175,7 @@ def serialize_document(d: Document, source: Source | None = None) -> dict: "published_at": d.published_at.isoformat() if d.published_at else None, "words": meta.get("words") or len((d.clean_text or "").split()), "evergreen_score": d.evergreen_score, + "locked": is_locked(d), } @@ -198,6 +211,11 @@ def get_document(document_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)): if not doc: raise HTTPException(404, "Post not found") source = ctx.db.get(Source, doc.source_id) if doc.source_id else None + if is_locked(doc): + # Listed, not indexed: there is no text to show until the workspace upgrades. + plan = effective_plan(ctx.db, ctx.workspace) + return {**serialize_document(doc, source), "lines": [f"# {doc.title}"], + "preview": True, "limit": plan.indexed_posts} lines: list[str] = [] if doc.path: corpus = Corpus(ctx.workspace.id) diff --git a/backend/app/routers/voice.py b/backend/app/routers/voice.py index 6d8da11b48017c0c55b2b0f8209fdcaf9823b578..4778b5907df74daea4436e7b5ac7d88dd2bf6f54 100644 --- a/backend/app/routers/voice.py +++ b/backend/app/routers/voice.py @@ -57,6 +57,23 @@ class PreviewIn(BaseModel): delivery: DeliveryIn | None = None +class DesignIn(BaseModel): + """Either picked options (Build from options) or a free description (Describe in your own words).""" + prompt: str | None = Field(None, max_length=1000) + gender: str = Field("", max_length=20) + age: str = Field("", max_length=20) + persona: str = Field("", max_length=30) + pace: str = Field("", max_length=20) + accent: str = Field("", max_length=40) + + +class DesignSaveIn(BaseModel): + generated_voice_id: str = Field(min_length=5, max_length=100) + name: str = Field(min_length=1, max_length=60) + description: str = Field("", max_length=1000) + use_as: str | None = None # "host_a" | "host_b" + + class LibraryAddIn(BaseModel): public_owner_id: str = Field(min_length=5, max_length=100) voice_id: str = Field(min_length=5, max_length=60) @@ -116,6 +133,8 @@ def _serialize(ctx: Ctx) -> dict: "error": job.error if job and job.status == "failed" else None, "preview_url": storage.presign_get(preview_key) if preview_key else None, }, + # Voices this workspace generated with Voice Design (the ElevenLabs account is shared across workspaces). + "custom_voices": voices.get("custom") or [], "tts_configured": bool(settings.elevenlabs_api_key), "model": tts.model_id(), "models": tts.MODELS, @@ -146,6 +165,11 @@ def update_voice(body: ProfileIn, ctx: Ctx = Depends(get_ctx)): @router.post("/build") def build(body: BuildIn, ctx: Ctx = Depends(get_ctx)): + if not body.document_ids: # the automatic build after a sync may already be running; follow that one + pending = ctx.db.scalar(select(Job).where(Job.workspace_id == ctx.workspace.id, Job.kind == "voice_profile", + Job.status.in_(("queued", "running"))).limit(1)) + if pending: + return serialize_job(pending) job = create_job(ctx.db, ctx.workspace.id, "voice_profile", {"document_ids": [str(i) for i in body.document_ids]}, max_attempts=2) return serialize_job(job) @@ -211,6 +235,61 @@ async def preview(body: PreviewIn, ctx: Ctx = Depends(get_ctx)): return {"url": storage.presign_get(key), "seconds": round(tts.mp3_seconds(clip), 1)} +def _design_description(body: DesignIn) -> str: + if body.prompt and body.prompt.strip(): + text = body.prompt.strip() + if len(text) < 20: + raise HTTPException(400, "Describe the voice in at least 20 characters.") + return text + parts = [f"A {body.age} {body.gender} voice".replace(" ", " ") if body.gender or body.age else "A voice"] + if body.accent: + parts.append(f"with a {body.accent} accent") + sentence = " ".join(parts) + "." + if body.persona: + sentence += f" Sounds {body.persona}." + if body.pace: + sentence += f" Speaks at a {body.pace} pace." + return sentence + " Clear studio quality, suited to podcasts and narration." + + +@router.post("/design") +async def design(body: DesignIn, ctx: Ctx = Depends(get_ctx)): + """Three previews of a new voice. Nothing is saved until the writer picks one.""" + if not settings.elevenlabs_api_key: + raise HTTPException(503, "Set ELEVENLABS_API_KEY in .env to generate voices.") + description = _design_description(body) + try: + previews = await run_in_threadpool(tts.design_voice, description) + except tts.TTSError as exc: + raise HTTPException(502, str(exc)) from exc + out = [] + for p in previews: + key = f"ws/{ctx.workspace.id}/voice-previews/{uuid.uuid4().hex}.mp3" + storage.put_bytes(key, p["audio"], "audio/mpeg") + out.append({"generated_voice_id": p["generated_voice_id"], "url": storage.presign_get(key), "seconds": p["seconds"]}) + record_usage(ctx.db, workspace_id=ctx.workspace.id, kind="voice_design", provider="elevenlabs", + quantity=len(out), unit="previews") + return {"description": description, "previews": out} + + +@router.post("/design/save") +async def save_design(body: DesignSaveIn, ctx: Ctx = Depends(get_ctx)): + try: + voice_id = await run_in_threadpool(tts.create_designed_voice, body.name, body.description, + body.generated_voice_id) + except tts.TTSError as exc: + raise HTTPException(502, str(exc)) from exc + vp = _ensure_profile(ctx) + voices = dict(vp.host_voices or {}) + voices["custom"] = [*(voices.get("custom") or []), + {"voice_id": voice_id, "name": body.name, "description": body.description[:300]}] + if body.use_as in ("host_a", "host_b"): + voices[body.use_as] = voice_id + vp.host_voices = voices + ctx.db.commit() + return {"voice_id": voice_id, **_serialize(ctx)} + + @router.get("/reading-script") def reading_script(ctx: Ctx = Depends(get_ctx)): """About two minutes of the writer's own prose to read aloud for the clone (their words, their rhythm).""" diff --git a/backend/app/services/billing.py b/backend/app/services/billing.py index 6bfef4d53503a623c74f22bf69bfd9a64d4c3658..0597b75dc7dff20caced186a5fc3775c33919e6f 100644 --- a/backend/app/services/billing.py +++ b/backend/app/services/billing.py @@ -112,7 +112,7 @@ def build_nudges(db: Session, workspace: Workspace, plan: Plan, meters: list[dic "Open Launchpad", "/app/launchpad", priority=35)) out.append(_nudge(f"act-resurface-{month}", "action", "Old posts, new orbit", "Resurfacing finds evergreen posts worth sharing again this month.", "Find evergreen posts", - "/app/resurface", priority=20)) + "/app/launchpad#ideas", priority=20)) return sorted(out, key=lambda n: -n["priority"]) @@ -146,6 +146,8 @@ def apply_subscription(db: Session, workspace_id, *, plan: str, status: str, pro if plan not in PLANS: raise ValueError(f"Unknown plan {plan}") sub = db.query(Subscription).filter_by(workspace_id=workspace_id).one_or_none() + workspace = db.get(Workspace, workspace_id) + before = effective_plan(db, workspace).indexed_posts if workspace else None if not sub: sub = Subscription(workspace_id=workspace_id) db.add(sub) @@ -154,9 +156,23 @@ def apply_subscription(db: Session, workspace_id, *, plan: str, status: str, pro sub.provider_subscription_id = subscription_id or sub.provider_subscription_id sub.current_period_end = current_period_end db.commit() + if workspace and effective_plan(db, workspace).indexed_posts != before: + resync_sources(db, workspace_id) return sub +def resync_sources(db: Session, workspace_id) -> None: + """The post limit changed: re-sync every feed so locked posts unlock on upgrade (or re-lock on downgrade).""" + from app.pipeline.ingest import IMPORTS_FEED + from app.services.jobs import create_job + + for source in db.scalars(select(Source).where(Source.workspace_id == workspace_id, + Source.feed_url != IMPORTS_FEED)): + source.sync_status = "pending" + create_job(db, workspace_id, "ingest", {"source_id": str(source.id)}) + db.commit() + + def return_urls() -> tuple[str, str]: """(success_url, cancel_url) for the hosted checkout page.""" base = settings.frontend_url.rstrip("/") diff --git a/backend/app/services/plans.py b/backend/app/services/plans.py index d45f043d14c625f7535fadab629e630d4805da1e..fcacf0faf6685a8c56e29a74e7b61fdfc9e9992e 100644 --- a/backend/app/services/plans.py +++ b/backend/app/services/plans.py @@ -41,7 +41,7 @@ PLANS: dict[str, Plan] = { voice_cloning=False, brand_kit=False, features=( - "1 source, your 5 latest posts", + "1 source, your latest 5 posts indexed", "Grounded research chat with citations", "3 min of audio overviews a month", "1 min of video a month", diff --git a/backend/app/services/tts.py b/backend/app/services/tts.py index 9a8ddf331c3769c46c863d6cffdfa56fb32225f2..ddede9e4d73e7fdfd6827b5e033a4de031840e8f 100644 --- a/backend/app/services/tts.py +++ b/backend/app/services/tts.py @@ -289,6 +289,29 @@ def clone_voice(name: str, samples: list[tuple[str, bytes, str]], *, remove_back return resp.json()["voice_id"] +def design_voice(description: str) -> list[dict]: + """Voice Design: three short previews of a new voice from a text description. + Each preview's generated_voice_id is temporary until saved with create_designed_voice.""" + data = _request("POST", "/text-to-voice/design", attempts=2, timeout=90, + json={"voice_description": description[:1000], "auto_generate_text": True}).json() + return [ + {"generated_voice_id": p["generated_voice_id"], "audio": base64.b64decode(p["audio_base_64"]), + "seconds": p.get("duration_secs")} + for p in data.get("previews", []) if p.get("generated_voice_id") and p.get("audio_base_64") + ] + + +def create_designed_voice(name: str, description: str, generated_voice_id: str) -> str: + """Turns a Voice Design preview into a permanent voice on the account.""" + resp = _request("POST", "/text-to-voice/create-voice-from-preview", attempts=2, timeout=30, json={ + "voice_name": name[:100], + "voice_description": (description or name)[:1000], + "generated_voice_id": generated_voice_id, + "labels": {"source": "notestack"}, + }) + return resp.json()["voice_id"] + + def delete_voice(voice_id: str) -> None: _request("DELETE", f"/voices/{voice_id}", attempts=2, timeout=30) diff --git a/backend/app/worker.py b/backend/app/worker.py index 51b2a8d66ddc65cfb605163f66561d7e7508794d..f99a74b996c2a9549182a42efeea4b89d6c76a37 100644 --- a/backend/app/worker.py +++ b/backend/app/worker.py @@ -67,10 +67,22 @@ HANDED_OFF = object() # the job continues elsewhere (renderer) and reports back # Handlers: (db, job) -> Done | HANDED_OFF. Raise to trigger a retry. +def _pending(db: Session, workspace_id: uuid.UUID, kind: str) -> bool: + return db.scalar(select(Job.id).where(Job.workspace_id == workspace_id, Job.kind == kind, + Job.status.in_(("queued", "running"))).limit(1)) is not None + + def _after_import(db: Session, job: Job, workspace_id: uuid.UUID, changed: list[str]) -> None: - """New or changed posts get mapped into the topic constellation automatically.""" - if changed and settings.llm_api_key: - create_job(db, workspace_id, "topics", {"document_ids": changed}, max_attempts=2) + """New or changed posts are mapped into the topic constellation and scored for resurfacing, and the + first import builds the writing voice, so none of these need a button press.""" + if not changed or not settings.llm_api_key: + return + create_job(db, workspace_id, "topics", {"document_ids": changed}, max_attempts=2) + if not _pending(db, workspace_id, "resurface_scan"): + create_job(db, workspace_id, "resurface_scan", {}, max_attempts=2) + vp = db.scalar(select(VoiceProfile).where(VoiceProfile.workspace_id == workspace_id)) + if not (vp and vp.profile_json) and not _pending(db, workspace_id, "voice_profile"): + create_job(db, workspace_id, "voice_profile", {"document_ids": []}, max_attempts=2) def handle_ingest(db: Session, job: Job): @@ -91,6 +103,9 @@ def handle_ingest(db: Session, job: Job): raise PermanentJobError(str(exc)) from exc raise _after_import(db, job, source.workspace_id, result.get("changed", [])) + if result.get("locked"): + return Done(result, f"Found {result['found']} posts. Your latest {result['available']} are indexed; " + f"upgrade to index the other {result['locked']}") return Done(result, f"All posts in orbit: {result['indexed']} new or updated") diff --git a/backend/tests/test_locked_posts.py b/backend/tests/test_locked_posts.py new file mode 100644 index 0000000000000000000000000000000000000000..008bd2aec247caba88fd8a616d009f452200eceb --- /dev/null +++ b/backend/tests/test_locked_posts.py @@ -0,0 +1,91 @@ +"""Posts past the plan's indexed_posts are listed, not indexed: no text, no corpus file, indexed on a bigger plan.""" + +import uuid +from datetime import UTC, datetime, timedelta + +from sqlalchemy import select + +from app.models import Document, Job, Source, Workspace +from app.pipeline import ingest +from app.pipeline.ingest import FeedEntry, ingest_source +from app.routers.sources import is_locked + + +def _entries(n: int) -> list[FeedEntry]: + start = datetime(2026, 1, 1, tzinfo=UTC) + # Oldest first on purpose: ingest must pick the newest posts regardless of feed order. + return [ + FeedEntry(title=f"Post {i}", url=f"https://ada.example/p/{i}", published_at=start + timedelta(days=i), + html=f"
Paragraph for post number {i} with enough words to keep.
") + for i in range(n) + ] + + +def _setup(db, monkeypatch, n: int): + monkeypatch.setattr(ingest, "fetch_feed", lambda url, limit: ("Ada", _entries(n)[:limit])) + ws = Workspace(name="w", owner_id=uuid.uuid4()) + db.add(ws) + db.flush() + source = Source(workspace_id=ws.id, feed_url="https://ada.example/feed", platform="rss", title="Ada") + db.add(source) + db.flush() + job = Job(workspace_id=ws.id, kind="ingest", params={"source_id": str(source.id)}) + db.add(job) + db.commit() + return ws, source, job + + +def _docs(db, source): + return {d.title: d for d in db.scalars(select(Document).where(Document.source_id == source.id))} + + +def test_lists_every_post_but_indexes_only_the_latest(db_session, monkeypatch): + ws, source, job = _setup(db_session, monkeypatch, 12) + result = ingest_source(db_session, source, job, max_posts=5) + + assert result["found"] == 12 and result["available"] == 5 and result["locked"] == 7 and result["capped"] + docs = _docs(db_session, source) + assert len(docs) == 12 + available = {t for t, d in docs.items() if d.path} + assert available == {f"Post {i}" for i in range(7, 12)} # the five newest + locked = [d for d in docs.values() if not d.path] + assert len(locked) == 7 + assert all(is_locked(d) and d.clean_text == "" and d.title for d in locked) + + +def test_upgrade_unlocks_and_newer_posts_relock_the_oldest(db_session, monkeypatch): + ws, source, job = _setup(db_session, monkeypatch, 12) + ingest_source(db_session, source, job, max_posts=5) + + ingest_source(db_session, source, job, max_posts=500) + docs = _docs(db_session, source) + assert all(d.path for d in docs.values()) + assert not any(is_locked(d) for d in docs.values()) + + ingest_source(db_session, source, job, max_posts=5) + docs = _docs(db_session, source) + assert sum(1 for d in docs.values() if d.path) == 5 + assert docs["Post 0"].path is None and is_locked(docs["Post 0"]) + + +def test_scan_is_capped_per_sync(db_session, monkeypatch): + ws, source, job = _setup(db_session, monkeypatch, ingest.SCAN_CAP + 50) + result = ingest_source(db_session, source, job, max_posts=5) + assert result["found"] == ingest.SCAN_CAP + + +def test_crawl_reads_only_the_pages_it_will_index(monkeypatch): + read = [] + urls = [f"https://ada.example/posts/post-{i}" for i in range(20)] + monkeypatch.setattr(ingest, "_plain_links", lambda url: urls) + monkeypatch.setattr(ingest, "_post_urls", lambda links, url, limit: links[:limit]) + + def fake_extract(u): + read.append(u) + return FeedEntry(title=u, url=u, published_at=None, html="body text for the page
") + + monkeypatch.setattr(ingest, "extract_article", fake_extract) + entries = ingest.crawl_site("https://ada.example", 200, read_limit=5) + assert len(read) == 5 and len(entries) == 20 + assert all(e.html == "" for e in entries[5:]) + assert entries[5].title == "Post 5" diff --git a/backend/tests/test_simplified_flows.py b/backend/tests/test_simplified_flows.py new file mode 100644 index 0000000000000000000000000000000000000000..d60dc10acf5903deab8a976d4a0bbe40c9751146 --- /dev/null +++ b/backend/tests/test_simplified_flows.py @@ -0,0 +1,47 @@ +"""One-click flows: the "All posts" notebook, whole-archive generation, and jobs that run after a sync.""" + +from sqlalchemy import select + +from app.config import settings +from app.models import Job +from tests.test_buildout import auth, connect, docs, feed, llm # noqa: F401 (fixtures) + + +def test_archive_notebook_is_created_once_and_listed_first(client, auth, run_jobs, feed): + connect(client, auth, run_jobs) + client.post("/api/notebooks", json={"title": "Newer"}, headers=auth) + first = client.post("/api/notebooks/archive", headers=auth).json() + again = client.post("/api/notebooks/archive", headers=auth).json() + assert first["id"] == again["id"] and first["title"] == "All posts" + + listed = client.get("/api/notebooks", headers=auth).json() + assert listed[0]["id"] == first["id"] and listed[0]["is_archive"] + indexed = [d for d in docs(client, auth) if d["path"]] + assert listed[0]["document_count"] == len(indexed) > 0 + + nb = client.get(f"/api/notebooks/{first['id']}", headers=auth).json() + assert nb["is_archive"] and len(nb["documents"]) == len(indexed) + + +def test_generate_from_whole_archive_uses_the_archive_notebook(client, auth, run_jobs, feed): + connect(client, auth, run_jobs) + art = client.post("/api/artifacts/generate", json={"type": "summary", "archive": True}, headers=auth).json() + archive = client.post("/api/notebooks/archive", headers=auth).json() + assert art["notebook_id"] == archive["id"] + + +def test_sync_queues_resurfacing_and_writing_voice_once(client, auth, feed, db_session, session_factory, + monkeypatch): + from app.services.jobs import claim_next + from app.worker import run_job + + monkeypatch.setattr(settings, "llm_api_key", "test-key") + client.post("/api/sources", json={"url": "ada.example.com"}, headers=auth) + with session_factory() as db: + job_id = claim_next(db, "test") + run_job(job_id, session_factory=session_factory) # the ingest job only; follow-ups stay queued + db_session.expire_all() + kinds = [j.kind for j in db_session.scalars(select(Job).where(Job.status == "queued"))] + assert kinds.count("resurface_scan") == 1 + assert kinds.count("voice_profile") == 1 + assert kinds.count("topics") == 1 diff --git a/frontend/index.html b/frontend/index.html index 8ce8a789d70f4af61ee44a64e3b44b0381e895dc..f47e99317b7e535b763b83dbb8bab7cb5ce0fd58 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -1,6 +1,15 @@ + + + diff --git a/frontend/public/_redirects b/frontend/public/_redirects new file mode 100644 index 0000000000000000000000000000000000000000..61e4a382f245470aaf1f83b0685daf85b230b033 --- /dev/null +++ b/frontend/public/_redirects @@ -0,0 +1,5 @@ +# Signed in and auth routes get the empty app shell (noindex), not the prerendered home page. +/app/* /app 200 +/auth /app 200 +/auth/* /app 200 +/welcome /app 200 diff --git a/frontend/seo/prerender.ts b/frontend/seo/prerender.ts new file mode 100644 index 0000000000000000000000000000000000000000..c9a79b46603f56dfc5479ee953b6fedddf53bfdc --- /dev/null +++ b/frontend/seo/prerender.ts @@ -0,0 +1,269 @@ +import { readFile, mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { Plugin, ResolvedConfig } from "vite"; +import { blogPosts } from "../src/content/blogPosts"; +import { NOTEBOOKLM_ALT_FAQ, NOTEBOOKLM_ALT_META, NOTEBOOKLM_ALT_ROWS } from "../src/content/notebooklmAlternative"; +import type { BlogPost } from "../src/content/seoTypes"; +import { tools, toolsHub, type ToolDef } from "../src/content/tools"; + +/** + * Writes a static HTML file per public route after `vite build`, so crawlers that do not run JavaScript still get the + * page's title, description, canonical, headings, copy and links. The React app replaces #root on load. + * + * Files are written as${inline(f.a)}
`).join("")}${inline(p)}
`).join("")}${ + s.bullets?.length ? `${inline(s.callout)}
` : ""}${esc(post.mission.question)} ${inline(post.mission.answer)}
` : ""; + const related = post.relatedPaths.length + ? `` + : ""; + const faq = post.faq.map((f) => ({ q: f.question, a: f.answer })); + return { + path: `/blogs/${post.slug}`, + title: `${post.title} | Notestack`, + description: post.description, + ogType: "article", + body: page( + `${esc(post.heroEyebrow)}
${esc(post.heroDescription)}
${mission}${sections}${faqHtml(faq)}${inline(p)}
`).join("")}${ + s.bullets?.length ? `${esc(tool.eyebrow)}
${esc(tool.heroDescription)}
Sign up free to use the ${esc( + tool.eyebrow.toLowerCase(), + )}
${sections}${faqHtml(faq)}Notestack turns your blog, newsletter, site or markdown files into a research notebook you can question, podcasts you can publish and launch kits for every platform. Every word traced back to what you actually wrote.
Start free or see Notestack vs NotebookLM.
NotebookLM alternative
NotebookLM is great for a pile of documents. Notestack is built for a body of work: it syncs your blog, newsletter, site or markdown, answers with citations to the exact lines you wrote, and turns your knowledge into audio, video and launch posts.
| Notestack | NotebookLM | |
|---|---|---|
| ${esc(r.feature)} | ${esc(r.notestack)} | ${esc(r.notebooklm)} |
If you are studying someone else's documents for a one off project, want a free tool from Google, or live in Google Drive, NotebookLM is excellent. Notestack is the NotebookLM alternative when the sources are your own writing and the goal is to reuse and publish it.
Comparing more tools? Read our guide to NotebookLM alternatives and NotebookLM vs ChatGPT.
$0. 1 source, your latest 5 posts indexed, 3 audio minutes, 1 video minute, 2 Launch Kits a month.
$24.99 a month ($18.99 a month billed annually). 3 sources, 500 posts, 60 audio minutes, 30 video minutes, 50 Launch Kits.
$48.99 a month ($36.99 a month billed annually). 10 sources, 5,000 posts, 240 audio minutes, 120 video minutes, unlimited Launch Kits.
${esc(p.description)}
${esc(toolsHub.heroDescription)}
${esc(t.description)}
Free tools
+ {tools.map((t) => ( + + {t.eyebrow} + + ))} +From the blog
{blogPosts.slice(0, 3).map((p) => ( diff --git a/frontend/src/components/Reader.tsx b/frontend/src/components/Reader.tsx index a1ab985ba532f7b5ca749320ab2e4edf2b686fba..42406479ce95e01ee2664d4e91878ffcb332002c 100644 --- a/frontend/src/components/Reader.tsx +++ b/frontend/src/components/Reader.tsx @@ -1,6 +1,7 @@ import { useEffect, useRef, useState } from "react"; import { docsApi } from "../api/endpoints"; import type { DocDetail } from "../api/types"; +import { useUpgrade } from "../hooks/useUpgrade"; import { Drawer, formatDate, Loading } from "./ui"; /** Reads a post from the corpus with the exact line numbers citations point at. */ @@ -16,6 +17,7 @@ export function Reader({ const [doc, setDoc] = useState+ Your plan indexes your latest {doc.limit ?? 5} posts. Upgrade to index this one and the rest of your + archive, so you can ask about it and turn it into audio, video and Launch Kits. +
+{total} artifacts
{!items &&Set GOOGLE_CLIENT_ID in .env to enable Google sign in.
; + return null; } return ; } @@ -85,7 +85,8 @@ export default function AuthPage() { const next = safeNext(params.get("next")); useEffect(() => { - if (user) navigate(isNew ? "/welcome" : next, { replace: true }); + // New accounts get onboarding, except people who signed up to use a free tool: send them straight back to it. + if (user) navigate(isNew && !next.startsWith("/tools/") ? "/welcome" : next, { replace: true }); }, [user, navigate, next, isNew]); useEffect(() => { @@ -170,9 +171,6 @@ export default function AuthPage() { <> {providers?.google === "redirect" &&Set GOOGLE_CLIENT_ID in .env to enable Google sign in.
- )} {!providers && }{notice}
} {error &&{error}
} -- Create a developer app, then add these redirect URIs and put the client id and secret in .env: -
-Nothing scheduled this month.
+ + Schedule your latest Launch Kit + +{upcoming.length} upcoming in view
} +Mission Control
-{error}
} - {checklist.some((c) => !c.done) && ( -Pre-flight checklist
-Connect a blog or newsletter, import an article or drop in files. Everything else starts from here.
+{next.text}
+ {next.action} +{totalDocs} posts indexed across your sources.
-Audio overviews, videos and launch kits will show up here.
} -Nothing scheduled. Quiet moon tonight.
} -Is there a NotebookLM built for my own writing?
NotebookLM is great for a pile of documents. Notestack is built for a body of work: it syncs your blog, newsletter, site or
markdown, answers with citations to the exact lines you wrote, and turns your knowledge into audio, video and launch posts.
diff --git a/frontend/src/pages/NotebookView.tsx b/frontend/src/pages/NotebookView.tsx
index 9cefb5deeaf4c3bbbdf6caae5ac8bec52cb4d2df..0f3621ae9f328bdabd6b7926f123a6ae60741e48 100644
--- a/frontend/src/pages/NotebookView.tsx
+++ b/frontend/src/pages/NotebookView.tsx
@@ -1,5 +1,5 @@
import { useCallback, useEffect, useRef, useState, type FormEvent } from "react";
-import { Link, useParams } from "react-router-dom";
+import { Link, useParams, useSearchParams } from "react-router-dom";
import { artifactsApi, notebooksApi, type GenerateBody } from "../api/endpoints";
import { streamSSE } from "../api/stream";
import type { Artifact, ChatSummary, Citation, Notebook } from "../api/types";
@@ -8,15 +8,19 @@ import { Markdown } from "../components/Markdown";
import { DocPicker } from "../components/DocPicker";
import { TelescopeIcon } from "../components/icons/Icons";
import { Reader } from "../components/Reader";
-import { ConfirmButton, errorMessage, formatDate, Loading, Modal } from "../components/ui";
+import { ConfirmButton, errorMessage, formatDate, Loading, Modal, Tabs } from "../components/ui";
type Turn = { role: "user" | "assistant"; text: string; citations?: Citation[]; status?: string; steps?: string[] };
+type Pane = "create" | "posts";
+
+const STARTERS = ["What are the strongest ideas across these posts?", "Where do I contradict myself?", "Which post is most worth updating, and why?"];
+
+/** The Create tab: one click per format with sensible defaults, options folded away. */
function StudioPanel({ notebookId, disabled }: { notebookId: string; disabled: boolean }) {
const [artifacts, setArtifacts] = useState Studio