Merge pull request #7 from FireBird-Technologies/feature/report

README.md

@@ -49,7 +49,7 @@ archive, stays in sync with it, cites the exact lines behind every answer, and t
 | Citations | Exact line ranges, verified before you see them | Passage level citations |
 | Not in your sources | Says so; unverifiable claims are dropped | Grounded, passage citations |
 | Audio | Two hosts on ElevenLabs voices, or your consented voice clone | Two host overviews, Google voices |
-| Video | 9:16 shorts, 16:9 explainers, 1:1 audiograms, quote cards, in your brand | Video overviews |
+| Video | 9:16 shorts, 16:9 explainers, 1:1 audiograms, quote cards | Video overviews |
 | Publishing | Launch Kits (threads, LinkedIn, Notes, Bluesky, SEO, carousels) and auto posting | Not a publishing tool |
 | Your voice | Voice profile learned from your posts, used in every draft | General purpose writing |
 | Seeing your archive | Topic constellation with rising and dormant topics, evergreen resurfacing | Idea Constellations per notebook |
@@ -121,7 +121,7 @@ cd renderer && npm install && npm start                    # only needed for vid
 | Launchpad | Calendar and agenda, auto posting to X, LinkedIn and Bluesky, email reminders, tracked links, engagement |
 | Archive | Every generated artifact with filters, players, downloads, retry and delete |
 | Resurfacing | Evergreen scores and reshare angles for old posts, on this day, straight into a kit or the calendar |
-| Settings | Profile, workspace, brand kit, privacy, plan usage meters, connections, sign out everywhere, delete |
+| Settings | Profile, workspace, privacy, plan usage meters, connections, sign out everywhere, delete |
 
 ## Key choices
 

backend/alembic/versions/0006_plans.py

@@ -57,12 +57,15 @@ def upgrade() -> None:
             sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
             sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
         )
+    # Columns a later migration drops (brand_kit) are not in a table 0001 built from today's models: seed what exists.
+    have = {c["name"] for c in sa.inspect(bind).get_columns("plans")}
+    seed = [{k: v for k, v in p.items() if k in have} for p in SEED]
     types = {"features": sa.JSON(), "voice_cloning": sa.Boolean(), "brand_kit": sa.Boolean(),
              "created_at": sa.DateTime(timezone=True), "updated_at": sa.DateTime(timezone=True)}
-    plans = sa.table("plans", *(sa.column(k, types.get(k)) for k in [*SEED[0], "created_at", "updated_at"]))
+    plans = sa.table("plans", *(sa.column(k, types.get(k)) for k in [*seed[0], "created_at", "updated_at"]))
     existing = {row[0] for row in bind.execute(sa.text("SELECT id FROM plans"))}
     now = datetime.now(UTC)
-    missing = [{**p, "created_at": now, "updated_at": now} for p in SEED if p["id"] not in existing]
+    missing = [{**p, "created_at": now, "updated_at": now} for p in seed if p["id"] not in existing]
     if missing:
         op.bulk_insert(plans, missing)
 

backend/alembic/versions/0021_artifact_feedback.py

@@ -0,0 +1,40 @@
+"""artifact_feedback: thumbs up/down on generated reports, quizzes, flashcards and infographics
+
+Revision ID: 0021_artifact_feedback
+Revises: 0020_calendar_kit
+Create Date: 2026-10-08
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0021_artifact_feedback"
+down_revision = "0020_calendar_kit"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # 0001 builds from the live models, so a fresh database may already have the table.
+    if "artifact_feedback" in sa.inspect(op.get_bind()).get_table_names():
+        return
+    op.create_table(
+        "artifact_feedback",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("workspace_id", sa.Uuid(), sa.ForeignKey("workspaces.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("artifact_id", sa.Uuid(), sa.ForeignKey("artifacts.id", ondelete="SET NULL")),
+        sa.Column("user_id", sa.Uuid(), sa.ForeignKey("users.id", ondelete="SET NULL")),
+        sa.Column("artifact_type", sa.String(30), nullable=False),
+        sa.Column("rating", sa.Integer(), nullable=False),
+        sa.Column("reasons", sa.JSON(), nullable=False),
+        sa.Column("comment", sa.Text()),
+        sa.Column("snapshot", sa.JSON(), nullable=False),
+        sa.UniqueConstraint("artifact_id", "user_id"),
+    )
+    op.create_index("ix_artifact_feedback_workspace_id", "artifact_feedback", ["workspace_id"])
+    op.create_index("ix_artifact_feedback_artifact_id", "artifact_feedback", ["artifact_id"])
+
+
+def downgrade() -> None:
+    op.drop_table("artifact_feedback")

backend/alembic/versions/0022_report_template_cache.py

@@ -0,0 +1,34 @@
+"""report_template_cache: suggested report templates kept, so the Create report dialog is instant
+
+Revision ID: 0022_report_template_cache
+Revises: 0021_artifact_feedback
+Create Date: 2026-10-08
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0022_report_template_cache"
+down_revision = "0021_artifact_feedback"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # 0001 builds from the live models, so a fresh database may already have the table.
+    if "report_template_cache" in sa.inspect(op.get_bind()).get_table_names():
+        return
+    op.create_table(
+        "report_template_cache",
+        sa.Column("id", sa.Uuid(), primary_key=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("workspace_id", sa.Uuid(), sa.ForeignKey("workspaces.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("key", sa.String(64), nullable=False),
+        sa.Column("templates", sa.JSON(), nullable=False),
+        sa.UniqueConstraint("workspace_id", "key"),
+    )
+    op.create_index("ix_report_template_cache_workspace_id", "report_template_cache", ["workspace_id"])
+
+
+def downgrade() -> None:
+    op.drop_table("report_template_cache")

backend/alembic/versions/0023_drop_brand_kit.py

@@ -0,0 +1,31 @@
+"""drop_brand_kit: the brand kit (workspace colors and logo, and the plan flag for it) is gone
+
+Revision ID: 0023_drop_brand_kit
+Revises: 0022_report_template_cache
+Create Date: 2026-10-08
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0023_drop_brand_kit"
+down_revision = "0022_report_template_cache"
+branch_labels = None
+depends_on = None
+
+
+def _has(table: str, column: str) -> bool:
+    return column in {c["name"] for c in sa.inspect(op.get_bind()).get_columns(table)}
+
+
+def upgrade() -> None:
+    for table, column in (("plans", "brand_kit"), ("workspaces", "brand_json")):
+        if _has(table, column):
+            with op.batch_alter_table(table) as batch:
+                batch.drop_column(column)
+
+
+def downgrade() -> None:
+    if not _has("plans", "brand_kit"):
+        op.add_column("plans", sa.Column("brand_kit", sa.Boolean(), nullable=False, server_default=sa.false()))
+    if not _has("workspaces", "brand_json"):
+        op.add_column("workspaces", sa.Column("brand_json", sa.JSON(), nullable=True))

backend/alembic/versions/0024_plan_limits.py

@@ -0,0 +1,49 @@
+"""plan_limits: posts indexed across every source, Free allowances that are totals, new Writer and Studio numbers
+
+Revision ID: 0024_plan_limits
+Revises: 0023_drop_brand_kit
+Create Date: 2026-10-08
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0024_plan_limits"
+down_revision = "0023_drop_brand_kit"
+branch_labels = None
+depends_on = None
+
+PLANS = {
+    "free": dict(sources=-1, indexed_posts=5, audio_minutes=-1, audio_overviews=1, videos=2, launch_kits=2, reports=2,
+                 infographics=2, videos_monthly=False, features=[
+                     "Up to 5 posts indexed, from any source", "Grounded research chat with citations",
+                     "1 audio overview, any length", "2 videos", "2 Launch Kits", "2 infographics", "2 reports a month"]),
+    "writer": dict(sources=-1, indexed_posts=100, audio_minutes=60, audio_overviews=-1, videos=10, launch_kits=20,
+                   reports=30, infographics=30, features=[
+                       "Up to 100 posts indexed, from any source", "60 min of audio overviews a month", "10 videos a month",
+                       "20 Launch Kits a month", "30 reports a month", "30 infographics a month",
+                       "Voice cloning with consent"]),
+    "studio": dict(sources=-1, indexed_posts=250, audio_minutes=240, audio_overviews=-1, videos=20, launch_kits=50,
+                   reports=-1, infographics=-1, features=[
+                       "Up to 250 posts indexed, from any source", "240 min of audio overviews a month",
+                       "20 videos a month", "50 Launch Kits a month", "Unlimited reports", "Unlimited infographics",
+                       "Launchpad calendar and resurfacing", "Priority rendering"]),
+}
+
+
+def upgrade() -> None:
+    columns = {c["name"] for c in sa.inspect(op.get_bind()).get_columns("plans")}
+    if "audio_overviews" not in columns:
+        op.add_column("plans", sa.Column("audio_overviews", sa.Integer(), nullable=False, server_default="-1"))
+    plans = sa.table("plans", sa.column("id", sa.String), sa.column("sources", sa.Integer),
+                     sa.column("indexed_posts", sa.Integer), sa.column("audio_minutes", sa.Integer),
+                     sa.column("audio_overviews", sa.Integer), sa.column("videos", sa.Integer),
+                     sa.column("launch_kits", sa.Integer), sa.column("reports", sa.Integer),
+                     sa.column("infographics", sa.Integer), sa.column("videos_monthly", sa.Boolean),
+                     sa.column("features", sa.JSON))
+    for plan_id, values in PLANS.items():
+        op.execute(plans.update().where(plans.c.id == plan_id).values(**values))
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("plans") as batch:
+        batch.drop_column("audio_overviews")

backend/alembic/versions/0025_drop_video_premium.py

@@ -0,0 +1,24 @@
+"""drop_video_premium: the premium video options are gone, so is the plan flag for them
+
+Revision ID: 0025_drop_video_premium
+Revises: 0024_plan_limits
+Create Date: 2026-10-08
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0025_drop_video_premium"
+down_revision = "0024_plan_limits"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    if "video_premium" in {c["name"] for c in sa.inspect(op.get_bind()).get_columns("plans")}:
+        with op.batch_alter_table("plans") as batch:
+            batch.drop_column("video_premium")
+
+
+def downgrade() -> None:
+    if "video_premium" not in {c["name"] for c in sa.inspect(op.get_bind()).get_columns("plans")}:
+        op.add_column("plans", sa.Column("video_premium", sa.Boolean(), nullable=False, server_default=sa.false()))

backend/alembic/versions/0026_plan_copy.py

@@ -0,0 +1,40 @@
+"""plan_copy: Free gets 1 video; Writer and Studio audio copy is relative to Free
+
+Revision ID: 0026_plan_copy
+Revises: 0025_drop_video_premium
+Create Date: 2026-10-08
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = "0026_plan_copy"
+down_revision = "0025_drop_video_premium"
+branch_labels = None
+depends_on = None
+
+SWAPS = {
+    "free": {"2 videos": "1 video"},
+    "writer": {"60 min of audio overviews a month": "10x more audio overviews than Free (60 min a month)"},
+    "studio": {"240 min of audio overviews a month": "40x more audio overviews than Free (240 min a month)"},
+}
+
+
+def _swap(rows: dict) -> None:
+    plans = sa.table("plans", sa.column("id", sa.String), sa.column("videos", sa.Integer), sa.column("features", sa.JSON))
+    bind = op.get_bind()
+    for plan_id, swaps in rows.items():
+        features = bind.execute(sa.select(plans.c.features).where(plans.c.id == plan_id)).scalar()
+        if features is None:
+            continue
+        values = {"features": [swaps.get(f, f) for f in features]}
+        if plan_id == "free":
+            values["videos"] = 1 if "1 video" in swaps.values() else 2
+        op.execute(plans.update().where(plans.c.id == plan_id).values(**values))
+
+
+def upgrade() -> None:
+    _swap(SWAPS)
+
+
+def downgrade() -> None:
+    _swap({k: {v: o for o, v in d.items()} for k, d in SWAPS.items()})

backend/app/config.py

@@ -64,6 +64,25 @@ class Settings(BaseSettings):
     r2_presign_ttl_seconds: int = 3600
     max_upload_bytes: int = 300 * 1024 * 1024
 
+    # Posts an infographic reads, and a report that is not built from themes (REPORT_PLANNER below). A quiz or flashcard set reads up to that many straight from the posts; with more (no limit
+    # on how many are picked) it reads a small sample of the ideas already extracted from them, so its size does not grow.
+    artifact_max_posts: int = 20
+    # Characters of post text a report, quiz or flashcard set may read in all (split evenly between the posts, 1,500 at least each).
+    artifact_passage_budget: int = 120_000
+    # The same for a quiz: it needs less text than a report to ask good questions.
+    artifact_quiz_passage_budget: int = 60_000
+    # Ideas (already extracted from each post) in the evidence of a flashcard set, or of a quiz above ARTIFACT_MAX_POSTS posts.
+    artifact_idea_sample: int = 30
+
+    # Reports. "themes": the posts' stored ideas are grouped into weighted themes, a blueprint plans the report, and each section is
+    # written from its own evidence (pipeline/report_themes.py). "legacy": the posts' text goes straight to the writer, at most
+    # ARTIFACT_MAX_POSTS of them. Chats, small selections and posts without stored ideas always use the legacy way.
+    report_planner: str = "themes"
+    # Posts a themed report reads: 0 = every post picked (the plan's indexed_posts still applies).
+    report_max_posts: int = 0
+    # Posts whose ideas are extracted while the report is made (in parallel); the rest are represented by their opening.
+    report_inline_ideas: int = 25
+
     # LLM (LiteLLM model strings; Z.ai GLM by default)
     llm_model: str = "openai/glm-5.3"
     llm_fast_model: str = "openai/glm-5.3-flash"
@@ -86,6 +105,9 @@ class Settings(BaseSettings):
     llm_temperature: float = 1.0  # Z.ai recommends 1.0 for GLM-5.x
     # Reasoning output counts against max_tokens, so leave headroom for the structured answer.
     llm_max_tokens: int = 10000
+    # The output cap of the fast model: quizzes, flashcards, infographics, idea extraction, template suggestions. A cap, not a target:
+    # only the tokens written are charged, and the thinking counts too, so keep room for it. Measured answers are under 3,000.
+    llm_fast_max_tokens: int = 6000
     # GLM-5.x always reasons (it cannot be disabled); this sets how hard. low | high | max
     llm_reasoning_effort: str = "low"
 

backend/app/db.py

@@ -11,13 +11,18 @@ class Base(DeclarativeBase):
 
 
 _sqlite = settings.database_url.startswith("sqlite")
-# Postgres: a connection that dies silently (laptop sleep, network change, Neon dropping it) otherwise leaves a query
-# waiting on the socket forever, and the worker loop with it. Keepalives notice a dead peer within about a minute
-# and the query fails instead; connect_timeout does the same for a connect that never completes.
+# A hosted Postgres (Neon) closes idle connections and drops the odd one on a network blip, and a connection that dies silently
+# (laptop sleep, network change) otherwise leaves a query waiting on the socket forever, and the worker loop with it. TCP
+# keepalives notice a dead peer within about a minute, so the query fails instead and a dead socket is noticed while it sits in
+# the pool; connect_timeout does the same for a connect that never completes. A pooled connection is also replaced before it
+# is handed out (pool_recycle, below Neon's idle timeout), since pool_pre_ping alone can fail on a dead SSL socket with a
+# ProgrammingError ("can't change 'autocommit' now") that SQLAlchemy does not treat as a lost connection, and the request
+# ends in a 500.
 _pg_connect_args = {"connect_timeout": 10, "keepalives": 1, "keepalives_idle": 30, "keepalives_interval": 10,
                     "keepalives_count": 3}
 _connect_args = {"check_same_thread": False, "timeout": 30} if _sqlite else _pg_connect_args
-engine = create_engine(settings.database_url, pool_pre_ping=True, pool_recycle=300, connect_args=_connect_args)
+engine = create_engine(settings.database_url, pool_pre_ping=True, connect_args=_connect_args,
+                       **({} if _sqlite else {"pool_recycle": 240, "pool_use_lifo": True}))
 
 if _sqlite:
     # Local dev: the API and the worker share one file. WAL lets reads run during a write, the busy

backend/app/infographics/design.py

@@ -0,0 +1,213 @@
+"""The AI-composed infographic: a page frame of ours (head, takeaway, background, mark) around a body the model writes in
+HTML from a small vocabulary of components (flow charts, timelines, hierarchies, comparisons, cards, numbers). The
+model's markup is never trusted: it is parsed and rebuilt from an allowlist of tags and class names, with no
+attributes but `class`, so it can carry no script, style, link, image or address."""
+
+import hashlib
+import random
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from bs4 import BeautifulSoup, Comment, NavigableString, Tag
+from jinja2 import Environment, FileSystemLoader, StrictUndefined, select_autoescape
+from markupsafe import Markup
+
+from app.infographics.render import LOGO_SVG, SIZES
+
+_DIR = Path(__file__).parent / "templates"
+CSS = (_DIR / "design.css").read_text(encoding="utf-8")
+
+
+@dataclass(frozen=True)
+class Look:
+    id: str
+    name: str
+    blurb: str
+    vars: dict  # CSS variables of the palette
+
+
+LOOKS: dict[str, Look] = {l.id: l for l in (
+    Look("midnight", "Midnight", "Black and blue, white type", {
+        "bg": "#03050a", "bg2": "#0b1a3a", "ink": "#ffffff", "mute": "#9fb0cc", "acc": "#2f7bff", "acc2": "#8db8ff",
+        "line": "rgba(130,170,255,.26)", "card": "rgba(255,255,255,.04)", "hl": "rgba(47,123,255,.18)", "paper": "#ffffff",
+        "paperink": "#03050a", "star": "#ffffff", "markbg": "rgba(3,5,10,.75)", "markline": "rgba(141,184,255,.4)", "markink": "#ffffff"}),
+    Look("paper", "Orbit", "White, black type, blue orbits and stars", {
+        "bg": "#ffffff", "bg2": "#eaf1ff", "ink": "#05070d", "mute": "#4b5668", "acc": "#1f5fe6", "acc2": "#1f5fe6",
+        "line": "rgba(5,7,13,.18)", "card": "#ffffff", "hl": "rgba(31,95,230,.09)", "paper": "#05070d",
+        "paperink": "#ffffff", "star": "#1f5fe6", "markbg": "#05070d", "markline": "#05070d", "markink": "#ffffff"}),
+)}
+DEFAULT_LOOK = "midnight"
+
+# What the person can ask the layout to lean towards; "auto" lets the AI choose from the content.
+STYLES = {
+    "auto": "Choose the layout that suits the content",
+    "flowchart": "A flow chart: boxes joined by arrows, steps in order",
+    "timeline": "A timeline: what happened or happens, in order",
+    "compare": "A comparison: two sides set against each other",
+    "hierarchy": "A hierarchy: one idea and the parts it breaks into",
+    "overview": "An overview: cards of the key ideas and the numbers that matter",
+}
+
+# Every class the model's markup may use (the components in design.css); anything else is dropped.
+ALLOWED_CLASSES = {
+    "sec", "sec-t", "lead", "mono", "small", "hl", "node", "n-no", "n-t", "n-d", "flow", "col", "grid", "c2", "c3", "c4",
+    "card", "c-k", "c-t", "c-d", "timeline", "t-item", "t-when", "t-t", "t-d", "tree", "t-root", "t-kids", "compare",
+    "side", "s-h", "stats", "stat", "s-n", "s-l", "cols", "pills", "pill", "callout",
+}
+ALLOWED_TAGS = {"div", "section", "span", "p", "ul", "ol", "li", "strong", "b", "em", "i", "small", "br", "h3", "h4"}
+DROP_WITH_CONTENT = {"script", "style", "iframe", "object", "embed", "link", "meta", "svg", "img", "video", "audio", "form",
+                     "input", "button", "textarea", "select", "canvas", "math", "head", "title", "noscript", "template"}
+MAX_BODY_CHARS = 14_000
+MAX_ELEMENTS = 320
+
+_SPACE = re.compile(r"\s+")
+
+
+def _clean_text(text: str) -> str:
+    return text.replace("—", ",").replace("–", "-")
+
+
+def sanitize_body(raw: object) -> str | None:
+    """The model's body markup rebuilt from the allowlist, or None when nothing readable is left."""
+    if not isinstance(raw, str) or not raw.strip():
+        return None
+    soup = BeautifulSoup(raw[: MAX_BODY_CHARS * 2], "html.parser")
+    for c in soup.find_all(string=lambda t: isinstance(t, Comment)):
+        c.extract()
+    for tag in soup.find_all(DROP_WITH_CONTENT):
+        tag.decompose()
+    count = 0
+    for tag in list(soup.find_all(True)):
+        count += 1
+        if tag.name not in ALLOWED_TAGS or count > MAX_ELEMENTS:
+            tag.unwrap() if count <= MAX_ELEMENTS else tag.decompose()
+            continue
+        classes = [c for c in (tag.get("class") or []) if c in ALLOWED_CLASSES]
+        tag.attrs = {"class": classes} if classes else {}
+    for node in soup.find_all(string=True):
+        if isinstance(node, NavigableString):
+            node.replace_with(NavigableString(_clean_text(str(node))))
+    html = _SPACE.sub(" ", str(soup)).strip()
+    text = soup.get_text(strip=True)
+    if len(html) > MAX_BODY_CHARS or len(text) < 40:
+        return None
+    return html
+
+
+def _clip(text: object, limit: int) -> str:
+    s = _SPACE.sub(" ", re.sub(r"<[^>]*>", " ", str(text or ""))).strip()
+    s = _clean_text(s)
+    if len(s) <= limit:
+        return s
+    cut = s[: limit - 1].rsplit(" ", 1)[0].rstrip(" ,;:.-") or s[: limit - 1]
+    return cut + "…"
+
+
+def build_content(raw: dict) -> dict | None:
+    """The stored content for a model answer (title, subtitle, eyebrow, key_figures, body_html, rule, notes), or None when it is
+    too thin to draw."""
+    if hasattr(raw, "model_dump"):
+        raw = raw.model_dump()
+    title = _clip(raw.get("title"), 70)
+    body = sanitize_body(raw.get("body_html"))
+    if not title or not body:
+        return None
+    figures = []
+    for f in raw.get("key_figures") or []:
+        f = f if isinstance(f, dict) else getattr(f, "__dict__", {})
+        value, label = _clip(f.get("value"), 14), _clip(f.get("label"), 44)
+        if value and label:
+            figures.append({"value": value, "label": label})
+    notes = [n for n in (_clip(n, 100) for n in raw.get("notes") or []) if n][:3]
+    return {"title": title, "subtitle": _clip(raw.get("subtitle"), 130), "eyebrow": _clip(raw.get("eyebrow"), 30) or "Key ideas",
+            "key_figures": figures[:3], "body_html": body, "rule": _clip(raw.get("rule"), 90), "notes": notes}
+
+
+def _stars(seed: str, w: int, h: int, n: int = 90) -> str:
+    rnd = random.Random(int(hashlib.sha1(seed.encode()).hexdigest()[:8], 16))
+    out = []
+    for _ in range(n):
+        r = rnd.choice((1.2, 1.5, 1.5, 2, 2.6))
+        out.append(f'<circle cx="{rnd.randint(0, w)}" cy="{rnd.randint(0, h)}" r="{r}" fill="var(--star)" opacity="{round(rnd.uniform(.12, .5), 2)}"/>')
+    return "".join(out)
+
+
+def _orbits(w: int, h: int) -> str:
+    """The Orbit look's scenery: a ringed planet like the app's icon, orbit rings with small moons, and shooting stars."""
+    big = round(w * 0.2)
+    cx, cy = round(w * 0.86), round(h * 0.085)
+    pr = round(w * 0.05)
+
+    def ring(x, y, rx, ry, op, dash=""):
+        return (f'<ellipse cx="{x}" cy="{y}" rx="{rx}" ry="{ry}" transform="rotate(-20 {x} {y})" fill="none" stroke="var(--acc)" '
+                f'stroke-width="2" opacity="{op}"{dash}/>')
+
+    def planet_ring(front: bool):
+        sweep = 0 if front else 1
+        return (f'<g transform="rotate(-22 {cx} {cy})"><path d="M{cx - pr * 2} {cy} A{pr * 2} {round(pr * .6)} 0 0 {sweep} {cx + pr * 2} {cy}" '
+                f'fill="none" stroke="var(--acc)" stroke-width="5" opacity="{1 if front else .5}"/></g>')
+
+    out = [
+        '<defs><linearGradient id="shoot" x1="0" y1="0" x2="1" y2="0"><stop offset="0" stop-color="var(--acc)" stop-opacity="0"/>'
+        '<stop offset="1" stop-color="var(--acc)" stop-opacity=".85"/></linearGradient></defs>',
+        ring(cx, cy, big, round(big * .3), .35, ' stroke-dasharray="3 12"'),
+        ring(cx, cy, round(big * 1.55), round(big * .46), .22),
+        ring(round(w * .06), round(h * .96), round(w * .5), round(w * .14), .2, ' stroke-dasharray="3 12"'),
+        ring(round(w * .06), round(h * .96), round(w * .34), round(w * .09), .28),
+        planet_ring(False),
+        f'<circle cx="{cx}" cy="{cy}" r="{pr}" fill="var(--bg)" stroke="var(--acc)" stroke-width="5"/>',
+        f'<circle cx="{cx}" cy="{cy}" r="{pr - 12}" fill="var(--acc)" opacity=".12"/>',
+        planet_ring(True),
+        f'<circle cx="{round(w * .6)}" cy="{round(h * .036)}" r="9" fill="var(--acc)" opacity=".7"/>',
+        f'<circle cx="{round(w * .31)}" cy="{round(h * .93)}" r="8" fill="var(--acc)" opacity=".6"/>',
+    ]
+    for fx, fy, length, op in ((.46, .2, 190, .8), (.7, .4, 150, .6), (.12, .55, 120, .5)):
+        x, y = round(w * fx), round(h * fy)
+        out.append(f'<g transform="rotate(28 {x} {y})" opacity="{op}"><rect x="{x - length}" y="{y - 2}" width="{length}" height="4" rx="2" fill="url(#shoot)"/>'
+                   f'<circle cx="{x}" cy="{y}" r="5" fill="var(--acc)"/></g>')
+    return "".join(out)
+
+
+_ENV = Environment(loader=FileSystemLoader(_DIR), autoescape=select_autoescape(["j2", "html"], default=True), undefined=StrictUndefined,
+                   trim_blocks=True, lstrip_blocks=True)
+
+
+def render_design(look: str, content: dict, layout: str = "portrait", fit: float | None = None) -> str:
+    """The page for an AI-composed infographic. `fit` is the scale that makes its body fit (stored with the content)."""
+    look = look if look in LOOKS else DEFAULT_LOOK
+    layout = layout if layout in SIZES else "portrait"
+    w, h = SIZES[layout]
+    if fit is None:
+        fit = float(((content.get("fit") or {}) if isinstance(content.get("fit"), dict) else {}).get(layout) or 1)
+    fit = max(0.5, min(1.0, fit))
+    vars_css = ";".join(f"--{k}:{v}" for k, v in LOOKS[look].vars.items())
+    return _ENV.get_template("design.html.j2").render(
+        c=content, look=look, layout=layout, w=w, h=h, fit=fit, vars_css=vars_css, css=Markup(CSS), logo=Markup(LOGO_SVG),
+        body=Markup(content["body_html"]),
+        stars=Markup(_stars(content["title"] + look, w, h) + (_orbits(w, h) if look == "paper" else "")),
+        fonts="https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@500;600;700&family=Inter:wght@300;400;500;600&family=JetBrains+Mono:wght@400;500;700&display=swap")
+
+
+SAMPLE_BODY = (
+    '<section class="sec"><div class="sec-t">The five layers</div><div class="flow">'
+    '<div class="node"><span class="n-no">01</span><span class="n-t">Prompt</span><span class="n-d">Wording, examples and format of one request.</span></div>'
+    '<div class="node"><span class="n-no">02</span><span class="n-t">Context</span><span class="n-d">Docs, history, tools and memory for each call.</span></div>'
+    '<div class="node hl"><span class="n-no">03</span><span class="n-t">Harness</span><span class="n-d">Tools, checks and guardrails around the model.</span></div>'
+    '<div class="node"><span class="n-no">04</span><span class="n-t">Loop</span><span class="n-d">Reason, act and observe until done.</span></div></div></section>'
+    '<section class="sec"><div class="sec-t">Every request goes through</div><div class="grid c4">'
+    + "".join(f'<div class="card{" hl" if i == 3 else ""}"><span class="c-k">Step {i + 1}</span><span class="c-t">{t}</span><span class="c-d">{d}</span></div>'
+              for i, (t, d) in enumerate([("Auth", "Identity, limits and input guards."), ("Cache", "Answers to repeat questions."),
+                                          ("Safety", "Scrub PII, catch injection."), ("Memory", "Session history, summarised."),
+                                          ("Retrieve", "Fetch docs, split queries."), ("Answer", "Sub-queries in parallel."),
+                                          ("Validate", "Check quality first."), ("Save", "Store and log one trace.")]))
+    + '</div></section>'
+    '<section class="sec"><div class="sec-t">Why it matters</div><div class="callout">Each layer builds on the one below it, so start at the bottom and add a layer only when the task needs it.</div></section>'
+)
+SAMPLE = {"title": "From Prompt to Production Agent", "subtitle": "Five engineering layers and the steps every request takes",
+          "eyebrow": "Agent engineering", "body_html": SAMPLE_BODY,
+          "key_figures": [{"value": "5", "label": "engineering layers"}, {"value": "8", "label": "steps on every request"},
+                          {"value": "30%", "label": "of traffic skips the pipeline"}],
+          "rule": "Most tasks only need the bottom layers; avoid complex graphs without cause.",
+          "notes": ["Each layer builds on the one below it.", "Cache and middleware run outside the graph.",
+                    "Every request produces one end-to-end trace."]}

backend/app/infographics/image.py

@@ -5,6 +5,7 @@ import functools
 import glob
 import hashlib
 import os
+import re
 import shutil
 import subprocess
 import tempfile
@@ -14,8 +15,9 @@ from threading import Lock
 
 from app.infographics.render import SIZES
 
-TIMEOUT = 60  # seconds for one drawing
-KEEP = 8  # drawings kept in memory
+TIMEOUT = 90  # seconds for one drawing
+SCALE = 2  # pixels per page pixel: a 1600 x 2260 page is drawn as 3200 x 4520, sharp on any screen and in print
+KEEP = 6  # drawings kept in memory
 
 _cache: OrderedDict[str, bytes] = OrderedDict()
 _lock = Lock()
@@ -51,9 +53,9 @@ def chrome() -> str | None:
 
 
 def render_png(html: str, layout: str = "portrait") -> bytes:
-    """The page as a PNG at its own size: tall (1600 x 2260) or wide (2260 x 1600)."""
+    """The page as a PNG, drawn at SCALE times its size: tall (1600 x 2260) or wide (2260 x 1600) before scaling."""
     width, height = SIZES.get(layout, SIZES["portrait"])
-    key = hashlib.sha256(f"{width}x{height}{html}".encode()).hexdigest()
+    key = hashlib.sha256(f"{width}x{height}@{SCALE}{html}".encode()).hexdigest()
     with _lock:
         if key in _cache:
             _cache.move_to_end(key)
@@ -65,7 +67,7 @@ def render_png(html: str, layout: str = "portrait") -> bytes:
     with tempfile.TemporaryDirectory() as tmp:
         src, out = Path(tmp) / "page.html", Path(tmp) / "out.png"
         src.write_text(page, encoding="utf-8")
-        cmd = [binary, "--headless=new", "--disable-gpu", "--no-sandbox", "--hide-scrollbars", "--force-device-scale-factor=1",
+        cmd = [binary, "--headless=new", "--disable-gpu", "--no-sandbox", "--hide-scrollbars", f"--force-device-scale-factor={SCALE}",
                f"--user-data-dir={tmp}/profile", f"--window-size={width},{height}", "--virtual-time-budget=8000",
                f"--screenshot={out}", f"file://{src}"]
         try:
@@ -80,3 +82,30 @@ def render_png(html: str, layout: str = "portrait") -> bytes:
         while len(_cache) > KEEP:
             _cache.popitem(last=False)
     return data
+
+
+_FIT = re.compile(r"FIT=([0-9.]+)")
+_MEASURE = ("<script>document.fonts.ready.then(function(){var b=document.querySelector('.ig-body'),i=document.querySelector('.ig-inner');"
+            "var s=1;if(b&&i){var inner=i.scrollHeight*parseFloat(getComputedStyle(i).getPropertyValue('--fit')||1);s=Math.min(1,b.clientHeight/Math.max(inner,1));}"
+            "document.title='FIT='+s.toFixed(3);});</script>")
+
+
+def measure_fit(html: str, layout: str) -> float:
+    """The scale (1 or less) that makes an AI-composed page's body fit its space: the page drawn once, in Chrome, with our own
+    small script measuring it (the page's markup is sanitised, so no other script is in it). 1 when Chrome is missing."""
+    width, height = SIZES.get(layout, SIZES["portrait"])
+    binary = chrome()
+    if not binary:
+        return 1.0
+    page = f'<!doctype html><meta charset="utf-8"><body style="margin:0">{html}{_MEASURE}</body>'
+    with tempfile.TemporaryDirectory() as tmp:
+        src = Path(tmp) / "page.html"
+        src.write_text(page, encoding="utf-8")
+        cmd = [binary, "--headless=new", "--disable-gpu", "--no-sandbox", "--hide-scrollbars", f"--user-data-dir={tmp}/profile",
+               f"--window-size={width},{height}", "--virtual-time-budget=10000", "--dump-dom", f"file://{src}"]
+        try:
+            out = subprocess.run(cmd, check=True, capture_output=True, timeout=TIMEOUT, text=True).stdout
+        except (subprocess.SubprocessError, OSError):
+            return 1.0
+    found = _FIT.search(out)
+    return max(0.55, min(1.0, float(found.group(1)))) if found else 1.0

backend/app/infographics/templates/design.css

@@ -0,0 +1,126 @@
+/* The infographic design system: black, blue and white, thin lines, flow charts. The page frame (head, takeaway, mark, the
+   background) is ours; the body is composed by the AI from the components below, and only these class names survive the
+   sanitiser (design.ALLOWED_CLASSES). */
+.ig2,.ig2 *{box-sizing:border-box;margin:0;padding:0}
+.ig2{position:relative;width:var(--w);height:var(--h);overflow:hidden;color:var(--ink);font-family:'Inter',sans-serif;
+  background:radial-gradient(1300px 900px at 82% 6%,var(--bg2),var(--bg) 68%)}
+.ig2 .ig-bg{position:absolute;inset:0;pointer-events:none}
+.ig2 .ig-main{position:absolute;left:110px;right:110px;top:84px;bottom:132px;display:flex;flex-direction:column;gap:34px}
+.ig2 .ig-left{display:contents}
+.ig2 .ig-head{order:1;position:relative;flex:none;min-height:286px}
+.ig2 .ig-body{order:2;flex:1;min-height:0;overflow:hidden;display:flex}
+.ig2 .ig-take{order:3;flex:none}
+
+.ig2 .ig-eyebrow{display:flex;align-items:center;gap:16px;font:500 20px 'JetBrains Mono',monospace;letter-spacing:.24em;text-transform:uppercase;color:var(--acc2);margin-bottom:20px}
+.ig2 .ig-eyebrow:before{content:"";width:44px;height:2px;background:var(--acc)}
+.ig2 h1{font:600 74px/1.04 'Space Grotesk',sans-serif;letter-spacing:-.02em;max-width:850px;margin-bottom:20px}
+.ig2 .ig-sub{font-size:30px;line-height:1.38;color:var(--mute);max-width:850px}
+/* The key figures: the few numbers or short facts that matter most, beside the title (or under it when wide) */
+.ig2 .ig-figs{position:absolute;right:0;top:0;width:430px;display:flex;flex-direction:column;gap:16px}
+.ig2 .fig{padding:6px 0 6px 24px;border-left:3px solid var(--acc)}
+.ig2 .fig-v{display:block;font:600 56px/1.02 'Space Grotesk',sans-serif;letter-spacing:-.02em;overflow-wrap:anywhere}
+.ig2 .fig-l{display:block;margin-top:6px;font:500 19px 'JetBrains Mono',monospace;letter-spacing:.1em;text-transform:uppercase;color:var(--mute)}
+
+.ig2 .ig-take{display:grid;grid-template-columns:1.05fr 1fr;gap:48px;align-items:center;padding:32px 46px;border-radius:16px;background:var(--paper);color:var(--paperink)}
+.ig2 .ig-take .q{font:600 38px/1.16 'Space Grotesk',sans-serif;letter-spacing:-.01em}
+.ig2 .ig-take ul{list-style:none;font-size:24px;line-height:1.38}
+.ig2 .ig-take li{position:relative;padding-left:28px;margin:8px 0}
+.ig2 .ig-take li:before{content:"";position:absolute;left:0;top:12px;width:11px;height:11px;border-radius:50%;background:var(--acc)}
+
+.ig2 .ig-mark{position:absolute;right:44px;bottom:34px;display:flex;align-items:center;gap:16px;padding:8px 26px 8px 10px;border-radius:999px;
+  background:var(--markbg);border:1px solid var(--markline);font:700 30px 'JetBrains Mono',monospace;letter-spacing:.04em;color:var(--markink)}
+.ig2 .ig-mark svg{width:52px;height:52px;display:block}
+
+/* The body: fitted by scaling when the AI wrote more than a poster holds (the scale is measured once, when it is made) */
+.ig2 .ig-inner{display:flex;flex-direction:column;justify-content:space-evenly;gap:34px;min-height:calc(100% / var(--fit,1));flex:none;width:calc(1380px / var(--fit,1));transform:scale(var(--fit,1));transform-origin:0 0}
+.ig2 .sec{display:flex;flex-direction:column;gap:20px}
+.ig2 .sec-t{display:flex;align-items:center;gap:18px;font:500 20px 'JetBrains Mono',monospace;letter-spacing:.22em;text-transform:uppercase;color:var(--acc2)}
+.ig2 .sec-t:after{content:"";flex:1;height:1.5px;background:var(--line)}
+.ig2 .lead{font-size:28px;line-height:1.4;color:var(--ink)}
+.ig2 .ig-inner p{font-size:24px;line-height:1.4;color:var(--mute)}
+.ig2 .ig-inner strong,.ig2 .ig-inner b{color:var(--ink);font-weight:600}
+.ig2 .ig-inner em,.ig2 .ig-inner i{color:var(--acc2);font-style:normal}
+.ig2 .mono{font-family:'JetBrains Mono',monospace}
+.ig2 .small{font-size:20px}
+
+/* A box in a flow: a small number, a title, a line of text */
+.ig2 .node{position:relative;padding:24px 26px;border:1.5px solid var(--line);border-radius:14px;background:linear-gradient(var(--card),var(--card)),linear-gradient(var(--bg),var(--bg))}
+.ig2 .node.hl{background:linear-gradient(var(--hl),var(--hl)),linear-gradient(var(--bg),var(--bg));border-color:var(--acc)}
+.ig2 .n-no{display:block;font:500 19px 'JetBrains Mono',monospace;letter-spacing:.14em;color:var(--acc2);margin-bottom:10px}
+.ig2 .n-t{display:block;font:600 31px/1.15 'Space Grotesk',sans-serif;margin-bottom:8px}
+.ig2 .n-d{display:block;font-size:23px;line-height:1.38;color:var(--mute)}
+
+/* Flow chart: boxes in a row (or a column) joined by arrows */
+.ig2 .flow{display:flex;gap:58px;align-items:stretch}
+.ig2 .flow>.node{flex:1;min-width:0}
+.ig2 .flow>.node+.node:before{content:"";position:absolute;left:-48px;top:50%;width:34px;height:2px;background:var(--acc)}
+.ig2 .flow>.node+.node:after{content:"";position:absolute;left:-15px;top:50%;margin-top:-7px;border-style:solid;border-width:7px 0 7px 11px;border-color:transparent transparent transparent var(--acc)}
+.ig2 .flow.col{flex-direction:column;gap:50px}
+.ig2 .flow.col>.node+.node:before{left:50%;top:-40px;width:2px;height:28px;margin-left:-1px}
+.ig2 .flow.col>.node+.node:after{left:50%;top:-13px;margin:0 0 0 -7px;border-width:11px 7px 0 7px;border-color:var(--acc) transparent transparent transparent}
+
+/* Cards in a grid */
+.ig2 .grid{display:grid;gap:22px}
+.ig2 .c2{grid-template-columns:repeat(2,minmax(0,1fr))}
+.ig2 .c3{grid-template-columns:repeat(3,minmax(0,1fr))}
+.ig2 .c4{grid-template-columns:repeat(4,minmax(0,1fr))}
+.ig2 .card{padding:24px 26px 26px;border:1.5px solid var(--line);border-top:3px solid var(--acc);border-radius:14px;background:linear-gradient(var(--card),var(--card)),linear-gradient(var(--bg),var(--bg))}
+.ig2 .card.hl{background:linear-gradient(var(--hl),var(--hl)),linear-gradient(var(--bg),var(--bg));border-color:var(--acc)}
+.ig2 .c-k{display:block;font:500 19px 'JetBrains Mono',monospace;letter-spacing:.14em;text-transform:uppercase;color:var(--acc2);margin-bottom:10px}
+.ig2 .c-t{display:block;font:600 29px/1.15 'Space Grotesk',sans-serif;margin-bottom:8px}
+.ig2 .c-d{display:block;font-size:22px;line-height:1.38;color:var(--mute)}
+
+/* A timeline: a line with a dot for each moment */
+.ig2 .timeline{position:relative;display:flex;flex-direction:column;gap:28px;padding-left:46px}
+.ig2 .timeline:before{content:"";position:absolute;left:10px;top:8px;bottom:8px;width:2px;background:var(--line)}
+.ig2 .t-item{position:relative}
+.ig2 .t-item:before{content:"";position:absolute;left:-46px;top:6px;width:16px;height:16px;border-radius:50%;background:var(--bg);border:3px solid var(--acc)}
+.ig2 .t-when{display:block;font:500 19px 'JetBrains Mono',monospace;letter-spacing:.14em;text-transform:uppercase;color:var(--acc2);margin-bottom:6px}
+.ig2 .t-t{display:block;font:600 30px/1.15 'Space Grotesk',sans-serif;margin-bottom:6px}
+.ig2 .t-d{display:block;font-size:23px;line-height:1.38;color:var(--mute)}
+
+/* A hierarchy: one box over the boxes it leads to, joined by lines */
+.ig2 .tree{display:flex;flex-direction:column;align-items:center}
+.ig2 .t-root{width:560px;max-width:100%;text-align:center;position:relative}
+.ig2 .t-root:after{content:"";position:absolute;left:50%;top:100%;width:2px;height:28px;margin-left:-1px;background:var(--acc)}
+.ig2 .t-kids{display:flex;gap:24px;width:100%;margin-top:28px}
+.ig2 .t-kids>*{flex:1;min-width:0;position:relative;margin-top:28px}
+.ig2 .t-kids>*:before{content:"";position:absolute;left:50%;top:-28px;width:2px;height:28px;margin-left:-1px;background:var(--acc)}
+.ig2 .t-kids>*:after{content:"";position:absolute;top:-28px;left:-12px;right:-12px;height:2px;background:var(--acc)}
+.ig2 .t-kids>*:first-child:after{left:50%}
+.ig2 .t-kids>*:last-child:after{right:50%}
+.ig2 .t-kids>*:only-child:after{display:none}
+
+/* Two sides compared */
+.ig2 .compare{display:grid;grid-template-columns:1fr 1fr;border:1.5px solid var(--line);border-radius:14px;overflow:hidden}
+.ig2 .side{padding:26px 32px 22px;background:linear-gradient(var(--card),var(--card)),linear-gradient(var(--bg),var(--bg))}
+.ig2 .side+.side{border-left:1.5px solid var(--line)}
+.ig2 .side.hl{background:linear-gradient(var(--hl),var(--hl)),linear-gradient(var(--bg),var(--bg))}
+.ig2 .s-h{display:block;font:600 32px/1.15 'Space Grotesk',sans-serif;margin-bottom:14px}
+.ig2 .side ul{list-style:none}
+.ig2 .side li{position:relative;padding:13px 0 13px 30px;border-top:1px solid var(--line);font-size:23px;line-height:1.35;color:var(--mute)}
+.ig2 .side li:before{content:"";position:absolute;left:2px;top:23px;width:10px;height:10px;border-radius:50%;background:var(--acc)}
+
+/* Big numbers */
+.ig2 .stats{display:grid;grid-auto-flow:column;grid-auto-columns:minmax(0,1fr);gap:22px}
+.ig2 .stat{padding:20px 26px 22px;border-left:3px solid var(--acc);background:linear-gradient(var(--card),var(--card)),linear-gradient(var(--bg),var(--bg));border-radius:0 12px 12px 0}
+.ig2 .s-n{display:block;font:600 62px/1 'Space Grotesk',sans-serif;letter-spacing:-.02em}
+.ig2 .s-l{display:block;font-size:22px;line-height:1.35;color:var(--mute);margin-top:8px}
+
+.ig2 .cols{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:40px}
+.ig2 .pills{display:flex;flex-wrap:wrap;gap:12px}
+.ig2 .pill{padding:10px 22px;border:1.5px solid var(--line);border-radius:999px;font-size:22px;background:linear-gradient(var(--card),var(--card)),linear-gradient(var(--bg),var(--bg))}
+.ig2 .callout{padding:24px 30px;border-left:4px solid var(--acc);background:linear-gradient(var(--hl),var(--hl)),linear-gradient(var(--bg),var(--bg));border-radius:0 12px 12px 0;font-size:27px;line-height:1.38}
+
+/* Wide: the title and the takeaway in a column, the body beside them */
+.ig2.landscape .ig-main{top:70px;bottom:104px;flex-direction:row;gap:60px}
+.ig2.landscape .ig-left{display:flex;flex-direction:column;justify-content:space-between;gap:28px;width:600px;flex:none;min-height:0}
+.ig2.landscape .ig-head{min-height:0}
+.ig2.landscape h1{font-size:58px;max-width:none}
+.ig2.landscape .ig-sub{font-size:27px;max-width:none}
+.ig2.landscape .ig-figs{position:static;width:100%;margin-top:34px;gap:14px}
+.ig2.landscape .fig-v{font-size:50px}
+.ig2.landscape .ig-body{order:0;width:1380px;flex:none}
+.ig2.landscape .ig-take{grid-template-columns:1fr;gap:18px;padding:30px 34px}
+.ig2.landscape .ig-take .q{font-size:35px}
+.ig2.landscape .ig-take ul{font-size:22px}

backend/app/infographics/templates/design.html.j2

@@ -0,0 +1,22 @@
+{#- The frame of an AI-composed infographic. `body` is sanitised markup (design.sanitize_body); every other value is autoescaped. -#}
+<link rel="stylesheet" href="{{ fonts }}">
+<style>{{ css }}</style>
+<div class="ig2 look-{{ look }} {{ layout }}" style="--w:{{ w }}px;--h:{{ h }}px;--fit:{{ fit }};{{ vars_css }}">
+  <svg class="ig-bg" width="{{ w }}" height="{{ h }}" viewBox="0 0 {{ w }} {{ h }}">{{ stars }}</svg>
+  <div class="ig-main">
+    <div class="ig-left">
+      <header class="ig-head">
+        <div class="ig-eyebrow">{{ c.eyebrow }}</div>
+        <h1>{{ c.title }}</h1>
+        <p class="ig-sub">{{ c.subtitle }}</p>
+        {% if c.key_figures %}<div class="ig-figs">{% for f in c.key_figures %}<div class="fig"><span class="fig-v">{{ f.value }}</span><span class="fig-l">{{ f.label }}</span></div>{% endfor %}</div>{% endif %}
+      </header>
+      <div class="ig-take">
+        <div class="q">{{ c.rule }}</div>
+        <ul>{% for n in c.notes %}<li>{{ n }}</li>{% endfor %}</ul>
+      </div>
+    </div>
+    <div class="ig-body"><div class="ig-inner">{{ body }}</div></div>
+  </div>
+  <div class="ig-mark">{{ logo }}<span>notestack.ai</span></div>
+</div>

backend/app/infographics/themes.py

@@ -19,8 +19,11 @@ THEMES: dict[str, Theme] = {t.id: t for t in (
     Theme("moonbase", "Moon Base", "A lunar surface, a dome and a flag", ("#2b2b30", "#d8d2c4", "#c8745f")),
     Theme("observatory", "Observatory", "A telescope under a chart of stars", ("#14282b", "#8fc2b4", "#d6b66a")),
 )}
-DEFAULT_THEME = "launch"
+DEFAULT_THEME = "midnight"  # the look new infographics start with (the six above are the older fixed themes)
 
 
 def theme_id(value: object) -> str:
-    return value if isinstance(value, str) and value in THEMES else DEFAULT_THEME
+    """A look (design.LOOKS) or one of the older themes; the default when unknown."""
+    from app.infographics.design import LOOKS
+
+    return value if isinstance(value, str) and (value in LOOKS or value in THEMES) else DEFAULT_THEME

backend/app/llm/provider.py

@@ -49,7 +49,7 @@ def main_lm(effort: str | None = None) -> dspy.LM:
 
 @lru_cache
 def fast_lm() -> dspy.LM:
-    return _build(settings.llm_fast_model, effort="low", temperature=None, max_tokens=8000)
+    return _build(settings.llm_fast_model, effort="low", temperature=None, max_tokens=settings.llm_fast_max_tokens)
 
 
 REPORT_MAX_TOKENS = 16000  # a few pages of prose plus GLM's low-effort reasoning
@@ -62,6 +62,27 @@ def report_lm() -> dspy.LM:
                   max_tokens=REPORT_MAX_TOKENS)
 
 
+REPORT_PLAN_MAX_TOKENS = 6000  # the blueprint: a storyline and up to eight short section plans plus reasoning
+REPORT_SECTION_MAX_TOKENS = 4000  # one section (two to five paragraphs) plus reasoning, a quarter of a whole report's cap
+
+
+def _report_model() -> str:
+    return settings.llm_report_model or settings.llm_fast_model
+
+
+@lru_cache
+def report_plan_lm() -> dspy.LM:
+    """Plans a themed report (the blueprint). The report model, with a small cap: the answer is a plan, not prose."""
+    return _build(_report_model(), effort="low", temperature=None, max_tokens=REPORT_PLAN_MAX_TOKENS)
+
+
+@lru_cache
+def report_section_lm(retry: bool = False) -> dspy.LM:
+    """Writes one section of a themed report. retry=True is the second try after a failed one: a different temperature,
+    so it is a fresh call and not the failed one again from the cache."""
+    return _build(_report_model(), effort="low", temperature=0.7 if retry else None, max_tokens=REPORT_SECTION_MAX_TOKENS)
+
+
 @lru_cache
 def triage_lm() -> dspy.LM:
     """Cheap model that decides whether a chat message needs research at all."""
@@ -82,6 +103,16 @@ def focus_lm(retry: bool = False) -> dspy.LM:
                   max_tokens=FOCUS_MAX_TOKENS)
 
 
+SUGGEST_TEMPLATES_MAX_TOKENS = 2000  # four short template ideas plus GLM's low-effort reasoning
+
+
+@lru_cache
+def suggest_templates_lm() -> dspy.LM:
+    """The Create report dialog's suggested templates: four short template ideas out, so a tighter cap than the
+    general fast_lm keeps this quick without cutting into GLM's reasoning headroom."""
+    return _build(settings.llm_fast_model, effort="low", temperature=None, max_tokens=SUGGEST_TEMPLATES_MAX_TOKENS)
+
+
 def configure_default() -> None:
     dspy.configure(lm=main_lm(), adapter=dspy.JSONAdapter())
 

backend/app/llm/run.py

@@ -77,7 +77,8 @@ def predict_many(
 def _record(db: Session, workspace_id: uuid.UUID, job: Job | None, signature: type[dspy.Signature], lm: dspy.LM,
             usage, calls: list[tuple[dict, dict]]) -> None:
     if usage.total:
-        record_usage(db, workspace_id=workspace_id, kind="llm", provider=provider_name(), model=settings.llm_model,
+        model = getattr(lm, "model", None) or settings.llm_model  # the model that answered, not always the main one
+        record_usage(db, workspace_id=workspace_id, kind="llm", provider=provider_name(model), model=model,
                      quantity=usage.total, unit="tokens", cost_usd=usage.cost_usd, job=job)
     workspace = db.get(Workspace, workspace_id)
     for inputs, outputs in calls:

backend/app/llm/signatures.py

@@ -451,6 +451,58 @@ class PlanInfographic(dspy.Signature):
     infographic: InfographicOut = dspy.OutputField()
 
 
+class KeyFigure(BaseModel):
+    value: str = Field(description="A number or a very short fact, at most 12 characters, e.g. 30%, 8 steps, Layer 1")
+    label: str = Field(description="What it counts or means, at most 6 words")
+
+
+class DesignOut(BaseModel):
+    title: str = Field(description="At most 8 words")
+    subtitle: str = Field(description="One line, at most 16 words")
+    eyebrow: str = Field(description="A 1 to 3 word topic label, e.g. Agent engineering")
+    key_figures: list[KeyFigure] = Field(description="The 2 or 3 most important numbers or short facts in the material, shown beside the title")
+    body_html: str = Field(description="The body of the page: 2 to 4 sections of HTML built only from the allowed components")
+    rule: str = Field(description="The one takeaway, at most 14 words")
+    notes: list[str] = Field(description="Exactly 3 supporting notes, at most 14 words each")
+
+
+class DesignInfographic(dspy.Signature):
+    """Design a one page infographic from the material. You write the body of the page as HTML, composed ONLY from the
+    components below, choosing the ones that make the content clearest (a flow chart for steps, a timeline for events,
+    a hierarchy for a whole and its parts, a comparison for two options, cards for the key ideas, stats for numbers that
+    matter). Use only what the material says. When a layout style is asked for, build the page around it. When a focus is
+    given, make it about that, as far as the material covers it. The page is black, blue and white and printed on a
+    poster: plain text only, short lines, no markdown, no emoji, no quotes around names, no em dashes. Write in the
+    language of the material.
+
+    HTML rules: tags allowed are div, section, span, p, ul, ol, li, strong, em, small, br. Use no style, id, href, src,
+    script, svg, img or any attribute except class. Only these class names exist: sec, sec-t, lead, hl, node, n-no, n-t,
+    n-d, flow, col, grid, c2, c3, c4, card, c-k, c-t, c-d, timeline, t-item, t-when, t-t, t-d, tree, t-root, t-kids,
+    compare, side, s-h, stats, stat, s-n, s-l, cols, pills, pill, callout.
+
+    Every part sits in a section: <section class="sec"><div class="sec-t">Short heading</div> COMPONENT </section>.
+    Components:
+    - Flow chart, left to right, at most 4 boxes, arrows are drawn between them for you:
+      <div class="flow"><div class="node"><span class="n-no">01</span><span class="n-t">Title</span><span class="n-d">One short line</span></div> ...</div>
+      Add class hl to the box that matters most (class="node hl"). For more than 4 steps use two flows, or a vertical one: class="flow col" (at most 5 boxes).
+    - Cards in a grid of 2, 3 or 4 columns (class grid c2, grid c3 or grid c4, at most 8 cards):
+      <div class="grid c3"><div class="card"><span class="c-k">Kicker</span><span class="c-t">Title</span><span class="c-d">One short line</span></div> ...</div>
+    - Timeline, at most 5 moments: <div class="timeline"><div class="t-item"><span class="t-when">When</span><span class="t-t">Title</span><span class="t-d">One short line</span></div> ...</div>
+    - Hierarchy, one root and 2 to 4 children: <div class="tree"><div class="node hl t-root"><span class="n-t">Whole</span><span class="n-d">Line</span></div><div class="t-kids"><div class="node"><span class="n-t">Part</span><span class="n-d">Line</span></div> ...</div></div>
+    - Comparison of two sides, 2 to 4 short lines each: <div class="compare"><div class="side"><span class="s-h">Side A</span><ul><li>Line</li></ul></div><div class="side hl"><span class="s-h">Side B</span><ul><li>Line</li></ul></div></div>
+    - Numbers, 2 to 4: <div class="stats"><div class="stat"><span class="s-n">30%</span><span class="s-l">What it counts</span></div> ...</div>
+    - A callout sentence: <div class="callout">One sentence.</div>
+    The key_figures are separate from the body and are shown beside the title, so do not repeat them in the body.
+    Lengths: a box or card title at most 4 words, its line at most 14 words, a heading at most 4 words. Use two to four
+    sections, and mix components (never one grid of 8 cards alone). Keep the whole page short enough for one poster."""
+
+    title: str = dspy.InputField(desc="What the material is from")
+    material: list[str] = dspy.InputField(desc="Ideas from posts, or chat transcripts")
+    focus: str = dspy.InputField(desc="What the infographic should be about, or (none)")
+    layout_style: str = dspy.InputField(desc="The kind of layout asked for, or auto to choose")
+    design: DesignOut = dspy.OutputField()
+
+
 class WriteReport(dspy.Signature):
     """Write the report the reader describes, using only the material given: never add facts from outside it. Follow
     the reader's instructions on structure, style, tone and length. Write in Markdown that starts straight at the
@@ -460,9 +512,8 @@ class WriteReport(dspy.Signature):
     unless the reader's instructions ask for a different structure. Every "## " heading is specific and descriptive of
     what is in that section (for example "The Context Window Bottleneck: Moving Beyond Basic Summarization", never
     "Section 2" or "Details"), and they read in a logical order. Use "### " inside sections, lists, and tables where
-    they help. When the material has numbered lines (FILE blocks), put a [n] marker after each
-    claim it supports and list each n in citations with the path and lines it came from; when it is a chat transcript,
-    write no markers and no citations. Do not write a quiz, flashcards, practice or review questions into the report
+    they help. Do not cite: no [n] markers, footnotes, references or lists of sources in the text; the app shows the reader
+    the sources separately. Do not write a quiz, flashcards, practice or review questions into the report
     unless the reader's instructions
     ask for one: those are offered to the reader separately as visuals they can add. Never write about visuals, mind
     maps, flashcards, quizzes, tables or timelines in the text, and never suggest or describe one, even if the
@@ -476,7 +527,6 @@ class WriteReport(dspy.Signature):
     language: str = dspy.InputField(desc="The language to write in")
     report_title: str = dspy.OutputField(desc="A specific title, 3 to 10 words")
     markdown: str = dspy.OutputField(desc="The report body in Markdown, starting at the first ## heading")
-    citations: list[FileCitation] = dspy.OutputField()
 
 
 class PlannedEmbed(BaseModel):
@@ -521,22 +571,166 @@ class WriteReportSection(dspy.Signature):
     and length; a section is two to five short paragraphs, or a list or table where that reads better. Do not write
     quiz questions, flashcards or
     review questions in the section (those are offered to the reader separately). Do not repeat
-    the heading, and do not use "#" or "##" headings; "###" subheadings are fine. When the material has numbered lines (FILE
-    blocks), put a [n] marker after each claim it supports and list each n in citations with the path and lines; when it
-    is a chat transcript, write no markers and no citations. Never write about visuals, mind maps, flashcards,
+    the heading, and do not use "#" or "##" headings; "###" subheadings are fine. Do not cite: no [n] markers, footnotes,
+    references or lists of sources in the text; the app shows the reader the sources separately. Never write about visuals, mind maps, flashcards,
     quizzes, tables or timelines in the text, and never suggest or describe one, even if the reader's instructions
     mention them: the app offers those to the reader separately, so write only the report itself. Write in the
     language requested. Never use em dashes."""
 
+    # The parts every section shares come first, so a provider can reuse them across the section calls.
     title: str = dspy.InputField()
     report_title: str = dspy.InputField()
+    request: str = dspy.InputField(desc="What the reader wants the report to be")
+    material: list[str] = dspy.InputField(desc="Posts with path and numbered lines, or chat transcripts")
+    language: str = dspy.InputField()
     heading: str = dspy.InputField()
     brief: str = dspy.InputField(desc="What this section should say")
+    markdown: str = dspy.OutputField(desc="The section body in Markdown, without its heading")
+
+
+# Themed reports: the posts' stored ideas are grouped into themes, a blueprint plans the whole report, and each section is
+# written from its own evidence (pipeline/report_themes.py, report_blueprint.py, report_evidence.py).
+
+
+class ReportTheme(BaseModel):
+    name: str = Field(description="2 to 5 words, a noun phrase a reader would recognise")
+    summary: str = Field(description="One or two plain sentences on what the posts say about it")
+    idea_ids: list[str] = Field(description="Ids of the ideas that belong to this theme, exactly as listed")
+    matches_request: bool = Field(False, description="True when the reader's request asks for this theme to be focused on, "
+                                                     "included or given more room")
+
+
+class ThemeRelation(BaseModel):
+    a: str = Field(description="A theme name, exactly as you wrote it in themes")
+    b: str = Field(description="Another theme name, exactly as you wrote it in themes")
+    relation: str = Field(description="How they connect, one short sentence")
+
+
+class GroupThemes(dspy.Signature):
+    """Group ideas taken from a writer's posts into themes. Every idea belongs to exactly one theme, listed by its id as
+    given. A theme is something several ideas are about, named the way a reader would name it: not a post title, not a
+    single idea, not 'Other'. Choose 4 to 12 themes, fewer when the ideas really are about few things. Also note how themes
+    connect (one cause of another, one the context of another, two sides of a choice), only where the ideas show it.
+    Where the reader's request asks to focus on, include or give more room to something, set matches_request on the theme
+    that is about it. Never use em dashes."""
+
+    title: str = dspy.InputField(desc="The notebook or posts the ideas come from")
+    request: str = dspy.InputField(desc="What the reader wants the report to be")
+    ideas: list[str] = dspy.InputField(desc="One per line: id | label: note")
+    themes: list[ReportTheme] = dspy.OutputField(desc="4 to 12 themes")
+    relations: list[ThemeRelation] = dspy.OutputField(desc="How themes connect, at most 8, only where the ideas show it")
+
+
+class BlueprintSection(BaseModel):
+    heading: str = Field(description="2 to 8 words, specific to what the section says, never generic")
+    role: Literal["intro", "body", "synthesis", "summary"] = Field(
+        "body", description="intro opens the report, summary closes it, synthesis ties two or more themes together, "
+                            "body covers its themes")
+    brief: str = Field(description="What this section says, 1 to 2 sentences")
+    theme_ids: list[str] = Field(description="Ids of the themes this section is about, as listed. A body section's "
+                                             "themes belong to it alone")
+    depth: Literal["deep", "standard", "brief"] = Field("standard", description="deep for a theme with a large weight "
+                                                        "or one the reader asked for, brief for a small one")
+    must_cover: list[str] = Field(description="2 to 4 specific points this section has to make")
+    embed: PlannedEmbed | None = Field(None, description="A visual to suggest right after this section, only where it "
+                                                         "really helps")
+
+
+class ThemeConnection(BaseModel):
+    theme_ids: list[str] = Field(description="Two or more theme ids that are connected")
+    relation: str = Field(description="How they connect, one short sentence")
+    section: str = Field(description="Heading of the section that should bridge them")
+
+
+class PlanReportBlueprint(dspy.Signature):
+    """Plan a whole report before any of it is written. The themes come with their weight (their share of what the posts say), a
+    tier (major, medium, minor) and whether the reader asked for them. Give every major theme and every theme the reader asked
+    for its own section with depth deep, in an order that builds an argument; put the ones the reader asked for early. Put medium
+    themes in standard sections and merge the minor ones into one brief section. A theme is the subject of exactly one section.
+    Write a storyline of three to five sentences: what the report as a whole says, so every section can be written toward it.
+    Start with an intro section and end with a summary section unless the reader asks for something else; the summary must
+    restate the major themes. Where themes connect, say which section bridges them (connections), and add a synthesis section
+    only if the connection deserves its own. Each section's must_cover is specific to it and shares no point with another.
+    Choose at most max_sections sections. Never plan a section that is itself a quiz, flashcards, practice questions or a
+    mind map: when allowed_kinds is not empty those are visuals, suggested through a section's embed (at most four in all, each
+    kind once, only the kinds allowed): a mind_map to show how many linked themes fit together (at most one, near the start or
+    the end), flashcards to learn a set of terms or facts, a quiz to check understanding after a dense section, and an
+    infographic, a one page picture of the main ideas, which every report with that kind allowed should suggest once, on the
+    last section before the summary. Suggest the others only where they really help; none where plain prose reads better. When
+    allowed_kinds is empty, plan no embeds, and a quiz or practice section is allowed only if the reader's request asks for
+    one. Use only what the themes cover. Never use em dashes."""
+
+    title: str = dspy.InputField()
+    request: str = dspy.InputField(desc="What the reader wants the report to be")
+    themes: list[str] = dspy.InputField(desc="One per line: id | name | tier | weight | posts | asked for: summary")
+    relations: list[str] = dspy.InputField(desc="One per line: id <-> id: how they connect, or empty")
+    allowed_kinds: list[str] = dspy.InputField(desc="The visuals that may be suggested, or empty")
+    language: str = dspy.InputField()
+    max_sections: int = dspy.InputField(desc="The most sections to plan")
+    report_title: str = dspy.OutputField(desc="A specific title, 3 to 10 words")
+    storyline: str = dspy.OutputField(desc="Three to five sentences on what the whole report says")
+    sections: list[BlueprintSection] = dspy.OutputField(desc="2 to max_sections sections, in reading order")
+    connections: list[ThemeConnection] = dspy.OutputField(desc="At most 6")
+
+
+class WriteThemedSection(dspy.Signature):
+    """Write one section of a report that has already been planned. You are given the storyline of the whole report and its
+    outline: write toward that storyline, and do not make points that another section owns (covered_elsewhere). Make the
+    points in must_cover. Use only the evidence given; never add facts from outside it. The evidence is grouped by post and
+    comes in two forms: text as the writer wrote it, and short summaries of other posts. Combine what
+    several posts say about the same thing: say where they agree, where they differ or where the writer changed their mind,
+    and never go through the posts one at a time. Where a connection is listed, bridge the themes in a sentence or two.
+    Follow the reader's instructions on style, tone and length; a section is two to five short paragraphs, or a list or table
+    where that reads better; a deep section is longer than a brief one. For role intro, say what the report covers and why it
+    matters; for role summary, restate the main points of the sections as written, in the digests given. Do not repeat the
+    heading, and do not use "#" or "##" headings; "###" subheadings are fine. Do not cite: no [n] markers, footnotes,
+    references or lists of sources in the text, and do not name the posts or their paths; the app shows the reader the
+    sources separately. Do not write quiz questions, flashcards or review questions, and never write about visuals, mind maps,
+    flashcards, quizzes, tables or timelines in the text: the app offers those separately. Write in the language requested.
+    Never use em dashes."""
+
+    # What every section shares comes first, so a provider can reuse it across the section calls.
+    storyline: str = dspy.InputField(desc="What the whole report says")
+    outline: list[str] = dspy.InputField(desc="Every section: number, heading, role, what it says")
+    connections: list[str] = dspy.InputField(desc="Themes the report connects, with the section that bridges them")
     request: str = dspy.InputField(desc="What the reader wants the report to be")
-    material: list[str] = dspy.InputField(desc="Posts with path and numbered lines, or chat transcripts")
     language: str = dspy.InputField()
+    heading: str = dspy.InputField(desc="The section to write")
+    role: str = dspy.InputField(desc="intro, body, synthesis or summary")
+    depth: str = dspy.InputField(desc="deep, standard or brief")
+    brief: str = dspy.InputField(desc="What this section should say")
+    must_cover: list[str] = dspy.InputField(desc="Points this section has to make")
+    covered_elsewhere: list[str] = dspy.InputField(desc="Points other sections own: do not make them here")
+    bridges: list[str] = dspy.InputField(desc="Connections this section should bridge, or empty")
+    evidence: list[str] = dspy.InputField(desc="Posts as written, short summaries of other posts, or digests of the written "
+                                               "sections")
     markdown: str = dspy.OutputField(desc="The section body in Markdown, without its heading")
-    citations: list[FileCitation] = dspy.OutputField()
+
+
+class WriteThemedReport(dspy.Signature):
+    """Write a whole report that has already been planned, in one piece. You are given the storyline and the planned sections
+    in order, each with what it must cover and the evidence for it. Write in Markdown that starts straight at the first "## "
+    heading (the title goes in report_title, not in the text), with one "## " heading per planned section, in that order and
+    with those headings, and write toward the storyline. Make each section's must_cover points, and do not make a point
+    another section owns. Use only the evidence given; never add facts from outside it. The evidence is grouped by post and
+    comes in two forms: text as the writer wrote it, and short summaries of other posts. Combine what
+    several posts say about the same thing: say where they agree, where they differ or where the writer changed their mind,
+    and never go through the posts one at a time. Where a connection is listed, bridge the themes. A deep section is longer
+    than a brief one. Follow the reader's instructions on style, tone and length. Use "### " inside sections, lists, and tables
+    where they help. Do not cite: no [n] markers, footnotes, references or lists of sources in the text, and do not name the
+    posts or their paths; the app shows the reader the sources separately. Do not write a quiz, flashcards, practice or
+    review questions unless the reader's instructions ask for one, and never write about visuals, mind maps, flashcards or
+    timelines in the text. Write in the language requested. Never use em dashes."""
+
+    title: str = dspy.InputField(desc="The notebook or posts the evidence comes from")
+    storyline: str = dspy.InputField(desc="What the whole report says")
+    request: str = dspy.InputField(desc="What the reader wants the report to be")
+    language: str = dspy.InputField(desc="The language to write in")
+    connections: list[str] = dspy.InputField(desc="Themes the report connects, with the section that bridges them")
+    sections: list[str] = dspy.InputField(desc="The planned sections in order: heading, role, depth, what it says, must cover, "
+                                               "followed by its evidence")
+    report_title: str = dspy.OutputField(desc="A specific title, 3 to 10 words")
+    markdown: str = dspy.OutputField(desc="The report body in Markdown, starting at the first ## heading")
 
 
 class ReportTemplateIdea(BaseModel):
@@ -552,7 +746,7 @@ class SuggestReportTemplates(dspy.Signature):
     a practical how-to, a comparison, a beginner primer and an in-depth analysis, when the material supports them).
     Never use em dashes."""
 
-    items: list[str] = dspy.InputField(desc="The title and the start of each source")
+    items: list[str] = dspy.InputField(desc="Each source: its title, topics and key ideas, or the start of it")
     topic: str = dspy.InputField(desc="What the reader wants the report to be about, or (none)")
     templates: list[ReportTemplateIdea] = dspy.OutputField(desc="Exactly 4")
 

backend/app/models/__init__.py

@@ -1,5 +1,6 @@
 from app.models.content import (
     Artifact,
+    ArtifactFeedback,
     ArtifactShare,
     B2VCustomVoice,
     B2VStyle,
@@ -17,6 +18,7 @@ from app.models.content import (
     MessageFeedback,
     Notebook,
     NotebookDocument,
+    ReportTemplateCache,
     SocialAccount,
     Source,
     Topic,
@@ -37,6 +39,7 @@ __all__ = [
     "SupportConversation",
     "SupportMessage",
     "Artifact",
+    "ArtifactFeedback",
     "ArtifactShare",
     "AuthProvider",
     "B2VCustomVoice",
@@ -58,6 +61,7 @@ __all__ = [
     "Notebook",
     "PlanRecord",
     "NotebookDocument",
+    "ReportTemplateCache",
     "SocialAccount",
     "Source",
     "Subscription",

backend/app/models/content.py

@@ -155,6 +155,37 @@ class MessageFeedback(IdMixin, TimestampMixin, Base):
     recall: Mapped[dict] = mapped_column(JSON, default=dict)  # copy of Message.recall_json
 
 
+class ArtifactFeedback(IdMixin, TimestampMixin, Base):
+    """A thumbs up or down on one generated report, quiz, flashcard set or infographic, with a snapshot of what it was
+    (title, prompt, sources, options) taken when it was rated, so the ratings stay useful after it is deleted (the link
+    then becomes null). One row per artifact and person: rating again replaces it."""
+
+    __tablename__ = "artifact_feedback"
+    __table_args__ = (UniqueConstraint("artifact_id", "user_id"),)
+
+    workspace_id: Mapped[uuid.UUID] = _ws_fk()
+    artifact_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, ForeignKey("artifacts.id", ondelete="SET NULL"), index=True)
+    user_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, ForeignKey("users.id", ondelete="SET NULL"))
+    artifact_type: Mapped[str] = mapped_column(String(30))
+    rating: Mapped[int] = mapped_column(Integer)  # 1 up, -1 down
+    reasons: Mapped[list] = mapped_column(JSON, default=list)
+    comment: Mapped[str | None] = mapped_column(Text)
+    snapshot: Mapped[dict] = mapped_column(JSON, default=dict)
+
+
+class ReportTemplateCache(IdMixin, TimestampMixin, Base):
+    """The suggested report templates already written for some sources, found again by a hash of exactly what the AI read (each
+    post's stored ideas and topics, and the topic). The same input gives the same templates at once, so the Create report dialog
+    is instant, and a changed post (its ideas change) gets new ones."""
+
+    __tablename__ = "report_template_cache"
+    __table_args__ = (UniqueConstraint("workspace_id", "key"),)
+
+    workspace_id: Mapped[uuid.UUID] = _ws_fk()
+    key: Mapped[str] = mapped_column(String(64))
+    templates: Mapped[list] = mapped_column(JSON, default=list)
+
+
 class WorkspaceMemory(IdMixin, TimestampMixin, Base):
     """The writer's standing notes, one row per workspace, shared by every notebook and chat.
 

backend/app/models/user.py

@@ -13,7 +13,6 @@ from sqlalchemy import (
     String,
     UniqueConstraint,
     Uuid,
-    false,
     true,
 )
 from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -56,7 +55,6 @@ class Workspace(IdMixin, TimestampMixin, Base):
     training_opt_in: Mapped[bool] = mapped_column(Boolean, default=False)
     # Off: no report of this workspace can be shared by link, and existing links stop working.
     allow_public_links: Mapped[bool] = mapped_column(Boolean, default=True, server_default=true())
-    brand_json: Mapped[dict] = mapped_column(JSON, default=dict)
 
     members: Mapped[list["WorkspaceMember"]] = relationship(back_populates="workspace")
 
@@ -112,11 +110,10 @@ class PlanRecord(TimestampMixin, Base):
     launch_kits: Mapped[int] = mapped_column(Integer, default=0)  # -1 = unlimited
     reports: Mapped[int] = mapped_column(Integer, default=0, server_default="0")  # per month, -1 = unlimited
     infographics: Mapped[int] = mapped_column(Integer, default=0, server_default="0")  # per month, -1 = unlimited
+    audio_overviews: Mapped[int] = mapped_column(Integer, default=-1, server_default="-1")  # allowed (in total on Free); -1 = no cap
     voice_cloning: Mapped[bool] = mapped_column(Boolean, default=False)
-    brand_kit: Mapped[bool] = mapped_column(Boolean, default=False)
     features: Mapped[list] = mapped_column(JSON, default=list)  # bullet points on the pricing page
     # Server defaults: 0006 seeds this table without these columns on a database 0001 built from the models.
-    video_premium: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false())  # premium options
     # False: `videos` is a lifetime total that never refills (Free). True: per month.
     videos_monthly: Mapped[bool] = mapped_column(Boolean, default=True, server_default=true())
     video_limits: Mapped[dict | None] = mapped_column(JSON, default=dict)  # see plans.VIDEO_LIMITS

backend/app/pipeline/flashcards.py

@@ -8,6 +8,7 @@ from app.llm import run
 from app.llm.provider import fast_lm
 from app.llm.signatures import GenerateFlashcards
 from app.models import Artifact, Job
+from app.pipeline import idea_pool
 from app.pipeline.generate import NothingToDo
 from app.pipeline.material import Material, load_material
 from app.services.jobs import update_job
@@ -16,20 +17,29 @@ SAME_LANGUAGE = "the same language as the sources"  # no language picked: write
 CARD_COUNTS = {"fewer": 8, "standard": 16, "more": 24}
 
 
+def _card(c: dict, refs) -> dict | None:
+    """One generated card as the deck plays it, or None when it has no front or no back."""
+    front, back = (c.get("front") or "").strip(), (c.get("back") or "").strip()
+    if not front or not back:
+        return None
+    return {"front": front[:300], "back": back[:800], "sources": refs(c.get("sources"))}
+
+
 def make_cards(db: Session, workspace_id: uuid.UUID, material: Material, *, topic: str = "", count: int = 16,
                difficulty: str = "medium", language: str = SAME_LANGUAGE, job: Job | None = None) -> list[dict]:
     """Cards about the material: a front, a back and verified source lines. Repeats and empty cards are dropped."""
-    out = run.predict(GenerateFlashcards, db=db, workspace_id=workspace_id, job=job, title=material.title,
-                      material=material.text, topic=topic or "(none)", difficulty=difficulty, count=count,
-                      language=language.strip() or SAME_LANGUAGE, lm=fast_lm())
+    inputs = dict(title=material.title, topic=topic or "(none)", difficulty=difficulty, language=language.strip() or SAME_LANGUAGE)
+    # Posts with their ideas stored (the Mind Constellation's): one call over a random sample of those ideas, not the posts' text.
+    text = idea_pool.sample(db, workspace_id, material.docs, topic, chats=material.text) if material.mode == "ideas" else material.text
+    out = run.predict(GenerateFlashcards, db=db, workspace_id=workspace_id, job=job, material=text, count=count,
+                      lm=fast_lm(), **inputs)
     cards, seen = [], set()
-    for c in out.get("cards") or []:
-        front, back = (c.get("front") or "").strip(), (c.get("back") or "").strip()
-        key = front.lower()
-        if not front or not back or key in seen:
+    for raw in out.get("cards") or []:
+        card = _card(raw, material.refs)
+        if not card or card["front"].lower() in seen:
             continue
-        seen.add(key)
-        cards.append({"front": front[:300], "back": back[:800], "sources": material.refs(c.get("sources"))})
+        seen.add(card["front"].lower())
+        cards.append(card)
     return cards[:count]
 
 
@@ -38,7 +48,8 @@ def build_flashcards(db: Session, job: Job, artifact: Artifact) -> dict:
     count = CARD_COUNTS.get(params.get("count") or "standard", CARD_COUNTS["standard"])
     topic = (params.get("topic") or "").strip()
     difficulty = params.get("difficulty") if params.get("difficulty") in ("easy", "medium", "hard") else "medium"
-    material = load_material(db, artifact.workspace_id, artifact.notebook_id, params, job=job, what="flashcard set")
+    material = load_material(db, artifact.workspace_id, artifact.notebook_id, params, job=job, what="flashcard set",
+                             large_ok=True, ideas_always=True)
     update_job(db, job, progress=0.4, message="Writing the cards")
     cards = make_cards(db, artifact.workspace_id, material, topic=topic, count=count, difficulty=difficulty,
                        language=params.get("language") or SAME_LANGUAGE, job=job)

backend/app/pipeline/generate.py

@@ -112,22 +112,39 @@ def ideas_fresh(d) -> bool:
     return bool(md.get("ideas")) and md.get("ideas_hash") == (d.content_hash or "")
 
 
-def extract_ideas(db: Session, job: Job | None, workspace_id: uuid.UUID, doc_ids: list[str] | None = None) -> dict:
+def extract_ideas(db: Session, job: Job | None, workspace_id: uuid.UUID, doc_ids: list[str] | None = None, *,
+                  parallel: bool = False, limit: int | None = None) -> dict:
     """Read each post that has no current ideas once and keep what it says on the post (label, note, points and
-    the lines they came from). Maps are arranged from these, so they never re-read the posts."""
+    the lines they came from). Maps are arranged from these, so they never re-read the posts. `parallel` reads the
+    posts together instead of one after the other (a report that needs many at once); `limit` reads at most that many."""
     docs = workspace_docs(db, workspace_id, [uuid.UUID(str(i)) for i in doc_ids] if doc_ids is not None else None)
     todo = [d for d in docs if not ideas_fresh(d)]
+    if limit is not None:
+        todo = todo[:limit]
     corpus = Corpus(workspace_id)
     lm = fast_lm()
     done = 0
-    for i, d in enumerate(todo):
+    answers: list[dict | None] | None = None
+    if parallel and todo:
         if job:
-            update_job(db, job, progress=0.05 + 0.85 * i / max(len(todo), 1), message=f"Reading {d.title[:70]}")
-        try:
-            out = run.predict(ExtractIdeas, db=db, workspace_id=workspace_id, job=job, lm=lm, title=d.title,
-                              passage="\n\n".join(passages_for(corpus, [d], budget_chars=IDEAS_BUDGET)))
-        except Exception:
-            continue
+            update_job(db, job, progress=0.05, message=f"Reading {len(todo)} posts for their key ideas")
+        answers = run.predict_many(
+            ExtractIdeas,
+            [dict(title=d.title, passage="\n\n".join(passages_for(corpus, [d], budget_chars=IDEAS_BUDGET))) for d in todo],
+            db=db, workspace_id=workspace_id, lm=lm)
+    for i, d in enumerate(todo):
+        if answers is not None:
+            out = answers[i]
+            if out is None:
+                continue
+        else:
+            if job:
+                update_job(db, job, progress=0.05 + 0.85 * i / max(len(todo), 1), message=f"Reading {d.title[:70]}")
+            try:
+                out = run.predict(ExtractIdeas, db=db, workspace_id=workspace_id, job=job, lm=lm, title=d.title,
+                                  passage="\n\n".join(passages_for(corpus, [d], budget_chars=IDEAS_BUDGET)))
+            except Exception:
+                continue
         tools = tools_for(corpus, [d])
 
         def keep(items, tools=tools, d=d) -> list[dict]:

backend/app/pipeline/idea_pool.py

@@ -0,0 +1,85 @@
+"""Flashcards (always) and quizzes (above ARTIFACT_MAX_POSTS posts) from the ideas already extracted from each post, the same stored
+ideas the Mind Constellation is drawn from. Instead of sending post text, a small random sample of those ideas goes to the model: each
+as one line with the post and the lines it came from, so the model can cite them and the existing check can verify the lines.
+Nothing here extracts anything and no post text is read: a post with no stored ideas yet is represented by a short opening, at most
+a few of them. A sample of ARTIFACT_IDEA_SAMPLE ideas is about 1,500 tokens whatever the size of the selection."""
+
+import math
+import random
+import re
+import uuid
+
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.corpus import Corpus
+from app.models import Document
+from app.pipeline.generate import ideas_fresh
+from app.pipeline.passages import passages_for
+
+IDEAS_PER_POST = 2  # so a few posts cannot fill the sample (more when the selection is small, to fill it)
+OPENING_BUDGET = 800  # characters of a post's opening, for a post with no stored ideas
+MAX_OPENINGS = 3  # posts with no stored ideas represented by their opening
+CHAT_BUDGET = 12_000  # characters of chat transcripts kept (the end of each) when chats are picked as well
+
+FORMAT_NOTE = ("Each item below is a key idea from one post, written as: IDEA [path lines a-b] label: note. The path and lines say "
+               "where in the post the idea comes from. When a question or card comes from an idea, cite that path and those lines.")
+
+_STOP = {"the", "and", "for", "with", "that", "this", "from", "how", "what", "why", "are", "was", "about", "your", "you",
+         "into", "over", "does", "can", "not", "but", "all", "any", "its", "their", "them", "will", "have", "has"}
+
+
+def _tokens(text: str) -> set[str]:
+    return {w for w in re.findall(r"[a-z0-9]{3,}", (text or "").lower()) if w not in _STOP}
+
+
+def has_ideas(docs: list[Document]) -> int:
+    """How many of the posts have their ideas stored and current."""
+    return sum(1 for d in docs if ideas_fresh(d))
+
+
+def _line(doc: Document, idea: dict) -> str:
+    """One stored idea as one line, tagged with where it came from (its first cited range, when it has one)."""
+    where = doc.path
+    for ref in idea.get("sources") or []:
+        try:
+            a, b = int(ref["line_start"]), int(ref["line_end"])
+        except (KeyError, TypeError, ValueError):
+            continue
+        where = f"{doc.path} lines {a}-{max(a, b)}"
+        break
+    label, note = (idea.get("label") or "").strip(), (idea.get("note") or "").strip()
+    return f"IDEA [{where}] {label}: {note}"
+
+
+def sample(db: Session, workspace_id: uuid.UUID, docs: list[Document], topic: str = "", chats: list[str] | None = None,
+           size: int | None = None, rng: random.Random | None = None) -> list[str]:
+    """The material for one call: the format note, a random sample of the posts' stored ideas, a few short openings for posts with
+    none, and the end of any chats. With a topic, ideas that match it come first. A fresh draw each time unless `rng` is given."""
+    rng = rng or random.Random()
+    size = size or max(settings.artifact_idea_sample, 1)
+    wanted = _tokens(topic)
+    have = [d for d in docs if ideas_fresh(d)]
+    per_post = max(IDEAS_PER_POST, math.ceil(size / max(len(have), 1)))
+    matching: list[tuple[Document, dict]] = []
+    rest: list[tuple[Document, dict]] = []
+    for d in have:
+        mine = [i for i in (d.metadata_json or {}).get("ideas") or [] if (i.get("label") or "").strip()]
+        rng.shuffle(mine)
+        for idea in mine[:per_post]:
+            hit = wanted and wanted & _tokens(" ".join([idea.get("label", ""), idea.get("note", ""), d.title]))
+            (matching if hit else rest).append((d, idea))
+    rng.shuffle(matching)
+    rng.shuffle(rest)
+    chosen = (matching + rest)[:size]
+    rng.shuffle(chosen)  # not grouped by relevance or by post
+    out = [FORMAT_NOTE] + [_line(d, i) for d, i in chosen]
+    bare = [d for d in docs if not ideas_fresh(d)]
+    if bare:
+        corpus = Corpus(workspace_id)
+        for d in rng.sample(bare, min(MAX_OPENINGS, len(bare))):
+            out.extend(passages_for(corpus, [d], budget_chars=OPENING_BUDGET))
+    if chats:
+        share = CHAT_BUDGET // len(chats)
+        out += [t[-share:] for t in chats]
+    return out

backend/app/pipeline/infographic.py

@@ -9,15 +9,18 @@ from sqlalchemy import select
 from sqlalchemy.orm import Session
 
 from app.corpus import Corpus
-from app.infographics.content import clamp
+from concurrent.futures import ThreadPoolExecutor
+
+from app.infographics.design import LOOKS, STYLES, build_content, render_design
+from app.infographics.image import measure_fit
 from app.infographics.render import build_html
 from app.infographics.themes import theme_id
 from app.llm import run
 from app.llm.provider import fast_lm
-from app.llm.signatures import PlanInfographic
+from app.llm.signatures import DesignInfographic
 from app.models import Artifact, Chat, Document, Job, Notebook
 from app.pipeline.generate import NothingToDo, extract_ideas, ideas_fresh
-from app.pipeline.material import MAX_POSTS, chat_material
+from app.pipeline.material import chat_material, max_posts
 from app.pipeline.passages import notebook_docs, passages_for, workspace_docs
 from app.services.jobs import update_job
 
@@ -67,7 +70,7 @@ def write_content(db: Session, job: Job, artifact: Artifact, params: dict) -> tu
         docs = workspace_docs(db, ws, picked)
     elif not chat_ids and nb:
         docs = notebook_docs(db, nb.id)
-    docs = docs[:MAX_POSTS]
+    docs = docs[:max_posts()]
     chats, chat_blocks = _chat_text(db, ws, chat_ids) if chat_ids else ([], [])
     if not docs and not chat_blocks:
         raise NothingToDo("Pick at least one post or chat for the infographic.")
@@ -76,12 +79,14 @@ def write_content(db: Session, job: Job, artifact: Artifact, params: dict) -> tu
     material = (_ideas_text(db, ws, docs, job) if docs else []) + chat_blocks
     prompt = (params.get("instructions") or "").strip()
     title = docs[0].title if docs else (chats[0].title or "Chat")
-    update_job(db, job, progress=0.35, message="Writing the infographic")
+    update_job(db, job, progress=0.35, message="Designing the infographic")
+    style = params.get("style") if params.get("style") in STYLES else "auto"
     content = None
     for _ in range(2):  # one retry when the model gives too little to draw
-        out = run.predict(PlanInfographic, db=db, workspace_id=ws, job=job, title=title, material=material,
-                          focus=prompt or "(none)", lm=fast_lm())
-        content = clamp(out.get("infographic") or {})
+        out = run.predict(DesignInfographic, db=db, workspace_id=ws, job=job, title=title, material=material,
+                          focus=prompt or "(none)", layout_style=f"{style}: {STYLES[style]}" if style != "auto" else "auto",
+                          lm=fast_lm())
+        content = build_content(out.get("design") or {})
         if content:
             break
     if not content:
@@ -101,6 +106,8 @@ def build_infographic(db: Session, job: Job, artifact: Artifact) -> dict:
     params = job.params
     theme = theme_id(params.get("theme"))
     content, source, _title = write_content(db, job, artifact, params)
+    update_job(db, job, progress=0.8, message="Fitting it to the page")
+    content["fit"] = fit_scales(theme, content)
     artifact.content_json = {**(artifact.content_json or {}), "title": f"Infographic: {content['title']}"[:140],
                              "theme": theme, "prompt": (params.get("instructions") or "").strip(), "source": source,
                              "content": content}
@@ -110,7 +117,22 @@ def build_infographic(db: Session, job: Job, artifact: Artifact) -> dict:
     return {"artifact_id": str(artifact.id)}
 
 
+def fit_scales(theme: str, content: dict) -> dict:
+    """The scale at which the body fits the page, tall and wide (measured in Chrome once, kept with the content)."""
+    look = theme if theme in LOOKS else "midnight"
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        wide, tall = (pool.submit(measure_fit, render_design(look, content, layout, fit=1.0), layout)
+                      for layout in ("landscape", "portrait"))
+        return {"landscape": wide.result(), "portrait": tall.result()}
+
+
 def infographic_html(content: dict, layout: str = "portrait") -> str | None:
-    """The page for a finished infographic's stored text and theme, tall or wide (None when it has none yet)."""
+    """The page for a finished infographic's stored content and look, tall or wide (None when it has none yet). Older
+    infographics (fixed text in a premade theme) still draw in that theme."""
     text = content.get("content")
-    return build_html(theme_id(content.get("theme")), text, layout) if text else None
+    if not text:
+        return None
+    theme = theme_id(content.get("theme"))
+    if "body_html" in text:
+        return render_design(theme if theme in LOOKS else "midnight", text, layout)
+    return build_html(theme, text, layout) if "items" in text else None

backend/app/pipeline/ingest.py

@@ -630,29 +630,39 @@ def store_locked(db: Session, source: Source, entries: list[FeedEntry]) -> tuple
 
 def ingest_source(db: Session, source: Source, job: Job, max_posts: int) -> dict:
     source.sync_status = "syncing"
-    update_job(db, job, status="running", progress=0.02, message="Contacting your feed")
     fetch_limit = max(SCAN_CAP, max_posts)
+    # Live counts the Sources page shows while this runs: found on the site, fetched (read), indexed.
+    live = {"stage": "fetching", "found": 0, "fetched": 0, "fetch_total": 0, "indexed": 0, "index_total": 0}
+
+    def report(progress: float | None = None, message: str | None = None, **counts) -> None:
+        live.update(counts)
+        update_job(db, job, progress=progress, message=message, result=dict(live))
+
+    update_job(db, job, status="running", progress=0.02, message="Contacting your feed", result=dict(live))
 
     if source.feed_url.startswith(SITE_PREFIX):
         update_job(db, job, message="Mapping your site")
         entries = crawl_site(
             source.feed_url.removeprefix(SITE_PREFIX), fetch_limit,
-            on_found=lambda n: update_job(db, job, progress=0.05, message=f"Found {n} pages, reading them"),
-            on_post=lambda done, total: update_job(db, job, progress=0.05 + 0.05 * done / total,
-                                                   message=f"Read {done} of {total} pages"),
-            read_limit=max_posts,
+            on_found=lambda n: report(0.05, f"Found {n} pages, reading them", found=n),
+            on_post=lambda done, total: report(0.05 + 0.05 * done / total, f"Read {done} of {total} pages",
+                                               fetched=done, fetch_total=total),
+            read_limit=max(max_posts, 1),
         )
         title = None
     else:
         title, entries = fetch_feed(source.feed_url, fetch_limit)
+        report(found=len(entries), fetched=len(entries), fetch_total=len(entries))
     source.title = source.title or title
     if source.platform == "substack" and len(entries) < fetch_limit:
         site = f"{urlparse(source.feed_url).scheme}://{urlparse(source.feed_url).netloc}"
-        update_job(db, job, progress=0.05, message="Reading your Substack archive")
+        report(0.05, "Reading your Substack archive")
         try:
             entries += fetch_substack_archive(
                 site, {e.url for e in entries}, fetch_limit,
-                on_page=lambda n: update_job(db, job, message=f"Found {len(entries) + n} posts in the archive"),
+                on_page=lambda n: report(message=f"Found {len(entries) + n} posts in the archive",
+                                         found=len(entries) + n, fetched=len(entries) + n,
+                                         fetch_total=len(entries) + n),
                 body_budget=max(0, max_posts - len(entries)),
             )
         except (httpx.HTTPError, ValueError):
@@ -662,9 +672,9 @@ def ingest_source(db: Session, source: Source, job: Job, max_posts: int) -> dict
     available, beyond = entries[:max_posts], entries[max_posts:]
 
     def progress(done: int, total: int, post_title: str) -> None:
-        update_job(db, job, progress=0.1 + 0.7 * done / total, message=f"{post_title[:80]} is in orbit")
+        report(0.1 + 0.7 * done / total, f"{post_title[:80]} is in orbit", stage="indexing", indexed=done)
 
-    update_job(db, job, progress=0.1, message=f"Indexing {len(available)} posts")
+    report(0.1, f"Indexing {len(available)} posts", stage="indexing", found=len(entries), index_total=len(available))
     indexed, skipped, changed = store_entries(db, source, available, progress)
 
     locked = 0

backend/app/pipeline/material.py

@@ -9,16 +9,36 @@ from dataclasses import dataclass, field
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
+from app.config import settings
 from app.corpus import Corpus
-from app.models import Chat, Document, Job, Message, Notebook
+from app.models import Chat, Document, Job, Message, Notebook, Workspace
 from app.pipeline.generate import NothingToDo
 from app.pipeline.passages import ScopedTools, notebook_docs, passages_for, tools_for, verify_refs, workspace_docs
 from app.services.jobs import update_job
+from app.services.plans import effective_plan
 
-MAX_POSTS = 20  # posts one artifact reads; the passage budget is shared between them
 CHAT_BUDGET = 60_000  # characters of chat transcripts
 
 
+def max_posts() -> int:
+    """Posts a report or infographic reads, and the most a quiz or flashcard set reads straight from the posts (ARTIFACT_MAX_POSTS);
+    the passage budget is shared between them."""
+    return max(settings.artifact_max_posts, 1)
+
+
+def use_ideas(docs: list[Document], *, large_ok: bool, ideas_always: bool) -> bool:
+    """Read the posts through their stored ideas instead of their text? Only when most of them have ideas stored (the rest are
+    represented by a short opening). A selection above max_posts() (quiz and flashcards) always, when it can; a flashcard set
+    (`ideas_always`) whatever the size. Otherwise, or with too few ideas stored, the posts' text is read as it always was."""
+    from app.pipeline.idea_pool import has_ideas
+
+    if not docs or not (large_ok or ideas_always):
+        return False
+    if len(docs) > max_posts() or ideas_always:
+        return has_ideas(docs) * 2 >= len(docs)
+    return False
+
+
 @dataclass
 class Material:
     title: str  # the notebook, the post, or the first chat's title
@@ -28,6 +48,9 @@ class Material:
     # Turns the model's {path, line_start, line_end} items into verified refs with their quote. Chats give none.
     refs: Callable[[list | None], list[dict]] = lambda _items: []
     tools: ScopedTools | None = None  # reads back cited lines (posts only)
+    # "passages": `text` holds the posts' passages. "ideas": more posts than max_posts(), so `text` holds only chat transcripts and
+    # the posts are read through their stored ideas (pipeline/idea_pool.py).
+    mode: str = "passages"
 
     @property
     def is_chat(self) -> bool:
@@ -52,7 +75,8 @@ def chat_material(db: Session, chats: list[Chat]) -> list[str]:
 
 
 def load_material(db: Session, workspace_id: uuid.UUID, notebook_id: uuid.UUID | None, params: dict, *,
-                  job: Job | None = None, what: str = "quiz") -> Material:
+                  job: Job | None = None, what: str = "quiz", large_ok: bool = False,
+                  ideas_always: bool = False, passage_budget: int | None = None) -> Material:
     """The source a job's params name: posts (`document_ids`, a notebook's, the archive's or a single post), chats
     (`chat_ids`), or both together. With `chat_ids` and no posts it reads only the chats; with neither it reads the whole
     notebook. Raises NothingToDo when there is nothing to read."""
@@ -65,7 +89,11 @@ def load_material(db: Session, workspace_id: uuid.UUID, notebook_id: uuid.UUID |
         docs = workspace_docs(db, workspace_id, picked)
     elif not chat_ids and nb:
         docs = notebook_docs(db, nb.id)
-    docs = docs[:MAX_POSTS]
+    # Quiz and flashcards take as many posts as the plan indexes: their ideas are sampled. The rest read max_posts().
+    docs = docs[:effective_plan(db, db.get(Workspace, workspace_id)).indexed_posts] if large_ok else docs[:max_posts()]
+    ideas_mode = use_ideas(docs, large_ok=large_ok, ideas_always=ideas_always)
+    if large_ok and not ideas_mode:
+        docs = docs[:max_posts()]  # their ideas are not stored yet: read the first posts' text, as a small selection is
 
     chats: list[Chat] = []
     chat_text: list[str] = []
@@ -84,8 +112,9 @@ def load_material(db: Session, workspace_id: uuid.UUID, notebook_id: uuid.UUID |
     if docs:
         corpus = Corpus(workspace_id)
         if job:
-            update_job(db, job, progress=0.1, message=f"Reading {len(docs)} posts")
-        text = passages_for(corpus, docs)
+            update_job(db, job, progress=0.1, message=f"Gathering the key ideas of {len(docs)} posts" if ideas_mode
+                       else f"Reading {len(docs)} posts")
+        text = [] if ideas_mode else passages_for(corpus, docs, budget_chars=passage_budget)
         tools = tools_for(corpus, docs)
         by_path = {d.path: d for d in docs}
 
@@ -110,4 +139,5 @@ def load_material(db: Session, workspace_id: uuid.UUID, notebook_id: uuid.UUID |
     if chat_text:
         source["chat_ids"] = [str(c.id) for c in chats]
     source["count"] = len(docs) + (len(chats) if chat_text else 0)
-    return Material(title=title, text=text + chat_text, docs=docs, refs=refs, tools=tools, source=source)
+    return Material(title=title, text=text + chat_text, docs=docs, refs=refs, tools=tools, source=source,
+                    mode="ideas" if ideas_mode else "passages")

backend/app/pipeline/media.py

@@ -17,7 +17,7 @@ from app.pipeline.passages import notebook_docs, passages_for, tools_for, verify
 from app.pipeline.report import SAME_LANGUAGE
 from app.services import tts
 from app.services.jobs import record_usage, update_job
-from app.services.renderer import PermanentJobError, brand_props, request_render
+from app.services.renderer import PermanentJobError, request_render
 from app.services.storage import keys, storage
 
 
@@ -160,14 +160,13 @@ def make_audiogram(db: Session, job: Job, artifact: Artifact) -> None:
     source = db.get(Artifact, uuid.UUID(job.params["audio_artifact_id"]))
     if not source or source.type != "audio_overview" or not source.storage_key:
         raise PermanentJobError("Pick a finished audio overview for the audiogram.")
-    workspace = db.get(Workspace, artifact.workspace_id)
     content = source.content_json or {}
     duration = float(content.get("duration_s") or 30)
     segments = [{"speaker": s["speaker"], "text": s["text"], "startMs": int(s["start"] * 1000),
                  "endMs": int(s["end"] * 1000)} for s in content.get("segments", [])]
     props = {"title": (content.get("title") or "Audio overview")[:140],
              "audioUrl": storage.presign_get(source.storage_key, ttl=6 * 3600),
-             "durationS": round(min(duration, 600), 2), "segments": segments, "brand": brand_props(workspace)}
+             "durationS": round(min(duration, 600), 2), "segments": segments }
     artifact.content_json = {**(artifact.content_json or {}), "style": "audiogram", "composition": "AudiogramSquare",
                              "audio_artifact_id": str(source.id), "duration_s": props["durationS"]}
     artifact.status = "rendering"
@@ -184,12 +183,11 @@ def make_audiogram(db: Session, job: Job, artifact: Artifact) -> None:
 def author_line(db: Session, workspace_id: uuid.UUID) -> str:
     ws = db.get(Workspace, workspace_id)
     owner = db.get(User, ws.owner_id) if ws else None
-    return (ws.brand_json or {}).get("name") or (owner.name if owner and owner.name else "") or "Notestack"
+    return (owner.name if owner and owner.name else "") or "Notestack"
 
 
 def make_quote_card(db: Session, job: Job, artifact: Artifact) -> None:
     content = artifact.content_json or {}
-    workspace = db.get(Workspace, artifact.workspace_id)
     if not content.get("quote"):
         docs = artifact_docs(db, artifact)
         if not docs:
@@ -212,8 +210,7 @@ def make_quote_card(db: Session, job: Job, artifact: Artifact) -> None:
     if doc and doc.source_id:
         src = db.get(Source, doc.source_id)
         publication = src.title if src else None
-    props = {"quote": content["quote"][:380], "author": author_line(db, artifact.workspace_id),
-             "brand": brand_props(workspace)}
+    props = {"quote": content["quote"][:380], "author": author_line(db, artifact.workspace_id)}
     if publication or (doc and doc.title):
         props["publication"] = (publication or doc.title)[:120]
     artifact.content_json = {**content, "composition": "QuoteCard"}
@@ -226,10 +223,8 @@ def render_carousel(db: Session, job: Job, artifact: Artifact) -> None:
     slides = (artifact.content_json or {}).get("slides") or []
     if not slides:
         raise PermanentJobError("This carousel has no slides.")
-    workspace = db.get(Workspace, artifact.workspace_id)
-    brand = brand_props(workspace)
     stills = [{"heading": s.get("heading", "")[:120], "body": s.get("body", "")[:500], "index": i + 1,
-               "total": len(slides), "brand": brand} for i, s in enumerate(slides)]
+               "total": len(slides)} for i, s in enumerate(slides)]
     artifact.status = "rendering"
     db.commit()
     request_render(job, artifact, "CarouselSlide", stills=stills)

backend/app/pipeline/passages.py

@@ -6,10 +6,11 @@ from types import SimpleNamespace
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
+from app.config import settings
 from app.corpus import Corpus, CorpusError, ScopedTools
 from app.models import Document, NotebookDocument, VoiceProfile
 
-DEFAULT_BUDGET = 120_000  # characters across all passages; GLM-5.3 has room, this keeps cost sane
+DEFAULT_BUDGET = 120_000  # the default of ARTIFACT_PASSAGE_BUDGET: characters across all passages; this keeps cost sane
 
 
 def notebook_docs(db: Session, notebook_id: uuid.UUID) -> list[Document]:
@@ -28,10 +29,26 @@ def workspace_docs(db: Session, workspace_id: uuid.UUID, ids: list[uuid.UUID] |
     return list(db.scalars(q.order_by(Document.published_at.desc())).all())
 
 
-def passages_for(corpus: Corpus, docs: list[Document], budget_chars: int = DEFAULT_BUDGET) -> list[str]:
+def _body_start(lines: list[str]) -> int:
+    """The index of the first line after the post file's header (the `---` block of title, url, date and source, and the repeated
+    `# Title`): none of it is worth sending, the `FILE path (title)` line already names the post."""
+    i = 0
+    if lines and lines[0].strip() == "---":
+        end = next((k for k in range(1, min(len(lines), 12)) if lines[k].strip() == "---"), None)
+        if end is not None:
+            i = end + 1
+    while i < len(lines) and not lines[i].strip():
+        i += 1
+    if i < len(lines) and lines[i].startswith("# "):
+        i += 1
+    return i
+
+
+def passages_for(corpus: Corpus, docs: list[Document], budget_chars: int | None = None) -> list[str]:
     """One string per post: a header line then `n| text` numbered lines (same numbering as read()).
     The budget is split evenly, so one long post cannot crowd out the rest."""
     corpus.sync()
+    budget_chars = budget_chars or settings.artifact_passage_budget
     per_doc = max(budget_chars // max(len(docs), 1), 1500)
     out = []
     for d in docs:
@@ -40,8 +57,9 @@ def passages_for(corpus: Corpus, docs: list[Document], budget_chars: int = DEFAU
         except CorpusError:
             continue
         body, used = [], 0
+        first = _body_start(lines)  # lines keep their number in the file, so a cited line is still the right one
         for n, line in enumerate(lines, start=1):
-            if not line.strip():
+            if n <= first or not line.strip():
                 continue
             entry = f"{n}| {line}"
             if used + len(entry) > per_doc:

backend/app/pipeline/quiz.py

@@ -4,10 +4,12 @@ import uuid
 
 from sqlalchemy.orm import Session
 
+from app.config import settings
 from app.llm import run
 from app.llm.provider import fast_lm
 from app.llm.signatures import GenerateQuiz
 from app.models import Artifact, Job
+from app.pipeline import idea_pool
 from app.pipeline.generate import NothingToDo
 from app.pipeline.material import Material, load_material
 from app.services.jobs import update_job
@@ -51,12 +53,14 @@ def make_questions(db: Session, workspace_id: uuid.UUID, material: Material, *,
                    language: str = SAME_LANGUAGE, job: Job | None = None) -> list[dict]:
     """Playable questions about the material (fewer than `count` when the model gave unusable ones)."""
     types = [t for t in types or [] if t in QUESTION_TYPES] or list(QUESTION_TYPES[:2])
-    out = run.predict(GenerateQuiz, db=db, workspace_id=workspace_id, job=job, title=material.title,
-                      material=material.text, topic=topic or "(none)", difficulty=difficulty, question_types=types,
-                      count=count, language=language.strip() or SAME_LANGUAGE, lm=fast_lm())
-    cleaned = [c for q in out.get("questions") or [] if (c := _clean(q, material.refs))]
     # A multiple select with a single right answer was turned into a multiple choice: fine when that was asked for too.
     allowed = set(types) | ({"multiple_choice"} if "multiple_select" in types else set())
+    inputs = dict(title=material.title, topic=topic or "(none)", difficulty=difficulty, question_types=types,
+                  language=language.strip() or SAME_LANGUAGE)
+    # Many posts: one call over a random sample of the ideas already extracted from them (idea_pool), not the posts' text.
+    text = idea_pool.sample(db, workspace_id, material.docs, topic, chats=material.text) if material.mode == "ideas" else material.text
+    out = run.predict(GenerateQuiz, db=db, workspace_id=workspace_id, job=job, material=text, count=count, lm=fast_lm(), **inputs)
+    cleaned = [c for q in out.get("questions") or [] if (c := _clean(q, material.refs))]
     return [c for c in cleaned if c["type"] in allowed][:count]
 
 
@@ -66,7 +70,8 @@ def build_quiz(db: Session, job: Job, artifact: Artifact) -> dict:
     count = QUESTION_COUNTS.get(params.get("count") or "standard", QUESTION_COUNTS["standard"])
     difficulty = params.get("difficulty") if params.get("difficulty") in DIFFICULTIES else "medium"
     topic = (params.get("topic") or "").strip()
-    material = load_material(db, artifact.workspace_id, artifact.notebook_id, params, job=job, what="quiz")
+    material = load_material(db, artifact.workspace_id, artifact.notebook_id, params, job=job, what="quiz", large_ok=True,
+                             passage_budget=settings.artifact_quiz_passage_budget)
     update_job(db, job, progress=0.4, message="Writing the questions")
     questions = make_questions(db, artifact.workspace_id, material, topic=topic, difficulty=difficulty, types=types,
                                count=count, language=params.get("language") or SAME_LANGUAGE, job=job)

backend/app/pipeline/report.py

@@ -1,42 +1,46 @@
 """Reports: a written document (or an interactive one, with visuals) made from a notebook's posts or chats.
 
 Every report is stored as `blocks`, even a plain document (a few prose blocks), so the same page can show, extend and
-share either kind. Each block is {"id", "type", ...}: `prose` has Markdown `text` whose [n] markers point into the
-report's `citations`."""
+share either kind. Each block is {"id", "type", ...}: `prose` has Markdown `text`. A report cites nothing: it carries the
+list of the posts it was made from (`sources`) and the instructions it was written to (`instructions`), which the page shows
+together in a drop-down. (`citations` stays in the content, empty, for the older reports that have them.)"""
 
 import logging
 import re
+import time
 import uuid
+from dataclasses import dataclass, field
 
+from sqlalchemy import select
 from sqlalchemy.orm import Session
 
+from app.config import settings
+from app.corpus import Corpus
 from app.infographics.themes import theme_id
 from app.llm import run
-from app.llm.provider import fast_lm, report_lm
+from app.llm.provider import report_lm, report_section_lm
 from app.llm.signatures import (
     PlanInteractiveReport,
-    SuggestEmbeds,
     WriteReport,
     WriteReportSection,
+    WriteThemedReport,
+    WriteThemedSection,
 )
-from app.models import Artifact, Job
+from app.models import Artifact, Document, Job, Notebook
+from app.pipeline import report_blueprint, report_evidence, report_themes
 from app.pipeline.flashcards import make_cards
 from app.pipeline.generate import NothingToDo, mind_map_content
 from app.pipeline.material import Material, load_material
-from app.pipeline.passages import as_citations
 from app.pipeline.quiz import make_questions
-from app.pipeline.research import verify_citations
 from app.services.artifacts import start_artifact
 from app.services.jobs import update_job
 from app.services.renderer import PermanentJobError
 from app.services.report_templates import default_instructions
 
-SUGGEST_KINDS = ("mind_map", "flashcards", "quiz", "infographic")  # what the AI suggests for a document report
-EMBED_KINDS = SUGGEST_KINDS  # every visual a reader can add with one click
+EMBED_KINDS = ("mind_map", "flashcards", "quiz", "infographic")  # every visual a reader can add with one click
 MAX_PLANNED_VISUALS = 4  # suggestions an interactive report's plan can make
 MAX_SECTIONS = 8
 SAME_LANGUAGE = "the same language as the sources"  # no language picked: write as the material is written
-MAX_SUGGESTIONS = 4
 BLOCK_CARDS, BLOCK_QUESTIONS = 8, 5  # a visual inside a report is a taste of the full thing
 
 log = logging.getLogger(__name__)
@@ -46,7 +50,6 @@ MARKER = re.compile(r"\s*\[\d+\](?:\[\d+\])*")
 
 QUIZ_HEADING = re.compile(r"\b(quiz|test yourself|check your understanding|practice questions|review questions|self[- ]?check)\b", re.I)
 FLASHCARD_HEADING = re.compile(r"\bflash ?cards?\b", re.I)
-ASKED_FOR_VISUAL = re.compile(r"\b(quiz|flash ?cards?|practice questions)\b", re.I)
 
 
 def visual_kind(heading: str) -> str | None:
@@ -62,24 +65,6 @@ MOVED_BRIEF = {"quiz": "Check understanding of the key ideas in this report.",
                "flashcards": "Learn the key terms and facts from this report."}
 
 
-def pull_visual_sections(blocks: list[dict], kinds: list[str]) -> tuple[list[dict], list[dict]]:
-    """A section written as a quiz or flashcards is not part of the text: it is taken out and offered as a suggestion
-    after the section before it, for the reader to add. Never takes out every section."""
-    kept: list[dict] = []
-    moved: list[tuple[int, str]] = []
-    for b in blocks:
-        first = b["text"].split("\n", 1)[0] if b["type"] == "prose" else ""
-        kind = visual_kind(first.lstrip("# ").strip()) if first.startswith("#") else None
-        if kind and kind in kinds and kept:
-            moved.append((len(kept) - 1, kind))
-        else:
-            kept.append(b)
-    number_blocks(kept)
-    return kept, [{"id": f"s{i}", "after_block_id": kept[idx]["id"], "kind": kind, "brief": MOVED_BRIEF[kind],
-                   "why": "Offered here for you to add, not written into the text."}
-                  for i, (idx, kind) in enumerate(moved, start=1)]
-
-
 def merge_suggestions(*groups: list[dict]) -> list[dict]:
     """Suggestions from several places, one per section and kind, renumbered."""
     seen, out = set(), []
@@ -92,6 +77,36 @@ def merge_suggestions(*groups: list[dict]) -> list[dict]:
     return out[:MAX_PLANNED_VISUALS + 2]
 
 
+FALLBACK_BRIEFS = {
+    "infographic": ("A one page picture of the main ideas of this report.", "A quick picture of the whole report."),
+    "flashcards": (MOVED_BRIEF["flashcards"], "Cards to learn the key terms and facts."),
+    "quiz": (MOVED_BRIEF["quiz"], "A short quiz to check understanding."),
+    "mind_map": ("How the main ideas of this report connect.", "See how the ideas fit together."),
+}
+MIN_SUGGESTIONS = 3  # a report always offers at least this many visuals (fewer only when fewer kinds are allowed)
+
+
+def fill_suggestions(suggestions: list[dict], blocks: list[dict], kinds: list[str]) -> list[dict]:
+    """An interactive report always offers something to add (a document never does). Whatever the plan or the writer suggested is
+    kept; then an infographic is added if there is none, and flashcards, a quiz and a mind map in turn until there are
+    MIN_SUGGESTIONS, each after a different section. Plain code, so it never depends on the model remembering to suggest."""
+    prose = [b for b in blocks if b.get("type") == "prose"]
+    if not prose:
+        return suggestions
+    inner = prose[1:-1] or prose  # not the introduction or the summary, when there are others
+    place = {"infographic": inner[-1]["id"], "flashcards": inner[0]["id"], "quiz": inner[len(inner) // 2]["id"],
+             "mind_map": prose[0]["id"]}
+    out = list(suggestions)
+    have = {s["kind"] for s in out}
+    for kind in ("infographic", "flashcards", "quiz", "mind_map"):
+        if kind in kinds and kind not in have and (kind == "infographic" or len(out) < MIN_SUGGESTIONS):
+            brief, why = FALLBACK_BRIEFS[kind]
+            out.append({"id": "", "after_block_id": place[kind], "kind": kind, "brief": brief, "why": why})
+            have.add(kind)
+            log.info("report: no %s suggested by the plan, offering one after %s", kind, place[kind])
+    return out
+
+
 def split_prose(markdown: str) -> list[dict]:
     """Markdown cut into prose blocks at each "## " heading, so a block is one section and can be followed by a visual."""
     parts = re.split(r"(?m)^(?=## )", markdown.strip())
@@ -108,81 +123,29 @@ def number_blocks(blocks: list[dict]) -> list[dict]:
     return blocks
 
 
-def citation_rows(cites, material: Material) -> list[dict]:
-    by_path = {d.path: d for d in material.docs}
-    return [{"marker": c.marker, "path": c.path, "line_start": c.line_start, "line_end": c.line_end,
-             "span": c.quote, "title": by_path[c.path].title, "url": by_path[c.path].url,
-             "document_id": str(by_path[c.path].id)}
-            for c in cites if c.path in by_path]
+def source_rows(material: Material) -> list[dict]:
+    """The posts a report was made from, for its Sources list: titles and the posts' own web addresses."""
+    return source_rows_for(material.docs)
 
 
-def source_rows(material: Material) -> list[dict]:
-    """Where the report came from, for the public page's Sources button: titles and the posts' own web addresses."""
-    return [{"title": d.title, "url": d.url, "document_id": str(d.id)} for d in material.docs]
+def source_rows_for(docs: list[Document]) -> list[dict]:
+    return [{"title": d.title, "url": d.url, "document_id": str(d.id)} for d in docs]
 
 
 def write_document(db: Session, workspace_id: uuid.UUID, job: Job | None, material: Material, instructions: str,
-                   language: str) -> tuple[str, str, list[dict]]:
-    """(title, Markdown, citation rows) for a written report. Citations are checked against the posts' lines and
-    renumbered 1..n; a chat has none, so any marker the model wrote is removed."""
+                   language: str) -> tuple[str, str]:
+    """(title, Markdown) for a written report. Nothing is cited; a marker the model wrote anyway is removed."""
     out = run.predict(WriteReport, db=db, workspace_id=workspace_id, job=job, lm=report_lm(), title=material.title,
                       material=material.text, request=instructions, language=language)
     markdown = (out.get("markdown") or "").strip()
     if not markdown:
         raise NothingToDo("The report came back empty. Try again, or describe it differently.")
-    if material.tools is None:
-        return (out.get("report_title") or material.title).strip(), MARKER.sub("", markdown), []
-    text, cites = verify_citations(material.tools, markdown, as_citations(out.get("citations")))
-    return (out.get("report_title") or material.title).strip(), text, citation_rows(cites, material)
-
-
-def allowed_kinds(material_or_source: Material | dict) -> list[str]:
-    """Which visuals fit this source: a mind map is drawn from posts' stored ideas, so a chat gets cards and a quiz."""
-    has_posts = bool(material_or_source.docs) if isinstance(material_or_source, Material) \
-        else bool(material_or_source.get("document_ids"))
-    return [k for k in SUGGEST_KINDS if k != "mind_map" or has_posts]
-
-
-def suggest_embeds(db: Session, workspace_id: uuid.UUID, job: Job | None, blocks: list[dict],
-                   kinds: list[str]) -> list[dict]:
-    """Up to MAX_SUGGESTIONS places for a visual: after which section, what kind, what it covers, why. Never fails
-    the report: no suggestions is a fine answer."""
-    sections = []
-    for b in blocks:
-        if b["type"] != "prose":
-            continue
-        first, _, rest = b["text"].partition("\n")
-        heading = first.lstrip("# ").strip() if first.startswith("#") else ""
-        start = re.sub(r"\s+", " ", rest or first)[:240]
-        sections.append(f"{b['id']} | {heading or 'Introduction'}: {start}")
-    if len(sections) < 2:
-        return []
-    try:
-        out = run.predict(SuggestEmbeds, db=db, workspace_id=workspace_id, job=job, lm=fast_lm(), sections=sections,
-                          allowed_kinds=kinds)
-    except Exception:
-        return []
-    ids = {b["id"] for b in blocks}
-    taken: set[tuple[str, str]] = {(b.get("after") or "", b["type"]) for b in blocks if b["type"] != "prose"}
-    found = []
-    for s in out.get("suggestions") or []:
-        key = (s.get("after_block_id"), s.get("kind"))
-        if s.get("after_block_id") not in ids or s.get("kind") not in kinds or key in taken or not (s.get("brief") or "").strip():
-            continue
-        taken.add(key)
-        found.append({"id": f"s{len(found) + 1}", "after_block_id": s["after_block_id"], "kind": s["kind"],
-                      "brief": s["brief"].strip()[:300], "why": (s.get("why") or "").strip()[:300]})
-    return found[:MAX_SUGGESTIONS]
-
-
-def shift_markers(text: str, offset: int) -> str:
-    """[n] markers renumbered to [n + offset], so sections written apart share one citation list."""
-    return re.sub(r"\[(\d+)\]", lambda m: f"[{int(m.group(1)) + offset}]", text) if offset else text
+    return (out.get("report_title") or material.title).strip(), MARKER.sub("", markdown)
 
 
 def write_interactive(db: Session, workspace_id: uuid.UUID, job: Job, material: Material, instructions: str,
-                      language: str) -> tuple[str, list[dict], list[dict], list[dict]] | None:
-    """(title, blocks, citation rows, suggestions) for an interactive report: a plan of sections, each written on its
+                      language: str) -> tuple[str, list[dict], list[dict]] | None:
+    """(title, blocks, suggestions) for an interactive report: a plan of sections, each written on its
     own (in parallel). The plan also says where a visual would fit; those spots come back as suggestions, and nothing
     is built until the reader clicks Add. None when planning fails, so the caller can write a plain document."""
     kinds = [k for k in EMBED_KINDS if k != "mind_map" or not material.is_chat]
@@ -219,21 +182,13 @@ def write_interactive(db: Session, workspace_id: uuid.UUID, job: Job, material:
         db=db, workspace_id=workspace_id, lm=report_lm())
 
     blocks: list[dict] = []
-    citations: list[dict] = []
     suggestions: list[dict] = []
     used_kinds: set[str] = set()
     anchor: dict[int, str] = {}  # planned section number -> the block it became
     for n, (sec, out) in enumerate(zip(sections, written, strict=True)):
-        body = ((out or {}).get("markdown") or "").strip()
+        body = MARKER.sub("", ((out or {}).get("markdown") or "").strip())
         if not body:
             continue
-        if material.tools is None:
-            body = MARKER.sub("", body)
-        else:
-            body, cites = verify_citations(material.tools, body, as_citations((out or {}).get("citations")))
-            offset = len(citations)
-            body = shift_markers(body, offset)
-            citations += [{**c, "marker": c["marker"] + offset} for c in citation_rows(cites, material)]
         block = {"type": "prose", "text": f"## {sec['heading'].strip()}\n\n{body}"}
         blocks.append(block)
         number_blocks(blocks)
@@ -252,7 +207,275 @@ def write_interactive(db: Session, workspace_id: uuid.UUID, job: Job, material:
         suggestions.append({"id": "", "after_block_id": after, "kind": kind,
                             "brief": (sec.get("brief") or MOVED_BRIEF[kind]).strip()[:300],
                             "why": "Offered here for you to add, not written into the text."})
-    return report_title, blocks, citations, merge_suggestions(suggestions)
+    return report_title, blocks, merge_suggestions(fill_suggestions(suggestions, blocks, kinds))
+
+
+# A report built around themes (report_themes.py, report_blueprint.py, report_evidence.py)
+
+EVIDENCE_TOTAL = 100_000  # characters of evidence in all for a report written in one piece (about 25k tokens)
+DIGEST_CHARS = 300  # of each written section, for the introduction and the summary that are written after them
+
+
+@dataclass
+class Themed:
+    title: str
+    blocks: list[dict]
+    suggestions: list[dict]
+    source: dict
+    sources: list[dict]
+    plan: dict
+    failed: list[str] = field(default_factory=list)
+
+
+@dataclass
+class ThemedRun:
+    """What the steps of a themed report share."""
+    db: Session
+    workspace_id: uuid.UUID
+    job: Job
+    docs: list[Document]
+    corpus: Corpus
+    bp: report_blueprint.Blueprint
+    themes: list[report_themes.Theme]
+    instructions: str
+    language: str
+    title: str
+
+    @property
+    def by_id(self) -> dict[str, report_themes.Theme]:
+        return {t.id: t for t in self.themes}
+
+    def names(self, s: report_blueprint.Section) -> list[str]:
+        by_id = self.by_id
+        return [by_id[i].name for i in s.theme_ids if i in by_id]
+
+    def sections_info(self) -> list[dict]:
+        return [{"heading": s.heading, "role": s.role, "depth": s.depth, "theme_ids": s.theme_ids, "must_cover": s.must_cover}
+                for s in self.bp.sections]
+
+    def finish(self, title: str, blocks: list[dict], suggestions: list[dict], used_posts: list[str],
+               failed: list[str]) -> Themed:
+        """The Sources list is every post the report was made from: the ones read as written and the ones summarised."""
+        used = [d for d in self.docs if d.path in set(used_posts)] or self.docs
+        log.info("report sources: job=%s posts_total=%d posts_used=%d", self.job.id, len(self.docs), len(used))
+        plan = {"planner": "themes", "storyline": self.bp.storyline, "posts_total": len(self.docs), "posts_used": len(used),
+                "themes": [report_themes.theme_summary(t) for t in self.themes], "sections": self.sections_info(),
+                "connections": self.bp.connections}
+        return Themed(title=title, blocks=blocks, suggestions=suggestions, plan=plan, failed=failed,
+                      source={"kind": "posts", "document_ids": [str(d.id) for d in self.docs], "count": len(self.docs),
+                              "read": len(used), "total": len(self.docs)},
+                      sources=source_rows_for(used))
+
+
+def themed_docs(db: Session, artifact: Artifact, params: dict) -> list[Document]:
+    """The posts a themed report would read, or [] when the report is made the usual way: chats, a small selection, or the
+    legacy planner."""
+    if settings.report_planner != "themes" or params.get("chat_ids"):
+        log.info("report route: legacy artifact=%s reason=%s", artifact.id,
+                 "chats picked" if params.get("chat_ids") else f"REPORT_PLANNER={settings.report_planner}")
+        return []
+    docs = report_themes.select_docs(db, artifact.workspace_id, artifact.notebook_id, params)
+    if len(docs) <= report_themes.SMALL_SELECTION:
+        log.info("report route: legacy artifact=%s reason=small selection (%d posts, themes need more than %d)", artifact.id,
+                 len(docs), report_themes.SMALL_SELECTION)
+        return []
+    return docs
+
+
+def themed_title(db: Session, artifact: Artifact, params: dict, docs: list[Document]) -> str:
+    """What the report is about, named the way load_material names its source."""
+    nb = db.get(Notebook, artifact.notebook_id) if artifact.notebook_id else None
+    if nb and not params.get("document_ids"):
+        return nb.title
+    return docs[0].title + (f" + {len(docs) - 1} more" if len(docs) > 1 else "")
+
+
+def _digest(markdown: str) -> str:
+    """The start of a written section, for the sections written after it to know what it says."""
+    text = MARKER.sub("", re.sub(r"(?m)^#+\s*", "", markdown))
+    return re.sub(r"\s+", " ", text).strip()[:DIGEST_CHARS]
+
+
+def _section_header(n: int, s: report_blueprint.Section, names: list[str]) -> str:
+    return (f"SECTION {n}: {s.heading} (role: {s.role}, depth: {s.depth})\nSays: {s.brief}\n"
+            f"Must cover: {'; '.join(s.must_cover) or '(see what it says)'}\nThemes: {', '.join(names) or '(the whole report)'}")
+
+
+def _section_input(ctx: ThemedRun, s: report_blueprint.Section, evidence: list[str]) -> dict:
+    """Everything WriteThemedSection is given. What every section shares comes first in the signature."""
+    must = list(s.must_cover)
+    if s.role == "summary" and ctx.names(s):
+        must.append(f"Restate the main points about: {', '.join(ctx.names(s))}")
+    return {"storyline": ctx.bp.storyline, "outline": report_blueprint.outline(ctx.bp),
+            "connections": report_blueprint.connection_lines(ctx.bp, ctx.themes), "request": ctx.instructions,
+            "language": ctx.language, "heading": s.heading, "role": s.role, "depth": s.depth, "brief": s.brief,
+            "must_cover": must, "covered_elsewhere": s.covered_elsewhere,
+            "bridges": report_blueprint.connection_lines(ctx.bp, ctx.themes, s.heading), "evidence": evidence}
+
+
+def _write_sections(ctx: ThemedRun, inputs: list[dict]) -> list[dict | None]:
+    """WriteThemedSection for each input, together; one that failed or came back empty gets one more try (another temperature)."""
+    kw = {"db": ctx.db, "workspace_id": ctx.workspace_id}
+    started = time.monotonic()
+    outs = run.predict_many(WriteThemedSection, inputs, lm=report_section_lm(), **kw)
+    again = [k for k, o in enumerate(outs) if not ((o or {}).get("markdown") or "").strip()]
+    log.info("report sections written: asked=%d ok=%d failed=%d seconds=%.1f", len(inputs), len(inputs) - len(again), len(again),
+             time.monotonic() - started)
+    if again:
+        log.warning("report sections failed, trying again: %s", [inputs[k]["heading"] for k in again])
+        redo = run.predict_many(WriteThemedSection, [inputs[k] for k in again], lm=report_section_lm(retry=True), **kw)
+        for k, o in zip(again, redo, strict=True):
+            outs[k] = o
+        gone = [inputs[k]["heading"] for k in again if not ((outs[k] or {}).get("markdown") or "").strip()]
+        log.info("report sections retried: asked=%d recovered=%d still_failed=%s", len(again), len(again) - len(gone), gone)
+    return outs
+
+
+def _themed_document(ctx: ThemedRun) -> Themed | None:
+    """The whole report in one piece: the plan and every section's evidence go to one writer, so one voice writes it all."""
+    budgets = report_evidence.scale_budgets(ctx.bp.sections, EVIDENCE_TOTAL)
+    parts: list[str] = []
+    used: list[str] = []
+    for n, s in enumerate(ctx.bp.sections, start=1):
+        head = _section_header(n, s, ctx.names(s))
+        if s.role in ("body", "synthesis"):
+            ev = report_evidence.build(ctx.corpus, s, ctx.by_id, budgets.get(s.heading))
+            used += ev.posts
+            if ev.blocks:
+                head += "\nEvidence:\n" + "\n\n".join(ev.blocks)
+        parts.append(head)
+    report_themes.progress(ctx.db, ctx.job, 0.5, "Writing the report")
+    log.info("report document: writing in one piece sections=%d evidence_chars=%d (budget %d) posts_used=%d", len(parts),
+             sum(len(p) for p in parts), EVIDENCE_TOTAL, len(set(used)))
+    started = time.monotonic()
+    out = run.predict(WriteThemedReport, db=ctx.db, workspace_id=ctx.workspace_id, job=ctx.job, lm=report_lm(),
+                      title=ctx.title, storyline=ctx.bp.storyline, request=ctx.instructions, language=ctx.language,
+                      connections=report_blueprint.connection_lines(ctx.bp, ctx.themes), sections=parts)
+    markdown = MARKER.sub("", (out.get("markdown") or "").strip())
+    if not markdown:
+        log.warning("report document came back empty, reading the posts instead")
+        return None
+    log.info("report document written: chars=%d seconds=%.1f", len(markdown), time.monotonic() - started)
+    return ctx.finish((out.get("report_title") or ctx.bp.title).strip(), number_blocks(split_prose(markdown)), [], used, [])
+
+
+def _themed_interactive(ctx: ThemedRun, moved: list[tuple[report_blueprint.Section, str, report_blueprint.Section]]) -> Themed | None:
+    """Wave 1: the body sections together, each given the storyline, the outline, what the others cover and only its own
+    evidence. Wave 2: the introduction and the summary, written from what wave 1 produced."""
+    sections = ctx.bp.sections
+    body = [(n, s) for n, s in enumerate(sections) if s.role in ("body", "synthesis")]
+    frame = [(n, s) for n, s in enumerate(sections) if s.role in ("intro", "summary")]
+    used: list[str] = []
+    inputs = []
+    for _, s in body:
+        ev = report_evidence.build(ctx.corpus, s, ctx.by_id)
+        used += ev.posts
+        inputs.append(_section_input(ctx, s, ev.blocks))
+    report_themes.progress(ctx.db, ctx.job, 0.45, f"Writing {len(body)} sections")
+    log.info("report wave 1: body sections=%d evidence_chars=%s", len(body), [sum(len(b) for b in i["evidence"]) for i in inputs])
+    written: dict[int, dict] = {}
+    for (n, _), out in zip(body, _write_sections(ctx, inputs), strict=True):
+        if ((out or {}).get("markdown") or "").strip():
+            written[n] = out
+    if not written:
+        log.warning("report wave 1 wrote nothing, reading the posts instead")
+        return None
+    digests = [f"{s.heading}: {_digest(written[n]['markdown'])}" for n, s in body if n in written]
+    report_themes.progress(ctx.db, ctx.job, 0.8, "Writing the introduction and summary")
+    log.info("report wave 2: intro/summary sections=%d digests=%d", len(frame), len(digests))
+    for (n, _), out in zip(frame, _write_sections(ctx, [_section_input(ctx, s, digests) for _, s in frame]), strict=True):
+        if ((out or {}).get("markdown") or "").strip():
+            written[n] = out
+
+    blocks: list[dict] = []
+    suggestions: list[dict] = []
+    anchor: dict[int, str] = {}
+    used_kinds: set[str] = set()
+    for n, s in enumerate(sections):
+        out = written.get(n)
+        if not out:
+            continue
+        blocks.append({"type": "prose", "text": f"## {s.heading}\n\n{MARKER.sub('', out['markdown'].strip())}"})
+        number_blocks(blocks)
+        anchor[id(s)] = blocks[-1]["id"]
+        embed = s.embed
+        if embed and len(suggestions) < MAX_PLANNED_VISUALS and embed.get("kind") in EMBED_KINDS \
+                and embed["kind"] not in used_kinds and (embed.get("brief") or "").strip():
+            used_kinds.add(embed["kind"])
+            suggestions.append({"id": f"s{len(suggestions) + 1}", "after_block_id": blocks[-1]["id"], "kind": embed["kind"],
+                                "brief": embed["brief"].strip()[:300], "why": (embed.get("why") or "").strip()[:300]})
+    for after, kind, sec in moved:
+        suggestions.append({"id": "", "after_block_id": anchor.get(id(after)) or blocks[0]["id"], "kind": kind,
+                            "brief": (sec.brief or MOVED_BRIEF[kind]).strip()[:300],
+                            "why": "Offered here for you to add, not written into the text."})
+    failed = [s.heading for n, s in enumerate(sections) if n not in written]
+    if failed:
+        log.warning("report sections left out after a retry: %s", failed)
+    log.info("report interactive written: blocks=%d suggestions=%d moved_to_suggestions=%d failed=%d", len(blocks),
+             len(suggestions), len(moved), len(failed))
+    suggestions = fill_suggestions(suggestions, blocks, list(EMBED_KINDS))
+    return ctx.finish(ctx.bp.title, blocks, merge_suggestions(suggestions), used, failed)
+
+
+def write_themed(db: Session, workspace_id: uuid.UUID, job: Job, docs: list[Document], title: str, instructions: str,
+                 language: str, interactive: bool) -> Themed | None:
+    """The report from the posts' themes: group their stored ideas into weighted themes, plan the whole report, build each
+    section's evidence from several posts, then write. A document is written in one piece from the plan; an interactive report
+    section by section, each knowing the storyline, the outline and what the others cover, with the introduction and the
+    summary written last from what the sections say. None when any step cannot be done (too few ideas, no usable plan, nothing
+    written), so the caller reads the posts as a report always did. Never makes a report fail."""
+    started = time.monotonic()
+    log.info("themed report starting: job=%s workspace=%s format=%s posts=%d language=%r", job.id, workspace_id,
+             "interactive" if interactive else "document", len(docs), language)
+    try:
+        report_themes.progress(db, job, 0.05, f"Reading {len(docs)} posts")
+        report_themes.ensure_ideas(db, job, workspace_id, docs)
+        share = report_themes.fresh_share(docs)
+        if share < report_themes.MIN_FRESH_SHARE:
+            log.warning("themed report not possible, reading the posts instead: job=%s only %.0f%% of %d posts have ideas "
+                        "(need %.0f%%)", job.id, share * 100, len(docs), report_themes.MIN_FRESH_SHARE * 100)
+            return None
+        corpus = Corpus(workspace_id)
+        corpus.sync()
+        ideas = report_themes.collect_ideas(docs, corpus)
+        report_themes.progress(db, job, 0.2, f"Finding the themes in {len(docs)} posts")
+        themes, relations = report_themes.group_themes(db, job, workspace_id, title, instructions, ideas)
+        report_themes.progress(db, job, 0.3, "Planning the report")
+        kinds = list(EMBED_KINDS) if interactive else []
+        bp = report_blueprint.plan_blueprint(db, job, workspace_id, title, instructions, themes, relations, kinds, language)
+        log.info("themed report planned: job=%s themes=%d sections=%d seconds=%.1f", job.id, len(themes), len(bp.sections),
+                 time.monotonic() - started)
+        moved: list[tuple[report_blueprint.Section, str, report_blueprint.Section]] = []
+        if interactive:  # a planned section that is really a quiz or flashcards is a suggestion, not text
+            kept: list[report_blueprint.Section] = []
+            for s in bp.sections:
+                kind = visual_kind(s.heading)
+                if kind and kept:
+                    log.info("report section %r is a %s, so it becomes a suggestion after %r", s.heading, kind, kept[-1].heading)
+                    moved.append((kept[-1], kind, s))
+                    if kept[-1].role == "body":  # what the section was about is still said, in the section before it
+                        kept[-1].theme_ids += [i for i in s.theme_ids if i not in kept[-1].theme_ids]
+                else:
+                    kept.append(s)
+            bp.sections = kept
+        if len(bp.sections) < 2:
+            log.warning("themed report not possible, reading the posts instead: job=%s fewer than two sections left", job.id)
+            return None
+        ctx = ThemedRun(db, workspace_id, job, docs, corpus, bp, themes, instructions, language, title)
+        themed = _themed_interactive(ctx, moved) if interactive else _themed_document(ctx)
+        if themed:
+            log.info("themed report done: job=%s posts_total=%d posts_used=%d blocks=%d suggestions=%d failed=%d seconds=%.1f",
+                     job.id, len(docs), themed.plan["posts_used"], len(themed.blocks), len(themed.suggestions),
+                     len(themed.failed), time.monotonic() - started)
+        else:
+            log.warning("themed report wrote nothing, reading the posts instead: job=%s", job.id)
+        return themed
+    except (report_themes.ThemesUnavailable, report_blueprint.BlueprintUnavailable) as exc:
+        log.warning("themed report not possible, reading the posts instead: job=%s reason=%s", job.id, exc)
+    except Exception:
+        db.rollback()
+        log.warning("themed report failed, reading the posts instead: job=%s", job.id, exc_info=True)
+    return None
 
 
 def build_report(db: Session, job: Job, artifact: Artifact) -> dict:
@@ -260,30 +483,40 @@ def build_report(db: Session, job: Job, artifact: Artifact) -> dict:
     topic = (params.get("topic") or "").strip()
     language = (params.get("language") or "").strip() or SAME_LANGUAGE
     report_format = "interactive" if params.get("report_format") == "interactive" else "document"
-    material = load_material(db, artifact.workspace_id, artifact.notebook_id, params, job=job, what="report")
+    # Posts with ideas stored are read through their themes, however many are picked; chats and small selections as always.
+    docs = themed_docs(db, artifact, params)
+    material = None if docs else load_material(db, artifact.workspace_id, artifact.notebook_id, params, job=job, what="report")
+    about = topic or (themed_title(db, artifact, params, docs) if docs else material.title)
     instructions = (params.get("instructions") or "").strip() or default_instructions(
-        report_format, params.get("template_id"), topic or material.title)
-
-    made = write_interactive(db, artifact.workspace_id, job, material, instructions, language) \
-        if report_format == "interactive" else None
-    if made:
-        title, blocks, citations, suggestions = made
+        report_format, params.get("template_id"), about)
+
+    themed = write_themed(db, artifact.workspace_id, job, docs, themed_title(db, artifact, params, docs), instructions, language,
+                          report_format == "interactive") if docs else None
+    log.info("report: job=%s artifact=%s format=%s way=%s", job.id, artifact.id, report_format,
+             "themes" if themed else "legacy (posts read as text)")
+    if themed:
+        extra = {"plan": themed.plan, **({"failed_sections": themed.failed} if themed.failed else {})}
+        title, blocks, suggestions = themed.title, themed.blocks, themed.suggestions
+        source, sources = themed.source, themed.sources
     else:
-        # A document, or an interactive report whose plan came back unusable: write it as one piece. Never a failure.
-        report_format = "document"
-        update_job(db, job, progress=0.3, message="Writing the report")
-        title, markdown, citations = write_document(db, artifact.workspace_id, job, material, instructions, language)
-        blocks = number_blocks(split_prose(markdown))
-        kinds = allowed_kinds(material)
-        pulled: list[dict] = []
-        if not ASKED_FOR_VISUAL.search(instructions):  # a quiz section the reader did not ask for becomes a suggestion
-            blocks, pulled = pull_visual_sections(blocks, kinds)
-        update_job(db, job, progress=0.9, message="Finding places for visuals")
-        suggestions = merge_suggestions(pulled, suggest_embeds(db, artifact.workspace_id, job, blocks, kinds))
+        material = material or load_material(db, artifact.workspace_id, artifact.notebook_id, params, job=job, what="report")
+        extra = {}
+        made = write_interactive(db, artifact.workspace_id, job, material, instructions, language) \
+            if report_format == "interactive" else None
+        if made:
+            title, blocks, suggestions = made
+        else:
+            # A document, or an interactive report whose plan came back unusable: write it as one piece. Never a failure.
+            report_format = "document"
+            update_job(db, job, progress=0.3, message="Writing the report")
+            title, markdown = write_document(db, artifact.workspace_id, job, material, instructions, language)
+            blocks = number_blocks(split_prose(markdown))
+            suggestions = []  # visuals are offered in interactive reports only, never in a document
+        source, sources = material.source, source_rows(material)
     artifact.content_json = {
         "title": title, "format": report_format, "template_id": params.get("template_id"),
-        "instructions": instructions, "language": language, "topic": topic, "source": material.source,
-        "blocks": blocks, "suggestions": suggestions, "citations": citations, "sources": source_rows(material),
+        "instructions": instructions, "language": language, "topic": topic, "source": source,
+        "blocks": blocks, "suggestions": suggestions, "suggestions_offered": True, "citations": [], "sources": sources, **extra,
     }
     artifact.status = "ready"
     db.commit()
@@ -308,7 +541,7 @@ def copy_block(kind: str, existing: Artifact) -> dict:
         return {"title": c.get("title"), "questions": c["questions"]}
     if kind == "flashcards" and c.get("cards"):
         return {"title": c.get("title"), "cards": c["cards"]}
-    if kind == "infographic" and existing.storage_key:  # the picture itself is shared, not copied
+    if kind == "infographic" and c.get("content"):  # the picture itself is shared, not copied
         return {"title": c.get("title"), "artifact_id": str(existing.id), "theme": c.get("theme")}
     raise NothingToDo("That one has nothing to add.")
 
@@ -381,6 +614,7 @@ def handle_report_block(db: Session, job: Job) -> dict:
             block = copy_block(kind, existing)
         else:
             block = make_block(db, job, report, kind, (p.get("brief") or "").strip(), p.get("theme"))
+            keep_visual(db, report, {"type": kind, **block})  # a new one also lands in the notebook's Studio list
     except NothingToDo as exc:
         raise PermanentJobError(str(exc)) from exc
     report.content_json = insert_block(report.content_json or {}, {"type": kind, **block}, p.get("after_block_id"),
@@ -389,8 +623,44 @@ def handle_report_block(db: Session, job: Job) -> dict:
     return {"artifact_id": str(report.id)}
 
 
+def keep_visual(db: Session, report: Artifact, block: dict) -> Artifact | None:
+    """A mind map, flashcards or quiz made inside a report would exist only as that block. It is also saved as an
+    artifact of its own in the notebook, so it shows in the Studio list and "use one you already made" offers it
+    again after the block is removed. Content that is already saved is not saved twice."""
+    kind = block.get("type")
+    source = (report.content_json or {}).get("source") or {}
+    if kind == "flashcards" and block.get("cards"):
+        key, content = "cards", {"cards": block["cards"], "card_count": len(block["cards"])}
+    elif kind == "quiz" and block.get("questions"):
+        key, content = "questions", {"questions": block["questions"], "question_count": len(block["questions"])}
+    elif kind == "mind_map" and block.get("root"):
+        key, content = "root", {"root": block["root"], "node_count": block.get("node_count"),
+                                "post_count": block.get("post_count"),
+                                "document_ids": source.get("document_ids") or []}
+    else:
+        return None  # an infographic already is its own artifact
+    same = db.scalars(select(Artifact).where(Artifact.workspace_id == report.workspace_id, Artifact.type == kind,
+                                             Artifact.notebook_id == report.notebook_id))
+    if any((a.content_json or {}).get(key) == content[key] for a in same):
+        return None
+    saved = Artifact(workspace_id=report.workspace_id, notebook_id=report.notebook_id, type=kind, status="ready",
+                     content_json={"title": block.get("title") or kind, "source": source, **content})
+    db.add(saved)
+    db.commit()
+    return saved
+
+
 def remove_block(content: dict, block_id: str) -> dict | None:
-    """The content without that visual (prose cannot be removed), or None when there is no such visual."""
+    """The content without that visual (prose cannot be removed), with the offer to add it again restored, or None
+    when there is no such visual."""
     blocks = list(content.get("blocks") or [])
-    keep = [b for b in blocks if not (b["id"] == block_id and b["type"] != "prose")]
-    return None if len(keep) == len(blocks) else {**content, "blocks": keep}
+    removed = next((b for b in blocks if b["id"] == block_id and b["type"] != "prose"), None)
+    if not removed:
+        return None
+    keep = [b for b in blocks if b["id"] != block_id]
+    suggestions = list(content.get("suggestions") or [])
+    kind = removed["type"]
+    if kind in FALLBACK_BRIEFS and not any(s["kind"] == kind for s in suggestions):
+        brief, why = FALLBACK_BRIEFS[kind]
+        suggestions.append({"id": "", "after_block_id": removed.get("after"), "kind": kind, "brief": brief, "why": why})
+    return {**content, "blocks": keep, "suggestions": suggestions}

backend/app/pipeline/report_blueprint.py

@@ -0,0 +1,220 @@
+"""The plan of a themed report, made before any of it is written.
+
+One model call (PlanReportBlueprint) sees the themes with their weights and the reader's request, never the posts, and plans the
+whole report: a storyline, the sections in order (what each is about, how deep, what it must cover), and which section bridges which
+themes. Code then checks the plan against the weights, because a plan that forgets a major theme or gives a footnote the room of a
+main theme is wrong however well it reads: every major or asked-for theme gets a section, depth follows the theme's tier, a body
+theme belongs to one section, and what other sections own is worked out here, not left to the model."""
+
+import logging
+import time
+import uuid
+from dataclasses import dataclass, field
+
+from sqlalchemy.orm import Session
+
+from app.llm import run
+from app.llm.provider import report_plan_lm
+from app.llm.signatures import PlanReportBlueprint
+from app.models import Job
+from app.pipeline.report_themes import Theme
+
+log = logging.getLogger(__name__)
+
+MAX_SECTIONS = 8
+MAX_CONNECTIONS = 6
+ROLES = ("intro", "body", "synthesis", "summary")
+MORE_HEADING = "More to know"  # where the themes no section took are gathered
+
+
+class BlueprintUnavailable(Exception):
+    """The plan was not usable: the caller reads the posts as before."""
+
+
+@dataclass
+class Section:
+    heading: str
+    role: str
+    brief: str
+    theme_ids: list[str]
+    depth: str = "standard"
+    must_cover: list[str] = field(default_factory=list)
+    embed: dict | None = None
+    covered_elsewhere: list[str] = field(default_factory=list)
+
+
+@dataclass
+class Blueprint:
+    title: str
+    storyline: str
+    sections: list[Section]
+    connections: list[dict]  # {"theme_ids": [...], "relation": str, "section": heading}
+
+
+def theme_lines(themes: list[Theme]) -> list[str]:
+    """The themes as the planner reads them: id | name | tier | weight | posts | asked for: summary, then a few of their ideas."""
+    return [f"{t.id} | {t.name} | {t.tier} | {t.weight:.2f} | {len(t.doc_ids)} posts | "
+            f"{'asked for' if t.pinned else 'not asked for'}: {t.summary} (e.g. {'; '.join(i.label for i in t.ideas[:3])})"
+            for t in themes]
+
+
+def relation_lines(relations: list[dict]) -> list[str]:
+    return [f"{r['a']} <-> {r['b']}: {r['relation']}" for r in relations]
+
+
+def derived_depth(section: Section, by_id: dict[str, Theme]) -> str:
+    """How deep a section goes follows its heaviest theme, whatever the model wrote: the weights are the plan's reason to exist."""
+    if section.role in ("intro", "summary"):
+        return "brief"
+    tiers = {by_id[i].tier + ("!" if by_id[i].pinned else "") for i in section.theme_ids if i in by_id}
+    if any(t.startswith("major") or t.endswith("!") for t in tiers):
+        return "deep"
+    if section.role == "synthesis" or any(t.startswith("medium") for t in tiers):
+        return "standard"
+    return "brief"
+
+
+def _first_index(sections: list[Section], role: str) -> int | None:
+    return next((n for n, s in enumerate(sections) if s.role == role), None)
+
+
+def _in_reading_order(sections: list[Section]) -> list[Section]:
+    """The intro first, the summary last, whatever order the model gave them in. One of each."""
+    intro = [s for s in sections if s.role == "intro"][:1]
+    summary = [s for s in sections if s.role == "summary"][-1:]
+    return intro + [s for s in sections if s.role in ("body", "synthesis")] + summary
+
+
+def repair(bp: Blueprint, themes: list[Theme]) -> Blueprint:
+    """Make the plan agree with the weights. Raises BlueprintUnavailable when fewer than two sections remain."""
+    by_id = {t.id: t for t in themes}
+    sections = [s for s in bp.sections if s.heading.strip()]
+    planned = len(sections)
+
+    # A body theme is the subject of one section; unknown ids go.
+    seen: set[str] = set()
+    for s in sections:
+        s.theme_ids = [i for n, i in enumerate(s.theme_ids) if i in by_id and i not in s.theme_ids[:n]]
+        if s.role == "body":
+            s.theme_ids = [i for i in s.theme_ids if i not in seen]
+            seen.update(s.theme_ids)
+    before = len(sections)
+    sections = [s for s in sections if s.role != "body" or s.theme_ids]
+    if len(sections) < before:
+        log.info("report blueprint: dropped %d body sections with no theme of their own", before - len(sections))
+    sections = _in_reading_order(sections)
+
+    def insert(section: Section) -> None:
+        at = _first_index(sections, "summary")
+        sections.insert(len(sections) if at is None else at, section)
+
+    # Every major or asked-for theme has a section; whatever else no section took is gathered into one.
+    for t in sorted(themes, key=lambda t: -t.weight):
+        if t.id not in seen and (t.tier == "major" or t.pinned):
+            log.info("report blueprint: no section for %s theme %s (%s), added one", "asked-for" if t.pinned else t.tier, t.id,
+                     t.name)
+            insert(Section(heading=t.name, role="body", brief=t.summary, theme_ids=[t.id], depth="deep"))
+            seen.add(t.id)
+    rest = [t.id for t in sorted(themes, key=lambda t: -t.weight) if t.id not in seen]
+    if rest:
+        log.info("report blueprint: themes in no section %s go to %r", rest, MORE_HEADING)
+        more = next((s for s in sections if s.role == "body" and s.heading == MORE_HEADING), None)
+        if more:
+            more.theme_ids += rest
+        else:
+            insert(Section(heading=MORE_HEADING, role="body", theme_ids=rest, depth="brief",
+                           brief="The smaller themes in the posts, in a line or two each."))
+        seen.update(rest)
+
+    # Too many sections: the lightest body sections (never one holding a major or asked-for theme) are gathered into one brief
+    # section, so a footnote never shares a section with a main theme.
+    def weight(s: Section) -> float:
+        return sum(by_id[i].weight for i in s.theme_ids)
+
+    def protected(s: Section) -> bool:
+        return any(by_id[i].tier == "major" or by_id[i].pinned for i in s.theme_ids)
+
+    while len(sections) > MAX_SECTIONS:
+        more = next((s for s in sections if s.role == "body" and s.heading == MORE_HEADING), None)
+        candidates = [s for s in sections if s.role == "body" and not protected(s) and s is not more]
+        if not candidates:
+            break
+        lightest = min(candidates, key=weight)
+        log.info("report blueprint: %d sections, over the %d allowed: merging %r (themes %s) into %r", len(sections),
+                 MAX_SECTIONS, lightest.heading, lightest.theme_ids, MORE_HEADING)
+        if more:
+            more.theme_ids += lightest.theme_ids
+            sections.remove(lightest)
+        else:
+            lightest.heading, lightest.must_cover = MORE_HEADING, []
+            lightest.brief = "The smaller themes in the posts, in a line or two each."
+
+    # intro first, summary last, depth from the weights, the summary restates the main themes.
+    sections = _in_reading_order(sections)
+    for s in [s for s in sections if s.role == "summary"]:
+        s.theme_ids = [t.id for t in themes if t.tier == "major" or t.pinned] or s.theme_ids
+    for s in sections:
+        s.depth = derived_depth(s, by_id)
+    if len(sections) < 2:
+        log.warning("report blueprint unusable: sections=%d planned=%d", len(sections), planned)
+        raise BlueprintUnavailable("the plan has fewer than two sections")
+
+    # What the other body sections own, so no section writes another's points. The intro and summary may touch everything.
+    for s in sections:
+        s.covered_elsewhere = [] if s.role in ("intro", "summary") else [
+            f"{o.heading}: {'; '.join(o.must_cover) or o.brief}"[:240] for o in sections
+            if o is not s and o.role in ("body", "synthesis")]
+
+    headings = {s.heading.lower(): s.heading for s in sections}
+    connections = []
+    for c in bp.connections:
+        ids = [i for i in c.get("theme_ids") or [] if i in by_id]
+        relation = (c.get("relation") or "").strip()
+        if len(ids) < 2 or not relation:
+            continue
+        home = headings.get(str(c.get("section", "")).strip().lower()) or next(
+            (s.heading for s in sections if s.role in ("body", "synthesis") and set(ids) & set(s.theme_ids)), None)
+        if home:
+            connections.append({"theme_ids": ids, "relation": relation[:200], "section": home})
+    log.info("report blueprint checked: planned=%d final=%d connections=%d/%d", planned, len(sections), len(connections),
+             len(bp.connections))
+    for n, s in enumerate(sections, start=1):
+        log.info("report section %d: heading=%r role=%s depth=%s themes=%s must_cover=%d embed=%s", n, s.heading, s.role, s.depth,
+                 s.theme_ids, len(s.must_cover), (s.embed or {}).get("kind"))
+    return Blueprint(title=bp.title, storyline=bp.storyline, sections=sections, connections=connections[:MAX_CONNECTIONS])
+
+
+def plan_blueprint(db: Session, job: Job | None, workspace_id: uuid.UUID, title: str, request: str,
+                   themes: list[Theme], relations: list[dict], kinds: list[str], language: str) -> Blueprint:
+    """The checked plan for these themes. Raises BlueprintUnavailable (or whatever the model call raised) when there is none."""
+    log.info("report blueprint: asking the model workspace=%s themes=%d relations=%d allowed_visuals=%s", workspace_id, len(themes),
+             len(relations), kinds)
+    started = time.monotonic()
+    out = run.predict(PlanReportBlueprint, db=db, workspace_id=workspace_id, job=job, lm=report_plan_lm(), title=title,
+                      request=request, themes=theme_lines(themes), relations=relation_lines(relations), allowed_kinds=kinds,
+                      language=language, max_sections=MAX_SECTIONS)
+    sections = []
+    for s in out.get("sections") or []:
+        heading = (s.get("heading") or "").strip()
+        if heading:
+            sections.append(Section(
+                heading=heading, role=s.get("role") if s.get("role") in ROLES else "body", brief=(s.get("brief") or "").strip(),
+                theme_ids=[str(i) for i in s.get("theme_ids") or []], depth=s.get("depth") or "standard",
+                must_cover=[str(p).strip() for p in s.get("must_cover") or [] if str(p).strip()][:4], embed=s.get("embed")))
+    log.info("report blueprint: model answered sections=%d connections=%d seconds=%.1f", len(sections),
+             len(out.get("connections") or []), time.monotonic() - started)
+    bp = Blueprint(title=(out.get("report_title") or title).strip(), storyline=(out.get("storyline") or "").strip(),
+                   sections=sections, connections=out.get("connections") or [])
+    return repair(bp, themes)
+
+
+def outline(bp: Blueprint) -> list[str]:
+    """Every section as every writer sees it: number, heading, role, what it says."""
+    return [f"{n}. {s.heading} ({s.role}): {s.brief}" for n, s in enumerate(bp.sections, start=1)]
+
+
+def connection_lines(bp: Blueprint, themes: list[Theme], heading: str | None = None) -> list[str]:
+    """The connections (all of them, or the ones one section bridges) as 'Pricing + Hiring: how they connect (section)'."""
+    names = {t.id: t.name for t in themes}
+    return [f"{' + '.join(names[i] for i in c['theme_ids'])}: {c['relation']} (bridged in: {c['section']})"
+            for c in bp.connections if heading is None or c["section"] == heading]

backend/app/pipeline/report_evidence.py

@@ -0,0 +1,166 @@
+"""The evidence a themed report section is written from. Plain code, no model.
+
+A section is about one or more themes (report_themes.py) and its depth (report_blueprint.py) says how much room it has. What
+the writer gets, from the posts that have ideas in those themes:
+
+  - the posts that say the most about them (most of the section's themes first, then most ideas) as the writer wrote them: numbered
+    lines taken from around the lines the ideas came from, with the start of the post for context; a short post that is mostly about
+    the theme goes in whole. Several posts, each with an equal share, so a section blends posts and does not walk through one;
+  - every other post in the themes as the short summaries already stored for it (label, note, points, and the lines to cite);
+  - nothing raw for a brief section: summaries only.
+
+Nothing is cited in the report, so the text carries no line numbers and no paths: a post is named by its title."""
+
+import logging
+import math
+import uuid
+from dataclasses import dataclass, field
+
+from app.corpus import Corpus
+from app.models import Document
+from app.pipeline.passages import _body_start
+from app.pipeline.report_blueprint import Section
+from app.pipeline.report_themes import Idea, Theme
+
+BUDGETS = {"deep": 32_000, "standard": 16_000, "brief": 4_000}  # characters of evidence (about 8k, 4k and 1k tokens)
+RAW_POSTS = {"deep": 5, "standard": 3, "brief": 0}  # posts read as written, at most
+RAW_SHARE = 0.75  # of the budget when other posts are summarised too
+WHOLE_POST_CHARS = 10_000  # a post this short, mostly about the theme, goes in whole (about 2,500 tokens)
+WHOLE_POST_SHARE = 0.5
+MARGIN = 6  # lines kept around the lines an idea came from
+LEAD_LINES = 6  # lines kept from the start of a post, for context
+JOIN_GAP = 2  # windows this close are one
+SUMMARY_DETAILS = 2
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class Evidence:
+    blocks: list[str] = field(default_factory=list)
+    posts: list[str] = field(default_factory=list)  # paths of every post used
+    raw_posts: int = 0  # how many of them were read as written
+
+
+def _merge(ranges: list[tuple[int, int]]) -> list[tuple[int, int]]:
+    out: list[tuple[int, int]] = []
+    for a, b in sorted(ranges):
+        if out and a <= out[-1][1] + JOIN_GAP:
+            out[-1] = (out[-1][0], max(out[-1][1], b))
+        else:
+            out.append((a, b))
+    return out
+
+
+def _text(lines: list[str], a: int, b: int) -> list[str]:
+    return [lines[n - 1] for n in range(a, min(b, len(lines)) + 1) if lines[n - 1].strip()]
+
+
+def _idea_ranges(d: Document, ideas: list[Idea], total: int, first: int) -> list[tuple[int, int]]:
+    out = []
+    for idea in ideas:
+        for s in idea.sources:
+            try:
+                a, b = int(s["line_start"]), int(s["line_end"])
+            except (KeyError, TypeError, ValueError):
+                continue
+            if s.get("path") == d.path and total:
+                out.append((max(first + 1, a - MARGIN), min(total, max(a, b) + MARGIN)))
+    return out
+
+
+def raw_post(corpus: Corpus, d: Document, ideas: list[Idea], names: list[str], cap: int) -> str | None:
+    """One post as written: the whole of it when it is short and mostly about the section's themes, else the lead and the windows
+    around its ideas' lines, merged, cut to `cap` characters (the windows that fit, in order of the post). None if unreadable."""
+    try:
+        lines = corpus.read_lines(d.path)
+    except Exception:
+        log.warning("report evidence: cannot read post %s, its summary stands in", d.path, exc_info=True)
+        return None
+    first = _body_start(lines)  # lines before this are the file's header
+    head = f"POST {d.title} [themes: {', '.join(names)}]"
+    stored = len((d.metadata_json or {}).get("ideas") or []) or 1
+    if len(d.clean_text or "") <= WHOLE_POST_CHARS and len(ideas) / stored >= WHOLE_POST_SHARE:
+        whole = "\n".join(_text(lines, first + 1, len(lines)))
+        if len(head) + len(whole) <= cap:
+            log.debug("report evidence: whole post %s (%d chars)", d.path, len(whole))
+            return f"{head}\n{whole}"
+    ranges = _idea_ranges(d, ideas, len(lines), first)
+    windows = _merge([(first + 1, min(len(lines), first + LEAD_LINES))] + ranges)
+    parts, used = [], len(head)
+    for a, b in windows:
+        text = "\n".join(_text(lines, a, b))
+        if not text:
+            continue
+        if parts and used + len(text) + 6 > cap:
+            break
+        parts.append(text[: max(cap - used, 200)] if not parts else text)
+        used += len(parts[-1]) + 6
+    return f"{head}\n" + "\n[...]\n".join(parts) if parts else None
+
+
+def _summary_line(idea: Idea, theme_name: str) -> str:
+    points = "; ".join(f"{x.get('label', '')}: {x.get('note', '')}".strip(": ") for x in idea.details[:SUMMARY_DETAILS]
+                       if isinstance(x, dict))
+    return f'- {idea.label}: {idea.note}' + (f" ({points})" if points else "") + f' (from "{idea.doc.title}", {theme_name})'
+
+
+def build(corpus: Corpus, section: Section, themes: dict[str, Theme], budget: int | None = None) -> Evidence:
+    """The evidence for one body or synthesis section: raw posts first, then summaries of the rest, within `budget` characters."""
+    budget = budget or BUDGETS.get(section.depth, BUDGETS["standard"])
+    items: list[tuple[Theme, Idea]] = [(themes[t], i) for t in section.theme_ids if t in themes for i in themes[t].ideas]
+    out = Evidence()
+    if not items:
+        log.warning("report evidence: section=%r has no ideas (themes %s)", section.heading, section.theme_ids)
+        return out
+    by_doc: dict[uuid.UUID, list[tuple[Theme, Idea]]] = {}
+    for t, i in items:
+        by_doc.setdefault(i.doc.id, []).append((t, i))
+
+    def key(entries: list[tuple[Theme, Idea]]) -> tuple:
+        d = entries[0][1].doc
+        stamp = d.published_at.timestamp() if d.published_at else 0
+        return (-len({t.id for t, _ in entries}), -len(entries), -stamp)
+
+    ranked = sorted(by_doc.values(), key=key)
+    wanted = RAW_POSTS.get(section.depth, 0)
+    raw_budget = (budget if len(ranked) <= wanted else int(budget * RAW_SHARE)) if wanted else 0
+    each = raw_budget // max(min(wanted, len(ranked)), 1)
+    rest: list[list[tuple[Theme, Idea]]] = []
+    for entries in ranked:
+        if out.raw_posts < wanted:
+            d = entries[0][1].doc
+            names = list(dict.fromkeys(t.name for t, _ in entries))
+            block = raw_post(corpus, d, [i for _, i in entries], names, each)
+            if block is not None:  # an unreadable post is passed over: the next one takes its place, its summary stands in
+                out.blocks.append(block)
+                out.posts.append(d.path)
+                out.raw_posts += 1
+                continue
+        rest.append(entries)
+
+    summary_budget = max(budget - sum(len(b) for b in out.blocks), 0)
+    lists = [[_summary_line(i, t.name) for t, i in entries] for entries in rest]
+    lines, used, rank = [], 0, 0
+    while any(rank < len(x) for x in lists):
+        for x in lists:
+            if rank < len(x) and used + len(x[rank]) <= summary_budget:
+                lines.append(x[rank])
+                used += len(x[rank]) + 1
+        rank += 1
+        if used >= summary_budget:
+            break
+    if lines:
+        out.blocks.append("SUMMARIES of other posts\n" + "\n".join(lines))
+    out.posts += [e[0][1].doc.path for e in rest if e[0][1].doc.path not in out.posts]
+    log.info("report evidence: section=%r depth=%s themes=%s ideas=%d posts=%d raw_posts=%d summarised_posts=%d summary_lines=%d "
+             "chars=%d budget=%d", section.heading, section.depth, section.theme_ids, len(items), len(by_doc), out.raw_posts,
+             len(rest), len(lines), sum(len(b) for b in out.blocks), budget)
+    return out
+
+
+def scale_budgets(sections: list[Section], total: int) -> dict[str, int]:
+    """Section heading -> budget, the standard budgets scaled down together when they add up to more than `total`."""
+    wanted = {s.heading: BUDGETS.get(s.depth, BUDGETS["standard"]) for s in sections if s.role in ("body", "synthesis")}
+    factor = min(1.0, total / max(sum(wanted.values()), 1))
+    return {h: max(int(math.floor(b * factor)), 1_000) for h, b in wanted.items()}

backend/app/pipeline/report_themes.py

@@ -0,0 +1,280 @@
+"""Themes for a report: what a notebook's posts are about, and how much of it each thing is.
+
+A report no longer reads the posts' text to find out. Every post already has its key ideas stored (documents.metadata_json["ideas"],
+written by generate.extract_ideas, each with the lines it came from), so those ideas stand for the posts, however many are picked:
+
+  1. one model call (GroupThemes) names themes and says which ideas belong to which (it sees at most MAX_LISTED_IDEAS of them);
+  2. code files every other idea into a theme and counts: how many posts and ideas each theme has gives its weight, so the weights
+     are true for the whole selection and not for the sample the model saw. A model groups well and counts badly.
+
+The weights decide how much room a theme gets in the report (report_blueprint.py) and how much of the posts' own text it is written
+from (report_evidence.py). A theme the reader asked for is moved up a tier whatever its share."""
+
+import logging
+import time
+import uuid
+from collections import Counter
+from dataclasses import dataclass, field
+
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.corpus import Corpus
+from app.llm import run
+from app.llm.provider import fast_lm
+from app.llm.signatures import GroupThemes
+from app.models import Document, Job, Notebook, Workspace
+from app.pipeline.generate import extract_ideas, ideas_fresh
+from app.pipeline.idea_pool import _tokens
+from app.pipeline.passages import _body_start, notebook_docs, workspace_docs
+from app.services.jobs import update_job
+from app.services.plans import effective_plan
+
+log = logging.getLogger(__name__)
+
+SMALL_SELECTION = 5  # posts or fewer: the posts are read whole, as a report always was
+MIN_FRESH_SHARE = 0.5  # fewer posts than this with ideas stored: read the posts' text as before
+MAX_LISTED_IDEAS = 300  # ideas the model sees; the rest are filed by code
+MAX_THEMES = 12
+MIN_THEMES = 2
+MAJOR_AT, MEDIUM_AT = 0.25, 0.10  # a theme's share of the report: major from, medium from
+POST_WEIGHT, IDEA_WEIGHT = 0.6, 0.4  # how many posts mention it counts for more than how many ideas
+CATCH_ALL = "More from your posts"
+OPENING_LINES = 8  # lines of a post's opening that stand for a post with no ideas stored yet
+
+
+class ThemesUnavailable(Exception):
+    """The ideas did not group into themes: the caller reads the posts as before."""
+
+
+@dataclass
+class Idea:
+    id: str
+    doc: Document
+    label: str
+    note: str
+    details: list[dict] = field(default_factory=list)
+    sources: list[dict] = field(default_factory=list)
+    bare: bool = False  # no ideas stored for this post: this is its title and opening
+
+
+@dataclass
+class Theme:
+    id: str
+    name: str
+    summary: str
+    ideas: list[Idea]
+    asked: bool = False  # the reader's request asks for it
+    weight: float = 0.0  # share of the whole, all themes add up to 1
+    tier: str = "minor"  # major | medium | minor
+    pinned: bool = False  # asked for: always given a deep section
+
+    @property
+    def doc_ids(self) -> set[uuid.UUID]:
+        return {i.doc.id for i in self.ideas}
+
+
+# The posts
+
+def select_docs(db: Session, workspace_id: uuid.UUID, notebook_id: uuid.UUID | None, params: dict) -> list[Document]:
+    """The posts a report is made from: the ones picked, or the notebook's, newest first. Not cut at ARTIFACT_MAX_POSTS: only
+    what the plan indexes (and REPORT_MAX_POSTS, when set) limits it."""
+    picked = [uuid.UUID(str(i)) for i in params.get("document_ids") or []]
+    nb = db.get(Notebook, notebook_id) if notebook_id else None
+    if picked:
+        docs = workspace_docs(db, workspace_id, picked)
+    elif nb:
+        docs = notebook_docs(db, nb.id)
+    else:
+        docs = []
+    cap = effective_plan(db, db.get(Workspace, workspace_id)).indexed_posts
+    if settings.report_max_posts > 0:
+        cap = min(cap, settings.report_max_posts) if cap and cap > 0 else settings.report_max_posts
+    chosen = docs[:cap] if cap and cap > 0 else docs
+    log.info("report posts: workspace=%s notebook=%s picked=%d found=%d used=%d plan_cap=%s report_max_posts=%s", workspace_id,
+             notebook_id, len(picked), len(docs), len(chosen), cap, settings.report_max_posts)
+    if len(chosen) < len(docs):
+        log.warning("report posts cut by the plan or REPORT_MAX_POSTS: workspace=%s found=%d used=%d", workspace_id, len(docs),
+                    len(chosen))
+    return chosen
+
+
+def ensure_ideas(db: Session, job: Job | None, workspace_id: uuid.UUID, docs: list[Document]) -> None:
+    """Extract the ideas of posts that have none yet, together, at most REPORT_INLINE_IDEAS of them. The rest are represented
+    by their opening; the background ideas job reaches them later."""
+    stale = [d for d in docs if not ideas_fresh(d)]
+    if not stale or settings.report_inline_ideas <= 0:
+        log.info("report ideas: workspace=%s posts=%d stale=%d inline_limit=%d, nothing extracted now", workspace_id, len(docs),
+                 len(stale), settings.report_inline_ideas)
+        return
+    started = time.monotonic()
+    result = extract_ideas(db, job, workspace_id, [str(d.id) for d in stale], parallel=True, limit=settings.report_inline_ideas)
+    for d in stale:
+        db.refresh(d)
+    log.info("report ideas: workspace=%s posts=%d stale=%d extracted=%s skipped=%s left_for_background=%d seconds=%.1f",
+             workspace_id, len(docs), len(stale), result.get("extracted"), result.get("skipped"),
+             sum(1 for d in stale if not ideas_fresh(d)), time.monotonic() - started)
+
+
+def fresh_share(docs: list[Document]) -> float:
+    return sum(1 for d in docs if ideas_fresh(d)) / max(len(docs), 1)
+
+
+def _opening_idea(d: Document, di: int, corpus: Corpus) -> Idea:
+    """A post with no ideas stored: its title, its topics and where its opening is, so it still counts and can be cited."""
+    meta = d.metadata_json or {}
+    tags = ", ".join(t["name"] for t in (meta.get("topics") or [])[:3] if t.get("name"))
+    try:
+        lines = corpus.read_lines(d.path)
+    except Exception:
+        lines = []
+    first = _body_start(lines) + 1 if lines else 1
+    sources = [{"path": d.path, "line_start": first, "line_end": min(first + OPENING_LINES, len(lines) or first)}]
+    return Idea(id=f"p{di}.0", doc=d, label=d.title[:80], bare=True, sources=sources,
+                note=f"Topics: {tags}" if tags else "Only the opening of this post was read.")
+
+
+def collect_ideas(docs: list[Document], corpus: Corpus) -> list[Idea]:
+    """Every post's stored ideas as Ideas with ids like p3.1 (the third post's second idea), or its opening when it has none."""
+    out: list[Idea] = []
+    for di, d in enumerate(docs):
+        stored = [i for i in ((d.metadata_json or {}).get("ideas") or []) if (i.get("label") or "").strip()] \
+            if ideas_fresh(d) else []
+        if not stored:
+            out.append(_opening_idea(d, di, corpus))
+            continue
+        for ii, i in enumerate(stored):
+            out.append(Idea(id=f"p{di}.{ii}", doc=d, label=i["label"].strip(), note=(i.get("note") or "").strip(),
+                            details=i.get("details") or [], sources=i.get("sources") or []))
+    log.info("report ideas collected: posts=%d ideas=%d posts_with_only_an_opening=%d", len(docs), len(out),
+             sum(1 for i in out if i.bare))
+    return out
+
+
+# Themes
+
+def listed_ideas(ideas: list[Idea]) -> list[Idea]:
+    """The ideas the model sees: all of them, or when there are too many, the same number from every post in turn."""
+    if len(ideas) <= MAX_LISTED_IDEAS:
+        return ideas
+    by_doc: dict[uuid.UUID, list[Idea]] = {}
+    for i in ideas:
+        by_doc.setdefault(i.doc.id, []).append(i)
+    lists, picked, rank = list(by_doc.values()), [], 0
+    while len(picked) < MAX_LISTED_IDEAS and any(rank < len(x) for x in lists):
+        for x in lists:
+            if rank < len(x) and len(picked) < MAX_LISTED_IDEAS:
+                picked.append(x[rank])
+        rank += 1
+    return picked
+
+
+def _file_leftovers(themes: list[Theme], leftovers: list[Idea]) -> None:
+    """Every idea the model did not place goes to the theme its post's other ideas are mostly in, else the theme whose words it
+    shares most, else a catch-all theme: the way the Mind Constellation files what the model leaves out."""
+    by_doc: dict[uuid.UUID, Counter] = {}
+    for ti, t in enumerate(themes):
+        for i in t.ideas:
+            by_doc.setdefault(i.doc.id, Counter())[ti] += 1
+    theme_words = [_tokens(" ".join([t.name, t.summary] + [i.label for i in t.ideas[:20]])) for t in themes]
+    # A word in most themes ("idea", "post") says nothing about which one an idea is about.
+    common = {w for w, n in Counter(w for tw in theme_words for w in tw).items() if n > max(1, len(themes) // 2)}
+    theme_words = [tw - common for tw in theme_words]
+    catch: Theme | None = None
+    for idea in leftovers:
+        counts = by_doc.get(idea.doc.id)
+        if counts:
+            themes[counts.most_common(1)[0][0]].ideas.append(idea)
+            continue
+        words = _tokens(f"{idea.label} {idea.note}") - common
+        best, score = max(((ti, len(words & tw)) for ti, tw in enumerate(theme_words)), key=lambda x: x[1])
+        if score > 0:
+            themes[best].ideas.append(idea)
+            continue
+        if catch is None:
+            catch = Theme(id="", name=CATCH_ALL, summary="Ideas that fit none of the other themes.", ideas=[])
+            themes.append(catch)
+        catch.ideas.append(idea)
+    log.info("report leftover ideas filed: leftovers=%d catch_all=%d", len(leftovers), len(catch.ideas) if catch else 0)
+
+
+def weigh(themes: list[Theme], total_posts: int, total_ideas: int) -> None:
+    """Each theme's share of the whole, from how many posts and ideas it has (not from anything the model said), and its tier. A
+    theme the reader asked for moves up one tier and is pinned. If nothing reached major, the heaviest theme becomes major."""
+    raw = [POST_WEIGHT * len(t.doc_ids) / max(total_posts, 1) + IDEA_WEIGHT * len(t.ideas) / max(total_ideas, 1) for t in themes]
+    total = sum(raw) or 1.0
+    for t, r in zip(themes, raw, strict=True):
+        t.weight = round(r / total, 4)
+        t.tier = "major" if t.weight >= MAJOR_AT else "medium" if t.weight >= MEDIUM_AT else "minor"
+    if themes and not any(t.tier == "major" for t in themes):
+        max(themes, key=lambda t: t.weight).tier = "major"
+    for t in themes:
+        if t.asked:
+            t.pinned = True
+            t.tier = {"minor": "medium", "medium": "major"}.get(t.tier, t.tier)
+
+
+def _relations(raw: list[dict] | None, themes: list[Theme]) -> list[dict]:
+    by_name = {t.name.lower(): t.id for t in themes}
+    out = []
+    for r in raw or []:
+        a, b = by_name.get(str(r.get("a", "")).strip().lower()), by_name.get(str(r.get("b", "")).strip().lower())
+        relation = (r.get("relation") or "").strip()
+        if a and b and a != b and relation:
+            out.append({"a": a, "b": b, "relation": relation[:200]})
+    return out[:8]
+
+
+def group_themes(db: Session, job: Job | None, workspace_id: uuid.UUID, title: str, request: str,
+                 ideas: list[Idea]) -> tuple[list[Theme], list[dict]]:
+    """Themes with their weights, and how they relate (by theme id). Raises ThemesUnavailable when the answer is not usable."""
+    by_id = {i.id: i for i in ideas}
+    listed = listed_ideas(ideas)
+    listing = [f"{i.id} | {i.label}: {i.note[:140]}" for i in listed]
+    log.info("report themes: asking the model workspace=%s ideas=%d listed=%d (cap %d) posts=%d", workspace_id, len(ideas),
+             len(listed), MAX_LISTED_IDEAS, len({i.doc.id for i in ideas}))
+    started = time.monotonic()
+    out = run.predict(GroupThemes, db=db, workspace_id=workspace_id, job=job, lm=fast_lm(), title=title,
+                      request=request, ideas=listing)
+    themes: list[Theme] = []
+    taken: set[str] = set()
+    dropped = 0
+    for t in out.get("themes") or []:
+        name = (t.get("name") or "").strip()
+        wanted = list(t.get("idea_ids") or [])
+        mine = [by_id[i] for i in wanted if i in by_id and i not in taken]
+        dropped += len(wanted) - len(mine)  # unknown ids, or ideas another theme already took
+        if not name or not mine:
+            log.info("report theme skipped: name=%r usable_ideas=%d", name, len(mine))
+            continue
+        taken.update(i.id for i in mine)
+        themes.append(Theme(id="", name=name[:80], summary=(t.get("summary") or "").strip()[:400], ideas=mine,
+                            asked=bool(t.get("matches_request"))))
+        if len(themes) == MAX_THEMES:
+            break
+    log.info("report themes: model answered themes=%d usable=%d ids_dropped=%d seconds=%.1f", len(out.get("themes") or []),
+             len(themes), dropped, time.monotonic() - started)
+    if len(themes) < MIN_THEMES:
+        log.warning("report themes unusable: usable=%d needed=%d", len(themes), MIN_THEMES)
+        raise ThemesUnavailable("the ideas did not group into themes")
+    _file_leftovers(themes, [i for i in ideas if i.id not in taken])
+    for n, t in enumerate(themes, start=1):
+        t.id = f"t{n}"
+    weigh(themes, len({i.doc.id for i in ideas}), len(ideas))
+    relations = _relations(out.get("relations"), themes)
+    for t in themes:
+        log.info("report theme: id=%s name=%r weight=%.3f tier=%s asked=%s pinned=%s posts=%d ideas=%d", t.id, t.name, t.weight,
+                 t.tier, t.asked, t.pinned, len(t.doc_ids), len(t.ideas))
+    log.info("report theme relations: model=%d kept=%d", len(out.get("relations") or []), len(relations))
+    return themes, relations
+
+
+def theme_summary(t: Theme) -> dict:
+    """What is kept with the report about a theme."""
+    return {"id": t.id, "name": t.name, "summary": t.summary, "weight": t.weight, "tier": t.tier, "pinned": t.pinned,
+            "posts": len(t.doc_ids), "ideas": len(t.ideas)}
+
+
+def progress(db: Session, job: Job | None, value: float, message: str) -> None:
+    if job:
+        update_job(db, job, progress=value, message=message)

backend/app/routers/artifacts.py

@@ -1,3 +1,4 @@
+import logging
 import re
 import uuid
 from typing import Literal
@@ -10,8 +11,9 @@ from app.auth import Ctx, get_ctx
 from app.config import settings
 from app.infographics.image import ImageUnavailable, render_png
 from app.infographics.themes import theme_id
-from app.models import Artifact, CalendarItem, Chat, Document, Job, Notebook, UserSavedVoice
+from app.models import Artifact, ArtifactFeedback, CalendarItem, Chat, Document, Job, Notebook, UserSavedVoice
 from app.routers.notebooks import ensure_archive_notebook
+from app.services import artifact_feedback
 from app.services.artifacts import latest_jobs, serialize_artifact, start_artifact
 from app.services.jobs import create_job, serialize_job
 from app.services.plans import effective_plan, plan_limit_error
@@ -25,6 +27,8 @@ from app.slides.build import stored as stored_deck
 from app.slides.build import view as slide_pages
 from app.slides.themes import theme_id as slide_theme_id
 
+log = logging.getLogger(__name__)
+
 router = APIRouter(prefix="/api/artifacts", tags=["artifacts"])
 
 ArtifactType = Literal["summary", "audio_overview", "video", "quote_card", "carousel", "launch_kit", "mind_map", "quiz",
@@ -69,7 +73,8 @@ class GenerateIn(BaseModel):
     question_types: list[Literal["multiple_choice", "multiple_select", "fill_blank", "short_answer"]] = Field(
         default_factory=lambda: ["multiple_choice", "multiple_select"], min_length=1)
     language: str | None = Field(None, max_length=40)
-    theme: str | None = Field(None, max_length=30)  # infographic: app.infographics.themes; slide_deck: app.slides.themes
+    theme: str | None = Field(None, max_length=30)  # infographic: one of the looks (app.infographics.design.LOOKS); slide_deck: app.slides.themes
+    ig_style: str | None = Field(None, max_length=20)  # infographic: the layout to lean towards (design.STYLES), auto when empty
     # slide_deck: read on its own ("detailed") or shown behind a speaker ("presenter"), and how many slides
     deck_format: Literal["detailed", "presenter"] = "detailed"
     deck_length: Literal["short", "default", "long"] = "default"
@@ -166,6 +171,7 @@ def generate(body: GenerateIn, ctx: Ctx = Depends(get_ctx)):
         if body.format == "debate" and body.hosts == 1:
             raise HTTPException(400, "A debate needs two hosts")
         check_limit(ctx.db, ctx.workspace, "audio_minutes", body.minutes)
+        check_limit(ctx.db, ctx.workspace, "audio_overviews", 1)
         ids = body.document_ids or ([body.document_id] if body.document_id else [])
         hosts = _hosts(ctx, body)
         if body.hosts == 1:
@@ -197,7 +203,8 @@ def generate(body: GenerateIn, ctx: Ctx = Depends(get_ctx)):
         check_limit(ctx.db, ctx.workspace, "infographics", 1)
         ids = body.document_ids or ([body.document_id] if body.document_id else [])
         params = {"document_ids": [str(i) for i in ids], "chat_ids": [str(i) for i in body.chat_ids or []],
-                  "theme": theme_id(body.theme), "instructions": (body.instructions or "").strip()[:600]}
+                  "theme": theme_id(body.theme), "style": body.ig_style or "auto",
+                  "instructions": (body.instructions or "").strip()[:600]}
         title = f"Infographic: {params['instructions'][:60] or target}"
     elif body.type == "slide_deck":
         ids = body.document_ids or ([body.document_id] if body.document_id else [])
@@ -257,13 +264,85 @@ def list_artifacts(
     rows = ctx.db.scalars(query.order_by(Artifact.created_at.desc()).offset(max(offset, 0)).limit(min(limit, 200)))
     rows = list(rows.all())
     jobs = latest_jobs(ctx.db, [a.id for a in rows])
-    return {"total": total, "items": [serialize_artifact(a, jobs.get(a.id)) for a in rows]}
+    items = [serialize_artifact(a, jobs.get(a.id)) for a in rows]
+    return {"total": total, "items": artifact_feedback.attach(ctx.db, ctx.user.id, items)}
+
+
+@router.get("/feedback/export")
+def export_artifact_feedback(ctx: Ctx = Depends(get_ctx), rating: Literal["up", "down"] | None = None, type: str | None = None,
+                             limit: int = 200):
+    """The ratings of generated reports, quizzes, flashcards and infographics, newest first, with what each one was, to see
+    which kinds do well and which do not."""
+    q = select(ArtifactFeedback).where(ArtifactFeedback.workspace_id == ctx.workspace.id)
+    if rating:
+        q = q.where(ArtifactFeedback.rating == (1 if rating == "up" else -1))
+    if type:
+        q = q.where(ArtifactFeedback.artifact_type == type)
+    rows = ctx.db.scalars(q.order_by(ArtifactFeedback.updated_at.desc()).limit(min(limit, 1000))).all()
+    log.info("feedback exported: rows=%d rating=%s type=%s user=%s workspace=%s", len(rows), rating, type, ctx.user.id,
+             ctx.workspace.id)
+    return [{"id": str(f.id), "artifact_id": str(f.artifact_id) if f.artifact_id else None, "type": f.artifact_type,
+             "rating": "up" if f.rating > 0 else "down", "reasons": f.reasons, "comment": f.comment, "snapshot": f.snapshot,
+             "rated_at": f.updated_at.isoformat() if f.updated_at else None} for f in rows]
 
 
 @router.get("/{artifact_id}")
 def get_artifact(artifact_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
     a = get_artifact_or_404(ctx, artifact_id)
-    return serialize_artifact(a, latest_jobs(ctx.db, [a.id]).get(a.id))
+    _offer_visuals_once(ctx, a)
+    return artifact_feedback.attach(ctx.db, ctx.user.id, [serialize_artifact(a, latest_jobs(ctx.db, [a.id]).get(a.id))])[0]
+
+
+def _offer_visuals_once(ctx: Ctx, a: Artifact) -> None:
+    """An interactive report made before interactive reports always offered visuals gets them the first time it is opened (once: after
+    that, the ones the reader has added or taken away stay as they are). A document report never offers any."""
+    c = a.content_json or {}
+    if a.type != "report" or a.status != "ready" or c.get("format") != "interactive" or c.get("suggestions_offered") \
+            or not c.get("blocks"):
+        return
+    from app.pipeline.report import EMBED_KINDS, fill_suggestions, merge_suggestions
+
+    has_posts = bool((c.get("source") or {}).get("document_ids"))
+    kinds = [k for k in EMBED_KINDS if k != "mind_map" or has_posts]
+    a.content_json = {**c, "suggestions": merge_suggestions(fill_suggestions(list(c.get("suggestions") or []), c["blocks"], kinds)),
+                      "suggestions_offered": True}
+    ctx.db.commit()
+
+
+class FeedbackIn(BaseModel):
+    rating: Literal["up", "down"] | None  # null takes the rating back
+    reasons: list[str] = []
+    comment: str | None = Field(None, max_length=1000)
+
+
+@router.put("/{artifact_id}/feedback")
+def rate_artifact(artifact_id: uuid.UUID, body: FeedbackIn, ctx: Ctx = Depends(get_ctx)):
+    """Thumbs up or down on a finished report, quiz, flashcard set or infographic. Rating again replaces it; null removes it."""
+    a = get_artifact_or_404(ctx, artifact_id)
+    allowed = artifact_feedback.REASONS.get(a.type)
+    if allowed is None or a.status != "ready":
+        log.warning("feedback refused: artifact=%s type=%s status=%s user=%s", a.id, a.type, a.status, ctx.user.id)
+        raise HTTPException(400, "Only a finished report, quiz, flashcard set or infographic can be rated")
+    row = ctx.db.scalar(select(ArtifactFeedback).where(ArtifactFeedback.artifact_id == a.id, ArtifactFeedback.user_id == ctx.user.id))
+    if body.rating is None:
+        if row:
+            ctx.db.delete(row)
+            ctx.db.commit()
+            log.info("feedback removed: artifact=%s type=%s user=%s workspace=%s", a.id, a.type, ctx.user.id, ctx.workspace.id)
+        return {"feedback": None}
+    if not row:
+        row = ArtifactFeedback(workspace_id=ctx.workspace.id, artifact_id=a.id, user_id=ctx.user.id)
+        ctx.db.add(row)
+    row.artifact_type = a.type
+    row.rating = 1 if body.rating == "up" else -1
+    row.reasons = [r for r in dict.fromkeys(body.reasons) if r in allowed] if body.rating == "down" else []
+    row.comment = (body.comment or "").strip() or None
+    row.snapshot = artifact_feedback.snapshot(a)
+    ctx.db.commit()
+    # The comment's text stays out of the logs (it is the user's own words); its length and the reasons are enough.
+    log.info("feedback saved: artifact=%s type=%s rating=%s reasons=%s comment_chars=%d user=%s workspace=%s",
+             a.id, a.type, body.rating, row.reasons, len(row.comment or ""), ctx.user.id, ctx.workspace.id)
+    return {"feedback": artifact_feedback.feedback_out(row)}
 
 
 @router.get("/{artifact_id}/image")

backend/app/routers/notebooks.py

@@ -36,6 +36,7 @@ from app.pipeline.research import Turn, research
 from app.pipeline.trace import ChatTrace, use_trace
 from app.routers.sources import serialize_document
 from app.services.artifacts import latest_jobs, serialize_artifact
+from app.services import report_suggestions
 from app.services.jobs import create_job, record_usage
 from app.services.memory import list_facts, profile_text
 from app.services.storage import storage
@@ -86,6 +87,8 @@ def _add_docs(ctx: Ctx, nb: Notebook, ids: list[uuid.UUID]) -> int:
     fresh = [d for d in ctx.db.scalars(select(Document).where(Document.id.in_(valid - existing))) if not ideas_fresh(d)]
     if fresh and settings.llm_api_key:
         create_job(ctx.db, ctx.workspace.id, "ideas", {"document_ids": [str(d.id) for d in fresh]}, max_attempts=2)
+    if valid - existing:
+        report_suggestions.queue_warmup(ctx.db, ctx.workspace.id)  # the Create report dialog's templates, ready before it is opened
     return len(valid - existing)
 
 
@@ -254,7 +257,9 @@ def notebook_artifacts(notebook_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
         select(Artifact).where(Artifact.notebook_id == nb.id).order_by(Artifact.created_at.desc())
     ).all())
     jobs = latest_jobs(ctx.db, [a.id for a in rows])
-    return [serialize_artifact(a, jobs.get(a.id)) for a in rows]
+    from app.services import artifact_feedback  # the ratings ride along on each artifact the panel lists
+
+    return artifact_feedback.attach(ctx.db, ctx.user.id, [serialize_artifact(a, jobs.get(a.id)) for a in rows])
 
 
 @router.get("/{notebook_id}/chats")

backend/app/routers/reports.py

@@ -7,12 +7,11 @@ from sqlalchemy import select
 
 from app.auth import Ctx, get_ctx
 from app.llm import run
-from app.llm.provider import fast_lm
-from app.llm.signatures import SuggestReportTemplates
 from app.models import Artifact
-from app.pipeline.report import EMBED_KINDS, remove_block
+from app.pipeline.report import EMBED_KINDS, keep_visual, remove_block
 from app.routers.artifacts import get_artifact_or_404
 from app.routers.videos import FocusIn, _focus_material
+from app.services import report_suggestions
 from app.services.artifacts import latest_jobs, serialize_artifact
 from app.services.jobs import create_job
 from app.services.report_templates import INTERACTIVE, TEMPLATES
@@ -20,7 +19,6 @@ from app.services.usage import check_limit
 
 router = APIRouter(prefix="/api/reports", tags=["reports"])
 
-SUGGEST_ITEMS = 5  # sources the AI reads to suggest templates
 
 
 @router.get("/templates")
@@ -35,25 +33,15 @@ class SuggestIn(FocusIn):
 
 @router.post("/suggest")
 def suggest_templates(body: SuggestIn, ctx: Ctx = Depends(get_ctx)):
-    """Four kinds of report these sources could make, each with its own ready-written instructions. Never an error:
-    when the AI has nothing usable the list is empty and the dialog shows only the fixed templates."""
-    material, _docs = _focus_material(ctx, body)
+    """Four kinds of report these sources could make, each with its own ready-written instructions. The AI reads each post's stored
+    ideas and topics (not its text); the answer is kept, so the same sources get it again at once (and it is prepared in the
+    background as soon as ideas are extracted). Never an error: when there is nothing usable the list is empty and the dialog shows
+    only the fixed templates."""
+    material, docs = _focus_material(ctx, body)
     if not material:
         return {"templates": []}
-    try:
-        out = run.predict(SuggestReportTemplates, db=ctx.db, workspace_id=ctx.workspace.id, lm=fast_lm(),
-                          items=list(material.values())[:SUGGEST_ITEMS], topic=(body.topic or "").strip() or "(none)")
-    except Exception:
-        return {"templates": []}
-    seen, found = set(), []
-    for t in out.get("templates") or []:
-        name, prompt = (t.get("name") or "").strip(), (t.get("prompt") or "").strip()
-        if not name or not prompt or name.lower() in seen:
-            continue
-        seen.add(name.lower())
-        found.append({"id": f"suggested-{len(found) + 1}", "name": name[:60],
-                      "description": (t.get("description") or "").strip()[:200], "prompt": prompt[:1500]})
-    return {"templates": found[:4]}
+    items = report_suggestions.suggestion_items(material, docs)
+    return {"templates": report_suggestions.suggest(ctx.db, ctx.workspace.id, items, (body.topic or "").strip())}
 
 
 class AddBlockIn(BaseModel):
@@ -105,9 +93,11 @@ def add_block(artifact_id: uuid.UUID, body: AddBlockIn, ctx: Ctx = Depends(get_c
 @router.delete("/{artifact_id}/blocks/{block_id}")
 def delete_block(artifact_id: uuid.UUID, block_id: str, ctx: Ctx = Depends(get_ctx)):
     a = _report_or_404(ctx, artifact_id)
+    removed = next((b for b in (a.content_json or {}).get("blocks") or [] if b.get("id") == block_id), None)
     content = remove_block(a.content_json or {}, block_id)
     if content is None:
         raise HTTPException(404, "Visual not found")
+    keep_visual(ctx.db, a, removed)
     a.content_json = content
     ctx.db.commit()
     return serialize_artifact(a, latest_jobs(ctx.db, [a.id]).get(a.id))

backend/app/routers/settings.py

@@ -1,14 +1,9 @@
-import re
-
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends
 from pydantic import BaseModel, Field
-from sqlalchemy import select
 
 from app.auth import Ctx, get_ctx
 from app.config import settings as app_settings
-from app.models import Upload
 from app.services.plans import effective_plan, plan_dict
-from app.services.storage import storage
 from app.services.usage import usage_report
 
 router = APIRouter(prefix="/api/settings", tags=["settings"])
@@ -17,24 +12,18 @@ router = APIRouter(prefix="/api/settings", tags=["settings"])
 class SettingsIn(BaseModel):
     name: str | None = Field(None, max_length=200)
     workspace_name: str | None = Field(None, min_length=1, max_length=200)
-    brand_name: str | None = Field(None, max_length=120)
-    brand_accent: str | None = None
-    logo_upload_id: str | None = None  # "" removes the logo
     training_opt_in: bool | None = None
     allow_public_links: bool | None = None
     email_unsubscribed: bool | None = None
 
 
 def _serialize(ctx: Ctx) -> dict:
-    brand = ctx.workspace.brand_json or {}
     return {
         "user": {"name": ctx.user.name, "email": ctx.user.email, "auth_provider": ctx.user.auth_provider.value,
                  "email_unsubscribed": ctx.user.email_unsubscribed},
         "workspace": {"id": str(ctx.workspace.id), "name": ctx.workspace.name,
                       "training_opt_in": ctx.workspace.training_opt_in,
                       "allow_public_links": ctx.workspace.allow_public_links},
-        "brand": {"name": brand.get("name"), "accent": brand.get("accent") or "#217cff",
-                  "logo_url": storage.presign_get(brand["logo_key"]) if brand.get("logo_key") else None},
         "plan": plan_dict(effective_plan(ctx.db, ctx.workspace)),
         "billing_enabled": app_settings.billing_enabled,
         "usage": usage_report(ctx.db, ctx.workspace),
@@ -45,6 +34,15 @@ def _serialize(ctx: Ctx) -> dict:
     }
 
 
+@router.get("/limits")
+def artifact_limits(_ctx: Ctx = Depends(get_ctx)):
+    """How many posts a report or infographic reads, and a dialog ticks to start with (ARTIFACT_MAX_POSTS), so the pickers match the
+    server. A quiz or flashcard set can read as many posts as the plan indexes (selectable_posts)."""
+    from app.pipeline.material import max_posts
+
+    return {"posts": max_posts(), "selectable_posts": effective_plan(_ctx.db, _ctx.workspace).indexed_posts}
+
+
 @router.get("")
 def get_settings(ctx: Ctx = Depends(get_ctx)):
     return _serialize(ctx)
@@ -62,23 +60,6 @@ def update_settings(body: SettingsIn, ctx: Ctx = Depends(get_ctx)):
         ctx.workspace.allow_public_links = body.allow_public_links
     if body.email_unsubscribed is not None:
         ctx.user.email_unsubscribed = body.email_unsubscribed
-    brand = dict(ctx.workspace.brand_json or {})
-    if body.brand_name is not None:
-        brand["name"] = body.brand_name.strip() or None
-    if body.brand_accent is not None:
-        if not re.fullmatch(r"#[0-9a-fA-F]{6}", body.brand_accent):
-            raise HTTPException(422, "Use a hex color like #217cff")
-        brand["accent"] = body.brand_accent.lower()
-    if body.logo_upload_id is not None:
-        if body.logo_upload_id == "":
-            brand.pop("logo_key", None)
-        else:
-            upload = ctx.db.scalar(select(Upload).where(Upload.id == body.logo_upload_id,
-                                                        Upload.workspace_id == ctx.workspace.id))
-            if not upload or upload.status != "complete" or not upload.content_type.startswith("image/"):
-                raise HTTPException(400, "Upload an image first")
-            brand["logo_key"] = upload.key
-    ctx.workspace.brand_json = brand
     ctx.db.commit()
     return _serialize(ctx)
 

backend/app/routers/sources.py

@@ -10,7 +10,7 @@ from app.corpus import INDEX, Corpus, CorpusError
 from app.models import Document, Job, NotebookDocument, Source, Upload
 from app.pipeline.ingest import IMPORTS_FEED, FeedNotFound, discover_feed, imports_source, rebuild_index
 from app.services.jobs import create_job, serialize_job
-from app.services.plans import effective_plan, plan_limit_error
+from app.services.plans import effective_plan, plan_limit_error, post_room
 from app.services.storage import storage
 
 router = APIRouter(prefix="/api/sources", tags=["sources"])
@@ -74,7 +74,7 @@ async def create_source(body: SourceIn, ctx: Ctx = Depends(get_ctx)):
         select(func.count()).select_from(Source)
         .where(Source.workspace_id == ctx.workspace.id, Source.feed_url != IMPORTS_FEED)
     )
-    if count >= plan.sources:
+    if 0 <= plan.sources <= count:
         raise plan_limit_error(plan, "sources", f"Your plan includes {plan.sources} sources.")
     source = Source(workspace_id=ctx.workspace.id, feed_url=found.feed_url, platform=found.platform,
                     site_url=found.site_url, title=found.title)
@@ -135,8 +135,17 @@ def delete_source(source_id: uuid.UUID, ctx: Ctx = Depends(get_ctx)):
     return {"ok": True, "removed_posts": len(docs)}
 
 
+def _require_room(ctx: Ctx) -> None:
+    """A link or a file adds a post: refused (with the upgrade prompt) once the plan's posts are all indexed."""
+    plan = effective_plan(ctx.db, ctx.workspace)
+    if post_room(ctx.db, ctx.workspace.id, plan) <= 0:
+        raise plan_limit_error(plan, "indexed_posts", f"Your plan indexes up to {plan.indexed_posts} posts, and they are all "
+                                                       "used. Remove a post or upgrade to add more.")
+
+
 @router.post("/url")
 def import_url(body: SourceIn, ctx: Ctx = Depends(get_ctx)):
+    _require_room(ctx)
     source = imports_source(ctx.db, ctx.workspace.id)
     ctx.db.commit()
     job = create_job(ctx.db, ctx.workspace.id, "import_url", {"source_id": str(source.id), "url": body.url})
@@ -148,6 +157,7 @@ def import_upload(body: UploadImportIn, ctx: Ctx = Depends(get_ctx)):
     upload = ctx.db.scalar(select(Upload).where(Upload.id == body.upload_id, Upload.workspace_id == ctx.workspace.id))
     if not upload or upload.status != "complete":
         raise HTTPException(404, "Upload not found or not finished")
+    _require_room(ctx)
     source = imports_source(ctx.db, ctx.workspace.id)
     ctx.db.commit()
     job = create_job(ctx.db, ctx.workspace.id, "import_upload",

backend/app/routers/video_edit.py

@@ -4,7 +4,7 @@
 
 Only paths in RULES pass (anything else is 404). Before forwarding, the video must be this workspace's
 (b2v_access.video_or_404), every template / voice / style id in a JSON body must be this workspace's (check_refs),
-premium (★) features need a paid plan, and AI edits come out of the workspace's own counter (given back if
+AI edits come out of the workspace's own counter (given back if
 blog2video refuses). A template switch also uses one video, as it does on blog2video.
 
 JSON bodies, multipart uploads (images, voiceovers, portraits) and binary answers (downloads, stills) pass through.
@@ -35,8 +35,6 @@ S = r"\d+"  # a scene id
 @dataclass(frozen=True)
 class Rule:
     ai_edits: int = 0  # AI edits this call uses (per workspace, video_limits "ai_edits")
-    per_scene: bool = False  # ai_edits is per scene_ids entry (avatar batches)
-    premium: str | None = None  # ★: what to call it in the upgrade message
     uses_video: bool = False  # a template switch: one video from the allowance
     raw: bool = False  # binary answer (download, still)
     timeout: float = 30
@@ -94,29 +92,12 @@ RULES: list[tuple[str, re.Pattern, Rule]] = [(m, re.compile(f"^{p}$"), r) for m,
     ("GET", "regenerate-script/preview", Rule()),
     ("POST", "regenerate-script/verify", Rule()),
     ("POST", "regenerate-script/regenerate", Rule()),
-    # AI chat editing ★
-    ("POST", "chat", Rule(ai_edits=1, premium="AI chat editing", timeout=120)),
-    ("GET", "chat/history", Rule(premium="AI chat editing")),
-    # Avatars ★ (blog2video charges the whole batch at authorize: 10 AI edits per scene)
-    ("POST", "avatar-batch/authorize", Rule(ai_edits=10, per_scene=True, premium="Avatars")),
-    ("POST", rf"scenes/{S}/avatar", Rule(premium="Avatars")),
-    ("GET", rf"scenes/{S}/avatar-status", Rule()),
-    ("DELETE", rf"scenes/{S}/avatar", Rule()),
-    ("POST", rf"scenes/{S}/avatar-matte", Rule(premium="Avatars")),
-    ("POST", "avatar-matte-all", Rule(premium="Avatars")),
-    ("POST", "avatar-retry-failed", Rule(premium="Avatars")),
-    ("GET", "avatar-progress", Rule()),
-    ("POST", "avatar-portrait", Rule(premium="Avatars", timeout=60)),
-    ("DELETE", "avatar-portrait", Rule()),
-    ("PATCH", rf"scenes/{S}/avatar-focus", Rule()),
-    ("PATCH", rf"scenes/{S}/avatar-appearance", Rule()),
     # Render, download, frames
     ("POST", "render", Rule(status="rendering")),
     ("GET", "render-status", Rule()),
     ("POST", "cancel-render", Rule()),
     ("GET", "download-url", Rule()),
     ("GET", "download", Rule(raw=True, timeout=120)),
-    ("GET", "download-studio", Rule(raw=True, premium="The Remotion source download", timeout=120)),
     ("GET", "render-still", Rule(raw=True, timeout=60)),
     ("POST", "render-stills", Rule(timeout=120)),
 ]]
@@ -157,19 +138,12 @@ async def _read_body(request: Request) -> tuple[dict | None, dict | None, list |
 
 
 def _charge(ctx: Ctx, rule: Rule, body: dict | None, form: dict | None) -> int:
-    """Premium, references and counters, all before blog2video is called. Returns AI edits taken."""
+    """References and counters, all before blog2video is called. Returns AI edits taken."""
     plan = effective_plan(ctx.db, ctx.workspace)
-    if rule.premium:
-        video_limits.require_premium(ctx.db, ctx.workspace, rule.premium, plan)
     refs = body if isinstance(body, dict) else (form or {})
     with upstream():  # checking a template or voice may need blog2video's catalog
         b2v_access.check_refs(ctx, refs, plan)
     edits = rule.ai_edits
-    if rule.per_scene:
-        scenes = (body or {}).get("scene_ids") if isinstance(body, dict) else None
-        if not isinstance(scenes, list) or not scenes:
-            raise HTTPException(400, "Pick at least one scene")
-        edits *= len(scenes)
     if edits:
         video_limits.take(ctx.db, ctx.workspace, "ai_edits", edits, plan=plan)
     if rule.uses_video and not video_quota.reserve(ctx.db, ctx.workspace.id):

backend/app/routers/video_templates.py

@@ -179,7 +179,6 @@ def delete_template(template_id: int, ctx: Ctx = Depends(get_ctx), _: None = Dep
 
 @dataclass(frozen=True)
 class Rule:
-    ai_edit: bool = False  # ★ one AI edit
     metric: str | None = None  # another counter this uses (a lifetime slot, a daily AI step)
     raw: bool = False
 
@@ -190,8 +189,6 @@ RULES: list[tuple[str, re.Pattern, Rule]] = [(m, re.compile(f"^{p}$"), r) for m,
     ("PUT", "", Rule()),
     ("GET", "code", Rule()),
     ("POST", "upload-logo", Rule()),
-    ("POST", f"scenes/{K}/ai-edit", Rule(ai_edit=True)),
-    ("GET", f"scenes/{K}/ai-edit/status", Rule()),
     ("GET", "scene-drafts", Rule()),
     ("GET", f"scenes/{K}/draft", Rule()),
     ("POST", f"scenes/{K}/draft/apply", Rule()),
@@ -214,10 +211,6 @@ def _rule(method: str, path: str) -> Rule | None:
 def _take(ctx: Ctx, rule: Rule) -> list[str]:
     taken: list[str] = []
     try:
-        if rule.ai_edit:
-            plan = video_limits.require_premium(ctx.db, ctx.workspace, "AI template editing")
-            video_limits.take(ctx.db, ctx.workspace, "ai_edits", plan=plan)
-            taken.append("ai_edits")
         if rule.metric:
             video_limits.take(ctx.db, ctx.workspace, rule.metric)
             taken.append(rule.metric)

backend/app/routers/video_voices.py

@@ -1,24 +1,19 @@
-"""Voices for videos: the workspace's saved voices, the built-in library, custom voices (design, clone), and its
-Notestack voices (made on the Voice page; blog2video speaks with the same ElevenLabs account).
+"""Voices for videos: the workspace's saved voices, the free built-in library, and its Notestack voices (made on the Voice
+page; they speak audio overviews, not videos).
 
 "Your voices" (user_saved_voices, as many as wanted) is the one list for audio overviews and videos, and lives only
 here: blog2video's saved-voice list is one list for the whole account.
-Custom voices are made on our blog2video account, so b2v_custom_voices records which workspace made each one;
-only that workspace can use, play or delete it. Designing, keeping, cloning and samples are premium (★) and come out
-of the workspace's own daily and total counters (video_limits).
 """
 
 import logging
-from typing import Literal
 
-from fastapi import APIRouter, Depends, File, Form, HTTPException, Response, UploadFile
+from fastapi import APIRouter, Depends, HTTPException
 from pydantic import BaseModel, Field
 from sqlalchemy import select
 
 from app.auth import Ctx, get_ctx
-from app.models import B2VCustomVoice, UserSavedVoice
+from app.models import UserSavedVoice
 from app.routers.videos import b2v_ready, upstream
-from app.services import b2v_access, video_limits
 from app.services import blog2video as b2v
 from app.services.notestack_voices import notestack_voices, save_notestack_voice
 
@@ -26,7 +21,6 @@ log = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/api/video-voices", tags=["videos"])
 
-CLONE_MAX_BYTES = 50 * 1024 * 1024
 STARTER_VOICES = 4  # a new workspace starts with this many free built-in voices saved
 
 
@@ -36,7 +30,7 @@ def _labels(v: dict) -> dict:
 
 def _voice_out(v: UserSavedVoice, ours: set[str] = frozenset()) -> dict:
     return {"voice_id": v.voice_id, "name": v.name, "preview_url": v.preview_url, "gender": v.gender,
-            "accent": v.accent, "premium": v.premium, "is_custom": v.is_custom,
+            "accent": v.accent,
             "source": "notestack" if v.voice_id in ours else "blog2video"}
 
 
@@ -45,7 +39,7 @@ def _library_out(v: dict, saved: set[str]) -> dict:
     return {"voice_id": v["voice_id"], "name": (v.get("name") or "Voice").split(" - ")[0].strip(),
             "description": v.get("description") or labels.get("description") or "", "preview_url": v.get("preview_url"),
             "gender": labels.get("gender"), "accent": labels.get("accent"), "age": labels.get("age"),
-            "premium": v.get("plan") == "paid", "saved": v["voice_id"] in saved}
+            "saved": v["voice_id"] in saved}
 
 
 def _save_builtin(ctx: Ctx, v: dict) -> UserSavedVoice:
@@ -53,37 +47,36 @@ def _save_builtin(ctx: Ctx, v: dict) -> UserSavedVoice:
     row = UserSavedVoice(workspace_id=ctx.workspace.id, voice_id=v["voice_id"],
                          name=(v.get("name") or "Voice").split(" - ")[0].strip()[:255],
                          preview_url=v.get("preview_url"),
-                         gender=labels.get("gender"), accent=labels.get("accent"), premium=v.get("plan") == "paid")
+                         gender=labels.get("gender"), accent=labels.get("accent"))
     ctx.db.merge(row)
     return row
 
 
-def _saved(ctx: Ctx) -> list[UserSavedVoice]:
-    return list(ctx.db.scalars(select(UserSavedVoice).where(UserSavedVoice.workspace_id == ctx.workspace.id)
-                               .order_by(UserSavedVoice.created_at)))
+def _saved(ctx: Ctx, ours: set[str] = frozenset()) -> list[UserSavedVoice]:
+    """The saved voices that can still be used: free built-in voices and the workspace's Notestack voices. Voices that
+    were designed or cloned for videos, and paid built-in ones, are gone."""
+    rows = ctx.db.scalars(select(UserSavedVoice).where(UserSavedVoice.workspace_id == ctx.workspace.id)
+                          .order_by(UserSavedVoice.created_at))
+    return [v for v in rows if v.voice_id in ours or not (v.is_custom or v.premium)]
 
 
 @router.get("")
 def voices(ctx: Ctx = Depends(get_ctx), _: None = Depends(b2v_ready)):
-    """Your voices (audio overviews and videos), the built-in library (to save from) and my custom voices."""
+    """Your voices (audio overviews and videos), the free built-in library (to save from) and your Notestack voices."""
     with upstream():
-        library = b2v.prebuilt_voices()
-    saved = _saved(ctx)
+        library = [v for v in b2v.prebuilt_voices() if v.get("plan") != "paid"]
+    mine = notestack_voices(ctx.db, ctx.workspace.id)
+    ours = {v["voice_id"] for v in mine}
+    saved = _saved(ctx, ours)
     if not saved:
-        for v in [v for v in library if v.get("plan") != "paid" and v.get("voice_id")][:STARTER_VOICES]:
+        for v in [v for v in library if v.get("voice_id")][:STARTER_VOICES]:
             _save_builtin(ctx, v)
         ctx.db.commit()
-        saved = _saved(ctx)
+        saved = _saved(ctx, ours)
     ids = {v.voice_id for v in saved}
-    custom = ctx.db.scalars(select(B2VCustomVoice).where(B2VCustomVoice.workspace_id == ctx.workspace.id)
-                            .order_by(B2VCustomVoice.created_at.desc()))
-    mine = notestack_voices(ctx.db, ctx.workspace.id)
-    ours = {v["voice_id"] for v in mine}
     return {"saved": [_voice_out(v, ours) for v in saved],
             "notestack": [{**v, "saved": v["voice_id"] in ids} for v in mine],
-            "library": [_library_out(v, ids) for v in library if v.get("voice_id")],
-            "custom": [{"id": c.b2v_custom_voice_id, "voice_id": c.voice_id, "name": c.name, "source": c.source,
-                        "preview_url": c.preview_url, "saved": c.voice_id in ids} for c in custom]}
+            "library": [_library_out(v, ids) for v in library if v.get("voice_id")]}
 
 
 class SaveIn(BaseModel):
@@ -94,25 +87,17 @@ class SaveIn(BaseModel):
 def save_voice(body: SaveIn, ctx: Ctx = Depends(get_ctx), _: None = Depends(b2v_ready)):
     if ctx.db.get(UserSavedVoice, {"workspace_id": ctx.workspace.id, "voice_id": body.voice_id}):
         return _voice_out(ctx.db.get(UserSavedVoice, {"workspace_id": ctx.workspace.id, "voice_id": body.voice_id}))
-    custom = ctx.db.scalar(select(B2VCustomVoice).where(B2VCustomVoice.voice_id == body.voice_id,
-                                                        B2VCustomVoice.workspace_id == ctx.workspace.id))
     mine = next((v for v in notestack_voices(ctx.db, ctx.workspace.id) if v["voice_id"] == body.voice_id), None)
     if mine:
-        # A Notestack voice: saved on any plan, premium to use in a video. No preview link (the Voice page plays it).
+        # A Notestack voice: saved on any plan, for audio overviews. No preview link (the Voice page plays it).
         row = save_notestack_voice(ctx.db, ctx.workspace.id, mine["voice_id"], mine["name"])
         ctx.db.commit()
         return _voice_out(row, {row.voice_id})
-    if custom:
-        row = ctx.db.merge(UserSavedVoice(workspace_id=ctx.workspace.id, voice_id=custom.voice_id, name=custom.name,
-                                          preview_url=custom.preview_url, premium=True, is_custom=True))
-    else:
-        with upstream():
-            v = next((v for v in b2v.prebuilt_voices() if v.get("voice_id") == body.voice_id), None)
-        if not v:
-            raise HTTPException(404, "Voice not found")
-        if v.get("plan") == "paid":
-            video_limits.require_premium(ctx.db, ctx.workspace, "This voice")
-        row = _save_builtin(ctx, v)
+    with upstream():
+        v = next((v for v in b2v.prebuilt_voices() if v.get("voice_id") == body.voice_id), None)
+    if not v or v.get("plan") == "paid":
+        raise HTTPException(404, "Voice not found")
+    row = _save_builtin(ctx, v)
     ctx.db.commit()
     return _voice_out(row)
 
@@ -125,158 +110,3 @@ def unsave_voice(voice_id: str, ctx: Ctx = Depends(get_ctx)):
     ctx.db.delete(row)
     ctx.db.commit()
     return {"ok": True}
-
-
-# Design a voice ★: a description or options -> a few previews; keep one to make it a custom voice
-
-
-class DesignPromptIn(BaseModel):
-    prompt: str = Field(min_length=20, max_length=1000)
-
-
-class DesignPresetIn(BaseModel):
-    gender: str = Field("", max_length=30)
-    age: str = Field("", max_length=30)
-    persona: str = Field("", max_length=100)
-    speed: str = Field("", max_length=30)
-    accent: str = Field("", max_length=60)
-
-
-def _design(ctx: Ctx, path: str, body: dict) -> dict:
-    plan = video_limits.require_premium(ctx.db, ctx.workspace, "Designing a voice")
-    video_limits.take(ctx.db, ctx.workspace, "voice_designs_daily", plan=plan)
-    try:
-        with upstream():
-            data = b2v.request("POST", path, json=body, timeout=90) or {}
-    except HTTPException:
-        video_limits.give_back(ctx.db, ctx.workspace.id, "voice_designs_daily")
-        raise
-    return {"previews": [{k: p.get(k) for k in ("generated_voice_id", "audio_base_64", "media_type", "duration_secs")}
-                         for p in data.get("previews") or []]}
-
-
-@router.post("/design/prompt")
-def design_from_prompt(body: DesignPromptIn, ctx: Ctx = Depends(get_ctx), _: None = Depends(b2v_ready)):
-    return _design(ctx, "/api/voices/design-from-prompt", body.model_dump())
-
-
-@router.post("/design/preset")
-def design_from_preset(body: DesignPresetIn, ctx: Ctx = Depends(get_ctx), _: None = Depends(b2v_ready)):
-    return _design(ctx, "/api/voices/design-from-preset", body.model_dump())
-
-
-class KeepIn(BaseModel):
-    generated_voice_id: str = Field(min_length=1, max_length=200)
-    source: Literal["prompt", "preset"]
-    name: str = Field(min_length=1, max_length=100)
-    prompt_text: str | None = Field(None, max_length=1000)
-    preview_url: str | None = Field(None, max_length=1000)
-
-
-def _record_custom(ctx: Ctx, made: dict, source: str) -> dict:
-    row = B2VCustomVoice(b2v_custom_voice_id=int(made["id"]), workspace_id=ctx.workspace.id,
-                         voice_id=made["voice_id"], name=(made.get("name") or "My voice")[:255], source=source,
-                         preview_url=made.get("preview_url"))
-    ctx.db.add(row)
-    ctx.db.merge(UserSavedVoice(workspace_id=ctx.workspace.id, voice_id=row.voice_id, name=row.name,
-                                preview_url=row.preview_url, premium=True, is_custom=True))  # a new voice is saved
-    ctx.db.commit()
-    return {"id": row.b2v_custom_voice_id, "voice_id": row.voice_id, "name": row.name, "source": source,
-            "preview_url": row.preview_url, "saved": True}
-
-
-def _make_custom(ctx: Ctx, source: str, send) -> dict:
-    plan = video_limits.require_premium(ctx.db, ctx.workspace, "Custom voices")
-    video_limits.take(ctx.db, ctx.workspace, "custom_voices", plan=plan)
-    try:
-        with upstream():
-            made = send()
-    except HTTPException:
-        video_limits.give_back(ctx.db, ctx.workspace.id, "custom_voices")
-        raise
-    return _record_custom(ctx, made, source)
-
-
-@router.post("/custom", status_code=201)
-def keep_designed_voice(body: KeepIn, ctx: Ctx = Depends(get_ctx), _: None = Depends(b2v_ready)):
-    """Keep one designed preview as a permanent custom voice."""
-    payload = {"voice_id": body.generated_voice_id, "source": "prompt" if body.source == "prompt" else "form",
-               "name": body.name, "prompt_text": body.prompt_text, "preview_url": body.preview_url}
-    return _make_custom(ctx, body.source,
-                        lambda: b2v.request("POST", "/api/voices/custom", json=payload, timeout=60))
-
-
-@router.post("/clone", status_code=201)
-async def clone_voice(name: str = Form(..., min_length=1, max_length=100), remove_background_noise: bool = Form(True),
-                      file: UploadFile = File(...), ctx: Ctx = Depends(get_ctx), _: None = Depends(b2v_ready)):
-    """Clone a voice from one audio or video file (up to 50 MB)."""
-    data = await file.read(CLONE_MAX_BYTES + 1)
-    if not data:
-        raise HTTPException(400, "The file is empty.")
-    if len(data) > CLONE_MAX_BYTES:
-        raise HTTPException(400, "The file must be under 50 MB.")
-    form = {"name": name, "remove_background_noise": "true" if remove_background_noise else "false"}
-    files = {"file": (file.filename or "voice.mp3", data, file.content_type or "application/octet-stream")}
-    return _make_custom(ctx, "clone",
-                        lambda: b2v.request("POST", "/api/voices/clone", data=form, files=files, timeout=180))
-
-
-@router.get("/custom/{custom_id}/preview")
-def custom_preview(custom_id: int, ctx: Ctx = Depends(get_ctx), _: None = Depends(b2v_ready)):
-    row = b2v_access.owned_custom_voice(ctx, custom_id)
-    if row.preview_url:
-        return {"preview_url": row.preview_url, "ready": True}
-    with upstream():
-        result = b2v.request("GET", f"/api/voices/custom/{custom_id}/preview") or {}
-    if result.get("preview_url"):
-        row.preview_url = result["preview_url"]
-        saved = ctx.db.get(UserSavedVoice, {"workspace_id": ctx.workspace.id, "voice_id": row.voice_id})
-        if saved:
-            saved.preview_url = row.preview_url
-        ctx.db.commit()
-    return {"preview_url": result.get("preview_url"), "ready": bool(result.get("ready"))}
-
-
-@router.delete("/custom/{custom_id}")
-def delete_custom(custom_id: int, ctx: Ctx = Depends(get_ctx), _: None = Depends(b2v_ready)):
-    row = b2v_access.owned_custom_voice(ctx, custom_id)
-    try:
-        with upstream():
-            b2v.request("DELETE", f"/api/voices/custom/{custom_id}")
-    except HTTPException as e:
-        if e.status_code != 404:
-            raise
-    saved = ctx.db.get(UserSavedVoice, {"workspace_id": ctx.workspace.id, "voice_id": row.voice_id})
-    if saved:
-        ctx.db.delete(saved)
-    ctx.db.delete(row)
-    ctx.db.commit()
-    video_limits.give_back(ctx.db, ctx.workspace.id, "custom_voices")
-    return {"ok": True}
-
-
-# Voice sample ★: a few seconds of the chosen voice with the chosen tuning
-
-
-class SampleIn(BaseModel):
-    voice_gender: Literal["female", "male"] = "female"
-    voice_accent: str = Field("american", max_length=30)
-    custom_voice_id: str | None = Field(None, max_length=100)
-    voice_emotion: str | None = Field(None, max_length=200)
-    video_style: str | None = Field(None, max_length=40)
-
-
-@router.post("/sample")
-def voice_sample(body: SampleIn, ctx: Ctx = Depends(get_ctx), _: None = Depends(b2v_ready)):
-    plan = video_limits.require_premium(ctx.db, ctx.workspace, "Voice samples")
-    payload = body.model_dump(exclude_none=True)
-    with upstream():  # checking the voice may need blog2video's catalog
-        b2v_access.check_refs(ctx, {k: v for k, v in payload.items() if k != "voice_emotion"}, plan)
-    video_limits.take(ctx.db, ctx.workspace, "voice_samples_daily", plan=plan)
-    try:
-        with upstream():
-            audio, ctype = b2v.request("POST", "/api/voice/preview", json=payload, timeout=60, raw=True)
-    except HTTPException:
-        video_limits.give_back(ctx.db, ctx.workspace.id, "voice_samples_daily")
-        raise
-    return Response(audio, media_type=ctype or "audio/mpeg", headers={"Cache-Control": "no-store"})

backend/app/routers/videos.py

@@ -140,7 +140,7 @@ class VideoOptions(BaseModel):
     # Step 1: Project (script review is always off: blog2video goes straight to the video)
     stock_footage_enabled: bool = True
     aspect_ratio: Literal["landscape", "portrait"] = "landscape"
-    video_length: Literal["short", "medium", "detailed", "more_detailed"] = "short"
+    video_length: Literal["short", "medium"] = "short"
     logo_upload_id: uuid.UUID | None = None  # a file uploaded through /api/storage/uploads
     logo_position: Literal["bottom_right", "bottom_left", "top_left", "top_right"] = "bottom_right"
     logo_opacity: float = Field(0.9, ge=0.1, le=1.0)
@@ -155,7 +155,6 @@ class VideoOptions(BaseModel):
     voice_gender: Literal["female", "male", "none"] = "female"
     voice_accent: str = Field("american", max_length=30)
     custom_voice_id: str | None = Field(None, max_length=100)
-    voice_emotion: str | None = Field(None, max_length=200)  # ["<expressiveness>","<speed>","<emotion>",...]
     bgm_track_id: str | None = Field(None, max_length=100)
     bgm_volume: float = Field(0.10, ge=0, le=1)
 
@@ -167,7 +166,6 @@ class VideoOptions(BaseModel):
         data["script_review_enabled"] = False
         if self.voice_gender == "none":
             data.pop("custom_voice_id", None)
-            data.pop("voice_emotion", None)
         if not self.bgm_track_id:
             data.pop("bgm_volume", None)
         return data
@@ -369,9 +367,6 @@ def _check(ctx: Ctx, options: VideoOptions) -> tuple[dict, Plan]:
         if length := b2v_access.house_length(payload.get("video_style")):
             payload["video_length"] = length
         plan = b2v_access.check_refs(ctx, payload)
-    accent = (ctx.workspace.brand_json or {}).get("accent")
-    if plan.brand_kit and accent and "accent_color" not in payload:
-        payload["accent_color"] = accent
     return payload, plan
 
 
@@ -626,10 +621,10 @@ def quota(ctx: Ctx = Depends(get_ctx)):
 
 def _with_posters(t: dict) -> dict:
     """Poster images for the template picker. Built-in templates have baked posters on blog2video's web app (a few
-    have none; the picker then shows the template's colors); designer templates carry their own image."""
+    have none; the picker then shows the template's colors)."""
     if t.get("preview_image_url") and not t.get("preview_url"):
         return {**t, "preview_url": t["preview_image_url"]}
-    if t.get("preview_url") or not t.get("id") or str(t["id"]).startswith("crafted_"):
+    if t.get("preview_url") or not t.get("id"):
         return t
     base = f"{settings.b2v_app_url.rstrip('/')}/template-posters/{t['id']}"
     return {**t, "preview_url": f"{base}.webp", "preview_portrait_url": f"{base}-portrait.webp"}
@@ -637,20 +632,15 @@ def _with_posters(t: dict) -> dict:
 
 @router.get("/catalog")
 def catalog(ctx: Ctx = Depends(get_ctx), _: None = Depends(b2v_ready)):
-    """What the wizard offers this workspace: built-in and designer templates, its own ready custom templates,
+    """What the wizard offers this workspace: built-in templates, its own ready custom templates,
     built-in and its own styles, music, languages. Nothing another workspace made."""
     with upstream():
         builtin = b2v.builtin_templates()
-        try:
-            crafted = b2v.crafted_templates()
-        except b2v.B2VError:
-            crafted = []
         mine = [{"id": f"custom_{t.b2v_template_id}", "name": t.name, "custom": True}
                 for t in b2v_access.my_templates(ctx, ready_only=True)]
         styles = b2v_access.visible_styles(ctx)
         music = b2v.music_tracks()
     return {"templates": [_with_posters(t) for t in builtin if isinstance(t, dict)],
-            "crafted_templates": [_with_posters(t) for t in crafted if isinstance(t, dict)],
             "my_templates": mine, "video_styles": styles,
             "music": music, "languages": [{"code": c, "name": n} for c, n in LANGUAGES]}
 
@@ -888,10 +878,6 @@ class ReviewPreviewIn(BaseModel):
     revision: int = Field(0, ge=0)
 
 
-class ReviewAiIn(ReviewPreviewIn):
-    instruction: str = Field(min_length=2, max_length=1000)
-
-
 class ReviewApproveScene(DraftScene):
     preferred_layout: str | None = Field(None, max_length=100)
     source_fingerprint: str | None = Field(None, max_length=200)
@@ -923,22 +909,6 @@ def narration_preview(artifact_id: uuid.UUID, scene_id: int, body: ReviewPreview
                                 json=body.model_dump(), timeout=60)
 
 
-@router.post("/{artifact_id}/script/scenes/{scene_id}/ai-preview")
-def ai_preview(artifact_id: uuid.UUID, scene_id: int, body: ReviewAiIn, ctx: Ctx = Depends(get_ctx),
-               _: None = Depends(b2v_ready)):
-    """★ Rewrite one scene from an instruction. Uses one AI edit."""
-    _, vid, _ = b2v_access.video_or_404(ctx, artifact_id)
-    plan = video_limits.require_premium(ctx.db, ctx.workspace, "AI rewrites")
-    video_limits.take(ctx.db, ctx.workspace, "ai_edits", plan=plan)
-    try:
-        with upstream():
-            return b2v.project_call("POST", vid, f"script-review/scenes/{scene_id}/ai-preview",
-                                    json=body.model_dump(), timeout=60)
-    except HTTPException:
-        video_limits.give_back(ctx.db, ctx.workspace.id, "ai_edits")
-        raise
-
-
 @router.post("/{artifact_id}/script/approve")
 def approve_script(artifact_id: uuid.UUID, body: ReviewApproveIn, ctx: Ctx = Depends(get_ctx),
                    _: None = Depends(b2v_ready)):

backend/app/services/artifact_feedback.py

@@ -0,0 +1,47 @@
+"""Thumbs up or down on generated reports, quizzes, flashcards and infographics: what can be rated, why a thumbs down can be
+given, and how the ratings ride along on the artifacts the app lists."""
+
+import uuid
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.models import Artifact, ArtifactFeedback
+
+# What each kind can be marked down for (the ids the app sends; anything else is dropped).
+REASONS: dict[str, set[str]] = {
+    "report": {"inaccurate", "missed_points", "structure", "tone", "too_long", "too_short", "other"},
+    "quiz": {"wrong_answers", "too_easy", "too_hard", "unclear", "off_topic", "other"},
+    "flashcards": {"inaccurate", "too_basic", "too_detailed", "unclear", "other"},
+    "infographic": {"messy_layout", "cut_off", "missing_content", "look", "hard_to_read", "other"},
+}
+RATEABLE = tuple(REASONS)
+
+
+def feedback_out(f: ArtifactFeedback | None) -> dict | None:
+    if not f:
+        return None
+    return {"rating": "up" if f.rating > 0 else "down", "reasons": f.reasons or [], "comment": f.comment}
+
+
+def attach(db: Session, user_id: uuid.UUID, items: list[dict]) -> list[dict]:
+    """Add `feedback` (this person's rating, or None) to each serialized artifact that can be rated. One query."""
+    ids = [uuid.UUID(i["id"]) for i in items if i.get("type") in RATEABLE]
+    rows = {f.artifact_id: f for f in db.scalars(select(ArtifactFeedback).where(
+        ArtifactFeedback.artifact_id.in_(ids), ArtifactFeedback.user_id == user_id))} if ids else {}
+    for i in items:
+        if i.get("type") in RATEABLE:
+            i["feedback"] = feedback_out(rows.get(uuid.UUID(i["id"])))
+    return items
+
+
+def snapshot(a: Artifact) -> dict:
+    """What the artifact was when it was rated: enough to tell what a rating is about."""
+    c = a.content_json or {}
+    inner = c.get("content") if isinstance(c.get("content"), dict) else {}
+    keep = {k: c.get(k) for k in ("title", "prompt", "source", "theme", "format", "template_id", "difficulty", "question_count",
+                                  "card_count", "language", "topic") if c.get(k) is not None}
+    if a.type == "infographic":
+        keep["layout"] = "composed" if "body_html" in inner else "fixed"
+        keep["fit"] = inner.get("fit")
+    return keep

backend/app/services/b2v_access.py

@@ -1,6 +1,6 @@
 """Who owns what on our one blog2video account.
 
-blog2video's ownership checks stop at the account: any video, custom template, custom voice or style made with our
+blog2video's ownership checks stop at the account: any video, custom template or style made with our
 key can be used, listed and edited with it. So every route that takes a blog2video id checks here first that it
 belongs to the current workspace (404 otherwise, so another workspace's ids are never confirmed), and every create or
 edit body has its references checked before it is forwarded (check_refs).
@@ -12,11 +12,9 @@ from fastapi import HTTPException
 from sqlalchemy import select
 
 from app.auth import Ctx
-from app.models import Artifact, B2VCustomVoice, B2VStyle, B2VTemplate, B2VVideo
+from app.models import Artifact, B2VStyle, B2VTemplate, B2VVideo
 from app.services import blog2video as b2v
-from app.services.notestack_voices import is_notestack_voice
 from app.services.plans import Plan, effective_plan
-from app.services.video_limits import require_premium
 
 BUILTIN_STYLES = {"auto", "explainer", "storytelling", "promotional"}
 # Custom styles made on our blog2video account itself, offered to every workspace in place of the others when they
@@ -29,7 +27,6 @@ HOUSE_STYLES = [  # (name on blog2video, video length, what the wizard says it i
                             "scene building on the last. Best for YouTube, tutorials and full explainers. Usually 1 to "
                             "2 minutes long."),
 ]
-PREMIUM_LENGTHS = {"detailed", "more_detailed", "mdetailed"}
 
 
 def not_found(what: str = "Not found") -> HTTPException:
@@ -79,14 +76,6 @@ def owned_template(ctx: Ctx, template_id: int) -> B2VTemplate:
     return row
 
 
-def owned_custom_voice(ctx: Ctx, custom_voice_id: int) -> B2VCustomVoice:
-    row = ctx.db.scalar(select(B2VCustomVoice).where(B2VCustomVoice.b2v_custom_voice_id == custom_voice_id,
-                                                     B2VCustomVoice.workspace_id == ctx.workspace.id))
-    if not row:
-        raise not_found("Voice not found")
-    return row
-
-
 def owned_style(ctx: Ctx, style_id: int) -> B2VStyle:
     row = ctx.db.scalar(select(B2VStyle).where(B2VStyle.b2v_style_id == style_id,
                                                B2VStyle.workspace_id == ctx.workspace.id))
@@ -100,15 +89,6 @@ def _custom_id(ref: str, prefix: str) -> int | None:
     return int(raw) if ref.startswith(prefix) and raw.isdigit() else None
 
 
-def crafted_ids() -> set[str]:
-    out = set()
-    for t in b2v.crafted_templates():
-        tid = t.get("id") if isinstance(t, dict) else t
-        if isinstance(tid, str) and tid.startswith("crafted_"):
-            out.add(tid)
-    return out
-
-
 def check_template(ctx: Ctx, template: str, plan: Plan) -> None:
     if template == "default":
         return
@@ -117,12 +97,6 @@ def check_template(ctx: Ctx, template: str, plan: Plan) -> None:
         if not row.ready:
             raise HTTPException(400, "That template is still being made.")
         return
-    if template.startswith("crafted_"):
-        # blog2video silently falls back to "default" for an unknown id, so check it exists.
-        if template not in crafted_ids():
-            raise not_found("Template not found")
-        require_premium(ctx.db, ctx.workspace, "Designer templates", plan)
-        return
     if template not in {t.get("id") for t in b2v.builtin_templates() if isinstance(t, dict)}:
         raise not_found("Template not found")
 
@@ -157,38 +131,26 @@ def check_style(ctx: Ctx, style: str) -> None:
     raise not_found("Video style not found")  # includes your_style, which learns from every workspace's edits
 
 
-def check_voice(ctx: Ctx, voice_id: str, plan: Plan) -> None:
-    """A built-in voice (premium if blog2video marks it paid), or one of this workspace's custom or Notestack
-    voices."""
-    prebuilt = {v.get("voice_id"): v for v in b2v.prebuilt_voices()}
-    if voice_id in prebuilt:
-        if prebuilt[voice_id].get("plan") == "paid":
-            require_premium(ctx.db, ctx.workspace, "This voice", plan)
-        return
-    if ctx.db.scalar(select(B2VCustomVoice).where(B2VCustomVoice.voice_id == voice_id,
-                                                  B2VCustomVoice.workspace_id == ctx.workspace.id)):
-        require_premium(ctx.db, ctx.workspace, "Custom voices", plan)
-        return
-    # One of this workspace's Notestack voices (clone, designed, added): same ElevenLabs account, premium like custom.
-    if is_notestack_voice(ctx.db, ctx.workspace.id, voice_id):
-        require_premium(ctx.db, ctx.workspace, "Custom voices", plan)
-        return
-    raise not_found("Voice not found")
+def check_voice(voice_id: str) -> None:
+    """A free built-in voice (blog2video marks the paid ones)."""
+    voice = next((v for v in b2v.prebuilt_voices() if v.get("voice_id") == voice_id), None)
+    if not voice or voice.get("plan") == "paid":
+        raise not_found("Voice not found")
 
 
 def check_refs(ctx: Ctx, body: dict, plan: Plan | None = None) -> Plan:
-    """Every id in a create or edit body must be this workspace's, and premium options need a paid plan."""
+    """Every id in a create or edit body must be this workspace's, and only the standard options are accepted."""
     plan = plan or effective_plan(ctx.db, ctx.workspace)
     if body.get("template"):
         check_template(ctx, str(body["template"]), plan)
     if body.get("video_style"):
         check_style(ctx, str(body["video_style"]))
     if body.get("custom_voice_id") and body.get("voice_gender") != "none":
-        check_voice(ctx, str(body["custom_voice_id"]), plan)
-    if str(body.get("video_length") or "") in PREMIUM_LENGTHS:
-        require_premium(ctx.db, ctx.workspace, "Longer videos", plan)
+        check_voice(str(body["custom_voice_id"]))
+    if str(body.get("video_length") or "") not in {"", "short", "medium"}:
+        raise HTTPException(400, "That video length is not available.")
     if body.get("voice_emotion"):
-        require_premium(ctx.db, ctx.workspace, "Advanced voice options", plan)
+        raise HTTPException(400, "Voice tuning is not available.")
     return plan
 
 

backend/app/services/billing.py

@@ -10,7 +10,7 @@ from sqlalchemy import func, select
 from sqlalchemy.orm import Session
 
 from app.config import settings
-from app.models import Artifact, CalendarItem, Notebook, Source, Subscription, User, VoiceProfile, Workspace
+from app.models import Artifact, CalendarItem, Document, Notebook, Source, Subscription, User, VoiceProfile, Workspace
 from app.services.plans import PLANS, Plan, effective_plan, next_plan, plan_dict
 from app.services.usage import month_usage
 from app.services.video_quota import sync_video_quota, video_usage
@@ -27,15 +27,30 @@ METERS = {
 
 def _meters(plan: Plan, used: dict, sources: int) -> list[dict]:
     rows = []
-    for key, (label, unit) in METERS.items():
+    meters = dict(METERS)
+    if plan.audio_overviews >= 0:  # a count of audio overviews (Free), not minutes
+        meters = {("audio_overviews" if k == "audio_minutes" else k): (("Audio overviews", "") if k == "audio_minutes" else v)
+                  for k, v in METERS.items()}
+    for key, (label, unit) in meters.items():
         limit = getattr(plan, key)
         rows.append({"key": key, "label": label, "unit": unit, "used": used[key], "limit": limit,
                      "pct": 0 if limit < 0 else round(min(1.0, used[key] / max(limit, 1)), 3)})
     rows.append({"key": "sources", "label": "Sources", "unit": "", "used": sources, "limit": plan.sources,
-                 "pct": round(min(1.0, sources / max(plan.sources, 1)), 3)})
+                 "pct": 0 if plan.sources < 0 else round(min(1.0, sources / max(plan.sources, 1)), 3)})
     return rows
 
 
+def _more_meters(db: Session, workspace: Workspace, plan: Plan, used: dict) -> list[dict]:
+    """The rest of the plan's limits, shown when the sidebar card is opened: indexed posts, reports and infographics."""
+    posts = db.scalar(select(func.count()).select_from(Document)
+                      .where(Document.workspace_id == workspace.id, Document.path.is_not(None))) or 0
+    rows = [("indexed_posts", "Posts indexed", posts, plan.indexed_posts),
+            ("reports", "Reports", used["reports"], plan.reports),
+            ("infographics", "Infographics", used["infographics"], plan.infographics)]
+    return [{"key": k, "label": label, "unit": "", "used": u, "limit": limit,
+             "pct": 0 if limit < 0 else round(min(1.0, u / max(limit, 1)), 3)} for k, label, u, limit in rows]
+
+
 def _count(db: Session, model, workspace_id, *where) -> int:
     return db.scalar(select(func.count()).select_from(model).where(model.workspace_id == workspace_id, *where)) or 0
 
@@ -57,23 +72,20 @@ def build_nudges(db: Session, workspace: Workspace, plan: Plan, meters: list[dic
                 continue
             label = m["label"].lower()
             amount = f"{m['limit']:g} min of {label}" if m["unit"] else f"{m['limit']:g} {label}"
+            when = "" if plan.lifetime else " this month"
             if m["pct"] >= 1:
-                out.append(_nudge(f"empty-{m['key']}-{month}", "limit", f"Out of {label} this month",
+                out.append(_nudge(f"empty-{m['key']}-{month}", "limit", f"Out of {label}{when}",
                                   f"You have used all {amount}. Upgrade to keep launching.", "Refuel now",
                                   priority=100))
             elif m["pct"] >= LOW_FUEL:
                 left = f"{m['limit'] - m['used']:g}" + (" min" if m["unit"] else "")
                 out.append(_nudge(f"low-{m['key']}-{month}", "limit", f"Running low on {label}",
-                                  f"Only {left} left this month. Upgrade before you stall mid launch.",
+                                  f"Only {left} left{when}. Upgrade before you stall mid launch.",
                                   "See plans", priority=90))
         if not plan.voice_cloning:
             out.append(_nudge("perk-voice", "upgrade", "Narrate in your own voice",
                               "Writer unlocks voice cloning, so every audio overview sounds like you.",
                               "Unlock voice", priority=40))
-        if not plan.brand_kit:
-            out.append(_nudge("perk-brand", "upgrade", "Put your brand on every render",
-                              "Your colors and logo on videos, quote cards and carousels.", "Unlock brand kit",
-                              priority=30))
 
     sources = _count(db, Source, ws)
     if sources == 0:
@@ -119,7 +131,7 @@ def build_nudges(db: Session, workspace: Workspace, plan: Plan, meters: list[dic
 
 def billing_status(db: Session, workspace: Workspace) -> dict:
     plan = effective_plan(db, workspace)
-    used = month_usage(db, workspace.id)
+    used = month_usage(db, workspace.id, plan.lifetime)
     videos = video_usage(db, workspace)
     used["videos"] = videos["used"]
     sources = _count(db, Source, workspace.id)
@@ -136,6 +148,7 @@ def billing_status(db: Session, workspace: Workspace) -> dict:
         "has_billing_account": bool(sub and sub.provider_customer_id),
         "period_end": sub.current_period_end.isoformat() if sub and sub.current_period_end else None,
         "meters": meters,
+        "more_meters": _more_meters(db, workspace, plan, used),
         "since": used["since"],
         "videos_resets_at": videos["resets_at"],
         "nudges": build_nudges(db, workspace, plan, meters, can_upgrade),

backend/app/services/notestack_voices.py

@@ -1,6 +1,5 @@
 """The workspace's own ElevenLabs voices, made or added on the Voice page: its clone, designed voices, and voices added
-from the ElevenLabs library. blog2video speaks with the same ElevenLabs account, so these can be used in videos too
-(as `custom_voice_id`); video_voices lists them and b2v_access accepts them."""
+from the ElevenLabs library. They speak audio overviews; video_voices lists them with the workspace's saved voices."""
 
 import uuid
 
@@ -32,8 +31,8 @@ def is_notestack_voice(db: Session, workspace_id: uuid.UUID, voice_id: str) -> b
 
 
 def save_notestack_voice(db: Session, workspace_id: uuid.UUID, voice_id: str, name: str) -> UserSavedVoice:
-    """Put a Notestack voice in the workspace's voices (the one list for audio and video), once. Saving needs no plan:
-    using it in a video is premium (b2v_access.check_voice). No preview link: it is played through /api/voice/preview.
+    """Put a Notestack voice in the workspace's voices (the one list for audio and video), once. Saving needs no plan; it speaks audio
+    overviews (a video uses the free built-in voices). No preview link: it is played through /api/voice/preview.
     The caller commits."""
     row = db.get(UserSavedVoice, {"workspace_id": workspace_id, "voice_id": voice_id})
     if row is None:

backend/app/services/plans.py

@@ -31,36 +31,32 @@ class Plan:
     indexed_posts: int
     audio_minutes: int
     videos: int  # blog2video videos per month, or in total when videos_monthly is False
-    launch_kits: int  # -1 = unlimited
+    launch_kits: int  # a month (or in total on a lifetime plan); -1 = unlimited
     voice_cloning: bool
-    brand_kit: bool
     features: tuple[str, ...]
-    # Premium (★) video options: long durations, paid and custom voices, voice tuning and samples, AI script
-    # rewrites, AI chat editing, avatars, the Remotion source download.
-    video_premium: bool = False
     # Per-workspace shares of what blog2video only limits account-wide (keys in VIDEO_LIMITS; 0 = none).
     video_limits: dict = field(default_factory=dict)
     videos_monthly: bool = True
     reports: int = 0  # reports a month; -1 = unlimited
-    infographics: int = 0  # infographics a month; -1 = unlimited
+    infographics: int = 0  # infographics a month (in total on a lifetime plan); -1 = unlimited
+    audio_overviews: int = -1  # audio overviews allowed (in total on a lifetime plan), whatever their length; -1 = no cap
+
+    @property
+    def lifetime(self) -> bool:
+        """The allowances for videos, audio overviews, Launch Kits and infographics are totals that never reset (Free)."""
+        return not self.videos_monthly
 
 
 # metric -> period it counts over (app/services/video_limits.py)
 VIDEO_LIMITS = {
-    "ai_edits": "month",  # scene regenerate/add, AI images, stock clips, AI chat, template AI edits, avatars
+    "ai_edits": "month",  # scene regenerate/add, AI images, stock clips
     "templates": "all",  # custom templates: blog2video's slots are lifetime, never given back
     "template_ai_daily": "day",  # template theme extraction from a prompt or document, code generation
-    "voice_designs_daily": "day",
-    "voice_samples_daily": "day",
-    "custom_voices": "all",  # kept voices; deleting one frees its place
 }
 
-FREE_VIDEO_LIMITS = {"ai_edits": 20, "templates": 0, "template_ai_daily": 0, "voice_designs_daily": 0,
-                     "voice_samples_daily": 0, "custom_voices": 0}
-WRITER_VIDEO_LIMITS = {"ai_edits": 300, "templates": 2, "template_ai_daily": 3, "voice_designs_daily": 5,
-                       "voice_samples_daily": 20, "custom_voices": 3}
-STUDIO_VIDEO_LIMITS = {"ai_edits": 1000, "templates": 5, "template_ai_daily": 5, "voice_designs_daily": 10,
-                       "voice_samples_daily": 40, "custom_voices": 10}
+FREE_VIDEO_LIMITS = {"ai_edits": 20, "templates": 0, "template_ai_daily": 0}
+WRITER_VIDEO_LIMITS = {"ai_edits": 300, "templates": 2, "template_ai_daily": 3}
+STUDIO_VIDEO_LIMITS = {"ai_edits": 1000, "templates": 5, "template_ai_daily": 5}
 
 
 DEFAULT_PLANS: dict[str, Plan] = {
@@ -69,23 +65,23 @@ DEFAULT_PLANS: dict[str, Plan] = {
         name="Free",
         tagline="For trying Notestack on your archive",
         price_monthly_usd=0,
-        sources=1,
+        sources=-1,
         indexed_posts=5,
-        audio_minutes=3,
+        audio_minutes=-1,
+        audio_overviews=1,
         videos=1,
         launch_kits=2,
         reports=2,
         infographics=2,
         voice_cloning=False,
-        brand_kit=False,
         features=(
-            "1 source, your latest 5 posts indexed",
+            "Up to 5 posts indexed, from any source",
             "Grounded research chat with citations",
-            "3 min of audio overviews a month",
-            "1 video to try it",
-            "2 Launch Kits a month",
+            "1 audio overview, any length",
+            "1 video",
+            "2 Launch Kits",
+            "2 infographics",
             "2 reports a month",
-            "2 infographics a month",
         ),
         video_limits=FREE_VIDEO_LIMITS,
         videos_monthly=False,
@@ -95,26 +91,23 @@ DEFAULT_PLANS: dict[str, Plan] = {
         name="Writer",
         tagline="For writers publishing every week",
         price_monthly_usd=24.99,
-        sources=3,
-        indexed_posts=500,
+        sources=-1,
+        indexed_posts=100,
         audio_minutes=60,
         videos=10,
-        launch_kits=50,
+        launch_kits=20,
         reports=30,
         infographics=30,
         voice_cloning=True,
-        brand_kit=True,
         features=(
-            "3 sources, 500 indexed posts",
-            "60 min of audio overviews a month",
+            "Up to 100 posts indexed, from any source",
+            "10x more audio overviews than Free (60 min a month)",
             "10 videos a month",
-            "50 Launch Kits a month",
+            "20 Launch Kits a month",
             "30 reports a month",
             "30 infographics a month",
             "Voice cloning with consent",
-            "Your brand colors and logo",
         ),
-        video_premium=True,
         video_limits=WRITER_VIDEO_LIMITS,
     ),
     "studio": Plan(
@@ -122,26 +115,24 @@ DEFAULT_PLANS: dict[str, Plan] = {
         name="Studio",
         tagline="For publications and power users",
         price_monthly_usd=48.99,
-        sources=10,
-        indexed_posts=5000,
+        sources=-1,
+        indexed_posts=250,
         audio_minutes=240,
         videos=20,
-        launch_kits=-1,
+        launch_kits=50,
         reports=-1,
         infographics=-1,
         voice_cloning=True,
-        brand_kit=True,
         features=(
-            "10 sources, 5,000 indexed posts",
-            "240 min of audio overviews a month",
+            "Up to 250 posts indexed, from any source",
+            "40x more audio overviews than Free (240 min a month)",
             "20 videos a month",
-            "Unlimited Launch Kits",
+            "50 Launch Kits a month",
             "Unlimited reports",
             "Unlimited infographics",
             "Launchpad calendar and resurfacing",
             "Priority rendering",
         ),
-        video_premium=True,
         video_limits=STUDIO_VIDEO_LIMITS,
     ),
 }
@@ -154,10 +145,10 @@ def _plan_from_row(row: PlanRecord) -> Plan:
     return Plan(id=row.id, name=row.name, tagline=row.tagline or "",
                 price_monthly_usd=float(row.price_monthly_usd or 0), sources=row.sources,
                 indexed_posts=row.indexed_posts, audio_minutes=row.audio_minutes,
-                videos=row.videos, launch_kits=row.launch_kits, reports=row.reports, infographics=row.infographics,
+                videos=row.videos, launch_kits=row.launch_kits, reports=row.reports, infographics=row.infographics, audio_overviews=row.audio_overviews,
                 voice_cloning=bool(row.voice_cloning),
-                brand_kit=bool(row.brand_kit), features=tuple(row.features or ()),
-                video_premium=bool(row.video_premium), videos_monthly=bool(row.videos_monthly),
+                features=tuple(row.features or ()),
+                videos_monthly=bool(row.videos_monthly),
                 video_limits={**{k: 0 for k in VIDEO_LIMITS}, **(row.video_limits or {})})
 
 
@@ -251,6 +242,22 @@ def effective_plan(db: Session, workspace: Workspace) -> Plan:
     return PLANS.get(sub.plan, PLANS["free"])
 
 
+def post_room(db: Session, workspace_id, plan: Plan, source_id=None) -> int:
+    """How many more posts the plan lets this workspace index: its limit across every source, link and upload, less the
+    posts indexed already. With `source_id`, that source's own posts are left out of the count, and a source never loses
+    posts it already has indexed (a plan that went down keeps what is stored; it only stops new ones)."""
+    from sqlalchemy import func
+
+    from app.models import Document
+
+    base = select(func.count()).select_from(Document).where(Document.workspace_id == workspace_id, Document.path.is_not(None))
+    total = db.scalar(base) or 0
+    if source_id is None:
+        return max(plan.indexed_posts - total, 0)
+    own = db.scalar(base.where(Document.source_id == source_id)) or 0
+    return max(plan.indexed_posts - (total - own), own)
+
+
 def plan_order() -> list[str]:
     return PLANS.order()
 

backend/app/services/renderer.py

@@ -57,15 +57,3 @@ def request_render(job: Job, artifact: Artifact, composition: str, props: dict |
     if resp.status_code == 400:
         raise PermanentJobError(f"Renderer rejected the props: {resp.text[:400]}")
     resp.raise_for_status()
-
-
-def brand_props(workspace, *, ttl: int = 6 * 3600) -> dict:
-    brand = workspace.brand_json or {}
-    out: dict = {}
-    if brand.get("accent"):
-        out["accent"] = brand["accent"]
-    if brand.get("name"):
-        out["name"] = brand["name"]
-    if brand.get("logo_key"):
-        out["logoUrl"] = storage.presign_get(brand["logo_key"], ttl=ttl)
-    return out

backend/app/services/report_suggestions.py

@@ -0,0 +1,148 @@
+"""The suggested templates in the Create report dialog: written by the AI from each post's stored ideas and topics, kept by a hash of
+what it read, prepared in the background as soon as ideas are extracted, and served at once whenever the dialog opens."""
+
+import hashlib
+import logging
+import uuid
+
+from sqlalchemy import select
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from app.config import settings
+from app.llm import run
+from app.llm.provider import suggest_templates_lm
+from app.llm.signatures import SuggestReportTemplates
+from app.models import Document, Notebook, ReportTemplateCache
+from app.pipeline.generate import ideas_fresh
+from app.pipeline.material import max_posts
+from app.pipeline.passages import notebook_docs
+from app.services.jobs import create_job, pending_job
+
+log = logging.getLogger("notestack")
+
+SUGGEST_ITEMS = 4  # sources the AI reads to suggest templates
+SUMMARY_IDEAS = 3  # a post's stored ideas shown to the AI
+SUMMARY_CHARS = 350  # most characters of one post's summary
+WARM_NOTEBOOKS = 30  # notebooks prepared per run (the most recently changed)
+KEEP_PER_WORKSPACE = 10  # stored sets kept per workspace; the oldest go
+
+
+def summary(d: Document) -> str:
+    """A post as the AI sees it when suggesting templates: its title, its topics and the key ideas already extracted from it (the
+    ones the Mind Constellation and the topic map are built from), a few hundred characters at most."""
+    meta = d.metadata_json or {}
+    ideas = [f"{i['label'].strip()}: {(i.get('note') or '').strip()}" for i in meta.get("ideas") or [] if (i.get("label") or "").strip()]
+    tags = [t["name"] for t in meta.get("topics") or [] if t.get("name")][:3]
+    head = d.title + (f"\nTopics: {', '.join(tags)}" if tags else "")
+    return (head + "\n" + "\n".join(f"- {x}" for x in ideas[:SUMMARY_IDEAS]))[:SUMMARY_CHARS]
+
+
+def stable_order(docs: list[Document]) -> list[Document]:
+    """The posts in an order that belongs to the posts, not to the selection: the same posts come first whenever they are picked.
+    The templates are read from the first few, so ticking or unticking any other post gives the same input, the same key, and the
+    templates kept for it (the ones prepared in the background for the whole notebook) are served at once, not written again."""
+    return sorted(docs, key=lambda d: hashlib.sha256(str(d.id).encode()).hexdigest())
+
+
+def suggestion_items(material: dict[str, str], docs: list[Document]) -> list[str]:
+    """What the AI reads: each post's stored summary when it has one (else the start of the post, as `material` holds it), or for
+    chats their first exchange. At most SUGGEST_ITEMS, from the posts that come first in stable_order."""
+    if not docs:
+        return list(material.values())[:SUGGEST_ITEMS]
+    out = []
+    for d in stable_order(docs):
+        item = summary(d) if ideas_fresh(d) else material.get(str(d.id))
+        if item:
+            out.append(item)
+        if len(out) == SUGGEST_ITEMS:
+            break
+    return out
+
+
+def cache_key(items: list[str], topic: str) -> str:
+    return hashlib.sha256(("\x1f".join(items) + "\x1e" + (topic or "").strip()).encode()).hexdigest()
+
+
+def _clean(out: dict) -> list[dict]:
+    seen, found = set(), []
+    for t in out.get("templates") or []:
+        name, prompt = (t.get("name") or "").strip(), (t.get("prompt") or "").strip()
+        if not name or not prompt or name.lower() in seen:
+            continue
+        seen.add(name.lower())
+        found.append({"id": f"suggested-{len(found) + 1}", "name": name[:60],
+                      "description": (t.get("description") or "").strip()[:200], "prompt": prompt[:1500]})
+    return found[:4]
+
+
+def suggest(db: Session, workspace_id: uuid.UUID, items: list[str], topic: str = "") -> list[dict]:
+    """Four templates for these sources: the stored ones when the same sources were seen before, else written now (one AI call) and
+    stored. A failed or empty answer is never stored. Never raises."""
+    if not items:
+        return []
+    key = cache_key(items, topic)
+    hit = db.scalar(select(ReportTemplateCache).where(ReportTemplateCache.workspace_id == workspace_id, ReportTemplateCache.key == key))
+    if hit and hit.templates:
+        log.info("report templates: kept set served workspace=%s key=%s", workspace_id, key[:10])
+        return hit.templates
+    log.info("report templates: no kept set, asking the model workspace=%s key=%s items=%d", workspace_id, key[:10], len(items))
+    try:
+        out = run.predict(SuggestReportTemplates, db=db, workspace_id=workspace_id, lm=suggest_templates_lm(), items=items,
+                          topic=(topic or "").strip() or "(none)")
+    except Exception:
+        log.warning("template suggestions failed", exc_info=True)
+        return []
+    templates = _clean(out)
+    if templates:
+        _store(db, workspace_id, key, templates)
+    return templates
+
+
+def _store(db: Session, workspace_id: uuid.UUID, key: str, templates: list[dict]) -> None:
+    try:
+        row = db.scalar(select(ReportTemplateCache).where(ReportTemplateCache.workspace_id == workspace_id, ReportTemplateCache.key == key))
+        if row:
+            row.templates = templates
+        else:
+            db.add(ReportTemplateCache(workspace_id=workspace_id, key=key, templates=templates))
+        db.commit()
+    except IntegrityError:  # another request stored the same set first
+        db.rollback()
+        return
+    old = db.scalars(select(ReportTemplateCache).where(ReportTemplateCache.workspace_id == workspace_id)
+                     .order_by(ReportTemplateCache.updated_at.desc()).offset(KEEP_PER_WORKSPACE)).all()
+    for row in old:
+        db.delete(row)
+    if old:
+        db.commit()
+
+
+def queue_warmup(db: Session, workspace_id: uuid.UUID) -> None:
+    """Have the templates for the workspace's notebooks prepared in the background (once at a time). Called when ideas have been
+    extracted and when posts join a notebook, so they are ready before anyone opens the dialog."""
+    if settings.llm_api_key and not pending_job(db, workspace_id, "report_templates"):
+        create_job(db, workspace_id, "report_templates", {}, max_attempts=2)
+
+
+def warm_workspace(db: Session, workspace_id: uuid.UUID) -> int:
+    """Write and keep the templates of each recently changed notebook for the posts its dialog starts with (the newest ones, as the
+    dialog ticks them), skipping a notebook whose posts have mostly no stored ideas yet (they are prepared once the ideas land) and
+    any set already stored. Returns how many were written."""
+    from app.routers.videos import _excerpt, is_locked  # the dialog's own preview of a post, so the keys match
+
+    written = 0
+    for nb in db.scalars(select(Notebook).where(Notebook.workspace_id == workspace_id)
+                         .order_by(Notebook.updated_at.desc()).limit(WARM_NOTEBOOKS)):
+        docs = [d for d in notebook_docs(db, nb.id) if not is_locked(d)][:max_posts()]
+        if not docs or sum(1 for d in docs if ideas_fresh(d)) * 2 < len(docs):
+            continue
+        docs.sort(key=lambda d: d.published_at or d.created_at, reverse=True)
+        each = 800 if len(docs) == 1 else 400
+        material = {str(d.id): "\n".join(p for p in (d.title or "", _excerpt(d.clean_text, each)) if p) for d in docs}
+        items = suggestion_items(material, docs)
+        key = cache_key(items, "")
+        if items and not db.scalar(select(ReportTemplateCache.id).where(ReportTemplateCache.workspace_id == workspace_id,
+                                                                         ReportTemplateCache.key == key)):
+            written += bool(suggest(db, workspace_id, items, ""))
+    return written