LETS GOOOOOOOOOOOOOOOOOOOOOOOOO

backend/app/pipeline/ingest.py

@@ -418,8 +418,15 @@ def _plain_links(url: str) -> list[str]:
     return links
 
 
-def crawl_site(url: str, limit: int, on_found=None, on_post=None) -> list[FeedEntry]:
-    """Find the posts on a site with no feed and read each one."""
+def _title_from_url(u: str) -> str:
+    slug = urlparse(u).path.rstrip("/").rsplit("/", 1)[-1]
+    slug = re.sub(r"\.[a-z0-9]+$", "", slug)
+    return slug.replace("-", " ").replace("_", " ").strip().capitalize() or urlparse(u).netloc
+
+
+def crawl_site(url: str, limit: int, on_found=None, on_post=None, read_limit: int | None = None) -> list[FeedEntry]:
+    """Find the posts on a site with no feed and read each one. Past read_limit, pages are only listed (title from the
+    URL, no html): they show the writer what the rest of their archive holds without paying to read it."""
     if firecrawl.enabled():
         links = firecrawl.map_site(url, max(limit * 4, 100))
     else:
@@ -437,20 +444,24 @@ def crawl_site(url: str, limit: int, on_found=None, on_post=None) -> list[FeedEn
         except (FeedNotFound, httpx.HTTPError, firecrawl.FirecrawlError):
             return None
 
+    to_read = urls if read_limit is None else urls[:read_limit]
     entries: list[FeedEntry] = []
     with ThreadPoolExecutor(max_workers=4) as pool:
-        for i, entry in enumerate(pool.map(read, urls)):
+        for i, entry in enumerate(pool.map(read, to_read)):
             if entry:
                 entries.append(entry)
             if on_post:
-                on_post(i + 1, len(urls))
+                on_post(i + 1, len(to_read))
+    entries += [FeedEntry(title=_title_from_url(u), url=u, published_at=None, html="") for u in urls[len(to_read):]]
     if not entries:
         raise FeedNotFound(f"We found {len(urls)} pages on {urlparse(url).netloc} but none had article text.")
     return entries
 
 
-def fetch_substack_archive(site: str, known_urls: set[str], limit: int, on_page=None) -> list[FeedEntry]:
-    """Posts beyond the ~20 the RSS feed carries, through Substack's public archive API."""
+def fetch_substack_archive(site: str, known_urls: set[str], limit: int, on_page=None,
+                           body_budget: int | None = None) -> list[FeedEntry]:
+    """Posts beyond the ~20 the RSS feed carries, through Substack's public archive API. Only the first body_budget
+    posts get their body fetched; the rest are listed from the archive metadata alone (title, date, link)."""
     out: list[FeedEntry] = []
     offset = 0
     with httpx.Client(follow_redirects=True, timeout=30, headers=UA) as client:
@@ -468,6 +479,10 @@ def fetch_substack_archive(site: str, known_urls: set[str], limit: int, on_page=
                     continue
                 if len(out) + len(known_urls) >= limit:
                     break
+                if body_budget is not None and len(out) >= body_budget:
+                    out.append(FeedEntry(title=item.get("title") or "Untitled", url=url,
+                                         published_at=_parse_date(item.get("post_date")), html=""))
+                    continue
                 post = client.get(f"{site}/api/v1/posts/{item['slug']}")
                 if post.status_code >= 400:
                     continue
@@ -575,9 +590,9 @@ def store_entries(
     return indexed, skipped, changed
 
 
-# A sync reads up to SCAN_CAP posts (or the plan's own limit, if higher). Posts past the plan's indexed_posts are
-# stored as locked: listed with their title and text, but with no corpus file, so chat, notebooks and generation
-# never see them. Upgrading re-syncs and unlocks them without asking the writer to reconnect anything.
+# A sync discovers up to SCAN_CAP posts (or the plan's own limit, if higher) but only indexes the plan's
+# indexed_posts. The rest are listed, not indexed: title, date and link only, no text and no corpus file. They exist
+# to show the writer how much of their archive is waiting behind an upgrade. Upgrading re-syncs and indexes them.
 SCAN_CAP = 200
 
 
@@ -588,14 +603,11 @@ def _newest_first(entries: list[FeedEntry]) -> list[FeedEntry]:
 
 
 def store_locked(db: Session, source: Source, entries: list[FeedEntry]) -> tuple[int, set[str]]:
-    """Index posts beyond the plan limit without making them available. Returns (count, corpus paths to remove):
-    a post that was available before (a newer post pushed it out, or the plan went down) loses its corpus file."""
+    """List posts beyond the plan limit without indexing them: title, date and link only. Returns (count, corpus paths
+    to remove): a post that was indexed before (a newer post pushed it out, or the plan went down) loses its file."""
     locked = 0
     remove: set[str] = set()
     for entry in entries:
-        sections = entry.sections if entry.sections is not None else html_to_sections(entry.html)
-        if not sections:
-            continue
         doc = db.scalar(select(Document).where(Document.source_id == source.id, Document.url == entry.url))
         if not doc:
             doc = Document(id=uuid.uuid4(), workspace_id=source.workspace_id, source_id=source.id, url=entry.url)
@@ -605,14 +617,9 @@ def store_locked(db: Session, source: Source, entries: list[FeedEntry]) -> tuple
             doc.path = None
         doc.title = entry.title
         doc.published_at = entry.published_at
-        doc.content_hash = None  # forces a full store (corpus file, raw HTML) once the post unlocks
-        doc.clean_text = "\n\n".join(f"## {h}\n{t}" if h else t for h, t in sections)
-        doc.metadata_json = {
-            **(doc.metadata_json or {}),
-            "headings": [h for h, _ in sections if h],
-            "words": len(doc.clean_text.split()),
-            "locked": True,
-        }
+        doc.content_hash = None  # indexed from scratch once the plan allows it
+        doc.clean_text = ""
+        doc.metadata_json = {"locked": True}
         locked += 1
     db.commit()
     return locked, remove
@@ -630,6 +637,7 @@ def ingest_source(db: Session, source: Source, job: Job, max_posts: int) -> dict
             on_found=lambda n: update_job(db, job, progress=0.05, message=f"Found {n} pages, reading them"),
             on_post=lambda done, total: update_job(db, job, progress=0.05 + 0.05 * done / total,
                                                    message=f"Read {done} of {total} pages"),
+            read_limit=max_posts,
         )
         title = None
     else:
@@ -642,6 +650,7 @@ def ingest_source(db: Session, source: Source, job: Job, max_posts: int) -> dict
             entries += fetch_substack_archive(
                 site, {e.url for e in entries}, fetch_limit,
                 on_page=lambda n: update_job(db, job, message=f"Found {len(entries) + n} posts in the archive"),
+                body_budget=max(0, max_posts - len(entries)),
             )
         except (httpx.HTTPError, ValueError):
             log.warning("substack archive fetch failed for %s", site, exc_info=True)
@@ -658,7 +667,7 @@ def ingest_source(db: Session, source: Source, job: Job, max_posts: int) -> dict
     locked = 0
     if beyond:
         update_job(db, job, progress=0.85,
-                   message=f"Indexing {len(beyond)} more posts. Your plan makes the latest {max_posts} available")
+                   message=f"Found {len(beyond)} more posts. Your plan indexes the latest {max_posts}")
         locked, remove = store_locked(db, source, beyond)
         if remove:
             Corpus(source.workspace_id).write_files({INDEX: rebuild_index(db, source.workspace_id)}, remove=remove)

backend/app/services/plans.py

@@ -41,7 +41,7 @@ PLANS: dict[str, Plan] = {
         voice_cloning=False,
         brand_kit=False,
         features=(
-            "1 source, your 5 latest posts",
+            "1 source, whole archive indexed, latest 5 posts available",
             "Grounded research chat with citations",
             "3 min of audio overviews a month",
             "1 min of video a month",

backend/app/worker.py

@@ -92,8 +92,8 @@ def handle_ingest(db: Session, job: Job):
         raise
     _after_import(db, job, source.workspace_id, result.get("changed", []))
     if result.get("locked"):
-        return Done(result, f"All {result['found']} posts indexed. Your latest {result['available']} are available; "
-                            f"upgrade to unlock the other {result['locked']}")
+        return Done(result, f"Found {result['found']} posts. Your latest {result['available']} are indexed; "
+                            f"upgrade to index the other {result['locked']}")
     return Done(result, f"All posts in orbit: {result['indexed']} new or updated")
 
 

backend/tests/test_locked_posts.py

@@ -0,0 +1,91 @@
+"""Posts past the plan's indexed_posts are listed, not indexed: no text, no corpus file, indexed on a bigger plan."""
+
+import uuid
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import select
+
+from app.models import Document, Job, Source, Workspace
+from app.pipeline import ingest
+from app.pipeline.ingest import FeedEntry, ingest_source
+from app.routers.sources import is_locked
+
+
+def _entries(n: int) -> list[FeedEntry]:
+    start = datetime(2026, 1, 1, tzinfo=UTC)
+    # Oldest first on purpose: ingest must pick the newest posts regardless of feed order.
+    return [
+        FeedEntry(title=f"Post {i}", url=f"https://ada.example/p/{i}", published_at=start + timedelta(days=i),
+                  html=f"<h2>Point {i}</h2><p>Paragraph for post number {i} with enough words to keep.</p>")
+        for i in range(n)
+    ]
+
+
+def _setup(db, monkeypatch, n: int):
+    monkeypatch.setattr(ingest, "fetch_feed", lambda url, limit: ("Ada", _entries(n)[:limit]))
+    ws = Workspace(name="w", owner_id=uuid.uuid4())
+    db.add(ws)
+    db.flush()
+    source = Source(workspace_id=ws.id, feed_url="https://ada.example/feed", platform="rss", title="Ada")
+    db.add(source)
+    db.flush()
+    job = Job(workspace_id=ws.id, kind="ingest", params={"source_id": str(source.id)})
+    db.add(job)
+    db.commit()
+    return ws, source, job
+
+
+def _docs(db, source):
+    return {d.title: d for d in db.scalars(select(Document).where(Document.source_id == source.id))}
+
+
+def test_lists_every_post_but_indexes_only_the_latest(db_session, monkeypatch):
+    ws, source, job = _setup(db_session, monkeypatch, 12)
+    result = ingest_source(db_session, source, job, max_posts=5)
+
+    assert result["found"] == 12 and result["available"] == 5 and result["locked"] == 7 and result["capped"]
+    docs = _docs(db_session, source)
+    assert len(docs) == 12
+    available = {t for t, d in docs.items() if d.path}
+    assert available == {f"Post {i}" for i in range(7, 12)}  # the five newest
+    locked = [d for d in docs.values() if not d.path]
+    assert len(locked) == 7
+    assert all(is_locked(d) and d.clean_text == "" and d.title for d in locked)
+
+
+def test_upgrade_unlocks_and_newer_posts_relock_the_oldest(db_session, monkeypatch):
+    ws, source, job = _setup(db_session, monkeypatch, 12)
+    ingest_source(db_session, source, job, max_posts=5)
+
+    ingest_source(db_session, source, job, max_posts=500)
+    docs = _docs(db_session, source)
+    assert all(d.path for d in docs.values())
+    assert not any(is_locked(d) for d in docs.values())
+
+    ingest_source(db_session, source, job, max_posts=5)
+    docs = _docs(db_session, source)
+    assert sum(1 for d in docs.values() if d.path) == 5
+    assert docs["Post 0"].path is None and is_locked(docs["Post 0"])
+
+
+def test_scan_is_capped_per_sync(db_session, monkeypatch):
+    ws, source, job = _setup(db_session, monkeypatch, ingest.SCAN_CAP + 50)
+    result = ingest_source(db_session, source, job, max_posts=5)
+    assert result["found"] == ingest.SCAN_CAP
+
+
+def test_crawl_reads_only_the_pages_it_will_index(monkeypatch):
+    read = []
+    urls = [f"https://ada.example/posts/post-{i}" for i in range(20)]
+    monkeypatch.setattr(ingest, "_plain_links", lambda url: urls)
+    monkeypatch.setattr(ingest, "_post_urls", lambda links, url, limit: links[:limit])
+
+    def fake_extract(u):
+        read.append(u)
+        return FeedEntry(title=u, url=u, published_at=None, html="<p>body text for the page</p>")
+
+    monkeypatch.setattr(ingest, "extract_article", fake_extract)
+    entries = ingest.crawl_site("https://ada.example", 200, read_limit=5)
+    assert len(read) == 5 and len(entries) == 20
+    assert all(e.html == "" for e in entries[5:])
+    assert entries[5].title == "Post 5"

frontend/seo/prerender.ts

@@ -182,7 +182,7 @@ function routes(): Rendered[] {
       description:
         "Notestack pricing: Free ($0), Writer ($24.99 a month, $18.99 billed annually) and Studio ($48.99 a month, $36.99 billed annually). Annual billing saves 24%.",
       body: page(
-        `<h1>Pick your trajectory</h1><ul><li><h2>Free</h2><p>$0. 1 source, first 5 posts indexed, 3 audio minutes, 1 video minute, 2 Launch Kits a month.</p></li><li><h2>Writer</h2><p>$24.99 a month ($18.99 a month billed annually). 3 sources, 500 posts, 60 audio minutes, 30 video minutes, 50 Launch Kits.</p></li><li><h2>Studio</h2><p>$48.99 a month ($36.99 a month billed annually). 10 sources, 5,000 posts, 240 audio minutes, 120 video minutes, unlimited Launch Kits.</p></li></ul><p><a href="/auth?mode=signup">Start free</a></p>`,
+        `<h1>Pick your trajectory</h1><ul><li><h2>Free</h2><p>$0. 1 source, whole archive indexed with your latest 5 posts available, 3 audio minutes, 1 video minute, 2 Launch Kits a month.</p></li><li><h2>Writer</h2><p>$24.99 a month ($18.99 a month billed annually). 3 sources, 500 posts, 60 audio minutes, 30 video minutes, 50 Launch Kits.</p></li><li><h2>Studio</h2><p>$48.99 a month ($36.99 a month billed annually). 10 sources, 5,000 posts, 240 audio minutes, 120 video minutes, unlimited Launch Kits.</p></li></ul><p><a href="/auth?mode=signup">Start free</a></p>`,
       ),
     },
     {

frontend/seo/sitemap.ts

@@ -96,7 +96,7 @@ differs from NotebookLM.
   SEO pack, carousel).
 - **Publishing:** a Launchpad calendar that auto posts to X, LinkedIn and Bluesky, plus a topic map of rising and
   dormant themes and evergreen resurfacing of old posts.
-- **Pricing (USD):** Free: 1 source, first 5 posts indexed, 3 audio minutes, 1 video minute, 2 launch kits a month.
+- **Pricing (USD):** Free: 1 source, whole archive indexed with the latest 5 posts available, 3 audio minutes, 1 video minute, 2 launch kits a month.
   Writer: $24.99 a month ($18.99 a month billed annually): 3 sources, 500 posts, 60 audio minutes, 30 video minutes,
   50 launch kits. Studio: $48.99 a month ($36.99 a month billed annually): 10 sources, 5,000 posts, 240 audio
   minutes, 120 video minutes, unlimited launch kits. Annual billing saves 24%.

frontend/src/api/types.ts

@@ -25,6 +25,8 @@ export type Source = {
   sync_error: string | null;
   last_synced_at: string | null;
   document_count: number;
+  /** Indexed but past the plan's post limit: listed, unlocked by upgrading. */
+  locked_count: number;
   is_imports: boolean;
 };
 
@@ -38,9 +40,11 @@ export type Doc = {
   published_at: string | null;
   words: number;
   evergreen_score: number | null;
+  /** Indexed but past the plan's post limit: preview only until the workspace upgrades. */
+  locked: boolean;
 };
 
-export type DocDetail = Doc & { lines: string[] };
+export type DocDetail = Doc & { lines: string[]; preview?: boolean; limit?: number };
 
 export type Page<T> = { total: number; items: T[] };
 

frontend/src/components/PricingTiers.tsx

@@ -26,7 +26,7 @@ const FALLBACK: Plan[] = [
     tagline: "For trying Notestack on your archive",
     price_monthly_usd: 0,
     features: [
-      "1 source, your 5 latest posts",
+      "1 source, whole archive indexed, latest 5 posts available",
       "Grounded research chat with citations",
       "3 min of audio overviews a month",
       "1 min of video a month",

frontend/src/components/Reader.tsx

@@ -1,6 +1,7 @@
 import { useEffect, useRef, useState } from "react";
 import { docsApi } from "../api/endpoints";
 import type { DocDetail } from "../api/types";
+import { useUpgrade } from "../hooks/useUpgrade";
 import { Drawer, formatDate, Loading } from "./ui";
 
 /** Reads a post from the corpus with the exact line numbers citations point at. */
@@ -16,6 +17,7 @@ export function Reader({
   const [doc, setDoc] = useState<DocDetail | null>(null);
   const [error, setError] = useState<string | null>(null);
   const first = useRef<HTMLDivElement>(null);
+  const { openUpgrade } = useUpgrade();
 
   useEffect(() => {
     setDoc(null);
@@ -69,6 +71,20 @@ export function Reader({
               );
             })}
           </div>
+          {doc.preview && (
+            <div className="welcome-cap locked-cap">
+              <div>
+                <strong>This post is indexed but locked</strong>
+                <p className="muted small">
+                  Your plan makes your latest {doc.limit ?? 5} posts available. Upgrade to read it in full and use it in
+                  notebooks, audio, video and Launch Kits.
+                </p>
+              </div>
+              <button type="button" className="btn btn-small btn-primary" onClick={() => openUpgrade("indexed_posts")}>
+                Upgrade to unlock
+              </button>
+            </div>
+          )}
         </article>
       )}
     </Drawer>

frontend/src/pages/Sources.tsx

@@ -4,6 +4,7 @@ import type { Doc, Job, Source } from "../api/types";
 import { Reader } from "../components/Reader";
 import { ConfirmButton, EmptyState, errorMessage, formatDate, JobProgress, Loading, PageHeader, StatusPill, Tabs } from "../components/ui";
 import { useJobMap } from "../hooks/useJob";
+import { useUpgrade } from "../hooks/useUpgrade";
 
 type AddMode = "feed" | "url" | "file";
 
@@ -86,6 +87,7 @@ export function AddSource({ onAdded }: { onAdded: (source: Source, job: Job) =>
 
 export default function Sources() {
   const [sources, setSources] = useState<Source[] | null>(null);
+  const { openUpgrade } = useUpgrade();
   const [docs, setDocs] = useState<Doc[]>([]);
   const [total, setTotal] = useState(0);
   const [q, setQ] = useState("");
@@ -160,6 +162,7 @@ export default function Sources() {
                   <strong>{s.title ?? s.feed_url}</strong>
                   <span className="mono muted">
                     {s.is_imports ? "imports" : s.platform} · {s.document_count} posts
+                    {s.locked_count > 0 ? ` (${s.document_count - s.locked_count} available, ${s.locked_count} locked)` : ""}
                     {s.last_synced_at ? ` · synced ${formatDate(s.last_synced_at, true)}` : ""}
                   </span>
                 </div>
@@ -171,6 +174,17 @@ export default function Sources() {
                 </a>
               )}
               {job && (active || job.status === "failed") && <JobProgress job={job} compact />}
+              {s.locked_count > 0 && !active && (
+                <div className="welcome-cap locked-cap">
+                  <div>
+                    <strong>{s.locked_count} posts indexed and locked</strong>
+                    <p className="muted small">Your plan makes your latest posts available. Upgrade to unlock the whole archive.</p>
+                  </div>
+                  <button type="button" className="btn btn-small btn-primary" onClick={() => openUpgrade("indexed_posts")}>
+                    Upgrade to unlock
+                  </button>
+                </div>
+              )}
               {s.sync_status === "error" && s.sync_error && !active && <p className="error-text">{s.sync_error}</p>}
               <footer className="row">
                 {!s.is_imports && (
@@ -222,6 +236,11 @@ export default function Sources() {
                 <button className="link-btn doc-title" onClick={() => setReading(d.id)}>
                   {d.title}
                 </button>
+                {d.locked && (
+                  <span className="mono muted small" title="Indexed. Upgrade to use this post">
+                    Locked
+                  </span>
+                )}
                 <span className="mono muted">
                   {d.source_title ?? ""} · {formatDate(d.published_at)} · {d.words} words
                 </span>

frontend/src/pages/Welcome.tsx

@@ -144,9 +144,34 @@ export default function Welcome() {
 
   const indexingDone = live?.status === "done";
   const indexingFailed = live?.status === "failed";
-  // The ingest job reports when the plan's post cap cut the archive short.
-  const capped = live?.result?.capped === true;
-  const capLimit = Number(live?.result?.limit ?? posts);
+  // Every post is indexed; the ingest job reports how many the plan makes available and how many are locked.
+  const locked = Number(live?.result?.locked ?? 0);
+  const capped = indexingDone && locked > 0;
+  const available = Number(live?.result?.available ?? posts);
+  const found = Number(live?.result?.found ?? posts);
+  const lockedNotice = capped ? (
+    <div className="welcome-cap">
+      <div>
+        <strong>
+          All {found} posts indexed. Your latest {available} are available on the free plan.
+        </strong>
+        <p className="muted small">
+          Upgrade to unlock the other {locked}, so answers, audio and Launch Kits draw on your whole archive. Nothing to
+          reconnect: they unlock the moment you upgrade.
+        </p>
+      </div>
+      <button
+        type="button"
+        className="btn btn-small"
+        onClick={() => {
+          markOnboarded();
+          navigate("/app?upgrade=1", { replace: true });
+        }}
+      >
+        Upgrade to unlock {locked}
+      </button>
+    </div>
+  ) : null;
   const first = (user.name ?? user.email).split(/[ @]/)[0];
 
   const finish = async (chosen: Goal | null) => {
@@ -288,6 +313,7 @@ export default function Welcome() {
             <p className="eyebrow">{source?.title ?? "Your archive"}</p>
             <h1>{indexingFailed ? "That did not land" : "Bringing your posts into orbit"}</h1>
             {live && <JobProgress job={live} />}
+            {lockedNotice}
             {indexingFailed ? (
               <div className="row">
                 <button className="btn btn-primary" onClick={() => setStep("connect")}>
@@ -310,26 +336,9 @@ export default function Welcome() {
 
         {step === "goal" && (
           <section className="card welcome-card">
-            <p className="eyebrow">{indexingDone ? (capped ? `${posts} posts indexed, free plan limit` : `${posts} posts indexed`) : "Still indexing in the background"}</p>
+            <p className="eyebrow">{indexingDone ? (capped ? `${available} of ${found} posts available` : `${posts} posts indexed`) : "Still indexing in the background"}</p>
             <h1>What should we launch first?</h1>
-            {capped && (
-              <div className="welcome-cap">
-                <div>
-                  <strong>Your free plan indexed your newest {capLimit} posts</strong>
-                  <p className="muted small">Your archive has more. Upgrade to bring every post into orbit, so answers, audio and Launch Kits draw on all of it.</p>
-                </div>
-                <button
-                  type="button"
-                  className="btn btn-small"
-                  onClick={() => {
-                    markOnboarded();
-                    navigate("/app?upgrade=1", { replace: true });
-                  }}
-                >
-                  Index everything
-                </button>
-              </div>
-            )}
+            {lockedNotice}
             <div className="goal-grid">
               {GOALS.map((g) => (
                 <button key={g.id} type="button" className={`goal${goal === g.id ? " on" : ""}`} onClick={() => setGoal(g.id)} aria-pressed={goal === g.id}>
@@ -350,26 +359,9 @@ export default function Welcome() {
         {step === "ready" && (
           <section className="card welcome-card">
             <p className="eyebrow">Pre-flight check complete</p>
-            <h1>{indexingDone ? `${posts} posts are in orbit` : "Almost there"}</h1>
+            <h1>{indexingDone ? `${capped ? available : posts} posts are in orbit` : "Almost there"}</h1>
             {!indexingDone && live && <JobProgress job={live} />}
-            {capped && (
-              <div className="welcome-cap">
-                <div>
-                  <strong>Your free plan indexed your newest {capLimit} posts</strong>
-                  <p className="muted small">Your archive has more. Upgrade to bring every post into orbit, so answers, audio and Launch Kits draw on all of it.</p>
-                </div>
-                <button
-                  type="button"
-                  className="btn btn-small"
-                  onClick={() => {
-                    markOnboarded();
-                    navigate("/app?upgrade=1", { replace: true });
-                  }}
-                >
-                  Index everything
-                </button>
-              </div>
-            )}
+            {lockedNotice}
             <p className="muted">
               {goal === "ask" && "We will put your whole archive in a notebook so you can start asking right away."}
               {goal === "audio" && "We will put your archive in a notebook with the audio studio ready."}