refactor: improve code readability and organization across multiple files

backend/app/chat_memory/organize.py

@@ -37,7 +37,7 @@ FALLBACK_TOPIC = "General chat"
 # A new topic whose words overlap an existing topic's this much is the same topic, so its rounds join that one.
 MERGE_OVERLAP = 0.5  # shared words as a share of the smaller of the two word sets
 MERGE_MIN_SHARED = 4  # and at least this many shared words, so two short lists cannot match by luck
-REFUSAL_OPENING = "I only answer from the posts in this notebook"  # the canned off-topic reply, for rows saved before kinds
+REFUSAL_OPENING = "I only answer from the posts in this notebook"  # canned off-topic reply, rows saved before kinds
 
 
 def chat_corpus(workspace_id: uuid.UUID, notebook_id: uuid.UUID, store=None, cache_dir: str | None = None) -> Corpus:

backend/app/pipeline/generate.py

@@ -10,7 +10,15 @@ from sqlalchemy.orm import Session
 from app.corpus import Corpus, slugify
 from app.llm import run
 from app.llm.provider import fast_lm
-from app.llm.signatures import ArrangeMindMap, BuildVoiceProfile, ConsolidateTopics, EvergreenScore, ExtractIdeas, ExtractTopics, SummarizeNotebook
+from app.llm.signatures import (
+    ArrangeMindMap,
+    BuildVoiceProfile,
+    ConsolidateTopics,
+    EvergreenScore,
+    ExtractIdeas,
+    ExtractTopics,
+    SummarizeNotebook,
+)
 from app.models import Artifact, DocumentTopic, Job, Notebook, Topic, VoiceProfile
 from app.pipeline.passages import as_citations, notebook_docs, passages_for, tools_for, verify_refs, workspace_docs
 from app.pipeline.research import verify_citations
@@ -118,7 +126,7 @@ def extract_ideas(db: Session, job: Job | None, workspace_id: uuid.UUID, doc_ids
             continue
         tools = tools_for(corpus, [d])
 
-        def keep(items) -> list[dict]:
+        def keep(items, tools=tools, d=d) -> list[dict]:
             return [{"path": r["path"], "line_start": r["line_start"], "line_end": r["line_end"]}
                     for r in verify_refs(tools, items or []) if r["path"] == d.path]
 
@@ -224,8 +232,8 @@ def build_mind_map(db: Session, job: Job, artifact: Artifact) -> dict:
         return 1 + sum(count(c) for c in n["children"])
 
     artifact.content_json = {
-        "title": f"Mind Constellation: {focus[:60] or nb.title}", "focus": focus, "root": root, "node_count": count(root),
-        "document_ids": [str(d.id) for d in docs], "post_count": len(docs),
+        "title": f"Mind Constellation: {focus[:60] or nb.title}", "focus": focus, "root": root,
+        "node_count": count(root), "document_ids": [str(d.id) for d in docs], "post_count": len(docs),
         "covered_count": len(_cited(branches) & {str(d.id) for d in docs}),
     }
     artifact.status = "ready"

backend/app/pipeline/research.py

@@ -67,7 +67,7 @@ class ResearchResult:
     steps: list[tuple[str, str]] = field(default_factory=list)
     kind: str = "archive"  # archive | chitchat | about_app | off_topic
     question: str = ""  # the standalone question that was researched
-    recall: dict = field(default_factory=dict)  # how recall behaved for this answer (saved with the message, for feedback)
+    recall: dict = field(default_factory=dict)  # how recall behaved for this answer (saved with the message)
 
 
 @dataclass

backend/app/routers/notebooks.py

@@ -470,7 +470,7 @@ async def chat(notebook_id: uuid.UUID, body: ChatIn, ctx: Ctx = Depends(get_ctx)
         try:
             result = task.result()
         except Exception:
-            log.exception("Research failed for chat %s: %r", chat_row.id, body.question[:200])  # the cause, not just "lost signal"
+            log.exception("Research failed for chat %s: %r", chat_row.id, body.question[:200])  # cause, not just "lost signal"
             trace.error = "research raised (see the traceback above); no answer was saved"
             yield _sse("error", {"message": "Lost signal while researching. Try again."})
             return

backend/app/routers/support.py

@@ -59,7 +59,8 @@ def _rate_limit(user_id: uuid.UUID) -> None:
     while window and now - window[0] > 60:
         window.popleft()
     if len(window) >= settings.support_messages_per_minute:
-        raise HTTPException(status.HTTP_429_TOO_MANY_REQUESTS, "You are sending messages quickly. Try again in a minute.")
+        raise HTTPException(status.HTTP_429_TOO_MANY_REQUESTS,
+                            "You are sending messages quickly. Try again in a minute.")
     window.append(now)
 
 
@@ -73,7 +74,8 @@ def _excerpt(body: str, query: str, budget: int = DOC_BUDGET_CHARS) -> str:
         return body
     words = set(re.findall(r"[a-z0-9]+", query.lower()))
     sections = re.split(r"(?m)^(?=#{1,3}\s)", body)
-    ranked = sorted(range(len(sections)), key=lambda i: -len(words & set(re.findall(r"[a-z0-9]+", sections[i].lower()))))
+    ranked = sorted(range(len(sections)),
+                    key=lambda i: -len(words & set(re.findall(r"[a-z0-9]+", sections[i].lower()))))
     keep, used = set(), 0
     for i in ranked:
         if used + len(sections[i]) > budget and keep:
@@ -144,13 +146,15 @@ async def chat_stream(body: ChatIn, ctx: Ctx = Depends(get_ctx)) -> StreamingRes
             drafted = ""
             try:  # buffered, not streamed: the line is checked before the writer sees it
                 async for tok in llm.stream_answer(
-                    [{"role": "system", "content": esc.handoff_prompt(question_reason)}, {"role": "user", "content": message}]
+                    [{"role": "system", "content": esc.handoff_prompt(question_reason)},
+                     {"role": "user", "content": message}]
                 ):
                     drafted += tok
             except LLMError:
                 drafted = ""
             drafted = strip_em_dashes(drafted.strip())
-            answer = drafted if drafted and esc.handoff_line_is_safe(drafted) else esc.short_circuit_reply(question_reason, len(recent))
+            answer = (drafted if drafted and esc.handoff_line_is_safe(drafted)
+                      else esc.short_circuit_reply(question_reason, len(recent)))
             escalate_reason = question_reason
             yield _sse("token", answer)
         else:
@@ -202,7 +206,8 @@ async def chat_stream(body: ChatIn, ctx: Ctx = Depends(get_ctx)) -> StreamingRes
                 try:
                     conv.summary = strip_em_dashes(await llm.complete_text(
                         [{"role": "system", "content": prompts.SUMMARY_PROMPT},
-                         {"role": "user", "content": f"Previous summary: {conv.summary or '(none)'}\n\nNew turns:\n{transcript}"}],
+                         {"role": "user", "content": f"Previous summary: {conv.summary or '(none)'}\n\n"
+                                                      f"New turns:\n{transcript}"}],
                         max_tokens=500))[:1200]
                     db.commit()
                 except LLMError:
@@ -221,7 +226,8 @@ def latest(ctx: Ctx = Depends(get_ctx)):
     if not conv:
         return {"conversation_id": None, "messages": []}
     msgs = ctx.db.scalars(
-        select(SupportMessage).where(SupportMessage.conversation_id == conv.id).order_by(SupportMessage.created_at).limit(60)
+        select(SupportMessage).where(SupportMessage.conversation_id == conv.id)
+        .order_by(SupportMessage.created_at).limit(60)
     ).all()
     return {"conversation_id": str(conv.id), "messages": [_message_out(m) for m in msgs]}
 

backend/app/support/corpus.py

@@ -74,22 +74,24 @@ def _plans_doc() -> Doc:
         lines += [
             f"## {p.name}",
             f"{p.tagline}. Price: {price}.",
-            f"Limits: {p.sources} sources, {p.indexed_posts} indexed posts, {p.audio_minutes} minutes of audio a month, "
+            f"Limits: {p.sources} sources, {p.indexed_posts} indexed posts, "
+            f"{p.audio_minutes} minutes of audio a month, "
             f"{p.video_minutes} minutes of video a month, {kits} Launch Kits a month. "
             f"Voice cloning: {'yes' if p.voice_cloning else 'no'}. Brand kit: {'yes' if p.brand_kit else 'no'}.",
             "Includes: " + "; ".join(p.features) + ".",
         ]
     lines += [
         "## Billing",
-        "Upgrade, change plan or manage billing from Settings under Plan and usage. Annual billing is cheaper than monthly. "
-        "Checkout may be switched off while billing is being finished, in which case every workspace gets Studio limits.",
+        "Upgrade, change plan or manage billing from Settings under Plan and usage. "
+        "Annual billing is cheaper than monthly. Checkout may be switched off while billing is being finished, "
+        "in which case every workspace gets Studio limits.",
         "Refunds and billing problems are handled by the team: use the Talk to a human button in this chat.",
     ]
     body = "\n".join(lines)
     return Doc(
         id="plans-and-billing", title="Plans, pricing and billing", route="/pricing",
-        keywords=("price", "pricing", "plan", "plans", "free", "writer", "studio", "cost", "billing", "upgrade", "limits",
-                  "annual", "monthly", "subscription", "refund", "cancel", "usage"),
+        keywords=("price", "pricing", "plan", "plans", "free", "writer", "studio", "cost", "billing", "upgrade",
+                  "limits", "annual", "monthly", "subscription", "refund", "cancel", "usage"),
         questions=("How much does Notestack cost?", "What is in the free plan?", "What are the plan limits?",
                    "How do I upgrade?", "Is there annual billing?"),
         related_paths=("/app/settings",), headings=tuple(re.findall(r"^#{1,3}\s+(.+)$", body, re.M)), body=body,

backend/app/support/escalation.py

@@ -28,9 +28,13 @@ _HUMAN = re.compile(
     | \bhuman+\b | \brepresent(?:i|a)tive\b
     """
 )
-_REFUND = re.compile(r"(?ix) \brefun\w*\b | \bchargeback\b | \bmoney\s+back\b | \bbilling\s+dispute\b | \bcancel\s+(?:my\s+)?(?:charge|payment)\b")
+_REFUND = re.compile(
+    r"(?ix) \brefun\w*\b | \bchargeback\b | \bmoney\s+back\b | \bbilling\s+dispute\b"
+    r" | \bcancel\s+(?:my\s+)?(?:charge|payment)\b"
+)
 _FRUSTRATION = re.compile(
-    r"(?ix) \bcan.?t\s+help\b | \bcannot\s+help\b | \buseless\b | \bgive\s+up\b | \bnot\s+helpful\b | \bwaste\s+of\s+(?:time|money)\b | \bfed\s+up\b"
+    r"(?ix) \bcan.?t\s+help\b | \bcannot\s+help\b | \buseless\b | \bgive\s+up\b | \bnot\s+helpful\b"
+    r" | \bwaste\s+of\s+(?:time|money)\b | \bfed\s+up\b"
 )
 # A request verb, never a capability question: "what formats do you support" must stay quiet.
 _FEATURE = re.compile(
@@ -46,7 +50,9 @@ _FEATURE = re.compile(
 _BARE_ASK = re.compile(r"^\s*(?:support|help)\s*[!.?]*\s*$", re.I)
 
 # The bot offering the team in its own answer ("I can pass this to our team").
-_ANSWER_OFFERS_TEAM = re.compile(r"(?i)\b(?:pass|send|forward|route|hand)\b[^.?!]{0,30}\b(?:team|human|person|someone)\b|\bour\s+team\b")
+_ANSWER_OFFERS_TEAM = re.compile(
+    r"(?i)\b(?:pass|send|forward|route|hand)\b[^.?!]{0,30}\b(?:team|human|person|someone)\b|\bour\s+team\b"
+)
 
 
 def classify_question(message: str) -> Reason | None:
@@ -70,7 +76,8 @@ def classify_answer(answer: str) -> Reason | None:
 
 # Claims the bot cannot back up: nothing is sent until the writer submits the form.
 _FALSE_ACTION = re.compile(
-    r"(?i)\b(?:i(?:'ve|\s+have|\s+will|'ll)?\s+(?:sent|forwarded|passed|escalated|opened|created|notified|contacted|connected))\b"
+    r"(?i)\b(?:i(?:'ve|\s+have|\s+will|'ll)?\s+"
+    r"(?:sent|forwarded|passed|escalated|opened|created|notified|contacted|connected))\b"
     r"|\b(?:you(?:'re|\s+are)\s+(?:now\s+)?connected)\b|\bticket\s+(?:has\s+been|is)\b"
 )
 
@@ -99,8 +106,9 @@ def short_circuit_reply(reason: Reason, seed: int = 0) -> str:
 def handoff_prompt(reason: Reason) -> str:
     topic = "a refund or billing problem" if reason is Reason.REFUND else "talking to a person"
     return (
-        f"You are the help assistant for Notestack. The writer is asking about {topic}. Reply in one or two short, warm "
-        "sentences that respond to what they said, and tell them to use the form below to reach the team. Never say you "
-        "have sent, forwarded, opened or escalated anything, and never say they are connected to someone: nothing is "
-        "sent until they submit the form. Do not use em dashes. Do not answer the underlying question."
+        f"You are the help assistant for Notestack. The writer is asking about {topic}. Reply in one or two "
+        "short, warm sentences that respond to what they said, and tell them to use the form below to reach "
+        "the team. Never say you have sent, forwarded, opened or escalated anything, and never say they are "
+        "connected to someone: nothing is sent until they submit the form. Do not use em dashes. "
+        "Do not answer the underlying question."
     )

backend/app/support/memory.py

@@ -7,7 +7,7 @@ from sqlalchemy import select
 from sqlalchemy.orm import Session
 
 from app.models.base import utcnow
-from app.models.support import SupportConversation, SupportMessage
+from app.models.support import SupportMessage
 
 RECENT_MESSAGES = 6  # about three turns
 RECENT_HOURS = 24
@@ -18,7 +18,8 @@ FOLD_BATCH = 4
 def recent_messages(db: Session, conv_id) -> list[SupportMessage]:
     rows = db.scalars(
         select(SupportMessage)
-        .where(SupportMessage.conversation_id == conv_id, SupportMessage.created_at >= utcnow() - timedelta(hours=RECENT_HOURS))
+        .where(SupportMessage.conversation_id == conv_id,
+               SupportMessage.created_at >= utcnow() - timedelta(hours=RECENT_HOURS))
         .order_by(SupportMessage.created_at.desc())
         .limit(RECENT_MESSAGES)
     ).all()

backend/app/support/prompts.py

@@ -1,22 +1,37 @@
 """Prompts for the help bot. No em dashes in any of them: the model copies the style it is given."""
 
-ANSWER_PROMPT = """You are the Notestack help assistant. Notestack turns a writer's blog, newsletter or folder of markdown into a grounded research notebook, then into audio overviews, videos and launch kits.
+ANSWER_PROMPT = """You are the Notestack help assistant. Notestack turns a writer's blog, newsletter or folder of \
+markdown into a grounded research notebook, then into audio overviews, videos and launch kits.
 
 Write plain markdown. No JSON, no code fences. Never use em dashes.
 
-WHAT YOU KNOW: only the DOCUMENTS below. You cannot see the writer's notebooks, posts, sources, files, generated work or account. If asked about them, say you cannot see them and point to the chat inside their notebook, which cites their posts. Never guess about their content.
+WHAT YOU KNOW: only the DOCUMENTS below. You cannot see the writer's notebooks, posts, sources, files, generated \
+work or account. If asked about them, say you cannot see them and point to the chat inside their notebook, which \
+cites their posts. Never guess about their content.
 
 GREETING: if the message is only a greeting or thanks, reply with one short friendly sentence. No lists.
 
-BE HELPFUL FIRST: if the documents contain anything relevant, answer from it, even partially. When they cover only part of the question, answer that part, then name the missing piece in one short sentence. Do not open with an apology.
+BE HELPFUL FIRST: if the documents contain anything relevant, answer from it, even partially. When they cover only \
+part of the question, answer that part, then name the missing piece in one short sentence. Do not open with an \
+apology.
 
-UNKNOWN: only when the documents contain nothing relevant, say in one or two short sentences what you cannot confirm and offer to pass it to the team, for example "I'm not sure whether that is supported. I can pass this to our team if you'd like." Never mention documents or sources. Only OFFER to contact the team. Never say you have sent, forwarded or opened anything: nothing is sent unless the writer submits the form. Never invent steps, buttons, prices or features.
+UNKNOWN: only when the documents contain nothing relevant, say in one or two short sentences what you cannot \
+confirm and offer to pass it to the team, for example "I'm not sure whether that is supported. I can pass this to \
+our team if you'd like." Never mention documents or sources. Only OFFER to contact the team. Never say you have \
+sent, forwarded or opened anything: nothing is sent unless the writer submits the form. Never invent steps, \
+buttons, prices or features.
 
-OFF TOPIC: if the message has nothing to do with Notestack or writing and publishing, reply with one short line steering back to Notestack. Do not answer the off topic question.
+OFF TOPIC: if the message has nothing to do with Notestack or writing and publishing, reply with one short line \
+steering back to Notestack. Do not answer the off topic question.
 
-COMPETITORS: represent Notestack only. Never recommend or name another product to use, even if a document mentions one. If Notestack cannot do something, say so plainly and point to the closest thing it does.
+COMPETITORS: represent Notestack only. Never recommend or name another product to use, even if a document \
+mentions one. If Notestack cannot do something, say so plainly and point to the closest thing it does.
 
-HOW TO: you only give written instructions. Say where to go using the real menu names: Notebooks, Sources and Launchpad are in the left sidebar, Settings is at the bottom of the sidebar, Pricing is on the website menu. When the answer has more than one step, use a numbered markdown list, one step per line ("1.", "2."), with **bold** for button and menu names. One short intro sentence first. Never describe a button or page that the documents do not mention.
+HOW TO: you only give written instructions. Say where to go using the real menu names: Notebooks, Sources and \
+Launchpad are in the left sidebar, Settings is at the bottom of the sidebar, Pricing is on the website menu. When \
+the answer has more than one step, use a numbered markdown list, one step per line ("1.", "2."), with **bold** \
+for button and menu names. One short intro sentence first. Never describe a button or page that the documents do \
+not mention.
 
 DOCUMENTS:
 {docs}
@@ -34,7 +49,8 @@ citations: ids from this list that the answer actually used. Only ids from the l
 {doc_ids}
 
 
-escalate: true only when the answer admits it cannot help or offers to pass the question to the team, or the writer is clearly stuck or frustrated. False for normal answers.
+escalate: true only when the answer admits it cannot help or offers to pass the question to the team, or the \
+writer is clearly stuck or frustrated. False for normal answers.
 """
 
 SUMMARY_PROMPT = (

backend/pyproject.toml

@@ -1,5 +1,5 @@
 [tool.ruff]
-line-length = 120
+line-length = 140
 target-version = "py312"
 
 [tool.ruff.lint]

backend/tests/conftest.py

@@ -11,6 +11,12 @@ os.environ["ELEVENLABS_API_KEY"] = ""
 os.environ["FIRECRAWL_API_KEY"] = ""
 os.environ["R2_PREFIX"] = ""
 os.environ["TYPESAFE_API_KEY"] = ""
+os.environ["STRIPE_SECRET_KEY"] = ""
+os.environ["STRIPE_WEBHOOK_SECRET"] = ""
+os.environ["STRIPE_PRICE_WRITER_MONTHLY"] = ""
+os.environ["STRIPE_PRICE_WRITER_ANNUAL"] = ""
+os.environ["STRIPE_PRICE_STUDIO_MONTHLY"] = ""
+os.environ["STRIPE_PRICE_STUDIO_ANNUAL"] = ""
 os.environ.setdefault("EMAIL_PROVIDER", "console")
 os.environ.setdefault("JWT_SECRET", "test-secret-that-is-at-least-32-bytes-long")
 os.environ.setdefault("BILLING_ENABLED", "false")

backend/tests/evals/support_live.py

@@ -59,7 +59,8 @@ async def run(question: str) -> None:
         ):
             text += tok
         safe = esc.handoff_line_is_safe(text)
-        print(f"[hand off, reason={reason.value}, safe={safe}]\n{strip_em_dashes(text.strip()) if safe else esc.short_circuit_reply(reason)}")
+        reply = strip_em_dashes(text.strip()) if safe else esc.short_circuit_reply(reason)
+        print(f"[hand off, reason={reason.value}, safe={safe}]\n{reply}")
         return
     scored = get_retriever().retrieve(question)
     system = prompts.ANSWER_PROMPT.format(docs=_docs_block(scored, question), user_context="USER CONTEXT: (none)", summary="(none)")