import { useCallback, useEffect, useRef, useState, type FormEvent } from "react"; import { NavLink } from "react-router-dom"; import { docsApi, jobsApi, sourcesApi } from "../api/endpoints"; import type { Doc, Job, Source } from "../api/types"; import { Reader } from "../components/Reader"; import { ConfirmButton, EmptyState, errorMessage, formatDate, JobProgress, Loading, PageHeader, StatusPill } from "../components/ui"; import { useJobMap } from "../hooks/useJob"; import { useUpgrade } from "../hooks/useUpgrade"; const PAGE = 50; /** Sources and the Topic map share one sidebar entry; this strip switches between them. */ export function SourcesNav() { return ( ); } /** Live counts from a running sync: posts found, fetched and indexed so far. */ function SyncStats({ job }: { job: Job }) { const r = job.result as { found?: number; fetched?: number; fetch_total?: number; indexed?: number; index_total?: number }; if (job.kind !== "ingest" || !r || r.found === undefined) return null; const of = (done?: number, total?: number) => (total ? `${done ?? 0} / ${total}` : `${done ?? 0}`); return (
Found
{r.found ?? 0}
Fetched
{of(r.fetched, r.fetch_total)}
Indexed
{of(r.indexed, r.index_total)}
); } export const FILE_ACCEPT = ".md,.markdown,.txt,.vtt,.html,.htm,.pdf,text/vtt,text/markdown,text/plain,text/html,application/pdf"; /** A bare domain, a Substack or a feed URL is a whole archive; a deep link is one article. */ export function looksLikeArticle(raw: string): boolean { const v = raw.trim(); if (!v) return false; try { const u = new URL(v.includes("://") ? v : `https://${v}`); const path = u.pathname.replace(/\/+$/, ""); if (!path || /(feed|rss|atom)(\.xml)?$/i.test(path) || /^\/@[^/]+$/.test(path)) return false; const parts = path.split("/").filter(Boolean); return /\/p\/|\/posts?\/|\/\d{4}\/|\.html?$/i.test(path) || parts.length >= 2 || path.length > 24; } catch { return false; } } /** Imports each file as its own post. Returns how many failed. */ export async function importFiles(files: File[], onAdded: (source: Source, job: Job) => void): Promise { let failed = 0; for (const f of files) { try { const res = await sourcesApi.importFile(f); onAdded(res.source, res.job); } catch { failed += 1; } } return failed; } /** One box for everything: paste a link (feed or article is worked out for you) or add files. */ export function AddSource({ onAdded }: { onAdded: (source: Source, job: Job) => void }) { const [value, setValue] = useState(""); const [override, setOverride] = useState<"feed" | "url" | null>(null); const [busy, setBusy] = useState(false); const [error, setError] = useState(null); const fileRef = useRef(null); const mode = override ?? (looksLikeArticle(value) ? "url" : "feed"); const submit = async (e: FormEvent) => { e.preventDefault(); if (!value.trim()) return; setError(null); setBusy(true); try { const res = mode === "feed" ? await sourcesApi.connect(value.trim()) : await sourcesApi.importUrl(value.trim()); onAdded(res.source, res.job); setValue(""); setOverride(null); } catch (err) { setError(errorMessage(err, mode === "feed" ? "We could not find a feed there." : "We could not import that page.")); } finally { setBusy(false); } }; const upload = async (files: FileList | null) => { const list = Array.from(files ?? []); if (!list.length) return; setError(null); setBusy(true); const failed = await importFiles(list, onAdded); if (failed) setError(`${failed} of ${list.length} files could not be uploaded.`); setBusy(false); if (fileRef.current) fileRef.current.value = ""; }; return (
{ setValue(e.target.value); setOverride(null); }} aria-label="Blog, newsletter or article link" /> or upload(e.target.files)} aria-label="Files to upload" />

{!value.trim() && "Substack, Ghost, WordPress, Medium or any site with a feed. Markdown, text, VTT transcripts, HTML and PDF files work too; drop them anywhere on this page."} {value.trim() && mode === "feed" && ( <> We will pull in the whole archive and keep it in sync.{" "} )} {value.trim() && mode === "url" && ( <> Looks like a single article, so we will import just this page.{" "} )}

{error && (

{error}

)}
); } export default function Sources() { const [sources, setSources] = useState(null); const { openUpgrade } = useUpgrade(); const [docs, setDocs] = useState([]); const [total, setTotal] = useState(0); const [q, setQ] = useState(""); const [filter, setFilter] = useState(""); const [removing, setRemoving] = useState>(new Set()); // sources being deleted: dimmed at once, gone when done const [offset, setOffset] = useState(0); const [reading, setReading] = useState(null); const [error, setError] = useState(null); const loadSources = useCallback(() => sourcesApi.list().then(setSources), []); const loadDocs = useCallback( (reset = true) => { const start = reset ? 0 : offset + PAGE; return docsApi.list({ q, source_id: filter || undefined, indexed_only: true, limit: PAGE, offset: start }).then((p) => { setTotal(p.total); setOffset(start); setDocs((d) => (reset ? p.items : [...d, ...p.items])); }); }, [q, filter, offset], ); const { jobs, watch } = useJobMap(() => { loadSources(); loadDocs(); }); // The post counts on the cards come from the server, so refresh them, and the posts list under them, while a sync is running: // posts show up as they arrive. (Not while a search is being typed or a "Load more" page is open: that would reset them.) const syncing = Object.values(jobs).some((j) => j.status === "queued" || j.status === "running"); const loadDocsRef = useRef(loadDocs); loadDocsRef.current = loadDocs; const paged = useRef(false); paged.current = docs.length > PAGE; useEffect(() => { if (!syncing) return; const t = setInterval(() => { loadSources().catch(() => {}); if (!paged.current) loadDocsRef.current(true).catch(() => {}); }, 3000); return () => clearInterval(t); }, [syncing, loadSources]); useEffect(() => { loadSources().catch(() => setError("Could not load your sources. Refresh to try again.")); }, [loadSources]); useEffect(() => { const t = setTimeout(() => loadDocs(true), 200); return () => clearTimeout(t); // eslint-disable-next-line react-hooks/exhaustive-deps }, [q, filter]); // Pick up syncs and imports that were already running when the page opened. useEffect(() => { jobsApi.active().then((active) => { for (const job of active) { if (typeof job.params.source_id === "string" && ["ingest", "import_url", "import_upload"].includes(job.kind)) { watch(job.params.source_id, job); } } }); // eslint-disable-next-line react-hooks/exhaustive-deps }, []); const added = (source: Source, job: Job) => { setSources((list) => { const rest = (list ?? []).filter((s) => s.id !== source.id); return [...rest, source]; }); watch(source.id, job); }; const [dropping, setDropping] = useState(false); return (
{ if (!e.dataTransfer.types.includes("Files")) return; e.preventDefault(); setDropping(true); }} onDragLeave={(e) => { if (e.currentTarget === e.target) setDropping(false); }} onDrop={async (e) => { if (!e.dataTransfer.files.length) return; e.preventDefault(); setDropping(false); const files = Array.from(e.dataTransfer.files); const failed = await importFiles(files, added); setError(failed ? `${failed} of ${files.length} files could not be uploaded.` : null); }} > {error &&

{error}

} {/* Outside the card grid, so the loader is centered across the page rather than in the first column */} {!sources && }
{sources?.map((s) => { const job = jobs[s.id]; const active = job && (job.status === "queued" || job.status === "running"); return (
{s.title ?? s.feed_url} {s.is_imports ? "imports" : s.platform} · {s.document_count - s.locked_count} posts {s.last_synced_at ? ` · synced ${formatDate(s.last_synced_at, true)}` : ""}
{!s.is_imports && ( {s.feed_url.replace(/^site:/, "")} )} {job && (active || job.status === "failed") && } {job && active && } {s.locked_count > 0 && !active && (
{s.locked_count} posts not indexed

Your plan only indexes your latest posts. Upgrade to index the rest so answers, audio and Launch Kits can use your whole archive.

)} {s.sync_status === "error" && s.sync_error && !active &&

{s.sync_error}

}
{!s.is_imports && ( )} {sources.length > 1 && ( )} { setRemoving((r) => new Set(r).add(s.id)); try { await sourcesApi.remove(s.id); } catch (e) { setRemoving((r) => { const next = new Set(r); next.delete(s.id); return next; }); throw e; } // Gone from the page now; the lists refresh behind it. setSources((list) => (list ?? []).filter((x) => x.id !== s.id)); setDocs((list) => list.filter((d) => d.source_id !== s.id)); if (filter === s.id) setFilter(""); loadSources(); loadDocs(); }} > Delete
); })}
{sources?.length === 0 && ( )} {/* The posts are here as soon as there are any: every source's posts together, or one source's when its button is on */} {sources && sources.length > 0 && (filter || total > 0) && (

Posts {total}

setQ(e.target.value)} aria-label="Search posts" />
    {docs.map((d) => (
  • {d.locked && ( Not indexed )} {d.source_title ?? ""} · {formatDate(d.published_at)} · {d.words} words
  • ))} {docs.length === 0 &&
  • No posts match.
  • }
{docs.length < total && ( )}
)} {reading && setReading(null)} />}
); }