import { useCallback, useEffect, useRef, useState, type FormEvent } from "react";
import { NavLink } from "react-router-dom";
import { docsApi, jobsApi, sourcesApi } from "../api/endpoints";
import type { Doc, Job, Source } from "../api/types";
import { Reader } from "../components/Reader";
import { ConfirmButton, EmptyState, errorMessage, formatDate, JobProgress, Loading, PageHeader, StatusPill } from "../components/ui";
import { useJobMap } from "../hooks/useJob";
import { useUpgrade } from "../hooks/useUpgrade";
const PAGE = 50;
/** Sources and the Topic map share one sidebar entry; this strip switches between them. */
export function SourcesNav() {
return (
);
}
/** Live counts from a running sync: posts found, fetched and indexed so far. */
function SyncStats({ job }: { job: Job }) {
const r = job.result as { found?: number; fetched?: number; fetch_total?: number; indexed?: number; index_total?: number };
if (job.kind !== "ingest" || !r || r.found === undefined) return null;
const of = (done?: number, total?: number) => (total ? `${done ?? 0} / ${total}` : `${done ?? 0}`);
return (
Found
{r.found ?? 0}
Fetched
{of(r.fetched, r.fetch_total)}
Indexed
{of(r.indexed, r.index_total)}
);
}
export const FILE_ACCEPT = ".md,.markdown,.txt,.vtt,.html,.htm,.pdf,text/vtt,text/markdown,text/plain,text/html,application/pdf";
/** A bare domain, a Substack or a feed URL is a whole archive; a deep link is one article. */
export function looksLikeArticle(raw: string): boolean {
const v = raw.trim();
if (!v) return false;
try {
const u = new URL(v.includes("://") ? v : `https://${v}`);
const path = u.pathname.replace(/\/+$/, "");
if (!path || /(feed|rss|atom)(\.xml)?$/i.test(path) || /^\/@[^/]+$/.test(path)) return false;
const parts = path.split("/").filter(Boolean);
return /\/p\/|\/posts?\/|\/\d{4}\/|\.html?$/i.test(path) || parts.length >= 2 || path.length > 24;
} catch {
return false;
}
}
/** Imports each file as its own post. Returns how many failed. */
export async function importFiles(files: File[], onAdded: (source: Source, job: Job) => void): Promise {
let failed = 0;
for (const f of files) {
try {
const res = await sourcesApi.importFile(f);
onAdded(res.source, res.job);
} catch {
failed += 1;
}
}
return failed;
}
/** One box for everything: paste a link (feed or article is worked out for you) or add files. */
export function AddSource({ onAdded }: { onAdded: (source: Source, job: Job) => void }) {
const [value, setValue] = useState("");
const [override, setOverride] = useState<"feed" | "url" | null>(null);
const [busy, setBusy] = useState(false);
const [error, setError] = useState(null);
const fileRef = useRef(null);
const mode = override ?? (looksLikeArticle(value) ? "url" : "feed");
const submit = async (e: FormEvent) => {
e.preventDefault();
if (!value.trim()) return;
setError(null);
setBusy(true);
try {
const res = mode === "feed" ? await sourcesApi.connect(value.trim()) : await sourcesApi.importUrl(value.trim());
onAdded(res.source, res.job);
setValue("");
setOverride(null);
} catch (err) {
setError(errorMessage(err, mode === "feed" ? "We could not find a feed there." : "We could not import that page."));
} finally {
setBusy(false);
}
};
const upload = async (files: FileList | null) => {
const list = Array.from(files ?? []);
if (!list.length) return;
setError(null);
setBusy(true);
const failed = await importFiles(list, onAdded);
if (failed) setError(`${failed} of ${list.length} files could not be uploaded.`);
setBusy(false);
if (fileRef.current) fileRef.current.value = "";
};
return (
{!value.trim() && "Substack, Ghost, WordPress, Medium or any site with a feed. Markdown, text, VTT transcripts, HTML and PDF files work too; drop them anywhere on this page."}
{value.trim() && mode === "feed" && (
<>
We will pull in the whole archive and keep it in sync.{" "}
>
)}
{value.trim() && mode === "url" && (
<>
Looks like a single article, so we will import just this page.{" "}
>
)}
{error && (
{error}
)}
);
}
export default function Sources() {
const [sources, setSources] = useState(null);
const { openUpgrade } = useUpgrade();
const [docs, setDocs] = useState([]);
const [total, setTotal] = useState(0);
const [q, setQ] = useState("");
const [filter, setFilter] = useState("");
const [removing, setRemoving] = useState>(new Set()); // sources being deleted: dimmed at once, gone when done
const [offset, setOffset] = useState(0);
const [reading, setReading] = useState(null);
const [error, setError] = useState(null);
const loadSources = useCallback(() => sourcesApi.list().then(setSources), []);
const loadDocs = useCallback(
(reset = true) => {
const start = reset ? 0 : offset + PAGE;
return docsApi.list({ q, source_id: filter || undefined, indexed_only: true, limit: PAGE, offset: start }).then((p) => {
setTotal(p.total);
setOffset(start);
setDocs((d) => (reset ? p.items : [...d, ...p.items]));
});
},
[q, filter, offset],
);
const { jobs, watch } = useJobMap(() => {
loadSources();
loadDocs();
});
// The post counts on the cards come from the server, so refresh them, and the posts list under them, while a sync is running:
// posts show up as they arrive. (Not while a search is being typed or a "Load more" page is open: that would reset them.)
const syncing = Object.values(jobs).some((j) => j.status === "queued" || j.status === "running");
const loadDocsRef = useRef(loadDocs);
loadDocsRef.current = loadDocs;
const paged = useRef(false);
paged.current = docs.length > PAGE;
useEffect(() => {
if (!syncing) return;
const t = setInterval(() => {
loadSources().catch(() => {});
if (!paged.current) loadDocsRef.current(true).catch(() => {});
}, 3000);
return () => clearInterval(t);
}, [syncing, loadSources]);
useEffect(() => {
loadSources().catch(() => setError("Could not load your sources. Refresh to try again."));
}, [loadSources]);
useEffect(() => {
const t = setTimeout(() => loadDocs(true), 200);
return () => clearTimeout(t);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [q, filter]);
// Pick up syncs and imports that were already running when the page opened.
useEffect(() => {
jobsApi.active().then((active) => {
for (const job of active) {
if (typeof job.params.source_id === "string" && ["ingest", "import_url", "import_upload"].includes(job.kind)) {
watch(job.params.source_id, job);
}
}
});
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
const added = (source: Source, job: Job) => {
setSources((list) => {
const rest = (list ?? []).filter((s) => s.id !== source.id);
return [...rest, source];
});
watch(source.id, job);
};
const [dropping, setDropping] = useState(false);
return (
{
if (!e.dataTransfer.types.includes("Files")) return;
e.preventDefault();
setDropping(true);
}}
onDragLeave={(e) => {
if (e.currentTarget === e.target) setDropping(false);
}}
onDrop={async (e) => {
if (!e.dataTransfer.files.length) return;
e.preventDefault();
setDropping(false);
const files = Array.from(e.dataTransfer.files);
const failed = await importFiles(files, added);
setError(failed ? `${failed} of ${files.length} files could not be uploaded.` : null);
}}
>
{error &&
{error}
}
{/* Outside the card grid, so the loader is centered across the page rather than in the first column */}
{!sources && }
{sources?.map((s) => {
const job = jobs[s.id];
const active = job && (job.status === "queued" || job.status === "running");
return (
}
);
})}
{sources?.length === 0 && (
)}
{/* The posts are here as soon as there are any: every source's posts together, or one source's when its button is on */}
{sources && sources.length > 0 && (filter || total > 0) && (