import { createFileRoute } from "@tanstack/react-router"; import { useState, useMemo, useEffect } from "react"; import { useMutation } from "@tanstack/react-query"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, } from "@/components/ui/select"; import { Checkbox } from "@/components/ui/checkbox"; import { Badge } from "@/components/ui/badge"; import { Plus, ChevronLeft, ChevronRight, Download, Search, Trash2, Loader2 } from "lucide-react"; import { useSites } from "@/lib/grabber/useSites"; import { searchPosts } from "@/lib/grabber/search.functions"; import type { NormalizedPost } from "@/lib/grabber/sites"; import { AddSiteDialog } from "@/components/grabber/AddSiteDialog"; import { Lightbox } from "@/components/grabber/Lightbox"; export const Route = createFileRoute("/")({ head: () => ({ meta: [ { title: "Imageboard Grabber" }, { name: "description", content: "Search booru-style imageboards, browse thumbnails, and batch-download as ZIP.", }, { property: "og:title", content: "Imageboard Grabber" }, { property: "og:description", content: "Search booru-style imageboards, browse thumbnails, and batch-download as ZIP.", }, ], }), component: Grabber, }); function Grabber() { const { sites, custom, addSite, removeSite, hydrated } = useSites(); const [siteId, setSiteId] = useState("e621"); const [tags, setTags] = useState(""); const [limit, setLimit] = useState(40); const [page, setPage] = useState(1); const [selected, setSelected] = useState>(new Set()); const [addOpen, setAddOpen] = useState(false); const [lightbox, setLightbox] = useState(null); const [downloading, setDownloading] = useState(false); const [excludeTags, setExcludeTags] = useState(""); const [hfToken, setHfToken] = useState(""); const [datasetName, setDatasetName] = useState(""); const [uploading, setUploading] = useState(false); const [uploadStatus, setUploadStatus] = useState(""); const [customFilename, setCustomFilename] = useState(""); const [customSubfolder, setCustomSubfolder] = useState(""); const [e621Username, setE621Username] = useState(""); const [e621ApiKey, setE621ApiKey] = useState(""); const [autocompleteEnabled, setAutocompleteEnabled] = useState(true); const [activeTagFile, setActiveTagFile] = useState("tags-selected.csv"); const [availableTagFiles, setAvailableTagFiles] = useState([]); const [newTagFileUrl, setNewTagFileUrl] = useState(""); const [downloadingTagFile, setDownloadingTagFile] = useState(false); const [tagSuggestions, setTagSuggestions] = useState<{ name: string; count: number }[]>([]); const [activeWordInfo, setActiveWordInfo] = useState<{ word: string; start: number; end: number } | null>(null); const [focusedSuggestionIdx, setFocusedSuggestionIdx] = useState(-1); const fetchTagFiles = async () => { try { const res = await fetch("/api/tags?action=list"); const data = await res.json(); if (data.success && Array.isArray(data.files)) { setAvailableTagFiles(data.files); if (data.files.length > 0 && !data.files.includes(activeTagFile)) { // Default to tags-selected.csv if available if (data.files.includes("tags-selected.csv")) { setActiveTagFile("tags-selected.csv"); } else { setActiveTagFile(data.files[0]); } } } } catch (err) { console.error("Failed to list tag files:", err); } }; const handleDownloadTagFile = async () => { if (!newTagFileUrl.trim()) return; setDownloadingTagFile(true); try { const res = await fetch("/api/tags", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ downloadUrl: newTagFileUrl.trim() }), }); const data = await res.json(); if (!res.ok) { alert(`Error: ${data.error}`); } else { alert(`Successfully downloaded and decompressed ${data.filename}!`); setNewTagFileUrl(""); await fetchTagFiles(); setActiveTagFile(data.filename); } } catch (err) { alert(`Download failed: ${err instanceof Error ? err.message : "unknown error"}`); } finally { setDownloadingTagFile(false); } }; const updateActiveWord = (el: HTMLInputElement) => { const val = el.value; const pos = el.selectionStart; if (pos === null || !autocompleteEnabled) { setTagSuggestions([]); setActiveWordInfo(null); return; } // Find start and end of word under cursor let start = pos; while (start > 0 && !/\s/.test(val[start - 1])) { start--; } let end = pos; while (end < val.length && !/\s/.test(val[end])) { end++; } const word = val.slice(start, end).trim(); // Exclude colon commands like rating:safe if (word && !word.includes(":")) { setActiveWordInfo({ word, start, end }); } else { setTagSuggestions([]); setActiveWordInfo(null); } }; const selectSuggestion = (tagName: string) => { if (!activeWordInfo) return; const before = tags.slice(0, activeWordInfo.start); const after = tags.slice(activeWordInfo.end); // Replace word and add a trailing space const newTags = before + tagName + " " + (after.trim() ? after : ""); setTags(newTags); setTagSuggestions([]); setActiveWordInfo(null); // Refocus input and place cursor after inserted word + space const inputEl = document.getElementById("search-tags-input") as HTMLInputElement; if (inputEl) { inputEl.focus(); const cursorTarget = before.length + tagName.length + 1; setTimeout(() => { inputEl.setSelectionRange(cursorTarget, cursorTarget); }, 0); } }; const handleAutocompleteKeyDown = (e: React.KeyboardEvent) => { if (tagSuggestions.length > 0) { if (e.key === "ArrowDown") { e.preventDefault(); setFocusedSuggestionIdx((prev) => (prev + 1) % tagSuggestions.length); } else if (e.key === "ArrowUp") { e.preventDefault(); setFocusedSuggestionIdx((prev) => (prev - 1 + tagSuggestions.length) % tagSuggestions.length); } else if (e.key === "Enter") { if (focusedSuggestionIdx >= 0 && focusedSuggestionIdx < tagSuggestions.length) { e.preventDefault(); selectSuggestion(tagSuggestions[focusedSuggestionIdx].name); } else { runSearch(1); } } else if (e.key === "Escape") { e.preventDefault(); setTagSuggestions([]); setActiveWordInfo(null); } } else { if (e.key === "Enter") { runSearch(1); } } }; useEffect(() => { if (typeof window !== "undefined") { setE621Username(localStorage.getItem("e621_username") || ""); setE621ApiKey(localStorage.getItem("e621_api_key") || ""); } fetchTagFiles(); }, []); useEffect(() => { if (!autocompleteEnabled || !activeWordInfo || !activeWordInfo.word) { setTagSuggestions([]); return; } const delay = setTimeout(async () => { try { const res = await fetch( `/api/tags?action=autocomplete&query=${encodeURIComponent( activeWordInfo.word )}&file=${encodeURIComponent(activeTagFile)}&limit=15` ); const data = await res.json(); if (data.success && Array.isArray(data.suggestions)) { setTagSuggestions(data.suggestions); setFocusedSuggestionIdx(-1); } } catch (err) { console.error("Autocomplete fetch failed:", err); } }, 150); return () => clearTimeout(delay); }, [activeWordInfo, activeTagFile, autocompleteEnabled]); const site = useMemo(() => sites.find((s) => s.id === siteId) ?? sites[0], [sites, siteId]); const searchMut = useMutation({ mutationFn: async (vars: { page: number }) => { return await searchPosts({ data: { site, tags, page: vars.page, limit, login: site.id === "e621" ? e621Username : undefined, apiKey: site.id === "e621" ? e621ApiKey : undefined, }, }); }, }); const posts = searchMut.data?.posts ?? []; const err = searchMut.data?.error; const runSearch = (p: number) => { setPage(p); setSelected(new Set()); searchMut.mutate({ page: p }); }; const toggle = (id: string) => { setSelected((prev) => { const n = new Set(prev); if (n.has(id)) n.delete(id); else n.add(id); return n; }); }; const selectAll = () => { if (selected.size === posts.length) setSelected(new Set()); else setSelected(new Set(posts.map((p) => p.id))); }; const downloadZip = async () => { const chosen = posts.filter((p) => selected.has(p.id)); if (!chosen.length) return; setDownloading(true); try { const res = await fetch("/api/download-zip", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ siteName: site.id, posts: chosen, excludeTags: excludeTags .split(/\s+|,/) .map((t) => t.trim().toLowerCase()) .filter(Boolean), zipName: customFilename.trim() || undefined, }), }); if (!res.ok) { const t = await res.text(); alert(`Download failed: ${t}`); return; } const blob = await res.blob(); const url = URL.createObjectURL(blob); const a = document.createElement("a"); a.href = url; let finalName = ""; if (customFilename.trim()) { finalName = customFilename.trim(); if (!finalName.endsWith(".zip")) { finalName += ".zip"; } } else { finalName = `grabber-${site.id}-${Date.now()}.zip`; } a.download = finalName; document.body.appendChild(a); a.click(); a.remove(); URL.revokeObjectURL(url); } finally { setDownloading(false); } }; const uploadToDataset = async () => { const chosen = posts.filter((p) => selected.has(p.id)); if (!chosen.length) { alert("Please select some images to upload."); return; } if (!hfToken.trim() || !datasetName.trim()) { alert("Hugging Face write token and dataset name are required."); return; } setUploading(true); setUploadStatus("Uploading to Hugging Face..."); try { const res = await fetch("/api/upload-to-dataset", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ siteName: site.id, posts: chosen, excludeTags: excludeTags .split(/\s+|,/) .map((t) => t.trim().toLowerCase()) .filter(Boolean), hfToken: hfToken.trim(), datasetName: datasetName.trim(), zipName: customFilename.trim() || undefined, subfolder: customSubfolder.trim() || undefined, }), }); const data = await res.json(); if (!res.ok) { setUploadStatus(`Upload failed: ${data.error || "Unknown error"}`); } else { setUploadStatus(`Success! Uploaded ZIP as ${data.filename}`); } } catch (err: unknown) { const errMsg = err instanceof Error ? err.message : "Network error"; setUploadStatus(`Upload failed: ${errMsg}`); } finally { setUploading(false); } }; return (

Grabber

{hydrated && custom.some((c) => c.id === siteId) && ( )}
{ setTags(e.target.value); updateActiveWord(e.target); }} onKeyUp={(e) => updateActiveWord(e.currentTarget)} onSelect={(e) => updateActiveWord(e.currentTarget)} onFocus={(e) => updateActiveWord(e.currentTarget)} onBlur={() => { // Short timeout to allow clicking suggestion buttons setTimeout(() => { setTagSuggestions([]); setActiveWordInfo(null); }, 200); }} onKeyDown={handleAutocompleteKeyDown} className="w-full font-sans text-sm" autoComplete="off" /> {tagSuggestions.length > 0 && (
{tagSuggestions.map((suggestion, idx) => ( ))}
)}
setLimit(Number(e.target.value) || 40)} className="w-24" />
page {page}
selected: {selected.size} setCustomFilename(e.target.value)} className="w-64 max-w-xs" /> setExcludeTags(e.target.value)} className="w-64 max-w-xs" />
{/* e621 Authentication Panel */} {siteId === "e621" && (
e621 Authentication: { setE621Username(e.target.value); localStorage.setItem("e621_username", e.target.value); }} className="w-48 max-w-xs" /> { setE621ApiKey(e.target.value); localStorage.setItem("e621_api_key", e.target.value); }} className="w-48 max-w-xs" /> (Credentials stored only in your local browser storage)
)} {/* Autocomplete Settings Panel */}
Autocomplete Settings:
setAutocompleteEnabled(!!checked)} />
{autocompleteEnabled && availableTagFiles.length > 0 && ( <> Tags File: )} Download Tags Link: setNewTagFileUrl(e.target.value)} className="w-64 max-w-xs h-9 text-xs" />
{/* Hugging Face Dataset Integration Panel */}
Hugging Face Upload: setHfToken(e.target.value)} className="w-48 max-w-xs" /> setDatasetName(e.target.value)} className="w-64 max-w-xs" /> setCustomSubfolder(e.target.value)} className="w-64 max-w-xs" /> {uploadStatus && ( {uploadStatus} )}
{err && (
{err}
)} {searchMut.isPending && (
Loading…
)} {!searchMut.isPending && posts.length === 0 && searchMut.isSuccess && !err && (
No results.
)} {!searchMut.isPending && !searchMut.isSuccess && (
Enter tags and press Search.
)}
{posts.map((p) => { const isSel = selected.has(p.id); return (
setLightbox(p)} >
{`post (e.currentTarget.style.opacity = "0.2")} />
{ e.stopPropagation(); toggle(p.id); }} >
{p.rating && ( {p.rating} )} {p.score !== undefined && ( ★{p.score} )}
); })}
{ addSite(s); setSiteId(s.id); }} /> setLightbox(null)} />
); }