Upload folder using huggingface_hub

ROADMAP.md

@@ -7,27 +7,31 @@ This document outlines a roadmap and future development plan for the **Imageboar
 ## 🚀 1. Feature Enhancements
 
 ### 1.1 Support for Authenticated APIs (API Keys & Username)
-* **Goal:** Allow users to safely search and download from private or rate-limited sites (like Danbooru premium tiers, e621 personal accounts).
-* **Implementation:**
+
+- **Goal:** Allow users to safely search and download from private or rate-limited sites (like Danbooru premium tiers, e621 personal accounts).
+- **Implementation:**
   - Add credential fields (API Key / Password Hash, Username) to the **Add Site** dialog.
   - Securely transmit credentials to the server using HTTP headers or request bodies.
   - Store credentials locally in the browser's `localStorage` or session-only cookies so they are never stored globally on the server.
 
 ### 1.2 Persistent Site Configuration
-* **Goal:** Avoid losing newly added schemas when refreshing or clearing session storage.
-* **Implementation:**
+
+- **Goal:** Avoid losing newly added schemas when refreshing or clearing session storage.
+- **Implementation:**
   - Sync the list of added custom sites with IndexedDB or `localStorage`.
   - Provide an **Export/Import Sites** button to allow users to backup their custom configurations as JSON.
 
 ### 1.3 Advanced Query Builder
-* **Goal:** Help non-technical users build complex booru search queries.
-* **Implementation:**
+
+- **Goal:** Help non-technical users build complex booru search queries.
+- **Implementation:**
   - Add UI controls for searching by rating (Safe, Questionable, Explicit), score range, upload date, or file extension.
   - Autocomplete / tag suggestion box by querying the active site's tag API endpoint (e.g., `/tags.json?search[name_matches]=*`).
 
 ### 1.4 Enhanced Lightbox & Media Viewer
-* **Goal:** Support video playback and metadata exploration.
-* **Implementation:**
+
+- **Goal:** Support video playback and metadata exploration.
+- **Implementation:**
   - Auto-detect video extensions (`.webm`, `.mp4`) in the Lightbox and render a native HTML `<video>` player with volume and speed controls.
   - Show side-panel info: Artist, Character, Copyright, and Meta tags. Click a tag to search.
 
@@ -36,14 +40,16 @@ This document outlines a roadmap and future development plan for the **Imageboar
 ## 🛠️ 2. Architectural & Performance Improvements
 
 ### 2.1 Concurrent / Streamed Server Downloads
-* **Goal:** Speed up ZIP compilation and handle massive file downloads without causing OOM (Out Of Memory) crashes.
-* **Implementation:**
+
+- **Goal:** Speed up ZIP compilation and handle massive file downloads without causing OOM (Out Of Memory) crashes.
+- **Implementation:**
   - Implement streaming ZIP using `fflate` on the server so bytes are sent back to the client immediately as they download, rather than buffering entire gigabytes in server memory.
   - Limit concurrent download workers per request to avoid IP bans from image hosters.
 
 ### 2.2 Client-Side Image Caching
-* **Goal:** Reduce bandwidth usage and accelerate browsing.
-* **Implementation:**
+
+- **Goal:** Reduce bandwidth usage and accelerate browsing.
+- **Implementation:**
   - Register a Service Worker to cache thumbnails and UI assets.
   - Use `Cache-Control` headers for downloaded image URLs.
 
@@ -52,14 +58,16 @@ This document outlines a roadmap and future development plan for the **Imageboar
 ## 🧪 3. Quality Assurance & Automated Testing
 
 ### 3.1 End-to-End Tests with Playwright
-* **Goal:** Ensure code modifications do not break key features like site additions or tag searches.
-* **Implementation:**
+
+- **Goal:** Ensure code modifications do not break key features like site additions or tag searches.
+- **Implementation:**
   - Add E2E test suites inside a `tests/` directory.
   - Mock API responses from e621/Danbooru to test search, grid rendering, and selection UI without hitting actual boorus.
 
 ### 3.2 Unit and API Integration Tests
-* **Goal:** Validate parsing logic, schema validation, and URL building helpers.
-* **Implementation:**
+
+- **Goal:** Validate parsing logic, schema validation, and URL building helpers.
+- **Implementation:**
   - Write Vitest suites to verify mapping functions in `src/lib/grabber/sites.ts` and zip streaming endpoints.
 
 ---
@@ -67,6 +75,7 @@ This document outlines a roadmap and future development plan for the **Imageboar
 ## 🌐 4. Deployment & Infrastructure Optimization
 
 ### 4.1 Hugging Face Storage Integration
-* **Goal:** Retain user session data or cache assets on persistent storage.
-* **Implementation:**
+
+- **Goal:** Retain user session data or cache assets on persistent storage.
+- **Implementation:**
   - Connect a Hugging Face Dataset or persistent Space storage to backup logs or temporary configuration files if requested.

src/components/grabber/AddSiteDialog.tsx

@@ -47,6 +47,9 @@ const DEFAULT: SiteConfig = {
 export function AddSiteDialog({ open, onOpenChange, onAdd }: Props) {
   const [s, setS] = useState<SiteConfig>(DEFAULT);
   const [headersText, setHeadersText] = useState("");
+  const [pasteSchemaText, setPasteSchemaText] = useState("");
+  const [parseError, setParseError] = useState("");
+
   const set = <K extends keyof SiteConfig>(k: K, v: SiteConfig[K]) =>
     setS((p) => ({ ...p, [k]: v }));
   const setField = (k: keyof SiteConfig["fields"], v: string) =>
@@ -54,6 +57,60 @@ export function AddSiteDialog({ open, onOpenChange, onAdd }: Props) {
   const setParam = (k: keyof SiteConfig["params"], v: string) =>
     setS((p) => ({ ...p, params: { ...p.params, [k]: v } }));
 
+  const handlePasteSchema = () => {
+    setParseError("");
+    if (!pasteSchemaText.trim()) return;
+    try {
+      const parsed = JSON.parse(pasteSchemaText);
+      if (typeof parsed !== "object" || parsed === null) {
+        throw new Error("Schema must be an object");
+      }
+
+      const merged: SiteConfig = {
+        id: parsed.id || s.id,
+        name: parsed.name || s.name,
+        baseUrl: parsed.baseUrl || s.baseUrl,
+        searchPath: parsed.searchPath || s.searchPath,
+        params: {
+          tags: parsed.params?.tags || s.params.tags,
+          page: parsed.params?.page || s.params.page,
+          limit: parsed.params?.limit || s.params.limit,
+        },
+        postsPath: typeof parsed.postsPath === "string" ? parsed.postsPath : s.postsPath,
+        fields: {
+          id: parsed.fields?.id || s.fields.id,
+          previewUrl: parsed.fields?.previewUrl || s.fields.previewUrl,
+          fileUrl: parsed.fields?.fileUrl || s.fields.fileUrl,
+          ext: parsed.fields?.ext || s.fields.ext,
+          tags: parsed.fields?.tags || s.fields.tags,
+          source: parsed.fields?.source || s.fields.source,
+          rating: parsed.fields?.rating || s.fields.rating,
+          score: parsed.fields?.score || s.fields.score,
+          sampleUrl: parsed.fields?.sampleUrl || s.fields.sampleUrl,
+          fileSize: parsed.fields?.fileSize || s.fields.fileSize,
+          width: parsed.fields?.width || s.fields.width,
+          height: parsed.fields?.height || s.fields.height,
+          md5: parsed.fields?.md5 || s.fields.md5,
+          uploader: parsed.fields?.uploader || s.fields.uploader,
+          createdAt: parsed.fields?.createdAt || s.fields.createdAt,
+        },
+      };
+
+      if (parsed.headers && typeof parsed.headers === "object") {
+        setHeadersText(JSON.stringify(parsed.headers, null, 2));
+      }
+
+      setS(merged);
+      setPasteSchemaText("");
+    } catch (e: unknown) {
+      if (e instanceof Error) {
+        setParseError(e.message);
+      } else {
+        setParseError("Failed to parse JSON");
+      }
+    }
+  };
+
   const submit = () => {
     let headers: Record<string, string> | undefined;
     if (headersText.trim()) {
@@ -74,6 +131,29 @@ export function AddSiteDialog({ open, onOpenChange, onAdd }: Props) {
         <DialogHeader>
           <DialogTitle>Add Site (schema)</DialogTitle>
         </DialogHeader>
+
+        {/* New Paste Schema Input Box */}
+        <div className="border p-3 rounded-md bg-muted/30 mb-4">
+          <Label className="text-xs font-semibold mb-1 block">
+            Paste Site Configuration JSON (API Support)
+          </Label>
+          <div className="flex gap-2">
+            <Textarea
+              rows={3}
+              value={pasteSchemaText}
+              onChange={(e) => setPasteSchemaText(e.target.value)}
+              placeholder={`Paste JSON schema here, e.g.:\n{\n  "id": "danbooru",\n  "name": "Danbooru",\n  "baseUrl": "https://danbooru.donmai.us",\n  "fields": { "fileUrl": "file_url" }\n}`}
+              className="font-mono text-xs flex-1"
+            />
+            <Button variant="secondary" size="sm" onClick={handlePasteSchema} className="self-end">
+              Apply Schema
+            </Button>
+          </div>
+          {parseError && (
+            <div className="text-xs text-destructive mt-1 font-semibold">{parseError}</div>
+          )}
+        </div>
+
         <div className="grid grid-cols-2 gap-3">
           <Field label="ID (unique)">
             <Input value={s.id} onChange={(e) => set("id", e.target.value)} />