export type FieldMap = { id: string; previewUrl: string; fileUrl: string; ext: string; tags: string; source?: string; rating?: string; score?: string; sampleUrl?: string; fileSize?: string; width?: string; height?: string; md5?: string; uploader?: string; createdAt?: string; }; export type SiteConfig = { id: string; name: string; baseUrl: string; searchPath: string; params: { tags: string; page: string; limit: string }; postsPath: string; // dotted path to array of posts in JSON response; "" means root fields: FieldMap; headers?: Record; pageStartsAt?: number; // default 1 }; export const BUILTIN_SITES: SiteConfig[] = [ { id: "e621", name: "e621", baseUrl: "https://e621.net", searchPath: "/posts.json", params: { tags: "tags", page: "page", limit: "limit" }, postsPath: "posts", fields: { id: "id", previewUrl: "preview.url", fileUrl: "file.url", ext: "file.ext", tags: "tags.general", source: "sources.0", rating: "rating", score: "score.total", sampleUrl: "sample.url", fileSize: "file.size", width: "file.width", height: "file.height", md5: "file.md5", uploader: "uploader_id", createdAt: "created_at", }, headers: { "User-Agent": "LovableGrabber/1.0 (booru client)" }, pageStartsAt: 1, }, ]; // Resolve a dotted path like "a.b.0.c" against an arbitrary value. export function getByPath(obj: unknown, path: string): unknown { if (!path) return obj; const parts = path.split("."); let cur: unknown = obj; for (const p of parts) { if (cur == null) return undefined; if (Array.isArray(cur)) { const idx = Number(p); if (!Number.isInteger(idx)) return undefined; cur = cur[idx]; } else if (typeof cur === "object") { cur = (cur as Record)[p]; } else { return undefined; } } return cur; } export type NormalizedPost = { id: string; previewUrl: string; fileUrl: string; ext: string; tags: string[]; source?: string; rating?: string; score?: number; sampleUrl?: string; fileSize?: number; width?: number; height?: number; md5?: string; uploader?: string; createdAt?: string; rawJson?: string; }; function optStr(v: unknown): string | undefined { if (v == null) return undefined; if (typeof v === "string") return v || undefined; return String(v); } function optNum(v: unknown): number | undefined { if (v == null || v === "") return undefined; const n = typeof v === "number" ? v : Number(v); return Number.isFinite(n) ? n : undefined; } export function normalizePost(raw: unknown, fields: FieldMap): NormalizedPost | null { const fileUrl = getByPath(raw, fields.fileUrl); const previewUrl = getByPath(raw, fields.previewUrl) ?? fileUrl; const id = getByPath(raw, fields.id); if (!fileUrl || !id) return null; const tagsRaw = getByPath(raw, fields.tags); let tags: string[] = []; if (Array.isArray(tagsRaw)) tags = tagsRaw.map(String); else if (typeof tagsRaw === "string") tags = tagsRaw.split(/\s+/).filter(Boolean); const extRaw = getByPath(raw, fields.ext); const ext = (typeof extRaw === "string" && extRaw) || (typeof fileUrl === "string" ? (fileUrl.split(".").pop()?.split("?")[0] ?? "jpg") : "jpg"); return { id: String(id), previewUrl: String(previewUrl), fileUrl: String(fileUrl), ext, tags, source: fields.source ? optStr(getByPath(raw, fields.source)) : undefined, rating: fields.rating ? optStr(getByPath(raw, fields.rating)) : undefined, score: fields.score ? optNum(getByPath(raw, fields.score)) : undefined, sampleUrl: fields.sampleUrl ? optStr(getByPath(raw, fields.sampleUrl)) : undefined, fileSize: fields.fileSize ? optNum(getByPath(raw, fields.fileSize)) : undefined, width: fields.width ? optNum(getByPath(raw, fields.width)) : undefined, height: fields.height ? optNum(getByPath(raw, fields.height)) : undefined, md5: fields.md5 ? optStr(getByPath(raw, fields.md5)) : undefined, uploader: fields.uploader ? optStr(getByPath(raw, fields.uploader)) : undefined, createdAt: fields.createdAt ? optStr(getByPath(raw, fields.createdAt)) : undefined, rawJson: safeStringify(raw), }; } function safeStringify(v: unknown): string | undefined { try { return JSON.stringify(v); } catch { return undefined; } }