src/routes/api/scrap-models/+server.ts

/** @type {import('./$types').RequestHandler} */

import { json } from '@sveltejs/kit';
import moment from 'moment';
import prisma from '$lib/prisma';

export async function POST({ request }) {
  const headers = Object.fromEntries(request.headers.entries());

  if (headers["x-hf-token"] !== process.env.SECRET_HF_TOKEN) {
    return Response.json({
      message: "Wrong castle fam :^)"
    }, { status: 401 });
  }

  const response_models = await fetch(`https://huggingface.co/api/models?limit=200&filter=lora%2Cdiffusers&sort=createdAt&full=true&config=true`)
  const new_models = await response_models.json();
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  let index = 0;
  let model_saved = 0;

  for (const model of new_models) {
    if (index % 50 === 0) {
      await new Promise(resolve => setTimeout(resolve, 2000));
    }
    index++
    if (moment(model.createdAt).isBefore(moment().subtract(1, "hour"))) {
      continue;
    }

    const hasImages = model?.siblings?.filter((sibling: Record<string, string>) => sibling?.rfilename.endsWith(".png") || sibling?.rfilename.endsWith(".jpeg") || sibling?.rfilename.endsWith(".jpg"))
    if (hasImages.length > 0) {
      model.image = hasImages[1]?.rfilename ? `https://huggingface.co/${model.id}/resolve/main/${hasImages[1]?.rfilename}` : `https://huggingface.co/${model.id}/resolve/main/${hasImages[0]?.rfilename}`
    } else {
      const hasReadme = model?.siblings?.find((sibling: Record<string, string>) => sibling?.rfilename === "README.md")
      if (hasReadme) {
        const readmeRes = await fetch(`https://huggingface.co/${model.id}/raw/main/README.md`)
        const readme = await readmeRes.text().catch(() => null)
        if (!readme) {
          continue
        }
        const imageRegex = /!\[.*\]\((.*)\)/
        let image = readme.match(imageRegex)?.[1]
    
        if (!image) {
          continue
        }
        image = image.replace(/&#x2F;/g, "/")
        if (image.startsWith("http")) model.image = image
        else model.image = `https://huggingface.co/${model.id}/resolve/main/${image.replace("./", "")}`
      }
    }

    await prisma.model.create({
      data: {
        id: model.id,
        image: model.image,
        likes: model.likes,
        downloads: model.downloads,
        createdAt: model.createdAt,
        isPublic: false,
      }
    }).catch(() => {})
    model_saved++
  }

  return json({
    message: `Successfully added ${model_saved} models`,
  })
}