saveTrendingToJSON.mjs

import { promises as fs } from "fs";

const fetchJson = async (url) => {
  const response = await fetch(url);
  if (!response.ok) throw new Error("Network response was not ok");
  return response.json();
};

const fetchAllPages = async () => {
  const pageNumbers = Array.from({ length: 2 }, (_, i) => i);
  const urls = pageNumbers.map((pageNumber) => {
    const url = new URL("https://huggingface.co/spaces-json");
    url.search = new URLSearchParams({
      p: pageNumber,
      sort: "trending",
    }).toString();
    return url;
  });
  const jsonResponses = await Promise.all(urls.map(fetchJson));
  return jsonResponses.flatMap((json) => json.spaces);
};

const addAppFileToSpaces = async (spaces) => {
  return Promise.all(
    spaces.map(async (space) => {
      try {
        const url = `https://huggingface.co/spaces/${space.id}/resolve/main/app.py`;
        const response = await fetch(url);
        if (response.ok) {
          console.log(`Fetched appFile for space ${space.id}`);
          const appFile = await response.text();
          return { ...space, appFile };
        }
      } catch (error) {
        console.error(
          `Failed to fetch appFile for space ${space.id}: ${error}`
        );
      }
      return space;
    })
  );
};

const saveAllTrendingSpaces = async () => {
  try {
    const spaces = await fetchAllPages();
    const spacesWithAppFile = await addAppFileToSpaces(spaces);
    const publicSpaces = spacesWithAppFile.filter(
      (space) => space.private === false
    );
    await fs.writeFile(
      "trendingSpaces.json",
      JSON.stringify(publicSpaces, null, 2)
    );
    console.log("Data written to file");
  } catch (error) {
    console.error(`An error occurred: ${error}`);
  }
};

saveAllTrendingSpaces();