fixed stuff

bun.lock

@@ -12,13 +12,11 @@
     },
   },
   "packages": {
-    "@types/bun": ["@types/bun@1.2.9", "", { "dependencies": { "bun-types": "1.2.9" } }, "sha512-epShhLGQYc4Bv/aceHbmBhOz1XgUnuTZgcxjxk+WXwNyDXavv5QHD1QEFV0FwbTSQtNq6g4ZcV6y0vZakTjswg=="],
+    "@types/bun": ["@types/bun@1.2.14", "", { "dependencies": { "bun-types": "1.2.14" } }, "sha512-VsFZKs8oKHzI7zwvECiAJ5oSorWndIWEVhfbYqZd4HI/45kzW7PN2Rr5biAzvGvRuNmYLSANY+H59ubHq8xw7Q=="],
 
     "@types/node": ["@types/node@22.14.1", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-u0HuPQwe/dHrItgHHpmw3N2fYCR6x4ivMNbPHRkBVP4CvN+kiRrKHWk3i8tXiO/joPwXLMYvF9TTF0eqgHIuOw=="],
 
-    "@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="],
-
-    "bun-types": ["bun-types@1.2.9", "", { "dependencies": { "@types/node": "*", "@types/ws": "*" } }, "sha512-dk/kOEfQbajENN/D6FyiSgOKEuUi9PWfqKQJEgwKrCMWbjS/S6tEXp178mWvWAcUSYm9ArDlWHZKO3T/4cLXiw=="],
+    "bun-types": ["bun-types@1.2.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-Kuh4Ub28ucMRWeiUUWMHsT9Wcbr4H3kLIO72RZZElSDxSu7vpetRvxIUDUaW6QtaIeixIpm7OXtNnZPf82EzwA=="],
 
     "typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="],
 

package.json

@@ -7,6 +7,6 @@
     "@types/bun": "latest"
   },
   "peerDependencies": {
-    "typescript": "^5"
+    "typescript": "^5.8.3"
   }
 }

src/index.ts

@@ -1,250 +1,386 @@
 import { serve } from "bun";
 import { type Repo } from "./types.ts";
 
-const packagesUrls = [
-  "https://raw.githubusercontent.com/Zigistry/database/refs/heads/main/database/packages.json",
-];
+// --- Configuration Constants ---
+const PORT = 7860;
+const REFRESH_INTERVAL_MS = 30 * 60 * 1000; // 30 minutes in milliseconds
+const MAX_SEARCH_RESULTS = 25;
+const DEFAULT_PAGINATION_SIZE = 10;
 
-const programsUrl =
+// Changed to a single URL for packages
+const PACKAGE_URL =
+  "https://raw.githubusercontent.com/Zigistry/database/refs/heads/main/database/packages.json";
+const PROGRAMS_URL =
   "https://raw.githubusercontent.com/Zigistry/database/refs/heads/main/database/programs.json";
 
-let packages: Repo[] = [];
-let programs: Repo[] = [];
-
-async function loadData() {
-  const packagesFile = await Promise.all(packagesUrls.map((url) => fetch(url)));
-  packages = (await Promise.all(packagesFile.map((file) => file.json())))
-    .flat() as Repo[];
-
-  const programsFile = await fetch(programsUrl);
-  programs = (await programsFile.json()) as Repo[];
-  console.log("Data loaded");
-}
-
-await loadData();
-
-// Refresh the data every 30 minutes
-setInterval(loadData, 60 * 60 * 500);
-
 // --- CORS Headers ---
-const corsHeaders: Record<string, string> = {
+const CORS_HEADERS: Record<string, string> = {
   "Content-Type": "application/json",
   "Access-Control-Allow-Origin": "*",
 };
 
-// --- Sorting Helpers ---
-const sortByDate = (a: Repo, b: Repo) =>
-  new Date(b.created_at ?? "").getTime() - new Date(a.created_at ?? "").getTime();
+// --- Data Store ---
+interface DataStore {
+  packages: Repo[];
+  programs: Repo[];
+  sortedPackages: {
+    latest: Repo[];
+    mostUsed: Repo[];
+  };
+  sortedPrograms: {
+    latest: Repo[];
+    mostUsed: Repo[];
+  };
+  lastLoaded: number; // Timestamp of last successful data load
+}
 
-const sortByUsage = (a: Repo, b: Repo) =>
-  (b.stargazers_count ?? 0) - (a.stargazers_count ?? 0);
+const dataStore: DataStore = {
+  packages: [],
+  programs: [],
+  sortedPackages: { latest: [], mostUsed: [] },
+  sortedPrograms: { latest: [], mostUsed: [] },
+  lastLoaded: 0,
+};
 
-// --- Helper to remove readme content ---
+// --- Helper Functions ---
+
+/**
+ * Removes the 'readme_content' field from an array of Repo objects.
+ * Creates a shallow copy of each repo to avoid modifying the original data.
+ * @param repos - An array of Repo objects.
+ * @returns A new array of Repo objects without the 'readme_content'.
+ */
 function removeReadmeContent(repos: Repo[]): Repo[] {
-  return repos.map(repo => ({
-    ...repo,
-    readme_content: ""
-  }));
+  return repos.map((repo) => {
+    const { readme_content, ...rest } = repo; // Destructure to exclude readme_content
+    return rest;
+  });
 }
 
-// --- Pre-sorted Data ---
-function getSorted() {
-  const packagesArray = packages as Repo[];
-  const programsArray = programs as Repo[];
-
-  return {
-    packages: {
-      latest: [...packagesArray].sort(sortByDate),
-      mostUsed: [...packagesArray].sort(sortByUsage),
-    },
-    programs: {
-      latest: [...programsArray].sort(sortByDate),
-      mostUsed: [...programsArray].sort(sortByUsage),
-    },
-  };
-}
+/**
+ * Sorts repositories by creation date (newest first).
+ * @param a - First Repo object.
+ * @param b - Second Repo object.
+ * @returns A number indicating sort order.
+ */
+const sortByDate = (a: Repo, b: Repo) =>
+  new Date(b.created_at ?? "").getTime() -
+  new Date(a.created_at ?? "").getTime();
+
+/**
+ * Sorts repositories by stargazers count (most used first).
+ * @param a - First Repo object.
+ * @param b - Second Repo object.
+ * @returns A number indicating sort order.
+ */
+const sortByUsage = (a: Repo, b: Repo) =>
+  (b.stargazers_count ?? 0) - (a.stargazers_count ?? 0);
 
-// --- Filtering ---
+/**
+ * Filters an array of Repo objects based on a search query and a topic filter.
+ * @param items - The array of Repo objects to filter.
+ * @param q - The search query string (case-insensitive).
+ * @param filter - The topic filter string (case-insensitive).
+ * @returns A new array of filtered Repo objects.
+ */
 function filterItems(
   items: Repo[],
   q: string | null,
   filter: string | null
 ): Repo[] {
-  return items.filter(({ name, full_name, description, topics, readme_content }) => {
-    if (filter && !topics?.some((t) => t.toLowerCase() === filter)) return false;
-    if (!q) return true;
-    const lowerQ = q.toLowerCase();
-    return [name, full_name, description, ...(topics ?? []), readme_content].some((field) =>
-      field?.toLowerCase().includes(lowerQ)
-    );
-  });
+  const lowerQ = q?.toLowerCase();
+  const lowerFilter = filter?.toLowerCase();
+
+  return items.filter(
+    ({ name, full_name, description, topics, readme_content }) => {
+      // Apply topic filter first
+      if (
+        lowerFilter &&
+        !topics?.some((t) => t.toLowerCase() === lowerFilter)
+      ) {
+        return false;
+      }
+
+      // If no search query, all items passing the filter are included
+      if (!lowerQ) {
+        return true;
+      }
+
+      // Check if any relevant field includes the search query
+      return [name, full_name, description, ...(topics ?? []), readme_content].some(
+        (field) => field?.toLowerCase().includes(lowerQ)
+      );
+    }
+  );
 }
 
-// --- Pagination ---
-function getPaginated(items: Repo[], page = 0, size = 10): Repo[] {
+/**
+ * Paginates an array of items.
+ * @param items - The array of items to paginate.
+ * @param page - The page number (0-indexed).
+ * @param size - The number of items per page.
+ * @returns A new array containing items for the specified page.
+ */
+function getPaginated<T>(items: T[], page = 0, size = DEFAULT_PAGINATION_SIZE): T[] {
   const start = page * size;
   return items.slice(start, start + size);
 }
 
-// --- Find by Owner/Repo ---
-function findItem(items: Repo[], owner: string, repo: string): Repo | undefined {
-  return items.find(
-    ({ full_name }) => full_name?.toLowerCase() === `${owner}/${repo}`
-  );
-}
-
-// --- Parse Range ---
+/**
+ * Parses a range string (e.g., "0..10") into start and end numbers.
+ * Ensures the range is within valid bounds.
+ * @param str - The range string.
+ * @param max - The maximum possible value for the end of the range.
+ * @returns A tuple [start, end].
+ */
 function parseRange(str: string | null, max: number): [number, number] {
   const match = str?.match(/^(\d+)\.\.(\d+)$/);
-  const [start, end] = match
-    ? [parseInt(match[1] ?? "0", 10), parseInt(match[2] ?? "10", 10)]
-    : [0, 10];
+  let start = 0;
+  let end = DEFAULT_PAGINATION_SIZE; // Default range if no match
+
+  if (match) {
+    start = parseInt(match[1] ?? "0", 10);
+    end = parseInt(match[2] ?? String(DEFAULT_PAGINATION_SIZE), 10);
+  }
+
   return [Math.max(0, start), Math.min(max, end)];
 }
 
+// --- Data Loading and Caching ---
+
+/**
+ * Fetches data from a given URL and parses it as JSON.
+ * @param url - The URL to fetch.
+ * @returns A Promise that resolves with the parsed JSON data, or null if an error occurs.
+ */
+async function fetchData<T>(url: string): Promise<T | null> {
+  try {
+    const response = await fetch(url);
+    if (!response.ok) {
+      console.error(`Failed to fetch ${url}: ${response.statusText}`);
+      return null;
+    }
+    return (await response.json()) as T;
+  } catch (error) {
+    console.error(`Error fetching or parsing ${url}:`, error);
+    return null;
+  }
+}
+
+/**
+ * Loads and processes all repository data (packages and programs).
+ * Updates the global dataStore and pre-sorts the data.
+ */
+async function loadData() {
+  console.log("Attempting to load data...");
+  try {
+    // Fetch packages (now from a single URL)
+    const newPackages = await fetchData<Repo[]>(PACKAGE_URL);
+
+    // Fetch programs
+    const newPrograms = await fetchData<Repo[]>(PROGRAMS_URL);
+
+    if (newPackages && newPrograms) {
+      dataStore.packages = newPackages;
+      dataStore.programs = newPrograms;
+
+      // Pre-sort data after successful load
+      dataStore.sortedPackages.latest = [...newPackages].sort(sortByDate);
+      dataStore.sortedPackages.mostUsed = [...newPackages].sort(sortByUsage);
+      dataStore.sortedPrograms.latest = [...newPrograms].sort(sortByDate);
+      dataStore.sortedPrograms.mostUsed = [...newPrograms].sort(sortByUsage);
+
+      dataStore.lastLoaded = Date.now();
+      console.log("Data loaded successfully.");
+    } else {
+      console.warn("Failed to load all data. Retaining old data if available.");
+    }
+  } catch (error) {
+    console.error("Critical error during data loading:", error);
+  }
+}
+
+/**
+ * Checks if data needs to be refreshed and triggers loadData if necessary.
+ */
+async function ensureDataLoaded() {
+  if (Date.now() - dataStore.lastLoaded > REFRESH_INTERVAL_MS) {
+    console.log("Data is stale, refreshing...");
+    await loadData();
+  } else if (dataStore.lastLoaded === 0) {
+    // Initial load if not already done
+    console.log("Initial data load...");
+    await loadData();
+  }
+}
+
+// Perform initial data load immediately on server start
+await loadData();
+
+// Set up interval for refreshing data
+setInterval(ensureDataLoaded, REFRESH_INTERVAL_MS / 2); // Check more frequently than refresh interval
+
+// --- API Route Handlers ---
+
+/**
+ * Handles search requests for packages or programs.
+ * @param items - The array of Repo items to search.
+ * @param searchParams - URLSearchParams object from the request.
+ * @returns A Response object with filtered and truncated results.
+ */
+function handleSearch(items: Repo[], searchParams: URLSearchParams): Response {
+  const q = searchParams.get("q")?.trim() ?? null;
+  const filter = searchParams.get("filter")?.trim() ?? null;
+  const result = filterItems(items, q, filter).slice(0, MAX_SEARCH_RESULTS);
+  return Response.json(removeReadmeContent(result), { headers: CORS_HEADERS });
+}
+
+/**
+ * Handles infinite scroll requests for packages or programs.
+ * @param items - The array of Repo items to paginate.
+ * @param searchParams - URLSearchParams object from the request.
+ * @returns A Response object with paginated results.
+ */
+function handleInfiniteScroll(items: Repo[], searchParams: URLSearchParams): Response {
+  const page = parseInt(searchParams.get("pageNumber") || "0", 10);
+  if (isNaN(page) || page < 0) {
+    return Response.json(
+      { error: "Invalid page number. Must be a non-negative integer." },
+      { status: 400, headers: CORS_HEADERS }
+    );
+  }
+  const result = getPaginated(items, page);
+  return Response.json(removeReadmeContent(result), { headers: CORS_HEADERS });
+}
+
+/**
+ * Handles requests for a single package or program by owner/repo name.
+ * @param items - The array of Repo items to search within.
+ * @param owner - The owner part of the full_name.
+ * @param repo - The repo part of the full_name.
+ * @param repoFrom - The repo_from identifier.
+ * @returns A Response object with the found item or a 404 error.
+ */
+function handleSingleItem(
+  items: Repo[],
+  owner: string,
+  repo: string,
+  repoFrom: string | null = null
+): Response {
+  const found = items.find(
+    (item) =>
+      item.full_name?.toLowerCase() === `${owner}/${repo}` &&
+      (repoFrom === null || item.repo_from?.toLowerCase() === repoFrom)
+  );
+
+  if (found) {
+    // Return the full item, including readme_content, for single item requests
+    return Response.json(found, { status: 200, headers: CORS_HEADERS });
+  }
+  return Response.json(
+    { error: "Item not found" },
+    { status: 404, headers: CORS_HEADERS }
+  );
+}
+
+/**
+ * Handles requests for index details (latest or most used).
+ * @param sortedData - The pre-sorted data object (e.g., dataStore.sortedPackages).
+ * @param searchParams - URLSearchParams object from the request.
+ * @returns A Response object with sliced sorted results.
+ */
+function handleIndexDetails(
+  sortedData: { latest: Repo[]; mostUsed: Repo[] },
+  searchParams: URLSearchParams
+): Response {
+  const section = searchParams.get("section");
+  if (section !== "latestRepos" && section !== "mostUsed") {
+    return Response.json(
+      { error: "Invalid section. Must be 'latestRepos' or 'mostUsed'." },
+      { status: 400, headers: CORS_HEADERS }
+    );
+  }
+
+  const sortKey = section === "latestRepos" ? "latest" : "mostUsed";
+  const data = sortedData[sortKey];
+  const [start, end] = parseRange(searchParams.get("range"), data.length);
+  const result = data.slice(start, end);
+  return Response.json(removeReadmeContent(result), { headers: CORS_HEADERS });
+}
+
+// --- Router Mapping ---
+const routes: Record<
+  string,
+  (url: URL) => Promise<Response> | Response
+> = {
+  "/api/searchPackages": (url) =>
+    handleSearch(dataStore.packages, url.searchParams),
+  "/api/searchPrograms": (url) =>
+    handleSearch(dataStore.programs, url.searchParams),
+  "/api/infiniteScrollPackages": (url) =>
+    handleInfiniteScroll(dataStore.packages, url.searchParams),
+  "/api/infiniteScrollPrograms": (url) =>
+    handleInfiniteScroll(dataStore.programs, url.searchParams),
+  "/api/indexDetailsPackages": (url) =>
+    handleIndexDetails(dataStore.sortedPackages, url.searchParams),
+  "/api/indexDetailsPrograms": (url) =>
+    handleIndexDetails(dataStore.sortedPrograms, url.searchParams),
+};
+
 // --- Server ---
 serve({
-  port: 7860,
+  port: PORT,
   async fetch(req) {
     const url = new URL(req.url);
     const { pathname, searchParams } = url;
-    const q = searchParams.get("q")?.trim().toLowerCase() ?? null;
-    const filter = searchParams.get("filter")?.trim().toLowerCase() ?? null;
-    const sorted = getSorted();
 
     // Handle CORS preflight
     if (req.method === "OPTIONS") {
       return new Response(null, {
         status: 204,
         headers: {
-          ...corsHeaders,
+          ...CORS_HEADERS,
           "Access-Control-Allow-Methods": "GET, POST, OPTIONS",
           "Access-Control-Allow-Headers": "Content-Type",
         },
       });
     }
 
-    // Search endpoints
-    if (pathname === "/api/searchPackages") {
-      const result = filterItems(packages, q, filter).slice(0, 25);
-      return Response.json(removeReadmeContent(result), { headers: corsHeaders });
-    }
-    if (pathname === "/api/searchPrograms") {
-      const result = filterItems(programs, q, filter).slice(0, 25);
-      return Response.json(removeReadmeContent(result), { headers: corsHeaders });
-    }
+    // Ensure data is loaded/refreshed before handling requests
+    await ensureDataLoaded();
 
-    // Infinite scroll endpoints
-    if (pathname === "/api/infiniteScrollPackages") {
-      const page = parseInt(searchParams.get("pageNumber") || "0", 10);
-      if (isNaN(page) || page < 0)
-        return Response.json(
-          { error: "Invalid page number" },
-          { status: 400, headers: corsHeaders }
-        );
-      const result = getPaginated(packages, page);
-      return Response.json(removeReadmeContent(result), {
-        headers: corsHeaders,
-      });
-    }
-    if (pathname === "/api/infiniteScrollPrograms") {
-      const page = parseInt(searchParams.get("pageNumber") || "0", 10);
-      if (isNaN(page) || page < 0)
-        return Response.json(
-          { error: "Invalid page number" },
-          { status: 400, headers: corsHeaders }
-        );
-      const result = getPaginated(programs, page);
-      return Response.json(removeReadmeContent(result), {
-        headers: corsHeaders,
-      });
+    // Direct route match
+    if (routes[pathname]) {
+      return routes[pathname](url);
     }
 
-    // Get single program by repo_from/owner/repo
-    const programMatch = pathname.match(/^\/api\/programs\/([^/]+)\/([^/]+)\/([^/]+)$/);
+    // Regex-based routes for single items
+    const programMatch = pathname.match(
+      /^\/api\/programs\/([^/]+)\/([^/]+)\/([^/]+)$/
+    );
     if (programMatch) {
       const repo_from = programMatch[1]?.toLowerCase() ?? "";
       const owner = programMatch[2]?.toLowerCase() ?? "";
       const repo = programMatch[3]?.toLowerCase() ?? "";
-      const found = programs.find(
-        (item) =>
-          item.repo_from?.toLowerCase() === repo_from &&
-          item.full_name?.toLowerCase() === `${owner}/${repo}`
-      );
-      if (found) {
-        const result = { ...found, readme_content: "" };
-        return Response.json(result, {
-          status: 200,
-          headers: corsHeaders,
-        });
-      }
-      return Response.json({ error: "Program not found" }, {
-        status: 404,
-        headers: corsHeaders,
-      });
+      return handleSingleItem(dataStore.programs, owner, repo, repo_from);
     }
 
-    // Get single package by repo_from/owner/repo
-    const packageMatch = pathname.match(/^\/api\/packages\/([^/]+)\/([^/]+)\/([^/]+)$/);
+    const packageMatch = pathname.match(
+      /^\/api\/packages\/([^/]+)\/([^/]+)\/([^/]+)$/
+    );
     if (packageMatch) {
       const repo_from = packageMatch[1]?.toLowerCase() ?? "";
       const owner = packageMatch[2]?.toLowerCase() ?? "";
       const repo = packageMatch[3]?.toLowerCase() ?? "";
-      const found = packages.find(
-        (item) =>
-          item.repo_from?.toLowerCase() === repo_from &&
-          item.full_name?.toLowerCase() === `${owner}/${repo}`
-      );
-      if (found) {
-        const result = { ...found, readme_content: "" };
-        return Response.json(result, {
-          status: 200,
-          headers: corsHeaders,
-        });
-      }
-      return Response.json({ error: "Package not found" }, {
-        status: 404,
-        headers: corsHeaders,
-      });
-    }
-
-
-    // Index details endpoints
-    if (pathname === "/api/indexDetailsPackages") {
-      const section = searchParams.get("section");
-      if (section !== "latestRepos" && section !== "mostUsed") {
-        return Response.json(
-          { error: "Invalid section" },
-          { status: 400, headers: corsHeaders }
-        );
-      }
-      const sortKey = section === "latestRepos" ? "latest" : "mostUsed";
-      const data = sorted.packages[sortKey as keyof typeof sorted.packages] ?? packages;
-      const [start, end] = parseRange(searchParams.get("range"), data.length);
-      const result = data.slice(start, end);
-      return Response.json(removeReadmeContent(result), { headers: corsHeaders });
-    }
-    if (pathname === "/api/indexDetailsPrograms") {
-      const section = searchParams.get("section");
-      if (section !== "latestRepos" && section !== "mostUsed") {
-        return Response.json(
-          { error: "Invalid section" },
-          { status: 400, headers: corsHeaders }
-        );
-      }
-      const sortKey = section === "latestRepos" ? "latest" : "mostUsed";
-      const data = sorted.programs[sortKey as keyof typeof sorted.programs] ?? programs;
-      const [start, end] = parseRange(searchParams.get("range"), data.length);
-      const result = data.slice(start, end);
-      return Response.json(removeReadmeContent(result), { headers: corsHeaders });
+      return handleSingleItem(dataStore.packages, owner, repo, repo_from);
     }
 
     // Not found
     return new Response("Not Found", {
       status: 404,
-      headers: corsHeaders,
+      headers: CORS_HEADERS,
     });
   },
 });
 
-console.log("Server running on http://localhost:7860");
+console.log(`Server running on http://localhost:${PORT}`);