lib/queue.ts

type Task<T> = () => Promise<T>;

class SerialQueue {
  private chain: Promise<unknown> = Promise.resolve();

  run<T>(task: Task<T>, retries = 1): Promise<T> {
    const attempt = async (remaining: number): Promise<T> => {
      try {
        return await task();
      } catch (err) {
        if (remaining > 0) {
          await new Promise((r) => setTimeout(r, 1500));
          return attempt(remaining - 1);
        }
        throw err;
      }
    };

    const result = this.chain.then(() => attempt(retries));
    this.chain = result.catch(() => {}); // keep the chain alive even after a failure
    return result;
  }
}

export const crawlQueue = new SerialQueue();