lib/chunker.ts

export function chunkText(text: string, size: number = 3000): string[] {
  const chunks: string[] = [];
  for (let i = 0; i < text.length; i += size) {
    chunks.push(text.slice(i, i + size));
  }
  return chunks.length > 0 ? chunks : [""];
}

export function chunkArray<T>(arr: T[], size: number = 20): T[][] {
  const chunks: T[][] = [];
  for (let i = 0; i < arr.length; i += size) {
    chunks.push(arr.slice(i, i + size));
  }
  return chunks.length > 0 ? chunks : [[]];
}

// Recursively walks the object. Any string longer than textChunkSize,
// or array longer than arrayChunkSize, becomes "<field>Chunks": [...].
// Short fields (title, url, favicon, etc.) pass through untouched.
export function chunkData(
  data: any,
  textChunkSize: number = 3000,
  arrayChunkSize: number = 20
): any {
  const result: any = {};

  for (const key of Object.keys(data)) {
    const value = data[key];

    if (typeof value === "string") {
      if (value.length > textChunkSize) {
        result[`${key}Chunks`] = chunkText(value, textChunkSize);
      } else {
        result[key] = value;
      }
    } else if (Array.isArray(value)) {
      if (value.length > arrayChunkSize) {
        result[`${key}Chunks`] = chunkArray(value, arrayChunkSize);
      } else {
        result[key] = value;
      }
    } else if (value && typeof value === "object") {
      result[key] = chunkData(value, textChunkSize, arrayChunkSize);
    } else {
      result[key] = value;
    }
  }

  return result;
}