src/ingest.js

import { readdirSync, readFileSync } from 'node:fs';
import { join } from 'node:path';
import { createRequire } from 'node:module';
import { config } from './config.js';
import { embed } from './embeddings.js';
import { save } from './vectorstore.js';

const require = createRequire(import.meta.url);
const pdfParse = require('pdf-parse');

function chunkText(text, size, overlap) {
  const clean = text.replace(/\s+/g, ' ').trim();
  const chunks = [];
  for (let i = 0; i < clean.length; i += size - overlap) {
    chunks.push(clean.slice(i, i + size));
  }
  return chunks.filter((c) => c.trim().length > 30);
}

async function main() {
  const files = readdirSync(config.pdfDir).filter((f) => f.toLowerCase().endsWith('.pdf'));
  if (!files.length) {
    console.error(`Nenhum PDF em ${config.pdfDir}. Coloque seus arquivos lá.`);
    process.exit(1);
  }

  const items = [];
  let id = 0;
  for (const file of files) {
    const buf = readFileSync(join(config.pdfDir, file));
    const { text } = await pdfParse(buf);
    const chunks = chunkText(text, config.chunkSize, config.chunkOverlap);
    console.log(`📄 ${file}: ${chunks.length} chunks`);

    // embeddings em lotes para não estourar o payload da API
    const batch = 16;
    for (let i = 0; i < chunks.length; i += batch) {
      const slice = chunks.slice(i, i + batch);
      const vectors = await embed(slice);
      slice.forEach((text, j) => {
        items.push({ id: id++, text, source: file, vector: vectors[j] });
      });
      process.stdout.write(`  embeddings ${Math.min(i + batch, chunks.length)}/${chunks.length}\r`);
    }
    console.log('');
  }

  save({ items });
  console.log(`✅ Indexados ${items.length} chunks em ${config.storePath}`);
}

main().catch((e) => {
  console.error(e);
  process.exit(1);
});