src/extract.js

// ----------------------------------------------------------------------------
// Text extraction for uploaded files. Runs server-side where real parsers exist.
//   pdf  -> pdf-parse
//   docx -> mammoth
//   html -> html-to-text
//   ipynb-> parse notebook JSON (markdown + code + text outputs)
//   txt/md/py/json/csv/... -> utf8
// ----------------------------------------------------------------------------

import mammoth from 'mammoth';
import { convert as htmlToText } from 'html-to-text';
import { createRequire } from 'module';

// pdf-parse is CommonJS; import it via createRequire in this ESM module.
const require = createRequire(import.meta.url);

export async function extractText(ext, buffer) {
  const e = String(ext || '').toLowerCase().replace(/^\./, '');

  try {
    if (e === 'pdf') {
      // FIX (BUG-2): require the library file directly, NOT the package root.
      // pdf-parse's index.js contains a debug block that runs when
      // `module.parent` is not set (which is the case under ESM/createRequire)
      // and tries to read './test/data/05-versions-space.pdf' — crashing every
      // PDF ingest with ENOENT. lib/pdf-parse.js is the real API without that
      // debug wrapper.
      const pdfParse = require('pdf-parse/lib/pdf-parse.js');
      const data = await pdfParse(buffer);
      return (data.text || '').trim();
    }

    if (e === 'docx') {
      const { value } = await mammoth.extractRawText({ buffer });
      return (value || '').trim();
    }

    if (e === 'html' || e === 'htm') {
      return htmlToText(buffer.toString('utf8'), {
        wordwrap: false,
        selectors: [
          { selector: 'a', options: { ignoreHref: true } },
          { selector: 'img', format: 'skip' },
        ],
      }).trim();
    }

    if (e === 'ipynb') {
      return extractNotebook(buffer.toString('utf8'));
    }

    // txt, md, py, json, csv, etc.
    return buffer.toString('utf8').trim();
  } catch (err) {
    console.error(`[extract] ${e} failed:`, err.message);
    return '';
  }
}

function extractNotebook(json) {
  try {
    const nb = JSON.parse(json);
    const cells = Array.isArray(nb.cells) ? nb.cells : [];
    const parts = [];
    for (const cell of cells) {
      const src = Array.isArray(cell.source) ? cell.source.join('') : cell.source || '';
      if (cell.cell_type === 'markdown') {
        parts.push(src);
      } else if (cell.cell_type === 'code') {
        parts.push('```python\n' + src + '\n```');
        for (const out of cell.outputs || []) {
          const txt =
            (out.text && (Array.isArray(out.text) ? out.text.join('') : out.text)) ||
            (out.data && out.data['text/plain'] &&
              (Array.isArray(out.data['text/plain'])
                ? out.data['text/plain'].join('')
                : out.data['text/plain']));
          if (txt) parts.push('Output:\n' + txt);
        }
      }
    }
    return parts.join('\n\n').trim();
  } catch {
    return json;
  }
}