File size: 2,331 Bytes
f8b5d42 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 | const fs = require("fs").promises;
class PDFLoader {
constructor(filePath, { splitPages = true } = {}) {
this.filePath = filePath;
this.splitPages = splitPages;
}
async load() {
const buffer = await fs.readFile(this.filePath);
const { getDocument, version } = await this.getPdfJS();
const pdf = await getDocument({
data: new Uint8Array(buffer),
useWorkerFetch: false,
isEvalSupported: false,
useSystemFonts: true,
}).promise;
const meta = await pdf.getMetadata().catch(() => null);
const documents = [];
for (let i = 1; i <= pdf.numPages; i += 1) {
const page = await pdf.getPage(i);
const content = await page.getTextContent();
if (content.items.length === 0) {
continue;
}
let lastY;
const textItems = [];
for (const item of content.items) {
if ("str" in item) {
if (lastY === item.transform[5] || !lastY) {
textItems.push(item.str);
} else {
textItems.push(`\n${item.str}`);
}
lastY = item.transform[5];
}
}
const text = textItems.join("");
documents.push({
pageContent: text.trim(),
metadata: {
source: this.filePath,
pdf: {
version,
info: meta?.info,
metadata: meta?.metadata,
totalPages: pdf.numPages,
},
loc: { pageNumber: i },
},
});
}
if (this.splitPages) {
return documents;
}
if (documents.length === 0) {
return [];
}
return [
{
pageContent: documents.map((doc) => doc.pageContent).join("\n\n"),
metadata: {
source: this.filePath,
pdf: {
version,
info: meta?.info,
metadata: meta?.metadata,
totalPages: pdf.numPages,
},
},
},
];
}
async getPdfJS() {
try {
const pdfjs = await import("pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js");
return { getDocument: pdfjs.getDocument, version: pdfjs.version };
} catch (e) {
console.error(e);
throw new Error(
"Failed to load pdf-parse. Please install it with eg. `npm install pdf-parse`."
);
}
}
}
module.exports = PDFLoader;
|