File size: 2,456 Bytes
f8b5d42 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | const path = require("path");
const fs = require("fs");
const {
WATCH_DIRECTORY,
SUPPORTED_FILETYPE_CONVERTERS,
} = require("../utils/constants");
const {
trashFile,
isTextType,
normalizePath,
isWithin,
} = require("../utils/files");
const RESERVED_FILES = ["__HOTDIR__.md"];
/**
* Process a single file and return the documents
* @param {string} targetFilename - The filename to process
* @param {Object} options - The options for the file processing
* @param {Object} metadata - The metadata for the file processing
* @returns {Promise<{success: boolean, reason: string, documents: Object[]}>} - The documents from the file processing
*/
async function processSingleFile(targetFilename, options = {}, metadata = {}) {
const fullFilePath = path.resolve(
WATCH_DIRECTORY,
normalizePath(targetFilename)
);
if (!isWithin(path.resolve(WATCH_DIRECTORY), fullFilePath))
return {
success: false,
reason: "Filename is a not a valid path to process.",
documents: [],
};
if (RESERVED_FILES.includes(targetFilename))
return {
success: false,
reason: "Filename is a reserved filename and cannot be processed.",
documents: [],
};
if (!fs.existsSync(fullFilePath))
return {
success: false,
reason: "File does not exist in upload directory.",
documents: [],
};
const fileExtension = path.extname(fullFilePath).toLowerCase();
if (fullFilePath.includes(".") && !fileExtension) {
return {
success: false,
reason: `No file extension found. This file cannot be processed.`,
documents: [],
};
}
let processFileAs = fileExtension;
if (!SUPPORTED_FILETYPE_CONVERTERS.hasOwnProperty(fileExtension)) {
if (isTextType(fullFilePath)) {
console.log(
`\x1b[33m[Collector]\x1b[0m The provided filetype of ${fileExtension} does not have a preset and will be processed as .txt.`
);
processFileAs = ".txt";
} else {
trashFile(fullFilePath);
return {
success: false,
reason: `File extension ${fileExtension} not supported for parsing and cannot be assumed as text file type.`,
documents: [],
};
}
}
const FileTypeProcessor = require(SUPPORTED_FILETYPE_CONVERTERS[
processFileAs
]);
return await FileTypeProcessor({
fullFilePath,
filename: targetFilename,
options,
metadata,
});
}
module.exports = {
processSingleFile,
};
|