File size: 3,251 Bytes
f0743f4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 | import axios from 'axios';
import FormData from 'form-data';
import { createReadStream } from 'fs';
import { logger } from '@librechat/data-schemas';
import { FileSources } from 'librechat-data-provider';
import type { ServerRequest } from '~/types';
import { logAxiosError, readFileAsString } from '~/utils';
import { generateShortLivedToken } from '~/crypto/jwt';
/**
* Attempts to parse text using RAG API, falls back to native text parsing
* @param params - The parameters object
* @param params.req - The Express request object
* @param params.file - The uploaded file
* @param params.file_id - The file ID
* @returns
*/
export async function parseText({
req,
file,
file_id,
}: {
req: ServerRequest;
file: Express.Multer.File;
file_id: string;
}): Promise<{ text: string; bytes: number; source: string }> {
if (!process.env.RAG_API_URL) {
logger.debug('[parseText] RAG_API_URL not defined, falling back to native text parsing');
return parseTextNative(file);
}
const userId = req.user?.id;
if (!userId) {
logger.debug('[parseText] No user ID provided, falling back to native text parsing');
return parseTextNative(file);
}
try {
const healthResponse = await axios.get(`${process.env.RAG_API_URL}/health`, {
timeout: 10000,
});
if (healthResponse?.statusText !== 'OK' && healthResponse?.status !== 200) {
logger.debug('[parseText] RAG API health check failed, falling back to native parsing');
return parseTextNative(file);
}
} catch (healthError) {
logAxiosError({
message: '[parseText] RAG API health check failed, falling back to native parsing:',
error: healthError,
});
return parseTextNative(file);
}
try {
const jwtToken = generateShortLivedToken(userId);
const formData = new FormData();
formData.append('file_id', file_id);
formData.append('file', createReadStream(file.path));
const formHeaders = formData.getHeaders();
const response = await axios.post(`${process.env.RAG_API_URL}/text`, formData, {
headers: {
Authorization: `Bearer ${jwtToken}`,
accept: 'application/json',
...formHeaders,
},
timeout: 300000,
});
const responseData = response.data;
logger.debug(`[parseText] RAG API completed successfully (${response.status})`);
if (!('text' in responseData)) {
throw new Error('RAG API did not return parsed text');
}
return {
text: responseData.text,
bytes: Buffer.byteLength(responseData.text, 'utf8'),
source: FileSources.text,
};
} catch (error) {
logAxiosError({
message: '[parseText] RAG API text parsing failed, falling back to native parsing',
error,
});
return parseTextNative(file);
}
}
/**
* Native JavaScript text parsing fallback
* Simple text file reading - complex formats handled by RAG API
* @param file - The uploaded file
* @returns
*/
export async function parseTextNative(file: Express.Multer.File): Promise<{
text: string;
bytes: number;
source: string;
}> {
const { content: text, bytes } = await readFileAsString(file.path, {
fileSize: file.size,
});
return {
text,
bytes,
source: FileSources.text,
};
}
|