Spaces:
Paused
Paused
File size: 6,411 Bytes
529090e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 | // LocalFileScanner – Scans local file system for documents and data
import { promises as fs } from 'fs';
import * as path from 'path';
import { DataSourceAdapter, IngestedEntity } from './DataIngestionEngine.js';
export interface LocalFileScannerConfig {
rootPaths: string[]; // Directories to scan
extensions?: string[]; // File extensions to include (e.g., ['.txt', '.pdf', '.docx'])
maxDepth?: number; // Maximum depth to recurse
maxFileSize?: number; // Maximum file size in bytes (default: 10MB)
excludePatterns?: string[]; // Glob patterns to exclude
}
export class LocalFileScanner implements DataSourceAdapter {
name = 'Local File Scanner';
type = 'local_files' as const;
private config: LocalFileScannerConfig;
constructor(config: LocalFileScannerConfig) {
this.config = {
maxDepth: 3,
maxFileSize: 10 * 1024 * 1024, // 10MB
extensions: ['.txt', '.md', '.pdf', '.docx', '.xlsx', '.csv', '.json'],
excludePatterns: ['node_modules', '.git', 'dist', 'build'],
...config
};
}
async isAvailable(): Promise<boolean> {
// Check if at least one root path exists
for (const rootPath of this.config.rootPaths) {
try {
await fs.access(rootPath);
return true;
} catch {
continue;
}
}
return false;
}
async fetch(): Promise<any[]> {
const files: any[] = [];
for (const rootPath of this.config.rootPaths) {
try {
await this.walkDirectory(rootPath, 0, files);
} catch (error: any) {
console.warn(`Failed to scan ${rootPath}:`, error.message);
}
}
return files;
}
private async walkDirectory(dirPath: string, depth: number, results: any[]): Promise<void> {
if (this.config.maxDepth && depth > this.config.maxDepth) {
return;
}
try {
const entries = await fs.readdir(dirPath, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dirPath, entry.name);
// Check exclusion patterns
if (this.shouldExclude(fullPath)) {
continue;
}
if (entry.isDirectory()) {
await this.walkDirectory(fullPath, depth + 1, results);
} else if (entry.isFile()) {
// Check extension
if (this.config.extensions && this.config.extensions.length > 0) {
const ext = path.extname(entry.name).toLowerCase();
if (!this.config.extensions.includes(ext)) {
continue;
}
}
// Check file size
const stat = await fs.stat(fullPath);
if (this.config.maxFileSize && stat.size > this.config.maxFileSize) {
console.warn(`Skipping large file: ${fullPath} (${stat.size} bytes)`);
continue;
}
results.push({
path: fullPath,
name: entry.name,
size: stat.size,
modifiedAt: stat.mtime,
extension: path.extname(entry.name).toLowerCase()
});
}
}
} catch (error: any) {
console.warn(`Failed to read directory ${dirPath}:`, error.message);
}
}
private shouldExclude(filePath: string): boolean {
if (!this.config.excludePatterns) return false;
return this.config.excludePatterns.some(pattern =>
filePath.includes(pattern)
);
}
async transform(files: any[]): Promise<IngestedEntity[]> {
const entities: IngestedEntity[] = [];
for (const file of files) {
try {
// For text files, read content
let content = '';
if (['.txt', '.md', '.json', '.csv'].includes(file.extension)) {
try {
content = await fs.readFile(file.path, 'utf-8');
} catch {
content = '(unable to read)';
}
} else if (file.extension === '.pdf') {
try {
const dataBuffer = await fs.readFile(file.path);
// pdf-parse v2.x uses a class-based API
const { PDFParse } = await import('pdf-parse');
const parser = new PDFParse({ data: dataBuffer });
const textResult = await parser.getText();
content = textResult.text;
await parser.destroy();
} catch (e) {
console.warn(`Failed to parse PDF ${file.path}:`, e);
content = '(unable to read pdf)';
}
}
// Truncate if too long (except for PDFs where we might want more context, but keeping safety limit)
if (content.length > 50000) {
content = content.substring(0, 50000) + '... (truncated)';
}
entities.push({
id: file.path,
type: 'local_file',
source: 'Local File Scanner',
title: file.name,
content,
metadata: {
path: file.path,
size: file.size,
extension: file.extension,
// Convert Date to ISO string for Neo4j compatibility
modifiedAt: file.modifiedAt instanceof Date
? file.modifiedAt.toISOString()
: String(file.modifiedAt),
},
timestamp: new Date()
});
} catch (error: any) {
console.warn(`Failed to transform file ${file.path}:`, error.message);
}
}
return entities;
}
}
|