import React, { useState, useCallback, useMemo, useRef, useEffect } from 'react'; import { Folder, FolderOpen, FileText, FileCode, Upload, Copy, Check, X, Trash2, CheckCircle2, AlertCircle, Maximize2, Minimize2, Search, WrapText, AlignLeft, Database, ChevronDown, ChevronUp, ChevronRight, ChevronsUpDown, Sparkles, Target, Activity, } from 'lucide-react'; import type { CorpusDataset, VirtualFile } from '../types/corpus'; import { validateAndParseCorpus, getSampleJsonlString, getSampleTsvString, MAX_CORPUS_DOCUMENTS, } from '../utils/corpusValidator'; interface FileExplorerModalProps { isOpen: boolean; onClose: () => void; datasets: CorpusDataset[]; activeDatasetId: string; onSelectDataset: (id: string) => void; onSaveCustomDataset: (dataset: CorpusDataset) => void; onDeleteCustomDataset: (id: string) => void; } // Token type for JSON syntax highlighting interface JsonToken { type: 'key' | 'string' | 'number' | 'literal' | 'punct' | 'text'; val: string; } // Fast tokenizer for a single line of JSON function tokenizeJsonLine(line: string): JsonToken[] { const tokens: JsonToken[] = []; const regex = /(\"(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\\"\n])*\")(?=\s*:)|(\"(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\\"\n])*\")|(\b(true|false|null)\b)|(-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?)|([\{\}\[\],:])/g; let lastIndex = 0; let match: RegExpExecArray | null; while ((match = regex.exec(line)) !== null) { if (match.index > lastIndex) { tokens.push({ type: 'text', val: line.slice(lastIndex, match.index) }); } if (match[1]) tokens.push({ type: 'key', val: match[1] }); else if (match[3]) tokens.push({ type: 'string', val: match[3] }); else if (match[5]) tokens.push({ type: 'literal', val: match[5] }); else if (match[7]) tokens.push({ type: 'number', val: match[7] }); else if (match[8]) tokens.push({ type: 'punct', val: match[8] }); lastIndex = regex.lastIndex; } if (lastIndex < line.length) { tokens.push({ type: 'text', val: line.slice(lastIndex) }); } return tokens; } // Format raw JSON or JSONL into pretty indented string function prettifyJsonContent(rawContent: string, isJsonl: boolean): string { if (isJsonl) { const lines = rawContent.trim().split(/\r?\n/); const prettyObjects: string[] = []; for (const line of lines) { if (!line.trim()) continue; try { const obj = JSON.parse(line); prettyObjects.push(JSON.stringify(obj, null, 2)); } catch { prettyObjects.push(line); } } return prettyObjects.join('\n\n'); } try { const parsed = JSON.parse(rawContent); return JSON.stringify(parsed, null, 2); } catch { return rawContent; } } // Split long passage text into clean, readable paragraphs function formatTextIntoParagraphs(text: string): string[] { if (!text) return []; // Split on existing double newlines if present if (text.includes('\n\n')) { return text.split(/\n\n+/).filter((p) => p.trim().length > 0); } // If text is one long paragraph with multiple sentences (e.g. medical abstract or MS MARCO passage), // split every 2-3 sentences to provide clean readability const sentences = text.match(/[^.!?]+[.!?]+(\s+|$)/g); if (!sentences || sentences.length <= 2) { return [text]; } const paragraphs: string[] = []; let current = ''; sentences.forEach((sentence, idx) => { current += sentence; if ((idx + 1) % 2 === 0 || idx === sentences.length - 1) { paragraphs.push(current.trim()); current = ''; } }); if (current.trim()) { paragraphs.push(current.trim()); } return paragraphs; } export const FileExplorerModal: React.FC = ({ isOpen, onClose, datasets, activeDatasetId, onSelectDataset, onSaveCustomDataset, onDeleteCustomDataset, }) => { const [activeTab, setActiveTab] = useState<'browse' | 'upload'>('browse'); const [isFullscreen, setIsFullscreen] = useState(false); // Browse tab state const [selectedFilePath, setSelectedFilePath] = useState(() => { const active = datasets.find((d) => d.id === activeDatasetId) || datasets[0]; return active?.files[0]?.path || '/collections/msmarco/collection_jsonl/docs00.json'; }); const [expandedFolders, setExpandedFolders] = useState>({ '/collections/msmarco': true, '/collections/msmarco/collection_jsonl': true, '/collections/nfcorpus': true, '/collections/nfcorpus/qrels': true, '/uploads/custom': true, }); const [viewMode, setViewMode] = useState<'raw' | 'parsed'>('raw'); const [formatJson, setFormatJson] = useState(true); const [wrapLines, setWrapLines] = useState(false); const [copied, setCopied] = useState(false); const [tableSearch, setTableSearch] = useState(''); const [expandedRowDocId, setExpandedRowDocId] = useState(null); // Fast Hover Tooltip Popover state const [tooltip, setTooltip] = useState<{ docId: string; field: 'title' | 'content'; text: string; top: number; left: number; } | null>(null); const [tooltipCopied, setTooltipCopied] = useState(false); const tooltipTimeoutRef = useRef | null>(null); // Upload tab state const [customName, setCustomName] = useState('My Custom Corpus'); const [customContent, setCustomContent] = useState(''); const [customFormat, setCustomFormat] = useState<'auto' | 'jsonl' | 'tsv'>('auto'); const [customQuery, setCustomQuery] = useState(''); const [selectedGoldDocIds, setSelectedGoldDocIds] = useState([]); const [goldRelevanceGrade, setGoldRelevanceGrade] = useState(2); const [isReadingFile, setIsReadingFile] = useState(false); // Upload validation const validation = customContent.trim() ? validateAndParseCorpus( customContent, customFormat === 'auto' ? undefined : customFormat ) : null; // Auto-select first document as ground truth target when valid documents are parsed useEffect(() => { if (validation?.isValid && validation.parsedDocuments.length > 0) { setSelectedGoldDocIds((prev) => { if (prev.length === 0 || !validation.parsedDocuments.some((d) => d.id === prev[0])) { return [validation.parsedDocuments[0].id]; } return prev; }); } }, [validation?.isValid, validation?.parsedDocuments]); const isUploadValid = Boolean( validation && validation.isValid && validation.parsedDocuments.length > 0 && customQuery.trim().length > 0 && selectedGoldDocIds.length > 0 ); const handleSaveUpload = useCallback(() => { if (!validation || !validation.isValid || validation.parsedDocuments.length === 0) return; if (!customQuery.trim() || selectedGoldDocIds.length === 0) return; const id = `custom-${Date.now().toString(36)}`; const format = customFormat === 'auto' ? (customContent.trim().startsWith('{') ? 'jsonl' : 'tsv') : customFormat; const virtualFile: VirtualFile = { path: `/uploads/custom/${id}.${format}`, name: `${customName.toLowerCase().replace(/\s+/g, '_')}.${format}`, type: format, sizeBytes: new TextEncoder().encode(customContent).length, lineCount: customContent.trim().split(/\r?\n/).length, datasetId: id, rawContent: customContent, }; const queryId = `q-${id}-1`; const cleanQuery = customQuery.trim(); const goldSet = new Set(selectedGoldDocIds); const newDataset: CorpusDataset = { id, name: customName.trim() || 'Custom Uploaded Corpus', description: `User corpus (${validation.documentCount} documents). Evaluation query: "${cleanQuery}"`, format, documentCount: validation.documentCount, documents: validation.parsedDocuments.map((d) => ({ ...d, goldRelevance: goldSet.has(d.id) ? goldRelevanceGrade : 0, })), queries: [ { id: queryId, text: cleanQuery, datasetId: id, relevantDocIds: selectedGoldDocIds, }, ], qrels: selectedGoldDocIds.map((docId) => ({ queryId, iteration: '0', docId, relevance: goldRelevanceGrade, })), files: [virtualFile], isCustom: true, createdAt: new Date().toISOString(), }; onSaveCustomDataset(newDataset); onSelectDataset(id); setSelectedFilePath(virtualFile.path); setActiveTab('browse'); setCustomContent(''); setCustomQuery(''); setSelectedGoldDocIds([]); onClose(); }, [ validation, customFormat, customContent, customName, customQuery, selectedGoldDocIds, goldRelevanceGrade, onSaveCustomDataset, onSelectDataset, onClose, ]); // Clean up tooltip timeout useEffect(() => { return () => { if (tooltipTimeoutRef.current) clearTimeout(tooltipTimeoutRef.current); }; }, []); // Collect all virtual files const allFiles: VirtualFile[] = datasets.flatMap((d) => d.files || []); const currentFile = allFiles.find((f) => f.path === selectedFilePath) || allFiles[0]; const currentDataset = datasets.find((d) => d.id === currentFile?.datasetId) || datasets.find((d) => d.id === activeDatasetId) || datasets[0]; const activeDataset = datasets.find((d) => d.id === activeDatasetId) || datasets[0]; // Group files by directory const filesByDir: Record = {}; allFiles.forEach((file) => { if (!file || !file.path) return; const dir = file.path.substring(0, file.path.lastIndexOf('/')) || '/'; if (!filesByDir[dir]) { filesByDir[dir] = []; } filesByDir[dir].push(file); }); const toggleFolder = (dir: string) => { setExpandedFolders((prev) => ({ ...prev, [dir]: !prev[dir] })); }; const isAllExpanded = useMemo(() => { const dirKeys = Object.keys(filesByDir); if (dirKeys.length === 0) return true; return dirKeys.every((dir) => expandedFolders[dir] !== false); }, [filesByDir, expandedFolders]); const toggleAllFolders = () => { const nextState: Record = {}; const targetState = !isAllExpanded; Object.keys(filesByDir).forEach((dir) => { nextState[dir] = targetState; }); setExpandedFolders(nextState); }; const handleCopyRaw = () => { if (!currentFile) return; navigator.clipboard.writeText(currentFile.rawContent); setCopied(true); setTimeout(() => setCopied(false), 2000); }; const handleCopyTooltip = (text: string) => { navigator.clipboard.writeText(text); setTooltipCopied(true); setTimeout(() => setTooltipCopied(false), 1500); }; // Hover Tooltip Triggers const handleCellMouseEnter = ( docId: string, field: 'title' | 'content', text: string, e: React.MouseEvent ) => { if (tooltipTimeoutRef.current) clearTimeout(tooltipTimeoutRef.current); const rect = e.currentTarget.getBoundingClientRect(); const tooltipWidth = 460; const tooltipHeight = 260; let left = rect.left; if (left + tooltipWidth > window.innerWidth - 16) { left = window.innerWidth - tooltipWidth - 16; } if (left < 16) left = 16; let top = rect.bottom + 8; if (top + tooltipHeight > window.innerHeight - 16) { top = Math.max(16, rect.top - tooltipHeight - 8); } setTooltip({ docId, field, text, top, left, }); }; const handleCellMouseLeave = () => { if (tooltipTimeoutRef.current) clearTimeout(tooltipTimeoutRef.current); tooltipTimeoutRef.current = setTimeout(() => { setTooltip(null); }, 180); }; const cancelTooltipHide = () => { if (tooltipTimeoutRef.current) clearTimeout(tooltipTimeoutRef.current); }; // Active corpus switch handler const handleSwitchActiveCorpus = (newId: string) => { onSelectDataset(newId); const targetDataset = datasets.find((d) => d.id === newId); if (targetDataset && targetDataset.files && targetDataset.files.length > 0) { setSelectedFilePath(targetDataset.files[0].path); // Auto expand folder const dir = targetDataset.files[0].path.substring(0, targetDataset.files[0].path.lastIndexOf('/')); setExpandedFolders((prev) => ({ ...prev, [dir]: true })); } }; // File type detection const isJsonFile = currentFile ? currentFile.type === 'jsonl' || (currentFile.type as string) === 'json' || (currentFile.name && currentFile.name.endsWith('.json')) || (currentFile.name && currentFile.name.endsWith('.jsonl')) || (currentFile.rawContent && currentFile.rawContent.trim().startsWith('{')) : false; const isJsonlFormat = currentFile ? currentFile.type === 'jsonl' || Boolean(currentFile.name && currentFile.name.endsWith('.jsonl')) : false; // Compute displayed lines for Raw / JSON view const formattedContent = useMemo(() => { if (!currentFile || !currentFile.rawContent) return ''; if (isJsonFile && formatJson) { return prettifyJsonContent(currentFile.rawContent, isJsonlFormat); } return currentFile.rawContent; }, [currentFile, isJsonFile, formatJson, isJsonlFormat]); const displayedLines = useMemo(() => { return (formattedContent || '').split(/\r?\n/); }, [formattedContent]); // Filtered documents for Schema Table view const filteredDocuments = useMemo(() => { if (!currentDataset || !currentDataset.documents) return []; if (!tableSearch.trim()) return currentDataset.documents; const q = tableSearch.toLowerCase(); return currentDataset.documents.filter( (d) => (d.id && d.id.toLowerCase().includes(q)) || (d.title && d.title.toLowerCase().includes(q)) || (d.text && d.text.toLowerCase().includes(q)) ); }, [currentDataset, tableSearch]); const handleFileDrop = (e: React.DragEvent) => { e.preventDefault(); const file = e.dataTransfer.files[0]; if (file) { setIsReadingFile(true); const reader = new FileReader(); reader.onload = (ev) => { const text = ev.target?.result; if (typeof text === 'string') { setCustomContent(text); setCustomName(file.name.replace(/\.[^/.]+$/, '')); } setIsReadingFile(false); }; reader.onerror = () => setIsReadingFile(false); reader.readAsText(file); } }; const handleFileInput = (e: React.ChangeEvent) => { const file = e.target.files?.[0]; if (file) { setIsReadingFile(true); const reader = new FileReader(); reader.onload = (ev) => { const text = ev.target?.result; if (typeof text === 'string') { setCustomContent(text); setCustomName(file.name.replace(/\.[^/.]+$/, '')); } setIsReadingFile(false); }; reader.onerror = () => setIsReadingFile(false); reader.readAsText(file); } }; if (!isOpen) return null; return (
{/* Modal Top Bar */}
📁
Virtual File System / Raw Collections & Schema Inspector
{/* View switcher */}
{/* Fullscreen Toggle */} {/* Close Button */}
{/* Modal Content */} {activeTab === 'browse' ? (
{/* Left Sidebar: Virtual File Tree & Active Corpus Switcher */}
Virtual Workspace
{allFiles.length} files
{/* File Tree List */}
{Object.entries(filesByDir).map(([dir, files]) => { const isExpanded = expandedFolders[dir] ?? true; const isCustomDir = dir.includes('custom'); const isDirActiveCorpus = files.some( (f) => f.datasetId === activeDatasetId ); return (
{isExpanded && (
{files.map((file) => { const isSelected = selectedFilePath === file.path; const isDatasetActive = file.datasetId === activeDatasetId; return (
setSelectedFilePath(file.path)} >
{file.type === 'jsonl' || (file.type as string) === 'json' || file.name.endsWith('.json') ? ( ) : ( )} {file.name}
{isDatasetActive ? ( active ) : ( )} {isCustomDir && ( )}
); })}
)}
); })}
{/* Active Corpus Switcher (Compact, Refined, Modern) */}
Mounted Corpus
live
{/* Styled Dropdown with Custom Chevron */}
{/* Compact Inline Stats Ribbon */}
DOCS: {activeDataset?.documentCount} • FMT: {activeDataset?.format} • QRY: {activeDataset?.queries?.length ?? 1}
{/* Right Column: File Content & Schema Inspector */}
{currentFile ? ( <> {/* File Metadata & Toolbar */}
{currentFile.path} • {(currentFile.sizeBytes / 1024).toFixed(1)} KB • {currentFile.lineCount} lines
{/* View mode switcher */}
{/* Raw Mode Controls: JSON Formatter & Line Wrap */} {viewMode === 'raw' && ( <> {isJsonFile && ( )} )} {/* Copy File Button */} {/* Mount Button if file belongs to another dataset */} {currentDataset && currentDataset.id !== activeDatasetId && ( )}
{/* Viewer Content Area */}
{viewMode === 'raw' ? ( /* Code / Raw Lines Viewer */
{displayedLines.map((line, idx) => { const tokens = isJsonFile && formatJson ? tokenizeJsonLine(line) : null; return ( {/* Line Number Gutter */} {/* Code Content */} ); })}
{idx + 1} {tokens ? ( {tokens.map((token, tIdx) => { if (token.type === 'key') { return ( {token.val} ); } if (token.type === 'string') { return ( {token.val} ); } if (token.type === 'number') { return ( {token.val} ); } if (token.type === 'literal') { return ( {token.val} ); } if (token.type === 'punct') { return ( {token.val} ); } return ( {token.val} ); })} ) : ( {line} )}
) : ( /* Enhanced Schema Table with Search, Expandable Rows, and Hover Tooltips */
{/* Table Search & Count Bar */}
setTableSearch(e.target.value)} placeholder="Search passages by docid, title, or keywords..." className="w-full bg-zinc-950 border border-zinc-800 rounded-md pl-8 pr-3 py-1.5 text-xs text-zinc-200 placeholder-zinc-600 focus:outline-none focus:border-zinc-500 font-sans" /> {tableSearch && ( )}
Showing {filteredDocuments.length} of {currentDataset?.documents.length || 0} passages • Hover cell for full formatted passage
{/* Horizontally Scrollable Table Container */}
{filteredDocuments.map((doc, idx) => { const isExpanded = expandedRowDocId === doc.id; return ( {/* Row Number */} {/* Doc ID */} {/* Document Title (with Hover Tooltip) */} {/* Passage Content (with Hover Tooltip and Full Readability) */} {/* Expand/Collapse Inline Row */} {/* Expanded Row Drawer */} {isExpanded && ( )} ); })}
# docid title passage content inspect
{idx + 1}
{doc.id} {doc.goldRelevance !== undefined && doc.goldRelevance > 0 && ( rel={doc.goldRelevance} )}
handleCellMouseEnter(doc.id, 'title', doc.title || doc.id, e) } onMouseLeave={handleCellMouseLeave} >
{doc.title || '-'}
handleCellMouseEnter(doc.id, 'content', doc.text, e) } onMouseLeave={handleCellMouseLeave} >
{doc.text}
[{doc.id}]

{doc.title || 'Untitled Passage'}

{formatTextIntoParagraphs(doc.text).map((p, pIdx) => (

{p}

))}
)}
) : (
Select a file from the sidebar.
)}
) : ( /* Tab 2: Upload Micro-Corpus */

Upload Micro-Corpus to Virtual Workspace

Paste or drop a JSONL (id, contents) or TSV (id \t text) file. Corpora are capped at {MAX_CORPUS_DOCUMENTS} documents for instantaneous indexing and benchmark evaluation.

{/* Template Presets */}
Format Templates:
{/* Inputs: Corpus Name & Format */}
setCustomName(e.target.value)} placeholder="e.g. Biomed COVID Subcorpus" className="w-full bg-zinc-900 border border-zinc-800 focus:border-zinc-600 rounded-md px-3 py-1.5 text-zinc-100 focus:outline-none text-xs" />
{/* Dropzone & Textarea */}
e.preventDefault()} onDrop={handleFileDrop} className="space-y-1.5">
Passage Data (Drop file or paste raw content):
{isReadingFile && (
Reading and parsing corpus file from disk...
)}