Natnael B. Haile
Cursor
feat: complete IR Studio with Git LFS tracking for all binary assets and NFCorpus index
fbad523 | import type { CorpusDocument, ValidationIssue, ValidationResult } from '../types/corpus'; | |
| export const MAX_CORPUS_DOCUMENTS = 100; | |
| export const MAX_FILE_SIZE_BYTES = 500 * 1024; // 500 KB | |
| /** | |
| * Validates and parses raw text content into structured CorpusDocuments. | |
| * Supports JSONL and TSV formats. | |
| */ | |
| export function validateAndParseCorpus( | |
| content: string, | |
| preferredFormat?: 'jsonl' | 'tsv' | |
| ): ValidationResult { | |
| const issues: ValidationIssue[] = []; | |
| const errors: string[] = []; | |
| const warnings: string[] = []; | |
| const trimmed = content.trim(); | |
| if (!trimmed) { | |
| errors.push('File content is empty.'); | |
| return { | |
| isValid: false, | |
| errors, | |
| warnings, | |
| issues: [{ severity: 'error', message: 'File is completely empty.' }], | |
| documentCount: 0, | |
| parsedDocuments: [], | |
| }; | |
| } | |
| const byteLength = new TextEncoder().encode(content).length; | |
| if (byteLength > MAX_FILE_SIZE_BYTES) { | |
| errors.push( | |
| `File size (${(byteLength / 1024).toFixed(1)} KB) exceeds the 500 KB pedagogical sandbox limit.` | |
| ); | |
| return { | |
| isValid: false, | |
| errors, | |
| warnings, | |
| issues: [ | |
| { | |
| severity: 'error', | |
| message: `File exceeds 500 KB (${(byteLength / 1024).toFixed(1)} KB). Keep corpora under 500 KB for sub-second in-browser indexing.`, | |
| }, | |
| ], | |
| documentCount: 0, | |
| parsedDocuments: [], | |
| }; | |
| } | |
| // Detect format if not specified | |
| const format = preferredFormat || detectFormat(trimmed); | |
| const lines = trimmed.split(/\r?\n/).filter((l) => l.trim().length > 0); | |
| const parsedDocuments: CorpusDocument[] = []; | |
| const seenIds = new Set<string>(); | |
| for (let idx = 0; idx < lines.length; idx++) { | |
| const lineNumber = idx + 1; | |
| const line = lines[idx].trim(); | |
| if (format === 'jsonl') { | |
| try { | |
| const parsed = JSON.parse(line); | |
| if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { | |
| issues.push({ | |
| line: lineNumber, | |
| severity: 'error', | |
| message: `Line ${lineNumber}: Expected JSON object, found ${typeof parsed}.`, | |
| }); | |
| continue; | |
| } | |
| // Support MS MARCO ("id", "contents") and BEIR/NFCorpus ("_id", "title", "text") | |
| const id = String(parsed.id ?? parsed._id ?? '').trim(); | |
| const text = String(parsed.contents ?? parsed.text ?? parsed.body ?? '').trim(); | |
| const title = parsed.title ? String(parsed.title).trim() : undefined; | |
| if (!id) { | |
| issues.push({ | |
| line: lineNumber, | |
| severity: 'error', | |
| message: `Line ${lineNumber}: Missing required document identifier ('id' or '_id').`, | |
| }); | |
| continue; | |
| } | |
| if (!text) { | |
| issues.push({ | |
| line: lineNumber, | |
| severity: 'error', | |
| message: `Line ${lineNumber}: Missing text payload ('contents', 'text', or 'body').`, | |
| }); | |
| continue; | |
| } | |
| if (seenIds.has(id)) { | |
| issues.push({ | |
| line: lineNumber, | |
| severity: 'warning', | |
| message: `Line ${lineNumber}: Duplicate docid "${id}" found. Subsequent instances will overwrite previous.`, | |
| }); | |
| } | |
| seenIds.add(id); | |
| parsedDocuments.push({ | |
| id, | |
| title, | |
| text, | |
| goldRelevance: 0, | |
| }); | |
| } catch (err) { | |
| const errorMsg = err instanceof Error ? err.message : String(err); | |
| issues.push({ | |
| line: lineNumber, | |
| severity: 'error', | |
| message: `Line ${lineNumber}: Invalid JSON syntax. ${errorMsg}`, | |
| }); | |
| } | |
| } else { | |
| // TSV format | |
| const cols = line.split('\t'); | |
| if (cols.length < 2) { | |
| issues.push({ | |
| line: lineNumber, | |
| severity: 'error', | |
| message: `Line ${lineNumber}: Expected at least 2 tab-separated columns ('id\\tcontents'), found ${cols.length}.`, | |
| }); | |
| continue; | |
| } | |
| const id = cols[0].trim(); | |
| let title: string | undefined; | |
| let text = ''; | |
| if (cols.length === 2) { | |
| text = cols[1].trim(); | |
| } else { | |
| title = cols[1].trim() || undefined; | |
| text = cols.slice(2).join('\t').trim(); | |
| } | |
| if (!id) { | |
| issues.push({ | |
| line: lineNumber, | |
| severity: 'error', | |
| message: `Line ${lineNumber}: Missing docid in first column.`, | |
| }); | |
| continue; | |
| } | |
| if (!text) { | |
| issues.push({ | |
| line: lineNumber, | |
| severity: 'error', | |
| message: `Line ${lineNumber}: Empty text column for docid "${id}".`, | |
| }); | |
| continue; | |
| } | |
| if (seenIds.has(id)) { | |
| issues.push({ | |
| line: lineNumber, | |
| severity: 'warning', | |
| message: `Line ${lineNumber}: Duplicate docid "${id}".`, | |
| }); | |
| } | |
| seenIds.add(id); | |
| parsedDocuments.push({ | |
| id, | |
| title, | |
| text, | |
| goldRelevance: 0, | |
| }); | |
| } | |
| } | |
| // Populate human-readable summary errors | |
| for (const issue of issues) { | |
| if (issue.severity === 'error') { | |
| errors.push(issue.message); | |
| } else if (issue.severity === 'warning') { | |
| warnings.push(issue.message); | |
| } | |
| } | |
| // Cap documents if exceeding maximum | |
| let finalDocs = parsedDocuments; | |
| if (parsedDocuments.length > MAX_CORPUS_DOCUMENTS) { | |
| warnings.push( | |
| `Corpus contains ${parsedDocuments.length} valid documents. Capped to the first ${MAX_CORPUS_DOCUMENTS} documents for responsive browser execution.` | |
| ); | |
| finalDocs = parsedDocuments.slice(0, MAX_CORPUS_DOCUMENTS); | |
| } | |
| const isValid = errors.length === 0 && finalDocs.length > 0; | |
| if (finalDocs.length === 0 && errors.length === 0) { | |
| errors.push('No valid documents could be parsed from the provided input.'); | |
| } | |
| return { | |
| isValid, | |
| errors, | |
| warnings, | |
| issues, | |
| documentCount: finalDocs.length, | |
| parsedDocuments: finalDocs, | |
| }; | |
| } | |
| /** | |
| * Detects whether content looks like JSONL or TSV. | |
| */ | |
| export function detectFormat(sample: string): 'jsonl' | 'tsv' { | |
| const firstNonEmptyLine = sample | |
| .split(/\r?\n/) | |
| .map((l) => l.trim()) | |
| .find((l) => l.length > 0); | |
| if (!firstNonEmptyLine) return 'jsonl'; | |
| if (firstNonEmptyLine.startsWith('{') && firstNonEmptyLine.endsWith('}')) { | |
| return 'jsonl'; | |
| } | |
| return 'tsv'; | |
| } | |
| /** | |
| * Generates an educational sample JSONL string with 5 documents. | |
| */ | |
| export function getSampleJsonlString(): string { | |
| return [ | |
| JSON.stringify({ | |
| id: 'doc-001', | |
| contents: | |
| 'Inverted indexes map vocabulary terms to posting lists containing document IDs, term frequencies, and positions.', | |
| }), | |
| JSON.stringify({ | |
| id: 'doc-002', | |
| contents: | |
| 'BM25 ranking incorporates term saturation via k1 and document length penalization via b to prevent long document bias.', | |
| }), | |
| JSON.stringify({ | |
| id: 'doc-003', | |
| contents: | |
| 'Dense retrieval leverages bi-encoder transformers like Contriever or DPR to project queries and passages into a shared vector space.', | |
| }), | |
| JSON.stringify({ | |
| id: 'doc-004', | |
| contents: | |
| 'SPLADE expands queries and documents with sparse vocabulary weights using masked language model logits and sparsity regularizers.', | |
| }), | |
| JSON.stringify({ | |
| id: 'doc-005', | |
| contents: | |
| 'Anserini builds on Apache Lucene to provide scalable reproducible first-stage lexical and impact search benchmarks.', | |
| }), | |
| ].join('\n'); | |
| } | |
| /** | |
| * Generates an educational sample TSV string with 5 documents. | |
| */ | |
| export function getSampleTsvString(): string { | |
| return [ | |
| 'doc-001\tInverted Index Basics\tInverted indexes map vocabulary terms to posting lists containing document IDs, term frequencies, and positions.', | |
| 'doc-002\tBM25 Saturation\tBM25 ranking incorporates term saturation via k1 and document length penalization via b to prevent long document bias.', | |
| 'doc-003\tDense Retrieval MIPS\tDense retrieval leverages bi-encoder transformers like Contriever or DPR to project queries and passages into a shared vector space.', | |
| 'doc-004\tSPLADE Learned Sparsity\tSPLADE expands queries and documents with sparse vocabulary weights using masked language model logits and sparsity regularizers.', | |
| 'doc-005\tAnserini Lucene Core\tAnserini builds on Apache Lucene to provide scalable reproducible first-stage lexical and impact search benchmarks.', | |
| ].join('\n'); | |
| } | |