use anyhow::{anyhow, Context, Result}; use memmap2::Mmap; use quick_xml::events::Event; use quick_xml::reader::Reader; use std::fs::File; use std::io::{BufRead, Cursor, Read}; use std::path::Path; use thiserror::Error; #[derive(Error, Debug)] pub enum ParserError { #[error("I/O error: {0}")] Io(#[from] std::io::Error), #[error("Unsupported file format: {0}")] UnsupportedFormat(String), #[error("Corrupt document: {0}")] CorruptDocument(String), } #[derive(Debug, Clone)] pub struct ParsedDocument { pub path: String, pub title: String, pub content: String, pub extension: String, pub size_bytes: u64, } pub struct DocumentParser; impl DocumentParser { /// Ingests a file with zero-copy mmap for large files or standard read for small files. pub fn parse_file>(path: P) -> Result { let path_ref = path.as_ref(); let metadata = std::fs::metadata(path_ref) .with_context(|| format!("Failed to read metadata for {:?}", path_ref))?; let size_bytes = metadata.len(); if size_bytes == 0 { return Ok(ParsedDocument { path: path_ref.to_string_lossy().to_string(), title: Self::extract_title(path_ref), content: String::new(), extension: Self::get_extension(path_ref), size_bytes: 0, }); } let extension = Self::get_extension(path_ref); let title = Self::extract_title(path_ref); let file = File::open(path_ref)?; // Zero-copy memory map for files >= 16 KB; heap read for smaller files let content = if size_bytes >= 16 * 1024 { let mmap = unsafe { Mmap::map(&file)? }; Self::extract_content_from_bytes(&mmap, &extension)? } else { let mut buffer = Vec::with_capacity(size_bytes as usize); let mut reader = std::io::BufReader::new(file); reader.read_to_end(&mut buffer)?; Self::extract_content_from_bytes(&buffer, &extension)? }; Ok(ParsedDocument { path: path_ref.to_string_lossy().to_string(), title, content, extension, size_bytes, }) } fn extract_content_from_bytes(bytes: &[u8], extension: &str) -> Result { match extension.to_lowercase().as_str() { "txt" | "md" | "markdown" | "log" | "rs" | "py" | "js" | "ts" | "toml" | "yaml" | "yml" => { match std::str::from_utf8(bytes) { Ok(valid_str) => Ok(valid_str.to_string()), Err(_) => Ok(String::from_utf8_lossy(bytes).into_owned()), } } "json" => { let val: serde_json::Value = serde_json::from_slice(bytes) .context("Invalid JSON document")?; if let Some(text) = val.as_str() { Ok(text.to_string()) } else { Ok(serde_json::to_string_pretty(&val)?) } } "csv" => Self::parse_csv(bytes), "docx" => Self::parse_docx(bytes), "pdf" => Self::parse_pdf(bytes), ext => Err(ParserError::UnsupportedFormat(ext.to_string()).into()), } } fn parse_csv(bytes: &[u8]) -> Result { let mut rdr = csv::ReaderBuilder::new() .flexible(true) .has_headers(true) .from_reader(Cursor::new(bytes)); let mut output = String::with_capacity(bytes.len()); if let Ok(headers) = rdr.headers() { output.push_str(&headers.iter().collect::>().join(" | ")); output.push('\n'); } for result in rdr.records() { let record = result?; output.push_str(&record.iter().collect::>().join(" | ")); output.push('\n'); } Ok(output) } fn parse_docx(bytes: &[u8]) -> Result { let cursor = Cursor::new(bytes); let mut archive = zip::ZipArchive::new(cursor) .context("Failed to open DOCX as ZIP archive")?; let mut document_xml = archive .by_name("word/document.xml") .context("Missing word/document.xml inside DOCX archive")?; let mut xml_bytes = Vec::new(); document_xml.read_to_end(&mut xml_bytes)?; let mut reader = Reader::from_reader(Cursor::new(xml_bytes)); reader.config_mut().trim_text(true); let mut txt = String::new(); let mut buf = Vec::new(); let mut in_text_node = false; loop { match reader.read_event_into(&mut buf) { Ok(Event::Start(ref e)) if e.name().as_ref() == b"w:t" => { in_text_node = true; } Ok(Event::End(ref e)) if e.name().as_ref() == b"w:t" => { in_text_node = false; } Ok(Event::End(ref e)) if e.name().as_ref() == b"w:p" => { txt.push('\n'); } Ok(Event::Text(e)) if in_text_node => { if let Ok(s) = e.unescape() { txt.push_str(&s); } } Ok(Event::Eof) => break, Err(e) => return Err(anyhow!("XML Parsing error in DOCX: {}", e)), _ => (), } buf.clear(); } Ok(txt) } fn parse_pdf(bytes: &[u8]) -> Result { pdf_extract::extract_text_from_mem(bytes) .context("Failed to extract text stream from PDF") } fn extract_title(path: &Path) -> String { path.file_name() .map(|f| f.to_string_lossy().to_string()) .unwrap_or_else(|| "Untitled".to_string()) } fn get_extension(path: &Path) -> String { path.extension() .map(|e| e.to_string_lossy().to_string()) .unwrap_or_else(|| "txt".to_string()) } }