Delete src

src/indexer.rs

@@ -1,236 +0,0 @@
-use crate::parser::ParsedDocument;
-use anyhow::{Context, Result};
-use chrono::Utc;
-use serde::{Deserialize, Serialize};
-use std::fs;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-use tantivy::collector::TopDocs;
-use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
-use tantivy::schema::*;
-use tantivy::snippet::SnippetGenerator;
-use tantivy::{doc, Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, Term};
-use tokio::sync::RwLock;
-
-#[derive(Clone)]
-pub struct IndexSchemaFields {
-    pub id: Field,
-    pub collection: Field,
-    pub path: Field,
-    pub title: Field,
-    pub content: Field,
-    pub extension: Field,
-    pub file_size: Field,
-    pub indexed_at: Field,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct SearchHit {
-    pub score: f32,
-    pub collection: String,
-    pub path: String,
-    pub title: String,
-    pub snippet: String,
-    pub extension: String,
-    pub file_size: u64,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct CollectionStats {
-    pub collection_name: String,
-    pub doc_count: u64,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct IndexStats {
-    pub total_documents: u64,
-    pub num_segments: usize,
-    pub index_path: String,
-    pub collections: Vec<CollectionStats>,
-}
-
-pub struct TantivyEngine {
-    index: Index,
-    reader: IndexReader,
-    writer: Arc<RwLock<IndexWriter>>,
-    fields: IndexSchemaFields,
-    base_path: PathBuf,
-}
-
-impl TantivyEngine {
-    pub fn new<P: AsRef<Path>>(base_path: P) -> Result<Self> {
-        let path_buf = base_path.as_ref().to_path_buf();
-        fs::create_dir_all(&path_buf)?;
-
-        let mut schema_builder = Schema::builder();
-        let id = schema_builder.add_text_field("id", STRING | STORED);
-        let collection = schema_builder.add_text_field("collection", STRING | STORED | FAST);
-        let path = schema_builder.add_text_field("path", STRING | STORED);
-        let title = schema_builder.add_text_field("title", TEXT | STORED);
-        let content = schema_builder.add_text_field("content", TEXT | STORED);
-        let extension = schema_builder.add_text_field("extension", STRING | STORED);
-        let file_size = schema_builder.add_u64_field("file_size", STORED | FAST);
-        let indexed_at = schema_builder.add_i64_field("indexed_at", STORED | FAST);
-
-        let schema = schema_builder.build();
-
-        let index = Index::open_or_create(
-            tantivy::directory::MmapDirectory::open(&path_buf)?,
-            schema.clone(),
-        )?;
-
-        // 100MB memory budget for the writer buffer
-        let writer = index.writer(100 * 1024 * 1024)?;
-        let reader = index
-            .reader_builder()
-            .reload_policy(ReloadPolicy::OnCommitWithDelay)
-            .try_into()?;
-
-        let fields = IndexSchemaFields {
-            id,
-            collection,
-            path,
-            title,
-            content,
-            extension,
-            file_size,
-            indexed_at,
-        };
-
-        Ok(Self {
-            index,
-            reader,
-            writer: Arc::new(RwLock::new(writer)),
-            fields,
-            base_path: path_buf,
-        })
-    }
-
-    pub async fn add_documents(&self, collection_name: &str, docs: Vec<ParsedDocument>) -> Result<usize> {
-        let count = docs.len();
-        let writer = self.writer.write().await;
-
-        for doc in docs {
-            let mut tantivy_doc = TantivyDocument::default();
-            let doc_id = uuid::Uuid::new_v4().to_string();
-
-            tantivy_doc.add_text(self.fields.id, &doc_id);
-            tantivy_doc.add_text(self.fields.collection, collection_name);
-            tantivy_doc.add_text(self.fields.path, &doc.path);
-            tantivy_doc.add_text(self.fields.title, &doc.title);
-            tantivy_doc.add_text(self.fields.content, &doc.content);
-            tantivy_doc.add_text(self.fields.extension, &doc.extension);
-            tantivy_doc.add_u64(self.fields.file_size, doc.size_bytes);
-            tantivy_doc.add_i64(self.fields.indexed_at, Utc::now().timestamp());
-
-            writer.add_document(tantivy_doc)?;
-        }
-
-        // Commit and reload index reader
-        let mut writer_guard = writer;
-        writer_guard.commit()?;
-        drop(writer_guard);
-
-        self.reader.reload()?;
-        Ok(count)
-    }
-
-    pub fn search(
-        &self,
-        query_str: &str,
-        collection_filter: Option<&str>,
-        limit: usize,
-    ) -> Result<Vec<SearchHit>> {
-        let searcher = self.reader.searcher();
-        let query_parser = QueryParser::for_index(
-            &self.index,
-            vec![self.fields.title, self.fields.content],
-        );
-
-        let parsed_query = query_parser
-            .parse_query(query_str)
-            .context("Query parse error")?;
-
-        let final_query: Box<dyn Query> = if let Some(coll) = collection_filter {
-            let coll_term = Term::from_field_text(self.fields.collection, coll);
-            let coll_query = TermQuery::new(coll_term, IndexRecordOption::Basic);
-
-            Box::new(BooleanQuery::new(vec![
-                (Occur::Must, parsed_query),
-                (Occur::Must, Box::new(coll_query)),
-            ]))
-        } else {
-            parsed_query
-        };
-
-        let top_docs = searcher.search(&final_query, &TopDocs::with_limit(limit))?;
-        let mut snippet_generator =
-            SnippetGenerator::create(&searcher, &*final_query, self.fields.content)?;
-        snippet_generator.set_max_num_chars(250);
-
-        let mut hits = Vec::with_capacity(top_docs.len());
-
-        for (score, doc_address) in top_docs {
-            let retrieved_doc: TantivyDocument = searcher.doc(doc_address)?;
-
-            let coll_val = retrieved_doc
-                .get_first(self.fields.collection)
-                .and_then(|v| v.as_str())
-                .unwrap_or("default")
-                .to_string();
-
-            let path_val = retrieved_doc
-                .get_first(self.fields.path)
-                .and_then(|v| v.as_str())
-                .unwrap_or("")
-                .to_string();
-
-            let title_val = retrieved_doc
-                .get_first(self.fields.title)
-                .and_then(|v| v.as_str())
-                .unwrap_or("")
-                .to_string();
-
-            let ext_val = retrieved_doc
-                .get_first(self.fields.extension)
-                .and_then(|v| v.as_str())
-                .unwrap_or("")
-                .to_string();
-
-            let size_val = retrieved_doc
-                .get_first(self.fields.file_size)
-                .and_then(|v| v.as_u64())
-                .unwrap_or(0);
-
-            let snippet = snippet_generator.snippet_from_doc(&retrieved_doc).to_html();
-
-            hits.push(SearchHit {
-                score,
-                collection: coll_val,
-                path: path_val,
-                title: title_val,
-                snippet,
-                extension: ext_val,
-                file_size: size_val,
-            });
-        }
-
-        Ok(hits)
-    }
-
-    pub fn get_stats(&self) -> Result<IndexStats> {
-        let searcher = self.reader.searcher();
-        let total_docs = searcher.num_docs();
-        let num_segments = searcher.segment_readers().len();
-
-        Ok(IndexStats {
-            total_documents: total_docs,
-            num_segments,
-            index_path: self.base_path.to_string_lossy().to_string(),
-            collections: vec![CollectionStats {
-                collection_name: "global".to_string(),
-                doc_count: total_docs,
-            }],
-        })
-    }
-}

src/main.rs

@@ -1,77 +0,0 @@
-mod indexer;
-mod mcp;
-mod parser;
-
-use clap::Parser;
-use indexer::TantivyEngine;
-use mcp::McpServer;
-use std::path::PathBuf;
-use std::sync::Arc;
-use tracing::info;
-
-#[derive(Parser, Debug)]
-#[command(name = "rust-mcp-search")]
-#[command(about = "Ultra-high-performance Tantivy-backed Model Context Protocol server", long_about = None)]
-struct Args {
-    /// Transport mode: 'stdio' or 'http'
-    #[arg(short, long, env = "MCP_TRANSPORT", default_value = "stdio")]
-    transport: String,
-
-    /// HTTP host binding
-    #[arg(long, env = "HOST", default_value = "0.0.0.0")]
-    host: String,
-
-    /// HTTP port binding (defaults to HF Spaces standard 7860)
-    #[arg(short, long, env = "PORT", default_value = "7860")]
-    port: u16,
-
-    /// Tantivy index directory path
-    #[arg(short, long, env = "DATA_DIR", default_value = "./data/index")]
-    index_dir: PathBuf,
-
-    /// Rayon thread pool concurrency limit (0 = auto-detect hardware concurrency)
-    #[arg(long, env = "RAYON_NUM_THREADS", default_value = "0")]
-    threads: usize,
-}
-
-#[tokio::main]
-async fn main() -> anyhow::Result<()> {
-    let args = Args::parse();
-
-    // Direct all logs strictly to stderr to prevent breaking the MCP STDIO JSON protocol
-    tracing_subscriber::fmt()
-        .with_writer(std::io::stderr)
-        .with_env_filter(
-            tracing_subscriber::EnvFilter::from_default_env()
-                .add_directive(tracing::Level::INFO.into()),
-        )
-        .init();
-
-    if args.threads > 0 {
-        rayon::ThreadPoolBuilder::new()
-            .num_threads(args.threads)
-            .build_global()?;
-        info!("Configured Rayon thread pool with {} threads", args.threads);
-    }
-
-    info!("Initializing Tantivy index engine at: {:?}", args.index_dir);
-    let engine = Arc::new(TantivyEngine::new(&args.index_dir)?);
-    let server = Arc::new(McpServer::new(engine));
-
-    match args.transport.to_lowercase().as_str() {
-        "http" | "sse" => {
-            info!("Starting MCP HTTP/SSE transport...");
-            server.run_http(&args.host, args.port).await?;
-        }
-        "stdio" => {
-            info!("Starting MCP STDIO transport...");
-            server.run_stdio().await?;
-        }
-        other => {
-            eprintln!("Unknown transport '{}'. Expected 'stdio' or 'http'.", other);
-            std::process::exit(1);
-        }
-    }
-
-    Ok(())
-}

src/mcp.rs

@@ -1,388 +0,0 @@
-use crate::indexer::TantivyEngine;
-use crate::parser::{DocumentParser, ParsedDocument};
-use anyhow::Result;
-use axum::{
-    extract::State,
-    http::{header, HeaderMap, HeaderValue},
-    response::sse::{Event, KeepAlive, Sse},
-    routing::{get, post},
-    Json, Router,
-};
-use futures::stream::Stream;
-use rayon::prelude::*;
-use serde::{Deserialize, Serialize};
-use serde_json::{json, Value};
-use std::convert::Infallible;
-use std::path::Path;
-use std::sync::Arc;
-use std::time::Duration;
-use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
-use tokio_stream::StreamExt;
-use tracing::info;
-use walkdir::WalkDir;
-
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct JsonRpcRequest {
-    pub jsonrpc: String,
-    pub id: Option<Value>,
-    pub method: String,
-    pub params: Option<Value>,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct JsonRpcResponse {
-    pub jsonrpc: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub id: Option<Value>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub result: Option<Value>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub error: Option<JsonRpcError>,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct JsonRpcError {
-    pub code: i64,
-    pub message: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub data: Option<Value>,
-}
-
-pub struct McpServer {
-    engine: Arc<TantivyEngine>,
-}
-
-impl McpServer {
-    pub fn new(engine: Arc<TantivyEngine>) -> Self {
-        Self { engine }
-    }
-
-    pub async fn handle_request(&self, req: JsonRpcRequest) -> JsonRpcResponse {
-        let req_id = req.id.clone();
-
-        match req.method.as_str() {
-            "initialize" => {
-                let init_result = json!({
-                    "protocolVersion": "2024-11-05",
-                    "capabilities": {
-                        "tools": {
-                            "listChanged": true
-                        }
-                    },
-                    "serverInfo": {
-                        "name": "rust-mcp-search",
-                        "version": "0.1.0"
-                    }
-                });
-                Self::success_response(req_id, init_result)
-            }
-            "notifications/initialized" | "ping" => {
-                Self::success_response(req_id, json!({}))
-            }
-            "tools/list" => {
-                let tools = json!({
-                    "tools": [
-                        {
-                            "name": "parse_and_index",
-                            "description": "Recursively scans a directory and indexes plain text, Markdown, JSON, PDF, CSV, and DOCX files in parallel using Rayon and Tantivy.",
-                            "inputSchema": {
-                                "type": "object",
-                                "properties": {
-                                    "directory_path": {
-                                        "type": "string",
-                                        "description": "Absolute or relative directory path to index"
-                                    },
-                                    "collection_name": {
-                                        "type": "string",
-                                        "description": "Collection namespace identifier (default: 'default')"
-                                    }
-                                },
-                                "required": ["directory_path"]
-                            }
-                        },
-                        {
-                            "name": "search_documents",
-                            "description": "Full-text BM25 search over indexed documents with highlighted snippets and relevance scores.",
-                            "inputSchema": {
-                                "type": "object",
-                                "properties": {
-                                    "query": {
-                                        "type": "string",
-                                        "description": "Tantivy BM25 search query"
-                                    },
-                                    "collection_name": {
-                                        "type": "string",
-                                        "description": "Filter results by collection"
-                                    },
-                                    "limit": {
-                                        "type": "integer",
-                                        "description": "Max hits to return (default: 10)"
-                                    }
-                                },
-                                "required": ["query"]
-                            }
-                        },
-                        {
-                            "name": "extract_document_text",
-                            "description": "High-speed raw text extraction from a single file (TXT, MD, PDF, DOCX, CSV, JSON).",
-                            "inputSchema": {
-                                "type": "object",
-                                "properties": {
-                                    "file_path": {
-                                        "type": "string",
-                                        "description": "Path to document"
-                                    }
-                                },
-                                "required": ["file_path"]
-                            }
-                        },
-                        {
-                            "name": "get_index_stats",
-                            "description": "Returns indexed document counts, segment counts, active collections, and memory/storage status.",
-                            "inputSchema": {
-                                "type": "object",
-                                "properties": {}
-                            }
-                        }
-                    ]
-                });
-                Self::success_response(req_id, tools)
-            }
-            "tools/call" => {
-                let params = req.params.unwrap_or_default();
-                let tool_name = params.get("name").and_then(|v| v.as_str()).unwrap_or("");
-                let args = params.get("arguments").cloned().unwrap_or(json!({}));
-
-                match self.dispatch_tool(tool_name, args).await {
-                    Ok(tool_output) => Self::success_response(
-                        req_id,
-                        json!({
-                            "content": [
-                                {
-                                    "type": "text",
-                                    "text": tool_output
-                                }
-                            ],
-                            "isError": false
-                        }),
-                    ),
-                    Err(e) => Self::success_response(
-                        req_id,
-                        json!({
-                            "content": [
-                                {
-                                    "type": "text",
-                                    "text": format!("Error executing tool '{}': {}", tool_name, e)
-                                }
-                            ],
-                            "isError": true
-                        }),
-                    ),
-                }
-            }
-            _ => Self::error_response(req_id, -32601, format!("Method '{}' not found", req.method)),
-        }
-    }
-
-    async fn dispatch_tool(&self, name: &str, args: Value) -> Result<String> {
-        match name {
-            "parse_and_index" => {
-                let dir = args
-                    .get("directory_path")
-                    .and_then(|v| v.as_str())
-                    .ok_or_else(|| anyhow::anyhow!("Missing 'directory_path' argument"))?;
-                let collection = args
-                    .get("collection_name")
-                    .and_then(|v| v.as_str())
-                    .unwrap_or("default");
-
-                let dir_path = Path::new(dir).to_path_buf();
-                if !dir_path.exists() {
-                    return Err(anyhow::anyhow!("Directory does not exist: {:?}", dir_path));
-                }
-
-                let paths: Vec<_> = WalkDir::new(&dir_path)
-                    .into_iter()
-                    .filter_map(|e| e.ok())
-                    .filter(|e| e.file_type().is_file())
-                    .map(|e| e.into_path())
-                    .collect();
-
-                let total_found = paths.len();
-
-                let parsed_docs: Vec<ParsedDocument> = paths
-                    .par_iter()
-                    .filter_map(|path| match DocumentParser::parse_file(path) {
-                        Ok(doc) => Some(doc),
-                        Err(err) => {
-                            tracing::warn!("Skipping {:?}: {}", path, err);
-                            None
-                        }
-                    })
-                    .collect();
-
-                let parsed_count = parsed_docs.len();
-                let indexed_count = self.engine.add_documents(collection, parsed_docs).await?;
-
-                Ok(json!({
-                    "status": "success",
-                    "collection": collection,
-                    "scanned_files": total_found,
-                    "parsed_documents": parsed_count,
-                    "indexed_documents": indexed_count,
-                    "source_path": dir
-                })
-                .to_string())
-            }
-            "search_documents" => {
-                let query = args
-                    .get("query")
-                    .and_then(|v| v.as_str())
-                    .ok_or_else(|| anyhow::anyhow!("Missing 'query' argument"))?;
-                let collection = args.get("collection_name").and_then(|v| v.as_str());
-                let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(10) as usize;
-
-                let hits = self.engine.search(query, collection, limit)?;
-                Ok(serde_json::to_string_pretty(&hits)?)
-            }
-            "extract_document_text" => {
-                let path_str = args
-                    .get("file_path")
-                    .and_then(|v| v.as_str())
-                    .ok_or_else(|| anyhow::anyhow!("Missing 'file_path' argument"))?;
-
-                let doc = DocumentParser::parse_file(path_str)?;
-                Ok(json!({
-                    "path": doc.path,
-                    "title": doc.title,
-                    "extension": doc.extension,
-                    "size_bytes": doc.size_bytes,
-                    "content": doc.content
-                })
-                .to_string())
-            }
-            "get_index_stats" => {
-                let stats = self.engine.get_stats()?;
-                Ok(serde_json::to_string_pretty(&stats)?)
-            }
-            unknown => Err(anyhow::anyhow!("Unknown tool name: {}", unknown)),
-        }
-    }
-
-    fn success_response(id: Option<Value>, result: Value) -> JsonRpcResponse {
-        JsonRpcResponse {
-            jsonrpc: "2.0".to_string(),
-            id,
-            result: Some(result),
-            error: None,
-        }
-    }
-
-    fn error_response(id: Option<Value>, code: i64, message: String) -> JsonRpcResponse {
-        JsonRpcResponse {
-            jsonrpc: "2.0".to_string(),
-            id,
-            result: None,
-            error: Some(JsonRpcError {
-                code,
-                message,
-                data: None,
-            }),
-        }
-    }
-
-    pub async fn run_stdio(self: Arc<Self>) -> Result<()> {
-        let stdin = tokio::io::stdin();
-        let mut reader = BufReader::new(stdin).lines();
-        let mut stdout = tokio::io::stdout();
-
-        while let Some(line) = reader.next_line().await? {
-            if line.trim().is_empty() {
-                continue;
-            }
-
-            match serde_json::from_str::<JsonRpcRequest>(&line) {
-                Ok(request) => {
-                    let response = self.handle_request(request).await;
-                    let out_json = serde_json::to_string(&response)?;
-                    stdout.write_all(out_json.as_bytes()).await?;
-                    stdout.write_all(b"\n").await?;
-                    stdout.flush().await?;
-                }
-                Err(e) => {
-                    let err_resp = Self::error_response(None, -32700, format!("Parse error: {}", e));
-                    let out_json = serde_json::to_string(&err_resp)?;
-                    stdout.write_all(out_json.as_bytes()).await?;
-                    stdout.write_all(b"\n").await?;
-                    stdout.flush().await?;
-                }
-            }
-        }
-        Ok(())
-    }
-
-    pub async fn run_http(self: Arc<Self>, host: &str, port: u16) -> Result<()> {
-        let app = Router::new()
-            .route("/rpc", post(handle_http_rpc))
-            .route("/sse", get(handle_sse))
-            .route("/health", get(|| async { "healthy" }))
-            .layer(tower_http::cors::CorsLayer::permissive())
-            .with_state(self);
-
-        let addr = format!("{}:{}", host, port);
-        info!("MCP HTTP/SSE server listening on http://{}", addr);
-        let listener = tokio::net::TcpListener::bind(&addr).await?;
-        axum::serve(listener, app).await?;
-        Ok(())
-    }
-}
-
-async fn handle_http_rpc(
-    State(server): State<Arc<McpServer>>,
-    Json(payload): Json<JsonRpcRequest>,
-) -> Json<JsonRpcResponse> {
-    let resp = server.handle_request(payload).await;
-    Json(resp)
-}
-
-async fn handle_sse(
-    State(_server): State<Arc<McpServer>>,
-) -> (HeaderMap, Sse<impl Stream<Item = Result<Event, Infallible>>>) {
-    let mut headers = HeaderMap::new();
-    headers.insert("x-accel-buffering", HeaderValue::from_static("no"));
-    headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache, no-transform"));
-    headers.insert(header::CONNECTION, HeaderValue::from_static("keep-alive"));
-    headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("text/event-stream"));
-
-    // Emit initial endpoint AND tool availability notification immediately
-    let initial_events = vec![
-        Ok(Event::default().event("endpoint").data("/rpc")),
-        Ok(Event::default().event("message").data(
-            json!({
-                "jsonrpc": "2.0",
-                "method": "notifications/tools/list_changed",
-                "params": {}
-            })
-            .to_string(),
-        )),
-    ];
-
-    let initial = tokio_stream::iter(initial_events);
-
-    // 5-second heartbeats keep connection alive on Hugging Face Spaces proxy
-    let interval = tokio_stream::wrappers::IntervalStream::new(tokio::time::interval(
-        Duration::from_secs(5),
-    ))
-    .map(|_| Ok(Event::default().comment("keep-alive")));
-
-    let stream = initial.chain(interval);
-
-    (
-        headers,
-        Sse::new(stream).keep_alive(
-            KeepAlive::new()
-                .interval(Duration::from_secs(5))
-                .text("keep-alive"),
-        ),
-    )
-}

src/parser.rs

@@ -1,179 +0,0 @@
-use anyhow::{anyhow, Context, Result};
-use memmap2::Mmap;
-use quick_xml::events::Event;
-use quick_xml::reader::Reader;
-use std::fs::File;
-use std::io::{Cursor, Read};
-use std::path::Path;
-use thiserror::Error;
-
-#[derive(Error, Debug)]
-pub enum ParserError {
-    #[error("I/O error: {0}")]
-    Io(#[from] std::io::Error),
-    #[error("Unsupported file format: {0}")]
-    UnsupportedFormat(String),
-    #[error("Corrupt document: {0}")]
-    CorruptDocument(String),
-}
-
-#[derive(Debug, Clone)]
-pub struct ParsedDocument {
-    pub path: String,
-    pub title: String,
-    pub content: String,
-    pub extension: String,
-    pub size_bytes: u64,
-}
-
-pub struct DocumentParser;
-
-impl DocumentParser {
-    pub fn parse_file<P: AsRef<Path>>(path: P) -> Result<ParsedDocument> {
-        let path_ref = path.as_ref();
-        let metadata = std::fs::metadata(path_ref)
-            .with_context(|| format!("Failed to read metadata for {:?}", path_ref))?;
-        let size_bytes = metadata.len();
-
-        if size_bytes == 0 {
-            return Ok(ParsedDocument {
-                path: path_ref.to_string_lossy().to_string(),
-                title: Self::extract_title(path_ref),
-                content: String::new(),
-                extension: Self::get_extension(path_ref),
-                size_bytes: 0,
-            });
-        }
-
-        let extension = Self::get_extension(path_ref);
-        let title = Self::extract_title(path_ref);
-
-        let file = File::open(path_ref)?;
-        
-        let content = if size_bytes >= 16 * 1024 {
-            let mmap = unsafe { Mmap::map(&file)? };
-            Self::extract_content_from_bytes(&mmap, &extension)?
-        } else {
-            let mut buffer = Vec::with_capacity(size_bytes as usize);
-            let mut reader = std::io::BufReader::new(file);
-            reader.read_to_end(&mut buffer)?;
-            Self::extract_content_from_bytes(&buffer, &extension)?
-        };
-
-        Ok(ParsedDocument {
-            path: path_ref.to_string_lossy().to_string(),
-            title,
-            content,
-            extension,
-            size_bytes,
-        })
-    }
-
-    fn extract_content_from_bytes(bytes: &[u8], extension: &str) -> Result<String> {
-        match extension.to_lowercase().as_str() {
-            "txt" | "md" | "markdown" | "log" | "rs" | "py" | "js" | "ts" | "toml" | "yaml" | "yml" => {
-                match std::str::from_utf8(bytes) {
-                    Ok(valid_str) => Ok(valid_str.to_string()),
-                    Err(_) => Ok(String::from_utf8_lossy(bytes).into_owned()),
-                }
-            }
-            "json" => {
-                let val: serde_json::Value = serde_json::from_slice(bytes)
-                    .context("Invalid JSON document")?;
-                if let Some(text) = val.as_str() {
-                    Ok(text.to_string())
-                } else {
-                    Ok(serde_json::to_string_pretty(&val)?)
-                }
-            }
-            "csv" => Self::parse_csv(bytes),
-            "docx" => Self::parse_docx(bytes),
-            "pdf" => Self::parse_pdf(bytes),
-            ext => Err(ParserError::UnsupportedFormat(ext.to_string()).into()),
-        }
-    }
-
-    fn parse_csv(bytes: &[u8]) -> Result<String> {
-        let mut rdr = csv::ReaderBuilder::new()
-            .flexible(true)
-            .has_headers(true)
-            .from_reader(Cursor::new(bytes));
-
-        let mut output = String::with_capacity(bytes.len());
-        
-        if let Ok(headers) = rdr.headers() {
-            output.push_str(&headers.iter().collect::<Vec<_>>().join(" | "));
-            output.push('\n');
-        }
-
-        for result in rdr.records() {
-            let record = result?;
-            output.push_str(&record.iter().collect::<Vec<_>>().join(" | "));
-            output.push('\n');
-        }
-
-        Ok(output)
-    }
-
-    fn parse_docx(bytes: &[u8]) -> Result<String> {
-        let cursor = Cursor::new(bytes);
-        let mut archive = zip::ZipArchive::new(cursor)
-            .context("Failed to open DOCX as ZIP archive")?;
-
-        let mut document_xml = archive
-            .by_name("word/document.xml")
-            .context("Missing word/document.xml inside DOCX archive")?;
-
-        let mut xml_bytes = Vec::new();
-        document_xml.read_to_end(&mut xml_bytes)?;
-
-        let mut reader = Reader::from_reader(Cursor::new(xml_bytes));
-        reader.config_mut().trim_text(true);
-
-        let mut txt = String::new();
-        let mut buf = Vec::new();
-        let mut in_text_node = false;
-
-        loop {
-            match reader.read_event_into(&mut buf) {
-                Ok(Event::Start(ref e)) if e.name().as_ref() == b"w:t" => {
-                    in_text_node = true;
-                }
-                Ok(Event::End(ref e)) if e.name().as_ref() == b"w:t" => {
-                    in_text_node = false;
-                }
-                Ok(Event::End(ref e)) if e.name().as_ref() == b"w:p" => {
-                    txt.push('\n');
-                }
-                Ok(Event::Text(e)) if in_text_node => {
-                    if let Ok(s) = e.unescape() {
-                        txt.push_str(&s);
-                    }
-                }
-                Ok(Event::Eof) => break,
-                Err(e) => return Err(anyhow!("XML Parsing error in DOCX: {}", e)),
-                _ => (),
-            }
-            buf.clear();
-        }
-
-        Ok(txt)
-    }
-
-    fn parse_pdf(bytes: &[u8]) -> Result<String> {
-        pdf_extract::extract_text_from_mem(bytes)
-            .context("Failed to extract text stream from PDF")
-    }
-
-    fn extract_title(path: &Path) -> String {
-        path.file_name()
-            .map(|f| f.to_string_lossy().to_string())
-            .unwrap_or_else(|| "Untitled".to_string())
-    }
-
-    fn get_extension(path: &Path) -> String {
-        path.extension()
-            .map(|e| e.to_string_lossy().to_string())
-            .unwrap_or_else(|| "txt".to_string())
-    }
-}