Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
File size: 2,216 Bytes
78a356b 5df4a2a 78a356b 5df4a2a 78a356b 5df4a2a 78a356b 5df4a2a 78a356b 5df4a2a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | """Document loader for markdown files - loads RAW markdown to preserve headers."""
from pathlib import Path
from typing import List
from haystack import Document
import logging
logger = logging.getLogger(__name__)
class MarkdownDocumentLoader:
"""Loads markdown documents from a directory, preserving markdown structure."""
def __init__(self, documents_path: str):
"""
Initialize the document loader.
Args:
documents_path: Path to directory containing markdown files
"""
self.documents_path = Path(documents_path)
def load_documents(self) -> List[Document]:
"""
Load all markdown documents from the configured directory.
Loads RAW markdown content to preserve headers for semantic chunking.
Returns:
List of Haystack Document objects with raw markdown content
"""
if not self.documents_path.exists():
raise FileNotFoundError(f"Documents path does not exist: {self.documents_path}")
documents = []
markdown_files = list(self.documents_path.glob("*.md"))
if not markdown_files:
logger.warning(f"No markdown files found in {self.documents_path}")
return documents
logger.info(f"Loading {len(markdown_files)} markdown files from {self.documents_path}")
for md_file in markdown_files:
try:
# Load RAW markdown content (preserving ## headers)
content = md_file.read_text(encoding='utf-8')
# Create Haystack Document with metadata
doc = Document(
content=content,
meta={
"source_file": md_file.name,
"file_name": md_file.stem,
"file_path": str(md_file),
}
)
documents.append(doc)
logger.info(f"Loaded document: {md_file.name}")
except Exception as e:
logger.error(f"Error loading {md_file.name}: {e}")
continue
logger.info(f"Successfully loaded {len(documents)} documents")
return documents
|