Update utils/ingestion.py

utils/ingestion.py

@@ -1,7 +1,9 @@
 import json
 import time
 import os
+import logging
 from pathlib import Path
+import yaml
 from typing import Dict, Any, List
 import chromadb
 
@@ -15,10 +17,11 @@ from docling.document_converter import (
 )
 from docling.pipeline.simple_pipeline import SimplePipeline
 from docling.pipeline.standard_pdf_pipeline import StandardPdfPipeline
-from docling.document import DoclingDocument
 from docling.chunking.hierarchical_chunker import HierarchicalChunker
 from langchain_community.embeddings.fastembed import FastEmbedEmbeddings
 
+_log = logging.getLogger(__name__)
+
 class DocumentProcessor:
     def __init__(self):
         """Initialize document processor with Docling v2 changes"""
@@ -41,11 +44,13 @@ class DocumentProcessor:
                 InputFormat.PPTX,
                 InputFormat.TXT,  # Added text format
                 InputFormat.CSV,  # Added CSV format
+                InputFormat.ASCIIDOC,  # Added AsciiDoc format
+                InputFormat.MD,  # Added Markdown format
             ],
             format_options={
                 InputFormat.PDF: PdfFormatOption(
-                    pipeline_options=pipeline_options,
-                    backend=PyPdfiumDocumentBackend()
+                    pipeline_cls=StandardPdfPipeline,
+                    backend=PyPdfiumDocumentBackend
                 ),
                 InputFormat.DOCX: WordFormatOption(
                     pipeline_cls=SimplePipeline
@@ -61,17 +66,24 @@ class DocumentProcessor:
 
         try:
             conv_result = self.converter.convert(file_path)
-            doc: DoclingDocument = conv_result.document
+            doc = conv_result.document
         except Exception as e:
             print(f"❌ Conversion failed: {e}")
             return None
 
-        # Save document as markdown
+        # Save document as markdown, JSON, and YAML
         output_dir = Path("parsed-doc")
         output_dir.mkdir(parents=True, exist_ok=True)
         doc_filename = Path(file_path).stem
-        md_filename = output_dir / f"{doc_filename}.md"
-        doc.save_as_markdown(md_filename)
+
+        with (output_dir / f"{doc_filename}.md").open("w") as fp:
+            fp.write(doc.export_to_markdown())
+
+        with (output_dir / f"{doc_filename}.json").open("w") as fp:
+            fp.write(json.dumps(doc.export_to_dict()))
+
+        with (output_dir / f"{doc_filename}.yaml").open("w") as fp:
+            fp.write(yaml.safe_dump(doc.export_to_dict()))
 
         chunker = HierarchicalChunker()
         chunks = list(chunker.chunk(doc))