tests/test_document_processor.py

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent.parent / "src"))

import pytest
import io
from unittest.mock import Mock, MagicMock, patch


class TestExtractPdfPages:
    """Tests for extract_pdf_pages function."""
    
    @patch("document_processor.pdfplumber")
    @patch("document_processor.tempfile.NamedTemporaryFile")
    @patch("document_processor.os.unlink")
    def test_extracts_pages_with_text(self, mock_unlink, mock_tempfile, mock_pdfplumber):
        """Should extract text from PDF pages."""
        from document_processor import extract_pdf_pages
        
        mock_tmp = MagicMock()
        mock_tmp.name = "/tmp/test.pdf"
        mock_tmp.__enter__ = Mock(return_value=mock_tmp)
        mock_tmp.__exit__ = Mock(return_value=False)
        mock_tempfile.return_value = mock_tmp
        
        mock_page = Mock()
        mock_page.extract_text.return_value = "Page content"
        
        mock_pdf = MagicMock()
        mock_pdf.pages = [mock_page]
        mock_pdf.__enter__ = Mock(return_value=mock_pdf)
        mock_pdf.__exit__ = Mock(return_value=False)
        mock_pdfplumber.open.return_value = mock_pdf
        
        file_obj = io.BytesIO(b"fake pdf content")
        result = extract_pdf_pages(file_obj, "test.pdf")
        
        assert len(result) == 1
        assert result[0]["text"] == "Page content"
        assert result[0]["page_number"] == 1
        assert result[0]["source"] == "test.pdf"
    
    @patch("document_processor.pdfplumber")
    @patch("document_processor.tempfile.NamedTemporaryFile")
    @patch("document_processor.os.unlink")
    def test_skips_empty_pages(self, mock_unlink, mock_tempfile, mock_pdfplumber):
        """Should skip pages with no text."""
        from document_processor import extract_pdf_pages
        
        mock_tmp = MagicMock()
        mock_tmp.name = "/tmp/test.pdf"
        mock_tmp.__enter__ = Mock(return_value=mock_tmp)
        mock_tmp.__exit__ = Mock(return_value=False)
        mock_tempfile.return_value = mock_tmp
        
        mock_page1 = Mock()
        mock_page1.extract_text.return_value = "Content"
        mock_page2 = Mock()
        mock_page2.extract_text.return_value = "   "
        
        mock_pdf = MagicMock()
        mock_pdf.pages = [mock_page1, mock_page2]
        mock_pdf.__enter__ = Mock(return_value=mock_pdf)
        mock_pdf.__exit__ = Mock(return_value=False)
        mock_pdfplumber.open.return_value = mock_pdf
        
        file_obj = io.BytesIO(b"fake pdf")
        result = extract_pdf_pages(file_obj, "test.pdf")
        
        assert len(result) == 1
    
    @patch("document_processor.pdfplumber")
    @patch("document_processor.tempfile.NamedTemporaryFile")
    @patch("document_processor.os.unlink")
    def test_cleans_up_temp_file(self, mock_unlink, mock_tempfile, mock_pdfplumber):
        """Should delete temp file after processing."""
        from document_processor import extract_pdf_pages
        
        mock_tmp = MagicMock()
        mock_tmp.name = "/tmp/test.pdf"
        mock_tmp.__enter__ = Mock(return_value=mock_tmp)
        mock_tmp.__exit__ = Mock(return_value=False)
        mock_tempfile.return_value = mock_tmp
        
        mock_pdf = MagicMock()
        mock_pdf.pages = []
        mock_pdf.__enter__ = Mock(return_value=mock_pdf)
        mock_pdf.__exit__ = Mock(return_value=False)
        mock_pdfplumber.open.return_value = mock_pdf
        
        file_obj = io.BytesIO(b"fake pdf")
        extract_pdf_pages(file_obj, "test.pdf")
        
        mock_unlink.assert_called_once_with("/tmp/test.pdf")


class TestChunkText:
    """Tests for chunk_text function."""
    
    def test_creates_chunks_with_metadata(self):
        """Should create chunks with correct metadata."""
        from document_processor import chunk_text
        
        text = "Short text."
        result = chunk_text(text, "doc.pdf", 5)
        
        assert len(result) >= 1
        assert result[0]["text"] == "Short text."
        assert result[0]["metadata"]["source"] == "doc.pdf"
        assert result[0]["metadata"]["page_number"] == 5
        assert result[0]["metadata"]["chunk_index"] == 0
    
    def test_splits_long_text(self):
        """Should split long text into multiple chunks."""
        from document_processor import chunk_text
        
        text = "Word " * 500
        result = chunk_text(text, "doc.pdf", 1, chunk_size=100, chunk_overlap=20)
        
        assert len(result) > 1
    
    def test_increments_chunk_index(self):
        """Should increment chunk_index for each chunk."""
        from document_processor import chunk_text
        
        text = "Paragraph one.\n\nParagraph two.\n\nParagraph three." * 50
        result = chunk_text(text, "doc.pdf", 1, chunk_size=100, chunk_overlap=10)
        
        indices = [c["metadata"]["chunk_index"] for c in result]
        assert indices == list(range(len(result)))
    
    def test_uses_custom_chunk_size(self):
        """Should respect custom chunk_size."""
        from document_processor import chunk_text
        
        text = "A" * 200
        result = chunk_text(text, "doc.pdf", 1, chunk_size=50, chunk_overlap=10)
        
        for chunk in result:
            assert len(chunk["text"]) <= 60


class TestProcessDocuments:
    """Tests for process_documents function."""
    
    def test_processes_multiple_pages(self):
        """Should process all pages."""
        from document_processor import process_documents
        
        pages = [
            {"text": "Page one content.", "source": "doc.pdf", "page_number": 1},
            {"text": "Page two content.", "source": "doc.pdf", "page_number": 2},
        ]
        
        result = process_documents(pages)
        
        assert len(result) >= 2
    
    def test_preserves_page_numbers(self):
        """Should preserve page numbers in metadata."""
        from document_processor import process_documents
        
        pages = [
            {"text": "Content", "source": "doc.pdf", "page_number": 5},
        ]
        
        result = process_documents(pages)
        
        assert result[0]["metadata"]["page_number"] == 5
    
    def test_empty_pages_list(self):
        """Should handle empty pages list."""
        from document_processor import process_documents
        
        result = process_documents([])
        
        assert result == []


class TestChunksToStoreFormat:
    """Tests for chunks_to_store_format function."""
    
    def test_separates_texts_and_metadatas(self):
        """Should separate texts and metadatas."""
        from document_processor import chunks_to_store_format
        
        chunks = [
            {"text": "Text 1", "metadata": {"source": "a.pdf"}},
            {"text": "Text 2", "metadata": {"source": "b.pdf"}},
        ]
        
        texts, metadatas = chunks_to_store_format(chunks)
        
        assert texts == ["Text 1", "Text 2"]
        assert metadatas == [{"source": "a.pdf"}, {"source": "b.pdf"}]
    
    def test_empty_chunks(self):
        """Should handle empty chunks list."""
        from document_processor import chunks_to_store_format
        
        texts, metadatas = chunks_to_store_format([])
        
        assert texts == []
        assert metadatas == []


class TestExtractPdfFromPath:
    """Tests for extract_pdf_from_path function."""
    
    @patch("document_processor.pdfplumber")
    def test_extracts_pages_from_path(self, mock_pdfplumber):
        """Should extract text from PDF file path."""
        from document_processor import extract_pdf_from_path
        
        mock_page = Mock()
        mock_page.extract_text.return_value = "Page content"
        
        mock_pdf = MagicMock()
        mock_pdf.pages = [mock_page]
        mock_pdf.__enter__ = Mock(return_value=mock_pdf)
        mock_pdf.__exit__ = Mock(return_value=False)
        mock_pdfplumber.open.return_value = mock_pdf
        
        result = extract_pdf_from_path("/path/to/test.pdf", "test.pdf")
        
        assert len(result) == 1
        assert result[0]["text"] == "Page content"
        assert result[0]["source"] == "test.pdf"
    
    @patch("document_processor.pdfplumber")
    def test_uses_basename_if_no_filename(self, mock_pdfplumber):
        """Should use basename if filename not provided."""
        from document_processor import extract_pdf_from_path
        
        mock_page = Mock()
        mock_page.extract_text.return_value = "Content"
        
        mock_pdf = MagicMock()
        mock_pdf.pages = [mock_page]
        mock_pdf.__enter__ = Mock(return_value=mock_pdf)
        mock_pdf.__exit__ = Mock(return_value=False)
        mock_pdfplumber.open.return_value = mock_pdf
        
        result = extract_pdf_from_path("/path/to/document.pdf")
        
        assert result[0]["source"] == "document.pdf"
    
    @patch("document_processor.pdfplumber")
    def test_returns_empty_on_error(self, mock_pdfplumber):
        """Should return empty list on error."""
        from document_processor import extract_pdf_from_path
        
        mock_pdfplumber.open.side_effect = Exception("File not found")
        
        result = extract_pdf_from_path("/invalid/path.pdf")
        
        assert result == []