Add PDF reader example

app.py

@@ -29,6 +29,7 @@ from chromadb.config import Settings
 from sentence_transformers import SentenceTransformer
 from huggingface_hub import InferenceClient
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from pypdf import PdfReader
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -86,6 +87,27 @@ logger.info(f"LLM client initialized for model: {LLM_MODEL_NAME}")
 # Helper Functions
 # ---------------------------------------------------------------------------
 
+def extract_text_from_pdf(pdf_path: Path) -> str:
+    """
+    Extract text content from a PDF file using pypdf.
+
+    This demonstrates how to convert PDF documents into plain text
+    for downstream embedding. Each page is extracted sequentially
+    and concatenated with page separators for traceability.
+    """
+    reader = PdfReader(str(pdf_path))
+    pages_text = []
+
+    for page_num, page in enumerate(reader.pages, start=1):
+        text = page.extract_text()
+        if text and text.strip():
+            pages_text.append(f"[Page {page_num}]\n{text.strip()}")
+
+    full_text = "\n\n".join(pages_text)
+    logger.info(f"Extracted {len(reader.pages)} pages from PDF: {pdf_path.name} ({len(full_text)} chars)")
+    return full_text
+
+
 def chunk_text(text: str, source: str = "unknown") -> list[dict]:
     """
     Split a document into smaller chunks for embedding.
@@ -278,12 +300,23 @@ def ingest_sample_documents():
         logger.warning("No sample_documents directory found.")
         return
 
+    # Ingest plain text files
     for file_path in SAMPLE_DOCS_DIR.glob("*.txt"):
-        logger.info(f"Ingesting: {file_path.name}")
+        logger.info(f"Ingesting text file: {file_path.name}")
         text = file_path.read_text(encoding="utf-8")
         chunks = chunk_text(text, source=file_path.name)
         add_documents_to_vectorstore(chunks)
 
+    # Ingest PDF files (e.g., ebook-a-compact-guide-to-pretraining-custom-llms.pdf)
+    for file_path in SAMPLE_DOCS_DIR.glob("*.pdf"):
+        logger.info(f"Ingesting PDF file: {file_path.name}")
+        text = extract_text_from_pdf(file_path)
+        if text.strip():
+            chunks = chunk_text(text, source=file_path.name)
+            add_documents_to_vectorstore(chunks)
+        else:
+            logger.warning(f"No extractable text found in: {file_path.name}")
+
     logger.info(f"Sample document ingestion complete. Total chunks: {collection.count()}")