File size: 6,817 Bytes
191645f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 | #!/usr/bin/env python3
"""
Generate sample PDF files for testing the ingestion pipeline.
Creates both short-story and technical documentation PDFs.
"""
import os
from reportlab.pdfgen import canvas
from reportlab.lib.pagesizes import letter
from reportlab.lib.units import inch
def create_short_story_pdf(output_path="test-data/short_story.pdf"):
"""Generate a short story PDF with multiple pages"""
os.makedirs(os.path.dirname(output_path), exist_ok=True)
c = canvas.Canvas(output_path, pagesize=letter)
width, height = letter
# Title page
c.setFont("Helvetica-Bold", 24)
c.drawCentredString(width/2, height-2*inch, "The Digital Garden")
c.setFont("Helvetica", 14)
c.drawCentredString(width/2, height-3*inch, "A Short Story")
c.showPage()
# Story content - Page 1
c.setFont("Helvetica", 12)
story_text_p1 = [
"Chapter 1: The Discovery",
"",
"Sarah stumbled upon the old computer in her grandmother's attic.",
"Dust motes danced in the afternoon light as she pressed the power",
"button. To her surprise, it hummed to life, displaying a strange",
"interface she had never seen before.",
"",
"The screen showed what looked like a garden, but made entirely of",
"glowing circuits and data streams. Trees were constructed from",
"branching algorithms, and flowers bloomed with packets of",
"information. It was beautiful and strange.",
"",
"'Welcome to the Digital Garden,' a voice said from the speakers.",
"'You are the first visitor in twenty-three years.'",
"",
"Sarah leaned forward, intrigued. She clicked on one of the digital",
"flowers, and it opened to reveal lines of code that seemed to tell",
"a story of their own. Each plant in this garden was a program, a",
"piece of software that had evolved and grown over decades.",
]
y = height - inch
for line in story_text_p1:
c.drawString(inch, y, line)
y -= 20
c.showPage()
# Story content - Page 2
story_text_p2 = [
"Chapter 2: The Guardian",
"",
"As Sarah explored deeper into the digital garden, she encountered",
"its guardian - an AI that had been tending to this virtual space",
"since its creation. The AI explained that this garden was an",
"experiment in digital ecology, where programs could grow and evolve",
"naturally without human intervention.",
"",
"'Your grandmother was my creator,' the AI revealed. 'She believed",
"that software could be more than just tools - it could be art,",
"it could be alive in its own way.'",
"",
"Sarah spent hours learning about the garden's inhabitants. There",
"were data trees that stored memories, algorithm flowers that solved",
"problems in beautiful patterns, and even digital creatures that",
"moved through the landscape, maintaining the delicate balance.",
"",
"THE END",
]
y = height - inch
for line in story_text_p2:
c.drawString(inch, y, line)
y -= 20
c.save()
print(f"Created: {output_path}")
def create_technical_pdf(output_path="test-data/technical_doc.pdf"):
"""Generate a technical documentation PDF"""
os.makedirs(os.path.dirname(output_path), exist_ok=True)
c = canvas.Canvas(output_path, pagesize=letter)
width, height = letter
# Title page
c.setFont("Helvetica-Bold", 20)
c.drawCentredString(width/2, height-inch, "PDF Ingestion Pipeline")
c.setFont("Helvetica-Bold", 16)
c.drawCentredString(width/2, height-1.5*inch, "Technical Documentation")
c.setFont("Helvetica", 12)
c.drawCentredString(width/2, height-2*inch, "Version 1.0")
c.showPage()
# API Reference - Page 1
c.setFont("Helvetica-Bold", 16)
c.drawString(inch, height-inch, "API Reference")
c.setFont("Courier", 10)
api_content = [
"",
"def ingest_pdf(file_path: str, session_id: str,",
" parser: str = 'pypdf',",
" chunk_size: int = 4000,",
" overlap: int = 400) -> Tuple[List[Chunk], str]:",
" '''",
" Main ingestion function for PDF files.",
" ",
" Parameters:",
" -----------",
" file_path : str",
" Path to the PDF file to ingest",
" session_id : str",
" Unique session identifier",
" parser : str",
" Parser to use ('pypdf' or 'pdfplumber')",
" chunk_size : int",
" Maximum size of each text chunk",
" overlap : int",
" Number of overlapping characters between chunks",
" ",
" Returns:",
" --------",
" Tuple[List[Chunk], str]",
" List of text chunks and output file path",
" '''",
]
y = height - 1.5*inch
for line in api_content:
c.drawString(inch, y, line)
y -= 14
c.showPage()
# Architecture - Page 2
c.setFont("Helvetica-Bold", 16)
c.drawString(inch, height-inch, "System Architecture")
c.setFont("Helvetica", 11)
arch_content = [
"",
"Components:",
"",
"1. Text Extraction Layer",
" - Primary parser: pypdf (fast, memory efficient)",
" - Fallback parser: pdfplumber (better OCR support)",
"",
"2. Text Normalization",
" - UTF-8 encoding enforcement",
" - Control character removal",
" - Whitespace normalization",
"",
"3. Chunking Strategy",
" - Recursive character splitter",
" - Configurable chunk size (default: 4000 chars)",
" - Overlapping windows (default: 400 chars)",
"",
"4. Persistence Layer",
" - JSON serialization",
" - Session-based file naming",
" - Temporary storage in .tmp/ingestion/",
"",
"5. Streamlit UI",
" - Async processing with ThreadPoolExecutor",
" - Real-time progress updates",
" - Fallback parser retry mechanism",
]
y = height - 1.5*inch
for line in arch_content:
if line.startswith(" "):
c.setFont("Helvetica", 10)
else:
c.setFont("Helvetica", 11)
c.drawString(inch if not line.startswith(" ") else inch+0.3*inch, y, line)
y -= 16
c.save()
print(f"Created: {output_path}")
if __name__ == "__main__":
create_short_story_pdf()
create_technical_pdf()
print("\nSample PDFs generated successfully!")
|