File size: 3,876 Bytes
7f6dd09 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | // Package parser extracts text from PDF files using ledongthuc/pdf (pure-Go).
// It classifies each page as DIGITAL / SCANNED / PARTIAL / BLANK and returns
// structured per-page text that the chunker package consumes.
package parser
import (
"fmt"
"log"
"os"
"path/filepath"
"strings"
"unicode/utf8"
pdf "github.com/ledongthuc/pdf"
)
// PageType classifies a single PDF page by text density.
const (
PageDigital = "digital" // >200 chars — born-digital text
PageScanned = "scanned" // <50 chars — image-based, needs OCR
PagePartial = "partial" // 50-200 chars — may have garbled OCR
PageBlank = "blank" // 0 chars
)
// PageResult holds extracted text and classification for a single page.
type PageResult struct {
PageNum int `json:"page_num"` // 1-indexed page number
Text string `json:"text"` // Raw extracted text
PageType string `json:"page_type"` // digital / scanned / partial / blank
CharLen int `json:"char_len"` // Number of UTF-8 characters
}
// PDFResult is the complete extraction result for one PDF file.
type PDFResult struct {
SourceFile string `json:"source_file"`
TotalPages int `json:"total_pages"`
DigitalPages int `json:"digital_pages"`
ScannedPages []int `json:"scanned_pages"` // 1-indexed scanned page numbers → sent to AI OCR
Pages []PageResult `json:"pages"`
Error string `json:"error,omitempty"`
}
// ParsePDF opens a PDF file and extracts text from every page.
func ParsePDF(pdfPath string) (*PDFResult, error) {
filename := filepath.Base(pdfPath)
f, reader, err := pdf.Open(pdfPath)
if err != nil {
return &PDFResult{
SourceFile: filename,
Error: fmt.Sprintf("failed to open PDF: %v", err),
}, err
}
defer f.Close()
totalPages := reader.NumPage()
result := &PDFResult{
SourceFile: filename,
TotalPages: totalPages,
ScannedPages: []int{},
Pages: make([]PageResult, 0, totalPages),
}
for i := 1; i <= totalPages; i++ {
page := reader.Page(i)
if page.V.IsNull() {
result.Pages = append(result.Pages, PageResult{
PageNum: i,
Text: "",
PageType: PageBlank,
CharLen: 0,
})
continue
}
text, err := page.GetPlainText(nil)
if err != nil {
log.Printf("[parser] page %d text extraction error: %v", i, err)
text = ""
}
text = strings.TrimSpace(text)
charLen := utf8.RuneCountInString(text)
pageType := classifyPage(charLen)
result.Pages = append(result.Pages, PageResult{
PageNum: i,
Text: text,
PageType: pageType,
CharLen: charLen,
})
if pageType == PageDigital {
result.DigitalPages++
}
if pageType == PageScanned || pageType == PagePartial {
result.ScannedPages = append(result.ScannedPages, i)
}
}
return result, nil
}
// classifyPage mirrors the Python page_classifier thresholds.
func classifyPage(charLen int) string {
switch {
case charLen > 200:
return PageDigital
case charLen >= 50:
return PagePartial
case charLen > 0:
return PageScanned
default:
return PageBlank
}
}
// ParseAllPDFs parses every PDF in a directory. Returns results keyed by filename.
func ParseAllPDFs(dirPath string) (map[string]*PDFResult, error) {
entries, err := os.ReadDir(dirPath)
if err != nil {
return nil, fmt.Errorf("failed to read directory %s: %w", dirPath, err)
}
results := make(map[string]*PDFResult)
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := entry.Name()
if !strings.HasSuffix(strings.ToLower(name), ".pdf") {
continue
}
fullPath := filepath.Join(dirPath, name)
res, err := ParsePDF(fullPath)
if err != nil {
log.Printf("[parser] error parsing %s: %v", name, err)
results[name] = &PDFResult{
SourceFile: name,
Error: err.Error(),
}
continue
}
results[name] = res
}
return results, nil
}
|