File size: 8,661 Bytes
ca9d078 227ab63 ca9d078 2e6b5c8 ca9d078 2e6b5c8 ca9d078 2e6b5c8 ca9d078 227ab63 ca9d078 227ab63 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 | """Inspectable PDF calibration pages; geometry truth is independent of OCR sorting."""
import hashlib
import os
from pathlib import Path
import sys
import pymupdf
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'pipeline'))
from scripts import ocr_layout, pdf_reading_v3 as v3, pdf_text_layer as text
def build_samples(bundle, *, table_rows=False):
document = pymupdf.open()
font = pymupdf.Font('china-s')
descriptors = []
for kind in ('horizontal', 'rotation-90', 'rotation-270', 'vertical', 'vertical-rotation-90', 'columns', 'table'):
page = document.new_page(width=600, height=900)
blocks = []
expected = []
def line(value, x, y, size=12):
if any(not font.has_glyph(ord(char)) for char in value):
raise ValueError('calibration font lacks a source glyph')
page.insert_text((x, y + size), value, fontsize=size, fontname='china-s')
width = sum(size if ord(c) > 255 else size / 2 for c in value)
box = [x / 600, y / 900, (x + width) / 600, (y + size) / 900]
blocks.append({'t': value, 'b': box, 'q': [[box[0], box[1]], [box[2], box[1]],
[box[2], box[3]], [box[0], box[3]]], 'c': 1, 's': 'native'})
expected.append(value)
options = {'writing_mode': 'horizontal-ltr', 'join_soft_lines': False}
if kind in ('horizontal', 'rotation-90', 'rotation-270'):
for row in range(20):
line(f'第{row + 1}行 校验甲 校验乙 校验丙', 55, 60 + row * 35)
if kind != 'horizontal':
page.set_rotation(int(kind.split('-')[1]))
elif kind.startswith('vertical'):
options['writing_mode'] = 'vertical-rl'
for x, value in ((460, '竖排右列 校验甲Aé漢 校验乙丙'), (320, '竖排左列校验庚辛壬校验癸子丑')):
if any(not font.has_glyph(ord(char)) for char in value):
raise ValueError('calibration font lacks a source glyph')
for index, char in enumerate(value):
page.insert_text((x, 60 + (index + 1) * 28), char, fontsize=20, fontname='china-s')
# Include real word spaces and Latin advances to exercise attached-run measurement.
box = [x / 600, 60 / 900, (x + 20) / 600, (60 + len(value) * 28) / 900]
blocks.append({'t': value, 'b': box, 'c': 1, 's': 'native'})
expected.append(value)
if kind == 'vertical-rotation-90':
page.set_rotation(90)
elif kind == 'columns':
line('通栏标题', 70, 20)
for column, x in (('左栏', 50), ('右栏', 340)):
for row in range(20):
line(f'{column}{row + 1} 校验甲 校验乙', x, 100 + row * 34)
else:
line('合并表头与跨列说明', 55, 30)
# Header spans all columns; the first body label spans two rows.
xs = [45, 170, 295, 420, 555]
ys = [100 + row * 105 for row in range(8)]
for index, y in enumerate(ys):
page.draw_line((xs[1] if index == 1 else xs[0], y), (xs[-1], y), color=(0, 0, 0))
for x in xs:
page.draw_line((x, ys[0]), (x, ys[-1]), color=(0, 0, 0))
for row in range(7):
for column in range(4):
if row == 1 and column == 0:
continue
line(f'{row + 1}项{column + 1} 校验甲', xs[column] + 5, ys[row] + 18, 9)
line('数值 校验乙', xs[column] + 5, ys[row] + 42, 9)
if table_rows:
options['regions'] = [{'box': [0, 0, 1, .09], 'writing_mode': 'horizontal-ltr'}] + [
{'box': [xs[column] / 600, ys[row] / 900, xs[column + 1] / 600,
ys[row + (2 if row == column == 0 else 1)] / 900], 'writing_mode': 'horizontal-ltr'}
for row in range(7) for column in range(4) if not (row == 1 and column == 0)]
descriptors.append({'kind': kind, 'blocks': blocks, 'expected_regions': expected, 'options': options,
'rotation': page.rotation})
pdf = document.tobytes(no_new_id=True)
source = hashlib.sha256(pdf).hexdigest()
root = Path('objects') / source[:2] / source / ('c' * 16)
primary_path = bundle / root / 'document.pdf'
primary_path.parent.mkdir(parents=True, exist_ok=True)
primary_path.write_bytes(pdf)
primary = {'bucket': v3.shared.PDF_PAGES_BUCKET, 'path': primary_path.relative_to(bundle).as_posix(),
'sha256': source, 'bytes': len(pdf), 'role': 'runtime'}
layers = []
for index, descriptor in enumerate(descriptors, 1):
arranged = ocr_layout.arrange(descriptor['blocks'], 600, 900, descriptor['options'], include_writing_modes=True)
rotation = descriptor['rotation']
if rotation:
for field in ('blocks', 'raw_blocks'):
for block in arranged[field]:
quad = block.get('q') or [[block['b'][0], block['b'][1]], [block['b'][2], block['b'][1]],
[block['b'][2], block['b'][3]], [block['b'][0], block['b'][3]]]
block['q'] = [list(ocr_layout.map_point_back(x, y, 360 - rotation)) for x, y in quad]
block['b'] = [min(p[0] for p in block['q']), min(p[1] for p in block['q']),
max(p[0] for p in block['q']), max(p[1] for p in block['q'])]
for span in arranged['text_spans']:
span['box'] = next(block['b'] for block in arranged['blocks'] if block['id'] == span['block'])
page = document[index - 1]
layer = text.from_page({'kind': 'pdf-ocr-page', 'page': index, 'source': 'native',
'width': page.rect.width, 'height': page.rect.height, **arranged}, source, 'ch')
layers.append(layer)
descriptor['page'] = index
descriptor['expected_boxes'] = [r['box'] for r in layer['regions']]
descriptor['actual_regions'] = [r['text'] for r in layer['regions']]
document.close()
layer_ref = v3.build_text_bundle(layers, [{'p': p['page']} for p in layers], root, bundle)
def read(ref):
return (bundle / ref['path']).read_bytes()
spec = {'source_key': 'calibration\0layout.pdf', 'source_sha256': source, 'primary': primary,
'text_layer': layer_ref, 'require_complete_preview': True,
'vector_previews': os.environ.get('READER_V3_VECTOR_PREVIEWS') == '1'}
spec = v3.generate_previews(spec, read, bundle, dpi=100)
ref, manifest = v3.build_reading(spec, read, bundle)
return {'reading': ref, 'manifest': manifest, 'layers': layers, 'samples': descriptors,
'query': '校验', 'expected_total': sum(value.count('校验') for d in descriptors for value in d['expected_regions'])}
def build_fractional_vector_sample(bundle):
"""A4-like native vector pages whose SVG intrinsic size is fractional (595.3 x 841.9)."""
document = pymupdf.open()
for number in range(1, 4):
page = document.new_page(width=595.3, height=841.9)
# Extractable text is what classifies a page as native-vector.
page.insert_text((60, 100), f'Fractional vector page {number}', fontsize=18)
page.draw_rect(pymupdf.Rect(40, 60, 555, 120), color=(0, 0, 0))
page.draw_line((40, 200 + number * 10), (555, 200 + number * 10), color=(0, 0, 0))
pdf = document.tobytes(no_new_id=True)
document.close()
source = hashlib.sha256(pdf).hexdigest()
primary_path = bundle / 'objects' / source[:2] / source / ('d' * 16) / 'document.pdf'
primary_path.parent.mkdir(parents=True, exist_ok=True)
primary_path.write_bytes(pdf)
primary = {'bucket': v3.shared.PDF_PAGES_BUCKET, 'path': primary_path.relative_to(bundle).as_posix(),
'sha256': source, 'bytes': len(pdf), 'role': 'runtime'}
def read(ref):
return (bundle / ref['path']).read_bytes()
spec = {'source_key': 'calibration\0fractional-vector.pdf', 'source_sha256': source, 'primary': primary,
'require_complete_preview': True, 'vector_previews': True}
spec = v3.generate_previews(spec, read, bundle, dpi=100)
if any(entry['codec'] != 'svg' for entry in spec['previews']):
raise ValueError('fractional fixture did not produce vector previews')
ref, manifest = v3.build_reading(spec, read, bundle)
return {'reading': ref, 'manifest': manifest, 'previews': spec['previews']}
|